📦 deps(thirdparty): update snapshots

2026-08-31 05:00:46 +08:00
parent 8fea3d18fe
commit 0b2b056032
347 changed files with 38446 additions and 136 deletions
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""benchmark.py — 聚合报告（Phase 3/4，路线 C）。
+
+汇总一次迭代的全部可复算指标为 benchmark.json + benchmark.md：
+- A 类静态 Token 指标（调用 count_tokens.py 的配置与产物）
+- 触发/输出评测判分（run_trigger_evals / run_output_evals 的报告产物）
+- 过程代理指标（runs/*/stage-usage.jsonl 若存在；标注为估算，非真实 token）
+
+用法: python3 scripts/benchmark.py --token-config <token-config.json> [--eval-reports <dir>] [--sidecar <.cangjie>] --out <dir>
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from cangjie_common import TOOL_VERSION, dump_json, load_json  # noqa: E402
+
+SCRIPTS = Path(__file__).parent
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--token-config", required=True)
+    ap.add_argument("--eval-reports", default=None)
+    ap.add_argument("--sidecar", default=None)
+    ap.add_argument("--out", required=True)
+    args = ap.parse_args()
+
+    out_dir = Path(args.out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # 1. A 类静态指标
+    rc = subprocess.call([sys.executable, str(SCRIPTS / "count_tokens.py"), args.token_config, "--out", str(out_dir)])
+    if rc != 0:
+        raise SystemExit("count_tokens 失败")
+    a_metrics = load_json(out_dir / "a-class-metrics.json")
+
+    # 2. 评测报告
+    eval_reports = []
+    if args.eval_reports:
+        for f in sorted(Path(args.eval_reports).glob("*.md")):
+            eval_reports.append({"file": str(f), "title": f.read_text(encoding="utf-8").splitlines()[0].lstrip("# ")})
+
+    # 3. 过程代理指标（B 类）
+    proxy = []
+    if args.sidecar:
+        for usage in sorted(Path(args.sidecar).glob("runs/*/stage-usage.jsonl")):
+            rows = [json.loads(l) for l in usage.read_text(encoding="utf-8").splitlines() if l.strip()]
+            proxy.append({
+                "run": usage.parent.name,
+                "tasks": len(rows),
+                "prepared_input_chars": sum(r.get("prepared_input_chars", 0) for r in rows),
+                "cache_hits": sum(1 for r in rows if r.get("reused_from_cache")),
+                "retries": sum(r.get("retry_count", 0) for r in rows),
+            })
+
+    benchmark = {
+        "tool": TOOL_VERSION,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "a_class_metrics": a_metrics,
+        "eval_reports": eval_reports,
+        "process_proxy_metrics": proxy,
+        "notes": [
+            "A 类为固定 tokenizer 的文件计数与静态载荷模型，不代表宿主实际计费（§5.1）",
+            "B 类为过程代理量（prepared_input），标注为估算，禁止改称真实 Prompt 输入",
+            "路线 C 下不存在可信的 input/output/cached tokens，本报告不含这三项",
+        ],
+    }
+    dump_json(out_dir / "benchmark.json", benchmark)
+
+    lines = [f"# Benchmark 报告（{benchmark['generated_at']}）", "",
+             f"- 工具: {TOOL_VERSION}",
+             f"- A 类静态指标: `{out_dir / 'a-class-metrics.md'}`"]
+    if eval_reports:
+        lines.append("- 评测判分:")
+        lines += [f"  - [{r['title']}]({r['file']})" for r in eval_reports]
+    if proxy:
+        lines.append("\n## 过程代理指标（估算，非真实 token）\n")
+        lines.append("| run | 任务数 | prepared_input_chars | 缓存命中 | 重试 |")
+        lines.append("|---|---|---|---|---|")
+        lines += [f"| {p['run']} | {p['tasks']} | {p['prepared_input_chars']} | {p['cache_hits']} | {p['retries']} |"
+                  for p in proxy]
+    lines += ["", *(f"> {n}" for n in benchmark["notes"])]
+    (out_dir / "benchmark.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"benchmark: {out_dir / 'benchmark.json'} / {out_dir / 'benchmark.md'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())