97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
||
"""benchmark.py — 聚合报告（Phase 3/4，路线 C）。
|
||
|
||
汇总一次迭代的全部可复算指标为 benchmark.json + benchmark.md：
|
||
- A 类静态 Token 指标（调用 count_tokens.py 的配置与产物）
|
||
- 触发/输出评测判分（run_trigger_evals / run_output_evals 的报告产物）
|
||
- 过程代理指标（runs/*/stage-usage.jsonl 若存在；标注为估算，非真实 token）
|
||
|
||
用法: python3 scripts/benchmark.py --token-config <token-config.json> [--eval-reports <dir>] [--sidecar <.cangjie>] --out <dir>
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).parent))
|
||
from cangjie_common import TOOL_VERSION, dump_json, load_json # noqa: E402
|
||
|
||
SCRIPTS = Path(__file__).parent
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--token-config", required=True)
|
||
ap.add_argument("--eval-reports", default=None)
|
||
ap.add_argument("--sidecar", default=None)
|
||
ap.add_argument("--out", required=True)
|
||
args = ap.parse_args()
|
||
|
||
out_dir = Path(args.out)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 1. A 类静态指标
|
||
rc = subprocess.call([sys.executable, str(SCRIPTS / "count_tokens.py"), args.token_config, "--out", str(out_dir)])
|
||
if rc != 0:
|
||
raise SystemExit("count_tokens 失败")
|
||
a_metrics = load_json(out_dir / "a-class-metrics.json")
|
||
|
||
# 2. 评测报告
|
||
eval_reports = []
|
||
if args.eval_reports:
|
||
for f in sorted(Path(args.eval_reports).glob("*.md")):
|
||
eval_reports.append({"file": str(f), "title": f.read_text(encoding="utf-8").splitlines()[0].lstrip("# ")})
|
||
|
||
# 3. 过程代理指标（B 类）
|
||
proxy = []
|
||
if args.sidecar:
|
||
for usage in sorted(Path(args.sidecar).glob("runs/*/stage-usage.jsonl")):
|
||
rows = [json.loads(l) for l in usage.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||
proxy.append({
|
||
"run": usage.parent.name,
|
||
"tasks": len(rows),
|
||
"prepared_input_chars": sum(r.get("prepared_input_chars", 0) for r in rows),
|
||
"cache_hits": sum(1 for r in rows if r.get("reused_from_cache")),
|
||
"retries": sum(r.get("retry_count", 0) for r in rows),
|
||
})
|
||
|
||
benchmark = {
|
||
"tool": TOOL_VERSION,
|
||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||
"a_class_metrics": a_metrics,
|
||
"eval_reports": eval_reports,
|
||
"process_proxy_metrics": proxy,
|
||
"notes": [
|
||
"A 类为固定 tokenizer 的文件计数与静态载荷模型，不代表宿主实际计费（§5.1）",
|
||
"B 类为过程代理量（prepared_input），标注为估算，禁止改称真实 Prompt 输入",
|
||
"路线 C 下不存在可信的 input/output/cached tokens，本报告不含这三项",
|
||
],
|
||
}
|
||
dump_json(out_dir / "benchmark.json", benchmark)
|
||
|
||
lines = [f"# Benchmark 报告（{benchmark['generated_at']}）", "",
|
||
f"- 工具: {TOOL_VERSION}",
|
||
f"- A 类静态指标: `{out_dir / 'a-class-metrics.md'}`"]
|
||
if eval_reports:
|
||
lines.append("- 评测判分:")
|
||
lines += [f" - [{r['title']}]({r['file']})" for r in eval_reports]
|
||
if proxy:
|
||
lines.append("\n## 过程代理指标（估算，非真实 token）\n")
|
||
lines.append("| run | 任务数 | prepared_input_chars | 缓存命中 | 重试 |")
|
||
lines.append("|---|---|---|---|---|")
|
||
lines += [f"| {p['run']} | {p['tasks']} | {p['prepared_input_chars']} | {p['cache_hits']} | {p['retries']} |"
|
||
for p in proxy]
|
||
lines += ["", *(f"> {n}" for n in benchmark["notes"])]
|
||
(out_dir / "benchmark.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
print(f"benchmark: {out_dir / 'benchmark.json'} / {out_dir / 'benchmark.md'}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|