📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""run_trigger_evals.py — 触发评测的确定性环节（Phase 3，路线 C）。
|
||||
|
||||
路线 C 下 CLI 不能替宿主跑模型；本脚本负责三件确定性的事：
|
||||
|
||||
split 固定种子做 60/40 train/validation 切分（validation 在选版前保持隐藏）
|
||||
prepare 生成盲测任务包：只含 prompt + 候选 skill 目录清单，隐藏 expected/notes，
|
||||
由主流程逐条交给干净 sub-agent，结果写 results.jsonl
|
||||
score 对照 suite 判分：precision/recall/F1、兄弟混淆率、逐条配对结果
|
||||
（不做统计非劣声明——那需要预注册界值与配对检验，见方案 §10.3）
|
||||
|
||||
results.jsonl 每行: {"case_id": "...", "run": 1, "selected_skill": "<slug>|none"}
|
||||
|
||||
用法:
|
||||
python3 scripts/run_trigger_evals.py split <suite.json>
|
||||
python3 scripts/run_trigger_evals.py prepare <suite.json> --skills <slug1,slug2,...> --out <dir> [--set train|validation|all]
|
||||
python3 scripts/run_trigger_evals.py score <suite.json> --results <results.jsonl> --out <report.md>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import dump_json, load_json # noqa: E402
|
||||
|
||||
|
||||
def split_cases(suite: dict) -> tuple[list[dict], list[dict]]:
|
||||
cases = list(suite["trigger_cases"])
|
||||
rng = random.Random(suite.get("split_seed", 42))
|
||||
rng.shuffle(cases)
|
||||
k = round(len(cases) * suite.get("train_ratio", 0.6))
|
||||
return cases[:k], cases[k:]
|
||||
|
||||
|
||||
def cmd_split(suite_path: Path) -> int:
|
||||
suite = load_json(suite_path)
|
||||
train, val = split_cases(suite)
|
||||
out = {"suite_id": suite["suite_id"], "seed": suite.get("split_seed", 42),
|
||||
"train": [c["case_id"] for c in train], "validation": [c["case_id"] for c in val]}
|
||||
dump_json(suite_path.with_suffix(".split.json"), out)
|
||||
print(f"train {len(train)} / validation {len(val)} → {suite_path.with_suffix('.split.json')}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_prepare(suite_path: Path, skills: list[str], out_dir: Path, which: str) -> int:
|
||||
suite = load_json(suite_path)
|
||||
train, val = split_cases(suite)
|
||||
cases = {"train": train, "validation": val, "all": train + val}[which]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for c in cases:
|
||||
runs = c.get("runs", 3)
|
||||
packet = {
|
||||
"case_id": c["case_id"],
|
||||
"runs": runs,
|
||||
"prompt": c["prompt"],
|
||||
"instruction": (
|
||||
"你是一个未参与蒸馏的干净 agent。给定用户 prompt 与已安装 skill 清单，"
|
||||
"判断该激活哪一个 skill（或 none）。输出 JSON: "
|
||||
'{"selected_skill": "<slug>|none", "reason": "..."}。'
|
||||
"每条 prompt 独立判断，重复运行之间不携带记忆。"
|
||||
),
|
||||
"installed_skills": skills,
|
||||
}
|
||||
dump_json(out_dir / f"{c['case_id']}.json", packet)
|
||||
(out_dir / "README.md").write_text(
|
||||
f"# 盲测任务包（{which}, {len(cases)} 条）\n\n"
|
||||
"每个 JSON 是一条盲测任务：把 prompt + installed_skills 交给干净 sub-agent，"
|
||||
"按 instruction 输出;结果按行追加到 results.jsonl:\n"
|
||||
'`{"case_id": ..., "run": 1, "selected_skill": ...}`\n\n'
|
||||
"**不要**把 suite 中的 expected/notes 给 sub-agent。\n", encoding="utf-8")
|
||||
print(f"已生成 {len(cases)} 个盲测任务包 → {out_dir}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_score(suite_path: Path, results_path: Path, out_path: Path) -> int:
|
||||
suite = load_json(suite_path)
|
||||
target = suite["target"]
|
||||
by_case = {c["case_id"]: c for c in suite["trigger_cases"]}
|
||||
results = [json.loads(l) for l in results_path.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
|
||||
tp = fp = fn = tn = 0
|
||||
sibling_total = sibling_confused = 0
|
||||
rows = ["| case | expected | selected | 判定 |", "|---|---|---|---|"]
|
||||
for r in results:
|
||||
c = by_case.get(r["case_id"])
|
||||
if not c:
|
||||
continue
|
||||
selected = r.get("selected_skill", "none")
|
||||
triggered = selected == target
|
||||
exp = c["expected"]
|
||||
if exp == "should_trigger":
|
||||
verdict = "TP" if triggered else "FN"
|
||||
tp += triggered
|
||||
fn += not triggered
|
||||
elif exp in ("should_not_trigger", "edge_case"):
|
||||
verdict = "FP" if triggered else "TN"
|
||||
fp += triggered
|
||||
tn += not triggered
|
||||
else: # sibling
|
||||
sibling_total += 1
|
||||
correct = selected == c.get("sibling_target")
|
||||
if triggered:
|
||||
sibling_confused += 1
|
||||
verdict = "混淆(FP)"
|
||||
fp += 1
|
||||
else:
|
||||
verdict = "OK" if correct else f"未中兄弟({selected})"
|
||||
tn += 1
|
||||
rows.append(f"| {r['case_id']}#r{r.get('run', 1)} | {exp} | {selected} | {verdict} |")
|
||||
|
||||
precision = tp / (tp + fp) if tp + fp else 0.0
|
||||
recall = tp / (tp + fn) if tp + fn else 0.0
|
||||
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
|
||||
confusion = sibling_confused / sibling_total if sibling_total else 0.0
|
||||
|
||||
report = (f"# 触发评测判分 — {target}\n\n"
|
||||
f"- runs: {len(results)}（TP {tp} / FP {fp} / FN {fn} / TN {tn}）\n"
|
||||
f"- precision {precision:.3f} / recall {recall:.3f} / **F1 {f1:.3f}**\n"
|
||||
f"- 兄弟混淆率: {confusion:.3f}（{sibling_confused}/{sibling_total}）\n\n"
|
||||
+ "\n".join(rows)
|
||||
+ "\n\n> 本报告只给原始配对计数与比率；统计非劣需预注册界值 + McNemar/配对 Bootstrap（§10.3），不在此自动宣布。\n")
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(report, encoding="utf-8")
|
||||
print(report)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
sub = ap.add_subparsers(dest="mode", required=True)
|
||||
s = sub.add_parser("split")
|
||||
s.add_argument("suite")
|
||||
p = sub.add_parser("prepare")
|
||||
p.add_argument("suite")
|
||||
p.add_argument("--skills", required=True)
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--set", dest="which", choices=["train", "validation", "all"], default="train")
|
||||
c = sub.add_parser("score")
|
||||
c.add_argument("suite")
|
||||
c.add_argument("--results", required=True)
|
||||
c.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.mode == "split":
|
||||
return cmd_split(Path(args.suite))
|
||||
if args.mode == "prepare":
|
||||
return cmd_prepare(Path(args.suite), args.skills.split(","), Path(args.out), args.which)
|
||||
return cmd_score(Path(args.suite), Path(args.results), Path(args.out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user