📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""apply_skill_patch.py — 最小补丁事务：快照 → 覆盖补丁文件 → 校验 → 可回滚（Phase 2B/3）。
|
||||
|
||||
repair/update 的落盘环节。补丁是"文件覆盖层"（patch 目录里的文件按相对路径覆盖目标），
|
||||
不做自由重写；应用前自动快照，校验失败自动回滚。
|
||||
|
||||
用法:
|
||||
python3 scripts/apply_skill_patch.py apply --target <skill-dir> --patch-dir <dir> --snapshots <dir>
|
||||
python3 scripts/apply_skill_patch.py restore --target <skill-dir> --snapshot <snapshot-dir>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import WriterLock, snapshot_dir # noqa: E402
|
||||
|
||||
SCRIPTS = Path(__file__).parent
|
||||
|
||||
|
||||
def apply_patch(target: Path, patch_dir: Path, snapshots: Path) -> int:
|
||||
patch_files = [p for p in patch_dir.rglob("*") if p.is_file()]
|
||||
if not patch_files:
|
||||
raise SystemExit(f"补丁目录为空: {patch_dir}")
|
||||
|
||||
with WriterLock(target):
|
||||
snap = snapshot_dir(target, snapshots, "pre-patch")
|
||||
print(f"已快照: {snap}")
|
||||
diff_lines = []
|
||||
for pf in patch_files:
|
||||
rel = pf.relative_to(patch_dir)
|
||||
dest = target / rel
|
||||
old = dest.read_text(encoding="utf-8") if dest.exists() else ""
|
||||
new = pf.read_text(encoding="utf-8")
|
||||
diff_lines.append(f"### {rel}: {'修改' if dest.exists() else '新增'}（{len(old)} → {len(new)} 字符）")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(pf, dest)
|
||||
|
||||
check = subprocess.run([sys.executable, str(SCRIPTS / "validate_skill_pack.py"), str(target)],
|
||||
capture_output=True, text=True)
|
||||
if check.returncode != 0:
|
||||
shutil.rmtree(target)
|
||||
shutil.copytree(snap, target)
|
||||
print(check.stdout + check.stderr)
|
||||
raise SystemExit("[rollback] 补丁后校验失败，已自动恢复快照")
|
||||
|
||||
print("\n".join(diff_lines))
|
||||
print(f"补丁已应用并通过校验（{len(patch_files)} 个文件）。回滚: apply_skill_patch.py restore --snapshot {snap}")
|
||||
return 0
|
||||
|
||||
|
||||
def restore(target: Path, snapshot: Path) -> int:
|
||||
if not snapshot.is_dir():
|
||||
raise SystemExit(f"快照不存在: {snapshot}")
|
||||
with WriterLock(target):
|
||||
if target.exists():
|
||||
shutil.rmtree(target)
|
||||
shutil.copytree(snapshot, target)
|
||||
print(f"已恢复 {target} ← {snapshot}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
sub = ap.add_subparsers(dest="mode", required=True)
|
||||
a = sub.add_parser("apply")
|
||||
a.add_argument("--target", required=True)
|
||||
a.add_argument("--patch-dir", required=True)
|
||||
a.add_argument("--snapshots", required=True)
|
||||
r = sub.add_parser("restore")
|
||||
r.add_argument("--target", required=True)
|
||||
r.add_argument("--snapshot", required=True)
|
||||
args = ap.parse_args()
|
||||
if args.mode == "apply":
|
||||
return apply_patch(Path(args.target), Path(args.patch_dir), Path(args.snapshots))
|
||||
return restore(Path(args.target), Path(args.snapshot))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""benchmark.py — 聚合报告（Phase 3/4，路线 C）。
|
||||
|
||||
汇总一次迭代的全部可复算指标为 benchmark.json + benchmark.md：
|
||||
- A 类静态 Token 指标（调用 count_tokens.py 的配置与产物）
|
||||
- 触发/输出评测判分（run_trigger_evals / run_output_evals 的报告产物）
|
||||
- 过程代理指标（runs/*/stage-usage.jsonl 若存在；标注为估算，非真实 token）
|
||||
|
||||
用法: python3 scripts/benchmark.py --token-config <token-config.json> [--eval-reports <dir>] [--sidecar <.cangjie>] --out <dir>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import TOOL_VERSION, dump_json, load_json # noqa: E402
|
||||
|
||||
SCRIPTS = Path(__file__).parent
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--token-config", required=True)
|
||||
ap.add_argument("--eval-reports", default=None)
|
||||
ap.add_argument("--sidecar", default=None)
|
||||
ap.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
|
||||
out_dir = Path(args.out)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1. A 类静态指标
|
||||
rc = subprocess.call([sys.executable, str(SCRIPTS / "count_tokens.py"), args.token_config, "--out", str(out_dir)])
|
||||
if rc != 0:
|
||||
raise SystemExit("count_tokens 失败")
|
||||
a_metrics = load_json(out_dir / "a-class-metrics.json")
|
||||
|
||||
# 2. 评测报告
|
||||
eval_reports = []
|
||||
if args.eval_reports:
|
||||
for f in sorted(Path(args.eval_reports).glob("*.md")):
|
||||
eval_reports.append({"file": str(f), "title": f.read_text(encoding="utf-8").splitlines()[0].lstrip("# ")})
|
||||
|
||||
# 3. 过程代理指标（B 类）
|
||||
proxy = []
|
||||
if args.sidecar:
|
||||
for usage in sorted(Path(args.sidecar).glob("runs/*/stage-usage.jsonl")):
|
||||
rows = [json.loads(l) for l in usage.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
proxy.append({
|
||||
"run": usage.parent.name,
|
||||
"tasks": len(rows),
|
||||
"prepared_input_chars": sum(r.get("prepared_input_chars", 0) for r in rows),
|
||||
"cache_hits": sum(1 for r in rows if r.get("reused_from_cache")),
|
||||
"retries": sum(r.get("retry_count", 0) for r in rows),
|
||||
})
|
||||
|
||||
benchmark = {
|
||||
"tool": TOOL_VERSION,
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"a_class_metrics": a_metrics,
|
||||
"eval_reports": eval_reports,
|
||||
"process_proxy_metrics": proxy,
|
||||
"notes": [
|
||||
"A 类为固定 tokenizer 的文件计数与静态载荷模型，不代表宿主实际计费（§5.1）",
|
||||
"B 类为过程代理量（prepared_input），标注为估算，禁止改称真实 Prompt 输入",
|
||||
"路线 C 下不存在可信的 input/output/cached tokens，本报告不含这三项",
|
||||
],
|
||||
}
|
||||
dump_json(out_dir / "benchmark.json", benchmark)
|
||||
|
||||
lines = [f"# Benchmark 报告（{benchmark['generated_at']}）", "",
|
||||
f"- 工具: {TOOL_VERSION}",
|
||||
f"- A 类静态指标: `{out_dir / 'a-class-metrics.md'}`"]
|
||||
if eval_reports:
|
||||
lines.append("- 评测判分:")
|
||||
lines += [f" - [{r['title']}]({r['file']})" for r in eval_reports]
|
||||
if proxy:
|
||||
lines.append("\n## 过程代理指标（估算，非真实 token）\n")
|
||||
lines.append("| run | 任务数 | prepared_input_chars | 缓存命中 | 重试 |")
|
||||
lines.append("|---|---|---|---|---|")
|
||||
lines += [f"| {p['run']} | {p['tasks']} | {p['prepared_input_chars']} | {p['cache_hits']} | {p['retries']} |"
|
||||
for p in proxy]
|
||||
lines += ["", *(f"> {n}" for n in benchmark["notes"])]
|
||||
(out_dir / "benchmark.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print(f"benchmark: {out_dir / 'benchmark.json'} / {out_dir / 'benchmark.md'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""build_chunks.py — 原生 Markdown/TXT → SourceDocument + 结构感知块（Phase 2A，纯确定性）。
|
||||
|
||||
产出（写入 <sidecar>/，默认 books/<slug>/.cangjie/）：
|
||||
normalized/<source-id>/<version-id>/document.json # SourceDocument（contracts/source-document.schema.json）
|
||||
chunks/chunks.jsonl # 结构感知块（contracts/chunk.schema.json）
|
||||
|
||||
块规则：尊重标题层级边界；单块目标 <= --max-chars（默认 4000 字符），超长段落按段切分；
|
||||
每块保留 heading_path 与 element_ids，可回溯到原文。重复运行命中确定性缓存时跳过重算。
|
||||
|
||||
用法: python3 scripts/build_chunks.py <source.md> --sidecar <dir> [--source-id src-main-book] [--max-chars 4000]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import ( # noqa: E402
|
||||
cache_lookup,
|
||||
cache_store,
|
||||
deterministic_cache_key,
|
||||
dump_json,
|
||||
sha256_text,
|
||||
)
|
||||
|
||||
IMPL_VERSION = "build_chunks v1.0"
|
||||
SCHEMA_VERSION = "source-document@1/chunk@1"
|
||||
|
||||
HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$")
|
||||
WORD_RE = re.compile(r"[\u4e00-\u9fff]|[A-Za-z0-9]+")
|
||||
|
||||
|
||||
def parse_elements(text: str) -> list[dict]:
|
||||
"""把 Markdown/TXT 解析为 SourceDocument elements（标题/段落/列表/代码）。"""
|
||||
elements: list[dict] = []
|
||||
heading_stack: list[tuple[int, str]] = []
|
||||
in_code = False
|
||||
buf: list[str] = []
|
||||
buf_type = "paragraph"
|
||||
|
||||
def flush():
|
||||
nonlocal buf, buf_type
|
||||
content = "\n".join(buf).strip()
|
||||
buf = []
|
||||
if not content:
|
||||
return
|
||||
elements.append({
|
||||
"element_id": f"el-{len(elements):06d}",
|
||||
"type": buf_type,
|
||||
"text": content,
|
||||
"heading_path": [h for _, h in heading_stack],
|
||||
"page": None,
|
||||
"time_start_ms": None,
|
||||
"time_end_ms": None,
|
||||
"bbox": None,
|
||||
"asset_ref": None,
|
||||
"confidence": None,
|
||||
"content_hash": f"sha256:{sha256_text(content)}",
|
||||
})
|
||||
|
||||
for line in text.splitlines():
|
||||
if line.strip().startswith("```"):
|
||||
if in_code:
|
||||
buf.append(line)
|
||||
flush()
|
||||
buf_type = "paragraph"
|
||||
in_code = False
|
||||
else:
|
||||
flush()
|
||||
buf_type = "code"
|
||||
buf.append(line)
|
||||
in_code = True
|
||||
continue
|
||||
if in_code:
|
||||
buf.append(line)
|
||||
continue
|
||||
m = HEADING_RE.match(line)
|
||||
if m:
|
||||
flush()
|
||||
level, title = len(m.group(1)), m.group(2).strip()
|
||||
while heading_stack and heading_stack[-1][0] >= level:
|
||||
heading_stack.pop()
|
||||
heading_stack.append((level, title))
|
||||
buf_type = "heading"
|
||||
buf = [title]
|
||||
flush()
|
||||
buf_type = "paragraph"
|
||||
continue
|
||||
if not line.strip():
|
||||
flush()
|
||||
buf_type = "paragraph"
|
||||
continue
|
||||
if re.match(r"^\s*([-*+]|\d+\.)\s", line) and buf_type != "list":
|
||||
flush()
|
||||
buf_type = "list"
|
||||
buf.append(line)
|
||||
flush()
|
||||
return elements
|
||||
|
||||
|
||||
def extract_keywords(text: str, limit: int = 12) -> list[str]:
|
||||
"""确定性关键词预筛：词频最高的中文单字组成的双字串 + 英文词。够 FTS5 使用即可。"""
|
||||
freq: dict[str, int] = {}
|
||||
for m in re.finditer(r"[\u4e00-\u9fff]{2,6}|[A-Za-z][A-Za-z0-9-]{2,}", text):
|
||||
w = m.group(0).lower()
|
||||
freq[w] = freq.get(w, 0) + 1
|
||||
return [w for w, _ in sorted(freq.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]]
|
||||
|
||||
|
||||
def group_chunks(elements: list[dict], source_id: str, version_id: str, max_chars: int) -> list[dict]:
|
||||
chunks: list[dict] = []
|
||||
cur: list[dict] = []
|
||||
cur_path: list[str] = []
|
||||
|
||||
def flush():
|
||||
nonlocal cur
|
||||
if not cur:
|
||||
return
|
||||
text = "\n\n".join(e["text"] for e in cur)
|
||||
chunks.append({
|
||||
"chunk_id": f"ck-{sha256_text(version_id + text)[:12]}",
|
||||
"source_id": source_id,
|
||||
"version_id": version_id,
|
||||
"heading_path": cur[0]["heading_path"],
|
||||
"element_ids": [e["element_id"] for e in cur],
|
||||
"text": text,
|
||||
"char_count": len(text),
|
||||
"keywords": extract_keywords(text),
|
||||
"content_hash": f"sha256:{sha256_text(text)}",
|
||||
})
|
||||
cur = []
|
||||
|
||||
for el in elements:
|
||||
# 标题边界或超长时开新块
|
||||
if el["type"] == "heading" and cur:
|
||||
flush()
|
||||
if cur and sum(len(e["text"]) for e in cur) + len(el["text"]) > max_chars:
|
||||
flush()
|
||||
if el["heading_path"] != cur_path:
|
||||
cur_path = el["heading_path"]
|
||||
cur.append(el)
|
||||
flush()
|
||||
return chunks
|
||||
|
||||
|
||||
def build(source_path: Path, sidecar: Path, source_id: str, max_chars: int) -> dict:
|
||||
raw = source_path.read_text(encoding="utf-8")
|
||||
normalized = raw.replace("\r\n", "\n").strip() + "\n"
|
||||
version_id = f"sha256:{sha256_text(normalized)}"
|
||||
|
||||
cache_root = sidecar / "cache"
|
||||
key = deterministic_cache_key("build_chunks", IMPL_VERSION, SCHEMA_VERSION,
|
||||
[version_id], {"max_chars": max_chars, "source_id": source_id})
|
||||
cached = cache_lookup(cache_root, "build_chunks", key)
|
||||
if cached:
|
||||
doc = json.loads((cached / "document.json").read_text(encoding="utf-8"))
|
||||
chunk_lines = (cached / "chunks.jsonl").read_text(encoding="utf-8")
|
||||
print(f"[cache-hit] build_chunks {key[:12]}")
|
||||
else:
|
||||
elements = parse_elements(normalized)
|
||||
doc = {
|
||||
"schema_version": 1,
|
||||
"source_id": source_id,
|
||||
"version_id": version_id,
|
||||
"title": source_path.stem,
|
||||
"media_type": "markdown" if source_path.suffix.lower() in (".md", ".markdown") else "txt",
|
||||
"language": ["zh-CN"],
|
||||
"parser": IMPL_VERSION,
|
||||
"elements": elements,
|
||||
}
|
||||
chunks = group_chunks(elements, source_id, version_id, max_chars)
|
||||
chunk_lines = "".join(json.dumps(c, ensure_ascii=False) + "\n" for c in chunks)
|
||||
cache_store(cache_root, "build_chunks", key, {
|
||||
"document.json": json.dumps(doc, ensure_ascii=False, indent=1).encode("utf-8"),
|
||||
"chunks.jsonl": chunk_lines.encode("utf-8"),
|
||||
})
|
||||
|
||||
out_doc = sidecar / "normalized" / source_id / version_id.removeprefix("sha256:")[:16] / "document.json"
|
||||
dump_json(out_doc, doc)
|
||||
chunks_path = sidecar / "chunks" / "chunks.jsonl"
|
||||
chunks_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
chunks_path.write_text(chunk_lines, encoding="utf-8")
|
||||
|
||||
n_chunks = chunk_lines.count("\n")
|
||||
print(f"SourceDocument: {out_doc}\nchunks: {chunks_path}（{len(doc['elements'])} elements → {n_chunks} chunks）")
|
||||
return {"version_id": version_id, "elements": len(doc["elements"]), "chunks": n_chunks}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("source")
|
||||
ap.add_argument("--sidecar", required=True, help="侧车目录，如 books/<slug>/.cangjie")
|
||||
ap.add_argument("--source-id", default="src-main-book")
|
||||
ap.add_argument("--max-chars", type=int, default=4000)
|
||||
args = ap.parse_args()
|
||||
build(Path(args.source), Path(args.sidecar), args.source_id, args.max_chars)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""build_index.py — chunks.jsonl → SQLite FTS5 词法索引（Phase 2A MVP，不引入向量库）。
|
||||
|
||||
建索引: python3 scripts/build_index.py <chunks.jsonl> [--db <lexical.sqlite>]
|
||||
查询: python3 scripts/build_index.py <chunks.jsonl> --query "杠杆 复利" [--limit 5] [--neighbors 1]
|
||||
|
||||
查询返回命中块及其邻接块（防断章取义），供检索式 extractor（case/counter-example/glossary）取材。
|
||||
中文按字符 bigram 分词（FTS5 unicode61 对 CJK 不分词，入库前预处理）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
CJK_RE = re.compile(r"[\u4e00-\u9fff]+")
|
||||
|
||||
|
||||
def cjk_bigram(text: str) -> str:
|
||||
"""把连续中文串展开为 bigram 词序列，使 FTS5 能做中文子串匹配。"""
|
||||
|
||||
def expand(m: re.Match) -> str:
|
||||
s = m.group(0)
|
||||
if len(s) == 1:
|
||||
return s
|
||||
return " ".join(s[i : i + 2] for i in range(len(s) - 1))
|
||||
|
||||
return CJK_RE.sub(expand, text)
|
||||
|
||||
|
||||
def build(chunks_path: Path, db_path: Path) -> int:
|
||||
chunks = [json.loads(line) for line in chunks_path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
db_path.unlink(missing_ok=True)
|
||||
con = sqlite3.connect(db_path)
|
||||
con.execute("CREATE TABLE chunks (seq INTEGER PRIMARY KEY, chunk_id TEXT, heading_path TEXT, text TEXT, keywords TEXT)")
|
||||
con.execute("CREATE VIRTUAL TABLE chunks_fts USING fts5(chunk_id, heading_path, body, keywords)")
|
||||
for seq, c in enumerate(chunks):
|
||||
hp = " / ".join(c["heading_path"])
|
||||
con.execute("INSERT INTO chunks VALUES (?,?,?,?,?)",
|
||||
(seq, c["chunk_id"], hp, c["text"], " ".join(c.get("keywords", []))))
|
||||
con.execute("INSERT INTO chunks_fts VALUES (?,?,?,?)",
|
||||
(c["chunk_id"], cjk_bigram(hp), cjk_bigram(c["text"]), cjk_bigram(" ".join(c.get("keywords", [])))))
|
||||
con.commit()
|
||||
con.close()
|
||||
print(f"索引已建: {db_path}（{len(chunks)} chunks）")
|
||||
return 0
|
||||
|
||||
|
||||
def query(db_path: Path, q: str, limit: int, neighbors: int) -> int:
|
||||
con = sqlite3.connect(db_path)
|
||||
terms = " OR ".join(f'"{t}"' for t in cjk_bigram(q).split())
|
||||
rows = con.execute(
|
||||
"SELECT chunk_id FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY rank LIMIT ?", (terms, limit)
|
||||
).fetchall()
|
||||
if not rows:
|
||||
print("（无命中；建议放宽查询词或回退全量扫描）")
|
||||
return 0
|
||||
seen: set[int] = set()
|
||||
for (chunk_id,) in rows:
|
||||
(seq,) = con.execute("SELECT seq FROM chunks WHERE chunk_id=?", (chunk_id,)).fetchone()
|
||||
for s in range(max(0, seq - neighbors), seq + neighbors + 1):
|
||||
seen.add(s)
|
||||
for seq in sorted(seen):
|
||||
row = con.execute("SELECT chunk_id, heading_path, text FROM chunks WHERE seq=?", (seq,)).fetchone()
|
||||
if row:
|
||||
cid, hp, text = row
|
||||
print(f"\n===== {cid} [{hp}] =====\n{text[:1500]}")
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("chunks")
|
||||
ap.add_argument("--db", default=None)
|
||||
ap.add_argument("--query", default=None)
|
||||
ap.add_argument("--limit", type=int, default=5)
|
||||
ap.add_argument("--neighbors", type=int, default=1)
|
||||
args = ap.parse_args()
|
||||
|
||||
chunks_path = Path(args.chunks)
|
||||
db_path = Path(args.db) if args.db else chunks_path.parent.parent / "index" / "lexical.sqlite"
|
||||
if args.query:
|
||||
return query(db_path, args.query, args.limit, args.neighbors)
|
||||
return build(chunks_path, db_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""cangjie.py — 仓颉统一薄 CLI（路线 C：只做编排与确定性操作，蒸馏仍由 Agent 完成）。
|
||||
|
||||
子命令：
|
||||
doctor 环境自检（无网络、无重依赖场景可通过核心检查）
|
||||
migrate-legacy 旧 one-to-one pack + 人工能力映射 → Capability Bundle（.cangjie/capabilities/）
|
||||
compile Capability Bundle → single | compact pack（auto 决策 + 锁 + staging + 原子发布）
|
||||
replan-output 重新评估输出策略，只生成 side-by-side 预览，不迁移
|
||||
update 登记新来源 → diff → change-set → 影响分析 → 生成待处理 Agent 任务（不自动改 Skill）
|
||||
repair 校验失败案例 → 快照 → 生成诊断任务（语义修复由 Agent 完成后回归）
|
||||
rollback 恢复最近快照（--list 查看）
|
||||
eval 分发到触发评测（默认）或输出评测
|
||||
benchmark 分发到 benchmark.py
|
||||
|
||||
`compile` 的输入必须是已验证 Capability Bundle；本 CLI 不声称能从原始书籍一键蒸馏。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import ( # noqa: E402
|
||||
TOOL_VERSION,
|
||||
WriterLock,
|
||||
atomic_publish,
|
||||
create_run_workdir,
|
||||
dump_json,
|
||||
load_yaml,
|
||||
new_run_id,
|
||||
sha256_file,
|
||||
snapshot_dir,
|
||||
split_frontmatter,
|
||||
)
|
||||
|
||||
SCRIPTS = Path(__file__).parent
|
||||
|
||||
|
||||
# ---------- doctor ----------
|
||||
|
||||
def cmd_doctor(_args) -> int:
|
||||
ok = True
|
||||
print(f"{TOOL_VERSION}\npython: {sys.version.split()[0]}")
|
||||
for mod, required in (("yaml", True), ("tiktoken", False), ("jsonschema", False)):
|
||||
try:
|
||||
__import__(mod)
|
||||
print(f" [ok] {mod}")
|
||||
except ImportError:
|
||||
level = "缺失(必需)" if required else "缺失(可选)"
|
||||
print(f" [{'FAIL' if required else 'warn'}] {mod} {level}")
|
||||
ok = ok and not required
|
||||
for rel in ("capability-bundle.schema.json", "registry-entry.schema.json", "contracts/source-document.schema.json"):
|
||||
p = SCRIPTS.parent / "schemas" / rel
|
||||
print(f" [{'ok' if p.exists() else 'FAIL'}] schemas/{rel}")
|
||||
ok = ok and p.exists()
|
||||
print("doctor:", "PASS" if ok else "FAIL")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
# ---------- migrate-legacy ----------
|
||||
|
||||
ROUTER_GATE = {"independent_intent": False, "independent_contract": True, "independent_run": True,
|
||||
"independent_reuse": False, "independent_eval": True}
|
||||
PROMOTED_GATE = {k: True for k in ROUTER_GATE}
|
||||
|
||||
|
||||
def cmd_migrate_legacy(args) -> int:
|
||||
"""人工能力映射 + 旧 pack → Capability Bundle。语义字段全部来自映射，不猜测。"""
|
||||
import yaml as _yaml
|
||||
|
||||
m = load_yaml(Path(args.map))
|
||||
pack = Path(args.pack)
|
||||
bundle_dir = pack / ".cangjie" / "capabilities"
|
||||
(bundle_dir / "cards").mkdir(parents=True, exist_ok=True)
|
||||
(bundle_dir / "book").mkdir(exist_ok=True)
|
||||
|
||||
capabilities = []
|
||||
for cap in m["capabilities"]:
|
||||
src_md = pack / cap["slug"] / "SKILL.md"
|
||||
fm, body = split_frontmatter(src_md.read_text(encoding="utf-8"))
|
||||
dest = cap.get("pack_destination", "router")
|
||||
header = (
|
||||
f"<!-- capability_id: {cap['id']} | revision: 1 | status: active -->\n"
|
||||
f"<!-- 来源: {m['book']['source_pack']}/{cap['slug']}/SKILL.md（正文逐字保留） -->\n\n"
|
||||
)
|
||||
(bundle_dir / "cards" / f"{cap['slug']}.md").write_text(header + body, encoding="utf-8")
|
||||
capabilities.append({
|
||||
"capability_id": cap["id"],
|
||||
"revision": 1,
|
||||
"status": "active",
|
||||
"aliases": [],
|
||||
"supersedes": [],
|
||||
"merged_into": None,
|
||||
"split_into": [],
|
||||
"slug": cap["slug"],
|
||||
"title": cap["title"],
|
||||
"importance": cap["importance"],
|
||||
"importance_rationale": cap["importance_rationale"],
|
||||
"one_liner": cap["one_liner"],
|
||||
"intents": cap["intents"],
|
||||
"keywords": cap["keywords"],
|
||||
"also_read": cap.get("also_read", []),
|
||||
"card": f"cards/{cap['slug']}.md",
|
||||
"frontmatter": {"description": str(fm.get("description", "")).strip(),
|
||||
"tags": fm.get("tags", [])},
|
||||
"source_evidence": [{"source_id": "src-main-book", "location": str(fm.get("source_chapter", ""))}],
|
||||
"promotion": {"destination": dest,
|
||||
"gate": PROMOTED_GATE if dest == "promoted" else dict(ROUTER_GATE),
|
||||
"notes": cap.get("promotion_notes", "")},
|
||||
})
|
||||
|
||||
bundle = {
|
||||
"schema_version": 1,
|
||||
"bundle_id": f"bundle.{m['entry']['name']}",
|
||||
"book": m["book"],
|
||||
"entry": m["entry"],
|
||||
"router_entry": m["router_entry"],
|
||||
"promotion_budget": m.get("promotion_budget", 8),
|
||||
"capabilities": capabilities,
|
||||
}
|
||||
(bundle_dir / "verified.yaml").write_text(
|
||||
_yaml.safe_dump(bundle, allow_unicode=True, sort_keys=False, width=120), encoding="utf-8")
|
||||
|
||||
dump_json(bundle_dir / "destinations.json", {
|
||||
"bundle_id": bundle["bundle_id"],
|
||||
"destinations": {c["capability_id"]: ({"promoted_to": c["slug"]} if c["promotion"]["destination"] == "promoted"
|
||||
else {"served_by": m["router_entry"]["name"]})
|
||||
for c in capabilities},
|
||||
})
|
||||
|
||||
for src_name, dst_name in (("BOOK_OVERVIEW.md", "overview.md"), ("GLOSSARY.md", "glossary.md")):
|
||||
src = pack / src_name
|
||||
if src.exists():
|
||||
shutil.copyfile(src, bundle_dir / "book" / dst_name)
|
||||
|
||||
print(f"Capability Bundle 已生成: {bundle_dir}（{len(capabilities)} 个能力）")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------- compile ----------
|
||||
|
||||
def cmd_compile(args) -> int:
|
||||
from compile_pack import compile_pack_tree
|
||||
from compile_single import build_tree, write_tree
|
||||
from select_output_strategy import decide, render_report
|
||||
|
||||
bundle_dir = Path(args.bundle)
|
||||
sidecar = bundle_dir.parent if bundle_dir.name == "capabilities" else bundle_dir
|
||||
run_id = new_run_id()
|
||||
workdir = create_run_workdir(sidecar, run_id)
|
||||
|
||||
decision = decide(bundle_dir, args.output, args.purpose)
|
||||
dump_json(workdir / "output-decision.json", decision)
|
||||
(workdir / "output-decision.md").write_text(render_report(decision, bundle_dir), encoding="utf-8")
|
||||
print(render_report(decision, bundle_dir))
|
||||
if args.output == "auto" and not args.yes:
|
||||
print("auto 模式需要用户轻确认；确认后请加 --yes 重新运行（按推荐），或显式 --output single|pack。")
|
||||
return 2
|
||||
|
||||
selected = decision["selected"]
|
||||
files = build_tree(bundle_dir, "single") if selected == "single" else compile_pack_tree(
|
||||
bundle_dir, allow_over_budget=args.allow_over_budget)
|
||||
|
||||
target = Path(args.out)
|
||||
staging = target.parent / f".staging-{run_id}-{target.name}"
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
write_tree(files, staging)
|
||||
|
||||
check = subprocess.run([sys.executable, str(SCRIPTS / "validate_skill_pack.py"), str(staging)],
|
||||
capture_output=True, text=True)
|
||||
(workdir / "staging-validation.log").write_text(check.stdout + check.stderr, encoding="utf-8")
|
||||
if check.returncode != 0:
|
||||
print(check.stdout + check.stderr)
|
||||
raise SystemExit("[hard-gate] staging 校验未通过，已保留 staging 供排查，不发布")
|
||||
|
||||
with WriterLock(target):
|
||||
if target.exists():
|
||||
snap = snapshot_dir(target, sidecar / "snapshots", f"pre-{run_id}")
|
||||
print(f"已快照旧版本: {snap}")
|
||||
atomic_publish(staging, target, {
|
||||
"bundle_id": load_yaml(bundle_dir / "verified.yaml")["bundle_id"],
|
||||
"bundle_sha256": sha256_file(bundle_dir / "verified.yaml"),
|
||||
"variant": selected,
|
||||
"run_id": run_id,
|
||||
"decision": decision,
|
||||
}, allow_overwrite_edits=args.force_overwrite)
|
||||
|
||||
print(f"已原子发布 {selected} 产物到 {target}（run: {run_id}）")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------- replan-output ----------
|
||||
|
||||
def cmd_replan_output(args) -> int:
|
||||
from select_output_strategy import decide, render_report
|
||||
|
||||
bundle_dir = Path(args.pack) / ".cangjie" / "capabilities"
|
||||
decision = decide(bundle_dir, "auto", args.purpose)
|
||||
print(render_report(decision, bundle_dir))
|
||||
print("（dry-run 预览：未做任何迁移；确认变更请显式运行 compile 并选择新的 --output）")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------- update / repair / rollback / eval / benchmark ----------
|
||||
|
||||
def cmd_update(args) -> int:
|
||||
from update_flow import run_update # Phase 2B
|
||||
|
||||
return run_update(Path(args.pack), Path(args.add))
|
||||
|
||||
|
||||
def cmd_repair(args) -> int:
|
||||
from repair_flow import run_repair # Phase 3
|
||||
|
||||
return run_repair(Path(args.pack), Path(args.case))
|
||||
|
||||
|
||||
def cmd_rollback(args) -> int:
|
||||
sidecar = Path(args.pack) / ".cangjie"
|
||||
snaps = sorted((sidecar / "snapshots").glob("*")) if (sidecar / "snapshots").exists() else []
|
||||
if args.list or not args.to:
|
||||
print("可用快照：" + ("\n " + "\n ".join(s.name for s in snaps) if snaps else " (无)"))
|
||||
return 0
|
||||
snap = sidecar / "snapshots" / args.to
|
||||
if not snap.is_dir():
|
||||
raise SystemExit(f"快照不存在: {snap}")
|
||||
target = Path(args.target)
|
||||
with WriterLock(target):
|
||||
if target.exists():
|
||||
snapshot_dir(target, sidecar / "snapshots", "pre-rollback")
|
||||
shutil.rmtree(target)
|
||||
shutil.copytree(snap, target)
|
||||
print(f"已回滚 {target} ← {snap.name}")
|
||||
return 0
|
||||
|
||||
|
||||
def _dispatch(script: str, extra: list[str]) -> int:
|
||||
return subprocess.call([sys.executable, str(SCRIPTS / script), *extra])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# 这两个命令需要把未知选项原样透传给子工具。argparse 的子解析器
|
||||
# 会在 `--token-config` 这类选项上提前报错，因此在根解析前直接分发。
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "benchmark":
|
||||
return _dispatch("benchmark.py", sys.argv[2:])
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "eval":
|
||||
extra = sys.argv[2:]
|
||||
script = "run_trigger_evals.py"
|
||||
if extra and extra[0] in {"trigger", "output"}:
|
||||
script = "run_output_evals.py" if extra[0] == "output" else script
|
||||
extra = extra[1:]
|
||||
return _dispatch(script, extra)
|
||||
|
||||
ap = argparse.ArgumentParser(prog="cangjie", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("doctor")
|
||||
|
||||
p = sub.add_parser("migrate-legacy")
|
||||
p.add_argument("--map", required=True, help="人工能力映射 capability-map.yaml")
|
||||
p.add_argument("--pack", required=True, help="旧 one-to-one pack 目录")
|
||||
|
||||
p = sub.add_parser("compile")
|
||||
p.add_argument("--bundle", required=True)
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--output", choices=["auto", "single", "pack"], default="auto")
|
||||
p.add_argument("--purpose", choices=["learning", "reference", "workflow", "distribution"], default=None)
|
||||
p.add_argument("--yes", action="store_true", help="auto 模式下按推荐执行（轻确认）")
|
||||
p.add_argument("--allow-over-budget", action="store_true")
|
||||
p.add_argument("--force-overwrite", action="store_true", help="丢弃已发布目录中的本地手工修改")
|
||||
|
||||
p = sub.add_parser("replan-output")
|
||||
p.add_argument("--pack", required=True)
|
||||
p.add_argument("--purpose", choices=["learning", "reference", "workflow", "distribution"], default=None)
|
||||
p.add_argument("--dry-run", action="store_true", default=True)
|
||||
|
||||
p = sub.add_parser("update")
|
||||
p.add_argument("--pack", required=True)
|
||||
p.add_argument("--add", required=True, help="新增来源文件（Markdown/TXT）")
|
||||
|
||||
p = sub.add_parser("repair")
|
||||
p.add_argument("--pack", required=True)
|
||||
p.add_argument("--case", required=True, help="failure-case YAML（failure-case.schema.json）")
|
||||
|
||||
p = sub.add_parser("rollback")
|
||||
p.add_argument("--pack", required=True)
|
||||
p.add_argument("--target", help="要恢复的已发布目录")
|
||||
p.add_argument("--to", default=None)
|
||||
p.add_argument("--list", action="store_true")
|
||||
|
||||
for name, script in (("eval", "run_trigger_evals.py"), ("benchmark", "benchmark.py")):
|
||||
p = sub.add_parser(name, add_help=False)
|
||||
p.add_argument("extra", nargs=argparse.REMAINDER)
|
||||
p.set_defaults(script=script)
|
||||
|
||||
args = ap.parse_args()
|
||||
if args.cmd == "doctor":
|
||||
return cmd_doctor(args)
|
||||
if args.cmd == "migrate-legacy":
|
||||
return cmd_migrate_legacy(args)
|
||||
if args.cmd == "compile":
|
||||
return cmd_compile(args)
|
||||
if args.cmd == "replan-output":
|
||||
return cmd_replan_output(args)
|
||||
if args.cmd == "update":
|
||||
return cmd_update(args)
|
||||
if args.cmd == "repair":
|
||||
return cmd_repair(args)
|
||||
if args.cmd == "rollback":
|
||||
return cmd_rollback(args)
|
||||
return _dispatch(args.script, args.extra)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""cangjie_common.py — 仓颉确定性脚本的共享工具（路线 C：纯本地、不调模型）。
|
||||
|
||||
提供：frontmatter 解析、哈希、确定性缓存键、per-run workdir、writer lock、
|
||||
staging + 原子发布、发布哈希登记与本地手改检测（方案 §4.4/§4.6.3/§13A 非功能矩阵）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
TOOL_VERSION = "cangjie-tools v2.5.0"
|
||||
|
||||
|
||||
# ---------- 基础 IO ----------
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
return sha256_bytes(text.encode("utf-8"))
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path}: 期望 YAML mapping")
|
||||
return data
|
||||
|
||||
|
||||
def dump_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def load_json(path: Path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
# ---------- frontmatter ----------
|
||||
|
||||
def split_frontmatter(text: str) -> tuple[dict, str]:
|
||||
"""返回 (frontmatter dict, body)。无 frontmatter 时返回 ({}, 原文)。"""
|
||||
if text.startswith("---"):
|
||||
end = text.find("\n---", 3)
|
||||
if end != -1:
|
||||
fm = yaml.safe_load(text[3:end])
|
||||
if isinstance(fm, dict):
|
||||
return fm, text[end + 4 :].lstrip("\n")
|
||||
return {}, text
|
||||
|
||||
|
||||
# ---------- 确定性缓存（方案 §4.4 A 类） ----------
|
||||
|
||||
def deterministic_cache_key(stage_name: str, implementation_version: str, stage_schema_version: str,
|
||||
ordered_input_hashes: list[str], normalized_parameters: dict) -> str:
|
||||
payload = "\n".join([
|
||||
stage_name,
|
||||
implementation_version,
|
||||
stage_schema_version,
|
||||
*ordered_input_hashes,
|
||||
json.dumps(normalized_parameters, ensure_ascii=False, sort_keys=True),
|
||||
])
|
||||
return sha256_text(payload)
|
||||
|
||||
|
||||
def cache_lookup(cache_root: Path, stage: str, key: str) -> Path | None:
|
||||
p = cache_root / stage / key
|
||||
return p if p.is_dir() else None
|
||||
|
||||
|
||||
def cache_store(cache_root: Path, stage: str, key: str, files: dict[str, bytes]) -> Path:
|
||||
"""临时目录写入后原子 rename，避免半写缓存。"""
|
||||
final = cache_root / stage / key
|
||||
if final.exists():
|
||||
return final
|
||||
tmp = cache_root / stage / f".tmp-{key}-{uuid.uuid4().hex[:8]}"
|
||||
tmp.mkdir(parents=True)
|
||||
for rel, data in files.items():
|
||||
f = tmp / rel
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
f.write_bytes(data)
|
||||
try:
|
||||
tmp.rename(final)
|
||||
except OSError:
|
||||
shutil.rmtree(tmp, ignore_errors=True) # 并发下另一个 writer 已完成
|
||||
return final
|
||||
|
||||
|
||||
# ---------- run workdir / writer lock（方案 §15.17） ----------
|
||||
|
||||
def new_run_id() -> str:
|
||||
return time.strftime("run-%Y%m%d-%H%M%S-") + uuid.uuid4().hex[:6]
|
||||
|
||||
|
||||
def create_run_workdir(sidecar: Path, run_id: str | None = None) -> Path:
|
||||
run_id = run_id or new_run_id()
|
||||
workdir = sidecar / "runs" / run_id
|
||||
workdir.mkdir(parents=True, exist_ok=False)
|
||||
return workdir
|
||||
|
||||
|
||||
class WriterLock:
|
||||
"""同一目标 pack 同时只允许一个 writer。O_EXCL 创建锁文件，崩溃后可依据 pid/时间人工清理。"""
|
||||
|
||||
def __init__(self, target: Path):
|
||||
self.lock_path = target.with_name(target.name + ".cangjie-lock")
|
||||
|
||||
def __enter__(self):
|
||||
try:
|
||||
fd = os.open(self.lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
except FileExistsError:
|
||||
raise SystemExit(
|
||||
f"目标已被另一个 writer 锁定: {self.lock_path}\n"
|
||||
f"若确认无并发运行，删除该锁文件后重试。"
|
||||
)
|
||||
os.write(fd, f"pid={os.getpid()} time={time.strftime('%F %T')}\n".encode())
|
||||
os.close(fd)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.lock_path.unlink(missing_ok=True)
|
||||
return False
|
||||
|
||||
|
||||
# ---------- staging + 原子发布 + 手改检测（方案 §4.6.3/§6.5） ----------
|
||||
|
||||
MANIFEST_NAME = "BUILD_MANIFEST.json"
|
||||
|
||||
|
||||
def collect_published_hashes(out_dir: Path) -> dict[str, str]:
|
||||
return {
|
||||
str(p.relative_to(out_dir)): sha256_file(p)
|
||||
for p in sorted(out_dir.rglob("*"))
|
||||
if p.is_file() and p.name != MANIFEST_NAME
|
||||
}
|
||||
|
||||
|
||||
def detect_local_edits(target: Path) -> list[str]:
|
||||
"""比对目标目录当前文件与 BUILD_MANIFEST 发布哈希，返回被手工修改/删除的文件列表。"""
|
||||
manifest_path = target / MANIFEST_NAME
|
||||
if not manifest_path.exists():
|
||||
return []
|
||||
published = load_json(manifest_path).get("published_hashes", {})
|
||||
edited = []
|
||||
for rel, digest in published.items():
|
||||
f = target / rel
|
||||
if not f.exists():
|
||||
edited.append(f"{rel} (已删除)")
|
||||
elif sha256_file(f) != digest:
|
||||
edited.append(rel)
|
||||
return edited
|
||||
|
||||
|
||||
def atomic_publish(staging: Path, target: Path, manifest_extra: dict, *, allow_overwrite_edits: bool = False) -> None:
|
||||
"""staging 校验通过后原子替换发布目录。检测到本地手改且未显式允许时中止（三选一保护）。"""
|
||||
edits = detect_local_edits(target)
|
||||
if edits and not allow_overwrite_edits:
|
||||
raise SystemExit(
|
||||
"检测到已发布目录中的本地手工修改，拒绝静默覆盖：\n - "
|
||||
+ "\n - ".join(edits)
|
||||
+ "\n请三选一：\n 1) --force-overwrite 丢弃本地修改\n 2) 把修改回填 Capability Bundle 后重编译\n 3) 中止（当前行为）"
|
||||
)
|
||||
manifest = {
|
||||
"tool": TOOL_VERSION,
|
||||
"published_hashes": collect_published_hashes(staging),
|
||||
"note": "update/重编译前比对 published_hashes；检测到本地手改不得静默覆盖",
|
||||
**manifest_extra,
|
||||
}
|
||||
dump_json(staging / MANIFEST_NAME, manifest)
|
||||
backup = None
|
||||
if target.exists():
|
||||
backup = target.with_name(target.name + f".prev-{uuid.uuid4().hex[:6]}")
|
||||
target.rename(backup)
|
||||
try:
|
||||
staging.rename(target)
|
||||
except OSError:
|
||||
if backup is not None:
|
||||
backup.rename(target) # 发布失败，恢复旧版本
|
||||
raise
|
||||
if backup is not None:
|
||||
shutil.rmtree(backup)
|
||||
|
||||
|
||||
def snapshot_dir(src: Path, snapshots_root: Path, label: str) -> Path:
|
||||
"""发布前快照，供 rollback 使用（RPO=0）。"""
|
||||
dest = snapshots_root / f"{time.strftime('%Y%m%d-%H%M%S')}-{label}"
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(src, dest)
|
||||
return dest
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""compile_pack.py — 从 Capability Bundle 确定性编译 compact pack（方案 §4.6.3）。
|
||||
|
||||
产物固定为「1 个来源路由入口 + 少量晋级 Skill + 内部能力卡」：
|
||||
|
||||
<out>/
|
||||
├── <router-name>/ # 来源路由入口（全部能力卡保留在其目录内）
|
||||
│ ├── SKILL.md
|
||||
│ └── references/...
|
||||
├── <promoted-slug>/ # 晋级 Skill：从同一 Bundle 编译的自包含入口
|
||||
│ └── SKILL.md
|
||||
└── capability-destinations.json # 发布审计清单（不作为宿主发现入口）
|
||||
|
||||
硬不变量（编译时校验，违反即失败）：
|
||||
1. 每个 active capability 恰好一个主要去向（promoted_to 或 served_by）；
|
||||
2. 未晋级能力必须可经来源路由入口到达（能力卡 + 索引行都存在）；
|
||||
3. 晋级 Skill 自包含，不引用跨 Skill 根目录的相对路径；
|
||||
4. 可发现入口总数 <= promotion_budget（超出需 --allow-over-budget 并逐项解释）。
|
||||
|
||||
用法: python3 scripts/compile_pack.py --bundle <dir> --out <dir> [--allow-over-budget]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import TOOL_VERSION, dump_json, load_yaml # noqa: E402
|
||||
from compile_single import active_capabilities, build_tree, promoted_capabilities, write_tree # noqa: E402
|
||||
|
||||
|
||||
def build_promoted_skill_md(cap: dict, bundle: dict, card_text: str) -> str:
|
||||
fm = cap.get("frontmatter", {})
|
||||
description = fm.get("description") or f"{'；'.join(cap['intents'])}。{cap['one_liner']}"
|
||||
tags = fm.get("tags", [])
|
||||
lines = [
|
||||
"---",
|
||||
f"name: {cap['slug']}",
|
||||
"description: |",
|
||||
*[f" {line}" for line in description.strip().splitlines()],
|
||||
"metadata:",
|
||||
f" cangjie.generated-by: {TOOL_VERSION}",
|
||||
f" cangjie.capability-id: {cap['capability_id']}",
|
||||
f" cangjie.capability-revision: {cap['revision']}",
|
||||
f" cangjie.bundle-id: {bundle['bundle_id']}",
|
||||
f" cangjie.source-title: {bundle['book']['title']}",
|
||||
]
|
||||
if tags:
|
||||
lines.append(f" cangjie.tags: {', '.join(tags)}")
|
||||
lines += ["---", ""]
|
||||
return "\n".join(lines) + card_text
|
||||
|
||||
|
||||
def compile_pack_tree(bundle_dir: Path, *, allow_over_budget: bool = False) -> dict[str, str]:
|
||||
"""返回 {相对路径: 内容}，含路由入口、晋级 Skill 与 destinations 审计清单。"""
|
||||
bundle = load_yaml(bundle_dir / "verified.yaml")
|
||||
caps = active_capabilities(bundle)
|
||||
promoted = promoted_capabilities(bundle)
|
||||
router_name = bundle["router_entry"]["name"]
|
||||
budget = int(bundle.get("promotion_budget", 8))
|
||||
|
||||
# 不变量 1：每个 active capability 恰好一个主要去向
|
||||
for cap in caps:
|
||||
dest = cap.get("promotion", {}).get("destination")
|
||||
if dest not in ("promoted", "router"):
|
||||
raise SystemExit(f"[invariant] {cap['capability_id']}: promotion.destination 必须是 promoted|router，当前 {dest!r}")
|
||||
|
||||
# 不变量 4：入口预算
|
||||
entrypoint_count = 1 + len(promoted)
|
||||
if entrypoint_count > budget and not allow_over_budget:
|
||||
raise SystemExit(f"[invariant] 可发现入口 {entrypoint_count} 超出软预算 {budget}；确需超出请 --allow-over-budget 并在发布说明逐项解释")
|
||||
|
||||
files: dict[str, str] = {}
|
||||
|
||||
# 来源路由入口（复用 single 编译逻辑的 router 视图；全部能力卡保留）
|
||||
for rel, content in build_tree(bundle_dir, "router").items():
|
||||
files[f"{router_name}/{rel}"] = content
|
||||
|
||||
# 晋级 Skill：自包含，不引用跨目录路径（不变量 3 由内容构造保证 + 校验兜底）
|
||||
for cap in promoted:
|
||||
card_text = (bundle_dir / cap["card"]).read_text(encoding="utf-8")
|
||||
skill_md = build_promoted_skill_md(cap, bundle, card_text)
|
||||
if "references/capabilities/" in skill_md or f"../{router_name}" in skill_md:
|
||||
raise SystemExit(f"[invariant] 晋级 Skill {cap['slug']} 引用了跨目录路径，必须自包含")
|
||||
files[f"{cap['slug']}/SKILL.md"] = skill_md
|
||||
|
||||
# 不变量 2：未晋级能力在路由入口可达
|
||||
for cap in caps:
|
||||
if cap["promotion"]["destination"] == "router":
|
||||
card_rel = f"{router_name}/references/capabilities/{cap['slug']}.md"
|
||||
if card_rel not in files:
|
||||
raise SystemExit(f"[invariant] 未晋级能力 {cap['capability_id']} 在路由入口不可达（缺 {card_rel}）")
|
||||
|
||||
destinations = {
|
||||
"generated_by": TOOL_VERSION,
|
||||
"bundle_id": bundle["bundle_id"],
|
||||
"router_entrypoint": router_name,
|
||||
"entrypoint_count": entrypoint_count,
|
||||
"capability_count": len(caps),
|
||||
"destinations": {
|
||||
cap["capability_id"]: (
|
||||
{"promoted_to": cap["slug"], "router_view": f"{router_name}/references/capabilities/{cap['slug']}.md"}
|
||||
if cap["promotion"]["destination"] == "promoted"
|
||||
else {"served_by": router_name, "card": f"{router_name}/references/capabilities/{cap['slug']}.md"}
|
||||
)
|
||||
for cap in caps
|
||||
},
|
||||
}
|
||||
import json
|
||||
|
||||
files["capability-destinations.json"] = json.dumps(destinations, ensure_ascii=False, indent=2) + "\n"
|
||||
return files
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--bundle", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--allow-over-budget", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
files = compile_pack_tree(Path(args.bundle), allow_over_budget=args.allow_over_budget)
|
||||
write_tree(files, Path(args.out))
|
||||
entry_count = sum(1 for rel in files if rel.endswith("/SKILL.md"))
|
||||
print(f"已生成 compact pack: {args.out}（{entry_count} 个入口，{len(files)} 个文件）")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""compile_single.py — 从 Capability Bundle 确定性编译 single 入口（v0.2，Bundle 原生）。
|
||||
|
||||
输入是唯一编译事实源 Capability Bundle（ADR-002）：
|
||||
|
||||
<bundle-dir>/ # 通常是 books/<slug>/.cangjie/capabilities/
|
||||
├── verified.yaml # capability-bundle.schema.json
|
||||
├── destinations.json # 去向映射
|
||||
├── cards/<slug>.md # RIA 能力卡（R/I/A1/A2/E/B，正文逐字保留）
|
||||
└── book/{overview.md,glossary.md}
|
||||
|
||||
输出（--variant single）：1 个发现入口 + 全部能力卡 + overview/glossary/cheatsheet/索引。
|
||||
--variant router 生成 compact pack 的来源路由入口视图（晋级能力在路由表标注改由独立 Skill 处理）。
|
||||
|
||||
本脚本纯确定性、不调用模型。主入口 description、核心原则、意图路由全部来自 Bundle，脚本不猜测。
|
||||
v0.1 的 --map 模式（Phase 0 原型）已由 `cangjie.py migrate-legacy` + Bundle 取代。
|
||||
|
||||
用法: python3 scripts/compile_single.py --bundle <dir> --out <dir> [--variant single|router]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import TOOL_VERSION, load_yaml # noqa: E402
|
||||
|
||||
MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)#\s]+?)(?:#[^)\s]*)?\)")
|
||||
|
||||
|
||||
def sanitize_links(text: str, available: set[str]) -> str:
|
||||
"""把指向未随产物分发文件的相对链接降级为纯文本，避免死链。"""
|
||||
|
||||
def repl(m: re.Match) -> str:
|
||||
target = m.group(2)
|
||||
if target.startswith(("http://", "https://", "mailto:", "/")):
|
||||
return m.group(0)
|
||||
return m.group(0) if target.lstrip("./") in available else m.group(1)
|
||||
|
||||
return MD_LINK_RE.sub(repl, text)
|
||||
|
||||
|
||||
def active_capabilities(bundle: dict) -> list[dict]:
|
||||
return [c for c in bundle["capabilities"] if c.get("status", "active") == "active"]
|
||||
|
||||
|
||||
def promoted_capabilities(bundle: dict) -> list[dict]:
|
||||
return [c for c in active_capabilities(bundle) if c.get("promotion", {}).get("destination") == "promoted"]
|
||||
|
||||
|
||||
def router_table(caps: list[dict], variant: str) -> str:
|
||||
rows = ["| 用户意图 | 先读 | 补读/备注 |", "|---|---|---|"]
|
||||
for cap in caps:
|
||||
intents = "；".join(cap["intents"])
|
||||
card = f"references/capabilities/{cap['slug']}.md"
|
||||
extra = "、".join(f"references/capabilities/{s}.md" for s in cap.get("also_read", [])) or "—"
|
||||
if variant == "router" and cap.get("promotion", {}).get("destination") == "promoted":
|
||||
extra = f"已晋级为独立 Skill `{cap['slug']}`（已安装时优先直接使用；本卡仅作原文与背景补充）"
|
||||
rows.append(f"| {intents} | {card} | {extra} |")
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def build_entry_md(bundle: dict, variant: str) -> str:
|
||||
entry = bundle["entry"] if variant == "single" else bundle["router_entry"]
|
||||
caps = active_capabilities(bundle)
|
||||
book = bundle["book"]
|
||||
entrypoint_count = 1 if variant == "single" else 1 + len(promoted_capabilities(bundle))
|
||||
fm_lines = [
|
||||
"---",
|
||||
f"name: {entry['name']}",
|
||||
"description: |",
|
||||
*[f" {line}" for line in entry["description"].strip().splitlines()],
|
||||
"metadata:",
|
||||
f" cangjie.generated-by: {TOOL_VERSION}",
|
||||
f" cangjie.variant: {variant}",
|
||||
f" cangjie.bundle-id: {bundle['bundle_id']}",
|
||||
f" cangjie.capability-count: {len(caps)}",
|
||||
f" cangjie.entrypoint-count: {entrypoint_count}",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
e = bundle["entry"]
|
||||
principles = "\n".join(f"{i}. {p}" for i, p in enumerate(e["core_principles"], 1))
|
||||
out_of_scope = "\n".join(f"- {x}" for x in e["out_of_scope"])
|
||||
stops = "\n".join(f"- {x}" for x in e["stop_conditions"])
|
||||
|
||||
body = f"""# {book['title']} — {'全书能力入口' if variant == 'single' else '来源路由入口（compact pack）'}
|
||||
|
||||
## 触发与不触发
|
||||
|
||||
**适用**：与本书能力域相关的咨询与任务（见下方路由表的意图列）。
|
||||
**不适用**：
|
||||
{out_of_scope}
|
||||
|
||||
## 核心原则（常驻速览，概览类问题读到这里即可回答）
|
||||
|
||||
{principles}
|
||||
|
||||
## 能力路由（先读本表，按意图加载 1 张能力卡）
|
||||
|
||||
{router_table(caps, variant)}
|
||||
|
||||
**非能力类查询**：
|
||||
- 书名/作者/章节/整书概览 → references/overview.md
|
||||
- 术语解释 → references/glossary.md
|
||||
- 决策规则速查（不需要原文依据时） → references/cheatsheet.md
|
||||
- 完整意图与关键词索引（本表未覆盖的意图先查这里） → references/capability-index.md
|
||||
|
||||
## 加载规则
|
||||
|
||||
- 每次任务先读本文件，再按路由表加载 **1** 张能力卡；任务明确跨域时最多加载 2 张。
|
||||
- 概览/书名类问题不加载能力卡，用「核心原则」与 overview.md 回答。
|
||||
- 路由表与 capability-index.md 都无法命中的意图，明确告知超出本书范围，不要硬套。
|
||||
|
||||
## 边界与判停
|
||||
|
||||
{stops}
|
||||
"""
|
||||
return "\n".join(fm_lines) + body
|
||||
|
||||
|
||||
def build_index_md(caps: list[dict]) -> str:
|
||||
rows = ["# 能力索引（完整版）", "", "| capability_id | 标题 | 重要度 | 意图 | 关键词 | 能力卡 |", "|---|---|---|---|---|---|"]
|
||||
for c in caps:
|
||||
rows.append(
|
||||
f"| {c['capability_id']} | {c['title']} | {c['importance']} | {'；'.join(c['intents'])} | "
|
||||
f"{'、'.join(c['keywords'])} | capabilities/{c['slug']}.md |"
|
||||
)
|
||||
return "\n".join(rows) + "\n"
|
||||
|
||||
|
||||
def build_cheatsheet_md(caps: list[dict], book: dict) -> str:
|
||||
rows = [f"# 决策规则速查 — {book['title']}", "", "| 能力 | 一句话规则 |", "|---|---|"]
|
||||
for c in caps:
|
||||
rows.append(f"| {c['title']} | {c['one_liner']} |")
|
||||
rows.append("")
|
||||
rows.append("> 速查只给结论；需要原文依据、案例或反例时读对应能力卡。")
|
||||
return "\n".join(rows) + "\n"
|
||||
|
||||
|
||||
def build_tree(bundle_dir: Path, variant: str) -> dict[str, str]:
|
||||
"""返回 {相对路径: 内容}。纯函数，供 compile_pack.py 与 cangjie.py 复用。"""
|
||||
bundle = load_yaml(bundle_dir / "verified.yaml")
|
||||
caps = active_capabilities(bundle)
|
||||
|
||||
files: dict[str, str] = {}
|
||||
files["SKILL.md"] = build_entry_md(bundle, variant)
|
||||
for cap in caps:
|
||||
files[f"references/capabilities/{cap['slug']}.md"] = (bundle_dir / cap["card"]).read_text(encoding="utf-8")
|
||||
files["references/capability-index.md"] = build_index_md(caps)
|
||||
files["references/cheatsheet.md"] = build_cheatsheet_md(caps, bundle["book"])
|
||||
|
||||
available = {p.removeprefix("references/") for p in files if p.startswith("references/")}
|
||||
for name in ("overview.md", "glossary.md"):
|
||||
src = bundle_dir / "book" / name
|
||||
if src.exists():
|
||||
files[f"references/{name}"] = sanitize_links(src.read_text(encoding="utf-8"), available)
|
||||
return files
|
||||
|
||||
|
||||
def write_tree(files: dict[str, str], out: Path) -> None:
|
||||
for rel, content in files.items():
|
||||
p = out / rel
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--bundle", required=True, help="Capability Bundle 目录（含 verified.yaml）")
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--variant", choices=["single", "router"], default="single")
|
||||
args = ap.parse_args()
|
||||
|
||||
files = build_tree(Path(args.bundle), args.variant)
|
||||
write_tree(files, Path(args.out))
|
||||
print(f"已生成 {args.variant} 产物: {args.out}（{len(files)} 个文件）")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""count_tokens.py — A 类静态 Token 指标计算（Phase 0 工具，纯确定性，不调用模型）。
|
||||
|
||||
对三种版本形态计算：
|
||||
- discovery_payload: 全部入口 SKILL.md 的 frontmatter name+description token 数之和（常驻发现目录成本）；
|
||||
- per_task_payload: 单任务静态加载模型（见各 type 的定义，输出 min/median/max）；
|
||||
- corpus_total: 版本目录内全部 .md 的 token 总量。
|
||||
|
||||
加载模型（静态假设，写死并随结果一起输出）：
|
||||
- atomic_pack: 命中 1 个 Skill = 该 Skill 完整 SKILL.md；
|
||||
- single: 任务 = 入口 SKILL.md + 1 张能力卡；upper = 入口 + 最大卡 + capability-index.md；
|
||||
- compact_pack: 晋级命中 = 晋级 Skill 完整 SKILL.md；路由命中 = 路由入口 SKILL.md + 1 张能力卡。
|
||||
|
||||
用法: python3 scripts/count_tokens.py <config.json> [--out <dir>]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import tiktoken
|
||||
import yaml
|
||||
|
||||
ENCODINGS = {}
|
||||
|
||||
|
||||
def tok(text: str, enc_name: str) -> int:
|
||||
if enc_name not in ENCODINGS:
|
||||
ENCODINGS[enc_name] = tiktoken.get_encoding(enc_name)
|
||||
return len(ENCODINGS[enc_name].encode(text))
|
||||
|
||||
|
||||
def frontmatter_name_desc(path: Path) -> str:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if not text.startswith("---"):
|
||||
return ""
|
||||
end = text.find("\n---", 3)
|
||||
fm = yaml.safe_load(text[3:end])
|
||||
if not isinstance(fm, dict):
|
||||
return ""
|
||||
return f"{fm.get('name', '')}\n{fm.get('description', '')}"
|
||||
|
||||
|
||||
def file_tokens(path: Path, enc: str) -> int:
|
||||
return tok(path.read_text(encoding="utf-8"), enc)
|
||||
|
||||
|
||||
def stats(values: list[int]) -> dict:
|
||||
return {
|
||||
"min": min(values),
|
||||
"median": int(statistics.median(values)),
|
||||
"max": max(values),
|
||||
"n": len(values),
|
||||
}
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
|
||||
|
||||
|
||||
def measure_version(name: str, spec: dict, enc: str) -> dict:
|
||||
root = Path(spec["root"])
|
||||
vtype = spec["type"]
|
||||
result: dict = {"type": vtype, "root": str(root)}
|
||||
|
||||
if vtype == "atomic_pack":
|
||||
entries = sorted(root.glob(spec.get("entry_glob", "*/SKILL.md")))
|
||||
result["entrypoint_count"] = len(entries)
|
||||
result["discovery_payload"] = sum(tok(frontmatter_name_desc(p), enc) for p in entries)
|
||||
result["per_task_payload"] = stats([file_tokens(p, enc) for p in entries])
|
||||
skill_dirs_md = [m for p in entries for m in p.parent.rglob("*.md")]
|
||||
result["corpus_installable"] = sum(file_tokens(m, enc) for m in skill_dirs_md)
|
||||
result["corpus_total"] = sum(file_tokens(m, enc) for m in sorted(root.rglob("*.md")))
|
||||
|
||||
elif vtype == "single":
|
||||
entry = root / "SKILL.md"
|
||||
cards = sorted((root / "references" / "capabilities").glob("*.md"))
|
||||
entry_tok = file_tokens(entry, enc)
|
||||
card_toks = [file_tokens(c, enc) for c in cards]
|
||||
index_tok = file_tokens(root / "references" / "capability-index.md", enc)
|
||||
result["entrypoint_count"] = 1
|
||||
result["capability_count"] = len(cards)
|
||||
result["discovery_payload"] = tok(frontmatter_name_desc(entry), enc)
|
||||
result["entry_tokens"] = entry_tok
|
||||
result["per_task_payload"] = stats([entry_tok + c for c in card_toks])
|
||||
result["per_task_upper_bound"] = entry_tok + max(card_toks) + index_tok
|
||||
result["corpus_total"] = sum(file_tokens(m, enc) for m in sorted(root.rglob("*.md")))
|
||||
|
||||
elif vtype == "compact_pack":
|
||||
router_root = root / spec["router"]
|
||||
router_entry = router_root / "SKILL.md"
|
||||
promoted = sorted(p for p in root.glob(spec.get("promoted_glob", "*/SKILL.md")) if p != router_entry)
|
||||
cards = sorted((router_root / "references" / "capabilities").glob("*.md"))
|
||||
router_tok = file_tokens(router_entry, enc)
|
||||
card_toks = [file_tokens(c, enc) for c in cards]
|
||||
promoted_toks = [file_tokens(p, enc) for p in promoted]
|
||||
entries = [router_entry, *promoted]
|
||||
result["entrypoint_count"] = len(entries)
|
||||
result["capability_count"] = len(cards)
|
||||
result["promoted_count"] = len(promoted)
|
||||
result["discovery_payload"] = sum(tok(frontmatter_name_desc(p), enc) for p in entries)
|
||||
result["per_task_payload_promoted_hit"] = stats(promoted_toks)
|
||||
result["per_task_payload_router_hit"] = stats([router_tok + c for c in card_toks])
|
||||
result["corpus_total"] = sum(file_tokens(m, enc) for m in sorted(root.rglob("*.md")))
|
||||
else:
|
||||
raise ValueError(f"未知版本类型: {vtype}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
cfg_path = Path(argv[1])
|
||||
out_dir = Path(argv[argv.index("--out") + 1]) if "--out" in argv else cfg_path.parent / "metrics"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
|
||||
report = {
|
||||
"tool": "count_tokens.py v0.1",
|
||||
"config": str(cfg_path),
|
||||
"config_sha256_16": sha256(cfg_path),
|
||||
"load_model_note": "见脚本 docstring；全部为静态假设，不代表任何宿主实际计费",
|
||||
"results": {},
|
||||
}
|
||||
for enc in cfg["tokenizers"]:
|
||||
report["results"][enc] = {
|
||||
vname: measure_version(vname, vspec, enc) for vname, vspec in cfg["versions"].items()
|
||||
}
|
||||
|
||||
json_path = out_dir / "a-class-metrics.json"
|
||||
json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
lines = ["# A 类静态 Token 指标\n", f"- 工具: {report['tool']} 配置: `{cfg_path}` (sha256:{report['config_sha256_16']})", "- 静态加载模型见 `scripts/count_tokens.py` docstring；不代表宿主实际计费。\n"]
|
||||
for enc, versions in report["results"].items():
|
||||
lines.append(f"## tokenizer = `{enc}`\n")
|
||||
lines.append("| 版本 | 入口数 | 发现负载(常驻) | 单任务负载 min/median/max | 语料总量 |")
|
||||
lines.append("|---|---|---|---|---|")
|
||||
for vname, r in versions.items():
|
||||
if r["type"] == "compact_pack":
|
||||
pt = r["per_task_payload_router_hit"]
|
||||
extra = r["per_task_payload_promoted_hit"]
|
||||
payload = f"路由 {pt['min']}/{pt['median']}/{pt['max']}；晋级 {extra['min']}/{extra['median']}/{extra['max']}"
|
||||
else:
|
||||
pt = r["per_task_payload"]
|
||||
payload = f"{pt['min']}/{pt['median']}/{pt['max']}"
|
||||
lines.append(f"| {vname} | {r['entrypoint_count']} | {r['discovery_payload']} | {payload} | {r['corpus_total']} |")
|
||||
lines.append("")
|
||||
md_path = out_dir / "a-class-metrics.md"
|
||||
md_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"已写出 {json_path} 与 {md_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""diff_sources.py — 两个版本的块清单 → change-set（Phase 2B，纯确定性）。
|
||||
|
||||
判定规则（确定性部分）：
|
||||
- content_hash 相同 → unchanged / exact_duplicate（同版本内重复出现时）
|
||||
- 同 heading_path 且哈希不同 → modified（是否为 correction/contradiction 属语义判断，
|
||||
标记 requires_human_confirmation 交给 Agent 复核）
|
||||
- 新版本独有 → additive
|
||||
- 旧版本独有 → deletion（不物理删除历史证据，只进影响分析）
|
||||
|
||||
near_duplicate / correction / contradiction 的语义定性不在本脚本伪造——路线 C 下由 Agent
|
||||
按 methodology 合并规则复核后回填 change-set。
|
||||
|
||||
用法: python3 scripts/diff_sources.py <old-chunks.jsonl> <new-chunks.jsonl> --out <change-set.json> [--pack <slug>]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import dump_json # noqa: E402
|
||||
|
||||
|
||||
def load_chunks(path: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def diff(old: list[dict], new: list[dict]) -> list[dict]:
|
||||
old_by_hash = {c["content_hash"]: c for c in old}
|
||||
new_by_hash = {c["content_hash"]: c for c in new}
|
||||
changes: list[dict] = []
|
||||
|
||||
unchanged_hashes = old_by_hash.keys() & new_by_hash.keys()
|
||||
|
||||
old_rest = [c for c in old if c["content_hash"] not in unchanged_hashes]
|
||||
new_rest = [c for c in new if c["content_hash"] not in unchanged_hashes]
|
||||
|
||||
# 同 heading_path 配对为 modified
|
||||
old_by_path: dict[str, list[dict]] = {}
|
||||
for c in old_rest:
|
||||
old_by_path.setdefault(" / ".join(c["heading_path"]), []).append(c)
|
||||
|
||||
for c in new_rest:
|
||||
path_key = " / ".join(c["heading_path"])
|
||||
pool = old_by_path.get(path_key)
|
||||
if pool:
|
||||
o = pool.pop(0)
|
||||
changes.append({
|
||||
"change_type": "modified",
|
||||
"chunk_id": c["chunk_id"],
|
||||
"old_hash": o["content_hash"],
|
||||
"new_hash": c["content_hash"],
|
||||
"heading_path": c["heading_path"],
|
||||
"requires_human_confirmation": True,
|
||||
"note": "同章节内容变化；是否为 correction/contradiction 需 Agent 语义复核",
|
||||
})
|
||||
else:
|
||||
changes.append({
|
||||
"change_type": "additive",
|
||||
"chunk_id": c["chunk_id"],
|
||||
"old_hash": None,
|
||||
"new_hash": c["content_hash"],
|
||||
"heading_path": c["heading_path"],
|
||||
"requires_human_confirmation": False,
|
||||
"note": "新增内容，进入影响分析与增量候选提取",
|
||||
})
|
||||
|
||||
for pool in old_by_path.values():
|
||||
for o in pool:
|
||||
changes.append({
|
||||
"change_type": "deletion",
|
||||
"chunk_id": o["chunk_id"],
|
||||
"old_hash": o["content_hash"],
|
||||
"new_hash": None,
|
||||
"heading_path": o["heading_path"],
|
||||
"requires_human_confirmation": True,
|
||||
"note": "来源撤回/删除；不立即物理删除历史证据，先计算受影响能力",
|
||||
})
|
||||
return changes
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("old_chunks")
|
||||
ap.add_argument("new_chunks")
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--pack", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
old = load_chunks(Path(args.old_chunks))
|
||||
new = load_chunks(Path(args.new_chunks))
|
||||
changes = diff(old, new)
|
||||
unchanged = len({c["content_hash"] for c in old} & {c["content_hash"] for c in new})
|
||||
|
||||
change_set = {
|
||||
"schema_version": 1,
|
||||
"change_id": time.strftime("chg-%Y%m%d-%H%M%S"),
|
||||
"content_pack": args.pack,
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"base_version": old[0]["version_id"] if old else None,
|
||||
"new_version": new[0]["version_id"] if new else None,
|
||||
"changes": changes,
|
||||
"summary": {
|
||||
"added": sum(1 for c in changes if c["change_type"] == "additive"),
|
||||
"removed": sum(1 for c in changes if c["change_type"] == "deletion"),
|
||||
"modified": sum(1 for c in changes if c["change_type"] == "modified"),
|
||||
"unchanged": unchanged,
|
||||
},
|
||||
}
|
||||
dump_json(Path(args.out), change_set)
|
||||
s = change_set["summary"]
|
||||
print(f"change-set: {args.out}（+{s['added']} / -{s['removed']} / ~{s['modified']} / ={s['unchanged']}）")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""impact_analysis.py — 依赖图构建 + change-set 影响范围分析（Phase 2B，纯确定性）。
|
||||
|
||||
build 模式：从 Capability Bundle（+ 可选 chunks.jsonl）构建带类型的依赖图
|
||||
（dependency-graph.schema.json）：source→chunk→capability→entrypoint→eval。
|
||||
|
||||
analyze 模式：给定 change-set，沿图向下找：
|
||||
1. 直接依赖变更块的能力（chunk_id 精确匹配，或 source_evidence.location 与块 heading_path 的文本匹配）；
|
||||
2. 这些能力编译成的入口（single 入口 / 晋级 Skill / 来源路由入口）；
|
||||
3. also_read 邻居能力（对比/组合关系需一并回归）；
|
||||
4. 覆盖这些能力的评测用例。
|
||||
|
||||
匹配不到任何能力的 additive 变更 → 标注为"新知识候选"，交给 Agent 做增量提取。
|
||||
|
||||
用法:
|
||||
python3 scripts/impact_analysis.py build --bundle <dir> [--chunks <chunks.jsonl>] --out <graph.json>
|
||||
python3 scripts/impact_analysis.py analyze --graph <graph.json> --change-set <cs.json> --out <report.md>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import TOOL_VERSION, dump_json, load_json, load_yaml # noqa: E402
|
||||
|
||||
|
||||
def build_graph(bundle_dir: Path, chunks_path: Path | None) -> dict:
|
||||
bundle = load_yaml(bundle_dir / "verified.yaml")
|
||||
router_name = bundle["router_entry"]["name"]
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
|
||||
def add_node(node_id: str, node_type: str, label: str = "") -> None:
|
||||
nodes.append({"node_id": node_id, "node_type": node_type, "label": label})
|
||||
|
||||
chunks = []
|
||||
if chunks_path and chunks_path.exists():
|
||||
chunks = [json.loads(l) for l in chunks_path.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
if chunks:
|
||||
sv = f"{chunks[0]['source_id']}@{chunks[0]['version_id']}"
|
||||
add_node(sv, "source_version")
|
||||
for c in chunks:
|
||||
add_node(c["chunk_id"], "chunk", " / ".join(c["heading_path"]))
|
||||
edges.append({"from": sv, "to": c["chunk_id"], "edge_type": "contains"})
|
||||
|
||||
add_node(router_name, "entrypoint", "来源路由入口")
|
||||
for cap in bundle["capabilities"]:
|
||||
cid = cap["capability_id"]
|
||||
add_node(cid, "capability", cap["title"])
|
||||
# 证据边：chunk_id 精确匹配 + location 与 heading_path 的文本匹配
|
||||
for ev in cap.get("source_evidence", []):
|
||||
for chunk_id in ev.get("chunk_ids", []):
|
||||
edges.append({"from": chunk_id, "to": cid, "edge_type": "supports", "evidence": "chunk_ids"})
|
||||
loc = str(ev.get("location", "")).strip()
|
||||
if loc and chunks:
|
||||
# 章节粒度匹配：location 的任一段与块 heading 的任一段互为子串即建边。
|
||||
# 宁可过近似（多回归）不可漏（少回归）。
|
||||
loc_segs = [s.strip() for s in loc.split("/") if len(s.strip()) >= 3]
|
||||
for c in chunks:
|
||||
hp_segs = [s for s in c["heading_path"] if len(s) >= 3]
|
||||
if any(ls in hs or hs in ls for ls in loc_segs for hs in hp_segs):
|
||||
edges.append({"from": c["chunk_id"], "to": cid, "edge_type": "supports",
|
||||
"evidence": f"location~heading(章节粒度): {loc}"})
|
||||
# 编译去向
|
||||
if cap.get("promotion", {}).get("destination") == "promoted":
|
||||
add_node(cap["slug"], "entrypoint", f"晋级 Skill: {cap['title']}")
|
||||
edges.append({"from": cid, "to": cap["slug"], "edge_type": "compiled_as"})
|
||||
edges.append({"from": cid, "to": router_name, "edge_type": "served_by"})
|
||||
# 邻居
|
||||
slug_to_id = {c["slug"]: c["capability_id"] for c in bundle["capabilities"]}
|
||||
for sib in cap.get("also_read", []):
|
||||
if sib in slug_to_id:
|
||||
edges.append({"from": cid, "to": slug_to_id[sib], "edge_type": "composes_with"})
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"content_pack": bundle["book"].get("source_pack", bundle["bundle_id"]),
|
||||
"generated_by": TOOL_VERSION,
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
}
|
||||
|
||||
|
||||
def analyze(graph: dict, change_set: dict) -> str:
|
||||
edges = graph["edges"]
|
||||
labels = {n["node_id"]: n.get("label", "") for n in graph["nodes"]}
|
||||
node_types = {n["node_id"]: n["node_type"] for n in graph["nodes"]}
|
||||
chunk_headings = {n["node_id"]: n.get("label", "") for n in graph["nodes"] if n["node_type"] == "chunk"}
|
||||
|
||||
affected_caps: dict[str, list[str]] = {}
|
||||
orphan_changes: list[dict] = []
|
||||
|
||||
for ch in change_set["changes"]:
|
||||
hit = False
|
||||
# 1) chunk_id 精确匹配
|
||||
for e in edges:
|
||||
if e["edge_type"] in ("supports", "contradicts", "examples") and e["from"] == ch["chunk_id"]:
|
||||
affected_caps.setdefault(e["to"], []).append(f"{ch['change_type']}:{ch['chunk_id']}")
|
||||
hit = True
|
||||
# 2) heading_path 文本匹配（新版本块的 chunk_id 不在旧图中时）
|
||||
if not hit and ch.get("heading_path"):
|
||||
hp_new = " / ".join(ch["heading_path"])
|
||||
for cid_chunk, hp in chunk_headings.items():
|
||||
if hp and (hp in hp_new or hp_new in hp):
|
||||
for e in edges:
|
||||
if e["edge_type"] == "supports" and e["from"] == cid_chunk:
|
||||
affected_caps.setdefault(e["to"], []).append(f"{ch['change_type']}:{hp_new}")
|
||||
hit = True
|
||||
if not hit and ch["change_type"] == "additive":
|
||||
orphan_changes.append(ch)
|
||||
|
||||
# 沿图向下：能力 → 入口 / 邻居 / 评测
|
||||
affected_entrypoints: set[str] = set()
|
||||
neighbor_caps: set[str] = set()
|
||||
affected_evals: set[str] = set()
|
||||
for cap in affected_caps:
|
||||
for e in edges:
|
||||
if e["from"] == cap and e["edge_type"] in ("compiled_as", "served_by"):
|
||||
affected_entrypoints.add(e["to"])
|
||||
if e["edge_type"] in ("composes_with", "compared_with", "depends_on") and cap in (e["from"], e["to"]):
|
||||
other = e["to"] if e["from"] == cap else e["from"]
|
||||
if node_types.get(other) == "capability" and other not in affected_caps:
|
||||
neighbor_caps.add(other)
|
||||
if e["edge_type"] == "covers" and e["to"] == cap:
|
||||
affected_evals.add(e["from"])
|
||||
|
||||
lines = ["# 影响范围分析", "",
|
||||
f"- change-set: `{change_set['change_id']}`（+{change_set['summary']['added']} / "
|
||||
f"-{change_set['summary']['removed']} / ~{change_set['summary']['modified']}）", ""]
|
||||
lines.append(f"## 受影响能力（{len(affected_caps)}）\n")
|
||||
for cap, reasons in sorted(affected_caps.items()):
|
||||
lines.append(f"- `{cap}` {labels.get(cap, '')} ← {'; '.join(sorted(set(reasons))[:3])}")
|
||||
lines.append(f"\n## 需重编译/回归的入口（{len(affected_entrypoints)}）\n")
|
||||
for ep in sorted(affected_entrypoints):
|
||||
lines.append(f"- `{ep}` {labels.get(ep, '')}")
|
||||
lines.append(f"\n## 需一并回归的邻居能力（{len(neighbor_caps)}）\n")
|
||||
for cap in sorted(neighbor_caps):
|
||||
lines.append(f"- `{cap}` {labels.get(cap, '')}")
|
||||
if affected_evals:
|
||||
lines.append(f"\n## 覆盖这些能力的评测（{len(affected_evals)}）\n")
|
||||
lines += [f"- `{e}`" for e in sorted(affected_evals)]
|
||||
lines.append(f"\n## 未命中任何既有能力的新增块（{len(orphan_changes)}）——新知识候选\n")
|
||||
for ch in orphan_changes:
|
||||
lines.append(f"- {ch['chunk_id']} [{' / '.join(ch.get('heading_path', []))}]")
|
||||
lines.append("\n> 未受影响的能力/入口不重编译，文件哈希保持不变（增量验收要求 §6.5）。")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
sub = ap.add_subparsers(dest="mode", required=True)
|
||||
b = sub.add_parser("build")
|
||||
b.add_argument("--bundle", required=True)
|
||||
b.add_argument("--chunks", default=None)
|
||||
b.add_argument("--out", required=True)
|
||||
a = sub.add_parser("analyze")
|
||||
a.add_argument("--graph", required=True)
|
||||
a.add_argument("--change-set", required=True)
|
||||
a.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.mode == "build":
|
||||
graph = build_graph(Path(args.bundle), Path(args.chunks) if args.chunks else None)
|
||||
dump_json(Path(args.out), graph)
|
||||
print(f"依赖图: {args.out}（{len(graph['nodes'])} nodes / {len(graph['edges'])} edges）")
|
||||
else:
|
||||
report = analyze(load_json(Path(args.graph)), load_json(Path(args.change_set)))
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text(report, encoding="utf-8")
|
||||
print(report)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""repair_flow.py — `cangjie.py repair` 的编排逻辑（Phase 3，路线 C）。
|
||||
|
||||
repair 是可回滚事务（方案 §7.3）。CLI 做确定性部分：校验失败案例 → 快照 → 生成诊断任务。
|
||||
语义诊断与最小补丁由 Agent 完成；补丁经 apply_skill_patch.py 落盘（自动校验 + 失败回滚），
|
||||
回归经 run_trigger_evals.py / run_output_evals.py 判分。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import create_run_workdir, load_yaml, new_run_id, snapshot_dir # noqa: E402
|
||||
|
||||
DIAGNOSIS_TABLE = """\
|
||||
| 类别 | 典型现象 | 主要修改点 | 必跑测试 |
|
||||
|---|---|---|---|
|
||||
| activation_miss | 该触发但没触发 | description、A2 | trigger 正例、验证集 |
|
||||
| false_activation | 不该触发却触发 | description、B、兄弟区分 | near-miss 负例、兄弟混淆 |
|
||||
| knowledge_gap | 缺事实、案例或术语 | references、I、A1 | 来源事实断言 |
|
||||
| execution_gap | 会讲道理但不会做 | E、脚本、输出契约 | output eval |
|
||||
| boundary_gap | 在不适用场景硬套 | B、判停条件 | edge/negative eval |
|
||||
| structure_gap | 步骤顺序错误或前置条件缺失 | E、checklist | 流程断言 |
|
||||
| tool_gap | 每次临时写重复脚本或工具调用失败 | scripts、compatibility | 集成测试 |
|
||||
| preprocessing_gap | 上游漏字、表格错、时间轴错 | parser/IR，不应修 Skill 文案 | 预处理 golden set |
|
||||
| eval_gap | 测试本身错误或过拟合 | eval 标签/断言 | 独立复核 |"""
|
||||
|
||||
REQUIRED_CASE_FIELDS = ("prompt", "actual", "expected", "severity")
|
||||
|
||||
|
||||
def run_repair(pack: Path, case_path: Path) -> int:
|
||||
case = load_yaml(case_path)
|
||||
fc = case.get("failure_case", {})
|
||||
missing = [f for f in REQUIRED_CASE_FIELDS if not str(fc.get(f, "")).strip()]
|
||||
if missing or not case.get("skill"):
|
||||
raise SystemExit(f"failure case 缺少必填字段: {['skill'] if not case.get('skill') else []} + {missing}\n"
|
||||
f"（schema: schemas/failure-case.schema.json；宿主拿不到执行轨迹时 actual 填 unavailable）")
|
||||
if fc["severity"] not in ("critical", "major", "minor"):
|
||||
raise SystemExit(f"severity 必须是 critical|major|minor，当前 {fc['severity']!r}")
|
||||
|
||||
sidecar = pack / ".cangjie"
|
||||
run_id = new_run_id()
|
||||
workdir = create_run_workdir(sidecar, run_id)
|
||||
|
||||
# 1. 只读快照目标 skill（能找到已发布目录时）
|
||||
target_hint = ""
|
||||
for candidate in (pack / case["skill"], pack.parent / case["skill"]):
|
||||
if candidate.is_dir():
|
||||
snap = snapshot_dir(candidate, sidecar / "snapshots", f"pre-repair-{case['skill']}")
|
||||
target_hint = f"已快照目标 skill: `{snap}`"
|
||||
break
|
||||
else:
|
||||
target_hint = "（未在本仓找到已发布 skill 目录；修复对象可能是 Bundle 能力卡，快照 Bundle 后再动手）"
|
||||
|
||||
task = f"""# repair 诊断任务（run: {run_id}）
|
||||
|
||||
## 失败案例（{fc['severity']}）
|
||||
|
||||
- **skill**: `{case['skill']}`
|
||||
- **prompt**: {fc['prompt']}
|
||||
- **actual**: {fc['actual']}
|
||||
- **expected**: {fc['expected']}
|
||||
|
||||
{target_hint}
|
||||
|
||||
## 你（Agent）要做的事，按序执行
|
||||
|
||||
1. **复现**：用 prompt 复现失败；宿主没有执行 Trace 就明确记录 unavailable，不得推测补齐；
|
||||
2. **诊断分类**（写回 failure-case 的 diagnosis 字段，类别必须取自下表）：
|
||||
|
||||
{DIAGNOSIS_TABLE}
|
||||
|
||||
3. **最小补丁**：只改诊断影响范围内的文件（能力卡/Bundle 字段），不自由重写。
|
||||
把补丁文件放入 `{workdir}/patch/`，用
|
||||
`python3 scripts/apply_skill_patch.py apply --target <skill-dir> --patch-dir {workdir}/patch --snapshots {sidecar}/snapshots` 落盘；
|
||||
4. **防过拟合**（方案 §7.4）：不把失败案例专有名词原样塞进 description；每修一个正例至少补一个语义近邻负例；
|
||||
5. **回归**：目标失败案例 + 该 skill 全部回归 + 相邻 skill 混淆回归（run_trigger_evals.py 判分，validation 集在选版前保持隐藏）；
|
||||
6. 通过后写 changelog；任何一步失败用 restore 回滚快照。
|
||||
|
||||
> preprocessing_gap 不要修 Skill 文案，去修上游解析；eval_gap 去修测试并记录理由。
|
||||
"""
|
||||
(workdir / "repair-task.md").write_text(task, encoding="utf-8")
|
||||
(workdir / "patch").mkdir()
|
||||
print(task)
|
||||
print(f"诊断任务已生成: {workdir / 'repair-task.md'}")
|
||||
return 0
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""run_output_evals.py — 输出评测的确定性环节（Phase 3，路线 C）。
|
||||
|
||||
prepare 为每条 output case 生成 old/new/without 三个匿名任务包（盲测：包名随机化标签，
|
||||
映射表单独存放，不给评审者）
|
||||
score 对记录的输出跑机械断言（contains/not_contains/regex/file_exists/json_path），
|
||||
机械断言先于 LLM judge；盲评分歧样本留给人工复核
|
||||
|
||||
outputs 目录约定: <outputs>/<case_id>/<variant-label>.md（variant-label 来自 prepare 的映射表）
|
||||
|
||||
用法:
|
||||
python3 scripts/run_output_evals.py prepare <suite.json> --out <dir> [--variants old_skill,new_skill,without_skill]
|
||||
python3 scripts/run_output_evals.py score <suite.json> --outputs <dir> --mapping <mapping.json> --out <report.md>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import dump_json, load_json # noqa: E402
|
||||
|
||||
|
||||
def check_assertion(a: dict, text: str, base: Path) -> bool:
|
||||
kind, value = a["kind"], a["value"]
|
||||
if kind == "contains":
|
||||
return value in text
|
||||
if kind == "not_contains":
|
||||
return value not in text
|
||||
if kind == "regex":
|
||||
return re.search(value, text) is not None
|
||||
if kind == "file_exists":
|
||||
return (base / value).exists()
|
||||
if kind == "json_path": # 简化版: 顶层 key 存在于输出中的 JSON 块
|
||||
try:
|
||||
data = json.loads(re.search(r"\{.*\}", text, re.S).group(0))
|
||||
except Exception:
|
||||
return False
|
||||
cur = data
|
||||
for part in value.lstrip("$.").split("."):
|
||||
if not isinstance(cur, dict) or part not in cur:
|
||||
return False
|
||||
cur = cur[part]
|
||||
return True
|
||||
raise ValueError(f"未知断言类型: {kind}")
|
||||
|
||||
|
||||
def cmd_prepare(suite_path: Path, out_dir: Path, variants: list[str]) -> int:
|
||||
suite = load_json(suite_path)
|
||||
rng = random.Random(suite.get("split_seed", 42))
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
mapping: dict[str, dict[str, str]] = {}
|
||||
for c in suite.get("output_cases", []):
|
||||
labels = [f"v{chr(65 + i)}" for i in range(len(variants))]
|
||||
rng.shuffle(labels)
|
||||
mapping[c["case_id"]] = dict(zip(labels, variants))
|
||||
case_dir = out_dir / c["case_id"]
|
||||
case_dir.mkdir(exist_ok=True)
|
||||
for label in labels:
|
||||
dump_json(case_dir / f"task-{label}.json", {
|
||||
"case_id": c["case_id"], "variant_label": label, "prompt": c["prompt"],
|
||||
"input_files": c.get("input_files", []),
|
||||
"instruction": "按 prompt 完成任务，输出保存为同目录 <label>.md。评审者不知道你是哪个版本。",
|
||||
})
|
||||
dump_json(out_dir / "mapping.json", mapping)
|
||||
print(f"已生成 {len(mapping)} 条 output 任务（每条 {len(variants)} 个匿名变体）→ {out_dir}\n"
|
||||
f"映射表 {out_dir / 'mapping.json'} 不要给评审 sub-agent。")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_score(suite_path: Path, outputs: Path, mapping_path: Path, out_path: Path) -> int:
|
||||
suite = load_json(suite_path)
|
||||
mapping = load_json(mapping_path)
|
||||
lines = [f"# 输出评测机械断言判分 — {suite['target']}", ""]
|
||||
totals: dict[str, list[int]] = {}
|
||||
for c in suite.get("output_cases", []):
|
||||
case_map = mapping.get(c["case_id"], {})
|
||||
lines.append(f"## {c['case_id']}\n")
|
||||
lines.append("| 变体 | 断言通过 | 明细 |")
|
||||
lines.append("|---|---|---|")
|
||||
for label, variant in sorted(case_map.items()):
|
||||
f = outputs / c["case_id"] / f"{label}.md"
|
||||
if not f.exists():
|
||||
lines.append(f"| {variant} | — | 输出缺失: {f.name} |")
|
||||
continue
|
||||
text = f.read_text(encoding="utf-8")
|
||||
results = [(a, check_assertion(a, text, f.parent)) for a in c["assertions"]]
|
||||
passed = sum(1 for _, ok in results if ok)
|
||||
detail = "; ".join(f"{'✓' if ok else '✗'}{a['kind']}:{a['value'][:24]}" for a, ok in results)
|
||||
lines.append(f"| {variant} | {passed}/{len(results)} | {detail} |")
|
||||
totals.setdefault(variant, []).append(int(passed == len(results)))
|
||||
lines.append("")
|
||||
lines.append("## 汇总（全部断言通过的 case 比例）\n")
|
||||
for variant, arr in sorted(totals.items()):
|
||||
lines.append(f"- {variant}: {sum(arr)}/{len(arr)}")
|
||||
lines.append("\n> 机械断言先于 LLM judge；A/B 盲评与分歧样本人工复核另行进行，此处不自动宣布非劣。")
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print("\n".join(lines))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
sub = ap.add_subparsers(dest="mode", required=True)
|
||||
p = sub.add_parser("prepare")
|
||||
p.add_argument("suite")
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--variants", default="old_skill,new_skill,without_skill")
|
||||
c = sub.add_parser("score")
|
||||
c.add_argument("suite")
|
||||
c.add_argument("--outputs", required=True)
|
||||
c.add_argument("--mapping", required=True)
|
||||
c.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
if args.mode == "prepare":
|
||||
return cmd_prepare(Path(args.suite), Path(args.out), args.variants.split(","))
|
||||
return cmd_score(Path(args.suite), Path(args.outputs), Path(args.mapping), Path(args.out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""run_trigger_evals.py — 触发评测的确定性环节（Phase 3，路线 C）。
|
||||
|
||||
路线 C 下 CLI 不能替宿主跑模型；本脚本负责三件确定性的事：
|
||||
|
||||
split 固定种子做 60/40 train/validation 切分（validation 在选版前保持隐藏）
|
||||
prepare 生成盲测任务包：只含 prompt + 候选 skill 目录清单，隐藏 expected/notes，
|
||||
由主流程逐条交给干净 sub-agent，结果写 results.jsonl
|
||||
score 对照 suite 判分：precision/recall/F1、兄弟混淆率、逐条配对结果
|
||||
（不做统计非劣声明——那需要预注册界值与配对检验，见方案 §10.3）
|
||||
|
||||
results.jsonl 每行: {"case_id": "...", "run": 1, "selected_skill": "<slug>|none"}
|
||||
|
||||
用法:
|
||||
python3 scripts/run_trigger_evals.py split <suite.json>
|
||||
python3 scripts/run_trigger_evals.py prepare <suite.json> --skills <slug1,slug2,...> --out <dir> [--set train|validation|all]
|
||||
python3 scripts/run_trigger_evals.py score <suite.json> --results <results.jsonl> --out <report.md>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import dump_json, load_json # noqa: E402
|
||||
|
||||
|
||||
def split_cases(suite: dict) -> tuple[list[dict], list[dict]]:
|
||||
cases = list(suite["trigger_cases"])
|
||||
rng = random.Random(suite.get("split_seed", 42))
|
||||
rng.shuffle(cases)
|
||||
k = round(len(cases) * suite.get("train_ratio", 0.6))
|
||||
return cases[:k], cases[k:]
|
||||
|
||||
|
||||
def cmd_split(suite_path: Path) -> int:
|
||||
suite = load_json(suite_path)
|
||||
train, val = split_cases(suite)
|
||||
out = {"suite_id": suite["suite_id"], "seed": suite.get("split_seed", 42),
|
||||
"train": [c["case_id"] for c in train], "validation": [c["case_id"] for c in val]}
|
||||
dump_json(suite_path.with_suffix(".split.json"), out)
|
||||
print(f"train {len(train)} / validation {len(val)} → {suite_path.with_suffix('.split.json')}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_prepare(suite_path: Path, skills: list[str], out_dir: Path, which: str) -> int:
|
||||
suite = load_json(suite_path)
|
||||
train, val = split_cases(suite)
|
||||
cases = {"train": train, "validation": val, "all": train + val}[which]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for c in cases:
|
||||
runs = c.get("runs", 3)
|
||||
packet = {
|
||||
"case_id": c["case_id"],
|
||||
"runs": runs,
|
||||
"prompt": c["prompt"],
|
||||
"instruction": (
|
||||
"你是一个未参与蒸馏的干净 agent。给定用户 prompt 与已安装 skill 清单，"
|
||||
"判断该激活哪一个 skill（或 none）。输出 JSON: "
|
||||
'{"selected_skill": "<slug>|none", "reason": "..."}。'
|
||||
"每条 prompt 独立判断，重复运行之间不携带记忆。"
|
||||
),
|
||||
"installed_skills": skills,
|
||||
}
|
||||
dump_json(out_dir / f"{c['case_id']}.json", packet)
|
||||
(out_dir / "README.md").write_text(
|
||||
f"# 盲测任务包（{which}, {len(cases)} 条）\n\n"
|
||||
"每个 JSON 是一条盲测任务：把 prompt + installed_skills 交给干净 sub-agent，"
|
||||
"按 instruction 输出;结果按行追加到 results.jsonl:\n"
|
||||
'`{"case_id": ..., "run": 1, "selected_skill": ...}`\n\n'
|
||||
"**不要**把 suite 中的 expected/notes 给 sub-agent。\n", encoding="utf-8")
|
||||
print(f"已生成 {len(cases)} 个盲测任务包 → {out_dir}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_score(suite_path: Path, results_path: Path, out_path: Path) -> int:
|
||||
suite = load_json(suite_path)
|
||||
target = suite["target"]
|
||||
by_case = {c["case_id"]: c for c in suite["trigger_cases"]}
|
||||
results = [json.loads(l) for l in results_path.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
|
||||
tp = fp = fn = tn = 0
|
||||
sibling_total = sibling_confused = 0
|
||||
rows = ["| case | expected | selected | 判定 |", "|---|---|---|---|"]
|
||||
for r in results:
|
||||
c = by_case.get(r["case_id"])
|
||||
if not c:
|
||||
continue
|
||||
selected = r.get("selected_skill", "none")
|
||||
triggered = selected == target
|
||||
exp = c["expected"]
|
||||
if exp == "should_trigger":
|
||||
verdict = "TP" if triggered else "FN"
|
||||
tp += triggered
|
||||
fn += not triggered
|
||||
elif exp in ("should_not_trigger", "edge_case"):
|
||||
verdict = "FP" if triggered else "TN"
|
||||
fp += triggered
|
||||
tn += not triggered
|
||||
else: # sibling
|
||||
sibling_total += 1
|
||||
correct = selected == c.get("sibling_target")
|
||||
if triggered:
|
||||
sibling_confused += 1
|
||||
verdict = "混淆(FP)"
|
||||
fp += 1
|
||||
else:
|
||||
verdict = "OK" if correct else f"未中兄弟({selected})"
|
||||
tn += 1
|
||||
rows.append(f"| {r['case_id']}#r{r.get('run', 1)} | {exp} | {selected} | {verdict} |")
|
||||
|
||||
precision = tp / (tp + fp) if tp + fp else 0.0
|
||||
recall = tp / (tp + fn) if tp + fn else 0.0
|
||||
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
|
||||
confusion = sibling_confused / sibling_total if sibling_total else 0.0
|
||||
|
||||
report = (f"# 触发评测判分 — {target}\n\n"
|
||||
f"- runs: {len(results)}（TP {tp} / FP {fp} / FN {fn} / TN {tn}）\n"
|
||||
f"- precision {precision:.3f} / recall {recall:.3f} / **F1 {f1:.3f}**\n"
|
||||
f"- 兄弟混淆率: {confusion:.3f}（{sibling_confused}/{sibling_total}）\n\n"
|
||||
+ "\n".join(rows)
|
||||
+ "\n\n> 本报告只给原始配对计数与比率；统计非劣需预注册界值 + McNemar/配对 Bootstrap（§10.3），不在此自动宣布。\n")
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(report, encoding="utf-8")
|
||||
print(report)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
sub = ap.add_subparsers(dest="mode", required=True)
|
||||
s = sub.add_parser("split")
|
||||
s.add_argument("suite")
|
||||
p = sub.add_parser("prepare")
|
||||
p.add_argument("suite")
|
||||
p.add_argument("--skills", required=True)
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--set", dest="which", choices=["train", "validation", "all"], default="train")
|
||||
c = sub.add_parser("score")
|
||||
c.add_argument("suite")
|
||||
c.add_argument("--results", required=True)
|
||||
c.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.mode == "split":
|
||||
return cmd_split(Path(args.suite))
|
||||
if args.mode == "prepare":
|
||||
return cmd_prepare(Path(args.suite), args.skills.split(","), Path(args.out), args.which)
|
||||
return cmd_score(Path(args.suite), Path(args.results), Path(args.out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""select_output_strategy.py — auto 输出决策器（方案 §4.6.5，策略 single-first-v1）。
|
||||
|
||||
规则（首版）：
|
||||
1. 用户明确目的（--purpose learning|reference|workflow|distribution）优先；
|
||||
2. 未明确目的时：至少 3 个能力通过晋级门、且晋级门触发验证达到预注册阈值（TBD-after-baseline，
|
||||
基线前该分支不启用）→ 推荐 pack；
|
||||
3. 其他情况一律推荐 single（single-first）。
|
||||
|
||||
输出可解释 decision report（output-decision.schema.json）。默认值只是推荐，不能取消用户显式选择。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from cangjie_common import dump_json, load_yaml # noqa: E402
|
||||
from compile_single import active_capabilities, promoted_capabilities # noqa: E402
|
||||
|
||||
POLICY = "single-first-v1"
|
||||
|
||||
PURPOSE_TO_MODE = {
|
||||
"learning": "single",
|
||||
"reference": "single",
|
||||
"workflow": "pack",
|
||||
"distribution": "pack",
|
||||
}
|
||||
|
||||
|
||||
def decide(bundle_dir: Path, requested: str, purpose: str | None, *, trigger_validation_ready: bool = False) -> dict:
|
||||
bundle = load_yaml(bundle_dir / "verified.yaml")
|
||||
caps = active_capabilities(bundle)
|
||||
promoted = promoted_capabilities(bundle)
|
||||
budget = int(bundle.get("promotion_budget", 8))
|
||||
|
||||
reasons: list[str] = []
|
||||
if requested in ("single", "pack"):
|
||||
selected = requested
|
||||
reasons.append(f"用户显式选择 {requested}，auto 决策器不覆盖用户选择")
|
||||
elif purpose:
|
||||
selected = PURPOSE_TO_MODE[purpose]
|
||||
reasons.append(f"用户目的为 {purpose} → {selected}")
|
||||
elif len(promoted) >= 3 and trigger_validation_ready:
|
||||
selected = "pack"
|
||||
reasons.append(f"{len(promoted)} 个能力通过晋级门（>=3）且触发验证达到预注册阈值")
|
||||
else:
|
||||
selected = "single"
|
||||
if len(promoted) >= 3:
|
||||
reasons.append(
|
||||
f"{len(promoted)} 个能力通过晋级门，但触发验证阈值为 TBD-after-baseline 尚未启用；"
|
||||
"按 single-first 原则先推荐 single，拆分由使用证据驱动"
|
||||
)
|
||||
else:
|
||||
reasons.append(f"仅 {len(promoted)} 个能力通过晋级门（<3），不满足 pack 推荐条件")
|
||||
|
||||
alternative = (
|
||||
f"compact pack（1 个来源路由入口 + {len(promoted)} 个晋级 Skill，共 {1 + len(promoted)} 个可发现入口）"
|
||||
if selected == "single"
|
||||
else f"single（1 个入口 + {len(caps)} 张内部能力卡）"
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"requested": requested,
|
||||
"selected": selected,
|
||||
"decision_policy": POLICY,
|
||||
"skill_budget": budget,
|
||||
"promoted_count": len(promoted),
|
||||
"capability_count": len(caps),
|
||||
"reasons": reasons,
|
||||
"alternative": alternative,
|
||||
"user_confirmed": requested != "auto",
|
||||
"preserve_strategy_on_update": True,
|
||||
}
|
||||
|
||||
|
||||
def render_report(decision: dict, bundle_dir: Path) -> str:
|
||||
bundle = load_yaml(bundle_dir / "verified.yaml")
|
||||
return f"""# 输出策略决策报告
|
||||
|
||||
推荐：**{decision['selected']}**（requested: {decision['requested']}，策略 {decision['decision_policy']}）
|
||||
|
||||
理由：
|
||||
{chr(10).join(f'- {r}' for r in decision['reasons'])}
|
||||
|
||||
产物：{'1 个 Skill + ' + str(decision['capability_count']) + ' 张内部能力卡 + 章节/术语/速查 references' if decision['selected'] == 'single' else f"1 个来源路由入口 + {decision['promoted_count']} 个晋级 Skill（共 {1 + decision['promoted_count']} 个可发现入口）"}
|
||||
备选：{decision['alternative']}
|
||||
|
||||
> 来源：{bundle['book']['title']}（{decision['capability_count']} 个 active 能力，晋级预算 {decision['skill_budget']}）
|
||||
> 请回答：按推荐 / 改成 single / 改成 pack。默认值只是推荐，不会取消你的显式选择。
|
||||
"""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--bundle", required=True)
|
||||
ap.add_argument("--requested", choices=["auto", "single", "pack"], default="auto")
|
||||
ap.add_argument("--purpose", choices=list(PURPOSE_TO_MODE), default=None)
|
||||
ap.add_argument("--out-json", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
decision = decide(Path(args.bundle), args.requested, args.purpose)
|
||||
print(render_report(decision, Path(args.bundle)))
|
||||
if args.out_json:
|
||||
dump_json(Path(args.out_json), decision)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""update_flow.py — `cangjie.py update` 的编排逻辑（Phase 2B，路线 C）。
|
||||
|
||||
CLI 只做确定性部分：登记来源 → 切块 → diff → change-set → 影响分析。
|
||||
需要语义判断的节点（增量候选提取、correction/contradiction 定性、合并规则应用）
|
||||
生成待处理 Agent 任务清单，由 Agent 按 methodology 完成后再交回 CLI 校验与重编译。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from build_chunks import build as build_chunks_for # noqa: E402
|
||||
from cangjie_common import create_run_workdir, load_yaml, new_run_id, sha256_file # noqa: E402
|
||||
from diff_sources import diff as diff_chunks, load_chunks # noqa: E402
|
||||
from impact_analysis import analyze, build_graph # noqa: E402
|
||||
from cangjie_common import dump_json # noqa: E402
|
||||
|
||||
|
||||
def ensure_manifest(pack: Path) -> dict:
|
||||
manifest_path = pack / ".cangjie" / "manifest.yaml"
|
||||
if manifest_path.exists():
|
||||
return load_yaml(manifest_path)
|
||||
manifest = {"schema_version": 1, "content_pack": pack.name, "sources": []}
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
|
||||
def run_update(pack: Path, new_source: Path) -> int:
|
||||
sidecar = pack / ".cangjie"
|
||||
bundle_dir = sidecar / "capabilities"
|
||||
if not (bundle_dir / "verified.yaml").exists():
|
||||
raise SystemExit(f"未找到 Capability Bundle: {bundle_dir}/verified.yaml（旧 pack 先运行 migrate-legacy）")
|
||||
|
||||
run_id = new_run_id()
|
||||
workdir = create_run_workdir(sidecar, run_id)
|
||||
|
||||
# 1. 登记来源版本
|
||||
manifest_path = sidecar / "manifest.yaml"
|
||||
manifest = ensure_manifest(pack)
|
||||
source_id = f"src-{new_source.stem.lower().replace(' ', '-')[:32]}"
|
||||
version_id = f"sha256:{sha256_file(new_source)}"
|
||||
entry = next((s for s in manifest["sources"] if s["source_id"] == source_id), None)
|
||||
if entry is None:
|
||||
entry = {"source_id": source_id, "kind": "other", "title": new_source.stem,
|
||||
"uri": new_source.resolve().as_uri(), "rights": "user-provided", "trust": "primary",
|
||||
"versions": []}
|
||||
manifest["sources"].append(entry)
|
||||
if any(v["version_id"] == version_id for v in entry["versions"]):
|
||||
print(f"[skip] 该来源版本已登记（exact duplicate）: {version_id[:23]}…")
|
||||
return 0
|
||||
for v in entry["versions"]:
|
||||
if v["status"] == "active":
|
||||
v["status"] = "superseded"
|
||||
entry["versions"].append({"version_id": version_id, "added_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"parser": "build_chunks v1.0", "status": "active"})
|
||||
manifest_path.write_text(yaml.safe_dump(manifest, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
|
||||
# 2. 切块（旧版本块先备份用于 diff）
|
||||
chunks_path = sidecar / "chunks" / "chunks.jsonl"
|
||||
old_chunks_backup = None
|
||||
if chunks_path.exists():
|
||||
old_chunks_backup = workdir / "old-chunks.jsonl"
|
||||
shutil.copyfile(chunks_path, old_chunks_backup)
|
||||
build_chunks_for(new_source, sidecar, source_id, 4000)
|
||||
|
||||
# 3. diff → change-set
|
||||
old = load_chunks(old_chunks_backup) if old_chunks_backup else []
|
||||
new = load_chunks(chunks_path)
|
||||
changes = diff_chunks(old, new)
|
||||
change_set = {
|
||||
"schema_version": 1,
|
||||
"change_id": f"chg-{run_id}",
|
||||
"content_pack": pack.name,
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"base_version": old[0]["version_id"] if old else None,
|
||||
"new_version": version_id,
|
||||
"changes": changes,
|
||||
"summary": {
|
||||
"added": sum(1 for c in changes if c["change_type"] == "additive"),
|
||||
"removed": sum(1 for c in changes if c["change_type"] == "deletion"),
|
||||
"modified": sum(1 for c in changes if c["change_type"] == "modified"),
|
||||
"unchanged": len({c["content_hash"] for c in old} & {c["content_hash"] for c in new}),
|
||||
},
|
||||
}
|
||||
cs_path = sidecar / "changes" / f"{change_set['change_id']}.json"
|
||||
dump_json(cs_path, change_set)
|
||||
|
||||
# 4. 影响分析
|
||||
graph = build_graph(bundle_dir, chunks_path)
|
||||
dump_json(sidecar / "graph" / "dependencies.json", graph)
|
||||
report = analyze(graph, change_set)
|
||||
(workdir / "impact-report.md").write_text(report, encoding="utf-8")
|
||||
|
||||
# 5. 生成待处理 Agent 任务（语义节点不由 CLI 伪造）
|
||||
pending = [f"# update 待处理任务（run: {run_id}）", "",
|
||||
f"change-set: `{cs_path}`；影响分析: `{workdir / 'impact-report.md'}`", "",
|
||||
"按 `methodology/` 合并规则逐项处理（CLI 不做语义判断）：", ""]
|
||||
need_confirm = [c for c in changes if c.get("requires_human_confirmation")]
|
||||
additive = [c for c in changes if c["change_type"] == "additive"]
|
||||
if additive:
|
||||
pending.append(f"1. **增量候选提取**：对 {len(additive)} 个新增块跑阶段 1（可用检索式取块），"
|
||||
"新候选走阶段 1.5 三重验证 → 阶段 1.6 晋级门 → 更新 Bundle；")
|
||||
if need_confirm:
|
||||
pending.append(f"2. **人工确认项（{len(need_confirm)}）**：modified/deletion 块需定性 "
|
||||
"correction / contradiction / near_duplicate，冲突不得静默综合（§6.4）；")
|
||||
pending += ["3. Bundle 更新后运行 `cangjie.py compile`（沿用原输出策略）重编译受影响入口；",
|
||||
"4. 运行受影响能力 + 邻居能力的回归评测（见影响分析报告）。", "",
|
||||
"> 未受影响的能力/入口不重编译，文件哈希保持不变。"]
|
||||
(workdir / "pending-tasks.md").write_text("\n".join(pending) + "\n", encoding="utf-8")
|
||||
|
||||
print(report)
|
||||
print(f"待处理任务: {workdir / 'pending-tasks.md'}")
|
||||
return 0
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""validate_skill_pack.py — 纯确定性 Skill 包静态校验（Phase 0 工具，不调用模型）。
|
||||
|
||||
校验内容：
|
||||
1. 每个含 SKILL.md 的目录视为一个 Skill：frontmatter 必须可解析且含非空 name/description；
|
||||
2. name 建议匹配 ^[a-z0-9][a-z0-9-]*$ 且与目录名一致（不一致仅告警）；
|
||||
3. 全部 .md 文件必须是合法 UTF-8；
|
||||
4. 全部 .md 文件中的相对引用（markdown 链接与裸 references/... 路径）必须存在；
|
||||
5. 输出每个 Skill 的行数/字节数概览。
|
||||
|
||||
用法: python3 scripts/validate_skill_pack.py <dir> [<dir> ...]
|
||||
退出码: 0 = 无 ERROR；1 = 存在 ERROR。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
MD_LINK_RE = re.compile(r"\]\(([^)#\s]+?)(?:#[^)\s]*)?\)")
|
||||
BARE_REF_RE = re.compile(r"(?<![\w/(])((?:references|assets)/[\w\-./]+\.[a-zA-Z0-9]+)")
|
||||
NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> dict | None:
|
||||
if not text.startswith("---"):
|
||||
return None
|
||||
end = text.find("\n---", 3)
|
||||
if end == -1:
|
||||
return None
|
||||
try:
|
||||
data = yaml.safe_load(text[3:end])
|
||||
except yaml.YAMLError:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def check_links(md_path: Path, text: str) -> None:
|
||||
base = md_path.parent
|
||||
seen: set[str] = set()
|
||||
for regex in (MD_LINK_RE, BARE_REF_RE):
|
||||
for m in regex.finditer(text):
|
||||
target = m.group(1).strip()
|
||||
if target in seen:
|
||||
continue
|
||||
seen.add(target)
|
||||
if target.startswith(("http://", "https://", "mailto:", "/")):
|
||||
continue
|
||||
if not (base / target).exists():
|
||||
errors.append(f"[broken-ref] {md_path}: `{target}` 不存在")
|
||||
|
||||
|
||||
def check_skill(skill_md: Path) -> None:
|
||||
text = skill_md.read_text(encoding="utf-8")
|
||||
fm = parse_frontmatter(text)
|
||||
if fm is None:
|
||||
errors.append(f"[frontmatter] {skill_md}: frontmatter 缺失或不可解析")
|
||||
return
|
||||
name = fm.get("name")
|
||||
desc = fm.get("description")
|
||||
if not (isinstance(name, str) and name.strip()):
|
||||
errors.append(f"[frontmatter] {skill_md}: 缺少非空 name")
|
||||
else:
|
||||
if not NAME_RE.match(name):
|
||||
warnings.append(f"[name-style] {skill_md}: name `{name}` 不符合小写连字符风格")
|
||||
if name != skill_md.parent.name:
|
||||
warnings.append(f"[name-dir] {skill_md}: name `{name}` 与目录名 `{skill_md.parent.name}` 不一致")
|
||||
if not (isinstance(desc, str) and desc.strip()):
|
||||
errors.append(f"[frontmatter] {skill_md}: 缺少非空 description")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
skill_count = 0
|
||||
md_count = 0
|
||||
for root_arg in argv[1:]:
|
||||
root = Path(root_arg)
|
||||
if not root.is_dir():
|
||||
errors.append(f"[input] 目录不存在: {root}")
|
||||
continue
|
||||
for md in sorted(root.rglob("*.md")):
|
||||
md_count += 1
|
||||
try:
|
||||
text = md.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
errors.append(f"[encoding] {md}: 非法 UTF-8")
|
||||
continue
|
||||
check_links(md, text)
|
||||
if md.name == "SKILL.md":
|
||||
skill_count += 1
|
||||
check_skill(md)
|
||||
lines = text.count("\n") + 1
|
||||
print(f" skill: {md.parent.name:<28} {lines:>4} 行 {len(text.encode('utf-8')):>7} 字节")
|
||||
|
||||
print(f"\n共扫描 {md_count} 个 .md，其中 {skill_count} 个 SKILL.md")
|
||||
for w in warnings:
|
||||
print(f"WARN {w}")
|
||||
for e in errors:
|
||||
print(f"ERROR {e}")
|
||||
print(f"\n结果: {len(errors)} errors, {len(warnings)} warnings")
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
Reference in New Issue
Block a user