#!/usr/bin/env python3 """Rebuild the bundled TSL API function index from the codegen markdown tree. The markdown tree is the source of truth: each `## `sig`` / `### `sig`` heading is one function entry. The TSV is a derived product; regenerate it whenever the leaf Markdown changes rather than editing it by hand. Columns (tab-separated, LF line endings, UTF-8): name scope module signature page anchor tags summary - name: signature text up to the first '(' - scope: first path segment under the codegen root (for example project) - module: second path segment for nested pages, else the flat file stem - signature: verbatim from the heading, backticks stripped - page: POSIX path relative to the codegen root - anchor: GitHub-style slug of the name (lowercased, chars outside [a-z0-9_] removed); per-page duplicate slugs get -1/-2 suffixes in document order, matching the rendered heading anchors. - tags: space-separated keywords from `` - summary: first prose line under the entry heading, empty for table, heading, or standalone return-type lines Usage (run from repo root; --skill-dir is required): SKILL=skills/tsl-api-reference python tools/tsl-codegen/scripts/build_index.py \ --skill-dir "$SKILL" # rewrite the TSV in place python tools/tsl-codegen/scripts/build_index.py \ --skill-dir "$SKILL" --check # verify against md tree, no write """ import argparse import sys from pathlib import Path import re ENTRY_RE = re.compile(r"^#{2,3}(?!#)\s+`(.+?)`\s*$") RETURN_RE = re.compile(r"^返回[:：]") TAGS_RE = re.compile(r"^$") HEADER = [ "name", "scope", "module", "signature", "page", "anchor", "tags", "summary", ] def slug(name): """GitHub-style anchor slug: lowercase, keep [a-z0-9_], drop the rest.""" return re.sub(r"[^a-z0-9_]", "", name.lower()) def extract_metadata(lines, heading_idx): """Return tags and the first prose line under a function entry heading.""" tags = "" for line in lines[heading_idx + 1:]: text = line.strip() if not text: continue tag_match = TAGS_RE.match(text) if tag_match: tags = " ".join(tag_match.group(1).split()).replace("\t", " ") continue if text.startswith("|") or text.startswith("#") or RETURN_RE.match(text): return tags, "" return tags, text.replace("\t", " ") return tags, "" def parse_page(codegen_root, md): """Yield index rows for one Markdown page.""" page = md.relative_to(codegen_root).as_posix() scope, module = scope_module(page) seen = {} rows = [] lines = md.read_text(encoding="utf-8").splitlines() for idx, line in enumerate(lines): m = ENTRY_RE.match(line) if not m: continue sig = m.group(1) name = sig.split("(", 1)[0] base = slug(name) n = seen.get(base, 0) seen[base] = n + 1 anchor = base if n == 0 else f"{base}-{n}" tags, summary = extract_metadata(lines, idx) rows.append([name, scope, module, sig, page, anchor, tags, summary]) return rows def scope_module(page): parts = page.split("/") scope = parts[0] module = parts[1] if len(parts) >= 3 else Path(parts[-1]).stem return scope, module def build_rows(codegen_root): """Scan the whole codegen tree and return sorted rows.""" rows = [] for md in sorted(codegen_root.rglob("*.md")): rows.extend(parse_page(codegen_root, md)) rows.sort(key=lambda r: (r[0].lower(), r[4], r[3])) return rows def render_tsv(rows): lines = ["\t".join(HEADER)] lines.extend("\t".join(r) for r in rows) return "\n".join(lines) + "\n" def read_tsv(tsv_path): text = tsv_path.read_text(encoding="utf-8") return [line.split("\t") for line in text.splitlines() if line.strip()] def main(argv=None): if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--skill-dir", required=True, help="tsl-api-reference skill dir; codegen root defaults to " "/references/codegen and tsv to " "/data/function_index.tsv", ) parser.add_argument( "--root", help="explicit codegen root, overriding the one derived from --skill-dir", ) parser.add_argument( "--tsv", help="explicit output tsv, overriding the one derived from --skill-dir", ) parser.add_argument( "--check", action="store_true", help="verify the tsv matches the md tree; exit 1 if not (no write)", ) args = parser.parse_args(argv) skill_root = Path(args.skill_dir) codegen_root = ( Path(args.root) if args.root else skill_root / "references" / "codegen" ) if not codegen_root.is_dir(): print(f"ERROR: codegen root not found: {codegen_root}", file=sys.stderr) return 1 tsv_path = ( Path(args.tsv) if args.tsv else skill_root / "data" / "function_index.tsv" ) rows = build_rows(codegen_root) new_text = render_tsv(rows) if args.check: if not tsv_path.is_file(): print(f"MISMATCH: tsv does not exist: {tsv_path}", file=sys.stderr) return 1 current = tsv_path.read_text(encoding="utf-8") tsv_ok = current == new_text if tsv_ok: print(f"OK: {tsv_path} matches md tree ({len(rows)} rows)") return 0 if not tsv_ok: cur_rows = read_tsv(tsv_path)[1:] cur_keys = {tuple(r) for r in cur_rows} new_keys = {tuple(r) for r in rows} print( f"MISMATCH: tsv out of date " f"(tsv {len(cur_rows)} rows, md {len(rows)} rows; " f"+{len(new_keys - cur_keys)} -{len(cur_keys - new_keys)}). " f"Run without --check to rebuild.", file=sys.stderr, ) return 1 tsv_path.parent.mkdir(parents=True, exist_ok=True) tsv_path.write_text(new_text, encoding="utf-8", newline="\n") print(f"wrote {tsv_path}: {len(rows)} rows") return 0 if __name__ == "__main__": raise SystemExit(main())