✨ feat(tsl-codegen): add decoupled documentation toolkit
Generate TSL API Markdown from YAML or JSON into a configurable project scope.\nAdd file and directory lint modes, tags-aware indexing, and keyword search across tags and descriptions.\nBundle the toolkit through the playbook build and sync workflows.
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild the bundled TSL API function index from the codegen markdown tree.
|
||||
|
||||
The markdown tree is the source of truth: each `## `sig`` / `### `sig`` heading
|
||||
is one function entry. The TSV is a derived product; regenerate it whenever the
|
||||
leaf Markdown changes rather than editing it by hand.
|
||||
|
||||
Columns (tab-separated, LF line endings, UTF-8):
|
||||
name scope module signature page anchor tags summary
|
||||
- name: signature text up to the first '('
|
||||
- scope: first path segment under the codegen root (for example project)
|
||||
- module: second path segment for nested pages, else the flat file stem
|
||||
- signature: verbatim from the heading, backticks stripped
|
||||
- page: POSIX path relative to the codegen root
|
||||
- anchor: GitHub-style slug of the name (lowercased, chars outside
|
||||
[a-z0-9_] removed); per-page duplicate slugs get -1/-2 suffixes
|
||||
in document order, matching the rendered heading anchors.
|
||||
- tags: space-separated keywords from `<!-- tags: ... -->`
|
||||
- summary: first prose line under the entry heading, empty for table,
|
||||
heading, or standalone return-type lines
|
||||
|
||||
Usage (run from repo root; --skill-dir is required):
|
||||
SKILL=skills/tsl-api-reference
|
||||
python tools/tsl-codegen/scripts/build_index.py \
|
||||
--skill-dir "$SKILL" # rewrite the TSV in place
|
||||
python tools/tsl-codegen/scripts/build_index.py \
|
||||
--skill-dir "$SKILL" --check # verify against md tree, no write
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
ENTRY_RE = re.compile(r"^#{2,3}(?!#)\s+`(.+?)`\s*$")
|
||||
RETURN_RE = re.compile(r"^返回[:：]")
|
||||
TAGS_RE = re.compile(r"^<!--\s*tags:\s*(.*?)\s*-->$")
|
||||
HEADER = [
|
||||
"name",
|
||||
"scope",
|
||||
"module",
|
||||
"signature",
|
||||
"page",
|
||||
"anchor",
|
||||
"tags",
|
||||
"summary",
|
||||
]
|
||||
|
||||
|
||||
def slug(name):
|
||||
"""GitHub-style anchor slug: lowercase, keep [a-z0-9_], drop the rest."""
|
||||
return re.sub(r"[^a-z0-9_]", "", name.lower())
|
||||
|
||||
|
||||
def extract_metadata(lines, heading_idx):
|
||||
"""Return tags and the first prose line under a function entry heading."""
|
||||
tags = ""
|
||||
for line in lines[heading_idx + 1:]:
|
||||
text = line.strip()
|
||||
if not text:
|
||||
continue
|
||||
tag_match = TAGS_RE.match(text)
|
||||
if tag_match:
|
||||
tags = " ".join(tag_match.group(1).split()).replace("\t", " ")
|
||||
continue
|
||||
if text.startswith("|") or text.startswith("#") or RETURN_RE.match(text):
|
||||
return tags, ""
|
||||
return tags, text.replace("\t", " ")
|
||||
return tags, ""
|
||||
|
||||
|
||||
def parse_page(codegen_root, md):
|
||||
"""Yield index rows for one Markdown page."""
|
||||
page = md.relative_to(codegen_root).as_posix()
|
||||
scope, module = scope_module(page)
|
||||
seen = {}
|
||||
rows = []
|
||||
lines = md.read_text(encoding="utf-8").splitlines()
|
||||
for idx, line in enumerate(lines):
|
||||
m = ENTRY_RE.match(line)
|
||||
if not m:
|
||||
continue
|
||||
sig = m.group(1)
|
||||
name = sig.split("(", 1)[0]
|
||||
base = slug(name)
|
||||
n = seen.get(base, 0)
|
||||
seen[base] = n + 1
|
||||
anchor = base if n == 0 else f"{base}-{n}"
|
||||
tags, summary = extract_metadata(lines, idx)
|
||||
rows.append([name, scope, module, sig, page, anchor, tags, summary])
|
||||
return rows
|
||||
|
||||
|
||||
def scope_module(page):
|
||||
parts = page.split("/")
|
||||
scope = parts[0]
|
||||
module = parts[1] if len(parts) >= 3 else Path(parts[-1]).stem
|
||||
return scope, module
|
||||
|
||||
|
||||
def build_rows(codegen_root):
|
||||
"""Scan the whole codegen tree and return sorted rows."""
|
||||
rows = []
|
||||
for md in sorted(codegen_root.rglob("*.md")):
|
||||
rows.extend(parse_page(codegen_root, md))
|
||||
rows.sort(key=lambda r: (r[0].lower(), r[4], r[3]))
|
||||
return rows
|
||||
|
||||
|
||||
def render_tsv(rows):
|
||||
lines = ["\t".join(HEADER)]
|
||||
lines.extend("\t".join(r) for r in rows)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def read_tsv(tsv_path):
|
||||
text = tsv_path.read_text(encoding="utf-8")
|
||||
return [line.split("\t") for line in text.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument(
|
||||
"--skill-dir",
|
||||
required=True,
|
||||
help="tsl-api-reference skill dir; codegen root defaults to "
|
||||
"<skill-dir>/references/codegen and tsv to "
|
||||
"<skill-dir>/data/function_index.tsv",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
help="explicit codegen root, overriding the one derived from --skill-dir",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tsv",
|
||||
help="explicit output tsv, overriding the one derived from --skill-dir",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="verify the tsv matches the md tree; exit 1 if not (no write)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
skill_root = Path(args.skill_dir)
|
||||
codegen_root = (
|
||||
Path(args.root) if args.root else skill_root / "references" / "codegen"
|
||||
)
|
||||
if not codegen_root.is_dir():
|
||||
print(f"ERROR: codegen root not found: {codegen_root}", file=sys.stderr)
|
||||
return 1
|
||||
tsv_path = (
|
||||
Path(args.tsv)
|
||||
if args.tsv
|
||||
else skill_root / "data" / "function_index.tsv"
|
||||
)
|
||||
|
||||
rows = build_rows(codegen_root)
|
||||
new_text = render_tsv(rows)
|
||||
|
||||
if args.check:
|
||||
if not tsv_path.is_file():
|
||||
print(f"MISMATCH: tsv does not exist: {tsv_path}", file=sys.stderr)
|
||||
return 1
|
||||
current = tsv_path.read_text(encoding="utf-8")
|
||||
tsv_ok = current == new_text
|
||||
if tsv_ok:
|
||||
print(f"OK: {tsv_path} matches md tree ({len(rows)} rows)")
|
||||
return 0
|
||||
if not tsv_ok:
|
||||
cur_rows = read_tsv(tsv_path)[1:]
|
||||
cur_keys = {tuple(r) for r in cur_rows}
|
||||
new_keys = {tuple(r) for r in rows}
|
||||
print(
|
||||
f"MISMATCH: tsv out of date "
|
||||
f"(tsv {len(cur_rows)} rows, md {len(rows)} rows; "
|
||||
f"+{len(new_keys - cur_keys)} -{len(cur_keys - new_keys)}). "
|
||||
f"Run without --check to rebuild.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
tsv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tsv_path.write_text(new_text, encoding="utf-8", newline="\n")
|
||||
print(f"wrote {tsv_path}: {len(rows)} rows")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user