✨ feat(tsl-codegen): add decoupled documentation toolkit
Generate TSL API Markdown from YAML or JSON into a configurable project scope.\nAdd file and directory lint modes, tags-aware indexing, and keyword search across tags and descriptions.\nBundle the toolkit through the playbook build and sync workflows.
This commit is contained in:
@@ -19,12 +19,13 @@ def build_agents_text(ruleset_path: Path) -> str:
|
||||
return text.rstrip("\n") + "\n"
|
||||
|
||||
|
||||
def ensure_sources(repo_root: Path) -> tuple[Path, Path, Path, Path]:
|
||||
def ensure_sources(repo_root: Path) -> tuple[Path, Path, Path, Path, Path]:
|
||||
docs_tsl = repo_root / "docs" / "tsl"
|
||||
syntax_skill = repo_root / "skills" / "tsl-syntax-reference"
|
||||
api_skill = repo_root / "skills" / "tsl-api-reference"
|
||||
ruleset = repo_root / "rulesets" / "tsl" / "index.md"
|
||||
sources = (docs_tsl, syntax_skill, api_skill, ruleset)
|
||||
codegen_toolkit = repo_root / "tools" / "tsl-codegen"
|
||||
sources = (docs_tsl, syntax_skill, api_skill, ruleset, codegen_toolkit)
|
||||
missing = [str(path) for path in sources if not path.exists()]
|
||||
if missing:
|
||||
raise FileNotFoundError("missing source path(s): " + ", ".join(missing))
|
||||
@@ -47,7 +48,9 @@ def clean_output(output: Path, repo_root: Path) -> None:
|
||||
|
||||
|
||||
def build(output: Path, repo_root: Path) -> None:
|
||||
docs_tsl, syntax_skill, api_skill, ruleset = ensure_sources(repo_root)
|
||||
docs_tsl, syntax_skill, api_skill, ruleset, codegen_toolkit = ensure_sources(
|
||||
repo_root
|
||||
)
|
||||
clean_output(output, repo_root)
|
||||
|
||||
(output / "AGENTS.md").write_text(
|
||||
@@ -56,6 +59,7 @@ def build(output: Path, repo_root: Path) -> None:
|
||||
copy_tree(docs_tsl, output / "docs" / "tsl")
|
||||
copy_tree(syntax_skill, output / "skills" / "tsl-syntax-reference")
|
||||
copy_tree(api_skill, output / "skills" / "tsl-api-reference")
|
||||
copy_tree(codegen_toolkit, output / "tools" / "tsl-codegen")
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
|
||||
@@ -1,361 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild the bundled TSL API function index from the codegen markdown tree.
|
||||
|
||||
The markdown tree is the source of truth: each `## `sig`` / `### `sig`` heading
|
||||
is one function entry. Both the tsv and every index.md are derived products;
|
||||
regenerate them whenever the leaf md changes rather than editing by hand. The
|
||||
top/scope/module index.md pages are rebuilt from the same entry headings the
|
||||
tsv uses, so their `函数数` counts can never drift from the tsv.
|
||||
|
||||
Columns (tab-separated, LF line endings, UTF-8):
|
||||
name scope module signature page anchor summary
|
||||
- name: signature text up to the first '('
|
||||
- scope: first path segment under the codegen root (builtin | dotnet)
|
||||
- module: second path segment for nested pages, else the flat file stem
|
||||
- signature: verbatim from the heading, backticks stripped
|
||||
- page: POSIX path relative to the codegen root
|
||||
- anchor: GitHub-style slug of the name (lowercased, chars outside
|
||||
[a-z0-9_] removed); per-page duplicate slugs get -1/-2 suffixes
|
||||
in document order, matching the rendered heading anchors.
|
||||
- summary: first prose line under the entry heading, empty for table,
|
||||
heading, or standalone return-type lines
|
||||
|
||||
Usage:
|
||||
python scripts/tsl_codegen_function_index.py # rewrite in place
|
||||
python scripts/tsl_codegen_function_index.py --check # verify, no write
|
||||
python scripts/tsl_codegen_function_index.py --root PATH --tsv PATH
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
ENTRY_RE = re.compile(r"^#{2,3}(?!#)\s+`(.+?)`\s*$")
|
||||
RETURN_RE = re.compile(r"^返回[:：]")
|
||||
HEADER = ["name", "scope", "module", "signature", "page", "anchor", "summary"]
|
||||
|
||||
|
||||
def slug(name):
|
||||
"""GitHub-style anchor slug: lowercase, keep [a-z0-9_], drop the rest."""
|
||||
return re.sub(r"[^a-z0-9_]", "", name.lower())
|
||||
|
||||
|
||||
def extract_summary(lines, heading_idx):
|
||||
"""Return the first prose line under a function entry heading."""
|
||||
for line in lines[heading_idx + 1:]:
|
||||
text = line.strip()
|
||||
if not text:
|
||||
continue
|
||||
if text.startswith("|") or text.startswith("#") or RETURN_RE.match(text):
|
||||
return ""
|
||||
return text.replace("\t", " ")
|
||||
return ""
|
||||
|
||||
|
||||
def parse_page(codegen_root, md):
|
||||
"""Yield [name, scope, module, signature, page, anchor, summary] rows."""
|
||||
page = md.relative_to(codegen_root).as_posix()
|
||||
scope, module = scope_module(page)
|
||||
seen = {}
|
||||
rows = []
|
||||
lines = md.read_text(encoding="utf-8").splitlines()
|
||||
for idx, line in enumerate(lines):
|
||||
m = ENTRY_RE.match(line)
|
||||
if not m:
|
||||
continue
|
||||
sig = m.group(1)
|
||||
name = sig.split("(", 1)[0]
|
||||
base = slug(name)
|
||||
n = seen.get(base, 0)
|
||||
seen[base] = n + 1
|
||||
anchor = base if n == 0 else f"{base}-{n}"
|
||||
summary = extract_summary(lines, idx)
|
||||
rows.append([name, scope, module, sig, page, anchor, summary])
|
||||
return rows
|
||||
|
||||
|
||||
def scope_module(page):
|
||||
parts = page.split("/")
|
||||
scope = parts[0]
|
||||
module = parts[1] if len(parts) >= 3 else Path(parts[-1]).stem
|
||||
return scope, module
|
||||
|
||||
|
||||
def build_rows(codegen_root):
|
||||
"""Scan the whole codegen tree, return sorted rows (skips index.md)."""
|
||||
rows = []
|
||||
for md in sorted(codegen_root.rglob("*.md")):
|
||||
if md.name == "index.md":
|
||||
continue
|
||||
rows.extend(parse_page(codegen_root, md))
|
||||
rows.sort(key=lambda r: (r[0].lower(), r[4], r[3]))
|
||||
return rows
|
||||
|
||||
|
||||
def render_tsv(rows):
|
||||
lines = ["\t".join(HEADER)]
|
||||
lines.extend("\t".join(r) for r in rows)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def read_tsv(tsv_path):
|
||||
text = tsv_path.read_text(encoding="utf-8")
|
||||
return [line.split("\t") for line in text.splitlines() if line.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# index.md generation
|
||||
#
|
||||
# index.md pages are navigation, derived entirely from the leaf pages:
|
||||
# - counts come from the same ENTRY_RE the tsv uses (single source of truth,
|
||||
# so an index count can never disagree with the tsv);
|
||||
# - labels come from each leaf page's H1;
|
||||
# - the module title is the leaf H1 with its last " / " segment stripped.
|
||||
# Per builtin-doc-template-rule, `函数数：N` lives only in index.md, and each
|
||||
# index.md uses at most `#`/`##`, so it never trips markdownlint MD001.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
H1_RE = re.compile(r"^#\s+(.+?)\s*$")
|
||||
|
||||
SCOPE_META = {
|
||||
"builtin": {
|
||||
"heading": "Builtin",
|
||||
"scope_desc": "本目录为本地 TSL 内置函数。",
|
||||
"link_desc": "本地 TSL 内置函数。",
|
||||
# builtin leaf H1 is 'Builtin - 基础 / 数组'; strip the scope prefix so
|
||||
# module labels read '基础', mirroring dotnet's prefix-free '债券'.
|
||||
"title_prefix": "Builtin - ",
|
||||
},
|
||||
"dotnet": {
|
||||
"heading": "Dotnet",
|
||||
"scope_desc": "本目录为 .NET 平台函数（按功能重分类）。",
|
||||
"link_desc": ".NET 平台函数。",
|
||||
"title_prefix": "",
|
||||
},
|
||||
}
|
||||
|
||||
TOP_PREAMBLE = (
|
||||
"# TSL Codegen\n\n"
|
||||
"本目录是 TSL 函数调用事实目录。\n\n"
|
||||
"目录链路表达函数性质；函数条目只保留调用事实。\n\n"
|
||||
"函数查询从这里开始。\n\n"
|
||||
"## 主目录\n\n"
|
||||
)
|
||||
|
||||
|
||||
def page_h1(md):
|
||||
"""First `# ` heading text, or the file stem if the page has no H1."""
|
||||
for line in md.read_text(encoding="utf-8").splitlines():
|
||||
m = H1_RE.match(line)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return md.stem
|
||||
|
||||
|
||||
def page_entry_count(md):
|
||||
"""Number of function entries on a leaf page (same rule as the tsv)."""
|
||||
return sum(1 for line in md.read_text(encoding="utf-8").splitlines()
|
||||
if ENTRY_RE.match(line))
|
||||
|
||||
|
||||
def module_title(h1):
|
||||
"""Leaf H1 with its last ' / ' segment stripped ('债券 / 基本信息'→'债券')."""
|
||||
return h1.rsplit(" / ", 1)[0] if " / " in h1 else h1
|
||||
|
||||
|
||||
def display_label(h1, prefix):
|
||||
"""Strip a scope's title_prefix from an H1 for display in index tables."""
|
||||
return h1[len(prefix):] if prefix and h1.startswith(prefix) else h1
|
||||
|
||||
|
||||
def collect_leaves(codegen_root):
|
||||
"""Return leaf-page dicts: rel/scope/module/file/h1/count (skips index.md).
|
||||
|
||||
module is None for flat pages that sit directly under a scope
|
||||
(e.g. dotnet/forex.md). Pages deeper than scope/module/leaf.md are skipped.
|
||||
"""
|
||||
leaves = []
|
||||
for md in sorted(codegen_root.rglob("*.md")):
|
||||
if md.name == "index.md":
|
||||
continue
|
||||
parts = md.relative_to(codegen_root).as_posix().split("/")
|
||||
if len(parts) == 3:
|
||||
scope, module = parts[0], parts[1]
|
||||
elif len(parts) == 2:
|
||||
scope, module = parts[0], None
|
||||
else:
|
||||
continue
|
||||
leaves.append(
|
||||
{
|
||||
"scope": scope,
|
||||
"module": module,
|
||||
"file": parts[-1],
|
||||
"h1": page_h1(md),
|
||||
"count": page_entry_count(md),
|
||||
}
|
||||
)
|
||||
return leaves
|
||||
|
||||
|
||||
def render_index_table(header_cols, rows):
|
||||
"""Render a 3-column markdown table; last column right-aligned."""
|
||||
out = [f"| {' | '.join(header_cols)} |", "| --- | --- | ---: |"]
|
||||
out.extend(f"| {label} | [{label}]({link}) | {count} |" for label, link, count in rows)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def build_indexes(codegen_root):
|
||||
"""Return {absolute Path: text} for every top/scope/module index.md."""
|
||||
leaves = collect_leaves(codegen_root)
|
||||
pages = {}
|
||||
|
||||
scopes = sorted({leaf["scope"] for leaf in leaves})
|
||||
|
||||
# top index.md
|
||||
top_links = []
|
||||
for scope in scopes:
|
||||
desc = SCOPE_META.get(scope, {}).get("link_desc", "")
|
||||
top_links.append(f"- [{scope}/]({scope}/)：{desc}")
|
||||
pages[codegen_root / "index.md"] = TOP_PREAMBLE + "\n".join(top_links) + "\n"
|
||||
|
||||
for scope in scopes:
|
||||
scope_leaves = [x for x in leaves if x["scope"] == scope]
|
||||
meta = SCOPE_META.get(scope, {"heading": scope, "scope_desc": ""})
|
||||
modules = sorted({x["module"] for x in scope_leaves if x["module"]})
|
||||
flat = sorted((x for x in scope_leaves if x["module"] is None),
|
||||
key=lambda x: x["file"])
|
||||
|
||||
prefix = meta.get("title_prefix", "")
|
||||
|
||||
scope_rows = []
|
||||
for module in modules:
|
||||
mod_leaves = sorted(
|
||||
(x for x in scope_leaves if x["module"] == module),
|
||||
key=lambda x: x["file"],
|
||||
)
|
||||
title = display_label(module_title(mod_leaves[0]["h1"]), prefix)
|
||||
total = sum(x["count"] for x in mod_leaves)
|
||||
scope_rows.append((title, f"{module}/", total))
|
||||
|
||||
# module index.md
|
||||
mod_rows = [
|
||||
(display_label(x["h1"], prefix), x["file"], x["count"])
|
||||
for x in mod_leaves
|
||||
]
|
||||
mod_text = (
|
||||
f"# {title}\n\n"
|
||||
f"函数数：{total}\n\n"
|
||||
f"{render_index_table(['叶子', '文件', '函数数'], mod_rows)}\n"
|
||||
)
|
||||
pages[codegen_root / scope / module / "index.md"] = mod_text
|
||||
|
||||
for leaf in flat:
|
||||
scope_rows.append(
|
||||
(display_label(leaf["h1"], prefix), leaf["file"], leaf["count"])
|
||||
)
|
||||
|
||||
scope_total = sum(x["count"] for x in scope_leaves)
|
||||
scope_text = (
|
||||
f"# {meta['heading']}\n\n"
|
||||
f"{meta['scope_desc']}\n\n"
|
||||
f"函数数：{scope_total}\n\n"
|
||||
f"{render_index_table(['模块', '目录', '函数数'], scope_rows)}\n"
|
||||
)
|
||||
pages[codegen_root / scope / "index.md"] = scope_text
|
||||
|
||||
return pages
|
||||
|
||||
|
||||
def check_indexes(codegen_root):
|
||||
"""Return list of (path, reason) where a generated index differs from disk."""
|
||||
problems = []
|
||||
for path, text in build_indexes(codegen_root).items():
|
||||
if not path.is_file():
|
||||
problems.append((path, "missing"))
|
||||
elif path.read_text(encoding="utf-8") != text:
|
||||
problems.append((path, "stale"))
|
||||
return problems
|
||||
|
||||
|
||||
def write_indexes(codegen_root):
|
||||
"""Write all generated index.md pages; return the count written."""
|
||||
pages = build_indexes(codegen_root)
|
||||
for path, text in pages.items():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8", newline="\n")
|
||||
return len(pages)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
help="codegen root (default: skills/tsl-api-reference/references/codegen)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tsv",
|
||||
help="output tsv (default: skills/tsl-api-reference/data/function_index.tsv)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="verify the tsv matches the md tree; exit 1 if not (no write)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
skill_root = repo_root / "skills" / "tsl-api-reference"
|
||||
codegen_root = (
|
||||
Path(args.root) if args.root else skill_root / "references" / "codegen"
|
||||
)
|
||||
if not codegen_root.is_dir():
|
||||
print(f"ERROR: codegen root not found: {codegen_root}", file=sys.stderr)
|
||||
return 1
|
||||
tsv_path = Path(args.tsv) if args.tsv else skill_root / "data" / "function_index.tsv"
|
||||
|
||||
rows = build_rows(codegen_root)
|
||||
new_text = render_tsv(rows)
|
||||
|
||||
if args.check:
|
||||
index_problems = check_indexes(codegen_root)
|
||||
if not tsv_path.is_file():
|
||||
print(f"MISMATCH: tsv does not exist: {tsv_path}", file=sys.stderr)
|
||||
return 1
|
||||
current = tsv_path.read_text(encoding="utf-8")
|
||||
tsv_ok = current == new_text
|
||||
if tsv_ok and not index_problems:
|
||||
print(
|
||||
f"OK: {tsv_path} matches md tree ({len(rows)} rows); "
|
||||
f"index.md pages up to date"
|
||||
)
|
||||
return 0
|
||||
if not tsv_ok:
|
||||
cur_rows = read_tsv(tsv_path)[1:]
|
||||
cur_keys = {tuple(r) for r in cur_rows}
|
||||
new_keys = {tuple(r) for r in rows}
|
||||
print(
|
||||
f"MISMATCH: tsv out of date "
|
||||
f"(tsv {len(cur_rows)} rows, md {len(rows)} rows; "
|
||||
f"+{len(new_keys - cur_keys)} -{len(cur_keys - new_keys)}). "
|
||||
f"Run without --check to rebuild.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
for path, reason in index_problems:
|
||||
print(
|
||||
f"MISMATCH: index {reason}: "
|
||||
f"{path.relative_to(codegen_root).as_posix()}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
tsv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tsv_path.write_text(new_text, encoding="utf-8", newline="\n")
|
||||
n_index = write_indexes(codegen_root)
|
||||
print(f"wrote {tsv_path}: {len(rows)} rows; {n_index} index.md pages")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user