csh/playbook
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ feat(tsl-codegen): add decoupled documentation toolkit

Browse Source 
Generate TSL API Markdown from YAML or JSON into a configurable project scope.\nAdd file and directory lint modes, tags-aware indexing, and keyword search across tags and descriptions.\nBundle the toolkit through the playbook build and sync workflows.
This commit is contained in:
csh
2026-07-20 09:08:38 +08:00
parent 1d5304e7b6
commit c69278283f
18 changed files with 13955 additions and 12821 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tsl-codegen/scripts/build_index.py
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""Rebuild the bundled TSL API function index from the codegen markdown tree.
The markdown tree is the source of truth: each `## `sig`` / `### `sig`` heading
is one function entry. The TSV is a derived product; regenerate it whenever the
leaf Markdown changes rather than editing it by hand.
Columns (tab-separated, LF line endings, UTF-8):
name scope module signature page anchor tags summary
- name: signature text up to the first '('
- scope: first path segment under the codegen root (for example project)
- module: second path segment for nested pages, else the flat file stem
- signature: verbatim from the heading, backticks stripped
- page: POSIX path relative to the codegen root
- anchor: GitHub-style slug of the name (lowercased, chars outside
[a-z0-9_] removed); per-page duplicate slugs get -1/-2 suffixes
in document order, matching the rendered heading anchors.
- tags: space-separated keywords from `<!-- tags: ... -->`
- summary: first prose line under the entry heading, empty for table,
heading, or standalone return-type lines
Usage (run from repo root; --skill-dir is required):
SKILL=skills/tsl-api-reference
python tools/tsl-codegen/scripts/build_index.py \
--skill-dir "$SKILL" # rewrite the TSV in place
python tools/tsl-codegen/scripts/build_index.py \
--skill-dir "$SKILL" --check # verify against md tree, no write
"""
import argparse
import sys
from pathlib import Path
import re
ENTRY_RE = re.compile(r"^#{2,3}(?!#)\s+`(.+?)`\s*$")
RETURN_RE = re.compile(r"^返回[:：]")
TAGS_RE = re.compile(r"^<!--\s*tags:\s*(.*?)\s*-->$")
HEADER = [
"name",
"scope",
"module",
"signature",
"page",
"anchor",
"tags",
"summary",
]
def slug(name):
"""GitHub-style anchor slug: lowercase, keep [a-z0-9_], drop the rest."""
return re.sub(r"[^a-z0-9_]", "", name.lower())
def extract_metadata(lines, heading_idx):
"""Return tags and the first prose line under a function entry heading."""
tags = ""
for line in lines[heading_idx + 1:]:
text = line.strip()
if not text:
continue
tag_match = TAGS_RE.match(text)
if tag_match:
tags = " ".join(tag_match.group(1).split()).replace("\t", " ")
continue
if text.startswith("|") or text.startswith("#") or RETURN_RE.match(text):
return tags, ""
return tags, text.replace("\t", " ")
return tags, ""
def parse_page(codegen_root, md):
"""Yield index rows for one Markdown page."""
page = md.relative_to(codegen_root).as_posix()
scope, module = scope_module(page)
seen = {}
rows = []
lines = md.read_text(encoding="utf-8").splitlines()
for idx, line in enumerate(lines):
m = ENTRY_RE.match(line)
if not m:
continue
sig = m.group(1)
name = sig.split("(", 1)[0]
base = slug(name)
n = seen.get(base, 0)
seen[base] = n + 1
anchor = base if n == 0 else f"{base}-{n}"
tags, summary = extract_metadata(lines, idx)
rows.append([name, scope, module, sig, page, anchor, tags, summary])
return rows
def scope_module(page):
parts = page.split("/")
scope = parts[0]
module = parts[1] if len(parts) >= 3 else Path(parts[-1]).stem
return scope, module
def build_rows(codegen_root):
"""Scan the whole codegen tree and return sorted rows."""
rows = []
for md in sorted(codegen_root.rglob("*.md")):
rows.extend(parse_page(codegen_root, md))
rows.sort(key=lambda r: (r[0].lower(), r[4], r[3]))
return rows
def render_tsv(rows):
lines = ["\t".join(HEADER)]
lines.extend("\t".join(r) for r in rows)
return "\n".join(lines) + "\n"
def read_tsv(tsv_path):
text = tsv_path.read_text(encoding="utf-8")
return [line.split("\t") for line in text.splitlines() if line.strip()]
def main(argv=None):
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument(
"--skill-dir",
required=True,
help="tsl-api-reference skill dir; codegen root defaults to "
"<skill-dir>/references/codegen and tsv to "
"<skill-dir>/data/function_index.tsv",
)
parser.add_argument(
"--root",
help="explicit codegen root, overriding the one derived from --skill-dir",
)
parser.add_argument(
"--tsv",
help="explicit output tsv, overriding the one derived from --skill-dir",
)
parser.add_argument(
"--check",
action="store_true",
help="verify the tsv matches the md tree; exit 1 if not (no write)",
)
args = parser.parse_args(argv)
skill_root = Path(args.skill_dir)
codegen_root = (
Path(args.root) if args.root else skill_root / "references" / "codegen"
)
if not codegen_root.is_dir():
print(f"ERROR: codegen root not found: {codegen_root}", file=sys.stderr)
return 1
tsv_path = (
Path(args.tsv)
if args.tsv
else skill_root / "data" / "function_index.tsv"
)
rows = build_rows(codegen_root)
new_text = render_tsv(rows)
if args.check:
if not tsv_path.is_file():
print(f"MISMATCH: tsv does not exist: {tsv_path}", file=sys.stderr)
return 1
current = tsv_path.read_text(encoding="utf-8")
tsv_ok = current == new_text
if tsv_ok:
print(f"OK: {tsv_path} matches md tree ({len(rows)} rows)")
return 0
if not tsv_ok:
cur_rows = read_tsv(tsv_path)[1:]
cur_keys = {tuple(r) for r in cur_rows}
new_keys = {tuple(r) for r in rows}
print(
f"MISMATCH: tsv out of date "
f"(tsv {len(cur_rows)} rows, md {len(rows)} rows; "
f"+{len(new_keys - cur_keys)} -{len(cur_keys - new_keys)}). "
f"Run without --check to rebuild.",
file=sys.stderr,
)
return 1
tsv_path.parent.mkdir(parents=True, exist_ok=True)
tsv_path.write_text(new_text, encoding="utf-8", newline="\n")
print(f"wrote {tsv_path}: {len(rows)} rows")
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/tsl-codegen/scripts/generate.py
+246
View File
@@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""Generate compliant TSL codegen markdown from a YAML/JSON entry file.
The recording format is one leaf page: a `module` title, a relative `path`, and
a `functions` list. This script renders it to the markdown the codegen tree
stores, matching tools/tsl-codegen/STANDARD.md.
Tables are emitted as valid Markdown with single-space padding. Prettier may be
used optionally to align columns.
Input dispatch is by extension: .json parses with the stdlib (keeping the
toolchain dependency-free); .yml/.yaml needs pyyaml. If pyyaml is missing the
script says so and points at the JSON path.
Entry schema (per function):
signature required verbatim, underscores/case untouched
desc required description; may contain multiple lines
tags optional list of Chinese keywords -> `<!-- tags: ... -->`
params required when the signature takes args; omit for nullary
returns required return type
example optional fenced tsl block, pasted verbatim
Each param: name/type/desc required; optional (bool) -> `可选。` prefix;
values (list of {value, desc}) -> a `**name 取值**` enum section.
Usage (run from repo root):
python tools/tsl-codegen/scripts/generate.py entry.yml
python tools/tsl-codegen/scripts/generate.py entry.json \
--scope my-project
"""
import argparse
import json
import sys
from pathlib import Path
def die(msg):
print(f"ERROR: {msg}", file=sys.stderr)
raise SystemExit(1)
def scope_name(value):
"""Validate a user-defined single directory name."""
if not value or value in {".", ".."} or "/" in value or "\\" in value:
raise argparse.ArgumentTypeError("scope 必须是单级目录名")
return value
def resolve_format(path, fmt):
"""Pick the parser: explicit --format wins, else derive from extension."""
if fmt:
return fmt
suffix = path.suffix.lower()
if suffix == ".json":
return "json"
if suffix in (".yml", ".yaml"):
return "yaml"
die(
f"cannot infer format from extension '{suffix}'; "
f"pass --format json|yaml"
)
def load_entries(path, fmt=None):
"""Parse a recording file as JSON (stdlib) or YAML (pyyaml).
Format is chosen by --format when given, else by file extension.
"""
text = path.read_text(encoding="utf-8")
fmt = resolve_format(path, fmt)
if fmt == "json":
try:
return json.loads(text)
except json.JSONDecodeError as exc:
die(f"invalid JSON in {path}: {exc}")
if fmt == "yaml":
try:
import yaml
except ImportError:
die(
"pyyaml is not installed; either `pip install pyyaml` or "
"convert the input to .json (json parses with the stdlib)"
)
try:
return yaml.safe_load(text)
except yaml.YAMLError as exc:
die(f"invalid YAML in {path}: {exc}")
die(f"unknown format '{fmt}'; use json or yaml")
def escape_cell(text):
"""Escape `|` so a value stays inside one markdown table cell."""
return str(text).replace("|", "\\|")
def require(cond, msg):
if not cond:
die(msg)
def param_desc(param, where):
"""Description column text: prepend `可选。` for optional params."""
desc = param.get("desc")
require(desc, f"{where}: param '{param.get('name', '?')}' missing desc")
if param.get("optional") and not desc.startswith("可选。"):
return "可选。" + desc
return desc
def render_param_table(params, where):
"""Three-column 参数/类型/说明 table with single-space padding."""
lines = ["| 参数 | 类型 | 说明 |", "| --- | --- | --- |"]
for param in params:
name = param.get("name")
ptype = param.get("type")
require(name, f"{where}: a param is missing 'name'")
require(ptype, f"{where}: param '{name}' missing 'type'")
desc = param_desc(param, where)
lines.append(
f"| `{escape_cell(name)}` | {escape_cell(ptype)} | {escape_cell(desc)} |"
)
return lines
def render_enum_sections(params):
"""`**name 取值**` sections for every param carrying a `values` list."""
lines = []
for param in params:
values = param.get("values")
if not values:
continue
lines.append(f"**{param['name']} 取值**")
lines.append("")
for item in values:
lines.append(f"- `{item['value']}` — {item['desc']}")
lines.append("")
return lines
def render_function(fn, index):
"""Render one function entry to a list of lines (no trailing blank)."""
where = f"functions[{index}]"
sig = fn.get("signature")
require(sig, f"{where}: missing 'signature'")
desc = fn.get("desc")
require(desc, f"{where} ({sig}): missing 'desc'")
returns = fn.get("returns")
require(returns, f"{where} ({sig}): missing 'returns'")
lines = [f"## `{sig}`"]
tags = fn.get("tags")
if tags:
lines.append(f"<!-- tags: {' '.join(str(t) for t in tags)} -->")
lines.append("")
lines.append(desc)
lines.append("")
params = fn.get("params") or []
if params:
lines.extend(render_param_table(params, f"{where} ({sig})"))
lines.append("")
lines.extend(render_enum_sections(params))
lines.append(f"返回：{returns}")
example = fn.get("example")
if example:
lines.append("")
lines.append("### 示例")
lines.append("")
lines.append("```tsl")
lines.extend(example.rstrip("\n").split("\n"))
lines.append("```")
return lines
def render_page(data):
"""Render a whole leaf page: H1 + every function entry."""
require(isinstance(data, dict), "input root must be a mapping")
module = data.get("module")
require(module, "input missing 'module'")
functions = data.get("functions")
require(functions, "input missing non-empty 'functions'")
out = [f"# {module}", ""]
for index, fn in enumerate(functions):
out.extend(render_function(fn, index))
out.append("")
return "\n".join(out).rstrip("\n") + "\n"
def output_path(data, scope):
"""Build the leaf-page destination from the recording file's relative path."""
relative = data.get("path")
require(relative, "input missing 'path'")
require(isinstance(relative, str), "'path' must be a string")
require("\\" not in relative, "'path' must use '/' as the separator")
relative_path = Path(relative)
require(not relative_path.is_absolute(), "'path' must be relative")
require(".." not in relative_path.parts, "'path' must not contain '..'")
require(relative_path.suffix == "", "'path' must not include a file extension")
return (
Path("skills/tsl-api-reference/references/codegen")
/ scope
/ relative_path.with_suffix(".md")
)
def main(argv=None):
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
parser = argparse.ArgumentParser(description="从 YAML/JSON 录入文件生成 TSL 函数文档")
parser.add_argument(
"input",
metavar="INPUT_FILE",
help="YAML/JSON 录入文件路径，例如 tmp/my-functions.yaml",
)
parser.add_argument(
"--scope",
type=scope_name,
default="project",
help="codegen 下的一级目录，可自定义（默认：project）",
)
parser.add_argument(
"--format",
choices=["json", "yaml"],
help="录入文件格式；默认根据文件扩展名判断",
)
args = parser.parse_args(argv)
in_path = Path(args.input)
if not in_path.is_file():
die(f"input not found: {in_path}")
data = load_entries(in_path, args.format)
text = render_page(data)
out_path = output_path(data, args.scope)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(text, encoding="utf-8", newline="\n")
print(
f"wrote {out_path}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/tsl-codegen/scripts/lint.py
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
"""Lint TSL codegen function-doc markdown against the house standard.
The standard lives in tools/tsl-codegen/STANDARD.md.
Each `## `sig`` / `### `sig`` heading starts one function entry. Rules split
into hard errors (CI-blocking) and soft warnings (style
convergence over the ~12k existing entries).
Hard errors:
- missing/empty description (first prose line after the signature)
- missing `返回：类型`
- signature has parameters but the entry has no parameter table
- signature has no parameters but a parameter table is present
- parameter table header is not the fixed 参数 / 类型 / 说明 三列
Soft warnings:
- optional-parameter wording not starting with `可选。`
- malformed / empty `<!-- tags: ... -->` line
Exit status: 1 if any error (or, with --strict, any warning); else 0.
Usage:
python lint.py --file path/to/page.md
python lint.py --dir path/to/codegen-dir
python lint.py --dir path/to/codegen-dir --strict
"""
import argparse
import re
import sys
from pathlib import Path
# Entry heading: `## `sig`` or `### `sig``. Matches the index generator's rule
# so the linter and the tsv agree on what a function entry is.
ENTRY_RE = re.compile(r"^(#{2,3})(?!#)\s+`(.+?)`\s*$")
RETURN_RE = re.compile(r"^返回[:：]")
TAGS_RE = re.compile(r"^<!--\s*tags:\s*(.*?)\s*-->\s*$")
FENCE_RE = re.compile(r"^(```|~~~)")
OPTIONAL_HINT_RE = re.compile(r"可选|可省略|省略")
# Split a table row on unescaped pipes so `nil\|array` stays one cell.
CELL_SPLIT_RE = re.compile(r"(?<!\\)\|")
SEP_CELL_RE = re.compile(r"^:?-+:?$")
PARAM_HEADER = ["参数", "类型", "说明"]
def iter_entries(lines):
"""Yield (start, end, signature): each entry spans one signature heading
to the next. Category headings without backticks fall to the tail of the
preceding entry (harmless — checks anchor on the entry's head)."""
starts = [
(idx, m.group(2))
for idx, line in enumerate(lines)
if (m := ENTRY_RE.match(line))
]
for i, (start, sig) in enumerate(starts):
end = starts[i + 1][0] if i + 1 < len(starts) else len(lines)
yield start, end, sig
def scan_body(lines, start, end):
"""Return [(lineno, raw, in_fence)] for the entry body (excludes the
signature line). Fence delimiter lines are marked in_fence so callers
skip both the fences and their contents."""
body = []
in_fence = False
for idx in range(start + 1, end):
raw = lines[idx]
if FENCE_RE.match(raw.strip()):
body.append((idx, raw, True))
in_fence = not in_fence
continue
body.append((idx, raw, in_fence))
return body
def has_params(sig):
"""True if the signature's parentheses hold anything (`...` counts)."""
left = sig.find("(")
right = sig.rfind(")")
if left == -1 or right == -1 or right < left:
return False
return bool(sig[left + 1:right].strip())
def split_row(text):
"""Split a markdown table row into trimmed cells, honoring `\\|` escapes."""
parts = CELL_SPLIT_RE.split(text.strip())
if parts and parts[0].strip() == "":
parts = parts[1:]
if parts and parts[-1].strip() == "":
parts = parts[:-1]
return [p.strip() for p in parts]
def is_separator_row(cells):
return bool(cells) and all(SEP_CELL_RE.match(c) for c in cells)
def find_table(body):
"""Return (header_lineno, header_cells, [(lineno, cells)] data_rows) for the
first pipe table in the body, or None. Skips fenced content."""
collected = []
for lineno, raw, in_fence in body:
if in_fence:
continue
stripped = raw.strip()
if stripped.startswith("|"):
collected.append((lineno, stripped))
elif collected:
break # blank/prose line ends the table
if not collected:
return None
header_lineno, header_text = collected[0]
header_cells = split_row(header_text)
data = []
for lineno, text in collected[1:]:
cells = split_row(text)
if is_separator_row(cells):
continue
data.append((lineno, cells))
return header_lineno, header_cells, data
def find_description(body):
"""Return (found, lineno_of_offending_line). found is True when the first
content line after the signature is prose. When False the lineno points at
the table/heading/return line that showed up where a description belongs
(or None if the entry is empty)."""
for lineno, raw, in_fence in body:
stripped = raw.strip()
if not stripped or in_fence:
continue
if stripped.startswith("<!--"): # tags or other comment: skip
continue
if stripped.startswith("|") or stripped.startswith("#") \
or RETURN_RE.match(stripped):
return False, lineno
return True, lineno
return False, None
def check_entry(md_display, lines, start, end, sig, findings):
entry_line = start + 1 # 1-based signature line, used for entry-level errors
body = scan_body(lines, start, end)
# description ----------------------------------------------------------
found, off_lineno = find_description(body)
if not found:
line = (off_lineno + 1) if off_lineno is not None else entry_line
findings.append((md_display, line, "error", "description",
f"`{sig}` 缺少描述（签名后第一行须为非空描述）"))
# return ---------------------------------------------------------------
has_return = any(
RETURN_RE.match(raw.strip())
for _, raw, in_fence in body if not in_fence
)
if not has_return:
findings.append((md_display, entry_line, "error", "return",
f"`{sig}` 缺少 `返回：类型` 行"))
# parameter table ------------------------------------------------------
table = find_table(body)
wants_params = has_params(sig)
if wants_params and table is None:
findings.append((md_display, entry_line, "error", "param-table",
f"`{sig}` 有参数但缺少参数表"))
elif not wants_params and table is not None:
header_lineno = table[0]
findings.append((md_display, header_lineno + 1, "error", "param-table",
f"`{sig}` 无参数却存在参数表"))
elif table is not None:
header_lineno, header_cells, data_rows = table
if header_cells != PARAM_HEADER:
findings.append((md_display, header_lineno + 1, "error",
"param-header",
f"参数表表头须为 {' / '.join(PARAM_HEADER)}，"
f"实为 {' / '.join(header_cells) or '(空)'}"))
# soft: optional-parameter wording
for lineno, cells in data_rows:
if len(cells) < 3:
continue
desc = cells[2]
if OPTIONAL_HINT_RE.search(desc) and not desc.startswith("可选。"):
findings.append((md_display, lineno + 1, "warning", "optional",
"可选参数说明建议以 `可选。` 开头"))
# soft: tags line ------------------------------------------------------
for lineno, raw, in_fence in body:
if in_fence:
continue
m = TAGS_RE.match(raw.strip())
if m and not m.group(1).split():
findings.append((md_display, lineno + 1, "warning", "tags",
"空的 tags 行；填入关键词或删除"))
def lint_file(md, root, findings):
try:
display = md.relative_to(root).as_posix()
except ValueError:
display = str(md)
lines = md.read_text(encoding="utf-8").splitlines()
for start, end, sig in iter_entries(lines):
check_entry(display, lines, start, end, sig, findings)
def gather_targets(paths, root):
"""Expand paths (files/dirs) into a sorted list of *.md."""
targets = []
for p in paths:
if p.is_dir():
targets.extend(p.rglob("*.md"))
elif p.is_file() and p.suffix == ".md":
targets.append(p)
return sorted(set(targets))
def main(argv=None):
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
parser = argparse.ArgumentParser(description="校验 Markdown 文件或目录")
target_group = parser.add_mutually_exclusive_group(required=True)
target_group.add_argument("--file", help="要校验的单个 Markdown 文件")
target_group.add_argument("--dir", help="要递归校验的目录")
parser.add_argument("--strict", action="store_true",
help="treat warnings as failures")
args = parser.parse_args(argv)
target = Path(args.file or args.dir)
if not target.exists():
print(f"ERROR: target not found: {target}", file=sys.stderr)
return 2
if args.file and (not target.is_file() or target.suffix.lower() != ".md"):
print(f"ERROR: --file requires a Markdown file: {target}", file=sys.stderr)
return 2
if args.dir and not target.is_dir():
print(f"ERROR: --dir requires a directory: {target}", file=sys.stderr)
return 2
root = target if target.is_dir() else target.parent
targets = gather_targets([target], root)
if not targets:
print("no markdown targets found", file=sys.stderr)
return 2
findings = []
for md in targets:
lint_file(md, root, findings)
findings.sort(key=lambda f: (f[0], f[1], 0 if f[2] == "error" else 1))
for display, line, level, rule, message in findings:
print(f"{display}:{line}: {level}: [{rule}] {message}")
errors = sum(1 for f in findings if f[2] == "error")
warnings = sum(1 for f in findings if f[2] == "warning")
print(
f"\n{len(targets)} files, {errors} error(s), {warnings} warning(s)",
file=sys.stderr,
)
if errors or (args.strict and warnings):
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())