diff --git a/tools/tsl-codegen/README.md b/tools/tsl-codegen/README.md index edd11454..cce92288 100644 --- a/tools/tsl-codegen/README.md +++ b/tools/tsl-codegen/README.md @@ -277,7 +277,19 @@ python tools/tsl-codegen/scripts/generate.py --dir tmp/api-recordings 生成器读取录入文件中的 `path`，默认写入 `skills/tsl-api-reference/references/codegen/project/.md`。不指定 -`--scope` 时，scope 就是 `project` +`--scope` 时，scope 就是 `project`。该路径默认相对于当前工作目录 + +从其他目录运行生成器时，使用 `--root` 指定包含 `skills/` 的项目根目录。例如在 +`tools/tsl-codegen` 目录中运行： + +```bash +python scripts/generate.py --file ../../tmp/my-api.json --root ../.. +``` + +此时仍会写入仓库根目录下的 +`skills/tsl-api-reference/references/codegen/project/.md`，不会在 +`tools/tsl-codegen` 下创建新的 `skills/` 目录。`--root` 接受绝对路径；相对路径按 +运行命令时的当前工作目录解析 写入前会自动使用仓库的 `.prettierrc.json` 格式化 markdown，使新页面与现有 builtin 页面保持一致。未安装 Prettier 或格式化失败时，生成器会停止且不写目标文件 diff --git a/tools/tsl-codegen/scripts/enrich_metadata.py b/tools/tsl-codegen/scripts/enrich_metadata.py deleted file mode 100644 index a4f97ad4..00000000 --- a/tools/tsl-codegen/scripts/enrich_metadata.py +++ /dev/null @@ -1,905 +0,0 @@ -#!/usr/bin/env python3 -"""Conservatively enrich TSL API descriptions and search tags. - -Markdown is the fact source. This tool never changes API headings, signatures, -parameter tables, return types, examples, or scope placement. It only updates -the first prose description line and the optional ```` line. -""" - -from __future__ import annotations - -import argparse -import csv -import html -import re -import sys -import unicodedata -from collections import defaultdict -from dataclasses import dataclass, field, replace -from pathlib import Path - -from bs4 import BeautifulSoup - - -SCRIPT_DIR = Path(__file__).resolve().parent -if str(SCRIPT_DIR) not in sys.path: - sys.path.insert(0, str(SCRIPT_DIR)) - -from api_markdown import DECLARATION_LINE_RE, iter_api_entries - - -TAGS_RE = re.compile(r"^$") -ACCESS_RE = re.compile(r"^访问[:：]\s*(.*?)\s*$", re.IGNORECASE) -H1_RE = re.compile(r"^#(?!#)\s+(.+?)\s*$") -TERMINAL_PUNCTUATION_RE = re.compile(r"[。．.!！?？；;，,：:]+$") -MULTISPACE_RE = re.compile(r"\s+") -FORMULA_RE = re.compile( - r"^([^。；;，,]{2,40}?)\s*(?=!])=(?!=)\s*(.+)$" -) -COMPARISON_RE = re.compile(r"<=|>=|==|!=|<|>") -RELATED_FUNCTION_RE = re.compile( - r"^([A-Za-z][A-Za-z0-9_]*)\s*相关函数$" -) -ACRONYM_RE = re.compile(r"(? None: - self.entries += other.entries - self.source_matches += other.source_matches - self.ambiguous_sources += other.ambiguous_sources - self.changed_descriptions += other.changed_descriptions - self.changed_tags += other.changed_tags - self.changes.extend(other.changes) - - -def normalize_key(value: str) -> str: - return unicodedata.normalize("NFKC", value).casefold().strip() - - -def clean_text(value: str) -> str: - value = html.unescape(value).replace("\xa0", " ") - value = value.replace("“", '"').replace("”", '"') - value = value.replace("‘", "'").replace("’", "'") - return MULTISPACE_RE.sub(" ", value).strip() - - -def strip_terminal_punctuation(value: str) -> str: - return TERMINAL_PUNCTUATION_RE.sub("", clean_text(value)).rstrip() - - -def safe_source_description(value: str) -> str: - value = strip_terminal_punctuation(value) - if not value or PROHIBITED_PROCESS_RE.search(value): - return "" - return value - - -def _extract_source_description(page: Path) -> str: - try: - soup = BeautifulSoup(page.read_text(encoding="utf-8"), "html.parser") - except (OSError, UnicodeError): - return "" - root = soup.select_one("#help_content") or soup - for expected in ("简述", "说明"): - for label in root.select(".DescriteTitle"): - if clean_text(label.get_text(" ", strip=True)) != expected: - continue - marker = label.parent if label.parent is not None else label - sibling = marker.find_next_sibling() - if sibling is None: - sibling = label.find_next_sibling() - if sibling is not None: - return safe_source_description(sibling.get_text(" ", strip=True)) - return "" - - -class SourceCatalog: - PREFERENCES = { - "builtin": ("tsl_base", "net_function", "knowledge_base"), - "third": ("net_function", "tsl_base", "knowledge_base"), - "deprecated": ("net_function", "tsl_base", "knowledge_base"), - "dotnet": ("net_function", "tsl_base", "knowledge_base"), - } - - def __init__(self, records: dict[str, list[SourceRecord]] | None = None): - self.records = records or {} - self._description_cache: dict[Path, str] = {} - - @classmethod - def empty(cls) -> "SourceCatalog": - return cls({}) - - @classmethod - def from_docs_root(cls, docs_root: Path) -> "SourceCatalog": - records: dict[str, list[SourceRecord]] = defaultdict(list) - for corpus in ("net_function", "tsl_base", "knowledge_base"): - root = docs_root / corpus - manifest = root / "manifest.tsv" - if not manifest.is_file(): - continue - with manifest.open(encoding="utf-8", newline="") as handle: - for row in csv.DictReader(handle, delimiter="\t"): - if row.get("status") != "ok" or not row.get("title"): - continue - record = SourceRecord( - corpus=corpus, - page_id=row.get("id", ""), - title=row["title"], - page=root / row.get("path", ""), - ) - records[normalize_key(record.title)].append(record) - return cls(dict(records)) - - def _load_description(self, record: SourceRecord) -> SourceRecord: - if record.description: - return record - if record.page not in self._description_cache: - self._description_cache[record.page] = _extract_source_description( - record.page - ) - return replace( - record, description=self._description_cache[record.page] - ) - - def match(self, entry: EntryContext) -> list[SourceRecord]: - matches = self.records.get(normalize_key(entry.name), []) - preference = self.PREFERENCES.get( - entry.scope, - ("net_function", "tsl_base", "knowledge_base"), - ) - rank = {name: index for index, name in enumerate(preference)} - - def page_order(record: SourceRecord): - try: - page_id = (0, int(record.page_id)) - except ValueError: - page_id = (1, record.page_id) - return rank.get(record.corpus, 99), page_id - - return sorted(matches, key=page_order) - - def describe(self, records: list[SourceRecord]) -> list[SourceRecord]: - return [self._load_description(record) for record in records] - - -def _starts_with_verb(value: str) -> bool: - return value.startswith(LEADING_VERBS) or bool( - ACTION_CLAUSE_RE.match(value) - ) - - -def _choose_source_description( - current: str, sources: list[SourceRecord] -) -> str: - candidates = [ - safe_source_description(record.description) for record in sources - ] - candidates = [value for value in candidates if value] - if not candidates: - return current - if len(sources) == 1 and current in GENERIC_SUMMARIES: - candidate = candidates[0] - if candidate not in GENERIC_SUMMARIES and len(candidate) > len(current): - return candidate - if PROHIBITED_PROCESS_RE.search(current): - return candidates[0] - return current - - -def _property_description(value: str, access: str) -> str: - if _starts_with_verb(value): - return value - if ( - len(value) > 30 - or any(mark in value for mark in "。；，") - or value.startswith(("功能同", "只读", "用于")) - ): - return value - normalized = normalize_key(access).replace(" ", "") - if value.startswith("是否"): - return ( - f"控制{value}" - if "write" in normalized - else f"指示{value}" - ) - if "read" in normalized and "write" in normalized: - return f"获取或设置{value}" - if "write" in normalized: - return f"设置{value}" - return f"获取{value}" - - -def _name_intent(name: str) -> str: - key = normalize_key(name) - rules = ( - (("is", "if", "has", "can", "check", "valid"), "判断"), - (("get", "find", "query", "search"), "获取"), - (("read", "load"), "读取"), - (("write", "save", "export"), "写入"), - (("set",), "设置"), - (("create", "make", "new"), "创建"), - (("delete", "remove", "drop"), "删除"), - (("clear", "reset"), "清除"), - (("add", "append", "insert"), "添加"), - (("parse", "decode"), "解析"), - (("encode", "convert", "to"), "转换"), - (("format",), "格式化"), - (("calc", "compute"), "计算"), - (("open",), "打开"), - (("close",), "关闭"), - (("send", "post"), "发送"), - (("connect", "login"), "连接"), - (("list",), "列出"), - ) - for prefixes, intent in rules: - if key.startswith(prefixes): - return intent - return "" - - -def improve_description( - entry: EntryContext, sources: list[SourceRecord] -) -> str: - current = strip_terminal_punctuation(entry.summary) - current = _choose_source_description(current, sources) - current = strip_terminal_punctuation(current) - if not current: - return entry.summary - - if entry.kind == "class": - replacement = CLASS_DESCRIPTIONS.get(normalize_key(entry.name)) - if replacement and re.fullmatch( - rf"{re.escape(entry.name)}\s*内置对象", current, re.IGNORECASE - ): - return replacement - - if entry.kind == "property": - return _property_description(current, entry.access) - - key = normalize_key(entry.name) - generic_override = GENERIC_DESCRIPTION_OVERRIDES.get(key) - if generic_override and current.endswith("相关函数"): - return generic_override - awkward_override = AWKWARD_DESCRIPTION_OVERRIDES.get(key) - if awkward_override and current.startswith( - ("返回根据", "返回将", "返回:", "返回：") - ): - return awkward_override - - current = re.sub( - r"^(?:该|本)函数(?:主要(?:是)?)?\s*", "", current - ) - current = re.sub(r"^功能[:：]\s*", "", current) - if current.startswith("是否"): - current = f"判断{current}" - - for old, new in ( - ("读出", "读取"), - ("取出", "获取"), - ("得到", "获取"), - ("获得", "获取"), - ("取得", "获取"), - ("新建", "创建"), - ): - if current.startswith(old): - current = new + current[len(old):] - break - if current.startswith("读") and not current.startswith("读取"): - current = "读取" + current[1:] - if current.startswith("写") and not current.startswith("写入"): - current = "写入" + current[1:] - current = current.replace("读出", "读取") - current = current.replace("装载内容", "加载内容") - - related_function = RELATED_FUNCTION_RE.fullmatch(current) - if related_function: - return f"返回 {related_function.group(1)} 对应的数据" - - formula = FORMULA_RE.match(current) - page_key = normalize_key(entry.page) - if ( - formula - and ("/financial/" in page_key or "/financial_report/" in page_key) - and not _starts_with_verb(current) - ): - left, right = formula.groups() - return f"计算{left.strip()}，公式为{right.strip()}" - - if _starts_with_verb(current): - return current - - if "/financial/" in page_key or "/financial_report/" in page_key: - if ( - re.match(r"^\d+(?:\s|[.．、)])", current) - or COMPARISON_RE.search(current) - ): - return current - return f"返回{current}" - - intent = _name_intent(entry.name) - if intent and len(current) <= 28 and not any( - verb in current[:16] for verb in LEADING_VERBS - ): - return f"{intent}{current}" - return current - - -DOMAIN_RULES = ( - (r"数组|列表|(?:^|[^a-z])(?:array|fmarray|list)(?:$|[^a-z])", ("数组", "列表", "array", "list")), - (r"矩阵|(?:^|[^a-z])matrix(?:$|[^a-z])|\bmt_", ("矩阵", "matrix")), - (r"字符串|文本|(?:^|[^a-z])(?:string|char|text)(?:$|[^a-z])", ("字符串", "文本", "string", "text")), - (r"日期|时间|(?:^|[^a-z])(?:datetime|date|time)(?:$|[^a-z])", ("日期时间", "日期", "时间", "datetime")), - (r"文件|目录|路径|(?:^|[^a-z])(?:file|folder|directory|path)(?:$|[^a-z])", ("文件", "目录", "路径", "file")), - (r"网络|(?:^|[^a-z])(?:http|https|cgi|cookie|url)(?:$|[^a-z])", ("网络", "HTTP", "请求", "network")), - (r"缓存|(?:^|[^a-z])cache(?:$|[^a-z])", ("缓存", "缓存管理", "cache")), - (r"数据库|(?:^|[^a-z])(?:sql|dbf|ini)(?:$|[^a-z])", ("数据库", "SQL", "database")), - (r"进程|线程|(?:^|[^a-z])(?:process|thread|pipe)(?:$|[^a-z])", ("进程", "线程", "process")), - (r"颜色|(?:^|[^a-z])(?:color|rgb|cmyk)(?:$|[^a-z])", ("颜色", "RGB", "color")), - (r"图形|图表|(?:^|[^a-z])(?:graph|chart)(?:$|[^a-z])", ("图形", "图表", "graph")), - (r"统计|概率|分布|(?:^|[^a-z])(?:statistics|cdf|pdf)(?:$|[^a-z])", ("统计", "概率", "statistics")), - (r"优化|线性规划|(?:^|[^a-z])optimization(?:$|[^a-z])", ("优化", "求解", "optimization")), - (r"数值|数学|(?:^|[^a-z])(?:numeric|math)(?:$|[^a-z])", ("数学", "数值", "numeric")), - (r"类型转换|(?:^|[^a-z])(?:convert|conversion)(?:$|[^a-z])", ("类型转换", "转换", "conversion")), - (r"对象|(?:^|[^a-z])(?:object|class)(?:$|[^a-z])", ("对象", "实例", "object")), - (r"(?:^|[^a-z])(?:com|ole)(?:$|[^a-z])|activex", ("COM", "OLE", "自动化")), - (r"(?:^|[^a-z])ftp(?:$|[^a-z])", ("FTP", "文件传输")), - (r"邮件|(?:^|[^a-z])(?:smtp|pop3|imap|mail)(?:$|[^a-z])", ("邮件", "SMTP", "POP3", "IMAP")), - (r"财务|金融|证券|股票|基金|债券|期货|期权", ("金融",)), - (r"报表|报告期|report", ("报表", "报告期")), -) - -SPECIAL_TAG_RULES = ( - (r"去重|删除重复|dropduplicate|dedup", ("去重", "删除重复", "deduplicate")), - (r"哈希索引|哈希表", ("哈希索引", "哈希表", "hash")), - ( - r"摘要|(?:^|[^a-z0-9_])(?:crc32|md5|sha1|sha224|sha256|sha384|sha512|sm3|digest|hash)(?:$|[^a-z0-9_])", - ("摘要", "哈希", "散列", "digest", "hash"), - ), - (r"排序|sort", ("排序", "sort")), - (r"过滤|筛选|filter", ("过滤", "筛选", "filter")), - (r"查找|搜索|find|search|lookup", ("查找", "搜索", "lookup")), -) - -INTENT_TAGS = { - "返回": ("返回", "获取"), - "获取": ("获取", "查询"), - "读取": ("读取", "获取"), - "写入": ("写入", "保存"), - "设置": ("设置", "修改"), - "创建": ("创建", "生成"), - "删除": ("删除", "移除"), - "清除": ("清除", "重置"), - "添加": ("添加", "追加"), - "查找": ("查找", "搜索"), - "判断": ("判断", "检查"), - "计算": ("计算", "求解"), - "转换": ("转换", "编码"), - "解析": ("解析", "parse"), - "格式化": ("格式化", "format"), - "打开": ("打开", "open"), - "关闭": ("关闭", "close"), - "发送": ("发送", "提交"), - "连接": ("连接", "登录"), - "列出": ("列出", "列表"), - "输出": ("输出", "打印"), -} - - -def _taxonomy_tags(title: str) -> list[str]: - result = [] - for raw in re.split(r"\s*/\s*|\s+-\s+", title): - value = raw.strip(" #") - value = re.sub(r"\([^)]*\)|（[^）]*）", "", value).strip() - for suffix in ("相关函数", "及其实现", "相关"): - if value.endswith(suffix): - value = value[: -len(suffix)].rstrip() - if not value or normalize_key(value) in GENERIC_TAGS: - continue - if len(value) > 18: - continue - chinese = "".join(re.findall(r"[\u3400-\u9fff]+", value)) - latin = re.sub(r"[\u3400-\u9fff]+", " ", value).strip() - if latin and normalize_key(latin) not in GENERIC_TAGS: - result.extend( - re.findall(r"[A-Za-z][A-Za-z0-9_+.-]*", latin) - ) - if chinese: - result.append(chinese) - return result - - -def _description_intent(summary: str) -> str: - value = strip_terminal_punctuation(summary) - replacements = { - "得到": "获取", - "获得": "获取", - "取得": "获取", - "新建": "创建", - } - for source, target in replacements.items(): - if value.startswith(source): - return target - for intent in INTENT_TAGS: - if value.startswith(intent): - return intent - for intent in ("读取", "写入", "保存", "计算", "判断", "转换", "解析"): - if intent in value[:16]: - return "写入" if intent == "保存" else intent - return "" - - -def derive_tags(entry: EntryContext, page: PageContext) -> list[str]: - tags: list[str] = [] - seen: set[str] = set() - - def add(value: str) -> None: - value = clean_text(value).strip() - key = normalize_key(value) - if ( - not value - or key in seen - or key in GENERIC_TAGS - or len(tags) >= 12 - ): - return - seen.add(key) - tags.append(value) - - for tag in entry.tags: - add(tag) - - for tag in _taxonomy_tags(page.title): - add(tag) - - haystack = " ".join( - (entry.name, entry.signature, entry.summary, page.title, page.path) - ) - domain_haystack = " ".join((page.title, page.path)) - for pattern, aliases in SPECIAL_TAG_RULES: - if re.search(pattern, haystack, re.IGNORECASE): - for alias in aliases: - add(alias) - - summary = strip_terminal_punctuation(entry.summary) - intent = _description_intent(summary) - if not intent and not _starts_with_verb(summary): - is_market_price = ( - normalize_key(entry.name) in {"open", "close"} - and "盘价" in summary - ) - if not is_market_price: - intent = _name_intent(entry.name) - for alias in INTENT_TAGS.get(intent, ()): - add(alias) - for pattern, aliases in DOMAIN_RULES: - if re.search(pattern, domain_haystack, re.IGNORECASE): - for alias in aliases: - add(alias) - for acronym in ACRONYM_RE.findall(entry.summary): - add(acronym) - return tags - - -def _scope_for_page(page: str) -> str: - parts = Path(page).parts - return parts[0] if len(parts) > 1 else Path(page).stem - - -def _page_title(lines: list[str], page: str) -> PageContext: - for line in lines: - match = H1_RE.match(line) - if match: - return PageContext(match.group(1), page) - return PageContext(Path(page).stem, page) - - -def _entry_metadata(lines: list[str], start: int, end: int): - declaration_seen = False - description_index = None - tag_index = None - access = "" - for index in range(start + 1, end): - text = lines[index].strip() - if not text: - continue - if not declaration_seen: - if DECLARATION_LINE_RE.fullmatch(text): - declaration_seen = True - continue - tag_match = TAGS_RE.fullmatch(text) - if tag_match: - tag_index = index - continue - access_match = ACCESS_RE.fullmatch(text) - if access_match: - access = access_match.group(1) - continue - if description_index is None: - if text.startswith(("|", "#", "返回：", "类型：", "可见性：", "值：")): - continue - description_index = index - tags = () - if tag_index is not None: - tags = tuple(TAGS_RE.fullmatch(lines[tag_index].strip()).group(1).split()) - return description_index, tag_index, access, tags - - -def enrich_markdown( - text: str, page: str, source_catalog: SourceCatalog -) -> tuple[str, Audit]: - had_final_newline = text.endswith("\n") - lines = text.splitlines() - page_context = _page_title(lines, page) - entries = [entry for entry in iter_api_entries(lines) if entry.heading.valid] - audit = Audit(entries=len(entries)) - root_owner = "" - owners: dict[int, str] = {} - for entry in entries: - if entry.heading.level == 2: - root_owner = entry.heading.name if entry.heading.kind in {"class", "unit"} else "" - owners[entry.start] = root_owner if entry.heading.level > 2 else "" - - for api in reversed(entries): - description_index, tag_index, access, existing_tags = _entry_metadata( - lines, api.start, api.end - ) - if description_index is None: - continue - old_description = lines[description_index].strip() - entry = EntryContext( - name=api.heading.name, - signature=api.heading.signature, - kind=api.heading.kind, - scope=_scope_for_page(page), - page=page, - summary=old_description, - access=access, - owner=owners.get(api.start, ""), - tags=existing_tags, - ) - sources = source_catalog.match(entry) - if sources: - audit.source_matches += 1 - if len(sources) > 1: - audit.ambiguous_sources += 1 - evidence = sources - normalized_summary = strip_terminal_punctuation(old_description) - if ( - normalized_summary in GENERIC_SUMMARIES - or PROHIBITED_PROCESS_RE.search(normalized_summary) - ): - evidence = source_catalog.describe(sources) - new_description = improve_description(entry, evidence) - entry_for_tags = replace(entry, summary=new_description) - new_tags = tuple(derive_tags(entry_for_tags, page_context)) - - if new_description != old_description: - lines[description_index] = new_description - audit.changed_descriptions += 1 - if new_tags != existing_tags: - rendered = f"" - if tag_index is not None: - lines[tag_index] = rendered - else: - insert_at = description_index + 1 - if insert_at < len(lines) and lines[insert_at].strip() == "": - insert_at += 1 - lines[insert_at:insert_at] = [rendered, ""] - else: - lines[insert_at:insert_at] = ["", rendered, ""] - audit.changed_tags += 1 - - if new_description != old_description or new_tags != existing_tags: - audit.changes.append( - AuditChange( - page=page, - name=api.heading.name, - kind=api.heading.kind, - old_description=old_description, - new_description=new_description, - old_tags=existing_tags, - new_tags=new_tags, - sources=tuple( - f"{record.corpus}:{record.page_id}" for record in sources - ), - ) - ) - - result = "\n".join(lines) - if had_final_newline: - result += "\n" - return result, audit - - -def _write_report(path: Path, audit: Audit) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.writer(handle, delimiter="\t", lineterminator="\n") - writer.writerow( - ( - "page", - "name", - "kind", - "old_description", - "new_description", - "old_tags", - "new_tags", - "sources", - ) - ) - for item in audit.changes: - writer.writerow( - ( - item.page, - item.name, - item.kind, - item.old_description, - item.new_description, - " ".join(item.old_tags), - " ".join(item.new_tags), - " ".join(item.sources), - ) - ) - - -def _selected(page: str, scopes: set[str], prefixes: tuple[str, ...]) -> bool: - if scopes and _scope_for_page(page) not in scopes: - return False - return not prefixes or page.startswith(prefixes) - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--audit", action="store_true", help="report changes only") - mode.add_argument("--apply", action="store_true", help="write proposed changes") - parser.add_argument("--root", type=Path, required=True, help="codegen Markdown root") - parser.add_argument( - "--docs-root", type=Path, required=True, help="cached tmp/docs root" - ) - parser.add_argument("--report", type=Path, help="optional TSV audit report") - parser.add_argument("--scope", action="append", default=[]) - parser.add_argument("--prefix", action="append", default=[]) - args = parser.parse_args(argv) - - catalog = SourceCatalog.from_docs_root(args.docs_root) - aggregate = Audit() - changed_files = 0 - for md in sorted(args.root.rglob("*.md")): - page = md.relative_to(args.root).as_posix() - if not _selected(page, set(args.scope), tuple(args.prefix)): - continue - before_stat = md.stat() - original = md.read_text(encoding="utf-8") - enriched, audit = enrich_markdown(original, page, catalog) - aggregate.merge(audit) - if enriched == original: - continue - changed_files += 1 - if args.apply: - after_stat = md.stat() - if ( - after_stat.st_mtime_ns != before_stat.st_mtime_ns - or after_stat.st_size != before_stat.st_size - ): - raise RuntimeError(f"target changed while processing: {md}") - md.write_text(enriched, encoding="utf-8", newline="\n") - - if args.report: - _write_report(args.report, aggregate) - print( - "entries={entries} source_matches={source_matches} " - "ambiguous_sources={ambiguous_sources} changed_files={changed_files} " - "changed_descriptions={changed_descriptions} changed_tags={changed_tags}".format( - entries=aggregate.entries, - source_matches=aggregate.source_matches, - ambiguous_sources=aggregate.ambiguous_sources, - changed_files=changed_files, - changed_descriptions=aggregate.changed_descriptions, - changed_tags=aggregate.changed_tags, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/tsl-codegen/scripts/generate.py b/tools/tsl-codegen/scripts/generate.py index a2980ffd..505602a0 100644 --- a/tools/tsl-codegen/scripts/generate.py +++ b/tools/tsl-codegen/scripts/generate.py @@ -21,7 +21,7 @@ Function declaration fields: tags optional list of Chinese keywords -> `` params required when the signature takes args; omit for nullary returns required return type - example optional fenced tsl block, pasted verbatim + examples optional list of structured examples with desc/code/output fields Each param: name/type/desc required; optional (bool) -> `可选。` prefix; values (list of {value, desc}) -> a `name 取值` enum section. @@ -29,6 +29,7 @@ Usage (run from repo root): python tools/tsl-codegen/scripts/generate.py --file entry.yml python tools/tsl-codegen/scripts/generate.py --file entry.json \ --scope my-project + python scripts/generate.py --file entry.json --root ../.. python tools/tsl-codegen/scripts/generate.py --dir recordings """ @@ -279,7 +280,6 @@ def validate_function(fn, where, *, returns_required, extra_fields=()): "tags", "params", "returns", - "example", "examples", *extra_fields, } @@ -292,12 +292,6 @@ def validate_function(fn, where, *, returns_required, extra_fields=()): non_empty_string(fn.get("returns"), f"{where}：缺少 returns") elif "returns" in fn: optional_draft_string(fn["returns"], f"{where}: returns") - require( - not ("example" in fn and "examples" in fn), - f"{where}：example 和 examples 不能同时存在", - ) - if "example" in fn: - non_empty_string(fn["example"], f"{where}: example") if "examples" in fn: validate_examples(fn["examples"], where) return name @@ -333,7 +327,6 @@ def validate_class_member(member, where): "params", "returns", "modifiers", - "example", "examples", }, where, @@ -600,14 +593,6 @@ def render_examples(fn, heading_level): while lines and not lines[-1]: lines.pop() return lines - if fn.get("example"): - return [ - f"{'#' * heading_level} 示例", - "", - "```tsl", - *fn["example"].rstrip("\n").split("\n"), - "```", - ] return [] @@ -822,7 +807,7 @@ def format_markdown(text): return result.stdout -def output_path(data, scope): +def output_path(data, scope, root=None): """Build the leaf-page destination from the recording file's relative path.""" relative = data.get("path") require(relative, "录入数据缺少 path") @@ -854,7 +839,8 @@ def output_path(data, scope): ) relative_path = Path(*recorded_path.parts) return ( - Path("skills/tsl-api-reference/references/codegen") + (Path(root) if root is not None else Path()) + / "skills/tsl-api-reference/references/codegen" / scope / relative_path.with_suffix(".md") ) @@ -902,14 +888,14 @@ def gather_directory_inputs(directory, fmt): return inputs -def prepare_input(in_path, input_format, scope): +def prepare_input(in_path, input_format, scope, root=None): data = load_entries(in_path, input_format) try: rendered = render_page(data) except GenerationError as exc: raise GenerationError(f"录入数据校验失败：{exc}") from exc try: - out_path = output_path(data, scope) + out_path = output_path(data, scope, root) except GenerationError as exc: raise GenerationError(f"输出路径无效：{exc}") from exc try: @@ -941,11 +927,14 @@ def find_output_collisions(prepared): def main(argv=None): if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") + if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8") parser = ChineseArgumentParser( description="从 YAML/JSON 录入文件生成 TSL API 文档", usage=( - "%(prog)s [--help] [--scope SCOPE] [--format {json,yaml}] " - "(--file INPUT_FILE | --dir INPUT_DIR | INPUT_FILE)" + "%(prog)s [--help] [--root ROOT_DIR] [--scope SCOPE] " + "[--format {json,yaml}] " + "(--file INPUT_FILE | --dir INPUT_DIR)" ), add_help=False, allow_abbrev=False, @@ -955,12 +944,6 @@ def main(argv=None): action="help", help="显示本帮助并退出（不提供 -h 短选项）", ) - parser.add_argument( - "legacy_input", - nargs="?", - metavar="INPUT_FILE", - help="已废弃，请使用 --file；暂时兼容 YAML/JSON 录入文件路径", - ) parser.add_argument( "--file", dest="input_file", @@ -973,6 +956,11 @@ def main(argv=None): metavar="INPUT_DIR", help="批量生成目录中的直属 YAML/JSON 录入文件", ) + parser.add_argument( + "--root", + metavar="ROOT_DIR", + help="包含 skills 目录的项目根目录（默认：当前工作目录）", + ) parser.add_argument( "--scope", type=scope_name, @@ -986,16 +974,16 @@ def main(argv=None): ) args = parser.parse_args(argv) - input_modes = (args.legacy_input, args.input_file, args.input_dir) + input_modes = (args.input_file, args.input_dir) if sum(value is not None for value in input_modes) != 1: - parser.error("必须且只能指定一种输入方式：INPUT_FILE、--file 或 --dir") + parser.error("必须且只能指定一种输入方式：--file 或 --dir") is_batch = args.input_dir is not None try: if is_batch: input_paths = gather_directory_inputs(Path(args.input_dir), args.format) input_format = None else: - in_path = Path(args.input_file or args.legacy_input) + in_path = Path(args.input_file) if not in_path.is_file(): die(f"输入文件不存在或不是普通文件：{in_path}") input_paths = [in_path] @@ -1008,7 +996,9 @@ def main(argv=None): input_errors = [] for in_path in input_paths: try: - prepared.append(prepare_input(in_path, input_format, args.scope)) + prepared.append( + prepare_input(in_path, input_format, args.scope, args.root) + ) except GenerationError as exc: input_errors.append((in_path, exc)) diff --git a/tools/tsl-codegen/tests/test_enrich_metadata.py b/tools/tsl-codegen/tests/test_enrich_metadata.py deleted file mode 100644 index 883111e4..00000000 --- a/tools/tsl-codegen/tests/test_enrich_metadata.py +++ /dev/null @@ -1,778 +0,0 @@ -import importlib.util -import sys -import tempfile -import unittest -from pathlib import Path - - -SCRIPT = Path(__file__).parents[1] / "scripts" / "enrich_metadata.py" - - -def load_script(): - spec = importlib.util.spec_from_file_location("tsl_enrich_metadata", SCRIPT) - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -class SourceCatalogTest(unittest.TestCase): - def setUp(self): - self.module = load_script() - self.temp_dir = tempfile.TemporaryDirectory() - self.docs = Path(self.temp_dir.name) - - def tearDown(self): - self.temp_dir.cleanup() - - def write_corpus(self, corpus, rows, descriptions): - root = self.docs / corpus - pages = root / "pages" - pages.mkdir(parents=True) - manifest = [ - "id\ttitle\turl\tpath\tsource_bytes\tutf8_bytes\tsha256\tstatus" - ] - for page_id, title in rows: - manifest.append( - f"{page_id}\t{title}\thttp://example/{page_id}\t" - f"pages/{page_id}.html\t1\t1\tsha\tok" - ) - description = descriptions[page_id] - (pages / f"{page_id}.html").write_text( - "

" - + title - + "

简述
" - + f"
{description}
", - encoding="utf-8", - ) - (root / "manifest.tsv").write_text( - "\n".join(manifest) + "\n", encoding="utf-8" - ) - - def test_exact_matching_preserves_underscores_and_prefers_scope_corpus(self): - self.write_corpus( - "tsl_base", - [("1", "alpha_test")], - {"1": "基础解释器说明"}, - ) - self.write_corpus( - "net_function", - [("2", "alpha_test"), ("3", "alphaTest")], - {"2": "NET 说明", "3": "无下划线的另一个函数"}, - ) - catalog = self.module.SourceCatalog.from_docs_root(self.docs) - - builtin = self.module.EntryContext( - name="alpha_test", - signature="alpha_test()", - kind="function", - scope="builtin", - page="builtin/demo.md", - summary="原说明", - ) - dotnet = self.module.EntryContext( - name="alpha_test", - signature="alpha_test()", - kind="function", - scope="dotnet", - page="dotnet/demo.md", - summary="原说明", - ) - - self.assertEqual("tsl_base", catalog.match(builtin)[0].corpus) - self.assertEqual("net_function", catalog.match(dotnet)[0].corpus) - self.assertNotIn( - "alphaTest", {record.title for record in catalog.match(dotnet)} - ) - - def test_duplicate_titles_remain_visible_for_audit(self): - self.write_corpus( - "net_function", - [("1", "same_name"), ("2", "same_name")], - {"1": "说明一", "2": "说明二"}, - ) - catalog = self.module.SourceCatalog.from_docs_root(self.docs) - entry = self.module.EntryContext( - name="same_name", - signature="same_name()", - kind="function", - scope="dotnet", - page="dotnet/demo.md", - summary="原说明", - ) - - self.assertEqual(["1", "2"], [item.page_id for item in catalog.match(entry)]) - - -class DescriptionQualityTest(unittest.TestCase): - def setUp(self): - self.module = load_script() - - def entry(self, **overrides): - values = { - "name": "profitRatio", - "signature": "profitRatio(report_date)", - "kind": "function", - "scope": "dotnet", - "page": "dotnet/financial/profitability.md", - "summary": "总资产收益率(%)。", - } - values.update(overrides) - return self.module.EntryContext(**values) - - def test_removes_only_terminal_sentence_punctuation(self): - entry = self.entry(summary="总资产收益率(%)。") - - self.assertEqual( - "返回总资产收益率(%)", - self.module.improve_description(entry, []), - ) - - def test_preserves_formula_and_identifiers(self): - entry = self.entry( - summary="总资产报酬率(%)=利润总额/平均资产总额*100。" - ) - - self.assertEqual( - "计算总资产报酬率(%)，公式为利润总额/平均资产总额*100", - self.module.improve_description(entry, []), - ) - - def test_rejects_source_process_prose(self): - entry = self.entry(summary="返回指定值") - source = self.module.SourceRecord( - corpus="net_function", - page_id="1", - title="profitRatio", - page=Path("1.html"), - description="Windows 已验证通过，Linux 返回 not found", - ) - - self.assertEqual( - "返回指定值", self.module.improve_description(entry, [source]) - ) - - def test_current_environment_can_be_real_api_semantics(self): - self.assertEqual( - "获取当前环境时间系统参数", - self.module.safe_source_description("获取当前环境时间系统参数。"), - ) - - def test_property_access_is_stated_explicitly(self): - entry = self.entry( - name="Host", - signature="Host", - kind="property", - scope="builtin", - page="builtin/object/ftp.md", - summary="远程服务器地址。", - access="read/write", - owner="FTP", - ) - - self.assertEqual( - "获取或设置远程服务器地址", - self.module.improve_description(entry, []), - ) - - def test_boolean_property_description_is_idempotent(self): - entry = self.entry( - name="UseTLS", - signature="UseTLS", - kind="property", - scope="builtin", - page="builtin/object/imap.md", - summary="是否采用 SSL 连接", - access="read/write", - owner="IMAP", - ) - - first = self.module.improve_description(entry, []) - repeated = self.module.improve_description( - self.module.EntryContext( - **{**entry.__dict__, "summary": first} - ), - [], - ) - - self.assertEqual("控制是否采用 SSL 连接", first) - self.assertEqual(first, repeated) - - def test_low_confidence_sentence_is_kept_except_terminal_punctuation(self): - entry = self.entry( - name="opaqueApi", - page="dotnet.md", - summary="根据调用上下文处理结果，具体规则取决于输入。", - ) - - self.assertEqual( - "根据调用上下文处理结果，具体规则取决于输入", - self.module.improve_description(entry, []), - ) - - def test_short_description_is_not_replaced_by_ambiguous_source_text(self): - entry = self.entry( - name="clRed", - page="builtin/color.md", - summary="红色", - ) - sources = [ - self.module.SourceRecord( - corpus="net_function", - page_id="1", - title="clRed", - page=Path("1.html"), - description="定义", - ) - ] - - self.assertEqual("红色", self.module.improve_description(entry, sources)) - - def test_existing_action_word_is_normalized_without_duplicate_verb(self): - getter = self.entry( - name="getCValue", - page="builtin/color.md", - summary="得到颜色的 CMYK 模式 C 值。", - ) - writer = self.entry( - name="write", - page="builtin/cgi.md", - summary="输出字符串。", - ) - - self.assertEqual( - "获取颜色的 CMYK 模式 C 值", - self.module.improve_description(getter, []), - ) - self.assertEqual("输出字符串", self.module.improve_description(writer, [])) - - def test_equation_inside_algorithm_sentence_is_not_rewritten_as_formula(self): - entry = self.entry( - name="se_Gauss", - page="builtin/optimization.md", - summary="用高斯消去法求解线性方程组 AX = B", - ) - - self.assertEqual( - "用高斯消去法求解线性方程组 AX = B", - self.module.improve_description(entry, []), - ) - - def test_builtin_class_gets_a_purpose_description(self): - entry = self.entry( - name="TStringList", - signature="TStringList", - kind="class", - scope="builtin", - page="builtin/object/tstringlist.md", - summary="TStringList 内置对象", - ) - - self.assertEqual( - "提供字符串集合存储、查找、排序和名称值管理能力的内置对象", - self.module.improve_description(entry, []), - ) - - def test_long_property_explanation_is_not_wrapped_in_access_boilerplate(self): - entry = self.entry( - name="CommaTextW", - signature="CommaTextW", - kind="property", - scope="builtin", - page="builtin/object/tstringlist.md", - summary=( - "功能同 CommaText，区别是在读取时返回宽字节字符串，" - "而 CommaText 返回多字节字符串" - ), - access="read/write", - owner="TStringList", - ) - - self.assertEqual( - entry.summary, self.module.improve_description(entry, []) - ) - - def test_short_read_and_load_phrases_are_normalized(self): - read_entry = self.entry( - name="read", - page="builtin/object/tstream.md", - summary="读出内容", - ) - load_entry = self.entry( - name="loadFromFile", - page="builtin/object/tstringlist.md", - summary="从指定的文件中装载内容", - ) - - self.assertEqual( - "读取内容", self.module.improve_description(read_entry, []) - ) - self.assertEqual( - "从指定的文件中加载内容", - self.module.improve_description(load_entry, []), - ) - - def test_embedded_read_out_is_normalized_without_duplicate_prefix(self): - entry = self.entry( - name="readExcelSheets", - page="dotnet.md", - summary="从Excel文件中读出Sheets列表。", - ) - - self.assertEqual( - "从Excel文件中读取Sheets列表", - self.module.improve_description(entry, []), - ) - - def test_existing_decomposition_verb_is_not_prefixed(self): - entry = self.entry( - name="decodeGraphGroup", - page="builtin/graph.md", - summary="分解图形组合并写入输出参数", - ) - - self.assertEqual( - entry.summary, self.module.improve_description(entry, []) - ) - - def test_comparison_expression_is_not_rewritten_as_formula(self): - entry = self.entry( - name="stockStepAmount", - page="dotnet/financial/stock-capital-flow.md", - summary="分档区间 V1<=Value&2\nexit 9\n", - encoding="utf-8", - ) - fake_npx.chmod(0o755) + if os.name == "nt": + fake_npx = bin_dir / "npx.cmd" + fake_npx.write_text( + "@echo off\r\necho formatter-failed 1>&2\r\nexit /b 9\r\n", + encoding="utf-8", + ) + else: + fake_npx = bin_dir / "npx" + fake_npx.write_text( + "#!/bin/sh\necho formatter-failed >&2\nexit 9\n", + encoding="utf-8", + ) + fake_npx.chmod(0o755) env = os.environ.copy() env["PATH"] = str(bin_dir) @@ -884,6 +935,46 @@ class DocGenCliTest(unittest.TestCase): self.assertIn("范例02：多行输出", text) self.assertIn("// 输出：\n// first\n// second", text) + def test_singular_example_is_rejected_without_overwrite(self): + self.write_input( + { + "module": "项目 / 旧示例格式", + "path": "base/singular_example", + "declarations": [ + { + "kind": "function", + "name": "demo", + "signature": "demo()", + "desc": "示例函数。", + "returns": "nil", + "example": "return demo();", + } + ], + } + ) + + self.assert_rejected_without_overwrite( + "base/singular_example", "存在未知字段：example" + ) + + def test_singular_method_example_is_rejected_without_overwrite(self): + self.write_class_member_input( + { + "kind": "method", + "name": "Run", + "visibility": "public", + "binding": "instance", + "signature": "Run()", + "desc": "运行。", + "example": "return self.Run();", + }, + "base/singular_method_example", + ) + + self.assert_rejected_without_overwrite( + "base/singular_method_example", "存在未知字段：example" + ) + def test_top_level_enum_sections_are_plain_text_not_emphasis_headings(self): self.write_input( {