ci[bot]
@@ -277,7 +277,19 @@ python tools/tsl-codegen/scripts/generate.py --dir tmp/api-recordings
|
||||
|
||||
生成器读取录入文件中的 `path`，默认写入
|
||||
`skills/tsl-api-reference/references/codegen/project/<path>.md`。不指定
|
||||
`--scope` 时，scope 就是 `project`
|
||||
`--scope` 时，scope 就是 `project`。该路径默认相对于当前工作目录
|
||||
|
||||
从其他目录运行生成器时，使用 `--root` 指定包含 `skills/` 的项目根目录。例如在
|
||||
`tools/tsl-codegen` 目录中运行：
|
||||
|
||||
```bash
|
||||
python scripts/generate.py --file ../../tmp/my-api.json --root ../..
|
||||
```
|
||||
|
||||
此时仍会写入仓库根目录下的
|
||||
`skills/tsl-api-reference/references/codegen/project/<path>.md`，不会在
|
||||
`tools/tsl-codegen` 下创建新的 `skills/` 目录。`--root` 接受绝对路径；相对路径按
|
||||
运行命令时的当前工作目录解析
|
||||
|
||||
写入前会自动使用仓库的 `.prettierrc.json` 格式化 markdown，使新页面与现有
|
||||
builtin 页面保持一致。未安装 Prettier 或格式化失败时，生成器会停止且不写目标文件
|
||||
|
||||
@@ -1,905 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Conservatively enrich TSL API descriptions and search tags.
|
||||
|
||||
Markdown is the fact source. This tool never changes API headings, signatures,
|
||||
parameter tables, return types, examples, or scope placement. It only updates
|
||||
the first prose description line and the optional ``<!-- tags: ... -->`` line.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import html
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from api_markdown import DECLARATION_LINE_RE, iter_api_entries
|
||||
|
||||
|
||||
TAGS_RE = re.compile(r"^<!--\s*tags:\s*(.*?)\s*-->$")
|
||||
ACCESS_RE = re.compile(r"^访问[:：]\s*(.*?)\s*$", re.IGNORECASE)
|
||||
H1_RE = re.compile(r"^#(?!#)\s+(.+?)\s*$")
|
||||
TERMINAL_PUNCTUATION_RE = re.compile(r"[。．.!！?？；;，,：:]+$")
|
||||
MULTISPACE_RE = re.compile(r"\s+")
|
||||
FORMULA_RE = re.compile(
|
||||
r"^([^。；;，,]{2,40}?)\s*(?<![<>=!])=(?!=)\s*(.+)$"
|
||||
)
|
||||
COMPARISON_RE = re.compile(r"<=|>=|==|!=|<|>")
|
||||
RELATED_FUNCTION_RE = re.compile(
|
||||
r"^([A-Za-z][A-Za-z0-9_]*)\s*相关函数$"
|
||||
)
|
||||
ACRONYM_RE = re.compile(r"(?<![A-Za-z0-9_])([A-Z][A-Z0-9_]{1,15})(?![A-Za-z0-9_])")
|
||||
ACTION_CLAUSE_RE = re.compile(
|
||||
r"^(?:用于|如果|由于|若(?!干)|支持|查询|输入|按照|按(?!钮)|与系统参数|"
|
||||
r"对.+?(?:进行|使用)|从.+?(?:获取|读取|计算|返回)|"
|
||||
r"由.+?(?:倒推|获取|计算|得到|生成))"
|
||||
)
|
||||
|
||||
PROHIBITED_PROCESS_RE = re.compile(
|
||||
r"Source\s*ID|来源[:：]|已验证|验证通过|not[_ ]?found|"
|
||||
r"测试过程|探测结果|官网定义",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
GENERIC_SUMMARIES = {
|
||||
"函数",
|
||||
"方法",
|
||||
"属性",
|
||||
"常量",
|
||||
"接口",
|
||||
"对象",
|
||||
"类",
|
||||
"内置函数",
|
||||
"内部函数",
|
||||
"统计分布相关函数",
|
||||
}
|
||||
GENERIC_TAGS = {
|
||||
"函数",
|
||||
"方法",
|
||||
"属性",
|
||||
"常量",
|
||||
"数据",
|
||||
"tsl",
|
||||
"builtin",
|
||||
"dotnet",
|
||||
"third",
|
||||
"deprecated",
|
||||
"function",
|
||||
"method",
|
||||
"property",
|
||||
}
|
||||
CLASS_DESCRIPTIONS = {
|
||||
"imap": "用于连接 IMAP 服务器、管理邮箱并读取邮件的内置对象",
|
||||
"mailmsg": "用于创建、解析、保存和发送邮件消息的内置对象",
|
||||
"messagepart": "表示邮件正文、附件等消息分段的内置对象",
|
||||
"pop3": "用于连接 POP3 服务器并读取或删除邮件的内置对象",
|
||||
"smtp": "用于连接 SMTP 服务器并发送邮件的内置对象",
|
||||
"tcipher": "提供对称加密和解密能力的内置对象",
|
||||
"tcookie": "表示 HTTP Cookie 名称、值和属性的内置对象",
|
||||
"tcookiecollection": "用于创建、查找和管理 HTTP Cookie 集合的内置对象",
|
||||
"tfilestream": "以文件为存储后端并提供流式读写能力的内置对象",
|
||||
"thandlestream": "封装系统句柄并提供流式读写能力的内置对象",
|
||||
"thashedstringlist": "使用哈希索引加速查找的字符串列表内置对象",
|
||||
"tinifile": "用于读取和写入 INI 配置文件的内置对象",
|
||||
"tmeminifile": "在内存中读取、修改和保存 INI 配置的内置对象",
|
||||
"tmemorystream": "以内存缓冲区保存和操作流数据的内置对象",
|
||||
"tregistryinifile": "以系统注册表为存储后端的 INI 配置内置对象",
|
||||
"trsa": "提供 RSA 非对称加密、解密和签名能力的内置对象",
|
||||
"tsession": "表示服务端会话及其键值数据的内置对象",
|
||||
"tsessionman": "用于创建、查找、维护和删除服务端会话的内置对象",
|
||||
"tstream": "定义顺序读写、定位和复制操作的流基类",
|
||||
"tstringlist": "提供字符串集合存储、查找、排序和名称值管理能力的内置对象",
|
||||
"twebrequest": "封装 HTTP 请求头、请求参数、Cookie 和请求正文的内置对象",
|
||||
"twebresponse": "用于设置 HTTP 状态、响应头、Cookie 和响应正文的内置对象",
|
||||
}
|
||||
GENERIC_DESCRIPTION_OVERRIDES = {
|
||||
"unicodeesc2": "将字符串转换为 \\uxxxx 形式的 Unicode 编码串",
|
||||
"sf_x2inv": "根据累计概率计算卡方分布的逆函数值",
|
||||
"sf_tinv": "根据累计概率计算 t 分布的逆函数值",
|
||||
"sf_sinx": "计算正弦积分",
|
||||
"sf_normalinv2": "根据累计概率计算标准正态分布的逆函数值",
|
||||
"sf_normalinv": "根据累计概率计算正态分布的逆函数值",
|
||||
"sf_normal": "计算正态分布函数值",
|
||||
"sf_gammainv": "根据累计概率计算标准 Gamma 分布的逆函数值",
|
||||
"sf_gamma": "计算伽马函数值",
|
||||
"sf_finv": "根据累计概率计算 F 分布的逆函数值",
|
||||
"sf_ex": "计算指数积分",
|
||||
"sf_error": "计算误差函数（概率函数）值",
|
||||
"sf_cosx": "计算余弦积分",
|
||||
"sf_betainv": "根据累计概率计算贝塔分布的逆函数值",
|
||||
"sf_beta": "计算贝塔函数值",
|
||||
"readfile": "读取本地文件中的数据",
|
||||
"millisecondspan": "计算两个日期时间之间相差的毫秒数",
|
||||
"importfile": "从文件中导入数据",
|
||||
"getmsgdigest": "计算字符串或文件的摘要并返回摘要字符串",
|
||||
"exportfile": "将数据导出到文件",
|
||||
}
|
||||
AWKWARD_DESCRIPTION_OVERRIDES = {
|
||||
"encodegraph": "根据指定的类型、名称、数据和属性数组生成图形并返回",
|
||||
"slndepreciation": (
|
||||
"按直线折旧法计算每期折旧额，计算公式为"
|
||||
"（资产原值 Cost - 资产残值 Salvage）/ 折旧周期数 Life"
|
||||
),
|
||||
"format": "根据 formatstr 指定的格式将参数连接成字符串并返回",
|
||||
"ansiformat": "根据 formatstr 指定的格式将参数连接成字符串并返回",
|
||||
"dupestring": "将字符串 AText 重复 ACount 次并返回结果",
|
||||
"stockpjcj": "返回区间平均成交价，计算公式为区间成交金额 / 区间成交量",
|
||||
}
|
||||
LEADING_VERBS = (
|
||||
"返回",
|
||||
"获取",
|
||||
"读取",
|
||||
"写入",
|
||||
"设置",
|
||||
"创建",
|
||||
"删除",
|
||||
"移除",
|
||||
"清除",
|
||||
"添加",
|
||||
"插入",
|
||||
"查找",
|
||||
"搜索",
|
||||
"判断",
|
||||
"检查",
|
||||
"验证",
|
||||
"计算",
|
||||
"转换",
|
||||
"解析",
|
||||
"格式化",
|
||||
"比较",
|
||||
"执行",
|
||||
"调用",
|
||||
"连接",
|
||||
"关闭",
|
||||
"打开",
|
||||
"发送",
|
||||
"接收",
|
||||
"保存",
|
||||
"加载",
|
||||
"列出",
|
||||
"表示",
|
||||
"提供",
|
||||
"释放",
|
||||
"交换",
|
||||
"排序",
|
||||
"过滤",
|
||||
"统计",
|
||||
"控制",
|
||||
"指示",
|
||||
"提取",
|
||||
"根据",
|
||||
"将",
|
||||
"把",
|
||||
"生成",
|
||||
"构造",
|
||||
"采用",
|
||||
"利用",
|
||||
"输出",
|
||||
"得到",
|
||||
"获得",
|
||||
"取得",
|
||||
"新建",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceRecord:
|
||||
corpus: str
|
||||
page_id: str
|
||||
title: str
|
||||
page: Path
|
||||
description: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntryContext:
|
||||
name: str
|
||||
signature: str
|
||||
kind: str
|
||||
scope: str
|
||||
page: str
|
||||
summary: str
|
||||
access: str = ""
|
||||
owner: str = ""
|
||||
tags: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PageContext:
|
||||
title: str
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuditChange:
|
||||
page: str
|
||||
name: str
|
||||
kind: str
|
||||
old_description: str
|
||||
new_description: str
|
||||
old_tags: tuple[str, ...]
|
||||
new_tags: tuple[str, ...]
|
||||
sources: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Audit:
|
||||
entries: int = 0
|
||||
source_matches: int = 0
|
||||
ambiguous_sources: int = 0
|
||||
changed_descriptions: int = 0
|
||||
changed_tags: int = 0
|
||||
changes: list[AuditChange] = field(default_factory=list)
|
||||
|
||||
def merge(self, other: "Audit") -> None:
|
||||
self.entries += other.entries
|
||||
self.source_matches += other.source_matches
|
||||
self.ambiguous_sources += other.ambiguous_sources
|
||||
self.changed_descriptions += other.changed_descriptions
|
||||
self.changed_tags += other.changed_tags
|
||||
self.changes.extend(other.changes)
|
||||
|
||||
|
||||
def normalize_key(value: str) -> str:
|
||||
return unicodedata.normalize("NFKC", value).casefold().strip()
|
||||
|
||||
|
||||
def clean_text(value: str) -> str:
|
||||
value = html.unescape(value).replace("\xa0", " ")
|
||||
value = value.replace("“", '"').replace("”", '"')
|
||||
value = value.replace("‘", "'").replace("’", "'")
|
||||
return MULTISPACE_RE.sub(" ", value).strip()
|
||||
|
||||
|
||||
def strip_terminal_punctuation(value: str) -> str:
|
||||
return TERMINAL_PUNCTUATION_RE.sub("", clean_text(value)).rstrip()
|
||||
|
||||
|
||||
def safe_source_description(value: str) -> str:
|
||||
value = strip_terminal_punctuation(value)
|
||||
if not value or PROHIBITED_PROCESS_RE.search(value):
|
||||
return ""
|
||||
return value
|
||||
|
||||
|
||||
def _extract_source_description(page: Path) -> str:
|
||||
try:
|
||||
soup = BeautifulSoup(page.read_text(encoding="utf-8"), "html.parser")
|
||||
except (OSError, UnicodeError):
|
||||
return ""
|
||||
root = soup.select_one("#help_content") or soup
|
||||
for expected in ("简述", "说明"):
|
||||
for label in root.select(".DescriteTitle"):
|
||||
if clean_text(label.get_text(" ", strip=True)) != expected:
|
||||
continue
|
||||
marker = label.parent if label.parent is not None else label
|
||||
sibling = marker.find_next_sibling()
|
||||
if sibling is None:
|
||||
sibling = label.find_next_sibling()
|
||||
if sibling is not None:
|
||||
return safe_source_description(sibling.get_text(" ", strip=True))
|
||||
return ""
|
||||
|
||||
|
||||
class SourceCatalog:
|
||||
PREFERENCES = {
|
||||
"builtin": ("tsl_base", "net_function", "knowledge_base"),
|
||||
"third": ("net_function", "tsl_base", "knowledge_base"),
|
||||
"deprecated": ("net_function", "tsl_base", "knowledge_base"),
|
||||
"dotnet": ("net_function", "tsl_base", "knowledge_base"),
|
||||
}
|
||||
|
||||
def __init__(self, records: dict[str, list[SourceRecord]] | None = None):
|
||||
self.records = records or {}
|
||||
self._description_cache: dict[Path, str] = {}
|
||||
|
||||
@classmethod
|
||||
def empty(cls) -> "SourceCatalog":
|
||||
return cls({})
|
||||
|
||||
@classmethod
|
||||
def from_docs_root(cls, docs_root: Path) -> "SourceCatalog":
|
||||
records: dict[str, list[SourceRecord]] = defaultdict(list)
|
||||
for corpus in ("net_function", "tsl_base", "knowledge_base"):
|
||||
root = docs_root / corpus
|
||||
manifest = root / "manifest.tsv"
|
||||
if not manifest.is_file():
|
||||
continue
|
||||
with manifest.open(encoding="utf-8", newline="") as handle:
|
||||
for row in csv.DictReader(handle, delimiter="\t"):
|
||||
if row.get("status") != "ok" or not row.get("title"):
|
||||
continue
|
||||
record = SourceRecord(
|
||||
corpus=corpus,
|
||||
page_id=row.get("id", ""),
|
||||
title=row["title"],
|
||||
page=root / row.get("path", ""),
|
||||
)
|
||||
records[normalize_key(record.title)].append(record)
|
||||
return cls(dict(records))
|
||||
|
||||
def _load_description(self, record: SourceRecord) -> SourceRecord:
|
||||
if record.description:
|
||||
return record
|
||||
if record.page not in self._description_cache:
|
||||
self._description_cache[record.page] = _extract_source_description(
|
||||
record.page
|
||||
)
|
||||
return replace(
|
||||
record, description=self._description_cache[record.page]
|
||||
)
|
||||
|
||||
def match(self, entry: EntryContext) -> list[SourceRecord]:
|
||||
matches = self.records.get(normalize_key(entry.name), [])
|
||||
preference = self.PREFERENCES.get(
|
||||
entry.scope,
|
||||
("net_function", "tsl_base", "knowledge_base"),
|
||||
)
|
||||
rank = {name: index for index, name in enumerate(preference)}
|
||||
|
||||
def page_order(record: SourceRecord):
|
||||
try:
|
||||
page_id = (0, int(record.page_id))
|
||||
except ValueError:
|
||||
page_id = (1, record.page_id)
|
||||
return rank.get(record.corpus, 99), page_id
|
||||
|
||||
return sorted(matches, key=page_order)
|
||||
|
||||
def describe(self, records: list[SourceRecord]) -> list[SourceRecord]:
|
||||
return [self._load_description(record) for record in records]
|
||||
|
||||
|
||||
def _starts_with_verb(value: str) -> bool:
|
||||
return value.startswith(LEADING_VERBS) or bool(
|
||||
ACTION_CLAUSE_RE.match(value)
|
||||
)
|
||||
|
||||
|
||||
def _choose_source_description(
|
||||
current: str, sources: list[SourceRecord]
|
||||
) -> str:
|
||||
candidates = [
|
||||
safe_source_description(record.description) for record in sources
|
||||
]
|
||||
candidates = [value for value in candidates if value]
|
||||
if not candidates:
|
||||
return current
|
||||
if len(sources) == 1 and current in GENERIC_SUMMARIES:
|
||||
candidate = candidates[0]
|
||||
if candidate not in GENERIC_SUMMARIES and len(candidate) > len(current):
|
||||
return candidate
|
||||
if PROHIBITED_PROCESS_RE.search(current):
|
||||
return candidates[0]
|
||||
return current
|
||||
|
||||
|
||||
def _property_description(value: str, access: str) -> str:
|
||||
if _starts_with_verb(value):
|
||||
return value
|
||||
if (
|
||||
len(value) > 30
|
||||
or any(mark in value for mark in "。；，")
|
||||
or value.startswith(("功能同", "只读", "用于"))
|
||||
):
|
||||
return value
|
||||
normalized = normalize_key(access).replace(" ", "")
|
||||
if value.startswith("是否"):
|
||||
return (
|
||||
f"控制{value}"
|
||||
if "write" in normalized
|
||||
else f"指示{value}"
|
||||
)
|
||||
if "read" in normalized and "write" in normalized:
|
||||
return f"获取或设置{value}"
|
||||
if "write" in normalized:
|
||||
return f"设置{value}"
|
||||
return f"获取{value}"
|
||||
|
||||
|
||||
def _name_intent(name: str) -> str:
|
||||
key = normalize_key(name)
|
||||
rules = (
|
||||
(("is", "if", "has", "can", "check", "valid"), "判断"),
|
||||
(("get", "find", "query", "search"), "获取"),
|
||||
(("read", "load"), "读取"),
|
||||
(("write", "save", "export"), "写入"),
|
||||
(("set",), "设置"),
|
||||
(("create", "make", "new"), "创建"),
|
||||
(("delete", "remove", "drop"), "删除"),
|
||||
(("clear", "reset"), "清除"),
|
||||
(("add", "append", "insert"), "添加"),
|
||||
(("parse", "decode"), "解析"),
|
||||
(("encode", "convert", "to"), "转换"),
|
||||
(("format",), "格式化"),
|
||||
(("calc", "compute"), "计算"),
|
||||
(("open",), "打开"),
|
||||
(("close",), "关闭"),
|
||||
(("send", "post"), "发送"),
|
||||
(("connect", "login"), "连接"),
|
||||
(("list",), "列出"),
|
||||
)
|
||||
for prefixes, intent in rules:
|
||||
if key.startswith(prefixes):
|
||||
return intent
|
||||
return ""
|
||||
|
||||
|
||||
def improve_description(
|
||||
entry: EntryContext, sources: list[SourceRecord]
|
||||
) -> str:
|
||||
current = strip_terminal_punctuation(entry.summary)
|
||||
current = _choose_source_description(current, sources)
|
||||
current = strip_terminal_punctuation(current)
|
||||
if not current:
|
||||
return entry.summary
|
||||
|
||||
if entry.kind == "class":
|
||||
replacement = CLASS_DESCRIPTIONS.get(normalize_key(entry.name))
|
||||
if replacement and re.fullmatch(
|
||||
rf"{re.escape(entry.name)}\s*内置对象", current, re.IGNORECASE
|
||||
):
|
||||
return replacement
|
||||
|
||||
if entry.kind == "property":
|
||||
return _property_description(current, entry.access)
|
||||
|
||||
key = normalize_key(entry.name)
|
||||
generic_override = GENERIC_DESCRIPTION_OVERRIDES.get(key)
|
||||
if generic_override and current.endswith("相关函数"):
|
||||
return generic_override
|
||||
awkward_override = AWKWARD_DESCRIPTION_OVERRIDES.get(key)
|
||||
if awkward_override and current.startswith(
|
||||
("返回根据", "返回将", "返回:", "返回：")
|
||||
):
|
||||
return awkward_override
|
||||
|
||||
current = re.sub(
|
||||
r"^(?:该|本)函数(?:主要(?:是)?)?\s*", "", current
|
||||
)
|
||||
current = re.sub(r"^功能[:：]\s*", "", current)
|
||||
if current.startswith("是否"):
|
||||
current = f"判断{current}"
|
||||
|
||||
for old, new in (
|
||||
("读出", "读取"),
|
||||
("取出", "获取"),
|
||||
("得到", "获取"),
|
||||
("获得", "获取"),
|
||||
("取得", "获取"),
|
||||
("新建", "创建"),
|
||||
):
|
||||
if current.startswith(old):
|
||||
current = new + current[len(old):]
|
||||
break
|
||||
if current.startswith("读") and not current.startswith("读取"):
|
||||
current = "读取" + current[1:]
|
||||
if current.startswith("写") and not current.startswith("写入"):
|
||||
current = "写入" + current[1:]
|
||||
current = current.replace("读出", "读取")
|
||||
current = current.replace("装载内容", "加载内容")
|
||||
|
||||
related_function = RELATED_FUNCTION_RE.fullmatch(current)
|
||||
if related_function:
|
||||
return f"返回 {related_function.group(1)} 对应的数据"
|
||||
|
||||
formula = FORMULA_RE.match(current)
|
||||
page_key = normalize_key(entry.page)
|
||||
if (
|
||||
formula
|
||||
and ("/financial/" in page_key or "/financial_report/" in page_key)
|
||||
and not _starts_with_verb(current)
|
||||
):
|
||||
left, right = formula.groups()
|
||||
return f"计算{left.strip()}，公式为{right.strip()}"
|
||||
|
||||
if _starts_with_verb(current):
|
||||
return current
|
||||
|
||||
if "/financial/" in page_key or "/financial_report/" in page_key:
|
||||
if (
|
||||
re.match(r"^\d+(?:\s|[.．、)])", current)
|
||||
or COMPARISON_RE.search(current)
|
||||
):
|
||||
return current
|
||||
return f"返回{current}"
|
||||
|
||||
intent = _name_intent(entry.name)
|
||||
if intent and len(current) <= 28 and not any(
|
||||
verb in current[:16] for verb in LEADING_VERBS
|
||||
):
|
||||
return f"{intent}{current}"
|
||||
return current
|
||||
|
||||
|
||||
DOMAIN_RULES = (
|
||||
(r"数组|列表|(?:^|[^a-z])(?:array|fmarray|list)(?:$|[^a-z])", ("数组", "列表", "array", "list")),
|
||||
(r"矩阵|(?:^|[^a-z])matrix(?:$|[^a-z])|\bmt_", ("矩阵", "matrix")),
|
||||
(r"字符串|文本|(?:^|[^a-z])(?:string|char|text)(?:$|[^a-z])", ("字符串", "文本", "string", "text")),
|
||||
(r"日期|时间|(?:^|[^a-z])(?:datetime|date|time)(?:$|[^a-z])", ("日期时间", "日期", "时间", "datetime")),
|
||||
(r"文件|目录|路径|(?:^|[^a-z])(?:file|folder|directory|path)(?:$|[^a-z])", ("文件", "目录", "路径", "file")),
|
||||
(r"网络|(?:^|[^a-z])(?:http|https|cgi|cookie|url)(?:$|[^a-z])", ("网络", "HTTP", "请求", "network")),
|
||||
(r"缓存|(?:^|[^a-z])cache(?:$|[^a-z])", ("缓存", "缓存管理", "cache")),
|
||||
(r"数据库|(?:^|[^a-z])(?:sql|dbf|ini)(?:$|[^a-z])", ("数据库", "SQL", "database")),
|
||||
(r"进程|线程|(?:^|[^a-z])(?:process|thread|pipe)(?:$|[^a-z])", ("进程", "线程", "process")),
|
||||
(r"颜色|(?:^|[^a-z])(?:color|rgb|cmyk)(?:$|[^a-z])", ("颜色", "RGB", "color")),
|
||||
(r"图形|图表|(?:^|[^a-z])(?:graph|chart)(?:$|[^a-z])", ("图形", "图表", "graph")),
|
||||
(r"统计|概率|分布|(?:^|[^a-z])(?:statistics|cdf|pdf)(?:$|[^a-z])", ("统计", "概率", "statistics")),
|
||||
(r"优化|线性规划|(?:^|[^a-z])optimization(?:$|[^a-z])", ("优化", "求解", "optimization")),
|
||||
(r"数值|数学|(?:^|[^a-z])(?:numeric|math)(?:$|[^a-z])", ("数学", "数值", "numeric")),
|
||||
(r"类型转换|(?:^|[^a-z])(?:convert|conversion)(?:$|[^a-z])", ("类型转换", "转换", "conversion")),
|
||||
(r"对象|(?:^|[^a-z])(?:object|class)(?:$|[^a-z])", ("对象", "实例", "object")),
|
||||
(r"(?:^|[^a-z])(?:com|ole)(?:$|[^a-z])|activex", ("COM", "OLE", "自动化")),
|
||||
(r"(?:^|[^a-z])ftp(?:$|[^a-z])", ("FTP", "文件传输")),
|
||||
(r"邮件|(?:^|[^a-z])(?:smtp|pop3|imap|mail)(?:$|[^a-z])", ("邮件", "SMTP", "POP3", "IMAP")),
|
||||
(r"财务|金融|证券|股票|基金|债券|期货|期权", ("金融",)),
|
||||
(r"报表|报告期|report", ("报表", "报告期")),
|
||||
)
|
||||
|
||||
SPECIAL_TAG_RULES = (
|
||||
(r"去重|删除重复|dropduplicate|dedup", ("去重", "删除重复", "deduplicate")),
|
||||
(r"哈希索引|哈希表", ("哈希索引", "哈希表", "hash")),
|
||||
(
|
||||
r"摘要|(?:^|[^a-z0-9_])(?:crc32|md5|sha1|sha224|sha256|sha384|sha512|sm3|digest|hash)(?:$|[^a-z0-9_])",
|
||||
("摘要", "哈希", "散列", "digest", "hash"),
|
||||
),
|
||||
(r"排序|sort", ("排序", "sort")),
|
||||
(r"过滤|筛选|filter", ("过滤", "筛选", "filter")),
|
||||
(r"查找|搜索|find|search|lookup", ("查找", "搜索", "lookup")),
|
||||
)
|
||||
|
||||
INTENT_TAGS = {
|
||||
"返回": ("返回", "获取"),
|
||||
"获取": ("获取", "查询"),
|
||||
"读取": ("读取", "获取"),
|
||||
"写入": ("写入", "保存"),
|
||||
"设置": ("设置", "修改"),
|
||||
"创建": ("创建", "生成"),
|
||||
"删除": ("删除", "移除"),
|
||||
"清除": ("清除", "重置"),
|
||||
"添加": ("添加", "追加"),
|
||||
"查找": ("查找", "搜索"),
|
||||
"判断": ("判断", "检查"),
|
||||
"计算": ("计算", "求解"),
|
||||
"转换": ("转换", "编码"),
|
||||
"解析": ("解析", "parse"),
|
||||
"格式化": ("格式化", "format"),
|
||||
"打开": ("打开", "open"),
|
||||
"关闭": ("关闭", "close"),
|
||||
"发送": ("发送", "提交"),
|
||||
"连接": ("连接", "登录"),
|
||||
"列出": ("列出", "列表"),
|
||||
"输出": ("输出", "打印"),
|
||||
}
|
||||
|
||||
|
||||
def _taxonomy_tags(title: str) -> list[str]:
|
||||
result = []
|
||||
for raw in re.split(r"\s*/\s*|\s+-\s+", title):
|
||||
value = raw.strip(" #")
|
||||
value = re.sub(r"\([^)]*\)|（[^）]*）", "", value).strip()
|
||||
for suffix in ("相关函数", "及其实现", "相关"):
|
||||
if value.endswith(suffix):
|
||||
value = value[: -len(suffix)].rstrip()
|
||||
if not value or normalize_key(value) in GENERIC_TAGS:
|
||||
continue
|
||||
if len(value) > 18:
|
||||
continue
|
||||
chinese = "".join(re.findall(r"[\u3400-\u9fff]+", value))
|
||||
latin = re.sub(r"[\u3400-\u9fff]+", " ", value).strip()
|
||||
if latin and normalize_key(latin) not in GENERIC_TAGS:
|
||||
result.extend(
|
||||
re.findall(r"[A-Za-z][A-Za-z0-9_+.-]*", latin)
|
||||
)
|
||||
if chinese:
|
||||
result.append(chinese)
|
||||
return result
|
||||
|
||||
|
||||
def _description_intent(summary: str) -> str:
|
||||
value = strip_terminal_punctuation(summary)
|
||||
replacements = {
|
||||
"得到": "获取",
|
||||
"获得": "获取",
|
||||
"取得": "获取",
|
||||
"新建": "创建",
|
||||
}
|
||||
for source, target in replacements.items():
|
||||
if value.startswith(source):
|
||||
return target
|
||||
for intent in INTENT_TAGS:
|
||||
if value.startswith(intent):
|
||||
return intent
|
||||
for intent in ("读取", "写入", "保存", "计算", "判断", "转换", "解析"):
|
||||
if intent in value[:16]:
|
||||
return "写入" if intent == "保存" else intent
|
||||
return ""
|
||||
|
||||
|
||||
def derive_tags(entry: EntryContext, page: PageContext) -> list[str]:
|
||||
tags: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(value: str) -> None:
|
||||
value = clean_text(value).strip()
|
||||
key = normalize_key(value)
|
||||
if (
|
||||
not value
|
||||
or key in seen
|
||||
or key in GENERIC_TAGS
|
||||
or len(tags) >= 12
|
||||
):
|
||||
return
|
||||
seen.add(key)
|
||||
tags.append(value)
|
||||
|
||||
for tag in entry.tags:
|
||||
add(tag)
|
||||
|
||||
for tag in _taxonomy_tags(page.title):
|
||||
add(tag)
|
||||
|
||||
haystack = " ".join(
|
||||
(entry.name, entry.signature, entry.summary, page.title, page.path)
|
||||
)
|
||||
domain_haystack = " ".join((page.title, page.path))
|
||||
for pattern, aliases in SPECIAL_TAG_RULES:
|
||||
if re.search(pattern, haystack, re.IGNORECASE):
|
||||
for alias in aliases:
|
||||
add(alias)
|
||||
|
||||
summary = strip_terminal_punctuation(entry.summary)
|
||||
intent = _description_intent(summary)
|
||||
if not intent and not _starts_with_verb(summary):
|
||||
is_market_price = (
|
||||
normalize_key(entry.name) in {"open", "close"}
|
||||
and "盘价" in summary
|
||||
)
|
||||
if not is_market_price:
|
||||
intent = _name_intent(entry.name)
|
||||
for alias in INTENT_TAGS.get(intent, ()):
|
||||
add(alias)
|
||||
for pattern, aliases in DOMAIN_RULES:
|
||||
if re.search(pattern, domain_haystack, re.IGNORECASE):
|
||||
for alias in aliases:
|
||||
add(alias)
|
||||
for acronym in ACRONYM_RE.findall(entry.summary):
|
||||
add(acronym)
|
||||
return tags
|
||||
|
||||
|
||||
def _scope_for_page(page: str) -> str:
|
||||
parts = Path(page).parts
|
||||
return parts[0] if len(parts) > 1 else Path(page).stem
|
||||
|
||||
|
||||
def _page_title(lines: list[str], page: str) -> PageContext:
|
||||
for line in lines:
|
||||
match = H1_RE.match(line)
|
||||
if match:
|
||||
return PageContext(match.group(1), page)
|
||||
return PageContext(Path(page).stem, page)
|
||||
|
||||
|
||||
def _entry_metadata(lines: list[str], start: int, end: int):
|
||||
declaration_seen = False
|
||||
description_index = None
|
||||
tag_index = None
|
||||
access = ""
|
||||
for index in range(start + 1, end):
|
||||
text = lines[index].strip()
|
||||
if not text:
|
||||
continue
|
||||
if not declaration_seen:
|
||||
if DECLARATION_LINE_RE.fullmatch(text):
|
||||
declaration_seen = True
|
||||
continue
|
||||
tag_match = TAGS_RE.fullmatch(text)
|
||||
if tag_match:
|
||||
tag_index = index
|
||||
continue
|
||||
access_match = ACCESS_RE.fullmatch(text)
|
||||
if access_match:
|
||||
access = access_match.group(1)
|
||||
continue
|
||||
if description_index is None:
|
||||
if text.startswith(("|", "#", "返回：", "类型：", "可见性：", "值：")):
|
||||
continue
|
||||
description_index = index
|
||||
tags = ()
|
||||
if tag_index is not None:
|
||||
tags = tuple(TAGS_RE.fullmatch(lines[tag_index].strip()).group(1).split())
|
||||
return description_index, tag_index, access, tags
|
||||
|
||||
|
||||
def enrich_markdown(
|
||||
text: str, page: str, source_catalog: SourceCatalog
|
||||
) -> tuple[str, Audit]:
|
||||
had_final_newline = text.endswith("\n")
|
||||
lines = text.splitlines()
|
||||
page_context = _page_title(lines, page)
|
||||
entries = [entry for entry in iter_api_entries(lines) if entry.heading.valid]
|
||||
audit = Audit(entries=len(entries))
|
||||
root_owner = ""
|
||||
owners: dict[int, str] = {}
|
||||
for entry in entries:
|
||||
if entry.heading.level == 2:
|
||||
root_owner = entry.heading.name if entry.heading.kind in {"class", "unit"} else ""
|
||||
owners[entry.start] = root_owner if entry.heading.level > 2 else ""
|
||||
|
||||
for api in reversed(entries):
|
||||
description_index, tag_index, access, existing_tags = _entry_metadata(
|
||||
lines, api.start, api.end
|
||||
)
|
||||
if description_index is None:
|
||||
continue
|
||||
old_description = lines[description_index].strip()
|
||||
entry = EntryContext(
|
||||
name=api.heading.name,
|
||||
signature=api.heading.signature,
|
||||
kind=api.heading.kind,
|
||||
scope=_scope_for_page(page),
|
||||
page=page,
|
||||
summary=old_description,
|
||||
access=access,
|
||||
owner=owners.get(api.start, ""),
|
||||
tags=existing_tags,
|
||||
)
|
||||
sources = source_catalog.match(entry)
|
||||
if sources:
|
||||
audit.source_matches += 1
|
||||
if len(sources) > 1:
|
||||
audit.ambiguous_sources += 1
|
||||
evidence = sources
|
||||
normalized_summary = strip_terminal_punctuation(old_description)
|
||||
if (
|
||||
normalized_summary in GENERIC_SUMMARIES
|
||||
or PROHIBITED_PROCESS_RE.search(normalized_summary)
|
||||
):
|
||||
evidence = source_catalog.describe(sources)
|
||||
new_description = improve_description(entry, evidence)
|
||||
entry_for_tags = replace(entry, summary=new_description)
|
||||
new_tags = tuple(derive_tags(entry_for_tags, page_context))
|
||||
|
||||
if new_description != old_description:
|
||||
lines[description_index] = new_description
|
||||
audit.changed_descriptions += 1
|
||||
if new_tags != existing_tags:
|
||||
rendered = f"<!-- tags: {' '.join(new_tags)} -->"
|
||||
if tag_index is not None:
|
||||
lines[tag_index] = rendered
|
||||
else:
|
||||
insert_at = description_index + 1
|
||||
if insert_at < len(lines) and lines[insert_at].strip() == "":
|
||||
insert_at += 1
|
||||
lines[insert_at:insert_at] = [rendered, ""]
|
||||
else:
|
||||
lines[insert_at:insert_at] = ["", rendered, ""]
|
||||
audit.changed_tags += 1
|
||||
|
||||
if new_description != old_description or new_tags != existing_tags:
|
||||
audit.changes.append(
|
||||
AuditChange(
|
||||
page=page,
|
||||
name=api.heading.name,
|
||||
kind=api.heading.kind,
|
||||
old_description=old_description,
|
||||
new_description=new_description,
|
||||
old_tags=existing_tags,
|
||||
new_tags=new_tags,
|
||||
sources=tuple(
|
||||
f"{record.corpus}:{record.page_id}" for record in sources
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
result = "\n".join(lines)
|
||||
if had_final_newline:
|
||||
result += "\n"
|
||||
return result, audit
|
||||
|
||||
|
||||
def _write_report(path: Path, audit: Audit) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.writer(handle, delimiter="\t", lineterminator="\n")
|
||||
writer.writerow(
|
||||
(
|
||||
"page",
|
||||
"name",
|
||||
"kind",
|
||||
"old_description",
|
||||
"new_description",
|
||||
"old_tags",
|
||||
"new_tags",
|
||||
"sources",
|
||||
)
|
||||
)
|
||||
for item in audit.changes:
|
||||
writer.writerow(
|
||||
(
|
||||
item.page,
|
||||
item.name,
|
||||
item.kind,
|
||||
item.old_description,
|
||||
item.new_description,
|
||||
" ".join(item.old_tags),
|
||||
" ".join(item.new_tags),
|
||||
" ".join(item.sources),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _selected(page: str, scopes: set[str], prefixes: tuple[str, ...]) -> bool:
|
||||
if scopes and _scope_for_page(page) not in scopes:
|
||||
return False
|
||||
return not prefixes or page.startswith(prefixes)
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--audit", action="store_true", help="report changes only")
|
||||
mode.add_argument("--apply", action="store_true", help="write proposed changes")
|
||||
parser.add_argument("--root", type=Path, required=True, help="codegen Markdown root")
|
||||
parser.add_argument(
|
||||
"--docs-root", type=Path, required=True, help="cached tmp/docs root"
|
||||
)
|
||||
parser.add_argument("--report", type=Path, help="optional TSV audit report")
|
||||
parser.add_argument("--scope", action="append", default=[])
|
||||
parser.add_argument("--prefix", action="append", default=[])
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
catalog = SourceCatalog.from_docs_root(args.docs_root)
|
||||
aggregate = Audit()
|
||||
changed_files = 0
|
||||
for md in sorted(args.root.rglob("*.md")):
|
||||
page = md.relative_to(args.root).as_posix()
|
||||
if not _selected(page, set(args.scope), tuple(args.prefix)):
|
||||
continue
|
||||
before_stat = md.stat()
|
||||
original = md.read_text(encoding="utf-8")
|
||||
enriched, audit = enrich_markdown(original, page, catalog)
|
||||
aggregate.merge(audit)
|
||||
if enriched == original:
|
||||
continue
|
||||
changed_files += 1
|
||||
if args.apply:
|
||||
after_stat = md.stat()
|
||||
if (
|
||||
after_stat.st_mtime_ns != before_stat.st_mtime_ns
|
||||
or after_stat.st_size != before_stat.st_size
|
||||
):
|
||||
raise RuntimeError(f"target changed while processing: {md}")
|
||||
md.write_text(enriched, encoding="utf-8", newline="\n")
|
||||
|
||||
if args.report:
|
||||
_write_report(args.report, aggregate)
|
||||
print(
|
||||
"entries={entries} source_matches={source_matches} "
|
||||
"ambiguous_sources={ambiguous_sources} changed_files={changed_files} "
|
||||
"changed_descriptions={changed_descriptions} changed_tags={changed_tags}".format(
|
||||
entries=aggregate.entries,
|
||||
source_matches=aggregate.source_matches,
|
||||
ambiguous_sources=aggregate.ambiguous_sources,
|
||||
changed_files=changed_files,
|
||||
changed_descriptions=aggregate.changed_descriptions,
|
||||
changed_tags=aggregate.changed_tags,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -21,7 +21,7 @@ Function declaration fields:
|
||||
tags optional list of Chinese keywords -> `<!-- tags: ... -->`
|
||||
params required when the signature takes args; omit for nullary
|
||||
returns required return type
|
||||
example optional fenced tsl block, pasted verbatim
|
||||
examples optional list of structured examples with desc/code/output fields
|
||||
Each param: name/type/desc required; optional (bool) -> `可选。` prefix;
|
||||
values (list of {value, desc}) -> a `name 取值` enum section.
|
||||
|
||||
@@ -29,6 +29,7 @@ Usage (run from repo root):
|
||||
python tools/tsl-codegen/scripts/generate.py --file entry.yml
|
||||
python tools/tsl-codegen/scripts/generate.py --file entry.json \
|
||||
--scope my-project
|
||||
python scripts/generate.py --file entry.json --root ../..
|
||||
python tools/tsl-codegen/scripts/generate.py --dir recordings
|
||||
"""
|
||||
|
||||
@@ -279,7 +280,6 @@ def validate_function(fn, where, *, returns_required, extra_fields=()):
|
||||
"tags",
|
||||
"params",
|
||||
"returns",
|
||||
"example",
|
||||
"examples",
|
||||
*extra_fields,
|
||||
}
|
||||
@@ -292,12 +292,6 @@ def validate_function(fn, where, *, returns_required, extra_fields=()):
|
||||
non_empty_string(fn.get("returns"), f"{where}：缺少 returns")
|
||||
elif "returns" in fn:
|
||||
optional_draft_string(fn["returns"], f"{where}: returns")
|
||||
require(
|
||||
not ("example" in fn and "examples" in fn),
|
||||
f"{where}：example 和 examples 不能同时存在",
|
||||
)
|
||||
if "example" in fn:
|
||||
non_empty_string(fn["example"], f"{where}: example")
|
||||
if "examples" in fn:
|
||||
validate_examples(fn["examples"], where)
|
||||
return name
|
||||
@@ -333,7 +327,6 @@ def validate_class_member(member, where):
|
||||
"params",
|
||||
"returns",
|
||||
"modifiers",
|
||||
"example",
|
||||
"examples",
|
||||
},
|
||||
where,
|
||||
@@ -600,14 +593,6 @@ def render_examples(fn, heading_level):
|
||||
while lines and not lines[-1]:
|
||||
lines.pop()
|
||||
return lines
|
||||
if fn.get("example"):
|
||||
return [
|
||||
f"{'#' * heading_level} 示例",
|
||||
"",
|
||||
"```tsl",
|
||||
*fn["example"].rstrip("\n").split("\n"),
|
||||
"```",
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
@@ -822,7 +807,7 @@ def format_markdown(text):
|
||||
return result.stdout
|
||||
|
||||
|
||||
def output_path(data, scope):
|
||||
def output_path(data, scope, root=None):
|
||||
"""Build the leaf-page destination from the recording file's relative path."""
|
||||
relative = data.get("path")
|
||||
require(relative, "录入数据缺少 path")
|
||||
@@ -854,7 +839,8 @@ def output_path(data, scope):
|
||||
)
|
||||
relative_path = Path(*recorded_path.parts)
|
||||
return (
|
||||
Path("skills/tsl-api-reference/references/codegen")
|
||||
(Path(root) if root is not None else Path())
|
||||
/ "skills/tsl-api-reference/references/codegen"
|
||||
/ scope
|
||||
/ relative_path.with_suffix(".md")
|
||||
)
|
||||
@@ -902,14 +888,14 @@ def gather_directory_inputs(directory, fmt):
|
||||
return inputs
|
||||
|
||||
|
||||
def prepare_input(in_path, input_format, scope):
|
||||
def prepare_input(in_path, input_format, scope, root=None):
|
||||
data = load_entries(in_path, input_format)
|
||||
try:
|
||||
rendered = render_page(data)
|
||||
except GenerationError as exc:
|
||||
raise GenerationError(f"录入数据校验失败：{exc}") from exc
|
||||
try:
|
||||
out_path = output_path(data, scope)
|
||||
out_path = output_path(data, scope, root)
|
||||
except GenerationError as exc:
|
||||
raise GenerationError(f"输出路径无效：{exc}") from exc
|
||||
try:
|
||||
@@ -941,11 +927,14 @@ def find_output_collisions(prepared):
|
||||
def main(argv=None):
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
if hasattr(sys.stderr, "reconfigure"):
|
||||
sys.stderr.reconfigure(encoding="utf-8")
|
||||
parser = ChineseArgumentParser(
|
||||
description="从 YAML/JSON 录入文件生成 TSL API 文档",
|
||||
usage=(
|
||||
"%(prog)s [--help] [--scope SCOPE] [--format {json,yaml}] "
|
||||
"(--file INPUT_FILE | --dir INPUT_DIR | INPUT_FILE)"
|
||||
"%(prog)s [--help] [--root ROOT_DIR] [--scope SCOPE] "
|
||||
"[--format {json,yaml}] "
|
||||
"(--file INPUT_FILE | --dir INPUT_DIR)"
|
||||
),
|
||||
add_help=False,
|
||||
allow_abbrev=False,
|
||||
@@ -955,12 +944,6 @@ def main(argv=None):
|
||||
action="help",
|
||||
help="显示本帮助并退出（不提供 -h 短选项）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"legacy_input",
|
||||
nargs="?",
|
||||
metavar="INPUT_FILE",
|
||||
help="已废弃，请使用 --file；暂时兼容 YAML/JSON 录入文件路径",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--file",
|
||||
dest="input_file",
|
||||
@@ -973,6 +956,11 @@ def main(argv=None):
|
||||
metavar="INPUT_DIR",
|
||||
help="批量生成目录中的直属 YAML/JSON 录入文件",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
metavar="ROOT_DIR",
|
||||
help="包含 skills 目录的项目根目录（默认：当前工作目录）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scope",
|
||||
type=scope_name,
|
||||
@@ -986,16 +974,16 @@ def main(argv=None):
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
input_modes = (args.legacy_input, args.input_file, args.input_dir)
|
||||
input_modes = (args.input_file, args.input_dir)
|
||||
if sum(value is not None for value in input_modes) != 1:
|
||||
parser.error("必须且只能指定一种输入方式：INPUT_FILE、--file 或 --dir")
|
||||
parser.error("必须且只能指定一种输入方式：--file 或 --dir")
|
||||
is_batch = args.input_dir is not None
|
||||
try:
|
||||
if is_batch:
|
||||
input_paths = gather_directory_inputs(Path(args.input_dir), args.format)
|
||||
input_format = None
|
||||
else:
|
||||
in_path = Path(args.input_file or args.legacy_input)
|
||||
in_path = Path(args.input_file)
|
||||
if not in_path.is_file():
|
||||
die(f"输入文件不存在或不是普通文件：{in_path}")
|
||||
input_paths = [in_path]
|
||||
@@ -1008,7 +996,9 @@ def main(argv=None):
|
||||
input_errors = []
|
||||
for in_path in input_paths:
|
||||
try:
|
||||
prepared.append(prepare_input(in_path, input_format, args.scope))
|
||||
prepared.append(
|
||||
prepare_input(in_path, input_format, args.scope, args.root)
|
||||
)
|
||||
except GenerationError as exc:
|
||||
input_errors.append((in_path, exc))
|
||||
|
||||
|
||||
@@ -1,778 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "enrich_metadata.py"
|
||||
|
||||
|
||||
def load_script():
|
||||
spec = importlib.util.spec_from_file_location("tsl_enrich_metadata", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class SourceCatalogTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.module = load_script()
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.docs = Path(self.temp_dir.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def write_corpus(self, corpus, rows, descriptions):
|
||||
root = self.docs / corpus
|
||||
pages = root / "pages"
|
||||
pages.mkdir(parents=True)
|
||||
manifest = [
|
||||
"id\ttitle\turl\tpath\tsource_bytes\tutf8_bytes\tsha256\tstatus"
|
||||
]
|
||||
for page_id, title in rows:
|
||||
manifest.append(
|
||||
f"{page_id}\t{title}\thttp://example/{page_id}\t"
|
||||
f"pages/{page_id}.html\t1\t1\tsha\tok"
|
||||
)
|
||||
description = descriptions[page_id]
|
||||
(pages / f"{page_id}.html").write_text(
|
||||
"<div id='help_content'><h3>"
|
||||
+ title
|
||||
+ "</h3><div><div class='DescriteTitle'>简述</div></div>"
|
||||
+ f"<div>{description}</div></div>",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "manifest.tsv").write_text(
|
||||
"\n".join(manifest) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
def test_exact_matching_preserves_underscores_and_prefers_scope_corpus(self):
|
||||
self.write_corpus(
|
||||
"tsl_base",
|
||||
[("1", "alpha_test")],
|
||||
{"1": "基础解释器说明"},
|
||||
)
|
||||
self.write_corpus(
|
||||
"net_function",
|
||||
[("2", "alpha_test"), ("3", "alphaTest")],
|
||||
{"2": "NET 说明", "3": "无下划线的另一个函数"},
|
||||
)
|
||||
catalog = self.module.SourceCatalog.from_docs_root(self.docs)
|
||||
|
||||
builtin = self.module.EntryContext(
|
||||
name="alpha_test",
|
||||
signature="alpha_test()",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/demo.md",
|
||||
summary="原说明",
|
||||
)
|
||||
dotnet = self.module.EntryContext(
|
||||
name="alpha_test",
|
||||
signature="alpha_test()",
|
||||
kind="function",
|
||||
scope="dotnet",
|
||||
page="dotnet/demo.md",
|
||||
summary="原说明",
|
||||
)
|
||||
|
||||
self.assertEqual("tsl_base", catalog.match(builtin)[0].corpus)
|
||||
self.assertEqual("net_function", catalog.match(dotnet)[0].corpus)
|
||||
self.assertNotIn(
|
||||
"alphaTest", {record.title for record in catalog.match(dotnet)}
|
||||
)
|
||||
|
||||
def test_duplicate_titles_remain_visible_for_audit(self):
|
||||
self.write_corpus(
|
||||
"net_function",
|
||||
[("1", "same_name"), ("2", "same_name")],
|
||||
{"1": "说明一", "2": "说明二"},
|
||||
)
|
||||
catalog = self.module.SourceCatalog.from_docs_root(self.docs)
|
||||
entry = self.module.EntryContext(
|
||||
name="same_name",
|
||||
signature="same_name()",
|
||||
kind="function",
|
||||
scope="dotnet",
|
||||
page="dotnet/demo.md",
|
||||
summary="原说明",
|
||||
)
|
||||
|
||||
self.assertEqual(["1", "2"], [item.page_id for item in catalog.match(entry)])
|
||||
|
||||
|
||||
class DescriptionQualityTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.module = load_script()
|
||||
|
||||
def entry(self, **overrides):
|
||||
values = {
|
||||
"name": "profitRatio",
|
||||
"signature": "profitRatio(report_date)",
|
||||
"kind": "function",
|
||||
"scope": "dotnet",
|
||||
"page": "dotnet/financial/profitability.md",
|
||||
"summary": "总资产收益率(%)。",
|
||||
}
|
||||
values.update(overrides)
|
||||
return self.module.EntryContext(**values)
|
||||
|
||||
def test_removes_only_terminal_sentence_punctuation(self):
|
||||
entry = self.entry(summary="总资产收益率(%)。")
|
||||
|
||||
self.assertEqual(
|
||||
"返回总资产收益率(%)",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_preserves_formula_and_identifiers(self):
|
||||
entry = self.entry(
|
||||
summary="总资产报酬率(%)=利润总额/平均资产总额*100。"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"计算总资产报酬率(%)，公式为利润总额/平均资产总额*100",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_rejects_source_process_prose(self):
|
||||
entry = self.entry(summary="返回指定值")
|
||||
source = self.module.SourceRecord(
|
||||
corpus="net_function",
|
||||
page_id="1",
|
||||
title="profitRatio",
|
||||
page=Path("1.html"),
|
||||
description="Windows 已验证通过，Linux 返回 not found",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"返回指定值", self.module.improve_description(entry, [source])
|
||||
)
|
||||
|
||||
def test_current_environment_can_be_real_api_semantics(self):
|
||||
self.assertEqual(
|
||||
"获取当前环境时间系统参数",
|
||||
self.module.safe_source_description("获取当前环境时间系统参数。"),
|
||||
)
|
||||
|
||||
def test_property_access_is_stated_explicitly(self):
|
||||
entry = self.entry(
|
||||
name="Host",
|
||||
signature="Host",
|
||||
kind="property",
|
||||
scope="builtin",
|
||||
page="builtin/object/ftp.md",
|
||||
summary="远程服务器地址。",
|
||||
access="read/write",
|
||||
owner="FTP",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"获取或设置远程服务器地址",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_boolean_property_description_is_idempotent(self):
|
||||
entry = self.entry(
|
||||
name="UseTLS",
|
||||
signature="UseTLS",
|
||||
kind="property",
|
||||
scope="builtin",
|
||||
page="builtin/object/imap.md",
|
||||
summary="是否采用 SSL 连接",
|
||||
access="read/write",
|
||||
owner="IMAP",
|
||||
)
|
||||
|
||||
first = self.module.improve_description(entry, [])
|
||||
repeated = self.module.improve_description(
|
||||
self.module.EntryContext(
|
||||
**{**entry.__dict__, "summary": first}
|
||||
),
|
||||
[],
|
||||
)
|
||||
|
||||
self.assertEqual("控制是否采用 SSL 连接", first)
|
||||
self.assertEqual(first, repeated)
|
||||
|
||||
def test_low_confidence_sentence_is_kept_except_terminal_punctuation(self):
|
||||
entry = self.entry(
|
||||
name="opaqueApi",
|
||||
page="dotnet.md",
|
||||
summary="根据调用上下文处理结果，具体规则取决于输入。",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"根据调用上下文处理结果，具体规则取决于输入",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_short_description_is_not_replaced_by_ambiguous_source_text(self):
|
||||
entry = self.entry(
|
||||
name="clRed",
|
||||
page="builtin/color.md",
|
||||
summary="红色",
|
||||
)
|
||||
sources = [
|
||||
self.module.SourceRecord(
|
||||
corpus="net_function",
|
||||
page_id="1",
|
||||
title="clRed",
|
||||
page=Path("1.html"),
|
||||
description="定义",
|
||||
)
|
||||
]
|
||||
|
||||
self.assertEqual("红色", self.module.improve_description(entry, sources))
|
||||
|
||||
def test_existing_action_word_is_normalized_without_duplicate_verb(self):
|
||||
getter = self.entry(
|
||||
name="getCValue",
|
||||
page="builtin/color.md",
|
||||
summary="得到颜色的 CMYK 模式 C 值。",
|
||||
)
|
||||
writer = self.entry(
|
||||
name="write",
|
||||
page="builtin/cgi.md",
|
||||
summary="输出字符串。",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"获取颜色的 CMYK 模式 C 值",
|
||||
self.module.improve_description(getter, []),
|
||||
)
|
||||
self.assertEqual("输出字符串", self.module.improve_description(writer, []))
|
||||
|
||||
def test_equation_inside_algorithm_sentence_is_not_rewritten_as_formula(self):
|
||||
entry = self.entry(
|
||||
name="se_Gauss",
|
||||
page="builtin/optimization.md",
|
||||
summary="用高斯消去法求解线性方程组 AX = B",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"用高斯消去法求解线性方程组 AX = B",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_builtin_class_gets_a_purpose_description(self):
|
||||
entry = self.entry(
|
||||
name="TStringList",
|
||||
signature="TStringList",
|
||||
kind="class",
|
||||
scope="builtin",
|
||||
page="builtin/object/tstringlist.md",
|
||||
summary="TStringList 内置对象",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"提供字符串集合存储、查找、排序和名称值管理能力的内置对象",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_long_property_explanation_is_not_wrapped_in_access_boilerplate(self):
|
||||
entry = self.entry(
|
||||
name="CommaTextW",
|
||||
signature="CommaTextW",
|
||||
kind="property",
|
||||
scope="builtin",
|
||||
page="builtin/object/tstringlist.md",
|
||||
summary=(
|
||||
"功能同 CommaText，区别是在读取时返回宽字节字符串，"
|
||||
"而 CommaText 返回多字节字符串"
|
||||
),
|
||||
access="read/write",
|
||||
owner="TStringList",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
entry.summary, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_short_read_and_load_phrases_are_normalized(self):
|
||||
read_entry = self.entry(
|
||||
name="read",
|
||||
page="builtin/object/tstream.md",
|
||||
summary="读出内容",
|
||||
)
|
||||
load_entry = self.entry(
|
||||
name="loadFromFile",
|
||||
page="builtin/object/tstringlist.md",
|
||||
summary="从指定的文件中装载内容",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"读取内容", self.module.improve_description(read_entry, [])
|
||||
)
|
||||
self.assertEqual(
|
||||
"从指定的文件中加载内容",
|
||||
self.module.improve_description(load_entry, []),
|
||||
)
|
||||
|
||||
def test_embedded_read_out_is_normalized_without_duplicate_prefix(self):
|
||||
entry = self.entry(
|
||||
name="readExcelSheets",
|
||||
page="dotnet.md",
|
||||
summary="从Excel文件中读出Sheets列表。",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"从Excel文件中读取Sheets列表",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_existing_decomposition_verb_is_not_prefixed(self):
|
||||
entry = self.entry(
|
||||
name="decodeGraphGroup",
|
||||
page="builtin/graph.md",
|
||||
summary="分解图形组合并写入输出参数",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
entry.summary, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_comparison_expression_is_not_rewritten_as_formula(self):
|
||||
entry = self.entry(
|
||||
name="stockStepAmount",
|
||||
page="dotnet/financial/stock-capital-flow.md",
|
||||
summary="分档区间 V1<=Value<V2 的成交金额",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
entry.summary, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_financial_action_sentence_is_not_prefixed_with_return(self):
|
||||
for summary in (
|
||||
"提取股票的财务比率，与系统证券相关",
|
||||
"根据报告期获取财务指标",
|
||||
"将日线数据转换为周线数据",
|
||||
"统计指定报告期的基金数量",
|
||||
"生成指定区间的分析结果",
|
||||
):
|
||||
with self.subTest(summary=summary):
|
||||
entry = self.entry(
|
||||
name="annualRatio",
|
||||
page="dotnet/financial/financial-analysis.md",
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
entry.summary, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_financial_action_clauses_are_not_prefixed_with_return(self):
|
||||
for summary in (
|
||||
"用于生成投资组合结果名称",
|
||||
"如果存在对应记录则返回证券代码",
|
||||
"对矩阵进行正交处理",
|
||||
"从净值表中获取基金列表",
|
||||
"由交易价格倒推出隐含波动率",
|
||||
"按照周期读取数据",
|
||||
"与系统参数证券和日期相关",
|
||||
):
|
||||
with self.subTest(summary=summary):
|
||||
entry = self.entry(
|
||||
name="financialHelper",
|
||||
page="dotnet/financial/financial-analysis.md",
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
entry.summary, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_financial_boolean_description_uses_judgment_wording(self):
|
||||
entry = self.entry(
|
||||
name="isBankBond",
|
||||
page="dotnet/financial/bond-basic-info.md",
|
||||
summary="是否银行间债券。如果是，返回1，否则返回0",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"判断是否银行间债券。如果是，返回1，否则返回0",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_function_boilerplate_is_removed_from_action_description(self):
|
||||
entry = self.entry(
|
||||
name="cb_ytm",
|
||||
page="dotnet/financial/convertible-bond.md",
|
||||
summary="该函数采用牛顿迭代法获取可转债的到期收益率",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"采用牛顿迭代法获取可转债的到期收益率",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_financial_numbered_explanation_is_not_prefixed_with_return(self):
|
||||
entry = self.entry(
|
||||
name="financialMode",
|
||||
page="dotnet/financial/financial-analysis.md",
|
||||
summary="1 表示合并报表，2 表示母公司报表",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
entry.summary, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_related_data_placeholder_is_made_explicit(self):
|
||||
entry = self.entry(
|
||||
name="cb_downPeriodConversionPeriod",
|
||||
page="dotnet/financial/convertible-bond.md",
|
||||
summary="CB_DownPeriodConversionPeriod 相关函数",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"返回 CB_DownPeriodConversionPeriod 对应的数据",
|
||||
self.module.improve_description(entry, []),
|
||||
)
|
||||
|
||||
def test_reviewed_generic_descriptions_use_precise_purpose_text(self):
|
||||
cases = (
|
||||
(
|
||||
"sf_Normal",
|
||||
"统计分布相关函数。",
|
||||
"计算正态分布函数值",
|
||||
),
|
||||
(
|
||||
"readFile",
|
||||
"文件访问函数相关函数。",
|
||||
"读取本地文件中的数据",
|
||||
),
|
||||
(
|
||||
"unicodeEsc2",
|
||||
"多语言支持函数相关函数。",
|
||||
"将字符串转换为 \\uxxxx 形式的 Unicode 编码串",
|
||||
),
|
||||
)
|
||||
for name, summary, expected in cases:
|
||||
with self.subTest(name=name):
|
||||
entry = self.entry(name=name, page="dotnet.md", summary=summary)
|
||||
self.assertEqual(
|
||||
expected, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
def test_reviewed_awkward_descriptions_are_rewritten_directly(self):
|
||||
cases = (
|
||||
(
|
||||
"encodeGraph",
|
||||
"返回根据指定的类型、名称、数据以及属性数组生成的图形。",
|
||||
"根据指定的类型、名称、数据和属性数组生成图形并返回",
|
||||
),
|
||||
(
|
||||
"dupeString",
|
||||
"返回将指定的字符串AText重复ACount次后的字符串",
|
||||
"将字符串 AText 重复 ACount 次并返回结果",
|
||||
),
|
||||
(
|
||||
"stockpjcj",
|
||||
"返回:区间平均成交,区间成交金额/区间成交量",
|
||||
"返回区间平均成交价，计算公式为区间成交金额 / 区间成交量",
|
||||
),
|
||||
)
|
||||
for name, summary, expected in cases:
|
||||
with self.subTest(name=name):
|
||||
entry = self.entry(
|
||||
name=name, page="builtin/string.md", summary=summary
|
||||
)
|
||||
self.assertEqual(
|
||||
expected, self.module.improve_description(entry, [])
|
||||
)
|
||||
|
||||
|
||||
class TagDerivationTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.module = load_script()
|
||||
|
||||
def test_array_deduplication_gets_controlled_chinese_and_english_aliases(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="arrDropDuplicate",
|
||||
signature="arrDropDuplicate(data)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/array.md",
|
||||
summary="删除数组中的重复元素",
|
||||
)
|
||||
page = self.module.PageContext(
|
||||
title="Builtin / 数组", path="builtin/array.md"
|
||||
)
|
||||
|
||||
tags = self.module.derive_tags(entry, page)
|
||||
|
||||
self.assertTrue(
|
||||
{"数组", "列表", "去重", "删除重复", "array", "deduplicate"}
|
||||
<= set(tags)
|
||||
)
|
||||
self.assertLessEqual(len(tags), 12)
|
||||
self.assertNotIn("函数", tags)
|
||||
self.assertNotIn("builtin", tags)
|
||||
|
||||
def test_file_digest_gets_digest_and_hash_aliases(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="getMsgDigest",
|
||||
signature="getMsgDigest(s, mode)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/string.md",
|
||||
summary="计算字符串摘要，支持 CRC32、MD5、SHA 和 SM3",
|
||||
)
|
||||
page = self.module.PageContext(
|
||||
title="Builtin / 字符串", path="builtin/string.md"
|
||||
)
|
||||
|
||||
tags = self.module.derive_tags(entry, page)
|
||||
|
||||
self.assertTrue(
|
||||
{"字符串", "摘要", "哈希", "散列", "digest", "hash"}
|
||||
<= set(tags)
|
||||
)
|
||||
|
||||
def test_existing_tags_are_preserved_first_and_output_is_idempotent(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="readFile",
|
||||
signature="readFile(alias, file_name)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/file.md",
|
||||
summary="读取文件内容",
|
||||
tags=("自定义", "读取"),
|
||||
)
|
||||
page = self.module.PageContext(
|
||||
title="Builtin / 文件", path="builtin/file.md"
|
||||
)
|
||||
|
||||
first = self.module.derive_tags(entry, page)
|
||||
second = self.module.derive_tags(
|
||||
self.module.EntryContext(**{**entry.__dict__, "tags": tuple(first)}),
|
||||
page,
|
||||
)
|
||||
|
||||
self.assertEqual(["自定义", "读取"], first[:2])
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_substrings_do_not_create_digest_or_com_tags(self):
|
||||
reshape = self.module.EntryContext(
|
||||
name="reshape",
|
||||
signature="reshape(data, shape)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/array.md",
|
||||
summary="重构数组形状",
|
||||
)
|
||||
uncompress = self.module.EntryContext(
|
||||
name="uniuncompress",
|
||||
signature="uniuncompress(data, type)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/string.md",
|
||||
summary="统一解压缩数据",
|
||||
)
|
||||
|
||||
reshape_tags = self.module.derive_tags(
|
||||
reshape,
|
||||
self.module.PageContext("Builtin / 数组", reshape.page),
|
||||
)
|
||||
uncompress_tags = self.module.derive_tags(
|
||||
uncompress,
|
||||
self.module.PageContext("Builtin / 字符串", uncompress.page),
|
||||
)
|
||||
|
||||
self.assertNotIn("摘要", reshape_tags)
|
||||
self.assertNotIn("hash", reshape_tags)
|
||||
self.assertNotIn("COM", uncompress_tags)
|
||||
self.assertNotIn("OLE", uncompress_tags)
|
||||
|
||||
def test_runtime_and_character_names_do_not_create_time_or_graph_tags(self):
|
||||
random_entry = self.module.EntryContext(
|
||||
name="randomfrom",
|
||||
signature="randomfrom(values)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/runtime.md",
|
||||
summary="返回随机数组元素",
|
||||
)
|
||||
char_entry = self.module.EntryContext(
|
||||
name="charToByteLen",
|
||||
signature="charToByteLen(s, max_len)",
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/string.md",
|
||||
summary="计算字符串占用的字节数",
|
||||
)
|
||||
|
||||
random_tags = self.module.derive_tags(
|
||||
random_entry,
|
||||
self.module.PageContext("Builtin / 运行时", random_entry.page),
|
||||
)
|
||||
char_tags = self.module.derive_tags(
|
||||
char_entry,
|
||||
self.module.PageContext("Builtin / 字符串", char_entry.page),
|
||||
)
|
||||
|
||||
self.assertNotIn("日期时间", random_tags)
|
||||
self.assertNotIn("图形", char_tags)
|
||||
|
||||
def test_parameter_name_does_not_create_unrelated_time_domain(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="futureValue",
|
||||
signature=(
|
||||
"futureValue(rate, n_periods, payment, present_value, payment_time)"
|
||||
),
|
||||
kind="function",
|
||||
scope="builtin",
|
||||
page="builtin/numeric.md",
|
||||
summary="返回一项投资的未来值",
|
||||
)
|
||||
tags = self.module.derive_tags(
|
||||
entry,
|
||||
self.module.PageContext("Builtin / 数值计算", entry.page),
|
||||
)
|
||||
|
||||
self.assertNotIn("日期时间", tags)
|
||||
|
||||
def test_hash_index_is_not_tagged_as_cryptographic_digest(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="THashedStringList",
|
||||
signature="THashedStringList",
|
||||
kind="class",
|
||||
scope="builtin",
|
||||
page="builtin/object/thashedstringlist.md",
|
||||
summary="使用哈希索引加速查找的字符串列表内置对象",
|
||||
)
|
||||
tags = self.module.derive_tags(
|
||||
entry,
|
||||
self.module.PageContext(
|
||||
"Object / THashedStringList 字符串列表", entry.page
|
||||
),
|
||||
)
|
||||
|
||||
self.assertIn("哈希索引", tags)
|
||||
self.assertIn("hash", tags)
|
||||
self.assertNotIn("摘要", tags)
|
||||
self.assertNotIn("digest", tags)
|
||||
|
||||
def test_business_statistics_verb_does_not_add_probability_domain(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="fundCount",
|
||||
signature="fundCount(report_date)",
|
||||
kind="function",
|
||||
scope="dotnet",
|
||||
page="dotnet/financial/fund-holdings.md",
|
||||
summary="统计指定报告期的基金数量",
|
||||
)
|
||||
tags = self.module.derive_tags(
|
||||
entry,
|
||||
self.module.PageContext("金融 / 基金 / 持仓", entry.page),
|
||||
)
|
||||
|
||||
self.assertNotIn("概率", tags)
|
||||
self.assertNotIn("statistics", tags)
|
||||
self.assertNotIn("统计指定报告期的基金数量", tags)
|
||||
self.assertNotIn("证券", tags)
|
||||
|
||||
def test_taxonomy_tags_remove_generic_suffixes(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="weightedMedian",
|
||||
signature="weightedMedian(data, weights)",
|
||||
kind="function",
|
||||
scope="dotnet",
|
||||
page="dotnet/financial/weighted-statistics.md",
|
||||
summary="计算加权中位数",
|
||||
)
|
||||
tags = self.module.derive_tags(
|
||||
entry,
|
||||
self.module.PageContext(
|
||||
"基础算法常见加权统计量及其实现 / 时间相关函数",
|
||||
entry.page,
|
||||
),
|
||||
)
|
||||
|
||||
self.assertIn("基础算法常见加权统计量", tags)
|
||||
self.assertIn("时间", tags)
|
||||
self.assertNotIn("基础算法常见加权统计量及其实现", tags)
|
||||
self.assertNotIn("时间相关函数", tags)
|
||||
|
||||
def test_taxonomy_tags_drop_parenthetical_explanations_and_punctuation(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="close",
|
||||
signature="close()",
|
||||
kind="function",
|
||||
scope="dotnet",
|
||||
page="dotnet/datawarehouse/after_market.md",
|
||||
summary="收盘价，与系统参数股票、时间、周期和复权相关",
|
||||
)
|
||||
tags = self.module.derive_tags(
|
||||
entry,
|
||||
self.module.PageContext(
|
||||
"数据仓库 / 盘后相关(在当日可用于盘中)", entry.page
|
||||
),
|
||||
)
|
||||
|
||||
self.assertIn("数据仓库", tags)
|
||||
self.assertIn("盘后", tags)
|
||||
self.assertNotIn("(", tags)
|
||||
self.assertNotIn(")", tags)
|
||||
self.assertNotIn("盘后相关在当日可用于盘中", tags)
|
||||
self.assertNotIn("关闭", tags)
|
||||
self.assertNotIn("close", tags)
|
||||
|
||||
def test_name_intent_does_not_override_an_explicit_action_clause(self):
|
||||
entry = self.module.EntryContext(
|
||||
name="openEndFundFilterByFundIndex",
|
||||
signature="openEndFundFilterByFundIndex(fund_inds)",
|
||||
kind="function",
|
||||
scope="dotnet",
|
||||
page="dotnet/financial/fund-pool.md",
|
||||
summary="根据基金指数获取成分基金，并对基金进行筛选",
|
||||
)
|
||||
tags = self.module.derive_tags(
|
||||
entry,
|
||||
self.module.PageContext("金融 / 基金 / 基金池", entry.page),
|
||||
)
|
||||
|
||||
self.assertNotIn("打开", tags)
|
||||
self.assertNotIn("open", tags)
|
||||
|
||||
|
||||
class MarkdownEnrichmentTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.module = load_script()
|
||||
|
||||
def test_inserts_tags_after_description_without_touching_code(self):
|
||||
text = (
|
||||
"# Builtin / 数组\n\n"
|
||||
"## `arrDropDuplicate(data)`\n\n"
|
||||
"声明：function\n\n"
|
||||
"删除数组中的重复元素。\n\n"
|
||||
"| 参数 | 类型 | 说明 |\n"
|
||||
"| --- | --- | --- |\n"
|
||||
"| `data` | array | 输入数组 |\n\n"
|
||||
"返回：array\n\n"
|
||||
"### 示例\n\n"
|
||||
"```tsl\nreturn arrDropDuplicate(array(1, 1));\n```\n"
|
||||
)
|
||||
|
||||
result, audit = self.module.enrich_markdown(
|
||||
text, "builtin/array.md", self.module.SourceCatalog.empty()
|
||||
)
|
||||
repeated, repeated_audit = self.module.enrich_markdown(
|
||||
result, "builtin/array.md", self.module.SourceCatalog.empty()
|
||||
)
|
||||
|
||||
self.assertIn("删除数组中的重复元素\n\n<!-- tags:", result)
|
||||
self.assertIn("return arrDropDuplicate(array(1, 1));", result)
|
||||
self.assertEqual(result, repeated)
|
||||
self.assertEqual(1, audit.changed_descriptions)
|
||||
self.assertEqual(1, audit.changed_tags)
|
||||
self.assertEqual(0, repeated_audit.changed_descriptions)
|
||||
self.assertEqual(0, repeated_audit.changed_tags)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -164,19 +164,21 @@ class DocGenCliTest(unittest.TestCase):
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertTrue(self.generated("base/my_functions").is_file())
|
||||
|
||||
def test_legacy_positional_input_remains_supported(self):
|
||||
def test_positional_input_is_rejected(self):
|
||||
result = self.run_raw_cli(self.input)
|
||||
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertTrue(self.generated("base/my_functions").is_file())
|
||||
self.assertEqual(2, result.returncode)
|
||||
self.assertIn(f"无法识别的参数： {self.input}", result.stderr)
|
||||
self.assertFalse(self.generated("base/my_functions").exists())
|
||||
|
||||
def test_help_marks_legacy_input_as_deprecated(self):
|
||||
def test_help_only_documents_named_input_options(self):
|
||||
result = self.run_raw_cli("--help")
|
||||
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertIn("--file INPUT_FILE", result.stdout)
|
||||
self.assertIn("--dir INPUT_DIR", result.stdout)
|
||||
self.assertIn("已废弃，请使用 --file", result.stdout)
|
||||
self.assertNotIn("positional arguments:", result.stdout)
|
||||
self.assertNotIn("已废弃", result.stdout)
|
||||
|
||||
def test_invalid_format_error_is_fully_chinese(self):
|
||||
result = self.run_raw_cli("--file", self.input, "--format", "xml")
|
||||
@@ -192,7 +194,6 @@ class DocGenCliTest(unittest.TestCase):
|
||||
cases = (
|
||||
(),
|
||||
("--file", self.input, "--dir", self.root),
|
||||
(self.input, "--file", self.input),
|
||||
)
|
||||
for args in cases:
|
||||
with self.subTest(args=args):
|
||||
@@ -200,7 +201,7 @@ class DocGenCliTest(unittest.TestCase):
|
||||
|
||||
self.assertEqual(2, result.returncode)
|
||||
self.assertIn(
|
||||
"必须且只能指定一种输入方式：INPUT_FILE、--file 或 --dir",
|
||||
"必须且只能指定一种输入方式：--file 或 --dir",
|
||||
result.stderr,
|
||||
)
|
||||
|
||||
@@ -399,6 +400,49 @@ class DocGenCliTest(unittest.TestCase):
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertTrue(output.is_file())
|
||||
|
||||
def test_root_writes_under_explicit_project_root(self):
|
||||
output_root = self.root / "explicit-root"
|
||||
|
||||
result = self.run_cli("--root", str(output_root))
|
||||
|
||||
output = (
|
||||
output_root
|
||||
/ "skills"
|
||||
/ "tsl-api-reference"
|
||||
/ "references"
|
||||
/ "codegen"
|
||||
/ "project"
|
||||
/ "base"
|
||||
/ "my_functions.md"
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertTrue(output.is_file())
|
||||
self.assertFalse(self.generated("base/my_functions").exists())
|
||||
|
||||
def test_relative_root_is_resolved_from_current_working_directory(self):
|
||||
result = self.run_cli("--root", "explicit-root")
|
||||
|
||||
output = (
|
||||
self.root
|
||||
/ "explicit-root"
|
||||
/ "skills"
|
||||
/ "tsl-api-reference"
|
||||
/ "references"
|
||||
/ "codegen"
|
||||
/ "project"
|
||||
/ "base"
|
||||
/ "my_functions.md"
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertTrue(output.is_file())
|
||||
|
||||
def test_help_documents_root_option(self):
|
||||
result = self.run_raw_cli("--help")
|
||||
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertIn("--root ROOT_DIR", result.stdout)
|
||||
self.assertIn("包含 skills 目录的项目根目录", result.stdout)
|
||||
|
||||
def test_output_option_is_rejected(self):
|
||||
result = self.run_cli("--output", str(self.root / "out.md"))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
@@ -498,12 +542,19 @@ class DocGenCliTest(unittest.TestCase):
|
||||
output.write_text("原内容\n", encoding="utf-8")
|
||||
bin_dir = self.root / "bin"
|
||||
bin_dir.mkdir()
|
||||
fake_npx = bin_dir / "npx"
|
||||
fake_npx.write_text(
|
||||
"#!/bin/sh\necho formatter-failed >&2\nexit 9\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_npx.chmod(0o755)
|
||||
if os.name == "nt":
|
||||
fake_npx = bin_dir / "npx.cmd"
|
||||
fake_npx.write_text(
|
||||
"@echo off\r\necho formatter-failed 1>&2\r\nexit /b 9\r\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
else:
|
||||
fake_npx = bin_dir / "npx"
|
||||
fake_npx.write_text(
|
||||
"#!/bin/sh\necho formatter-failed >&2\nexit 9\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_npx.chmod(0o755)
|
||||
env = os.environ.copy()
|
||||
env["PATH"] = str(bin_dir)
|
||||
|
||||
@@ -884,6 +935,46 @@ class DocGenCliTest(unittest.TestCase):
|
||||
self.assertIn("范例02：多行输出", text)
|
||||
self.assertIn("// 输出：\n// first\n// second", text)
|
||||
|
||||
def test_singular_example_is_rejected_without_overwrite(self):
|
||||
self.write_input(
|
||||
{
|
||||
"module": "项目 / 旧示例格式",
|
||||
"path": "base/singular_example",
|
||||
"declarations": [
|
||||
{
|
||||
"kind": "function",
|
||||
"name": "demo",
|
||||
"signature": "demo()",
|
||||
"desc": "示例函数。",
|
||||
"returns": "nil",
|
||||
"example": "return demo();",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
self.assert_rejected_without_overwrite(
|
||||
"base/singular_example", "存在未知字段：example"
|
||||
)
|
||||
|
||||
def test_singular_method_example_is_rejected_without_overwrite(self):
|
||||
self.write_class_member_input(
|
||||
{
|
||||
"kind": "method",
|
||||
"name": "Run",
|
||||
"visibility": "public",
|
||||
"binding": "instance",
|
||||
"signature": "Run()",
|
||||
"desc": "运行。",
|
||||
"example": "return self.Run();",
|
||||
},
|
||||
"base/singular_method_example",
|
||||
)
|
||||
|
||||
self.assert_rejected_without_overwrite(
|
||||
"base/singular_method_example", "存在未知字段：example"
|
||||
)
|
||||
|
||||
def test_top_level_enum_sections_are_plain_text_not_emphasis_headings(self):
|
||||
self.write_input(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user