From 5e31a7ccf610f25ade77e401fe3a2b52a5feb7b5 Mon Sep 17 00:00:00 2001 From: csh Date: Sat, 11 Jul 2026 12:20:58 +0800 Subject: [PATCH] :sparkles: feat(tsl-syntax): add section lookup core --- skills/tsl-syntax-reference/scripts/lookup.py | 434 ++++++++++++++++++ test/test_tsl_syntax_lookup.py | 122 +++++ test/test_tsl_syntax_reference_skill.py | 18 +- 3 files changed, 570 insertions(+), 4 deletions(-) create mode 100644 skills/tsl-syntax-reference/scripts/lookup.py create mode 100644 test/test_tsl_syntax_lookup.py diff --git a/skills/tsl-syntax-reference/scripts/lookup.py b/skills/tsl-syntax-reference/scripts/lookup.py new file mode 100644 index 00000000..cc492eb0 --- /dev/null +++ b/skills/tsl-syntax-reference/scripts/lookup.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +import argparse +import re +import sys +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import unquote + + +SKILL_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_REFERENCES_DIR = SKILL_ROOT / "references" +HEADING_RE = re.compile(r"^(#{1,6})(?!#)\s+(.+?)\s*$") +FENCE_RE = re.compile(r"^```([^`]*)$") +MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)") +ASCII_TOKEN_RE = re.compile(r"[a-z_][a-z0-9_.$:+-]*", re.IGNORECASE) +CHINESE_RUN_RE = re.compile(r"[\u3400-\u9fff]+") +IDENTITY_PREFIX = "代码块身份：" +ALLOWED_IDENTITIES = { + "可直接照写示例", + "反例 / 不可照写", + "输出片段", + "配置片段 / 概念骨架", + "仅服务端可执行示例", +} +ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断") +EXCLUDED_REFERENCE_FILES = {"index.md"} + +TOKEN_IN_HEADING_SCORE = 12 +TOKEN_IN_BODY_SCORE = 3 +EXACT_QUERY_SCORE = 20 +PAGE_NAME_SCORE = 5 +DIRECT_EXAMPLE_BOOST = 8 +PITFALL_PAGE_BOOST = 18 +COUNTEREXAMPLE_BOOST = 8 +EXACT_ERROR_BOOST = 14 +MIXED_QUERY_MIN_SCORE = 10 + + +@dataclass(frozen=True) +class Section: + id: str + page: Path + page_title: str + heading_path: tuple[str, ...] + body: str + identities: tuple[str, ...] + searchable_text: str + + +@dataclass(frozen=True) +class ValidationProblem: + page: Path + line: int + message: str + + +@dataclass(frozen=True) +class QueryMatch: + section: Section + score: int + + +@dataclass +class QueryResult: + query: str + mode: str + matches: list[QueryMatch] + prelude: list[Section] + + +def normalize(value: str) -> str: + return unicodedata.normalize("NFKC", value).casefold() + + +def _slug(value: str) -> str: + normalized = normalize(value) + slug = re.sub(r"[^\w]+", "-", normalized, flags=re.UNICODE).strip("-_") + return slug or "section" + + +def section_id(relative_page: Path | str, heading_path: tuple[str, ...]) -> str: + page = Path(relative_page) + parts = [_slug(page.with_suffix("").as_posix()), *(_slug(item) for item in heading_path)] + return "--".join(parts) + + +def _reference_pages(references_dir: Path) -> list[Path]: + return [ + page + for page in sorted(references_dir.glob("*.md"), key=lambda item: item.name) + if page.name not in EXCLUDED_REFERENCE_FILES + ] + + +def _heading_records(lines: list[str]) -> tuple[str, list[tuple[int, int, str]]]: + page_title = "" + records: list[tuple[int, int, str]] = [] + in_fence = False + for index, line in enumerate(lines): + if FENCE_RE.match(line.rstrip("\r\n")): + in_fence = not in_fence + continue + if in_fence: + continue + match = HEADING_RE.match(line.rstrip("\r\n")) + if not match: + continue + level = len(match.group(1)) + title = match.group(2) + if level == 1 and not page_title: + page_title = title + elif level in (2, 3): + records.append((index, level, title)) + return page_title, records + + +def _identities(body: str) -> tuple[str, ...]: + identities: list[str] = [] + lines = body.splitlines() + in_fence = False + for index, line in enumerate(lines): + if not FENCE_RE.match(line): + continue + if not in_fence: + previous = index - 1 + while previous >= 0 and not lines[previous].strip(): + previous -= 1 + if previous >= 0 and lines[previous].strip().startswith(IDENTITY_PREFIX): + identities.append(lines[previous].strip()[len(IDENTITY_PREFIX) :].strip()) + in_fence = not in_fence + return tuple(identities) + + +def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section]: + sections: list[Section] = [] + id_counts: dict[str, int] = {} + for page in _reference_pages(Path(references_dir)): + text = page.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + page_title, headings = _heading_records(lines) + h2_title = "" + for position, (start, level, title) in enumerate(headings): + if level == 2: + h2_title = title + heading_path = (title,) + else: + heading_path = (h2_title, title) if h2_title else (title,) + end = len(lines) + for next_start, next_level, _ in headings[position + 1 :]: + if next_level <= level: + end = next_start + break + body = "".join(lines[start:end]) + base_id = section_id(page.relative_to(references_dir), heading_path) + id_counts[base_id] = id_counts.get(base_id, 0) + 1 + unique_id = base_id + if id_counts[base_id] > 1: + unique_id = f"{base_id}-{id_counts[base_id]}" + searchable_text = normalize( + "\n".join((page.stem, page_title, *heading_path, body)) + ) + sections.append( + Section( + id=unique_id, + page=page, + page_title=page_title, + heading_path=heading_path, + body=body, + identities=_identities(body), + searchable_text=searchable_text, + ) + ) + return sections + + +def _identity_problems(page: Path, lines: list[str]) -> list[ValidationProblem]: + problems: list[ValidationProblem] = [] + for index, line in enumerate(lines, start=1): + stripped = line.strip() + if stripped.startswith(IDENTITY_PREFIX): + identity = stripped[len(IDENTITY_PREFIX) :].strip() + if identity not in ALLOWED_IDENTITIES: + problems.append(ValidationProblem(page, index, f"未知身份：{identity}")) + in_fence = False + for index, line in enumerate(lines): + if not FENCE_RE.match(line): + continue + if in_fence: + in_fence = False + continue + previous = index - 1 + identities: list[tuple[int, str]] = [] + while previous >= 0: + stripped = lines[previous].strip() + if FENCE_RE.match(stripped) or HEADING_RE.match(stripped): + break + if stripped.startswith(IDENTITY_PREFIX): + identity = stripped[len(IDENTITY_PREFIX) :].strip() + identities.append((previous, identity)) + previous -= 1 + recognized = [identity for _, identity in identities if identity in ALLOWED_IDENTITIES] + if len(recognized) != 1 or len(identities) != 1: + problems.append( + ValidationProblem(page, index + 1, "每个代码围栏必须关联恰好一个代码块身份") + ) + in_fence = True + if in_fence: + problems.append(ValidationProblem(page, len(lines), "代码围栏未闭合")) + return problems + + +def _local_link_problems( + page: Path, text: str, references_dir: Path +) -> list[ValidationProblem]: + problems: list[ValidationProblem] = [] + without_fences = re.sub(r"```.*?```", "", text, flags=re.DOTALL) + searchable_markdown = re.sub(r"`[^`\n]*`", "", without_fences) + for match in MARKDOWN_LINK_RE.finditer(searchable_markdown): + target = match.group(1).strip().split(maxsplit=1)[0].strip("<>") + if target.startswith(("#", "http://", "https://", "mailto:")): + continue + target_path = unquote(target.split("#", 1)[0].replace("\\", "/")) + resolved = (page.parent / target_path).resolve() + try: + resolved.relative_to(references_dir.resolve()) + except ValueError: + exists = False + else: + exists = resolved.is_file() + if not exists: + line = searchable_markdown.count("\n", 0, match.start()) + 1 + problems.append(ValidationProblem(page, line, f"本地链接不存在：{target}")) + return problems + + +def validate_references( + references_dir: Path = DEFAULT_REFERENCES_DIR, +) -> list[ValidationProblem]: + references_dir = Path(references_dir) + problems: list[ValidationProblem] = [] + index_page = references_dir / "index.md" + if index_page.exists(): + problems.append(ValidationProblem(index_page, 1, "references 中不得保留 index.md")) + for page in sorted(references_dir.glob("*.md"), key=lambda item: item.name): + text = page.read_text(encoding="utf-8") + lines = text.splitlines() + problems.extend(_identity_problems(page, lines)) + problems.extend(_local_link_problems(page, text, references_dir)) + for phrase in ROUTER_PHRASES: + for index, line in enumerate(lines, start=1): + if phrase in line: + problems.append( + ValidationProblem(page, index, f"包含人工路由协议：{phrase}") + ) + sections = load_sections(references_dir) + ids: dict[str, Section] = {} + for section in sections: + if section.id in ids: + problems.append(ValidationProblem(section.page, 1, f"重复 section ID：{section.id}")) + ids[section.id] = section + return problems + + +def query_tokens(text: str) -> set[str]: + normalized = normalize(text) + tokens = set(ASCII_TOKEN_RE.findall(normalized)) + for run in CHINESE_RUN_RE.findall(normalized): + tokens.add(run) + tokens.update(run[index : index + 2] for index in range(len(run) - 1)) + return {token for token in tokens if token.strip()} + + +def _score_section(section: Section, query: str, mode: str) -> int: + normalized_query = normalize(query).strip() + tokens = query_tokens(query) + heading_text = normalize("\n".join((section.page_title, *section.heading_path))) + score = 0 + for token in tokens: + if token in heading_text: + score += TOKEN_IN_HEADING_SCORE + elif token in section.searchable_text: + score += TOKEN_IN_BODY_SCORE + if normalized_query and normalized_query in section.searchable_text: + score += EXACT_QUERY_SCORE + if any(token in normalize(section.page.stem) for token in tokens): + score += PAGE_NAME_SCORE + if mode == "write" and "可直接照写示例" in section.identities: + score += DIRECT_EXAMPLE_BOOST + if mode == "diagnose": + if section.page.name == "11_pitfalls.md": + score += PITFALL_PAGE_BOOST + if "反例 / 不可照写" in section.identities: + score += COUNTEREXAMPLE_BOOST + if normalized_query and normalized_query in section.searchable_text: + score += EXACT_ERROR_BOOST + return score + + +def _write_prelude(sections: list[Section]) -> list[Section]: + preferred = [ + ("02_core_model.md", "文件模型核心规则"), + ("01_quickstart.md", "语言核心事实速查"), + ] + prelude: list[Section] = [] + for page_name, heading in preferred: + match = next( + ( + section + for section in sections + if section.page.name == page_name and heading in section.heading_path + ), + None, + ) + if match is not None: + prelude.append(match) + return prelude + + +def query_sections( + query: str, + mode: str, + limit: int = 5, + references_dir: Path = DEFAULT_REFERENCES_DIR, +) -> QueryResult: + if mode not in {"write", "diagnose", "explain"}: + raise ValueError(f"unsupported mode: {mode}") + if not 1 <= limit <= 10: + raise ValueError("limit must be between 1 and 10") + sections = load_sections(references_dir) + ascii_tokens = { + token for token in query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token) + } + if ascii_tokens: + sections = [ + section + for section in sections + if any(token in section.searchable_text for token in ascii_tokens) + ] + ranked = [ + QueryMatch(section, _score_section(section, query, mode)) for section in sections + ] + has_chinese = bool(CHINESE_RUN_RE.search(normalize(query))) + minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1 + ranked = [match for match in ranked if match.score >= minimum_score] + ranked.sort(key=lambda match: (-match.score, match.section.page.as_posix(), match.section.id)) + matches = ranked[:limit] + prelude = _write_prelude(sections) if mode == "write" else [] + match_ids = {match.section.id for match in matches} + prelude = [section for section in prelude if section.id not in match_ids] + return QueryResult(query=query, mode=mode, matches=matches, prelude=prelude) + + +def render_result(result: QueryResult) -> str: + lines = [ + "# TSL Syntax Lookup", + "", + f"Mode: `{result.mode}`", + f"Query: {result.query}", + ] + if result.prelude: + lines.extend(["", "## Required context"]) + for section in result.prelude: + lines.extend(["", f"Section ID: `{section.id}`", "", section.body.rstrip()]) + for index, match in enumerate(result.matches, start=1): + lines.extend( + [ + "", + f"## Match {index}", + "", + f"Score: {match.score}", + f"Section ID: `{match.section.id}`", + f"Source: `{match.section.page.as_posix()}`", + "", + match.section.body.rstrip(), + ] + ) + lines.extend(["", "## Narrower section IDs"]) + lines.extend(f"- `{match.section.id}`" for match in result.matches) + return "\n".join(lines).rstrip() + "\n" + + +def _configure_utf8() -> None: + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + reconfigure(encoding="utf-8") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Search TSL syntax reference sections") + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--query") + action.add_argument("--section") + action.add_argument("--check", action="store_true") + parser.add_argument("--mode", choices=("write", "diagnose", "explain")) + parser.add_argument("--limit", type=int, default=5) + parser.add_argument("--references-dir", type=Path, default=DEFAULT_REFERENCES_DIR) + return parser + + +def main(argv: list[str] | None = None) -> int: + _configure_utf8() + parser = _parser() + args = parser.parse_args(argv) + if not 1 <= args.limit <= 10: + parser.error("--limit must be between 1 and 10") + if args.query is not None and args.mode is None: + parser.error("--mode is required with --query") + if args.check: + problems = validate_references(args.references_dir) + for problem in problems: + print(f"{problem.page}:{problem.line}: {problem.message}", file=sys.stderr) + return 1 if problems else 0 + if args.section is not None: + section = next( + (item for item in load_sections(args.references_dir) if item.id == args.section), + None, + ) + if section is None: + print(f"section not found: {args.section}", file=sys.stderr) + return 2 + result = QueryResult("", "section", [QueryMatch(section, 0)], []) + else: + result = query_sections(args.query, args.mode, args.limit, args.references_dir) + if not result.matches: + print("no matching sections", file=sys.stderr) + return 2 + print(render_result(result), end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test_tsl_syntax_lookup.py b/test/test_tsl_syntax_lookup.py new file mode 100644 index 00000000..fb042259 --- /dev/null +++ b/test/test_tsl_syntax_lookup.py @@ -0,0 +1,122 @@ +import importlib.util +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills" / "tsl-syntax-reference" / "scripts" / "lookup.py" + +spec = importlib.util.spec_from_file_location("tsl_syntax_lookup", SCRIPT) +lookup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lookup) + + +class TslSyntaxLookupTests(unittest.TestCase): + def test_parser_creates_unique_h2_h3_section_ids(self): + sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR) + ids = [section.id for section in sections] + self.assertEqual(len(ids), len(set(ids))) + self.assertTrue(any("05_functions_and_calls" in value for value in ids)) + + def test_write_mode_includes_file_model_and_direct_example(self): + result = lookup.query_sections("写函数 命名参数", "write", limit=4) + rendered = lookup.render_result(result) + self.assertIn("文件模型", rendered) + self.assertIn("代码块身份：可直接照写示例", rendered) + + def test_diagnose_mode_prioritizes_invalid_statement_pitfall(self): + result = lookup.query_sections("invalid statement 声明区", "diagnose", limit=4) + self.assertIn("11_pitfalls.md", result.matches[0].section.page.as_posix()) + + def test_explain_mode_does_not_force_write_prelude(self): + result = lookup.query_sections("数组下标", "explain", limit=3) + self.assertEqual(result.prelude, []) + + def test_no_match_returns_exit_code_two(self): + completed = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--query", + "不存在的孤立语法词xyz", + "--mode", + "explain", + ], + capture_output=True, + text=True, + ) + self.assertEqual(completed.returncode, 2) + + def test_check_rejects_unknown_code_block_identity(self): + with tempfile.TemporaryDirectory() as tmp_dir: + references = Path(tmp_dir) + (references / "sample.md").write_text( + "# Sample\n\n" + "## Example\n\n" + "代码块身份：未知身份\n\n" + "```tsl\n" + "return 1;\n" + "```\n", + encoding="utf-8", + newline="\n", + ) + + problems = lookup.validate_references(references) + + self.assertTrue( + any("未知身份" in problem.message for problem in problems), problems + ) + + def test_check_accumulates_fence_identity_and_link_problems(self): + with tempfile.TemporaryDirectory() as tmp_dir: + references = Path(tmp_dir) + (references / "sample.md").write_text( + "# Sample\n\n" + "## Example\n\n" + "[missing](missing.md)\n\n" + "代码块身份：输出片段\n" + "代码块身份：可直接照写示例\n\n" + "```tsl\n" + "return 1;\n", + encoding="utf-8", + newline="\n", + ) + + problems = lookup.validate_references(references) + + messages = "\n".join(problem.message for problem in problems) + self.assertIn("本地链接不存在", messages) + self.assertIn("代码围栏未闭合", messages) + self.assertIn("恰好一个代码块身份", messages) + + def test_check_ignores_markdown_link_shapes_inside_inline_code(self): + with tempfile.TemporaryDirectory() as tmp_dir: + references = Path(tmp_dir) + (references / "sample.md").write_text( + "# Sample\n\n" + "## Operators\n\n" + "Use `function operator[](index);`.\n", + encoding="utf-8", + newline="\n", + ) + + problems = lookup.validate_references(references) + + self.assertEqual(problems, []) + + def test_natural_chinese_query_finds_object_creation(self): + result = lookup.query_sections("写一个类并创建对象", "write", limit=5) + sources = "\n".join(match.section.id for match in result.matches) + self.assertIn("08_objects_and_classes", sources) + + def test_tsl_identifier_is_preserved(self): + result = lookup.query_sections("varByRef 命名参数", "explain", limit=5) + rendered = lookup.render_result(result) + self.assertRegex(rendered, r"(?i)varByRef") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_tsl_syntax_reference_skill.py b/test/test_tsl_syntax_reference_skill.py index 90df20eb..2ec4f8cc 100644 --- a/test/test_tsl_syntax_reference_skill.py +++ b/test/test_tsl_syntax_reference_skill.py @@ -36,7 +36,6 @@ TOPIC_REFERENCES = { "23_fmarray.md", "24_object_overloads_and_iteration.md", } -EXPECTED_REFERENCES = {"index.md", *TOPIC_REFERENCES} MARKDOWN_LINK = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)") @@ -83,9 +82,20 @@ class TslSyntaxReferenceSkillStructureTest(unittest.TestCase): self.assertRegex(description, r"(?i)\bTSF\b") self.assertRegex(description, r"写|编写|修改|审查|解释|语法错误") - def test_reference_inventory_is_exact(self) -> None: - actual = {path.name for path in REFERENCES_DIR.iterdir()} - self.assertEqual(actual, EXPECTED_REFERENCES) + def test_reference_inventory_contains_topics_without_router(self) -> None: + actual = {path.name for path in REFERENCES_DIR.iterdir() if path.is_file()} + self.assertEqual(actual, TOPIC_REFERENCES) + self.assertFalse((REFERENCES_DIR / "index.md").exists()) + + def test_lookup_script_is_bundled(self) -> None: + self.assertTrue((SKILL_DIR / "scripts" / "lookup.py").is_file()) + + def test_manual_router_protocol_is_absent(self) -> None: + corpus = read_text(SKILL_FILE) + "\n" + "\n".join( + read_text(path) for path in REFERENCES_DIR.glob("*.md") + ) + for forbidden in ("路由中心", "选择一个主专题", "候选页继续判断"): + self.assertNotIn(forbidden, corpus) def test_json_router_is_absent(self) -> None: self.assertFalse((REFERENCES_DIR / "00_agent_index.json").exists())