✨ feat(tsl-syntax): add section lookup core

skills/tsl-syntax-reference/scripts/lookup.py

@@ -0,0 +1,434 @@
+#!/usr/bin/env python3
+import argparse
+import re
+import sys
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import unquote
+
+
+SKILL_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_REFERENCES_DIR = SKILL_ROOT / "references"
+HEADING_RE = re.compile(r"^(#{1,6})(?!#)\s+(.+?)\s*$")
+FENCE_RE = re.compile(r"^```([^`]*)$")
+MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
+ASCII_TOKEN_RE = re.compile(r"[a-z_][a-z0-9_.$:+-]*", re.IGNORECASE)
+CHINESE_RUN_RE = re.compile(r"[\u3400-\u9fff]+")
+IDENTITY_PREFIX = "代码块身份："
+ALLOWED_IDENTITIES = {
+    "可直接照写示例",
+    "反例 / 不可照写",
+    "输出片段",
+    "配置片段 / 概念骨架",
+    "仅服务端可执行示例",
+}
+ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断")
+EXCLUDED_REFERENCE_FILES = {"index.md"}
+
+TOKEN_IN_HEADING_SCORE = 12
+TOKEN_IN_BODY_SCORE = 3
+EXACT_QUERY_SCORE = 20
+PAGE_NAME_SCORE = 5
+DIRECT_EXAMPLE_BOOST = 8
+PITFALL_PAGE_BOOST = 18
+COUNTEREXAMPLE_BOOST = 8
+EXACT_ERROR_BOOST = 14
+MIXED_QUERY_MIN_SCORE = 10
+
+
+@dataclass(frozen=True)
+class Section:
+    id: str
+    page: Path
+    page_title: str
+    heading_path: tuple[str, ...]
+    body: str
+    identities: tuple[str, ...]
+    searchable_text: str
+
+
+@dataclass(frozen=True)
+class ValidationProblem:
+    page: Path
+    line: int
+    message: str
+
+
+@dataclass(frozen=True)
+class QueryMatch:
+    section: Section
+    score: int
+
+
+@dataclass
+class QueryResult:
+    query: str
+    mode: str
+    matches: list[QueryMatch]
+    prelude: list[Section]
+
+
+def normalize(value: str) -> str:
+    return unicodedata.normalize("NFKC", value).casefold()
+
+
+def _slug(value: str) -> str:
+    normalized = normalize(value)
+    slug = re.sub(r"[^\w]+", "-", normalized, flags=re.UNICODE).strip("-_")
+    return slug or "section"
+
+
+def section_id(relative_page: Path | str, heading_path: tuple[str, ...]) -> str:
+    page = Path(relative_page)
+    parts = [_slug(page.with_suffix("").as_posix()), *(_slug(item) for item in heading_path)]
+    return "--".join(parts)
+
+
+def _reference_pages(references_dir: Path) -> list[Path]:
+    return [
+        page
+        for page in sorted(references_dir.glob("*.md"), key=lambda item: item.name)
+        if page.name not in EXCLUDED_REFERENCE_FILES
+    ]
+
+
+def _heading_records(lines: list[str]) -> tuple[str, list[tuple[int, int, str]]]:
+    page_title = ""
+    records: list[tuple[int, int, str]] = []
+    in_fence = False
+    for index, line in enumerate(lines):
+        if FENCE_RE.match(line.rstrip("\r\n")):
+            in_fence = not in_fence
+            continue
+        if in_fence:
+            continue
+        match = HEADING_RE.match(line.rstrip("\r\n"))
+        if not match:
+            continue
+        level = len(match.group(1))
+        title = match.group(2)
+        if level == 1 and not page_title:
+            page_title = title
+        elif level in (2, 3):
+            records.append((index, level, title))
+    return page_title, records
+
+
+def _identities(body: str) -> tuple[str, ...]:
+    identities: list[str] = []
+    lines = body.splitlines()
+    in_fence = False
+    for index, line in enumerate(lines):
+        if not FENCE_RE.match(line):
+            continue
+        if not in_fence:
+            previous = index - 1
+            while previous >= 0 and not lines[previous].strip():
+                previous -= 1
+            if previous >= 0 and lines[previous].strip().startswith(IDENTITY_PREFIX):
+                identities.append(lines[previous].strip()[len(IDENTITY_PREFIX) :].strip())
+        in_fence = not in_fence
+    return tuple(identities)
+
+
+def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section]:
+    sections: list[Section] = []
+    id_counts: dict[str, int] = {}
+    for page in _reference_pages(Path(references_dir)):
+        text = page.read_text(encoding="utf-8")
+        lines = text.splitlines(keepends=True)
+        page_title, headings = _heading_records(lines)
+        h2_title = ""
+        for position, (start, level, title) in enumerate(headings):
+            if level == 2:
+                h2_title = title
+                heading_path = (title,)
+            else:
+                heading_path = (h2_title, title) if h2_title else (title,)
+            end = len(lines)
+            for next_start, next_level, _ in headings[position + 1 :]:
+                if next_level <= level:
+                    end = next_start
+                    break
+            body = "".join(lines[start:end])
+            base_id = section_id(page.relative_to(references_dir), heading_path)
+            id_counts[base_id] = id_counts.get(base_id, 0) + 1
+            unique_id = base_id
+            if id_counts[base_id] > 1:
+                unique_id = f"{base_id}-{id_counts[base_id]}"
+            searchable_text = normalize(
+                "\n".join((page.stem, page_title, *heading_path, body))
+            )
+            sections.append(
+                Section(
+                    id=unique_id,
+                    page=page,
+                    page_title=page_title,
+                    heading_path=heading_path,
+                    body=body,
+                    identities=_identities(body),
+                    searchable_text=searchable_text,
+                )
+            )
+    return sections
+
+
+def _identity_problems(page: Path, lines: list[str]) -> list[ValidationProblem]:
+    problems: list[ValidationProblem] = []
+    for index, line in enumerate(lines, start=1):
+        stripped = line.strip()
+        if stripped.startswith(IDENTITY_PREFIX):
+            identity = stripped[len(IDENTITY_PREFIX) :].strip()
+            if identity not in ALLOWED_IDENTITIES:
+                problems.append(ValidationProblem(page, index, f"未知身份：{identity}"))
+    in_fence = False
+    for index, line in enumerate(lines):
+        if not FENCE_RE.match(line):
+            continue
+        if in_fence:
+            in_fence = False
+            continue
+        previous = index - 1
+        identities: list[tuple[int, str]] = []
+        while previous >= 0:
+            stripped = lines[previous].strip()
+            if FENCE_RE.match(stripped) or HEADING_RE.match(stripped):
+                break
+            if stripped.startswith(IDENTITY_PREFIX):
+                identity = stripped[len(IDENTITY_PREFIX) :].strip()
+                identities.append((previous, identity))
+            previous -= 1
+        recognized = [identity for _, identity in identities if identity in ALLOWED_IDENTITIES]
+        if len(recognized) != 1 or len(identities) != 1:
+            problems.append(
+                ValidationProblem(page, index + 1, "每个代码围栏必须关联恰好一个代码块身份")
+            )
+        in_fence = True
+    if in_fence:
+        problems.append(ValidationProblem(page, len(lines), "代码围栏未闭合"))
+    return problems
+
+
+def _local_link_problems(
+    page: Path, text: str, references_dir: Path
+) -> list[ValidationProblem]:
+    problems: list[ValidationProblem] = []
+    without_fences = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
+    searchable_markdown = re.sub(r"`[^`\n]*`", "", without_fences)
+    for match in MARKDOWN_LINK_RE.finditer(searchable_markdown):
+        target = match.group(1).strip().split(maxsplit=1)[0].strip("<>")
+        if target.startswith(("#", "http://", "https://", "mailto:")):
+            continue
+        target_path = unquote(target.split("#", 1)[0].replace("\\", "/"))
+        resolved = (page.parent / target_path).resolve()
+        try:
+            resolved.relative_to(references_dir.resolve())
+        except ValueError:
+            exists = False
+        else:
+            exists = resolved.is_file()
+        if not exists:
+            line = searchable_markdown.count("\n", 0, match.start()) + 1
+            problems.append(ValidationProblem(page, line, f"本地链接不存在：{target}"))
+    return problems
+
+
+def validate_references(
+    references_dir: Path = DEFAULT_REFERENCES_DIR,
+) -> list[ValidationProblem]:
+    references_dir = Path(references_dir)
+    problems: list[ValidationProblem] = []
+    index_page = references_dir / "index.md"
+    if index_page.exists():
+        problems.append(ValidationProblem(index_page, 1, "references 中不得保留 index.md"))
+    for page in sorted(references_dir.glob("*.md"), key=lambda item: item.name):
+        text = page.read_text(encoding="utf-8")
+        lines = text.splitlines()
+        problems.extend(_identity_problems(page, lines))
+        problems.extend(_local_link_problems(page, text, references_dir))
+        for phrase in ROUTER_PHRASES:
+            for index, line in enumerate(lines, start=1):
+                if phrase in line:
+                    problems.append(
+                        ValidationProblem(page, index, f"包含人工路由协议：{phrase}")
+                    )
+    sections = load_sections(references_dir)
+    ids: dict[str, Section] = {}
+    for section in sections:
+        if section.id in ids:
+            problems.append(ValidationProblem(section.page, 1, f"重复 section ID：{section.id}"))
+        ids[section.id] = section
+    return problems
+
+
+def query_tokens(text: str) -> set[str]:
+    normalized = normalize(text)
+    tokens = set(ASCII_TOKEN_RE.findall(normalized))
+    for run in CHINESE_RUN_RE.findall(normalized):
+        tokens.add(run)
+        tokens.update(run[index : index + 2] for index in range(len(run) - 1))
+    return {token for token in tokens if token.strip()}
+
+
+def _score_section(section: Section, query: str, mode: str) -> int:
+    normalized_query = normalize(query).strip()
+    tokens = query_tokens(query)
+    heading_text = normalize("\n".join((section.page_title, *section.heading_path)))
+    score = 0
+    for token in tokens:
+        if token in heading_text:
+            score += TOKEN_IN_HEADING_SCORE
+        elif token in section.searchable_text:
+            score += TOKEN_IN_BODY_SCORE
+    if normalized_query and normalized_query in section.searchable_text:
+        score += EXACT_QUERY_SCORE
+    if any(token in normalize(section.page.stem) for token in tokens):
+        score += PAGE_NAME_SCORE
+    if mode == "write" and "可直接照写示例" in section.identities:
+        score += DIRECT_EXAMPLE_BOOST
+    if mode == "diagnose":
+        if section.page.name == "11_pitfalls.md":
+            score += PITFALL_PAGE_BOOST
+        if "反例 / 不可照写" in section.identities:
+            score += COUNTEREXAMPLE_BOOST
+        if normalized_query and normalized_query in section.searchable_text:
+            score += EXACT_ERROR_BOOST
+    return score
+
+
+def _write_prelude(sections: list[Section]) -> list[Section]:
+    preferred = [
+        ("02_core_model.md", "文件模型核心规则"),
+        ("01_quickstart.md", "语言核心事实速查"),
+    ]
+    prelude: list[Section] = []
+    for page_name, heading in preferred:
+        match = next(
+            (
+                section
+                for section in sections
+                if section.page.name == page_name and heading in section.heading_path
+            ),
+            None,
+        )
+        if match is not None:
+            prelude.append(match)
+    return prelude
+
+
+def query_sections(
+    query: str,
+    mode: str,
+    limit: int = 5,
+    references_dir: Path = DEFAULT_REFERENCES_DIR,
+) -> QueryResult:
+    if mode not in {"write", "diagnose", "explain"}:
+        raise ValueError(f"unsupported mode: {mode}")
+    if not 1 <= limit <= 10:
+        raise ValueError("limit must be between 1 and 10")
+    sections = load_sections(references_dir)
+    ascii_tokens = {
+        token for token in query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token)
+    }
+    if ascii_tokens:
+        sections = [
+            section
+            for section in sections
+            if any(token in section.searchable_text for token in ascii_tokens)
+        ]
+    ranked = [
+        QueryMatch(section, _score_section(section, query, mode)) for section in sections
+    ]
+    has_chinese = bool(CHINESE_RUN_RE.search(normalize(query)))
+    minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
+    ranked = [match for match in ranked if match.score >= minimum_score]
+    ranked.sort(key=lambda match: (-match.score, match.section.page.as_posix(), match.section.id))
+    matches = ranked[:limit]
+    prelude = _write_prelude(sections) if mode == "write" else []
+    match_ids = {match.section.id for match in matches}
+    prelude = [section for section in prelude if section.id not in match_ids]
+    return QueryResult(query=query, mode=mode, matches=matches, prelude=prelude)
+
+
+def render_result(result: QueryResult) -> str:
+    lines = [
+        "# TSL Syntax Lookup",
+        "",
+        f"Mode: `{result.mode}`",
+        f"Query: {result.query}",
+    ]
+    if result.prelude:
+        lines.extend(["", "## Required context"])
+        for section in result.prelude:
+            lines.extend(["", f"Section ID: `{section.id}`", "", section.body.rstrip()])
+    for index, match in enumerate(result.matches, start=1):
+        lines.extend(
+            [
+                "",
+                f"## Match {index}",
+                "",
+                f"Score: {match.score}",
+                f"Section ID: `{match.section.id}`",
+                f"Source: `{match.section.page.as_posix()}`",
+                "",
+                match.section.body.rstrip(),
+            ]
+        )
+    lines.extend(["", "## Narrower section IDs"])
+    lines.extend(f"- `{match.section.id}`" for match in result.matches)
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _configure_utf8() -> None:
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            reconfigure(encoding="utf-8")
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Search TSL syntax reference sections")
+    action = parser.add_mutually_exclusive_group(required=True)
+    action.add_argument("--query")
+    action.add_argument("--section")
+    action.add_argument("--check", action="store_true")
+    parser.add_argument("--mode", choices=("write", "diagnose", "explain"))
+    parser.add_argument("--limit", type=int, default=5)
+    parser.add_argument("--references-dir", type=Path, default=DEFAULT_REFERENCES_DIR)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    _configure_utf8()
+    parser = _parser()
+    args = parser.parse_args(argv)
+    if not 1 <= args.limit <= 10:
+        parser.error("--limit must be between 1 and 10")
+    if args.query is not None and args.mode is None:
+        parser.error("--mode is required with --query")
+    if args.check:
+        problems = validate_references(args.references_dir)
+        for problem in problems:
+            print(f"{problem.page}:{problem.line}: {problem.message}", file=sys.stderr)
+        return 1 if problems else 0
+    if args.section is not None:
+        section = next(
+            (item for item in load_sections(args.references_dir) if item.id == args.section),
+            None,
+        )
+        if section is None:
+            print(f"section not found: {args.section}", file=sys.stderr)
+            return 2
+        result = QueryResult("", "section", [QueryMatch(section, 0)], [])
+    else:
+        result = query_sections(args.query, args.mode, args.limit, args.references_dir)
+        if not result.matches:
+            print("no matching sections", file=sys.stderr)
+            return 2
+    print(render_result(result), end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

test/test_tsl_syntax_lookup.py

@@ -0,0 +1,122 @@
+import importlib.util
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "skills" / "tsl-syntax-reference" / "scripts" / "lookup.py"
+
+spec = importlib.util.spec_from_file_location("tsl_syntax_lookup", SCRIPT)
+lookup = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(lookup)
+
+
+class TslSyntaxLookupTests(unittest.TestCase):
+    def test_parser_creates_unique_h2_h3_section_ids(self):
+        sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR)
+        ids = [section.id for section in sections]
+        self.assertEqual(len(ids), len(set(ids)))
+        self.assertTrue(any("05_functions_and_calls" in value for value in ids))
+
+    def test_write_mode_includes_file_model_and_direct_example(self):
+        result = lookup.query_sections("写函数 命名参数", "write", limit=4)
+        rendered = lookup.render_result(result)
+        self.assertIn("文件模型", rendered)
+        self.assertIn("代码块身份：可直接照写示例", rendered)
+
+    def test_diagnose_mode_prioritizes_invalid_statement_pitfall(self):
+        result = lookup.query_sections("invalid statement 声明区", "diagnose", limit=4)
+        self.assertIn("11_pitfalls.md", result.matches[0].section.page.as_posix())
+
+    def test_explain_mode_does_not_force_write_prelude(self):
+        result = lookup.query_sections("数组下标", "explain", limit=3)
+        self.assertEqual(result.prelude, [])
+
+    def test_no_match_returns_exit_code_two(self):
+        completed = subprocess.run(
+            [
+                sys.executable,
+                str(SCRIPT),
+                "--query",
+                "不存在的孤立语法词xyz",
+                "--mode",
+                "explain",
+            ],
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(completed.returncode, 2)
+
+    def test_check_rejects_unknown_code_block_identity(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            (references / "sample.md").write_text(
+                "# Sample\n\n"
+                "## Example\n\n"
+                "代码块身份：未知身份\n\n"
+                "```tsl\n"
+                "return 1;\n"
+                "```\n",
+                encoding="utf-8",
+                newline="\n",
+            )
+
+            problems = lookup.validate_references(references)
+
+        self.assertTrue(
+            any("未知身份" in problem.message for problem in problems), problems
+        )
+
+    def test_check_accumulates_fence_identity_and_link_problems(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            (references / "sample.md").write_text(
+                "# Sample\n\n"
+                "## Example\n\n"
+                "[missing](missing.md)\n\n"
+                "代码块身份：输出片段\n"
+                "代码块身份：可直接照写示例\n\n"
+                "```tsl\n"
+                "return 1;\n",
+                encoding="utf-8",
+                newline="\n",
+            )
+
+            problems = lookup.validate_references(references)
+
+        messages = "\n".join(problem.message for problem in problems)
+        self.assertIn("本地链接不存在", messages)
+        self.assertIn("代码围栏未闭合", messages)
+        self.assertIn("恰好一个代码块身份", messages)
+
+    def test_check_ignores_markdown_link_shapes_inside_inline_code(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            (references / "sample.md").write_text(
+                "# Sample\n\n"
+                "## Operators\n\n"
+                "Use `function operator[](index);`.\n",
+                encoding="utf-8",
+                newline="\n",
+            )
+
+            problems = lookup.validate_references(references)
+
+        self.assertEqual(problems, [])
+
+    def test_natural_chinese_query_finds_object_creation(self):
+        result = lookup.query_sections("写一个类并创建对象", "write", limit=5)
+        sources = "\n".join(match.section.id for match in result.matches)
+        self.assertIn("08_objects_and_classes", sources)
+
+    def test_tsl_identifier_is_preserved(self):
+        result = lookup.query_sections("varByRef 命名参数", "explain", limit=5)
+        rendered = lookup.render_result(result)
+        self.assertRegex(rendered, r"(?i)varByRef")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_tsl_syntax_reference_skill.py

@@ -36,7 +36,6 @@ TOPIC_REFERENCES = {
     "23_fmarray.md",
     "24_object_overloads_and_iteration.md",
 }
-EXPECTED_REFERENCES = {"index.md", *TOPIC_REFERENCES}
 MARKDOWN_LINK = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
 
 
@@ -83,9 +82,20 @@ class TslSyntaxReferenceSkillStructureTest(unittest.TestCase):
         self.assertRegex(description, r"(?i)\bTSF\b")
         self.assertRegex(description, r"写|编写|修改|审查|解释|语法错误")
 
-    def test_reference_inventory_is_exact(self) -> None:
+    def test_reference_inventory_contains_topics_without_router(self) -> None:
-        actual = {path.name for path in REFERENCES_DIR.iterdir()}
+        actual = {path.name for path in REFERENCES_DIR.iterdir() if path.is_file()}
-        self.assertEqual(actual, EXPECTED_REFERENCES)
+        self.assertEqual(actual, TOPIC_REFERENCES)
+        self.assertFalse((REFERENCES_DIR / "index.md").exists())
+
+    def test_lookup_script_is_bundled(self) -> None:
+        self.assertTrue((SKILL_DIR / "scripts" / "lookup.py").is_file())
+
+    def test_manual_router_protocol_is_absent(self) -> None:
+        corpus = read_text(SKILL_FILE) + "\n" + "\n".join(
+            read_text(path) for path in REFERENCES_DIR.glob("*.md")
+        )
+        for forbidden in ("路由中心", "选择一个主专题", "候选页继续判断"):
+            self.assertNotIn(forbidden, corpus)
 
     def test_json_router_is_absent(self) -> None:
         self.assertFalse((REFERENCES_DIR / "00_agent_index.json").exists())