🐛 fix(tsl-syntax): tighten lookup contracts

skills/tsl-syntax-reference/scripts/lookup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import argparse
+import difflib
 import re
 import sys
 import unicodedata
@@ -15,7 +16,10 @@ FENCE_RE = re.compile(r"^```([^`]*)$")
 MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
 ASCII_TOKEN_RE = re.compile(r"[a-z_][a-z0-9_.$:+-]*", re.IGNORECASE)
 CHINESE_RUN_RE = re.compile(r"[\u3400-\u9fff]+")
+INLINE_CODE_RE = re.compile(r"`([^`\n]+)`")
+FENCED_CODE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL)
 IDENTITY_PREFIX = "代码块身份："
+BLOCK_DESCRIPTION_PREFIX = "代码块说明："
 ALLOWED_IDENTITIES = {
     "可直接照写示例",
     "反例 / 不可照写",
@@ -26,10 +30,14 @@ ALLOWED_IDENTITIES = {
 ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断")
 EXCLUDED_REFERENCE_FILES = {"index.md"}
 
-TOKEN_IN_HEADING_SCORE = 12
+HEADING_TOKEN_SCORE = 12
-TOKEN_IN_BODY_SCORE = 3
+HEADING_EXACT_SCORE = 20
-EXACT_QUERY_SCORE = 20
+TERM_TOKEN_SCORE = 10
-PAGE_NAME_SCORE = 5
+TERM_EXACT_SCORE = 16
+PAGE_TITLE_TOKEN_SCORE = 5
+PAGE_TITLE_EXACT_SCORE = 8
+BODY_TOKEN_SCORE = 3
+BODY_EXACT_SCORE = 4
 DIRECT_EXAMPLE_BOOST = 8
 PITFALL_PAGE_BOOST = 18
 COUNTEREXAMPLE_BOOST = 8
@@ -59,6 +67,43 @@ class ValidationProblem:
 class QueryMatch:
     section: Section
     score: int
+    priority: tuple[int, int, int, int, int] = (0, 0, 0, 0, 0)
+
+
+@dataclass(frozen=True)
+class ScoreBreakdown:
+    heading_path: int
+    exact_term: int
+    page_title: int
+    body: int
+    mode_boost: int
+    diagnose_priority: bool = False
+
+    @property
+    def lexical_total(self) -> int:
+        return self.heading_path + self.exact_term + self.page_title + self.body
+
+    @property
+    def total(self) -> int:
+        return self.lexical_total + self.mode_boost
+
+    @property
+    def priority(self) -> tuple[int, int, int, int, int]:
+        if self.diagnose_priority:
+            return (
+                self.mode_boost,
+                self.heading_path,
+                self.exact_term,
+                self.page_title,
+                self.body,
+            )
+        return (
+            self.heading_path,
+            self.exact_term,
+            self.page_title,
+            self.mode_boost,
+            self.body,
+        )
 
 
 @dataclass
@@ -115,6 +160,28 @@ def _heading_records(lines: list[str]) -> tuple[str, list[tuple[int, int, str]]]
     return page_title, records
 
 
+def _associated_identity(lines: list[str], opening_fence: int) -> str | None:
+    previous = opening_fence - 1
+    while previous >= 0 and not lines[previous].strip():
+        previous -= 1
+    while previous >= 0 and lines[previous].strip().startswith(BLOCK_DESCRIPTION_PREFIX):
+        previous -= 1
+        while previous >= 0 and not lines[previous].strip():
+            previous -= 1
+    if previous < 0:
+        return None
+    metadata = lines[previous].strip()
+    if not metadata.startswith(IDENTITY_PREFIX):
+        return None
+    identity = metadata[len(IDENTITY_PREFIX) :].strip()
+    earlier = previous - 1
+    while earlier >= 0 and not lines[earlier].strip():
+        earlier -= 1
+    if earlier >= 0 and lines[earlier].strip().startswith(IDENTITY_PREFIX):
+        return None
+    return identity if identity in ALLOWED_IDENTITIES else None
+
+
 def _identities(body: str) -> tuple[str, ...]:
     identities: list[str] = []
     lines = body.splitlines()
@@ -123,11 +190,9 @@ def _identities(body: str) -> tuple[str, ...]:
         if not FENCE_RE.match(line):
             continue
         if not in_fence:
-            previous = index - 1
+            identity = _associated_identity(lines, index)
-            while previous >= 0 and not lines[previous].strip():
+            if identity is not None:
-                previous -= 1
+                identities.append(identity)
-            if previous >= 0 and lines[previous].strip().startswith(IDENTITY_PREFIX):
-                identities.append(lines[previous].strip()[len(IDENTITY_PREFIX) :].strip())
         in_fence = not in_fence
     return tuple(identities)
 
@@ -189,18 +254,7 @@ def _identity_problems(page: Path, lines: list[str]) -> list[ValidationProblem]:
         if in_fence:
             in_fence = False
             continue
-        previous = index - 1
+        if _associated_identity(lines, index) is None:
-        identities: list[tuple[int, str]] = []
-        while previous >= 0:
-            stripped = lines[previous].strip()
-            if FENCE_RE.match(stripped) or HEADING_RE.match(stripped):
-                break
-            if stripped.startswith(IDENTITY_PREFIX):
-                identity = stripped[len(IDENTITY_PREFIX) :].strip()
-                identities.append((previous, identity))
-            previous -= 1
-        recognized = [identity for _, identity in identities if identity in ALLOWED_IDENTITIES]
-        if len(recognized) != 1 or len(identities) != 1:
             problems.append(
                 ValidationProblem(page, index + 1, "每个代码围栏必须关联恰好一个代码块身份")
             )
@@ -271,36 +325,72 @@ def query_tokens(text: str) -> set[str]:
     return {token for token in tokens if token.strip()}
 
 
-def _lexical_score(section: Section, query: str) -> int:
+def _has_chinese_context(section: Section, query: str) -> bool:
+    runs = CHINESE_RUN_RE.findall(normalize(query))
+    for run in runs:
+        tokens = (
+            [run]
+            if len(run) < 2
+            else [run[index : index + 2] for index in range(len(run) - 1)]
+        )
+        matched = sum(token in section.searchable_text for token in tokens)
+        if matched >= (len(tokens) + 1) // 2:
+            return True
+    return not runs
+
+
+def _code_text(body: str) -> str:
+    inline = INLINE_CODE_RE.findall(body)
+    fenced = FENCED_CODE_RE.findall(body)
+    return normalize("\n".join((*inline, *fenced)))
+
+
+def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
     normalized_query = normalize(query).strip()
     tokens = query_tokens(query)
-    heading_text = normalize("\n".join((section.page_title, *section.heading_path)))
+    heading_text = normalize("\n".join(section.heading_path))
-    score = 0
+    page_title_text = normalize(section.page_title)
+    body_text = normalize(section.body)
+    term_text = _code_text(section.body)
+    heading_score = 0
+    term_score = 0
+    page_title_score = 0
+    body_score = 0
     for token in tokens:
         if token in heading_text:
-            score += TOKEN_IN_HEADING_SCORE
+            heading_score += HEADING_TOKEN_SCORE
-        elif token in section.searchable_text:
+        if token in term_text:
-            score += TOKEN_IN_BODY_SCORE
+            term_score += TERM_TOKEN_SCORE
-    if normalized_query and normalized_query in section.searchable_text:
+        if token in page_title_text:
-        score += EXACT_QUERY_SCORE
+            page_title_score += PAGE_TITLE_TOKEN_SCORE
-    if any(token in normalize(section.page.stem) for token in tokens):
+        if token in body_text:
-        score += PAGE_NAME_SCORE
+            body_score += BODY_TOKEN_SCORE
-    return score
+    if normalized_query and normalized_query in heading_text:
-
+        heading_score += HEADING_EXACT_SCORE
-
+    if normalized_query and normalized_query in term_text:
-def _score_section(section: Section, query: str, mode: str, lexical_score: int) -> int:
+        term_score += TERM_EXACT_SCORE
-    normalized_query = normalize(query).strip()
+    if normalized_query and normalized_query in page_title_text:
-    score = lexical_score
+        page_title_score += PAGE_TITLE_EXACT_SCORE
+    if normalized_query and normalized_query in body_text:
+        body_score += BODY_EXACT_SCORE
+    mode_boost = 0
     if mode == "write" and "可直接照写示例" in section.identities:
-        score += DIRECT_EXAMPLE_BOOST
+        mode_boost += DIRECT_EXAMPLE_BOOST
     if mode == "diagnose":
         if section.page.name == "11_pitfalls.md":
-            score += PITFALL_PAGE_BOOST
+            mode_boost += PITFALL_PAGE_BOOST
         if "反例 / 不可照写" in section.identities:
-            score += COUNTEREXAMPLE_BOOST
+            mode_boost += COUNTEREXAMPLE_BOOST
         if normalized_query and normalized_query in section.searchable_text:
-            score += EXACT_ERROR_BOOST
+            mode_boost += EXACT_ERROR_BOOST
-    return score
+    return ScoreBreakdown(
+        heading_path=heading_score,
+        exact_term=term_score,
+        page_title=page_title_score,
+        body=body_score,
+        mode_boost=mode_boost,
+        diagnose_priority=mode == "diagnose",
+    )
 
 
 def _write_prelude(sections: list[Section]) -> list[Section]:
@@ -348,13 +438,19 @@ def query_sections(
     minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
     ranked: list[QueryMatch] = []
     for section in sections:
-        lexical_score = _lexical_score(section, query)
+        if ascii_tokens and has_chinese and not _has_chinese_context(section, query):
-        if lexical_score < minimum_score:
             continue
-        ranked.append(
+        score = _score_section(section, query, mode)
-            QueryMatch(section, _score_section(section, query, mode, lexical_score))
+        if score.lexical_total < minimum_score:
+            continue
+        ranked.append(QueryMatch(section, score.total, score.priority))
+    ranked.sort(
+        key=lambda match: (
+            *(-value for value in match.priority),
+            match.section.page.as_posix(),
+            match.section.id,
         )
-    ranked.sort(key=lambda match: (-match.score, match.section.page.as_posix(), match.section.id))
+    )
     matches = ranked[:limit]
     prelude = _write_prelude(all_sections) if mode == "write" else []
     match_ids = {match.section.id for match in matches}
@@ -372,7 +468,15 @@ def render_result(result: QueryResult) -> str:
     if result.prelude:
         lines.extend(["", "## Required context"])
         for section in result.prelude:
-            lines.extend(["", f"Section ID: `{section.id}`", "", section.body.rstrip()])
+            lines.extend(
+                [
+                    "",
+                    f"Section ID: `{section.id}`",
+                    f"Source: `{section.page.as_posix()}`",
+                    "",
+                    section.body.rstrip(),
+                ]
+            )
     for index, match in enumerate(result.matches, start=1):
         lines.extend(
             [
@@ -410,6 +514,19 @@ def _parser() -> argparse.ArgumentParser:
     return parser
 
 
+def _nearest_section_ids(
+    requested: str, sections: list[Section], limit: int = 5
+) -> list[str]:
+    ranked = sorted(
+        sections,
+        key=lambda section: (
+            -difflib.SequenceMatcher(None, requested, section.id).ratio(),
+            section.id,
+        ),
+    )
+    return [section.id for section in ranked[:limit]]
+
+
 def main(argv: list[str] | None = None) -> int:
     _configure_utf8()
     parser = _parser()
@@ -424,12 +541,16 @@ def main(argv: list[str] | None = None) -> int:
             print(f"{problem.page}:{problem.line}: {problem.message}", file=sys.stderr)
         return 1 if problems else 0
     if args.section is not None:
+        sections = load_sections(args.references_dir)
         section = next(
-            (item for item in load_sections(args.references_dir) if item.id == args.section),
+            (item for item in sections if item.id == args.section),
             None,
         )
         if section is None:
             print(f"section not found: {args.section}", file=sys.stderr)
+            print("Nearest section IDs:", file=sys.stderr)
+            for candidate in _nearest_section_ids(args.section, sections):
+                print(f"- {candidate}", file=sys.stderr)
             return 2
         result = QueryResult("", "section", [QueryMatch(section, 0)], [])
     else:

test/test_tsl_syntax_lookup.py

@@ -53,11 +53,46 @@ class TslSyntaxLookupTests(unittest.TestCase):
                 )
                 self.assertEqual(completed.returncode, 2)
 
+    def test_missing_section_returns_nearest_section_ids(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            (references / "sample.md").write_text(
+                "# Sample\n\n## Exact Section\n\nDetails.\n\n## Other\n\nMore.\n",
+                encoding="utf-8",
+                newline="\n",
+            )
+            expected_id = lookup.load_sections(references)[0].id
+            completed = subprocess.run(
+                [
+                    sys.executable,
+                    str(SCRIPT),
+                    "--section",
+                    f"{expected_id}-typo",
+                    "--references-dir",
+                    str(references),
+                ],
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+            )
+
+        self.assertEqual(completed.returncode, 2)
+        self.assertIn("Nearest section IDs:", completed.stderr)
+        self.assertIn(f"- {expected_id}\n", completed.stderr)
+
     def test_write_identifier_query_keeps_required_context(self):
         result = lookup.query_sections("varByRef 命名参数", "write", limit=5)
         prelude_pages = {section.page.name for section in result.prelude}
         self.assertEqual(prelude_pages, {"01_quickstart.md", "02_core_model.md"})
 
+    def test_write_prelude_renders_source_for_every_section(self):
+        result = lookup.query_sections("varByRef 命名参数", "write", limit=5)
+        rendered = lookup.render_result(result)
+        required_context = rendered.split("## Match 1", 1)[0]
+        self.assertEqual(required_context.count("Source: `"), len(result.prelude))
+        for section in result.prelude:
+            self.assertIn(f"Source: `{section.page.as_posix()}`", required_context)
+
     def test_check_rejects_unknown_code_block_identity(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             references = Path(tmp_dir)
@@ -78,6 +113,50 @@ class TslSyntaxLookupTests(unittest.TestCase):
             any("未知身份" in problem.message for problem in problems), problems
         )
 
+    def test_parser_and_validator_reject_prose_between_identity_and_fence(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            page = references / "sample.md"
+            page.write_text(
+                "# Sample\n\n"
+                "## Example\n\n"
+                "代码块身份：可直接照写示例\n"
+                "This arbitrary prose breaks the metadata association.\n\n"
+                "```tsl\n"
+                "return 1;\n"
+                "```\n",
+                encoding="utf-8",
+                newline="\n",
+            )
+
+            sections = lookup.load_sections(references)
+            problems = lookup.validate_references(references)
+
+        self.assertEqual(sections[0].identities, ())
+        self.assertTrue(any("恰好一个代码块身份" in item.message for item in problems))
+
+    def test_parser_and_validator_accept_structured_block_description(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            page = references / "sample.md"
+            page.write_text(
+                "# Sample\n\n"
+                "## Example\n\n"
+                "代码块身份：配置片段 / 概念骨架\n"
+                "代码块说明：This is structured metadata.\n\n"
+                "```text\n"
+                "example\n"
+                "```\n",
+                encoding="utf-8",
+                newline="\n",
+            )
+
+            sections = lookup.load_sections(references)
+            problems = lookup.validate_references(references)
+
+        self.assertEqual(sections[0].identities, ("配置片段 / 概念骨架",))
+        self.assertEqual(problems, [])
+
     def test_check_accumulates_fence_identity_and_link_problems(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             references = Path(tmp_dir)
@@ -120,6 +199,29 @@ class TslSyntaxLookupTests(unittest.TestCase):
         sources = "\n".join(match.section.id for match in result.matches)
         self.assertIn("08_objects_and_classes", sources)
 
+    def test_ranking_tiers_beat_adversarial_page_order(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            references = Path(tmp_dir)
+            pages = {
+                "a_body.md": "# Other\n\n## Other\n\nneedle appears as plain prose.\n",
+                "b_title.md": "# needle\n\n## Other\n\nNo body match.\n",
+                "c_identifier.md": "# Other\n\n## Other\n\nUse `needle` here.\n",
+                "z_heading.md": "# Other\n\n## needle\n\nNo body detail.\n",
+            }
+            for name, text in pages.items():
+                (references / name).write_text(
+                    text, encoding="utf-8", newline="\n"
+                )
+
+            result = lookup.query_sections(
+                "needle", "explain", limit=4, references_dir=references
+            )
+
+        self.assertEqual(
+            [match.section.page.name for match in result.matches],
+            ["z_heading.md", "c_identifier.md", "b_title.md", "a_body.md"],
+        )
+
     def test_tsl_identifier_is_preserved(self):
         result = lookup.query_sections("varByRef 命名参数", "explain", limit=5)
         rendered = lookup.render_result(result)