csh/playbook
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐛 fix(tsl-syntax): tighten lookup contracts

Browse Source
This commit is contained in:
csh
2026-07-13 09:16:55 +08:00
parent 9b9e0dfcb7
commit 923bce91b0
2 changed files with 272 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/tsl-syntax-reference/scripts/lookup.py
+170 -49
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
import difflib
import re
import sys
import unicodedata
@@ -15,7 +16,10 @@ FENCE_RE = re.compile(r"^```([^`]*)$")
MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
ASCII_TOKEN_RE = re.compile(r"[a-z_][a-z0-9_.$:+-]*", re.IGNORECASE)
CHINESE_RUN_RE = re.compile(r"[\u3400-\u9fff]+")
INLINE_CODE_RE = re.compile(r"`([^`\n]+)`")
FENCED_CODE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL)
IDENTITY_PREFIX = "代码块身份："
BLOCK_DESCRIPTION_PREFIX = "代码块说明："
ALLOWED_IDENTITIES = {
"可直接照写示例",
"反例 / 不可照写",
@@ -26,10 +30,14 @@ ALLOWED_IDENTITIES = {
ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断")
EXCLUDED_REFERENCE_FILES = {"index.md"}
TOKEN_IN_HEADING_SCORE = 12
TOKEN_IN_BODY_SCORE = 3
EXACT_QUERY_SCORE = 20
PAGE_NAME_SCORE = 5
HEADING_TOKEN_SCORE = 12
HEADING_EXACT_SCORE = 20
TERM_TOKEN_SCORE = 10
TERM_EXACT_SCORE = 16
PAGE_TITLE_TOKEN_SCORE = 5
PAGE_TITLE_EXACT_SCORE = 8
BODY_TOKEN_SCORE = 3
BODY_EXACT_SCORE = 4
DIRECT_EXAMPLE_BOOST = 8
PITFALL_PAGE_BOOST = 18
COUNTEREXAMPLE_BOOST = 8
@@ -59,6 +67,43 @@ class ValidationProblem:
class QueryMatch:
section: Section
score: int
priority: tuple[int, int, int, int, int] = (0, 0, 0, 0, 0)
@dataclass(frozen=True)
class ScoreBreakdown:
heading_path: int
exact_term: int
page_title: int
body: int
mode_boost: int
diagnose_priority: bool = False
@property
def lexical_total(self) -> int:
return self.heading_path + self.exact_term + self.page_title + self.body
@property
def total(self) -> int:
return self.lexical_total + self.mode_boost
@property
def priority(self) -> tuple[int, int, int, int, int]:
if self.diagnose_priority:
return (
self.mode_boost,
self.heading_path,
self.exact_term,
self.page_title,
self.body,
)
return (
self.heading_path,
self.exact_term,
self.page_title,
self.mode_boost,
self.body,
)
@dataclass
@@ -115,6 +160,28 @@ def _heading_records(lines: list[str]) -> tuple[str, list[tuple[int, int, str]]]
return page_title, records
def _associated_identity(lines: list[str], opening_fence: int) -> str | None:
previous = opening_fence - 1
while previous >= 0 and not lines[previous].strip():
previous -= 1
while previous >= 0 and lines[previous].strip().startswith(BLOCK_DESCRIPTION_PREFIX):
previous -= 1
while previous >= 0 and not lines[previous].strip():
previous -= 1
if previous < 0:
return None
metadata = lines[previous].strip()
if not metadata.startswith(IDENTITY_PREFIX):
return None
identity = metadata[len(IDENTITY_PREFIX) :].strip()
earlier = previous - 1
while earlier >= 0 and not lines[earlier].strip():
earlier -= 1
if earlier >= 0 and lines[earlier].strip().startswith(IDENTITY_PREFIX):
return None
return identity if identity in ALLOWED_IDENTITIES else None
def _identities(body: str) -> tuple[str, ...]:
identities: list[str] = []
lines = body.splitlines()
@@ -123,11 +190,9 @@ def _identities(body: str) -> tuple[str, ...]:
if not FENCE_RE.match(line):
continue
if not in_fence:
previous = index - 1
while previous >= 0 and not lines[previous].strip():
previous -= 1
if previous >= 0 and lines[previous].strip().startswith(IDENTITY_PREFIX):
identities.append(lines[previous].strip()[len(IDENTITY_PREFIX) :].strip())
identity = _associated_identity(lines, index)
if identity is not None:
identities.append(identity)
in_fence = not in_fence
return tuple(identities)
@@ -189,18 +254,7 @@ def _identity_problems(page: Path, lines: list[str]) -> list[ValidationProblem]:
if in_fence:
in_fence = False
continue
previous = index - 1
identities: list[tuple[int, str]] = []
while previous >= 0:
stripped = lines[previous].strip()
if FENCE_RE.match(stripped) or HEADING_RE.match(stripped):
break
if stripped.startswith(IDENTITY_PREFIX):
identity = stripped[len(IDENTITY_PREFIX) :].strip()
identities.append((previous, identity))
previous -= 1
recognized = [identity for _, identity in identities if identity in ALLOWED_IDENTITIES]
if len(recognized) != 1 or len(identities) != 1:
if _associated_identity(lines, index) is None:
problems.append(
ValidationProblem(page, index + 1, "每个代码围栏必须关联恰好一个代码块身份")
)
@@ -271,36 +325,72 @@ def query_tokens(text: str) -> set[str]:
return {token for token in tokens if token.strip()}
def _lexical_score(section: Section, query: str) -> int:
def _has_chinese_context(section: Section, query: str) -> bool:
runs = CHINESE_RUN_RE.findall(normalize(query))
for run in runs:
tokens = (
[run]
if len(run) < 2
else [run[index : index + 2] for index in range(len(run) - 1)]
)
matched = sum(token in section.searchable_text for token in tokens)
if matched >= (len(tokens) + 1) // 2:
return True
return not runs
def _code_text(body: str) -> str:
inline = INLINE_CODE_RE.findall(body)
fenced = FENCED_CODE_RE.findall(body)
return normalize("\n".join((*inline, *fenced)))
def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
normalized_query = normalize(query).strip()
tokens = query_tokens(query)
heading_text = normalize("\n".join((section.page_title, *section.heading_path)))
score = 0
heading_text = normalize("\n".join(section.heading_path))
page_title_text = normalize(section.page_title)
body_text = normalize(section.body)
term_text = _code_text(section.body)
heading_score = 0
term_score = 0
page_title_score = 0
body_score = 0
for token in tokens:
if token in heading_text:
score += TOKEN_IN_HEADING_SCORE
elif token in section.searchable_text:
score += TOKEN_IN_BODY_SCORE
if normalized_query and normalized_query in section.searchable_text:
score += EXACT_QUERY_SCORE
if any(token in normalize(section.page.stem) for token in tokens):
score += PAGE_NAME_SCORE
return score
def _score_section(section: Section, query: str, mode: str, lexical_score: int) -> int:
normalized_query = normalize(query).strip()
score = lexical_score
heading_score += HEADING_TOKEN_SCORE
if token in term_text:
term_score += TERM_TOKEN_SCORE
if token in page_title_text:
page_title_score += PAGE_TITLE_TOKEN_SCORE
if token in body_text:
body_score += BODY_TOKEN_SCORE
if normalized_query and normalized_query in heading_text:
heading_score += HEADING_EXACT_SCORE
if normalized_query and normalized_query in term_text:
term_score += TERM_EXACT_SCORE
if normalized_query and normalized_query in page_title_text:
page_title_score += PAGE_TITLE_EXACT_SCORE
if normalized_query and normalized_query in body_text:
body_score += BODY_EXACT_SCORE
mode_boost = 0
if mode == "write" and "可直接照写示例" in section.identities:
score += DIRECT_EXAMPLE_BOOST
mode_boost += DIRECT_EXAMPLE_BOOST
if mode == "diagnose":
if section.page.name == "11_pitfalls.md":
score += PITFALL_PAGE_BOOST
mode_boost += PITFALL_PAGE_BOOST
if "反例 / 不可照写" in section.identities:
score += COUNTEREXAMPLE_BOOST
mode_boost += COUNTEREXAMPLE_BOOST
if normalized_query and normalized_query in section.searchable_text:
score += EXACT_ERROR_BOOST
return score
mode_boost += EXACT_ERROR_BOOST
return ScoreBreakdown(
heading_path=heading_score,
exact_term=term_score,
page_title=page_title_score,
body=body_score,
mode_boost=mode_boost,
diagnose_priority=mode == "diagnose",
)
def _write_prelude(sections: list[Section]) -> list[Section]:
@@ -348,13 +438,19 @@ def query_sections(
minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
ranked: list[QueryMatch] = []
for section in sections:
lexical_score = _lexical_score(section, query)
if lexical_score < minimum_score:
if ascii_tokens and has_chinese and not _has_chinese_context(section, query):
continue
ranked.append(
QueryMatch(section, _score_section(section, query, mode, lexical_score))
score = _score_section(section, query, mode)
if score.lexical_total < minimum_score:
continue
ranked.append(QueryMatch(section, score.total, score.priority))
ranked.sort(
key=lambda match: (
*(-value for value in match.priority),
match.section.page.as_posix(),
match.section.id,
)
ranked.sort(key=lambda match: (-match.score, match.section.page.as_posix(), match.section.id))
)
matches = ranked[:limit]
prelude = _write_prelude(all_sections) if mode == "write" else []
match_ids = {match.section.id for match in matches}
@@ -372,7 +468,15 @@ def render_result(result: QueryResult) -> str:
if result.prelude:
lines.extend(["", "## Required context"])
for section in result.prelude:
lines.extend(["", f"Section ID: `{section.id}`", "", section.body.rstrip()])
lines.extend(
[
"",
f"Section ID: `{section.id}`",
f"Source: `{section.page.as_posix()}`",
"",
section.body.rstrip(),
]
)
for index, match in enumerate(result.matches, start=1):
lines.extend(
[
@@ -410,6 +514,19 @@ def _parser() -> argparse.ArgumentParser:
return parser
def _nearest_section_ids(
requested: str, sections: list[Section], limit: int = 5
) -> list[str]:
ranked = sorted(
sections,
key=lambda section: (
-difflib.SequenceMatcher(None, requested, section.id).ratio(),
section.id,
),
)
return [section.id for section in ranked[:limit]]
def main(argv: list[str] | None = None) -> int:
_configure_utf8()
parser = _parser()
@@ -424,12 +541,16 @@ def main(argv: list[str] | None = None) -> int:
print(f"{problem.page}:{problem.line}: {problem.message}", file=sys.stderr)
return 1 if problems else 0
if args.section is not None:
sections = load_sections(args.references_dir)
section = next(
(item for item in load_sections(args.references_dir) if item.id == args.section),
(item for item in sections if item.id == args.section),
None,
)
if section is None:
print(f"section not found: {args.section}", file=sys.stderr)
print("Nearest section IDs:", file=sys.stderr)
for candidate in _nearest_section_ids(args.section, sections):
print(f"- {candidate}", file=sys.stderr)
return 2
result = QueryResult("", "section", [QueryMatch(section, 0)], [])
else: