🐛 fix(tsl_syntax): repair lookup engine and reconcile reference facts

lookup.py: exempt code-anchored ASCII identifiers from the mixed
zh/en gate so exact hits are no longer dropped; dedup parent/child
sections in results; validate write-prelude anchors in --check.

references: correct interpreter-verified facts (case-as-expression,
control-flow semicolons, __line__/__stack_frame, tslObjects order,
destroy timing, ErrDefine, truncated outputs), fix headings, scope
qualifiers and reversed quotes; strip dead preamble metadata from all
24 pages.

packaging: exclude __pycache__/*.pyc from playbook and bundle copies;
update build test file-count assertion to match.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
csh
2026-07-13 09:16:57 +08:00
co-authored by Claude Fable 5
parent 923bce91b0
commit b597edfc70
29 changed files with 356 additions and 256 deletions
+106 -16
View File
@@ -29,6 +29,11 @@ ALLOWED_IDENTITIES = {
}
ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断")
EXCLUDED_REFERENCE_FILES = {"index.md"}
SUSPICIOUS_FENCE_RE = re.compile(r"^(?:\s+`{3}|`{4,})")
WRITE_PRELUDE_ANCHORS = (
("02_core_model.md", "文件模型核心规则"),
("01_quickstart.md", "语言核心事实速查"),
)
HEADING_TOKEN_SCORE = 12
HEADING_EXACT_SCORE = 20
@@ -249,16 +254,24 @@ def _identity_problems(page: Path, lines: list[str]) -> list[ValidationProblem]:
problems.append(ValidationProblem(page, index, f"未知身份:{identity}"))
in_fence = False
for index, line in enumerate(lines):
if not FENCE_RE.match(line):
if FENCE_RE.match(line):
if in_fence:
in_fence = False
else:
if _associated_identity(lines, index) is None:
problems.append(
ValidationProblem(
page, index + 1, "每个代码围栏必须关联恰好一个代码块身份"
)
)
in_fence = True
continue
if in_fence:
in_fence = False
continue
if _associated_identity(lines, index) is None:
if not in_fence and SUSPICIOUS_FENCE_RE.match(line):
problems.append(
ValidationProblem(page, index + 1, "每个代码围栏必须关联恰好一个代码块身份")
ValidationProblem(
page, index + 1, "不支持的代码围栏形态(缩进围栏或四个及以上反引号)"
)
)
in_fence = True
if in_fence:
problems.append(ValidationProblem(page, len(lines), "代码围栏未闭合"))
return problems
@@ -268,8 +281,17 @@ def _local_link_problems(
page: Path, text: str, references_dir: Path
) -> list[ValidationProblem]:
problems: list[ValidationProblem] = []
without_fences = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
searchable_markdown = re.sub(r"`[^`\n]*`", "", without_fences)
# Blank out fenced/inline code without changing line offsets, so reported
# line numbers line up with the original file.
without_fences = re.sub(
r"```.*?```",
lambda match: re.sub(r"[^\n]", " ", match.group(0)),
text,
flags=re.DOTALL,
)
searchable_markdown = re.sub(
r"`[^`\n]*`", lambda match: " " * len(match.group(0)), without_fences
)
for match in MARKDOWN_LINK_RE.finditer(searchable_markdown):
target = match.group(1).strip().split(maxsplit=1)[0].strip("<>")
if target.startswith(("#", "http://", "https://", "mailto:")):
@@ -313,6 +335,22 @@ def validate_references(
if section.id in ids:
problems.append(ValidationProblem(section.page, 1, f"重复 section ID{section.id}"))
ids[section.id] = section
for page_name, heading in WRITE_PRELUDE_ANCHORS:
page_sections = [
section for section in sections if section.page.name == page_name
]
# Only enforce the anchor when the page is present, so validating a
# synthetic references dir (tests) does not demand the bundled pages.
if page_sections and not any(
heading in section.heading_path for section in page_sections
):
problems.append(
ValidationProblem(
references_dir / page_name,
1,
f"write 模式前置章节缺失:{page_name} 的「{heading}",
)
)
return problems
@@ -394,12 +432,8 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
def _write_prelude(sections: list[Section]) -> list[Section]:
preferred = [
("02_core_model.md", "文件模型核心规则"),
("01_quickstart.md", "语言核心事实速查"),
]
prelude: list[Section] = []
for page_name, heading in preferred:
for page_name, heading in WRITE_PRELUDE_ANCHORS:
match = next(
(
section
@@ -413,6 +447,43 @@ def _write_prelude(sections: list[Section]) -> list[Section]:
return prelude
STRING_LITERAL_RE = re.compile(r'"[^"\n]*"|\'[^\'\n]*\'')
def _bare_tokens(tokens: set[str]) -> set[str]:
# ASCII_TOKEN_RE 的连续字符类会把尾缀符号吞进 token(如编译开关
# `{$varByRef-}` 产出 `varbyref-`);按剥掉尾部符号的裸形态比较。
return {token.rstrip(".$:+-") for token in tokens} - {""}
def _ascii_anchor(section: Section, ascii_tokens: set[str]) -> bool:
# 只有足够长的标识符按标识符位置命中(标题、行内代码或剥离字符串
# 字面量后的围栏代码)才豁免中文门控;短 token 与字符串样例数据
# (如 "XYZ"、"xyz*")不算点名,避免样例值驱动召回。
anchor_tokens = {
token for token in _bare_tokens(ascii_tokens) if len(token) >= 4
}
if not anchor_tokens:
return False
heading_tokens = set(
ASCII_TOKEN_RE.findall(normalize("\n".join(section.heading_path)))
)
code_text = "\n".join(
(*INLINE_CODE_RE.findall(section.body), *FENCED_CODE_RE.findall(section.body))
)
code_tokens = set(
ASCII_TOKEN_RE.findall(normalize(STRING_LITERAL_RE.sub(" ", code_text)))
)
return bool(anchor_tokens & _bare_tokens(heading_tokens | code_tokens))
def _related_sections(a: Section, b: Section) -> bool:
if a.page != b.page or len(a.heading_path) == len(b.heading_path):
return False
shorter, longer = sorted((a.heading_path, b.heading_path), key=len)
return longer[: len(shorter)] == shorter
def query_sections(
query: str,
mode: str,
@@ -438,7 +509,14 @@ def query_sections(
minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
ranked: list[QueryMatch] = []
for section in sections:
if ascii_tokens and has_chinese and not _has_chinese_context(section, query):
# 中文上下文门控只裁剪正文级 ASCII 噪声;标识符在标题或代码里
# 精确命中的 section 不因中文措辞不同而被丢弃。
if (
ascii_tokens
and has_chinese
and not _ascii_anchor(section, ascii_tokens)
and not _has_chinese_context(section, query)
):
continue
score = _score_section(section, query, mode)
if score.lexical_total < minimum_score:
@@ -451,7 +529,17 @@ def query_sections(
match.section.id,
)
)
matches = ranked[:limit]
# H2 聚合 section 的正文逐字包含其 H3 子节;父子同时入选时只保留
# 排名更高的一个,避免同一内容重复返回。
matches: list[QueryMatch] = []
for match in ranked:
if any(
_related_sections(match.section, kept.section) for kept in matches
):
continue
matches.append(match)
if len(matches) == limit:
break
prelude = _write_prelude(all_sections) if mode == "write" else []
match_ids = {match.section.id for match in matches}
prelude = [section for section in prelude if section.id not in match_ids]
@@ -535,6 +623,8 @@ def main(argv: list[str] | None = None) -> int:
parser.error("--limit must be between 1 and 10")
if args.query is not None and args.mode is None:
parser.error("--mode is required with --query")
if args.mode is not None and args.query is None:
parser.error("--mode only applies to --query")
if args.check:
problems = validate_references(args.references_dir)
for problem in problems: