🐛 fix(tsl_syntax): repair lookup engine and reconcile reference facts
lookup.py: exempt code-anchored ASCII identifiers from the mixed zh/en gate so exact hits are no longer dropped; dedup parent/child sections in results; validate write-prelude anchors in --check. references: correct interpreter-verified facts (case-as-expression, control-flow semicolons, __line__/__stack_frame, tslObjects order, destroy timing, ErrDefine, truncated outputs), fix headings, scope qualifiers and reversed quotes; strip dead preamble metadata from all 24 pages. packaging: exclude __pycache__/*.pyc from playbook and bundle copies; update build test file-count assertion to match. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -29,6 +29,11 @@ ALLOWED_IDENTITIES = {
|
||||
}
|
||||
ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断")
|
||||
EXCLUDED_REFERENCE_FILES = {"index.md"}
|
||||
SUSPICIOUS_FENCE_RE = re.compile(r"^(?:\s+`{3}|`{4,})")
|
||||
WRITE_PRELUDE_ANCHORS = (
|
||||
("02_core_model.md", "文件模型核心规则"),
|
||||
("01_quickstart.md", "语言核心事实速查"),
|
||||
)
|
||||
|
||||
HEADING_TOKEN_SCORE = 12
|
||||
HEADING_EXACT_SCORE = 20
|
||||
@@ -249,16 +254,24 @@ def _identity_problems(page: Path, lines: list[str]) -> list[ValidationProblem]:
|
||||
problems.append(ValidationProblem(page, index, f"未知身份:{identity}"))
|
||||
in_fence = False
|
||||
for index, line in enumerate(lines):
|
||||
if not FENCE_RE.match(line):
|
||||
if FENCE_RE.match(line):
|
||||
if in_fence:
|
||||
in_fence = False
|
||||
else:
|
||||
if _associated_identity(lines, index) is None:
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
page, index + 1, "每个代码围栏必须关联恰好一个代码块身份"
|
||||
)
|
||||
)
|
||||
in_fence = True
|
||||
continue
|
||||
if in_fence:
|
||||
in_fence = False
|
||||
continue
|
||||
if _associated_identity(lines, index) is None:
|
||||
if not in_fence and SUSPICIOUS_FENCE_RE.match(line):
|
||||
problems.append(
|
||||
ValidationProblem(page, index + 1, "每个代码围栏必须关联恰好一个代码块身份")
|
||||
ValidationProblem(
|
||||
page, index + 1, "不支持的代码围栏形态(缩进围栏或四个及以上反引号)"
|
||||
)
|
||||
)
|
||||
in_fence = True
|
||||
if in_fence:
|
||||
problems.append(ValidationProblem(page, len(lines), "代码围栏未闭合"))
|
||||
return problems
|
||||
@@ -268,8 +281,17 @@ def _local_link_problems(
|
||||
page: Path, text: str, references_dir: Path
|
||||
) -> list[ValidationProblem]:
|
||||
problems: list[ValidationProblem] = []
|
||||
without_fences = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
|
||||
searchable_markdown = re.sub(r"`[^`\n]*`", "", without_fences)
|
||||
# Blank out fenced/inline code without changing line offsets, so reported
|
||||
# line numbers line up with the original file.
|
||||
without_fences = re.sub(
|
||||
r"```.*?```",
|
||||
lambda match: re.sub(r"[^\n]", " ", match.group(0)),
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
searchable_markdown = re.sub(
|
||||
r"`[^`\n]*`", lambda match: " " * len(match.group(0)), without_fences
|
||||
)
|
||||
for match in MARKDOWN_LINK_RE.finditer(searchable_markdown):
|
||||
target = match.group(1).strip().split(maxsplit=1)[0].strip("<>")
|
||||
if target.startswith(("#", "http://", "https://", "mailto:")):
|
||||
@@ -313,6 +335,22 @@ def validate_references(
|
||||
if section.id in ids:
|
||||
problems.append(ValidationProblem(section.page, 1, f"重复 section ID:{section.id}"))
|
||||
ids[section.id] = section
|
||||
for page_name, heading in WRITE_PRELUDE_ANCHORS:
|
||||
page_sections = [
|
||||
section for section in sections if section.page.name == page_name
|
||||
]
|
||||
# Only enforce the anchor when the page is present, so validating a
|
||||
# synthetic references dir (tests) does not demand the bundled pages.
|
||||
if page_sections and not any(
|
||||
heading in section.heading_path for section in page_sections
|
||||
):
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
references_dir / page_name,
|
||||
1,
|
||||
f"write 模式前置章节缺失:{page_name} 的「{heading}」",
|
||||
)
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
@@ -394,12 +432,8 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
|
||||
|
||||
|
||||
def _write_prelude(sections: list[Section]) -> list[Section]:
|
||||
preferred = [
|
||||
("02_core_model.md", "文件模型核心规则"),
|
||||
("01_quickstart.md", "语言核心事实速查"),
|
||||
]
|
||||
prelude: list[Section] = []
|
||||
for page_name, heading in preferred:
|
||||
for page_name, heading in WRITE_PRELUDE_ANCHORS:
|
||||
match = next(
|
||||
(
|
||||
section
|
||||
@@ -413,6 +447,43 @@ def _write_prelude(sections: list[Section]) -> list[Section]:
|
||||
return prelude
|
||||
|
||||
|
||||
STRING_LITERAL_RE = re.compile(r'"[^"\n]*"|\'[^\'\n]*\'')
|
||||
|
||||
|
||||
def _bare_tokens(tokens: set[str]) -> set[str]:
|
||||
# ASCII_TOKEN_RE 的连续字符类会把尾缀符号吞进 token(如编译开关
|
||||
# `{$varByRef-}` 产出 `varbyref-`);按剥掉尾部符号的裸形态比较。
|
||||
return {token.rstrip(".$:+-") for token in tokens} - {""}
|
||||
|
||||
|
||||
def _ascii_anchor(section: Section, ascii_tokens: set[str]) -> bool:
|
||||
# 只有足够长的标识符按标识符位置命中(标题、行内代码或剥离字符串
|
||||
# 字面量后的围栏代码)才豁免中文门控;短 token 与字符串样例数据
|
||||
# (如 "XYZ"、"xyz*")不算点名,避免样例值驱动召回。
|
||||
anchor_tokens = {
|
||||
token for token in _bare_tokens(ascii_tokens) if len(token) >= 4
|
||||
}
|
||||
if not anchor_tokens:
|
||||
return False
|
||||
heading_tokens = set(
|
||||
ASCII_TOKEN_RE.findall(normalize("\n".join(section.heading_path)))
|
||||
)
|
||||
code_text = "\n".join(
|
||||
(*INLINE_CODE_RE.findall(section.body), *FENCED_CODE_RE.findall(section.body))
|
||||
)
|
||||
code_tokens = set(
|
||||
ASCII_TOKEN_RE.findall(normalize(STRING_LITERAL_RE.sub(" ", code_text)))
|
||||
)
|
||||
return bool(anchor_tokens & _bare_tokens(heading_tokens | code_tokens))
|
||||
|
||||
|
||||
def _related_sections(a: Section, b: Section) -> bool:
|
||||
if a.page != b.page or len(a.heading_path) == len(b.heading_path):
|
||||
return False
|
||||
shorter, longer = sorted((a.heading_path, b.heading_path), key=len)
|
||||
return longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
def query_sections(
|
||||
query: str,
|
||||
mode: str,
|
||||
@@ -438,7 +509,14 @@ def query_sections(
|
||||
minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
|
||||
ranked: list[QueryMatch] = []
|
||||
for section in sections:
|
||||
if ascii_tokens and has_chinese and not _has_chinese_context(section, query):
|
||||
# 中文上下文门控只裁剪正文级 ASCII 噪声;标识符在标题或代码里
|
||||
# 精确命中的 section 不因中文措辞不同而被丢弃。
|
||||
if (
|
||||
ascii_tokens
|
||||
and has_chinese
|
||||
and not _ascii_anchor(section, ascii_tokens)
|
||||
and not _has_chinese_context(section, query)
|
||||
):
|
||||
continue
|
||||
score = _score_section(section, query, mode)
|
||||
if score.lexical_total < minimum_score:
|
||||
@@ -451,7 +529,17 @@ def query_sections(
|
||||
match.section.id,
|
||||
)
|
||||
)
|
||||
matches = ranked[:limit]
|
||||
# H2 聚合 section 的正文逐字包含其 H3 子节;父子同时入选时只保留
|
||||
# 排名更高的一个,避免同一内容重复返回。
|
||||
matches: list[QueryMatch] = []
|
||||
for match in ranked:
|
||||
if any(
|
||||
_related_sections(match.section, kept.section) for kept in matches
|
||||
):
|
||||
continue
|
||||
matches.append(match)
|
||||
if len(matches) == limit:
|
||||
break
|
||||
prelude = _write_prelude(all_sections) if mode == "write" else []
|
||||
match_ids = {match.section.id for match in matches}
|
||||
prelude = [section for section in prelude if section.id not in match_ids]
|
||||
@@ -535,6 +623,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.error("--limit must be between 1 and 10")
|
||||
if args.query is not None and args.mode is None:
|
||||
parser.error("--mode is required with --query")
|
||||
if args.mode is not None and args.query is None:
|
||||
parser.error("--mode only applies to --query")
|
||||
if args.check:
|
||||
problems = validate_references(args.references_dir)
|
||||
for problem in problems:
|
||||
|
||||
Reference in New Issue
Block a user