From d663c2dae56ec0038e9962bb52db4b7f1e30c19a Mon Sep 17 00:00:00 2001 From: csh Date: Sun, 12 Jul 2026 21:31:03 +0800 Subject: [PATCH] :bug: fix(tsl-syntax): close lookup review gaps --- skills/tsl-syntax-reference/scripts/lookup.py | 214 ++++++++++++------ test/test_tsl_syntax_lookup.py | 151 +++++++++++- 2 files changed, 299 insertions(+), 66 deletions(-) diff --git a/skills/tsl-syntax-reference/scripts/lookup.py b/skills/tsl-syntax-reference/scripts/lookup.py index b06f92ab..76d8fcc0 100644 --- a/skills/tsl-syntax-reference/scripts/lookup.py +++ b/skills/tsl-syntax-reference/scripts/lookup.py @@ -46,7 +46,7 @@ PAGE_TITLE_EXACT_SCORE = 8 BODY_TOKEN_SCORE = 3 BODY_EXACT_SCORE = 4 DIRECT_EXAMPLE_BOOST = 8 -PITFALL_PAGE_BOOST = 18 +PITFALL_PAGE_BOOST = 30 COUNTEREXAMPLE_BOOST = 8 EXACT_ERROR_BOOST = 14 MIXED_QUERY_MIN_SCORE = 10 @@ -67,22 +67,36 @@ CHINESE_STOP_TOKENS = { "里面", } +ASCII_FILTER_STOP_TOKENS = { + "debug", + "please", + "program", + "tinysoft", + "tsl", + "tsf", +} + QUERY_SYNONYMS = { "打印": ("输出", "writeLn"), "打出来": ("输出", "writeLn"), "左连接": ("左联接", "left join", "TS-SQL"), + "左外连接": ("左联接", "left join", "TS-SQL"), + "left outer join": ("left join", "左联接", "TS-SQL"), "列表": ("数组",), "复用文件": ("tsf", "unit"), "多个文件": ("unit", "uses", "作用域"), "跳出去": ("break", "控制流"), "程序慢": ("性能分析", "计时", "profiler"), "瓶颈": ("性能分析", "profiler"), + "debug": ("调试", "性能分析"), + "program": ("脚本",), + "tinysoft": ("天软", "TSL"), "字符串转整数": ("类型转换", "strToInt"), "高性能矩阵": ("FMArray",), } PAGE_INTENT_ALIASES = { - "01_quickstart.md": ("最简单能跑", "天软脚本"), + "01_quickstart.md": ("最简单能跑", "天软脚本", "tinysoft"), "02_core_model.md": ("脚本和可复用", "可复用函数文件"), "03_values_and_literals.md": ("字符串和数组下标",), "04_variables_and_constants.md": ("常量怎么声明", "变量能不能直接赋值"), @@ -95,8 +109,8 @@ PAGE_INTENT_ALIASES = { "11_pitfalls.md": ("声明函数后面写代码", "语法报错"), "12_matrix_and_collections.md": ("某行存在", "二维数组怎么判断"), "13_resultset_and_filters.md": ("保留匹配行", "按某一列"), - "14_ts_sql.md": ("左连接", "左联接", "数据库", "分组排序"), - "15_debug_and_profiler.md": ("程序慢", "计时找瓶颈", "性能瓶颈"), + "14_ts_sql.md": ("左连接", "左外连接", "左联接", "数据库", "分组排序", "聚合排序"), + "15_debug_and_profiler.md": ("程序慢", "计时找瓶颈", "性能瓶颈", "性能问题", "debug"), "16_lexical_structure_and_compile_options.md": ("变量名区分大小写", "注释怎么写"), "17_types_and_conversions.md": ("字符串转整数", "类型转换"), "18_external_calls_and_threads.md": ("调用 dll", "开线程"), @@ -116,6 +130,7 @@ class Section: page_title: str heading_path: tuple[str, ...] body: str + local_body: str identities: tuple[str, ...] searchable_text: str @@ -132,6 +147,7 @@ class QueryMatch: section: Section score: int priority: tuple[int, ...] = () + reasons: tuple[str, ...] = () @dataclass(frozen=True) @@ -142,6 +158,7 @@ class ScoreBreakdown: page_title: int body: int mode_boost: int + synonym_hits: int diagnose_priority: bool = False @property @@ -185,6 +202,7 @@ class QueryResult: mode: str matches: list[QueryMatch] prelude: list[Section] + limit: int def normalize(value: str) -> str: @@ -304,9 +322,15 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section end = next_start break body = "".join(lines[start:end]) + local_end = ( + headings[position + 1][0] + if position + 1 < len(headings) + else len(lines) + ) + local_body = "".join(lines[start:local_end]) base_id = section_id(page.relative_to(references_dir), heading_path) searchable_text = normalize( - "\n".join((page.stem, page_title, *heading_path, body)) + "\n".join((page.stem, page_title, *heading_path, local_body)) ) sections.append( Section( @@ -315,7 +339,8 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section page_title=page_title, heading_path=heading_path, body=body, - identities=_identities(body), + local_body=local_body, + identities=_identities(local_body), searchable_text=searchable_text, ) ) @@ -513,17 +538,43 @@ def _base_query_tokens(text: str) -> set[str]: } -def query_tokens(text: str) -> set[str]: - tokens = _base_query_tokens(text) - normalized = normalize(text) +def _ascii_token_sequence(text: str) -> list[str]: + return [ + token.rstrip(".$:+-") + for token in ASCII_TOKEN_RE.findall(normalize(text)) + if token.rstrip(".$:+-") + ] + + +def _query_contains_phrase(query: str, phrase: str) -> bool: + normalized_phrase = normalize(phrase) + if CHINESE_RUN_RE.search(normalized_phrase): + return normalized_phrase in normalize(query) + phrase_tokens = _ascii_token_sequence(phrase) + query_tokens_in_order = _ascii_token_sequence(query) + if not phrase_tokens: + return False + width = len(phrase_tokens) + return any( + query_tokens_in_order[index : index + width] == phrase_tokens + for index in range(len(query_tokens_in_order) - width + 1) + ) + + +def _synonym_tokens(text: str) -> set[str]: + tokens: set[str] = set() for phrase, synonyms in QUERY_SYNONYMS.items(): - if normalize(phrase) not in normalized: + if not _query_contains_phrase(text, phrase): continue for synonym in synonyms: tokens.update(_base_query_tokens(synonym)) return tokens +def query_tokens(text: str) -> set[str]: + return _base_query_tokens(text) | _synonym_tokens(text) + + def _text_contains_token(text: str, token: str) -> bool: normalized_text = normalize(text) if ASCII_TOKEN_RE.fullmatch(token): @@ -532,11 +583,17 @@ def _text_contains_token(text: str, token: str) -> bool: return token in normalized_text +def _text_contains_exact_query(text: str, query: str) -> bool: + normalized_query = normalize(query).strip() + if ASCII_TOKEN_RE.fullmatch(normalized_query): + return _text_contains_token(text, normalized_query) + return bool(normalized_query and normalized_query in normalize(text)) + + def _intent_score(section: Section, query: str) -> int: - normalized_query = normalize(query) aliases = PAGE_INTENT_ALIASES.get(section.page.name, ()) return PAGE_INTENT_SCORE * sum( - normalize(alias) in normalized_query for alias in aliases + _query_contains_phrase(query, alias) for alias in aliases ) @@ -565,8 +622,16 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown: tokens = query_tokens(query) heading_text = normalize("\n".join(section.heading_path)) page_title_text = normalize(section.page_title) - body_text = normalize(section.body) - term_text = _code_text(section.body) + body_text = normalize(section.local_body) + term_text = _code_text(section.local_body) + expanded_only_tokens = _synonym_tokens(query) - _base_query_tokens(query) + synonym_hits = sum( + any( + _text_contains_token(text, token) + for text in (heading_text, term_text, page_title_text, body_text) + ) + for token in expanded_only_tokens + ) heading_score = 0 term_score = 0 page_title_score = 0 @@ -580,13 +645,13 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown: page_title_score += PAGE_TITLE_TOKEN_SCORE if _text_contains_token(body_text, token): body_score += BODY_TOKEN_SCORE - if normalized_query and normalized_query in heading_text: + if _text_contains_exact_query(heading_text, query): heading_score += HEADING_EXACT_SCORE - if normalized_query and normalized_query in term_text: + if _text_contains_exact_query(term_text, query): term_score += TERM_EXACT_SCORE - if normalized_query and normalized_query in page_title_text: + if _text_contains_exact_query(page_title_text, query): page_title_score += PAGE_TITLE_EXACT_SCORE - if normalized_query and normalized_query in body_text: + if _text_contains_exact_query(body_text, query): body_score += BODY_EXACT_SCORE mode_boost = 0 if mode == "write" and "可直接照写示例" in section.identities: @@ -605,10 +670,26 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown: page_title=page_title_score, body=body_score, mode_boost=mode_boost, + synonym_hits=synonym_hits, diagnose_priority=mode == "diagnose", ) +def _score_reasons(score: ScoreBreakdown) -> tuple[str, ...]: + components = ( + ("intent", score.intent), + ("heading", score.heading_path), + ("identifier", score.exact_term), + ("page_title", score.page_title), + ("body", score.body), + ("mode", score.mode_boost), + ) + reasons = tuple(f"{name}={value}" for name, value in components if value) + if score.synonym_hits: + reasons += (f"synonym={score.synonym_hits}",) + return reasons + + def _write_prelude(sections: list[Section]) -> list[Section]: prelude: list[Section] = [] for page_name, heading in WRITE_PRELUDE_ANCHORS: @@ -647,7 +728,10 @@ def _ascii_anchor(section: Section, ascii_tokens: set[str]) -> bool: ASCII_TOKEN_RE.findall(normalize("\n".join(section.heading_path))) ) code_text = "\n".join( - (*INLINE_CODE_RE.findall(section.body), *FENCED_CODE_RE.findall(section.body)) + ( + *INLINE_CODE_RE.findall(section.local_body), + *FENCED_CODE_RE.findall(section.local_body), + ) ) code_tokens = set( ASCII_TOKEN_RE.findall(normalize(STRING_LITERAL_RE.sub(" ", code_text))) @@ -674,8 +758,18 @@ def query_sections( raise ValueError("limit must be between 1 and 10") all_sections = load_sections(references_dir) sections = all_sections + prelude = _write_prelude(all_sections) if mode == "write" else [] + required_ids = {section.id for section in prelude} + raw_ascii_tokens = { + token + for token in _base_query_tokens(query) + if ASCII_TOKEN_RE.fullmatch(token) + and token not in ASCII_FILTER_STOP_TOKENS + } ascii_tokens = { - token for token in _base_query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token) + token + for token in raw_ascii_tokens + if any(_ascii_anchor(section, {token}) for section in all_sections) } if ascii_tokens: sections = [ @@ -702,9 +796,17 @@ def query_sections( score = _score_section(section, query, mode) if score.lexical_total < minimum_score: continue - ranked.append(QueryMatch(section, score.total, score.priority)) + ranked.append( + QueryMatch( + section, + score.total, + score.priority, + _score_reasons(score), + ) + ) ranked.sort( key=lambda match: ( + -match.score, *(-value for value in match.priority), match.section.page.as_posix(), match.section.id, @@ -715,6 +817,8 @@ def query_sections( matches: list[QueryMatch] = [] page_counts: dict[Path, int] = {} for match in ranked: + if match.section.id in required_ids: + continue if any( _related_sections(match.section, kept.section) for kept in matches ): @@ -725,16 +829,26 @@ def query_sections( page_counts[match.section.page] = page_counts.get(match.section.page, 0) + 1 if len(matches) == limit: break - prelude = _write_prelude(all_sections) if mode == "write" else [] - match_ids = {match.section.id for match in matches} - prelude = [section for section in prelude if section.id not in match_ids] - return QueryResult(query=query, mode=mode, matches=matches, prelude=prelude) + return QueryResult( + query=query, + mode=mode, + matches=matches, + prelude=prelude, + limit=limit, + ) def _logical_source(section: Section) -> str: return f"references/{section.page.name}" +def _safe_json_string(value: str) -> str: + encoded = json.dumps(value, ensure_ascii=False) + for separator in ("\u0085", "\u2028", "\u2029"): + encoded = encoded.replace(separator, f"\\u{ord(separator):04x}") + return encoded + + def _plain_text_summary(body: str, limit: int = 180) -> str: without_fences = FENCED_CODE_RE.sub(" ", body) without_links = re.sub( @@ -758,14 +872,16 @@ def render_candidates(result: QueryResult) -> str: "# TSL Syntax Candidates", "", f"Mode: `{result.mode}`", - f"Query: {json.dumps(result.query, ensure_ascii=False)}", + f"Query: {_safe_json_string(result.query)}", ] candidates = [ - (section, 0, True, (0, 0, 0, 0, 0)) for section in result.prelude + (section, 0, True, ("required=1",)) for section in result.prelude ] + [ - (match.section, match.score, False, match.priority) for match in result.matches + (match.section, match.score, False, match.reasons) for match in result.matches ] - for index, (section, score, required, priority) in enumerate(candidates, start=1): + for index, (section, score, required, reasons) in enumerate( + candidates[: result.limit], start=1 + ): lines.extend( [ "", @@ -776,8 +892,8 @@ def render_candidates(result: QueryResult) -> str: f"Section ID: `{section.id}`", f"Source: `{_logical_source(section)}`", f"Heading: `{' > '.join(section.heading_path)}`", - f"Why: `rank={','.join(str(value) for value in priority)}`", - f"Summary: {_plain_text_summary(section.body)}", + f"Why: `{', '.join(reasons) or 'lexical=1'}`", + f"Summary: {_plain_text_summary(section.local_body)}", ] ) return "\n".join(lines).rstrip() + "\n" @@ -795,39 +911,6 @@ def render_section(section: Section) -> str: return "\n".join(lines).rstrip() + "\n" -def _duty_summary(body: str) -> str: - lines = body.splitlines() - paragraph: list[str] = [] - for line in lines[1:]: - stripped = line.strip() - if not stripped: - if paragraph: - break - continue - paragraph.append(stripped) - summary = " ".join(paragraph) - # 职责段以冒号引出列表时（如 09），首段本身空洞，把随后的 - # 列表项折叠进摘要才能保留实际覆盖面。 - if summary.endswith(("：", ":")): - items: list[str] = [] - seen_paragraph = False - for line in lines[1:]: - stripped = line.strip() - if not stripped: - continue - if not seen_paragraph: - if stripped == summary or stripped in summary: - seen_paragraph = True - continue - if stripped.startswith(("-", "*", "·")): - items.append(stripped.lstrip("-*· ").strip()) - elif items: - break - if items: - summary = summary + "；".join(items) + "。" - return summary - - def build_concept_map(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[tuple[str, str, str]]: entries: list[tuple[str, str, str]] = [] seen_pages: set[str] = set() @@ -837,7 +920,7 @@ def build_concept_map(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[tup continue if section.heading_path != (DUTY_HEADING,): continue - summary = _plain_text_summary(section.body, limit=600) + summary = _plain_text_summary(section.local_body, limit=600) if not summary: continue seen_pages.add(page_name) @@ -926,6 +1009,7 @@ def main(argv: list[str] | None = None) -> int: else: result = query_sections(args.query, args.mode, args.limit, args.references_dir) if not result.matches: + print(render_candidates(result), end="") print("no matching sections", file=sys.stderr) return 2 print(render_candidates(result), end="") diff --git a/test/test_tsl_syntax_lookup.py b/test/test_tsl_syntax_lookup.py index 715271f7..3414708f 100644 --- a/test/test_tsl_syntax_lookup.py +++ b/test/test_tsl_syntax_lookup.py @@ -72,6 +72,61 @@ class TslSyntaxLookupTests(unittest.TestCase): result = lookup.query_sections("数据库左连接", "write") self.assertEqual(result.matches[0].section.page.name, "14_ts_sql.md") + self.assertTrue( + any(reason.startswith("synonym=") for reason in result.matches[0].reasons) + ) + + def test_synonym_reason_only_appears_when_expansion_matches_candidate(self): + with tempfile.TemporaryDirectory() as tmp_dir: + references = Path(tmp_dir) + (references / "sample.md").write_text( + "# Sample\n\n" + "## 本篇职责\n\n" + "Synthetic synonym provenance.\n\n" + "## Exact\n\n" + "needle appears here without the expanded collection term.\n", + encoding="utf-8", + newline="\n", + ) + + result = lookup.query_sections( + "列表 needle", "explain", references_dir=references + ) + + exact = next( + match for match in result.matches if match.section.heading_path == ("Exact",) + ) + self.assertFalse( + any(reason.startswith("synonym=") for reason in exact.reasons), + exact.reasons, + ) + + def test_ascii_synonym_and_intent_triggers_require_token_boundaries(self): + with tempfile.TemporaryDirectory() as tmp_dir: + references = Path(tmp_dir) + (references / "15_debug_and_profiler.md").write_text( + "# Debug Probe\n\n" + "## 本篇职责\n\n" + "Synthetic boundary probe.\n\n" + "## Probe\n\n" + "probe body.\n", + encoding="utf-8", + newline="\n", + ) + + result = lookup.query_sections( + "debugger programmer probe", + "explain", + references_dir=references, + ) + + probe = next( + match for match in result.matches if match.section.heading_path == ("Probe",) + ) + self.assertFalse( + any(reason.startswith(("synonym=", "intent=")) for reason in probe.reasons), + probe.reasons, + ) def test_query_renders_compact_candidates_without_bodies_or_absolute_paths(self): result = lookup.query_sections("函数 默认参数", "write", limit=5) @@ -81,6 +136,8 @@ class TslSyntaxLookupTests(unittest.TestCase): self.assertIn("## Candidate 1", rendered) self.assertNotIn("```tsl", rendered) self.assertNotIn(str(lookup.DEFAULT_REFERENCES_DIR.resolve()), rendered) + self.assertNotIn("Why: `rank=", rendered) + self.assertRegex(rendered, r"Why: `[^`]*(intent|heading|identifier|body)=\d+") self.assertLess(len(rendered.encode("utf-8")), 8192) def test_query_echo_is_json_encoded_and_cannot_inject_markdown(self): @@ -96,6 +153,31 @@ class TslSyntaxLookupTests(unittest.TestCase): self.assertEqual(rendered.count("## Match 999"), 1) self.assertNotIn("\n## Match 999\n", rendered) + def test_query_echo_escapes_all_unicode_line_separators(self): + for separator in ("\u0085", "\u2028", "\u2029"): + with self.subTest(separator=hex(ord(separator))): + query = f"数组{separator}## Candidate 999" + rendered = lookup.render_candidates( + lookup.query_sections(query, "explain") + ) + + self.assertNotIn(separator, rendered) + self.assertIn(f"\\u{ord(separator):04x}", rendered) + self.assertNotIn("\n## Candidate 999\n", rendered) + + def test_mixed_language_paraphrases_ignore_unknown_ascii_fillers(self): + cases = ( + ("please 帮我写个最简单能跑的天软脚本", "write", "01_quickstart.md"), + ("TSL 中两个表如何做左外连接并聚合排序", "write", "14_ts_sql.md"), + ("debug 一下程序的性能问题", "diagnose", "15_debug_and_profiler.md"), + ("请写一个可以运行的最小 Tinysoft program", "write", "01_quickstart.md"), + ) + for query, mode, expected_page in cases: + with self.subTest(query=query): + result = lookup.query_sections(query, mode) + self.assertTrue(result.matches, query) + self.assertEqual(result.matches[0].section.page.name, expected_page) + def test_section_is_only_cli_path_that_returns_body(self): result = lookup.query_sections("基础函数", "write", limit=5) section = result.matches[0].section @@ -124,6 +206,61 @@ class TslSyntaxLookupTests(unittest.TestCase): self.assertNotIn(section.body.rstrip(), query_completed.stdout) self.assertIn(section.body.rstrip(), section_completed.stdout) + def test_body_only_match_prefers_deepest_section(self): + with tempfile.TemporaryDirectory() as tmp_dir: + references = Path(tmp_dir) + (references / "sample.md").write_text( + "# Sample\n\n" + "## 本篇职责\n\n" + "Synthetic hierarchy.\n\n" + "## Parent\n\n" + "Parent introduction.\n\n" + "### Child\n\n" + "Child introduction.\n\n" + "#### Exact Leaf\n\n" + "needle appears only in the leaf body.\n", + encoding="utf-8", + newline="\n", + ) + + result = lookup.query_sections( + "needle", "explain", references_dir=references + ) + + self.assertEqual(result.matches[0].section.heading_path[-1], "Exact Leaf") + + def test_limit_applies_to_final_rendered_candidate_list(self): + for limit in (1, 2, 5): + with self.subTest(limit=limit): + result = lookup.query_sections("函数 默认参数", "write", limit=limit) + rendered = lookup.render_candidates(result) + self.assertEqual(rendered.count("## Candidate "), limit) + + def test_required_context_stays_required_when_query_also_matches_it(self): + result = lookup.query_sections("语言核心事实速查", "write", limit=5) + rendered = lookup.render_candidates(result) + required_id = "01_quickstart--语言核心事实速查" + + self.assertIn( + f"Required: yes\nSection ID: `{required_id}`", + rendered, + ) + self.assertEqual(rendered.count(f"Section ID: `{required_id}`"), 1) + + def test_displayed_scores_are_sorted_descending(self): + for query, mode in ( + ("函数 默认参数", "write"), + ("变量怎么改", "write"), + ("数组", "diagnose"), + ("invalid statement 声明区", "diagnose"), + ("程序慢怎么计时找瓶颈", "diagnose"), + ): + with self.subTest(query=query, mode=mode): + result = lookup.query_sections(query, mode, limit=5) + scores = [match.score for match in result.matches] + + self.assertEqual(scores, sorted(scores, reverse=True)) + def test_parser_creates_unique_h2_h3_h4_section_ids(self): sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR) ids = [section.id for section in sections] @@ -143,6 +280,14 @@ class TslSyntaxLookupTests(unittest.TestCase): self.assertNotEqual(star, double_star) self.assertNotRegex(double_star, r"-2$") + def test_all_promised_symbolic_heading_slugs_are_distinct(self): + ids = { + lookup.section_id("sample.md", ("Examples", heading)) + for heading in ("with *", "with **", "operator[]", "walk ::", "walk :.") + } + + self.assertEqual(len(ids), 5) + def test_check_requires_one_nonempty_duty_section_per_page(self): with tempfile.TemporaryDirectory() as tmp_dir: references = Path(tmp_dir) @@ -206,6 +351,7 @@ class TslSyntaxLookupTests(unittest.TestCase): self.assertEqual(result.prelude, []) def test_no_match_returns_exit_code_two_in_every_mode(self): + query = "夔魍魉xyzqv" for mode in ("write", "diagnose", "explain"): with self.subTest(mode=mode): completed = subprocess.run( @@ -213,7 +359,7 @@ class TslSyntaxLookupTests(unittest.TestCase): sys.executable, str(SCRIPT), "--query", - "不存在的孤立语法词xyz", + query, "--mode", mode, ], @@ -222,6 +368,9 @@ class TslSyntaxLookupTests(unittest.TestCase): encoding="utf-8", ) self.assertEqual(completed.returncode, 2) + self.assertIn("# TSL Syntax Candidates", completed.stdout) + self.assertIn(f'Query: "{query}"', completed.stdout) + self.assertIn("no matching sections", completed.stderr) def test_missing_section_returns_nearest_section_ids(self): with tempfile.TemporaryDirectory() as tmp_dir: