diff --git a/skills/tsl-syntax-reference/scripts/lookup.py b/skills/tsl-syntax-reference/scripts/lookup.py index bfb86a51..31c0ea9e 100644 --- a/skills/tsl-syntax-reference/scripts/lookup.py +++ b/skills/tsl-syntax-reference/scripts/lookup.py @@ -50,6 +50,63 @@ PITFALL_PAGE_BOOST = 18 COUNTEREXAMPLE_BOOST = 8 EXACT_ERROR_BOOST = 14 MIXED_QUERY_MIN_SCORE = 10 +PAGE_INTENT_SCORE = 80 + +CHINESE_STOP_TOKENS = { + "一个", + "为什", + "什么", + "怎么", + "怎样", + "是否", + "能不", + "不能", + "帮我", + "想要", + "然后", + "里面", +} + +QUERY_SYNONYMS = { + "打印": ("输出", "writeLn"), + "打出来": ("输出", "writeLn"), + "左连接": ("左联接", "left join", "TS-SQL"), + "列表": ("数组",), + "复用文件": ("tsf", "unit"), + "多个文件": ("unit", "uses", "作用域"), + "跳出去": ("break", "控制流"), + "程序慢": ("性能分析", "计时", "profiler"), + "瓶颈": ("性能分析", "profiler"), + "字符串转整数": ("类型转换", "strToInt"), + "高性能矩阵": ("FMArray",), +} + +PAGE_INTENT_ALIASES = { + "01_quickstart.md": ("最简单能跑", "天软脚本"), + "02_core_model.md": ("脚本和可复用", "可复用函数文件"), + "03_values_and_literals.md": ("字符串和数组下标",), + "04_variables_and_constants.md": ("常量怎么声明", "变量能不能直接赋值"), + "05_functions_and_calls.md": ("默认参数", "函数怎么带"), + "06_expressions_and_operators.md": ("赋值和相等比较",), + "07_control_flow.md": ("跳出去", "循环里满足条件"), + "08_objects_and_classes.md": ("定义类", "创建对象"), + "09_units_and_scope.md": ("多个文件", "复用一组函数"), + "10_runtime_context_and_with.md": ("临时切换系统参数",), + "11_pitfalls.md": ("声明函数后面写代码", "语法报错"), + "12_matrix_and_collections.md": ("某行存在", "二维数组怎么判断"), + "13_resultset_and_filters.md": ("保留匹配行", "按某一列"), + "14_ts_sql.md": ("左连接", "左联接", "数据库", "分组排序"), + "15_debug_and_profiler.md": ("程序慢", "计时找瓶颈", "性能瓶颈"), + "16_lexical_structure_and_compile_options.md": ("变量名区分大小写", "注释怎么写"), + "17_types_and_conversions.md": ("字符串转整数", "类型转换"), + "18_external_calls_and_threads.md": ("调用 dll", "开线程"), + "19_namespace_libpath_and_unit_runtime.md": ("找不到 tsf", "搜索路径"), + "20_object_runtime_and_introspection.md": ("查看对象属于哪个类", "运行时对象"), + "21_builtin_runtime_objects.md": ("内存流",), + "22_matrix_deep_dive.md": ("矩阵求逆", "矩阵转置", "求逆和转置"), + "23_fmarray.md": ("高性能矩阵", "fmarray"), + "24_object_overloads_and_iteration.md": ("自定义对象支持下标", "for in"), +} @dataclass(frozen=True) @@ -74,11 +131,12 @@ class ValidationProblem: class QueryMatch: section: Section score: int - priority: tuple[int, int, int, int, int] = (0, 0, 0, 0, 0) + priority: tuple[int, ...] = () @dataclass(frozen=True) class ScoreBreakdown: + intent: int heading_path: int exact_term: int page_title: int @@ -88,16 +146,23 @@ class ScoreBreakdown: @property def lexical_total(self) -> int: - return self.heading_path + self.exact_term + self.page_title + self.body + return ( + self.intent + + self.heading_path + + self.exact_term + + self.page_title + + self.body + ) @property def total(self) -> int: return self.lexical_total + self.mode_boost @property - def priority(self) -> tuple[int, int, int, int, int]: + def priority(self) -> tuple[int, ...]: if self.diagnose_priority: return ( + self.intent, self.mode_boost, self.heading_path, self.exact_term, @@ -105,6 +170,7 @@ class ScoreBreakdown: self.body, ) return ( + self.intent, self.heading_path, self.exact_term, self.page_title, @@ -434,13 +500,44 @@ def validate_references( return problems -def query_tokens(text: str) -> set[str]: +def _base_query_tokens(text: str) -> set[str]: normalized = normalize(text) tokens = set(ASCII_TOKEN_RE.findall(normalized)) for run in CHINESE_RUN_RE.findall(normalized): tokens.add(run) tokens.update(run[index : index + 2] for index in range(len(run) - 1)) - return {token for token in tokens if token.strip()} + return { + token + for token in tokens + if token.strip() and token not in CHINESE_STOP_TOKENS + } + + +def query_tokens(text: str) -> set[str]: + tokens = _base_query_tokens(text) + normalized = normalize(text) + for phrase, synonyms in QUERY_SYNONYMS.items(): + if normalize(phrase) not in normalized: + continue + for synonym in synonyms: + tokens.update(_base_query_tokens(synonym)) + return tokens + + +def _text_contains_token(text: str, token: str) -> bool: + normalized_text = normalize(text) + if ASCII_TOKEN_RE.fullmatch(token): + text_tokens = _bare_tokens(set(ASCII_TOKEN_RE.findall(normalized_text))) + return token.rstrip(".$:+-") in text_tokens + return token in normalized_text + + +def _intent_score(section: Section, query: str) -> int: + normalized_query = normalize(query) + aliases = PAGE_INTENT_ALIASES.get(section.page.name, ()) + return PAGE_INTENT_SCORE * sum( + normalize(alias) in normalized_query for alias in aliases + ) def _has_chinese_context(section: Section, query: str) -> bool: @@ -475,13 +572,13 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown: page_title_score = 0 body_score = 0 for token in tokens: - if token in heading_text: + if _text_contains_token(heading_text, token): heading_score += HEADING_TOKEN_SCORE - if token in term_text: + if _text_contains_token(term_text, token): term_score += TERM_TOKEN_SCORE - if token in page_title_text: + if _text_contains_token(page_title_text, token): page_title_score += PAGE_TITLE_TOKEN_SCORE - if token in body_text: + if _text_contains_token(body_text, token): body_score += BODY_TOKEN_SCORE if normalized_query and normalized_query in heading_text: heading_score += HEADING_EXACT_SCORE @@ -502,6 +599,7 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown: if normalized_query and normalized_query in section.searchable_text: mode_boost += EXACT_ERROR_BOOST return ScoreBreakdown( + intent=_intent_score(section, query), heading_path=heading_score, exact_term=term_score, page_title=page_title_score, @@ -577,13 +675,16 @@ def query_sections( all_sections = load_sections(references_dir) sections = all_sections ascii_tokens = { - token for token in query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token) + token for token in _base_query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token) } if ascii_tokens: sections = [ section for section in sections - if any(token in section.searchable_text for token in ascii_tokens) + if any( + _text_contains_token(section.searchable_text, token) + for token in ascii_tokens + ) ] has_chinese = bool(CHINESE_RUN_RE.search(normalize(query))) minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1 @@ -612,12 +713,16 @@ def query_sections( # H2 聚合 section 的正文逐字包含其 H3 子节；父子同时入选时只保留 # 排名更高的一个，避免同一内容重复返回。 matches: list[QueryMatch] = [] + page_counts: dict[Path, int] = {} for match in ranked: if any( _related_sections(match.section, kept.section) for kept in matches ): continue + if page_counts.get(match.section.page, 0) >= 2: + continue matches.append(match) + page_counts[match.section.page] = page_counts.get(match.section.page, 0) + 1 if len(matches) == limit: break prelude = _write_prelude(all_sections) if mode == "write" else [] diff --git a/test/test_tsl_syntax_lookup.py b/test/test_tsl_syntax_lookup.py index 301546ae..715271f7 100644 --- a/test/test_tsl_syntax_lookup.py +++ b/test/test_tsl_syntax_lookup.py @@ -14,7 +14,65 @@ lookup = importlib.util.module_from_spec(spec) spec.loader.exec_module(lookup) +NATURAL_LANGUAGE_CASES = ( + ("帮我写个最简单能跑的天软脚本", "write", "01_quickstart.md"), + ("脚本和可复用函数文件有什么区别", "explain", "02_core_model.md"), + ("字符串和数组下标从几开始", "explain", "03_values_and_literals.md"), + ("常量怎么声明，变量能不能直接赋值", "explain", "04_variables_and_constants.md"), + ("函数怎么带默认参数", "write", "05_functions_and_calls.md"), + ("赋值和相等比较分别怎么写", "explain", "06_expressions_and_operators.md"), + ("循环里满足条件就跳出去", "write", "07_control_flow.md"), + ("怎么定义类并创建对象", "write", "08_objects_and_classes.md"), + ("多个文件复用一组函数怎么组织", "write", "09_units_and_scope.md"), + ("临时切换系统参数再调用函数", "write", "10_runtime_context_and_with.md"), + ("为什么声明函数后面写代码会报错", "diagnose", "11_pitfalls.md"), + ("二维数组怎么判断某行存在", "write", "12_matrix_and_collections.md"), + ("二维结果按某一列保留匹配行", "write", "13_resultset_and_filters.md"), + ("数据库左连接后分组排序", "write", "14_ts_sql.md"), + ("程序慢怎么计时找瓶颈", "diagnose", "15_debug_and_profiler.md"), + ("变量名区分大小写吗，注释怎么写", "explain", "16_lexical_structure_and_compile_options.md"), + ("字符串转整数失败怎么办", "diagnose", "17_types_and_conversions.md"), + ("调用 DLL 并开线程", "write", "18_external_calls_and_threads.md"), + ("找不到 tsf 文件怎么改搜索路径", "diagnose", "19_namespace_libpath_and_unit_runtime.md"), + ("运行时怎么查看对象属于哪个类", "explain", "20_object_runtime_and_introspection.md"), + ("内存流怎么读写", "write", "21_builtin_runtime_objects.md"), + ("矩阵求逆和转置", "write", "22_matrix_deep_dive.md"), + ("高性能矩阵怎么排序", "write", "23_fmarray.md"), + ("让自定义对象支持下标和 for in", "write", "24_object_overloads_and_iteration.md"), +) + + class TslSyntaxLookupTests(unittest.TestCase): + def test_natural_language_topic_matrix(self): + top1 = 0 + top5 = 0 + misses = [] + for query, mode, expected_page in NATURAL_LANGUAGE_CASES: + pages = [ + match.section.page.name + for match in lookup.query_sections(query, mode).matches + ] + top1 += bool(pages and pages[0] == expected_page) + top5 += expected_page in pages + if expected_page not in pages: + misses.append((query, expected_page, pages)) + + self.assertGreaterEqual(top1, 20, (top1, misses)) + self.assertEqual(top5, len(NATURAL_LANGUAGE_CASES), (top5, misses)) + + def test_short_ascii_tokens_use_identifier_boundaries(self): + result = lookup.query_sections("if", "explain") + ids = "\n".join(match.section.id for match in result.matches) + + self.assertIn("07_control_flow", ids) + self.assertNotIn("tinifile", ids) + self.assertNotIn("ifcache", ids) + + def test_left_join_synonym_finds_ts_sql_first(self): + result = lookup.query_sections("数据库左连接", "write") + + self.assertEqual(result.matches[0].section.page.name, "14_ts_sql.md") + def test_query_renders_compact_candidates_without_bodies_or_absolute_paths(self): result = lookup.query_sections("函数 默认参数", "write", limit=5) rendered = lookup.render_candidates(result)