csh/playbook
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐛 fix(tsl-syntax): close lookup review gaps

Browse Source
This commit is contained in:
csh
2026-07-13 09:17:00 +08:00
parent 455661c936
commit d663c2dae5
2 changed files with 299 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/tsl-syntax-reference/scripts/lookup.py
+149 -65
View File
@@ -46,7 +46,7 @@ PAGE_TITLE_EXACT_SCORE = 8
BODY_TOKEN_SCORE = 3
BODY_EXACT_SCORE = 4
DIRECT_EXAMPLE_BOOST = 8
PITFALL_PAGE_BOOST = 18
PITFALL_PAGE_BOOST = 30
COUNTEREXAMPLE_BOOST = 8
EXACT_ERROR_BOOST = 14
MIXED_QUERY_MIN_SCORE = 10
@@ -67,22 +67,36 @@ CHINESE_STOP_TOKENS = {
"里面",
}
ASCII_FILTER_STOP_TOKENS = {
"debug",
"please",
"program",
"tinysoft",
"tsl",
"tsf",
}
QUERY_SYNONYMS = {
"打印": ("输出", "writeLn"),
"打出来": ("输出", "writeLn"),
"左连接": ("左联接", "left join", "TS-SQL"),
"左外连接": ("左联接", "left join", "TS-SQL"),
"left outer join": ("left join", "左联接", "TS-SQL"),
"列表": ("数组",),
"复用文件": ("tsf", "unit"),
"多个文件": ("unit", "uses", "作用域"),
"跳出去": ("break", "控制流"),
"程序慢": ("性能分析", "计时", "profiler"),
"瓶颈": ("性能分析", "profiler"),
"debug": ("调试", "性能分析"),
"program": ("脚本",),
"tinysoft": ("天软", "TSL"),
"字符串转整数": ("类型转换", "strToInt"),
"高性能矩阵": ("FMArray",),
}
PAGE_INTENT_ALIASES = {
"01_quickstart.md": ("最简单能跑", "天软脚本"),
"01_quickstart.md": ("最简单能跑", "天软脚本", "tinysoft"),
"02_core_model.md": ("脚本和可复用", "可复用函数文件"),
"03_values_and_literals.md": ("字符串和数组下标",),
"04_variables_and_constants.md": ("常量怎么声明", "变量能不能直接赋值"),
@@ -95,8 +109,8 @@ PAGE_INTENT_ALIASES = {
"11_pitfalls.md": ("声明函数后面写代码", "语法报错"),
"12_matrix_and_collections.md": ("某行存在", "二维数组怎么判断"),
"13_resultset_and_filters.md": ("保留匹配行", "按某一列"),
"14_ts_sql.md": ("左连接", "左联接", "数据库", "分组排序"),
"15_debug_and_profiler.md": ("程序慢", "计时找瓶颈", "性能瓶颈"),
"14_ts_sql.md": ("左连接", "左外连接", "左联接", "数据库", "分组排序", "聚合排序"),
"15_debug_and_profiler.md": ("程序慢", "计时找瓶颈", "性能瓶颈", "性能问题", "debug"),
"16_lexical_structure_and_compile_options.md": ("变量名区分大小写", "注释怎么写"),
"17_types_and_conversions.md": ("字符串转整数", "类型转换"),
"18_external_calls_and_threads.md": ("调用 dll", "开线程"),
@@ -116,6 +130,7 @@ class Section:
page_title: str
heading_path: tuple[str, ...]
body: str
local_body: str
identities: tuple[str, ...]
searchable_text: str
@@ -132,6 +147,7 @@ class QueryMatch:
section: Section
score: int
priority: tuple[int, ...] = ()
reasons: tuple[str, ...] = ()
@dataclass(frozen=True)
@@ -142,6 +158,7 @@ class ScoreBreakdown:
page_title: int
body: int
mode_boost: int
synonym_hits: int
diagnose_priority: bool = False
@property
@@ -185,6 +202,7 @@ class QueryResult:
mode: str
matches: list[QueryMatch]
prelude: list[Section]
limit: int
def normalize(value: str) -> str:
@@ -304,9 +322,15 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section
end = next_start
break
body = "".join(lines[start:end])
local_end = (
headings[position + 1][0]
if position + 1 < len(headings)
else len(lines)
)
local_body = "".join(lines[start:local_end])
base_id = section_id(page.relative_to(references_dir), heading_path)
searchable_text = normalize(
"\n".join((page.stem, page_title, *heading_path, body))
"\n".join((page.stem, page_title, *heading_path, local_body))
)
sections.append(
Section(
@@ -315,7 +339,8 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section
page_title=page_title,
heading_path=heading_path,
body=body,
identities=_identities(body),
local_body=local_body,
identities=_identities(local_body),
searchable_text=searchable_text,
)
)
@@ -513,17 +538,43 @@ def _base_query_tokens(text: str) -> set[str]:
}
def query_tokens(text: str) -> set[str]:
tokens = _base_query_tokens(text)
normalized = normalize(text)
def _ascii_token_sequence(text: str) -> list[str]:
return [
token.rstrip(".$:+-")
for token in ASCII_TOKEN_RE.findall(normalize(text))
if token.rstrip(".$:+-")
]
def _query_contains_phrase(query: str, phrase: str) -> bool:
normalized_phrase = normalize(phrase)
if CHINESE_RUN_RE.search(normalized_phrase):
return normalized_phrase in normalize(query)
phrase_tokens = _ascii_token_sequence(phrase)
query_tokens_in_order = _ascii_token_sequence(query)
if not phrase_tokens:
return False
width = len(phrase_tokens)
return any(
query_tokens_in_order[index : index + width] == phrase_tokens
for index in range(len(query_tokens_in_order) - width + 1)
)
def _synonym_tokens(text: str) -> set[str]:
tokens: set[str] = set()
for phrase, synonyms in QUERY_SYNONYMS.items():
if normalize(phrase) not in normalized:
if not _query_contains_phrase(text, phrase):
continue
for synonym in synonyms:
tokens.update(_base_query_tokens(synonym))
return tokens
def query_tokens(text: str) -> set[str]:
return _base_query_tokens(text) | _synonym_tokens(text)
def _text_contains_token(text: str, token: str) -> bool:
normalized_text = normalize(text)
if ASCII_TOKEN_RE.fullmatch(token):
@@ -532,11 +583,17 @@ def _text_contains_token(text: str, token: str) -> bool:
return token in normalized_text
def _text_contains_exact_query(text: str, query: str) -> bool:
normalized_query = normalize(query).strip()
if ASCII_TOKEN_RE.fullmatch(normalized_query):
return _text_contains_token(text, normalized_query)
return bool(normalized_query and normalized_query in normalize(text))
def _intent_score(section: Section, query: str) -> int:
normalized_query = normalize(query)
aliases = PAGE_INTENT_ALIASES.get(section.page.name, ())
return PAGE_INTENT_SCORE * sum(
normalize(alias) in normalized_query for alias in aliases
_query_contains_phrase(query, alias) for alias in aliases
)
@@ -565,8 +622,16 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
tokens = query_tokens(query)
heading_text = normalize("\n".join(section.heading_path))
page_title_text = normalize(section.page_title)
body_text = normalize(section.body)
term_text = _code_text(section.body)
body_text = normalize(section.local_body)
term_text = _code_text(section.local_body)
expanded_only_tokens = _synonym_tokens(query) - _base_query_tokens(query)
synonym_hits = sum(
any(
_text_contains_token(text, token)
for text in (heading_text, term_text, page_title_text, body_text)
)
for token in expanded_only_tokens
)
heading_score = 0
term_score = 0
page_title_score = 0
@@ -580,13 +645,13 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
page_title_score += PAGE_TITLE_TOKEN_SCORE
if _text_contains_token(body_text, token):
body_score += BODY_TOKEN_SCORE
if normalized_query and normalized_query in heading_text:
if _text_contains_exact_query(heading_text, query):
heading_score += HEADING_EXACT_SCORE
if normalized_query and normalized_query in term_text:
if _text_contains_exact_query(term_text, query):
term_score += TERM_EXACT_SCORE
if normalized_query and normalized_query in page_title_text:
if _text_contains_exact_query(page_title_text, query):
page_title_score += PAGE_TITLE_EXACT_SCORE
if normalized_query and normalized_query in body_text:
if _text_contains_exact_query(body_text, query):
body_score += BODY_EXACT_SCORE
mode_boost = 0
if mode == "write" and "可直接照写示例" in section.identities:
@@ -605,10 +670,26 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
page_title=page_title_score,
body=body_score,
mode_boost=mode_boost,
synonym_hits=synonym_hits,
diagnose_priority=mode == "diagnose",
)
def _score_reasons(score: ScoreBreakdown) -> tuple[str, ...]:
components = (
("intent", score.intent),
("heading", score.heading_path),
("identifier", score.exact_term),
("page_title", score.page_title),
("body", score.body),
("mode", score.mode_boost),
)
reasons = tuple(f"{name}={value}" for name, value in components if value)
if score.synonym_hits:
reasons += (f"synonym={score.synonym_hits}",)
return reasons
def _write_prelude(sections: list[Section]) -> list[Section]:
prelude: list[Section] = []
for page_name, heading in WRITE_PRELUDE_ANCHORS:
@@ -647,7 +728,10 @@ def _ascii_anchor(section: Section, ascii_tokens: set[str]) -> bool:
ASCII_TOKEN_RE.findall(normalize("\n".join(section.heading_path)))
)
code_text = "\n".join(
(*INLINE_CODE_RE.findall(section.body), *FENCED_CODE_RE.findall(section.body))
(
*INLINE_CODE_RE.findall(section.local_body),
*FENCED_CODE_RE.findall(section.local_body),
)
)
code_tokens = set(
ASCII_TOKEN_RE.findall(normalize(STRING_LITERAL_RE.sub(" ", code_text)))
@@ -674,8 +758,18 @@ def query_sections(
raise ValueError("limit must be between 1 and 10")
all_sections = load_sections(references_dir)
sections = all_sections
prelude = _write_prelude(all_sections) if mode == "write" else []
required_ids = {section.id for section in prelude}
raw_ascii_tokens = {
token
for token in _base_query_tokens(query)
if ASCII_TOKEN_RE.fullmatch(token)
and token not in ASCII_FILTER_STOP_TOKENS
}
ascii_tokens = {
token for token in _base_query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token)
token
for token in raw_ascii_tokens
if any(_ascii_anchor(section, {token}) for section in all_sections)
}
if ascii_tokens:
sections = [
@@ -702,9 +796,17 @@ def query_sections(
score = _score_section(section, query, mode)
if score.lexical_total < minimum_score:
continue
ranked.append(QueryMatch(section, score.total, score.priority))
ranked.append(
QueryMatch(
section,
score.total,
score.priority,
_score_reasons(score),
)
)
ranked.sort(
key=lambda match: (
-match.score,
*(-value for value in match.priority),
match.section.page.as_posix(),
match.section.id,
@@ -715,6 +817,8 @@ def query_sections(
matches: list[QueryMatch] = []
page_counts: dict[Path, int] = {}
for match in ranked:
if match.section.id in required_ids:
continue
if any(
_related_sections(match.section, kept.section) for kept in matches
):
@@ -725,16 +829,26 @@ def query_sections(
page_counts[match.section.page] = page_counts.get(match.section.page, 0) + 1
if len(matches) == limit:
break
prelude = _write_prelude(all_sections) if mode == "write" else []
match_ids = {match.section.id for match in matches}
prelude = [section for section in prelude if section.id not in match_ids]
return QueryResult(query=query, mode=mode, matches=matches, prelude=prelude)
return QueryResult(
query=query,
mode=mode,
matches=matches,
prelude=prelude,
limit=limit,
)
def _logical_source(section: Section) -> str:
return f"references/{section.page.name}"
def _safe_json_string(value: str) -> str:
encoded = json.dumps(value, ensure_ascii=False)
for separator in ("\u0085", "\u2028", "\u2029"):
encoded = encoded.replace(separator, f"\\u{ord(separator):04x}")
return encoded
def _plain_text_summary(body: str, limit: int = 180) -> str:
without_fences = FENCED_CODE_RE.sub(" ", body)
without_links = re.sub(
@@ -758,14 +872,16 @@ def render_candidates(result: QueryResult) -> str:
"# TSL Syntax Candidates",
"",
f"Mode: `{result.mode}`",
f"Query: {json.dumps(result.query, ensure_ascii=False)}",
f"Query: {_safe_json_string(result.query)}",
]
candidates = [
(section, 0, True, (0, 0, 0, 0, 0)) for section in result.prelude
(section, 0, True, ("required=1",)) for section in result.prelude
] + [
(match.section, match.score, False, match.priority) for match in result.matches
(match.section, match.score, False, match.reasons) for match in result.matches
]
for index, (section, score, required, priority) in enumerate(candidates, start=1):
for index, (section, score, required, reasons) in enumerate(
candidates[: result.limit], start=1
):
lines.extend(
[
"",
@@ -776,8 +892,8 @@ def render_candidates(result: QueryResult) -> str:
f"Section ID: `{section.id}`",
f"Source: `{_logical_source(section)}`",
f"Heading: `{' > '.join(section.heading_path)}`",
f"Why: `rank={','.join(str(value) for value in priority)}`",
f"Summary: {_plain_text_summary(section.body)}",
f"Why: `{', '.join(reasons) or 'lexical=1'}`",
f"Summary: {_plain_text_summary(section.local_body)}",
]
)
return "\n".join(lines).rstrip() + "\n"
@@ -795,39 +911,6 @@ def render_section(section: Section) -> str:
return "\n".join(lines).rstrip() + "\n"
def _duty_summary(body: str) -> str:
lines = body.splitlines()
paragraph: list[str] = []
for line in lines[1:]:
stripped = line.strip()
if not stripped:
if paragraph:
break
continue
paragraph.append(stripped)
summary = " ".join(paragraph)
# 职责段以冒号引出列表时（如 09），首段本身空洞，把随后的
# 列表项折叠进摘要才能保留实际覆盖面。
if summary.endswith(("：", ":")):
items: list[str] = []
seen_paragraph = False
for line in lines[1:]:
stripped = line.strip()
if not stripped:
continue
if not seen_paragraph:
if stripped == summary or stripped in summary:
seen_paragraph = True
continue
if stripped.startswith(("-", "*", "·")):
items.append(stripped.lstrip("-*· ").strip())
elif items:
break
if items:
summary = summary + "；".join(items) + "。"
return summary
def build_concept_map(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[tuple[str, str, str]]:
entries: list[tuple[str, str, str]] = []
seen_pages: set[str] = set()
@@ -837,7 +920,7 @@ def build_concept_map(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[tup
continue
if section.heading_path != (DUTY_HEADING,):
continue
summary = _plain_text_summary(section.body, limit=600)
summary = _plain_text_summary(section.local_body, limit=600)
if not summary:
continue
seen_pages.add(page_name)
@@ -926,6 +1009,7 @@ def main(argv: list[str] | None = None) -> int:
else:
result = query_sections(args.query, args.mode, args.limit, args.references_dir)
if not result.matches:
print(render_candidates(result), end="")
print("no matching sections", file=sys.stderr)
return 2
print(render_candidates(result), end="")
test/test_tsl_syntax_lookup.py
+150 -1
View File
@@ -72,6 +72,61 @@ class TslSyntaxLookupTests(unittest.TestCase):
result = lookup.query_sections("数据库左连接", "write")
self.assertEqual(result.matches[0].section.page.name, "14_ts_sql.md")
self.assertTrue(
any(reason.startswith("synonym=") for reason in result.matches[0].reasons)
)
def test_synonym_reason_only_appears_when_expansion_matches_candidate(self):
with tempfile.TemporaryDirectory() as tmp_dir:
references = Path(tmp_dir)
(references / "sample.md").write_text(
"# Sample\n\n"
"## 本篇职责\n\n"
"Synthetic synonym provenance.\n\n"
"## Exact\n\n"
"needle appears here without the expanded collection term.\n",
encoding="utf-8",
newline="\n",
)
result = lookup.query_sections(
"列表 needle", "explain", references_dir=references
)
exact = next(
match for match in result.matches if match.section.heading_path == ("Exact",)
)
self.assertFalse(
any(reason.startswith("synonym=") for reason in exact.reasons),
exact.reasons,
)
def test_ascii_synonym_and_intent_triggers_require_token_boundaries(self):
with tempfile.TemporaryDirectory() as tmp_dir:
references = Path(tmp_dir)
(references / "15_debug_and_profiler.md").write_text(
"# Debug Probe\n\n"
"## 本篇职责\n\n"
"Synthetic boundary probe.\n\n"
"## Probe\n\n"
"probe body.\n",
encoding="utf-8",
newline="\n",
)
result = lookup.query_sections(
"debugger programmer probe",
"explain",
references_dir=references,
)
probe = next(
match for match in result.matches if match.section.heading_path == ("Probe",)
)
self.assertFalse(
any(reason.startswith(("synonym=", "intent=")) for reason in probe.reasons),
probe.reasons,
)
def test_query_renders_compact_candidates_without_bodies_or_absolute_paths(self):
result = lookup.query_sections("函数 默认参数", "write", limit=5)
@@ -81,6 +136,8 @@ class TslSyntaxLookupTests(unittest.TestCase):
self.assertIn("## Candidate 1", rendered)
self.assertNotIn("```tsl", rendered)
self.assertNotIn(str(lookup.DEFAULT_REFERENCES_DIR.resolve()), rendered)
self.assertNotIn("Why: `rank=", rendered)
self.assertRegex(rendered, r"Why: `[^`]*(intent|heading|identifier|body)=\d+")
self.assertLess(len(rendered.encode("utf-8")), 8192)
def test_query_echo_is_json_encoded_and_cannot_inject_markdown(self):
@@ -96,6 +153,31 @@ class TslSyntaxLookupTests(unittest.TestCase):
self.assertEqual(rendered.count("## Match 999"), 1)
self.assertNotIn("\n## Match 999\n", rendered)
def test_query_echo_escapes_all_unicode_line_separators(self):
for separator in ("\u0085", "\u2028", "\u2029"):
with self.subTest(separator=hex(ord(separator))):
query = f"数组{separator}## Candidate 999"
rendered = lookup.render_candidates(
lookup.query_sections(query, "explain")
)
self.assertNotIn(separator, rendered)
self.assertIn(f"\\u{ord(separator):04x}", rendered)
self.assertNotIn("\n## Candidate 999\n", rendered)
def test_mixed_language_paraphrases_ignore_unknown_ascii_fillers(self):
cases = (
("please 帮我写个最简单能跑的天软脚本", "write", "01_quickstart.md"),
("TSL 中两个表如何做左外连接并聚合排序", "write", "14_ts_sql.md"),
("debug 一下程序的性能问题", "diagnose", "15_debug_and_profiler.md"),
("请写一个可以运行的最小 Tinysoft program", "write", "01_quickstart.md"),
)
for query, mode, expected_page in cases:
with self.subTest(query=query):
result = lookup.query_sections(query, mode)
self.assertTrue(result.matches, query)
self.assertEqual(result.matches[0].section.page.name, expected_page)
def test_section_is_only_cli_path_that_returns_body(self):
result = lookup.query_sections("基础函数", "write", limit=5)
section = result.matches[0].section
@@ -124,6 +206,61 @@ class TslSyntaxLookupTests(unittest.TestCase):
self.assertNotIn(section.body.rstrip(), query_completed.stdout)
self.assertIn(section.body.rstrip(), section_completed.stdout)
def test_body_only_match_prefers_deepest_section(self):
with tempfile.TemporaryDirectory() as tmp_dir:
references = Path(tmp_dir)
(references / "sample.md").write_text(
"# Sample\n\n"
"## 本篇职责\n\n"
"Synthetic hierarchy.\n\n"
"## Parent\n\n"
"Parent introduction.\n\n"
"### Child\n\n"
"Child introduction.\n\n"
"#### Exact Leaf\n\n"
"needle appears only in the leaf body.\n",
encoding="utf-8",
newline="\n",
)
result = lookup.query_sections(
"needle", "explain", references_dir=references
)
self.assertEqual(result.matches[0].section.heading_path[-1], "Exact Leaf")
def test_limit_applies_to_final_rendered_candidate_list(self):
for limit in (1, 2, 5):
with self.subTest(limit=limit):
result = lookup.query_sections("函数 默认参数", "write", limit=limit)
rendered = lookup.render_candidates(result)
self.assertEqual(rendered.count("## Candidate "), limit)
def test_required_context_stays_required_when_query_also_matches_it(self):
result = lookup.query_sections("语言核心事实速查", "write", limit=5)
rendered = lookup.render_candidates(result)
required_id = "01_quickstart--语言核心事实速查"
self.assertIn(
f"Required: yes\nSection ID: `{required_id}`",
rendered,
)
self.assertEqual(rendered.count(f"Section ID: `{required_id}`"), 1)
def test_displayed_scores_are_sorted_descending(self):
for query, mode in (
("函数 默认参数", "write"),
("变量怎么改", "write"),
("数组", "diagnose"),
("invalid statement 声明区", "diagnose"),
("程序慢怎么计时找瓶颈", "diagnose"),
):
with self.subTest(query=query, mode=mode):
result = lookup.query_sections(query, mode, limit=5)
scores = [match.score for match in result.matches]
self.assertEqual(scores, sorted(scores, reverse=True))
def test_parser_creates_unique_h2_h3_h4_section_ids(self):
sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR)
ids = [section.id for section in sections]
@@ -143,6 +280,14 @@ class TslSyntaxLookupTests(unittest.TestCase):
self.assertNotEqual(star, double_star)
self.assertNotRegex(double_star, r"-2$")
def test_all_promised_symbolic_heading_slugs_are_distinct(self):
ids = {
lookup.section_id("sample.md", ("Examples", heading))
for heading in ("with *", "with **", "operator[]", "walk ::", "walk :.")
}
self.assertEqual(len(ids), 5)
def test_check_requires_one_nonempty_duty_section_per_page(self):
with tempfile.TemporaryDirectory() as tmp_dir:
references = Path(tmp_dir)
@@ -206,6 +351,7 @@ class TslSyntaxLookupTests(unittest.TestCase):
self.assertEqual(result.prelude, [])
def test_no_match_returns_exit_code_two_in_every_mode(self):
query = "夔魍魉xyzqv"
for mode in ("write", "diagnose", "explain"):
with self.subTest(mode=mode):
completed = subprocess.run(
@@ -213,7 +359,7 @@ class TslSyntaxLookupTests(unittest.TestCase):
sys.executable,
str(SCRIPT),
"--query",
"不存在的孤立语法词xyz",
query,
"--mode",
mode,
],
@@ -222,6 +368,9 @@ class TslSyntaxLookupTests(unittest.TestCase):
encoding="utf-8",
)
self.assertEqual(completed.returncode, 2)
self.assertIn("# TSL Syntax Candidates", completed.stdout)
self.assertIn(f'Query: "{query}"', completed.stdout)
self.assertIn("no matching sections", completed.stderr)
def test_missing_section_returns_nearest_section_ids(self):
with tempfile.TemporaryDirectory() as tmp_dir: