🐛 fix(tsl-syntax): gate lookup mode boosts
This commit is contained in:
@@ -271,7 +271,7 @@ def query_tokens(text: str) -> set[str]:
|
|||||||
return {token for token in tokens if token.strip()}
|
return {token for token in tokens if token.strip()}
|
||||||
|
|
||||||
|
|
||||||
def _score_section(section: Section, query: str, mode: str) -> int:
|
def _lexical_score(section: Section, query: str) -> int:
|
||||||
normalized_query = normalize(query).strip()
|
normalized_query = normalize(query).strip()
|
||||||
tokens = query_tokens(query)
|
tokens = query_tokens(query)
|
||||||
heading_text = normalize("\n".join((section.page_title, *section.heading_path)))
|
heading_text = normalize("\n".join((section.page_title, *section.heading_path)))
|
||||||
@@ -285,6 +285,12 @@ def _score_section(section: Section, query: str, mode: str) -> int:
|
|||||||
score += EXACT_QUERY_SCORE
|
score += EXACT_QUERY_SCORE
|
||||||
if any(token in normalize(section.page.stem) for token in tokens):
|
if any(token in normalize(section.page.stem) for token in tokens):
|
||||||
score += PAGE_NAME_SCORE
|
score += PAGE_NAME_SCORE
|
||||||
|
return score
|
||||||
|
|
||||||
|
|
||||||
|
def _score_section(section: Section, query: str, mode: str, lexical_score: int) -> int:
|
||||||
|
normalized_query = normalize(query).strip()
|
||||||
|
score = lexical_score
|
||||||
if mode == "write" and "可直接照写示例" in section.identities:
|
if mode == "write" and "可直接照写示例" in section.identities:
|
||||||
score += DIRECT_EXAMPLE_BOOST
|
score += DIRECT_EXAMPLE_BOOST
|
||||||
if mode == "diagnose":
|
if mode == "diagnose":
|
||||||
@@ -327,7 +333,8 @@ def query_sections(
|
|||||||
raise ValueError(f"unsupported mode: {mode}")
|
raise ValueError(f"unsupported mode: {mode}")
|
||||||
if not 1 <= limit <= 10:
|
if not 1 <= limit <= 10:
|
||||||
raise ValueError("limit must be between 1 and 10")
|
raise ValueError("limit must be between 1 and 10")
|
||||||
sections = load_sections(references_dir)
|
all_sections = load_sections(references_dir)
|
||||||
|
sections = all_sections
|
||||||
ascii_tokens = {
|
ascii_tokens = {
|
||||||
token for token in query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token)
|
token for token in query_tokens(query) if ASCII_TOKEN_RE.fullmatch(token)
|
||||||
}
|
}
|
||||||
@@ -337,15 +344,19 @@ def query_sections(
|
|||||||
for section in sections
|
for section in sections
|
||||||
if any(token in section.searchable_text for token in ascii_tokens)
|
if any(token in section.searchable_text for token in ascii_tokens)
|
||||||
]
|
]
|
||||||
ranked = [
|
|
||||||
QueryMatch(section, _score_section(section, query, mode)) for section in sections
|
|
||||||
]
|
|
||||||
has_chinese = bool(CHINESE_RUN_RE.search(normalize(query)))
|
has_chinese = bool(CHINESE_RUN_RE.search(normalize(query)))
|
||||||
minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
|
minimum_score = MIXED_QUERY_MIN_SCORE if ascii_tokens and has_chinese else 1
|
||||||
ranked = [match for match in ranked if match.score >= minimum_score]
|
ranked: list[QueryMatch] = []
|
||||||
|
for section in sections:
|
||||||
|
lexical_score = _lexical_score(section, query)
|
||||||
|
if lexical_score < minimum_score:
|
||||||
|
continue
|
||||||
|
ranked.append(
|
||||||
|
QueryMatch(section, _score_section(section, query, mode, lexical_score))
|
||||||
|
)
|
||||||
ranked.sort(key=lambda match: (-match.score, match.section.page.as_posix(), match.section.id))
|
ranked.sort(key=lambda match: (-match.score, match.section.page.as_posix(), match.section.id))
|
||||||
matches = ranked[:limit]
|
matches = ranked[:limit]
|
||||||
prelude = _write_prelude(sections) if mode == "write" else []
|
prelude = _write_prelude(all_sections) if mode == "write" else []
|
||||||
match_ids = {match.section.id for match in matches}
|
match_ids = {match.section.id for match in matches}
|
||||||
prelude = [section for section in prelude if section.id not in match_ids]
|
prelude = [section for section in prelude if section.id not in match_ids]
|
||||||
return QueryResult(query=query, mode=mode, matches=matches, prelude=prelude)
|
return QueryResult(query=query, mode=mode, matches=matches, prelude=prelude)
|
||||||
|
|||||||
@@ -35,20 +35,28 @@ class TslSyntaxLookupTests(unittest.TestCase):
|
|||||||
result = lookup.query_sections("数组下标", "explain", limit=3)
|
result = lookup.query_sections("数组下标", "explain", limit=3)
|
||||||
self.assertEqual(result.prelude, [])
|
self.assertEqual(result.prelude, [])
|
||||||
|
|
||||||
def test_no_match_returns_exit_code_two(self):
|
def test_no_match_returns_exit_code_two_in_every_mode(self):
|
||||||
completed = subprocess.run(
|
for mode in ("write", "diagnose", "explain"):
|
||||||
[
|
with self.subTest(mode=mode):
|
||||||
sys.executable,
|
completed = subprocess.run(
|
||||||
str(SCRIPT),
|
[
|
||||||
"--query",
|
sys.executable,
|
||||||
"不存在的孤立语法词xyz",
|
str(SCRIPT),
|
||||||
"--mode",
|
"--query",
|
||||||
"explain",
|
"不存在的孤立语法词xyz",
|
||||||
],
|
"--mode",
|
||||||
capture_output=True,
|
mode,
|
||||||
text=True,
|
],
|
||||||
)
|
capture_output=True,
|
||||||
self.assertEqual(completed.returncode, 2)
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
self.assertEqual(completed.returncode, 2)
|
||||||
|
|
||||||
|
def test_write_identifier_query_keeps_required_context(self):
|
||||||
|
result = lookup.query_sections("varByRef 命名参数", "write", limit=5)
|
||||||
|
prelude_pages = {section.page.name for section in result.prelude}
|
||||||
|
self.assertEqual(prelude_pages, {"01_quickstart.md", "02_core_model.md"})
|
||||||
|
|
||||||
def test_check_rejects_unknown_code_block_identity(self):
|
def test_check_rejects_unknown_code_block_identity(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
|
|||||||
Reference in New Issue
Block a user