✨ feat(tsl-syntax-reference): harden retrieval and restructure pages
- flag weak candidates (no intent/heading/identifier/tag hit) and exit 2 when every candidate is weak: mis-hits used to be indistinguishable from real hits, so the retry-with-better-terms loop never fired - accept multiple ids per --section for batch fetch, failing atomically on any unknown id so a partial fetch cannot pass as complete - move query synonyms and page intent aliases to data/lexicon.json and enforce alias/page correspondence in --check; curation data no longer lives in the engine - document the weak-hit rule, batch fetch and prelude-once guidance in SKILL.md, with curation discipline in data/README.md - drop 11_pitfalls.md, renumber the trailing pages and spread retrieval tags across topics; lexicon keys are page filenames, so the renumbering and the new --check rule cannot land in separate commits
This commit is contained in:
@@ -12,13 +12,20 @@ from urllib.parse import unquote
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REFERENCES_DIR = SKILL_ROOT / "references"
|
||||
DEFAULT_LEXICON_PATH = SKILL_ROOT / "data" / "lexicon.json"
|
||||
HEADING_RE = re.compile(r"^(#{1,6})(?!#)\s+(.+?)\s*$")
|
||||
FENCE_RE = re.compile(r"^```([^`]*)$")
|
||||
MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
|
||||
ASCII_TOKEN_RE = re.compile(r"[a-z_][a-z0-9_.$:+-]*", re.IGNORECASE)
|
||||
CHINESE_RUN_RE = re.compile(r"[\u3400-\u9fff]+")
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
INLINE_CODE_RE = re.compile(r"`([^`\n]+)`")
|
||||
FENCED_CODE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL)
|
||||
# 章节级检索标签:写在标题下方的 HTML 注释里,渲染后不可见。
|
||||
# 用途是补用户侧说法与正文词面之间的缺口(正文写 `var` / `operator[]`,
|
||||
# 用户说"传引用" / "中括号")。逗号或顿号分隔,只影响检索,不是事实正文。
|
||||
SECTION_TAG_RE = re.compile(r"<!--\s*tags?\s*:\s*(.*?)\s*-->", re.DOTALL | re.IGNORECASE)
|
||||
TAG_SEPARATOR_RE = re.compile(r"[,,、]\s*")
|
||||
IDENTITY_PREFIX = "代码块身份:"
|
||||
BLOCK_DESCRIPTION_PREFIX = "代码块说明:"
|
||||
ALLOWED_IDENTITIES = {
|
||||
@@ -31,6 +38,21 @@ ALLOWED_IDENTITIES = {
|
||||
ROUTER_PHRASES = ("路由中心", "选择一个主专题", "候选页继续判断")
|
||||
EXCLUDED_REFERENCE_FILES = {"index.md"}
|
||||
DUTY_HEADING = "本篇职责"
|
||||
# 这些标题在多页重复出现,不承载单一事实,不参与「必须有 tag」的约束。
|
||||
GENERIC_HEADINGS = frozenset(
|
||||
{
|
||||
"本篇职责",
|
||||
"核心规则",
|
||||
"禁止项",
|
||||
"可直接照写示例",
|
||||
"默认生成模板",
|
||||
"本页不生成的范围",
|
||||
"示例与行为",
|
||||
"决策边界和禁止项",
|
||||
"文件模型示例",
|
||||
"术语对照",
|
||||
}
|
||||
)
|
||||
SUSPICIOUS_FENCE_RE = re.compile(r"^(?:\s+`{3}|`{4,})")
|
||||
WRITE_PRELUDE_ANCHORS = (
|
||||
("02_core_model.md", "文件模型核心规则"),
|
||||
@@ -39,6 +61,15 @@ WRITE_PRELUDE_ANCHORS = (
|
||||
|
||||
HEADING_TOKEN_SCORE = 12
|
||||
HEADING_EXACT_SCORE = 20
|
||||
# 标签是人工策展的检索意图,权重与标识符同级:足以在页内区分章节,
|
||||
# 但压不过标题精确命中,避免标签写宽了就绑架整页。
|
||||
TAG_TOKEN_SCORE = 10
|
||||
TAG_EXACT_SCORE = 16
|
||||
# 标签按「被查询覆盖的比例」判命中,而不是逐 token 累加。中文按 2-gram 切分,
|
||||
# 逐 token 累加会让「参数」「函数」这类泛化词命中一整页的标签,把整页抬起来;
|
||||
# 要求覆盖过半,则「只读参数」不会被「临时改系统参数」点亮,而单 token 的
|
||||
# 精确标签(lambda)仍然 100% 覆盖、照常命中。
|
||||
TAG_COVERAGE_THRESHOLD = 0.5
|
||||
TERM_TOKEN_SCORE = 10
|
||||
TERM_EXACT_SCORE = 16
|
||||
PAGE_TITLE_TOKEN_SCORE = 5
|
||||
@@ -46,11 +77,14 @@ PAGE_TITLE_EXACT_SCORE = 8
|
||||
BODY_TOKEN_SCORE = 3
|
||||
BODY_EXACT_SCORE = 4
|
||||
DIRECT_EXAMPLE_BOOST = 8
|
||||
PITFALL_PAGE_BOOST = 30
|
||||
COUNTEREXAMPLE_BOOST = 8
|
||||
COUNTEREXAMPLE_BOOST = 30
|
||||
EXACT_ERROR_BOOST = 14
|
||||
MIXED_QUERY_MIN_SCORE = 10
|
||||
PAGE_INTENT_SCORE = 80
|
||||
# 单页最多贡献几条候选。上限保证候选跨页分散,但页级 intent 命中(+80)会把
|
||||
# 整页抬起来,页内只剩十几分的词法差异在排序;上限过小时正确章节会被同页
|
||||
# 邻居挤掉,且加大 --limit 也救不回来。3 是实测下节级准确率与跨页分散的平衡点。
|
||||
PAGE_MATCH_CAP = 3
|
||||
|
||||
CHINESE_STOP_TOKENS = {
|
||||
"一个",
|
||||
@@ -76,51 +110,45 @@ ASCII_FILTER_STOP_TOKENS = {
|
||||
"tsf",
|
||||
}
|
||||
|
||||
QUERY_SYNONYMS = {
|
||||
"打印": ("输出", "writeLn"),
|
||||
"打出来": ("输出", "writeLn"),
|
||||
"左连接": ("左联接", "left join", "TS-SQL"),
|
||||
"左外连接": ("左联接", "left join", "TS-SQL"),
|
||||
"left outer join": ("left join", "左联接", "TS-SQL"),
|
||||
"列表": ("数组",),
|
||||
"复用文件": ("tsf", "unit"),
|
||||
"多个文件": ("unit", "uses", "作用域"),
|
||||
"跳出去": ("break", "控制流"),
|
||||
"程序慢": ("性能分析", "计时", "profiler"),
|
||||
"瓶颈": ("性能分析", "profiler"),
|
||||
"debug": ("调试", "性能分析"),
|
||||
"program": ("脚本",),
|
||||
"tinysoft": ("天软", "TSL"),
|
||||
"字符串转整数": ("类型转换", "strToInt"),
|
||||
"高性能矩阵": ("FMArray",),
|
||||
}
|
||||
QUERY_SYNONYMS: dict[str, tuple[str, ...]]
|
||||
PAGE_INTENT_ALIASES: dict[str, tuple[str, ...]]
|
||||
|
||||
PAGE_INTENT_ALIASES = {
|
||||
"01_quickstart.md": ("最简单能跑", "天软脚本", "tinysoft"),
|
||||
"02_core_model.md": ("脚本和可复用", "可复用函数文件"),
|
||||
"03_values_and_literals.md": ("字符串和数组下标",),
|
||||
"04_variables_and_constants.md": ("常量怎么声明", "变量能不能直接赋值"),
|
||||
"05_functions_and_calls.md": ("默认参数", "函数怎么带"),
|
||||
"06_expressions_and_operators.md": ("赋值和相等比较",),
|
||||
"07_control_flow.md": ("跳出去", "循环里满足条件"),
|
||||
"08_objects_and_classes.md": ("定义类", "创建对象"),
|
||||
"09_units_and_scope.md": ("多个文件", "复用一组函数"),
|
||||
"10_runtime_context_and_with.md": ("临时切换系统参数",),
|
||||
"11_pitfalls.md": ("声明函数后面写代码", "语法报错"),
|
||||
"12_matrix_and_collections.md": ("某行存在", "二维数组怎么判断"),
|
||||
"13_resultset_and_filters.md": ("保留匹配行", "按某一列"),
|
||||
"14_ts_sql.md": ("左连接", "左外连接", "左联接", "数据库", "分组排序", "聚合排序"),
|
||||
"15_debug_and_profiler.md": ("程序慢", "计时找瓶颈", "性能瓶颈", "性能问题", "debug"),
|
||||
"16_lexical_structure_and_compile_options.md": ("变量名区分大小写", "注释怎么写"),
|
||||
"17_types_and_conversions.md": ("字符串转整数", "类型转换"),
|
||||
"18_external_calls_and_threads.md": ("调用 dll", "开线程"),
|
||||
"19_namespace_libpath_and_unit_runtime.md": ("找不到 tsf", "搜索路径"),
|
||||
"20_object_runtime_and_introspection.md": ("查看对象属于哪个类", "运行时对象"),
|
||||
"21_builtin_runtime_objects.md": ("内存流",),
|
||||
"22_matrix_deep_dive.md": ("矩阵求逆", "矩阵转置", "求逆和转置"),
|
||||
"23_fmarray.md": ("高性能矩阵", "fmarray"),
|
||||
"24_object_overloads_and_iteration.md": ("自定义对象支持下标", "for in"),
|
||||
}
|
||||
|
||||
def _load_lexicon(
|
||||
path: Path = DEFAULT_LEXICON_PATH,
|
||||
) -> tuple[dict[str, tuple[str, ...]], dict[str, tuple[str, ...]]]:
|
||||
"""加载策展词表(口语同义词与页级意图短语)。
|
||||
|
||||
词表是持续生长的策展数据,与检索引擎分离维护在 data/lexicon.json;
|
||||
策展纪律见 data/README.md。加载失败必须响亮报错——静默回退为空表
|
||||
会让全部自然语言入口消失而检索仍然"正常"返回。
|
||||
"""
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
raise SystemExit(f"词表文件缺失:{path};检查 skill 安装是否完整")
|
||||
except json.JSONDecodeError as error:
|
||||
raise SystemExit(f"词表文件不是合法 JSON:{path}:{error}")
|
||||
|
||||
def _table(name: str) -> dict[str, tuple[str, ...]]:
|
||||
table = data.get(name)
|
||||
if not isinstance(table, dict):
|
||||
raise SystemExit(f"{path}: 缺少 {name} 表或不是对象")
|
||||
result: dict[str, tuple[str, ...]] = {}
|
||||
for key, values in table.items():
|
||||
if (
|
||||
not isinstance(values, list)
|
||||
or not values
|
||||
or not all(isinstance(item, str) and item.strip() for item in values)
|
||||
):
|
||||
raise SystemExit(f"{path}: {name}[{key!r}] 必须是非空字符串数组")
|
||||
result[key] = tuple(values)
|
||||
return result
|
||||
|
||||
return _table("query_synonyms"), _table("page_intent_aliases")
|
||||
|
||||
|
||||
QUERY_SYNONYMS, PAGE_INTENT_ALIASES = _load_lexicon()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -132,6 +160,7 @@ class Section:
|
||||
body: str
|
||||
local_body: str
|
||||
identities: tuple[str, ...]
|
||||
tags: tuple[str, ...]
|
||||
searchable_text: str
|
||||
|
||||
|
||||
@@ -148,6 +177,7 @@ class QueryMatch:
|
||||
score: int
|
||||
priority: tuple[int, ...] = ()
|
||||
reasons: tuple[str, ...] = ()
|
||||
weak: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -155,6 +185,7 @@ class ScoreBreakdown:
|
||||
intent: int
|
||||
heading_path: int
|
||||
exact_term: int
|
||||
tag: int
|
||||
page_title: int
|
||||
body: int
|
||||
mode_boost: int
|
||||
@@ -167,10 +198,19 @@ class ScoreBreakdown:
|
||||
self.intent
|
||||
+ self.heading_path
|
||||
+ self.exact_term
|
||||
+ self.tag
|
||||
+ self.page_title
|
||||
+ self.body
|
||||
)
|
||||
|
||||
@property
|
||||
def weak(self) -> bool:
|
||||
# 没有任何强字段命中(意图短语、标题、标识符、标签),只靠正文 /
|
||||
# 页标题的低分撞词进入候选。实测这是"查询根本不在本 skill 事实域"
|
||||
# 时的典型形态(如通用词二字撞上正文),而策展用例从不落进来;
|
||||
# 全部候选皆弱时按无匹配处理。
|
||||
return self.intent + self.heading_path + self.exact_term + self.tag == 0
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return self.lexical_total + self.mode_boost
|
||||
@@ -183,6 +223,7 @@ class ScoreBreakdown:
|
||||
self.mode_boost,
|
||||
self.heading_path,
|
||||
self.exact_term,
|
||||
self.tag,
|
||||
self.page_title,
|
||||
self.body,
|
||||
)
|
||||
@@ -190,6 +231,7 @@ class ScoreBreakdown:
|
||||
self.intent,
|
||||
self.heading_path,
|
||||
self.exact_term,
|
||||
self.tag,
|
||||
self.page_title,
|
||||
self.mode_boost,
|
||||
self.body,
|
||||
@@ -299,6 +341,16 @@ def _identities(body: str) -> tuple[str, ...]:
|
||||
return tuple(identities)
|
||||
|
||||
|
||||
def _section_tags(body: str) -> tuple[str, ...]:
|
||||
tags: list[str] = []
|
||||
for block in SECTION_TAG_RE.findall(body):
|
||||
for tag in TAG_SEPARATOR_RE.split(block.replace("\n", " ")):
|
||||
tag = tag.strip()
|
||||
if tag and tag not in tags:
|
||||
tags.append(tag)
|
||||
return tuple(tags)
|
||||
|
||||
|
||||
def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section]:
|
||||
sections: list[Section] = []
|
||||
for page in _reference_pages(Path(references_dir)):
|
||||
@@ -329,8 +381,9 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section
|
||||
)
|
||||
local_body = "".join(lines[start:local_end])
|
||||
base_id = section_id(page.relative_to(references_dir), heading_path)
|
||||
tags = _section_tags(local_body)
|
||||
searchable_text = normalize(
|
||||
"\n".join((page.stem, page_title, *heading_path, local_body))
|
||||
"\n".join((page.stem, page_title, *heading_path, *tags, local_body))
|
||||
)
|
||||
sections.append(
|
||||
Section(
|
||||
@@ -341,6 +394,7 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section
|
||||
body=body,
|
||||
local_body=local_body,
|
||||
identities=_identities(local_body),
|
||||
tags=tags,
|
||||
searchable_text=searchable_text,
|
||||
)
|
||||
)
|
||||
@@ -413,11 +467,61 @@ def _local_link_problems(
|
||||
return problems
|
||||
|
||||
|
||||
def _tag_problems(sections: list[Section]) -> list[ValidationProblem]:
|
||||
"""章节 tag 的结构校验。
|
||||
|
||||
tag 是页内区分章节的主要信号,写空、漏写或页内重复都会静默削弱检索,
|
||||
而其余校验一概发现不了。含代码围栏的具体章节是事实落点,必须可被
|
||||
自然语言命中;纯交接说明(正文只指向别页、没有围栏)反而不该有 tag,
|
||||
否则会和真正拥有事实的那一页抢候选。
|
||||
"""
|
||||
problems: list[ValidationProblem] = []
|
||||
seen_per_page: dict[str, dict[str, str]] = {}
|
||||
for section in sections:
|
||||
if not section.heading_path:
|
||||
continue
|
||||
heading = section.heading_path[-1]
|
||||
has_fence = bool(FENCED_CODE_RE.search(section.local_body))
|
||||
if has_fence and heading not in GENERIC_HEADINGS and not section.tags:
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
section.page, 1, f"含代码围栏的章节缺少检索 tag:{heading}"
|
||||
)
|
||||
)
|
||||
for tag in section.tags:
|
||||
if not tag.strip():
|
||||
problems.append(
|
||||
ValidationProblem(section.page, 1, f"空 tag:{heading}")
|
||||
)
|
||||
continue
|
||||
owners = seen_per_page.setdefault(section.page.name, {})
|
||||
if tag in owners:
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
section.page,
|
||||
1,
|
||||
f"页内 tag 重复:「{tag}」同时属于「{owners[tag]}」和「{heading}」",
|
||||
)
|
||||
)
|
||||
else:
|
||||
owners[tag] = heading
|
||||
return problems
|
||||
|
||||
|
||||
def validate_references(
|
||||
references_dir: Path = DEFAULT_REFERENCES_DIR,
|
||||
) -> list[ValidationProblem]:
|
||||
references_dir = Path(references_dir)
|
||||
problems: list[ValidationProblem] = []
|
||||
# 逐页校验在零页时全部静默通过;参考页缺失属于安装/路径错误,必须报错,
|
||||
# 否则 --check 会为一个空目录返回成功。
|
||||
if not _reference_pages(references_dir):
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
references_dir, 1, "references 中没有可校验的参考页;检查路径或重新安装 skill"
|
||||
)
|
||||
)
|
||||
return problems
|
||||
index_page = references_dir / "index.md"
|
||||
if index_page.exists():
|
||||
problems.append(ValidationProblem(index_page, 1, "references 中不得保留 index.md"))
|
||||
@@ -477,6 +581,7 @@ def validate_references(
|
||||
ValidationProblem(page, index, f"包含人工路由协议:{phrase}")
|
||||
)
|
||||
sections = load_sections(references_dir)
|
||||
problems.extend(_tag_problems(sections))
|
||||
ids: dict[str, Section] = {}
|
||||
for section in sections:
|
||||
if section.id in ids:
|
||||
@@ -522,6 +627,26 @@ def validate_references(
|
||||
f"概念地图摘要为空:{section.page.name} 的「{DUTY_HEADING}」",
|
||||
)
|
||||
)
|
||||
# 词表校验只对内置参考目录有意义:alias 键指向的是内置页文件名,
|
||||
# 用 --references-dir 校验合成目录(测试)时跳过,避免整表误报。
|
||||
if references_dir.resolve() == DEFAULT_REFERENCES_DIR.resolve():
|
||||
page_names = {page.name for page in _reference_pages(references_dir)}
|
||||
for stale in sorted(set(PAGE_INTENT_ALIASES) - page_names):
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
DEFAULT_LEXICON_PATH,
|
||||
1,
|
||||
f"page_intent_aliases 指向不存在的参考页:{stale}",
|
||||
)
|
||||
)
|
||||
for missing in sorted(page_names - set(PAGE_INTENT_ALIASES)):
|
||||
problems.append(
|
||||
ValidationProblem(
|
||||
DEFAULT_LEXICON_PATH,
|
||||
1,
|
||||
f"参考页缺少 page_intent_aliases 自然语言入口:{missing}",
|
||||
)
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
@@ -549,7 +674,12 @@ def _ascii_token_sequence(text: str) -> list[str]:
|
||||
def _query_contains_phrase(query: str, phrase: str) -> bool:
|
||||
normalized_phrase = normalize(phrase)
|
||||
if CHINESE_RUN_RE.search(normalized_phrase):
|
||||
return normalized_phrase in normalize(query)
|
||||
# 含中文的短语按去空白后的串比较。SKILL.md 要求智能体传「术语」而不是
|
||||
# 用户原话,术语常以空格分隔(「数组 下标 起点」),逐字子串匹配会
|
||||
# 整条落空;去空白后 phrase 仍要求连续出现,不放宽词序。
|
||||
return _WHITESPACE_RE.sub("", normalized_phrase) in _WHITESPACE_RE.sub(
|
||||
"", normalize(query)
|
||||
)
|
||||
phrase_tokens = _ascii_token_sequence(phrase)
|
||||
query_tokens_in_order = _ascii_token_sequence(query)
|
||||
if not phrase_tokens:
|
||||
@@ -611,6 +741,23 @@ def _has_chinese_context(section: Section, query: str) -> bool:
|
||||
return not runs
|
||||
|
||||
|
||||
def _tag_matched_tokens(tags: tuple[str, ...], query_token_set: set[str]) -> int:
|
||||
"""标签命中的 token 数;覆盖率不过门槛的标签整条不计分。
|
||||
|
||||
覆盖率只做门控,计分仍按命中 token 数——否则一条深度吻合的标签
|
||||
(命中 4 个 token)和一条勉强擦边的标签得分相同,信号被抹平。
|
||||
"""
|
||||
matched = 0
|
||||
for tag in tags:
|
||||
tag_tokens = _base_query_tokens(tag)
|
||||
if not tag_tokens:
|
||||
continue
|
||||
covered = sum(1 for token in tag_tokens if token in query_token_set)
|
||||
if covered / len(tag_tokens) >= TAG_COVERAGE_THRESHOLD:
|
||||
matched += covered
|
||||
return matched
|
||||
|
||||
|
||||
def _code_text(body: str) -> str:
|
||||
inline = INLINE_CODE_RE.findall(body)
|
||||
fenced = FENCED_CODE_RE.findall(body)
|
||||
@@ -622,18 +769,20 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
|
||||
tokens = query_tokens(query)
|
||||
heading_text = normalize("\n".join(section.heading_path))
|
||||
page_title_text = normalize(section.page_title)
|
||||
body_text = normalize(section.local_body)
|
||||
body_text = normalize(SECTION_TAG_RE.sub(" ", section.local_body))
|
||||
tag_text = normalize("\n".join(section.tags))
|
||||
term_text = _code_text(section.local_body)
|
||||
expanded_only_tokens = _synonym_tokens(query) - _base_query_tokens(query)
|
||||
synonym_hits = sum(
|
||||
any(
|
||||
_text_contains_token(text, token)
|
||||
for text in (heading_text, term_text, page_title_text, body_text)
|
||||
for text in (heading_text, term_text, tag_text, page_title_text, body_text)
|
||||
)
|
||||
for token in expanded_only_tokens
|
||||
)
|
||||
heading_score = 0
|
||||
term_score = 0
|
||||
tag_score = TAG_TOKEN_SCORE * _tag_matched_tokens(section.tags, tokens)
|
||||
page_title_score = 0
|
||||
body_score = 0
|
||||
for token in tokens:
|
||||
@@ -649,6 +798,10 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
|
||||
heading_score += HEADING_EXACT_SCORE
|
||||
if _text_contains_exact_query(term_text, query):
|
||||
term_score += TERM_EXACT_SCORE
|
||||
if section.tags and any(
|
||||
normalize(tag) == normalize(query).strip() for tag in section.tags
|
||||
):
|
||||
tag_score += TAG_EXACT_SCORE
|
||||
if _text_contains_exact_query(page_title_text, query):
|
||||
page_title_score += PAGE_TITLE_EXACT_SCORE
|
||||
if _text_contains_exact_query(body_text, query):
|
||||
@@ -657,8 +810,8 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
|
||||
if mode == "write" and "可直接照写示例" in section.identities:
|
||||
mode_boost += DIRECT_EXAMPLE_BOOST
|
||||
if mode == "diagnose":
|
||||
if section.page.name == "11_pitfalls.md":
|
||||
mode_boost += PITFALL_PAGE_BOOST
|
||||
# 反例按代码块身份加分,不按页名。反例分散在各专题页里,
|
||||
# 没有一页专门收口它们。
|
||||
if "反例 / 不可照写" in section.identities:
|
||||
mode_boost += COUNTEREXAMPLE_BOOST
|
||||
if normalized_query and normalized_query in section.searchable_text:
|
||||
@@ -667,6 +820,7 @@ def _score_section(section: Section, query: str, mode: str) -> ScoreBreakdown:
|
||||
intent=_intent_score(section, query),
|
||||
heading_path=heading_score,
|
||||
exact_term=term_score,
|
||||
tag=tag_score,
|
||||
page_title=page_title_score,
|
||||
body=body_score,
|
||||
mode_boost=mode_boost,
|
||||
@@ -680,6 +834,7 @@ def _score_reasons(score: ScoreBreakdown) -> tuple[str, ...]:
|
||||
("intent", score.intent),
|
||||
("heading", score.heading_path),
|
||||
("identifier", score.exact_term),
|
||||
("tag", score.tag),
|
||||
("page_title", score.page_title),
|
||||
("body", score.body),
|
||||
("mode", score.mode_boost),
|
||||
@@ -802,6 +957,7 @@ def query_sections(
|
||||
score.total,
|
||||
score.priority,
|
||||
_score_reasons(score),
|
||||
score.weak,
|
||||
)
|
||||
)
|
||||
ranked.sort(
|
||||
@@ -823,7 +979,7 @@ def query_sections(
|
||||
_related_sections(match.section, kept.section) for kept in matches
|
||||
):
|
||||
continue
|
||||
if page_counts.get(match.section.page, 0) >= 2:
|
||||
if page_counts.get(match.section.page, 0) >= PAGE_MATCH_CAP:
|
||||
continue
|
||||
matches.append(match)
|
||||
page_counts[match.section.page] = page_counts.get(match.section.page, 0) + 1
|
||||
@@ -850,7 +1006,9 @@ def _safe_json_string(value: str) -> str:
|
||||
|
||||
|
||||
def _plain_text_summary(body: str, limit: int = 180) -> str:
|
||||
without_fences = FENCED_CODE_RE.sub(" ", body)
|
||||
# 标签是检索元数据,不是事实正文;不能泄进候选摘要。
|
||||
without_tags = SECTION_TAG_RE.sub(" ", body)
|
||||
without_fences = FENCED_CODE_RE.sub(" ", without_tags)
|
||||
without_links = re.sub(
|
||||
r"!?\[([^\]]*)\]\([^)]+\)", lambda match: match.group(1), without_fences
|
||||
)
|
||||
@@ -875,12 +1033,16 @@ def render_candidates(result: QueryResult) -> str:
|
||||
f"Query: {_safe_json_string(result.query)}",
|
||||
]
|
||||
candidates = [
|
||||
(section, 0, True, ("required=1",)) for section in result.prelude
|
||||
(section, 0, True, ("required=1",), False) for section in result.prelude
|
||||
] + [
|
||||
(match.section, match.score, False, match.reasons) for match in result.matches
|
||||
(match.section, match.score, False, match.reasons, match.weak)
|
||||
for match in result.matches
|
||||
]
|
||||
for index, (section, score, required, reasons) in enumerate(
|
||||
candidates[: result.limit], start=1
|
||||
# --limit is the budget for query matches only; write 模式的前置章节额外附加,
|
||||
# 否则 limit 小于前置章节数时会一条真实候选都不返回。
|
||||
budget = result.limit + len(result.prelude)
|
||||
for index, (section, score, required, reasons, weak) in enumerate(
|
||||
candidates[:budget], start=1
|
||||
):
|
||||
lines.extend(
|
||||
[
|
||||
@@ -889,6 +1051,13 @@ def render_candidates(result: QueryResult) -> str:
|
||||
"",
|
||||
f"Score: {score}",
|
||||
f"Required: {'yes' if required else 'no'}",
|
||||
]
|
||||
)
|
||||
if weak:
|
||||
# 只在弱命中时输出该行:没有 Weak 行即为强命中。
|
||||
lines.append("Weak: yes")
|
||||
lines.extend(
|
||||
[
|
||||
f"Section ID: `{section.id}`",
|
||||
f"Source: `{_logical_source(section)}`",
|
||||
f"Heading: `{' > '.join(section.heading_path)}`",
|
||||
@@ -948,16 +1117,72 @@ def _configure_utf8() -> None:
|
||||
reconfigure(encoding="utf-8")
|
||||
|
||||
|
||||
HELP_EPILOG = """\
|
||||
检索分两步,缺一步都不算取回事实:
|
||||
|
||||
1. 先取候选(只有摘要和 Section ID,不含事实正文)
|
||||
lookup.py --query "命名参数 默认参数" --mode write
|
||||
2. 再按候选里的 Section ID 取回正文;多个要素各自跑完第 1 步后,
|
||||
选定的 Section ID 可以合并成一次取回
|
||||
lookup.py --section "05_functions_and_calls--可直接照写示例--基础函数-过程骨架" \\
|
||||
"02_core_model--文件模型核心规则"
|
||||
|
||||
查询词无从下手时先 --map 把需求映射到 TSL 概念;改动参考页或 data/ 词表后用 --check 校验。
|
||||
"""
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Search TSL syntax reference sections")
|
||||
action = parser.add_mutually_exclusive_group(required=True)
|
||||
action.add_argument("--query")
|
||||
action.add_argument("--section")
|
||||
action.add_argument("--map", dest="show_map", action="store_true")
|
||||
action.add_argument("--check", action="store_true")
|
||||
parser.add_argument("--mode", choices=("write", "diagnose", "explain"))
|
||||
parser.add_argument("--limit", type=int, default=5)
|
||||
parser.add_argument("--references-dir", type=Path, default=DEFAULT_REFERENCES_DIR)
|
||||
parser = argparse.ArgumentParser(
|
||||
description="检索 TSL 语法参考页,取回可照写的语法事实。",
|
||||
epilog=HELP_EPILOG,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
action = parser.add_argument_group("动作(必选其一)").add_mutually_exclusive_group(
|
||||
required=True
|
||||
)
|
||||
action.add_argument(
|
||||
"--query", help="按用户原话、报错原文或语法要素名检索候选章节;需配合 --mode"
|
||||
)
|
||||
action.add_argument(
|
||||
"--section",
|
||||
nargs="+",
|
||||
metavar="SECTION_ID",
|
||||
help="按 Section ID 取回章节正文,可一次传多个 ID 批量取回"
|
||||
"(ID 抄自 --query 输出);这是唯一的事实来源",
|
||||
)
|
||||
action.add_argument(
|
||||
"--map",
|
||||
dest="show_map",
|
||||
action="store_true",
|
||||
help="输出各专题的职责摘要,用于把自然语言需求映射到 TSL 概念;不含可照写事实",
|
||||
)
|
||||
action.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="校验参考页的结构、代码块身份和本地链接;发现问题时退出码为 1",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=("write", "diagnose", "explain"),
|
||||
help="检索意图,仅用于 --query:"
|
||||
"write 编写或修改代码,额外附加文件模型与核心事实速查;"
|
||||
"diagnose 定位语法错误,优先易错点与反例;"
|
||||
"explain 解释语言规则或代码含义",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=5,
|
||||
help="--query 返回的候选条数上限,取值 1..10(默认 %(default)s);"
|
||||
"write 模式的前置章节不占该预算",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--references-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_REFERENCES_DIR,
|
||||
metavar="DIR",
|
||||
help="参考页目录(默认为本 skill 内置的 references/)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
@@ -979,11 +1204,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser = _parser()
|
||||
args = parser.parse_args(argv)
|
||||
if not 1 <= args.limit <= 10:
|
||||
parser.error("--limit must be between 1 and 10")
|
||||
parser.error("--limit 取值必须在 1..10 之间")
|
||||
if args.query is not None and args.mode is None:
|
||||
parser.error("--mode is required with --query")
|
||||
parser.error("--query 必须同时指定 --mode")
|
||||
if args.mode is not None and args.query is None:
|
||||
parser.error("--mode only applies to --query")
|
||||
parser.error("--mode 仅用于 --query")
|
||||
if args.show_map:
|
||||
print(render_concept_map(build_concept_map(args.references_dir)), end="")
|
||||
return 0
|
||||
@@ -994,25 +1219,35 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 1 if problems else 0
|
||||
if args.section is not None:
|
||||
sections = load_sections(args.references_dir)
|
||||
section = next(
|
||||
(item for item in sections if item.id == args.section),
|
||||
None,
|
||||
by_id = {item.id: item for item in sections}
|
||||
requested = list(dict.fromkeys(args.section))
|
||||
missing = [item for item in requested if item not in by_id]
|
||||
if missing:
|
||||
# 原子失败:只要有一个 ID 不存在就不输出任何正文,
|
||||
# 避免智能体把"部分取回"误当作全部要素已取回。
|
||||
for requested_id in missing:
|
||||
print(f"section not found: {requested_id}", file=sys.stderr)
|
||||
print("Nearest section IDs:", file=sys.stderr)
|
||||
for candidate in _nearest_section_ids(requested_id, sections):
|
||||
print(f"- {candidate}", file=sys.stderr)
|
||||
return 2
|
||||
print(
|
||||
"\n".join(render_section(by_id[item]) for item in requested),
|
||||
end="",
|
||||
)
|
||||
if section is None:
|
||||
print(f"section not found: {args.section}", file=sys.stderr)
|
||||
print("Nearest section IDs:", file=sys.stderr)
|
||||
for candidate in _nearest_section_ids(args.section, sections):
|
||||
print(f"- {candidate}", file=sys.stderr)
|
||||
return 2
|
||||
print(render_section(section), end="")
|
||||
return 0
|
||||
else:
|
||||
result = query_sections(args.query, args.mode, args.limit, args.references_dir)
|
||||
if not result.matches:
|
||||
print(render_candidates(result), end="")
|
||||
print("no matching sections", file=sys.stderr)
|
||||
return 2
|
||||
result = query_sections(args.query, args.mode, args.limit, args.references_dir)
|
||||
print(render_candidates(result), end="")
|
||||
if not result.matches:
|
||||
print("no matching sections", file=sys.stderr)
|
||||
return 2
|
||||
if all(match.weak for match in result.matches):
|
||||
print(
|
||||
"only weak candidates (no intent/heading/identifier/tag hit); "
|
||||
"视同无匹配,改进查询词后重试",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user