🐛 fix(tsl-syntax): index precise sections and enforce map integrity
This commit is contained in:
@@ -125,8 +125,19 @@ def normalize(value: str) -> str:
|
|||||||
return unicodedata.normalize("NFKC", value).casefold()
|
return unicodedata.normalize("NFKC", value).casefold()
|
||||||
|
|
||||||
|
|
||||||
|
SYMBOL_SLUG_REPLACEMENTS = (
|
||||||
|
("**", " double-star "),
|
||||||
|
("[]", " index "),
|
||||||
|
("::", " double-colon "),
|
||||||
|
(":.", " colon-dot "),
|
||||||
|
("*", " star "),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _slug(value: str) -> str:
|
def _slug(value: str) -> str:
|
||||||
normalized = normalize(value)
|
normalized = normalize(value)
|
||||||
|
for symbol, replacement in SYMBOL_SLUG_REPLACEMENTS:
|
||||||
|
normalized = normalized.replace(symbol, replacement)
|
||||||
slug = re.sub(r"[^\w]+", "-", normalized, flags=re.UNICODE).strip("-_")
|
slug = re.sub(r"[^\w]+", "-", normalized, flags=re.UNICODE).strip("-_")
|
||||||
return slug or "section"
|
return slug or "section"
|
||||||
|
|
||||||
@@ -162,7 +173,7 @@ def _heading_records(lines: list[str]) -> tuple[str, list[tuple[int, int, str]]]
|
|||||||
title = match.group(2)
|
title = match.group(2)
|
||||||
if level == 1 and not page_title:
|
if level == 1 and not page_title:
|
||||||
page_title = title
|
page_title = title
|
||||||
elif level in (2, 3):
|
elif level in (2, 3, 4):
|
||||||
records.append((index, level, title))
|
records.append((index, level, title))
|
||||||
return page_title, records
|
return page_title, records
|
||||||
|
|
||||||
@@ -206,18 +217,21 @@ def _identities(body: str) -> tuple[str, ...]:
|
|||||||
|
|
||||||
def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section]:
|
def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section]:
|
||||||
sections: list[Section] = []
|
sections: list[Section] = []
|
||||||
id_counts: dict[str, int] = {}
|
|
||||||
for page in _reference_pages(Path(references_dir)):
|
for page in _reference_pages(Path(references_dir)):
|
||||||
text = page.read_text(encoding="utf-8")
|
text = page.read_text(encoding="utf-8")
|
||||||
lines = text.splitlines(keepends=True)
|
lines = text.splitlines(keepends=True)
|
||||||
page_title, headings = _heading_records(lines)
|
page_title, headings = _heading_records(lines)
|
||||||
h2_title = ""
|
heading_stack: dict[int, str] = {}
|
||||||
for position, (start, level, title) in enumerate(headings):
|
for position, (start, level, title) in enumerate(headings):
|
||||||
if level == 2:
|
for stacked_level in tuple(heading_stack):
|
||||||
h2_title = title
|
if stacked_level >= level:
|
||||||
heading_path = (title,)
|
del heading_stack[stacked_level]
|
||||||
else:
|
heading_stack[level] = title
|
||||||
heading_path = (h2_title, title) if h2_title else (title,)
|
heading_path = tuple(
|
||||||
|
heading_stack[item]
|
||||||
|
for item in range(2, level + 1)
|
||||||
|
if item in heading_stack
|
||||||
|
)
|
||||||
end = len(lines)
|
end = len(lines)
|
||||||
for next_start, next_level, _ in headings[position + 1 :]:
|
for next_start, next_level, _ in headings[position + 1 :]:
|
||||||
if next_level <= level:
|
if next_level <= level:
|
||||||
@@ -225,16 +239,12 @@ def load_sections(references_dir: Path = DEFAULT_REFERENCES_DIR) -> list[Section
|
|||||||
break
|
break
|
||||||
body = "".join(lines[start:end])
|
body = "".join(lines[start:end])
|
||||||
base_id = section_id(page.relative_to(references_dir), heading_path)
|
base_id = section_id(page.relative_to(references_dir), heading_path)
|
||||||
id_counts[base_id] = id_counts.get(base_id, 0) + 1
|
|
||||||
unique_id = base_id
|
|
||||||
if id_counts[base_id] > 1:
|
|
||||||
unique_id = f"{base_id}-{id_counts[base_id]}"
|
|
||||||
searchable_text = normalize(
|
searchable_text = normalize(
|
||||||
"\n".join((page.stem, page_title, *heading_path, body))
|
"\n".join((page.stem, page_title, *heading_path, body))
|
||||||
)
|
)
|
||||||
sections.append(
|
sections.append(
|
||||||
Section(
|
Section(
|
||||||
id=unique_id,
|
id=base_id,
|
||||||
page=page,
|
page=page,
|
||||||
page_title=page_title,
|
page_title=page_title,
|
||||||
heading_path=heading_path,
|
heading_path=heading_path,
|
||||||
@@ -325,6 +335,50 @@ def validate_references(
|
|||||||
lines = text.splitlines()
|
lines = text.splitlines()
|
||||||
problems.extend(_identity_problems(page, lines))
|
problems.extend(_identity_problems(page, lines))
|
||||||
problems.extend(_local_link_problems(page, text, references_dir))
|
problems.extend(_local_link_problems(page, text, references_dir))
|
||||||
|
page_title, headings = _heading_records(text.splitlines(keepends=True))
|
||||||
|
h1_count = 0
|
||||||
|
in_fence = False
|
||||||
|
previous_level = 1
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
if FENCE_RE.match(line):
|
||||||
|
in_fence = not in_fence
|
||||||
|
continue
|
||||||
|
if in_fence:
|
||||||
|
continue
|
||||||
|
match = HEADING_RE.match(line)
|
||||||
|
if match is None:
|
||||||
|
continue
|
||||||
|
level = len(match.group(1))
|
||||||
|
if level == 1:
|
||||||
|
h1_count += 1
|
||||||
|
previous_level = 1
|
||||||
|
continue
|
||||||
|
if level not in (2, 3, 4):
|
||||||
|
continue
|
||||||
|
if level > previous_level + 1:
|
||||||
|
problems.append(
|
||||||
|
ValidationProblem(page, index + 1, "H2/H3/H4 标题层级跳跃")
|
||||||
|
)
|
||||||
|
previous_level = level
|
||||||
|
if h1_count != 1 or not page_title:
|
||||||
|
problems.append(ValidationProblem(page, 1, "每页必须有且仅有一个非空 H1"))
|
||||||
|
duty_sections = []
|
||||||
|
for position, (start, level, title) in enumerate(headings):
|
||||||
|
if level != 2 or title != DUTY_HEADING:
|
||||||
|
continue
|
||||||
|
end = len(lines)
|
||||||
|
for next_start, next_level, _ in headings[position + 1 :]:
|
||||||
|
if next_level <= level:
|
||||||
|
end = next_start
|
||||||
|
break
|
||||||
|
body = "\n".join(lines[start + 1 : end]).strip()
|
||||||
|
duty_sections.append(body)
|
||||||
|
if len(duty_sections) != 1 or not duty_sections[0]:
|
||||||
|
problems.append(
|
||||||
|
ValidationProblem(
|
||||||
|
page, 1, "每页必须有且仅有一个非空「本篇职责」"
|
||||||
|
)
|
||||||
|
)
|
||||||
for phrase in ROUTER_PHRASES:
|
for phrase in ROUTER_PHRASES:
|
||||||
for index, line in enumerate(lines, start=1):
|
for index, line in enumerate(lines, start=1):
|
||||||
if phrase in line:
|
if phrase in line:
|
||||||
@@ -356,6 +410,16 @@ def validate_references(
|
|||||||
# 概念地图逐页从「本篇职责」段生成;有该段的页必须产出非空摘要,
|
# 概念地图逐页从「本篇职责」段生成;有该段的页必须产出非空摘要,
|
||||||
# 否则某页职责段被清空/写坏时地图会静默缺页。
|
# 否则某页职责段被清空/写坏时地图会静默缺页。
|
||||||
mapped_pages = {page_name for page_name, _, _ in build_concept_map(references_dir)}
|
mapped_pages = {page_name for page_name, _, _ in build_concept_map(references_dir)}
|
||||||
|
reference_pages = {page.name for page in _reference_pages(references_dir)}
|
||||||
|
if mapped_pages != reference_pages:
|
||||||
|
missing = ", ".join(sorted(reference_pages - mapped_pages)) or "none"
|
||||||
|
problems.append(
|
||||||
|
ValidationProblem(
|
||||||
|
references_dir,
|
||||||
|
1,
|
||||||
|
f"概念地图页数与参考页不一致;缺失:{missing}",
|
||||||
|
)
|
||||||
|
)
|
||||||
for section in sections:
|
for section in sections:
|
||||||
if section.heading_path != (DUTY_HEADING,):
|
if section.heading_path != (DUTY_HEADING,):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -66,12 +66,67 @@ class TslSyntaxLookupTests(unittest.TestCase):
|
|||||||
self.assertNotIn(section.body.rstrip(), query_completed.stdout)
|
self.assertNotIn(section.body.rstrip(), query_completed.stdout)
|
||||||
self.assertIn(section.body.rstrip(), section_completed.stdout)
|
self.assertIn(section.body.rstrip(), section_completed.stdout)
|
||||||
|
|
||||||
def test_parser_creates_unique_h2_h3_section_ids(self):
|
def test_parser_creates_unique_h2_h3_h4_section_ids(self):
|
||||||
sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR)
|
sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR)
|
||||||
ids = [section.id for section in sections]
|
ids = [section.id for section in sections]
|
||||||
self.assertEqual(len(ids), len(set(ids)))
|
self.assertEqual(len(ids), len(set(ids)))
|
||||||
self.assertTrue(any("05_functions_and_calls" in value for value in ids))
|
self.assertTrue(any("05_functions_and_calls" in value for value in ids))
|
||||||
|
|
||||||
|
def test_parser_indexes_h4_pitfall_sections(self):
|
||||||
|
sections = lookup.load_sections(lookup.DEFAULT_REFERENCES_DIR)
|
||||||
|
|
||||||
|
self.assertTrue(any(len(section.heading_path) == 3 for section in sections))
|
||||||
|
self.assertTrue(any("把-当成赋值" in section.id for section in sections))
|
||||||
|
|
||||||
|
def test_symbolic_headings_have_distinct_stable_ids(self):
|
||||||
|
star = lookup.section_id("sample.md", ("Examples", "with *"))
|
||||||
|
double_star = lookup.section_id("sample.md", ("Examples", "with **"))
|
||||||
|
|
||||||
|
self.assertNotEqual(star, double_star)
|
||||||
|
self.assertNotRegex(double_star, r"-2$")
|
||||||
|
|
||||||
|
def test_check_requires_one_nonempty_duty_section_per_page(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
|
references = Path(tmp_dir)
|
||||||
|
(references / "missing.md").write_text(
|
||||||
|
"# Missing\n\n## Rules\n\nText.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
newline="\n",
|
||||||
|
)
|
||||||
|
(references / "duplicate.md").write_text(
|
||||||
|
"# Duplicate\n\n"
|
||||||
|
"## 本篇职责\n\n"
|
||||||
|
"One.\n\n"
|
||||||
|
"## 本篇职责\n\n"
|
||||||
|
"Two.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
newline="\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
problems = lookup.validate_references(references)
|
||||||
|
|
||||||
|
messages = "\n".join(problem.message for problem in problems)
|
||||||
|
self.assertIn("必须有且仅有一个非空「本篇职责」", messages)
|
||||||
|
|
||||||
|
def test_check_rejects_h2_h3_h4_heading_level_jumps(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
|
references = Path(tmp_dir)
|
||||||
|
(references / "sample.md").write_text(
|
||||||
|
"# Sample\n\n"
|
||||||
|
"## 本篇职责\n\n"
|
||||||
|
"Summary.\n\n"
|
||||||
|
"#### Skipped H3\n\n"
|
||||||
|
"Details.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
newline="\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
problems = lookup.validate_references(references)
|
||||||
|
|
||||||
|
self.assertTrue(
|
||||||
|
any("标题层级跳跃" in problem.message for problem in problems), problems
|
||||||
|
)
|
||||||
|
|
||||||
def test_write_mode_includes_file_model_and_direct_example(self):
|
def test_write_mode_includes_file_model_and_direct_example(self):
|
||||||
result = lookup.query_sections("写函数 命名参数", "write", limit=4)
|
result = lookup.query_sections("写函数 命名参数", "write", limit=4)
|
||||||
rendered = lookup.render_candidates(result)
|
rendered = lookup.render_candidates(result)
|
||||||
@@ -200,6 +255,8 @@ class TslSyntaxLookupTests(unittest.TestCase):
|
|||||||
page = references / "sample.md"
|
page = references / "sample.md"
|
||||||
page.write_text(
|
page.write_text(
|
||||||
"# Sample\n\n"
|
"# Sample\n\n"
|
||||||
|
"## 本篇职责\n\n"
|
||||||
|
"Structured metadata sample.\n\n"
|
||||||
"## Example\n\n"
|
"## Example\n\n"
|
||||||
"代码块身份:配置片段 / 概念骨架\n"
|
"代码块身份:配置片段 / 概念骨架\n"
|
||||||
"代码块说明:This is structured metadata.\n\n"
|
"代码块说明:This is structured metadata.\n\n"
|
||||||
@@ -213,7 +270,10 @@ class TslSyntaxLookupTests(unittest.TestCase):
|
|||||||
sections = lookup.load_sections(references)
|
sections = lookup.load_sections(references)
|
||||||
problems = lookup.validate_references(references)
|
problems = lookup.validate_references(references)
|
||||||
|
|
||||||
self.assertEqual(sections[0].identities, ("配置片段 / 概念骨架",))
|
example = next(
|
||||||
|
section for section in sections if section.heading_path == ("Example",)
|
||||||
|
)
|
||||||
|
self.assertEqual(example.identities, ("配置片段 / 概念骨架",))
|
||||||
self.assertEqual(problems, [])
|
self.assertEqual(problems, [])
|
||||||
|
|
||||||
def test_check_accumulates_fence_identity_and_link_problems(self):
|
def test_check_accumulates_fence_identity_and_link_problems(self):
|
||||||
@@ -243,6 +303,8 @@ class TslSyntaxLookupTests(unittest.TestCase):
|
|||||||
references = Path(tmp_dir)
|
references = Path(tmp_dir)
|
||||||
(references / "sample.md").write_text(
|
(references / "sample.md").write_text(
|
||||||
"# Sample\n\n"
|
"# Sample\n\n"
|
||||||
|
"## 本篇职责\n\n"
|
||||||
|
"Inline-code link sample.\n\n"
|
||||||
"## Operators\n\n"
|
"## Operators\n\n"
|
||||||
"Use `function operator[](index);`.\n",
|
"Use `function operator[](index);`.\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
|
|||||||
Reference in New Issue
Block a user