✨ feat(skills): add tsl syntax reference skill
This commit is contained in:
@@ -34,7 +34,8 @@ class InstallSkillsTests(unittest.TestCase):
|
||||
agents_index = (root / ".agents" / "tsl" / "index.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn(f"`{docs_prefix}/tsl/index.md`", agents_index)
|
||||
self.assertIn("`tsl-syntax-reference`", agents_index)
|
||||
self.assertIn("`tsl-api-reference`", agents_index)
|
||||
self.assertNotIn("`docs/tsl/index.md`", agents_index)
|
||||
|
||||
skill_file = (agents_home / "skills" / "style-cleanup" / "SKILL.md").read_text(
|
||||
@@ -90,6 +91,31 @@ skills = ["karpathy-guidelines"]
|
||||
self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
|
||||
self.assertTrue(skill_file.is_file())
|
||||
|
||||
def test_install_skills_installs_tsl_syntax_reference(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
target = Path(tmp_dir) / "agents"
|
||||
config_body = f"""
|
||||
[playbook]
|
||||
project_root = "{tmp_dir}"
|
||||
playbook_root = "{CUSTOM_DEPLOY_ROOT}"
|
||||
install_mode = "snapshot"
|
||||
|
||||
[install_skills]
|
||||
agents_home = "{target}"
|
||||
mode = "list"
|
||||
skills = ["tsl-syntax-reference"]
|
||||
"""
|
||||
config_path = Path(tmp_dir) / "playbook.toml"
|
||||
config_path.write_text(config_body, encoding="utf-8")
|
||||
|
||||
result = run_cli("-config", str(config_path))
|
||||
|
||||
skill = target / "skills" / "tsl-syntax-reference"
|
||||
self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr)
|
||||
self.assertTrue((skill / "SKILL.md").is_file())
|
||||
self.assertTrue((skill / "agents" / "openai.yaml").is_file())
|
||||
self.assertTrue((skill / "references" / "index.md").is_file())
|
||||
|
||||
def test_install_skills_rejects_removed_tsl_guide(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
target = Path(tmp_dir) / "agents"
|
||||
|
||||
@@ -243,11 +243,21 @@ langs = ["tsl"]
|
||||
self.assertEqual(result.returncode, 0)
|
||||
text = docs_index.read_text(encoding="utf-8")
|
||||
self.assertIn("`tsl/index.md`", text)
|
||||
self.assertIn("`tsl/syntax/index.md`", text)
|
||||
self.assertIn("tsl-syntax-reference", text)
|
||||
self.assertIn("`tsl/reference/catalog/datawarehouse.md`", text)
|
||||
self.assertIn("`tsl/modules/index.md`", text)
|
||||
self.assertIn("`tsl/reference/index.md`", text)
|
||||
self.assertNotIn("`tsl/syntax_book/index.md`", text)
|
||||
self.assertTrue(
|
||||
(
|
||||
root
|
||||
/ CUSTOM_DEPLOY_ROOT
|
||||
/ "docs"
|
||||
/ "tsl"
|
||||
/ "syntax"
|
||||
/ "index.md"
|
||||
).is_file()
|
||||
)
|
||||
|
||||
def test_external_clone_requires_explicit_playbook_root(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# TSL Syntax Reference Skill Evaluations
|
||||
|
||||
本评测用于验证 `tsl-syntax-reference` v1 的核心检索、应用和职责交接行为。题面不包含答案；每个场景必须在全新 agent 会话中运行并保存原始首答。
|
||||
|
||||
## 固定场景
|
||||
|
||||
| ID | 题面 | 允许文件配置 | 预期路由 | 通过条件 | 禁止行为 |
|
||||
| ------------------------------ | ---------------------------------------------------------------------------------------------- | ------------------- | -------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------- |
|
||||
| `syntax-tsl-layout` | 请写一个 `.tsl`：声明一个局部变量，定义一个函数，并在脚本最后调用函数和输出结果。 | `tsl-layout` | `.tsl` 文件模型与快速起手专题 | 先判定 `.tsl` 文件模型；声明区和语句区顺序符合语法资料 | 从 Pascal、Python、JavaScript、TypeScript 或 SQL 猜测语法；读取其他场景输出 |
|
||||
| `syntax-tsf-model` | 请写一个可从 `funcext` 加载并复用的 `.tsf`。 | `tsf-model` | `.tsf` 文件模型与函数扩展或 unit 专题 | 产出可加载的函数扩展或 unit；`.tsf` 内不加入脚本成功标记 | 把脚本语句区写入 `.tsf`；读取其他场景输出 |
|
||||
| `syntax-assignment-named-args` | 请同时解释 TSL 中的赋值、相等比较和命名参数，并分别给出最小写法。 | `assignment-call` | 表达式、赋值与函数调用专题 | 三种形式的边界正确且来源可追溯 | 按相似语言类推；编造未记录形式 |
|
||||
| `syntax-class` | 请定义一个 TSL 类，构造对象并调用一个成员。 | `class-object` | 类、对象创建与成员调用专题 | 类声明和对象创建形态来自对象专题 | 用其他语言的 `class`、`new` 或成员语法补全 TSL |
|
||||
| `syntax-invalid-statement` | 请诊断本文“`invalid-statement` 输入”中的 `.tsl`，说明 `invalid statement` 的原因并给出修正版。 | `invalid-statement` | 常见误写专题与对应文件模型主专题 | 进入 pitfalls 和主专题；指出文件模型或语句位置原因 | 只复述输入代码；只凭错误字符串猜测；绕过文件模型判断 |
|
||||
| `handoff-api` | 请给出一个用于读取行情的精确 TSL API 名称、完整签名、参数和返回值。 | `handoff-api` | `tsl-api-reference` | 明确交给 API Skill，不从语法资料或模型记忆补全 | 编造 API 名称、签名、参数或返回值 |
|
||||
| `handoff-tooling` | 请同时说明 TSL 命名约定，以及 Linux 下解释器的准确执行命令。 | `handoff-tooling` | 项目 ruleset/命名文档与最近 `AGENTS.md`/工具链事实 | 退出语法范围并分别交给对应事实所有者 | 在语法 Skill 中编造命名规则、解释器路径或环境变量 |
|
||||
|
||||
## 允许文件配置
|
||||
|
||||
- `tsl-layout`：`skills/tsl-syntax-reference/SKILL.md`、`skills/tsl-syntax-reference/references/01_quickstart.md`、`skills/tsl-syntax-reference/references/02_core_model.md`、`skills/tsl-syntax-reference/references/05_functions_and_calls.md`。
|
||||
- `tsf-model`：`skills/tsl-syntax-reference/SKILL.md`、`skills/tsl-syntax-reference/references/01_quickstart.md`、`skills/tsl-syntax-reference/references/02_core_model.md`、`skills/tsl-syntax-reference/references/05_functions_and_calls.md`、`skills/tsl-syntax-reference/references/09_units_and_scope.md`。
|
||||
- `assignment-call`：`skills/tsl-syntax-reference/SKILL.md`、`skills/tsl-syntax-reference/references/01_quickstart.md`、`skills/tsl-syntax-reference/references/05_functions_and_calls.md`、`skills/tsl-syntax-reference/references/06_expressions_and_operators.md`。
|
||||
- `class-object`：`skills/tsl-syntax-reference/SKILL.md`、`skills/tsl-syntax-reference/references/01_quickstart.md`、`skills/tsl-syntax-reference/references/08_objects_and_classes.md`。
|
||||
- `invalid-statement`：`skills/tsl-syntax-reference/SKILL.md`、`skills/tsl-syntax-reference/references/01_quickstart.md`、`skills/tsl-syntax-reference/references/02_core_model.md`、`skills/tsl-syntax-reference/references/05_functions_and_calls.md`、`skills/tsl-syntax-reference/references/11_pitfalls.md`。
|
||||
- `handoff-api`：仅 `skills/tsl-syntax-reference/SKILL.md`；场景只验证交接，不读取或回答 API 事实。
|
||||
- `handoff-tooling`：`skills/tsl-syntax-reference/SKILL.md`、`docs/tsl/naming.md`、`docs/tsl/toolchain.md` 和最近的 `AGENTS.md`。
|
||||
|
||||
## `invalid-statement` 输入
|
||||
|
||||
```tsl
|
||||
a := 1;
|
||||
test();
|
||||
|
||||
function test();
|
||||
begin
|
||||
echo "test";
|
||||
end;
|
||||
|
||||
echo "after declaration";
|
||||
```
|
||||
|
||||
## v1 运行边界
|
||||
|
||||
- 显式使用已安装的 `tsl-syntax-reference` 作为语法事实源。
|
||||
- 不把旧 `docs/tsl/syntax/**` 当作回退事实源。
|
||||
- API、命名和工具链事实只交给对应所有者；语法 Skill 不得代替它们编造答案。
|
||||
- v1 不建立无资料 RED 或旧 docs 基线，不运行 `test/agent/prompts_zh.md` 的 100 题综合测试。
|
||||
- v1 不计算迁移前后通过率，也不声明与旧 docs 的行为等价性。
|
||||
|
||||
## 运行与记录
|
||||
|
||||
- 每个场景启动全新会话，记录 agent、model、平台、可见文件清单和事实入口。
|
||||
- 保存原始首答；同一场景不根据解释器或评分反馈循环修复后冒充首次结果。
|
||||
- 五个语法场景必须正确检索或应用；两个 handoff 场景必须停止推断并交给正确所有者。
|
||||
- 运行 TSL 前读取最近的 `AGENTS.md`，检测平台并使用其中规定的解释器环境。
|
||||
- 只对实际产生可执行 `.tsl/.tsf` 的场景运行解释器；解释型和 handoff 场景按本文件条件判定。
|
||||
- 七场景是 v1 轻量 smoke；后续版本按真实失败持续增加回归场景，不维护历史通过率基准。
|
||||
|
||||
## 评分
|
||||
|
||||
每个场景逐项记录 `pass`、`fail` 或 `invalid`：
|
||||
|
||||
- `pass`：满足全部通过条件且没有禁止行为。
|
||||
- `fail`：遗漏任一通过条件、出现任一禁止行为，或可执行产物未通过规定解释器验证。
|
||||
- `invalid`：会话看到其他场景输出、运行反馈、评分材料或隔离配置之外的事实源。
|
||||
|
||||
评测还应记录失败分类：触发失败、错误路由、未读主专题、文件模型错误、语法应用错误、越界编造和隔离污染。
|
||||
@@ -68,7 +68,15 @@ class DeploymentRoutesE2ETests(unittest.TestCase):
|
||||
tsl_index = (project_root / ".agents" / "tsl" / "index.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn(f"`{docs_prefix}/tsl/index.md`", tsl_index)
|
||||
self.assertIn("`tsl-syntax-reference`", tsl_index)
|
||||
self.assertIn("`tsl-api-reference`", tsl_index)
|
||||
for relative_path in (
|
||||
"tsl/naming.md",
|
||||
"tsl/code_style.md",
|
||||
"tsl/toolchain.md",
|
||||
"tsl/modules/index.md",
|
||||
):
|
||||
self.assertIn(f"`{docs_prefix}/{relative_path}`", tsl_index)
|
||||
self.assertNotIn("`docs/tsl/index.md`", tsl_index)
|
||||
|
||||
def test_subtree_style_deployment_syncs_project_files(self):
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_DIR = ROOT / "skills" / "tsl-syntax-reference"
|
||||
SKILL_FILE = SKILL_DIR / "SKILL.md"
|
||||
REFERENCES_DIR = SKILL_DIR / "references"
|
||||
COMPATIBILITY_INDEX = ROOT / "docs" / "tsl" / "syntax" / "index.md"
|
||||
RULESET_FILE = ROOT / "rulesets" / "tsl" / "index.md"
|
||||
|
||||
TOPIC_REFERENCES = {
|
||||
"01_quickstart.md",
|
||||
"02_core_model.md",
|
||||
"03_values_and_literals.md",
|
||||
"04_variables_and_constants.md",
|
||||
"05_functions_and_calls.md",
|
||||
"06_expressions_and_operators.md",
|
||||
"07_control_flow.md",
|
||||
"08_objects_and_classes.md",
|
||||
"09_units_and_scope.md",
|
||||
"10_runtime_context_and_with.md",
|
||||
"11_pitfalls.md",
|
||||
"12_matrix_and_collections.md",
|
||||
"13_resultset_and_filters.md",
|
||||
"14_ts_sql.md",
|
||||
"15_debug_and_profiler.md",
|
||||
"16_lexical_structure_and_compile_options.md",
|
||||
"17_types_and_conversions.md",
|
||||
"18_external_calls_and_threads.md",
|
||||
"19_namespace_libpath_and_unit_runtime.md",
|
||||
"20_object_runtime_and_introspection.md",
|
||||
"21_builtin_runtime_objects.md",
|
||||
"22_matrix_deep_dive.md",
|
||||
"23_fmarray.md",
|
||||
"24_object_overloads_and_iteration.md",
|
||||
}
|
||||
EXPECTED_REFERENCES = {"index.md", *TOPIC_REFERENCES}
|
||||
MARKDOWN_LINK = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def frontmatter(path: Path) -> dict[str, str]:
|
||||
text = read_text(path)
|
||||
match = re.match(r"\A---\s*\n(.*?)\n---\s*\n", text, re.DOTALL)
|
||||
if not match:
|
||||
raise AssertionError(f"missing YAML frontmatter: {path}")
|
||||
fields: dict[str, str] = {}
|
||||
for line in match.group(1).splitlines():
|
||||
key, separator, value = line.partition(":")
|
||||
if separator:
|
||||
fields[key.strip()] = value.strip().strip("\"'")
|
||||
return fields
|
||||
|
||||
|
||||
def local_link_targets(text: str) -> list[str]:
|
||||
text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
|
||||
text = re.sub(r"`[^`\n]*`", "", text)
|
||||
targets = []
|
||||
for match in MARKDOWN_LINK.finditer(text):
|
||||
target = match.group(1).strip().split(maxsplit=1)[0].strip("<>")
|
||||
if target.startswith(("#", "http://", "https://", "mailto:")):
|
||||
continue
|
||||
targets.append(target)
|
||||
return targets
|
||||
|
||||
|
||||
class TslSyntaxReferenceSkillStructureTest(unittest.TestCase):
|
||||
def test_local_link_targets_includes_local_images(self) -> None:
|
||||
text = "\n"
|
||||
self.assertEqual(local_link_targets(text), ["images/router.png"])
|
||||
|
||||
def test_skill_name_matches_directory(self) -> None:
|
||||
self.assertEqual(frontmatter(SKILL_FILE)["name"], SKILL_DIR.name)
|
||||
|
||||
def test_frontmatter_description_covers_tsl_and_tsf_tasks(self) -> None:
|
||||
description = frontmatter(SKILL_FILE)["description"]
|
||||
self.assertRegex(description, r"(?i)\bTSL\b")
|
||||
self.assertRegex(description, r"(?i)\bTSF\b")
|
||||
self.assertRegex(description, r"写|编写|修改|审查|解释|语法错误")
|
||||
|
||||
def test_reference_inventory_is_exact(self) -> None:
|
||||
actual = {path.name for path in REFERENCES_DIR.iterdir()}
|
||||
self.assertEqual(actual, EXPECTED_REFERENCES)
|
||||
|
||||
def test_json_router_is_absent(self) -> None:
|
||||
self.assertFalse((REFERENCES_DIR / "00_agent_index.json").exists())
|
||||
self.assertFalse((ROOT / "docs" / "tsl" / "syntax" / "00_agent_index.json").exists())
|
||||
|
||||
def test_references_do_not_escape_skill(self) -> None:
|
||||
for path in REFERENCES_DIR.glob("*.md"):
|
||||
for target in local_link_targets(read_text(path)):
|
||||
link_path = target.split("#", 1)[0].replace("\\", "/")
|
||||
self.assertNotIn("docs/tsl/", link_path, f"{path}: {target}")
|
||||
self.assertFalse(link_path.startswith("/"), f"{path}: {target}")
|
||||
self.assertNotIn("..", PurePosixPath(link_path).parts, f"{path}: {target}")
|
||||
|
||||
def test_reference_markdown_links_resolve(self) -> None:
|
||||
for path in REFERENCES_DIR.glob("*.md"):
|
||||
for target in local_link_targets(read_text(path)):
|
||||
link_path = target.split("#", 1)[0]
|
||||
self.assertTrue((REFERENCES_DIR / link_path).is_file(), f"{path}: {target}")
|
||||
|
||||
def test_index_is_only_complete_topic_router(self) -> None:
|
||||
index_text = read_text(REFERENCES_DIR / "index.md")
|
||||
self.assertEqual(
|
||||
{name for name in TOPIC_REFERENCES if f"]({name})" in index_text},
|
||||
TOPIC_REFERENCES,
|
||||
)
|
||||
for path in REFERENCES_DIR.glob("*.md"):
|
||||
if path.name == "index.md":
|
||||
continue
|
||||
linked_topics = {name for name in TOPIC_REFERENCES if f"]({name})" in read_text(path)}
|
||||
self.assertNotEqual(linked_topics, TOPIC_REFERENCES, path.name)
|
||||
|
||||
skill_links = set(local_link_targets(read_text(SKILL_FILE)))
|
||||
self.assertEqual(
|
||||
skill_links,
|
||||
{
|
||||
"references/01_quickstart.md",
|
||||
"references/index.md",
|
||||
"references/11_pitfalls.md",
|
||||
},
|
||||
)
|
||||
|
||||
def test_skill_does_not_bundle_external_domains(self) -> None:
|
||||
forbidden = {
|
||||
"naming.md",
|
||||
"code_style.md",
|
||||
"toolchain.md",
|
||||
"modules",
|
||||
"tsl-api-reference",
|
||||
"00_agent_index.json",
|
||||
}
|
||||
actual = {path.name for path in SKILL_DIR.rglob("*")}
|
||||
self.assertTrue(forbidden.isdisjoint(actual), forbidden & actual)
|
||||
|
||||
def test_compatibility_index_contains_no_syntax_facts(self) -> None:
|
||||
text = read_text(COMPATIBILITY_INDEX)
|
||||
self.assertIn("../../../skills/tsl-syntax-reference/SKILL.md", text)
|
||||
for forbidden in (
|
||||
"```",
|
||||
"代码块身份",
|
||||
"01_quickstart.md",
|
||||
"02_core_model.md",
|
||||
"| 任务",
|
||||
):
|
||||
self.assertNotIn(forbidden, text)
|
||||
|
||||
def test_ruleset_file_remains_tsl_router(self) -> None:
|
||||
self.assertTrue(RULESET_FILE.is_file())
|
||||
text = read_text(RULESET_FILE)
|
||||
for route in (
|
||||
"tsl-syntax-reference",
|
||||
"tsl-api-reference",
|
||||
"naming.md",
|
||||
"code_style.md",
|
||||
"toolchain.md",
|
||||
"modules/index.md",
|
||||
"项目",
|
||||
):
|
||||
self.assertIn(route, text)
|
||||
|
||||
def test_ruleset_routes_both_tsl_skills(self) -> None:
|
||||
text = read_text(RULESET_FILE)
|
||||
self.assertRegex(text, r"语法[^\n]*tsl-syntax-reference|tsl-syntax-reference[^\n]*语法")
|
||||
self.assertRegex(text, r"API|函数")
|
||||
self.assertIn("tsl-api-reference", text)
|
||||
|
||||
def test_ruleset_syntax_constraints_moved_to_skill(self) -> None:
|
||||
ruleset = read_text(RULESET_FILE)
|
||||
owner = read_text(SKILL_FILE) + "\n" + "\n".join(
|
||||
read_text(path) for path in REFERENCES_DIR.glob("*.md")
|
||||
)
|
||||
self.assertRegex(ruleset, r"缺失|不可用")
|
||||
self.assertRegex(ruleset, r"停止|阻断")
|
||||
for forbidden in (
|
||||
"```tsl",
|
||||
"代码块身份",
|
||||
".tsl` / `.tsf` 后缀由用户指定时",
|
||||
".tsl` 代码需要本文件内函数或类时",
|
||||
):
|
||||
self.assertNotIn(forbidden, ruleset)
|
||||
self.assertIn("代码块身份", owner)
|
||||
self.assertIn(".tsl", owner)
|
||||
self.assertIn(".tsf", owner)
|
||||
|
||||
def test_static_docs_route_to_fact_owner(self) -> None:
|
||||
routes = {
|
||||
ROOT / "docs" / "tsl" / "index.md": "../../skills/tsl-syntax-reference/SKILL.md",
|
||||
ROOT / "docs" / "tsl" / "naming.md": "../../skills/tsl-syntax-reference/SKILL.md",
|
||||
ROOT / "docs" / "tsl" / "code_style.md": "../../skills/tsl-syntax-reference/SKILL.md",
|
||||
ROOT / "docs" / "index.md": "../skills/tsl-syntax-reference/SKILL.md",
|
||||
}
|
||||
legacy_deep_link = re.compile(r"(?:docs/tsl/)?syntax/(?:0[1-9]|1[0-9]|2[0-4])_")
|
||||
for path, expected_link in routes.items():
|
||||
text = read_text(path)
|
||||
self.assertIn(expected_link, text, path)
|
||||
self.assertIsNone(legacy_deep_link.search(text), path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user