Files
playbook/test/test_playbook.py
T

774 lines
29 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "playbook.py"
MODE_ROOTS = {
"snapshot": Path("custom/playbook"),
"subtree": Path("docs/standards/playbook"),
}
def run_playbook(
script: Path, config: Path, project_root: Path
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, str(script), "-config", str(config)],
cwd=project_root,
capture_output=True,
text=True,
)
def copy_subtree_source(destination: Path) -> None:
destination.mkdir(parents=True)
shutil.copy2(ROOT / ".gitattributes", destination / ".gitattributes")
for name in ("scripts", "templates"):
shutil.copytree(
ROOT / name,
destination / name,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
)
(destination / "docs").mkdir()
for name in ("common", "tsl", "markdown"):
shutil.copytree(ROOT / "docs" / name, destination / "docs" / name)
(destination / "rulesets").mkdir()
shutil.copy2(
ROOT / "rulesets" / "index.md",
destination / "rulesets" / "index.md",
)
for name in ("tsl", "markdown"):
shutil.copytree(
ROOT / "rulesets" / name,
destination / "rulesets" / name,
)
(destination / "skills").mkdir()
for name in ("commit-message", "cook-it-through"):
shutil.copytree(
ROOT / "skills" / name,
destination / "skills" / name,
)
def write_config(project_root: Path, install_mode: str, playbook_root: Path) -> Path:
config = project_root / "playbook.toml"
config.write_text(
f"""
[playbook]
project_root = "."
playbook_root = "{playbook_root.as_posix()}"
install_mode = "{install_mode}"
[sync_rules]
date = "2026-01-01"
no_backup = true
[sync_memory_bank]
project_name = "Demo"
no_backup = true
[sync_standards]
langs = ["tsl", "markdown"]
gitattr_mode = "overwrite"
no_backup = true
[install_skills]
agents_home = ".test-agents"
mode = "list"
skills = ["commit-message", "cook-it-through"]
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
return config
def seed_custom_files(project_root: Path) -> None:
custom_memory = project_root / "memory-bank" / "custom.md"
custom_memory.parent.mkdir(parents=True)
custom_memory.write_text("custom memory\n", encoding="utf-8", newline="\n")
(project_root / "AGENTS.md").write_text(
"# Existing Agents\n\nKeep this agent note.\n\nSee `.agents/index.md`.\n",
encoding="utf-8",
newline="\n",
)
(project_root / "CLAUDE.md").write_text(
"# Existing Claude\n\nKeep this Claude note.\n\n@AGENTS.md\n",
encoding="utf-8",
newline="\n",
)
(project_root / ".gitignore").write_text(
"build/\n.scratch/\n",
encoding="utf-8",
newline="\n",
)
class PlaybookDeploymentTests(unittest.TestCase):
def test_help_lists_only_current_cli_options(self):
with tempfile.TemporaryDirectory() as tmp_dir:
result = subprocess.run(
[sys.executable, str(SCRIPT), "--help"],
cwd=Path(tmp_dir),
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, msg=result.stderr)
self.assertIn("-config PATH", result.stdout)
self.assertIn("-h, --help", result.stdout)
self.assertNotIn("-h, -help", result.stdout)
def test_install_all_honors_configured_and_legacy_exclusions(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
config = project_root / "playbook.toml"
config.write_text(
"""
[playbook]
project_root = "."
playbook_root = "custom/playbook"
install_mode = "snapshot"
[install_skills]
agents_home = ".test-agents"
mode = "all"
exclude = ["cook-it-through", "to-tickets"]
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(
result.returncode,
0,
msg=f"all-skills install failed\n{result.stdout}{result.stderr}",
)
installed = {
path.name
for path in (project_root / ".test-agents" / "skills").iterdir()
if path.is_dir()
}
self.assertIn("grill-with-docs", installed)
self.assertNotIn("to-tickets", installed)
self.assertNotIn("cook-it-through", installed)
self.assertTrue(
{
"using-superpowers",
"brainstorming",
"writing-plans",
"executing-plans",
}.isdisjoint(installed)
)
def test_sync_rules_rejects_missing_workflow_skill_before_any_write(self):
invalid_install_configs = {
"excluded": '\n'.join(
(
'mode = "all"',
'exclude = ["cook-it-through"]',
)
),
"omitted-from-list": '\n'.join(
(
'mode = "list"',
'skills = ["commit-message"]',
)
),
}
for case, install_config in invalid_install_configs.items():
with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
config = project_root / "playbook.toml"
config.write_text(
f"""
[playbook]
project_root = "."
playbook_root = "custom/playbook"
install_mode = "snapshot"
[sync_rules]
no_backup = true
[install_skills]
agents_home = ".test-agents"
{install_config}
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(result.returncode, 2, msg=result.stdout)
self.assertIn("cook-it-through", result.stderr)
for untouched in (
"custom/playbook",
"AGENTS.md",
"AGENT_RULES.md",
"AGENT_RULES.local.md",
".test-agents",
):
self.assertFalse(
(project_root / untouched).exists(),
msg=f"invalid config wrote {untouched}",
)
def test_sync_rules_requires_an_install_skills_action_before_any_write(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
config = project_root / "playbook.toml"
config.write_text(
"""
[playbook]
project_root = "."
playbook_root = "custom/playbook"
install_mode = "snapshot"
[sync_rules]
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(result.returncode, 2, msg=result.stdout)
self.assertIn("[install_skills]", result.stderr)
self.assertIn("cook-it-through", result.stderr)
for untouched in (
"custom/playbook",
"AGENTS.md",
"AGENT_RULES.md",
"AGENT_RULES.local.md",
):
self.assertFalse(
(project_root / untouched).exists(),
msg=f"invalid config wrote {untouched}",
)
def test_install_skills_list_requires_explicit_skills(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
config = project_root / "playbook.toml"
config.write_text(
"""
[playbook]
project_root = "."
playbook_root = "custom/playbook"
install_mode = "snapshot"
[install_skills]
agents_home = ".test-agents"
mode = "list"
bundles = ["matt-pocock-workflow"]
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(
result.returncode,
2,
msg=f"bundles unexpectedly supported\n{result.stdout}{result.stderr}",
)
self.assertIn("ERROR: skills is required", result.stderr)
def test_install_skills_accepts_an_empty_exclusion_list(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
config = project_root / "playbook.toml"
config.write_text(
"""
[playbook]
project_root = "."
playbook_root = "custom/playbook"
install_mode = "snapshot"
[install_skills]
agents_home = ".test-agents"
mode = "list"
skills = ["commit-message"]
exclude = []
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(
result.returncode,
0,
msg=f"empty exclusion failed\n{result.stdout}{result.stderr}",
)
self.assertTrue(
(
project_root
/ ".test-agents"
/ "skills"
/ "commit-message"
/ "SKILL.md"
).is_file()
)
def test_invalid_toml_is_reported_without_a_traceback(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
config = project_root / "playbook.toml"
config.write_text(
r"""
[playbook]
project_root = "C:\workspace\project"
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(result.returncode, 2)
self.assertIn("ERROR: invalid TOML", result.stderr)
self.assertNotIn("Traceback", result.stderr)
def test_playbook_deployment_modes(self):
for install_mode, playbook_root in MODE_ROOTS.items():
with self.subTest(install_mode=install_mode):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
if install_mode == "subtree":
source_root = project_root / playbook_root
copy_subtree_source(source_root)
script = source_root / "scripts" / "playbook.py"
else:
source_root = ROOT
script = SCRIPT
seed_custom_files(project_root)
config = write_config(project_root, install_mode, playbook_root)
for run_number in (1, 2):
result = run_playbook(script, config, project_root)
self.assertEqual(
result.returncode,
0,
msg=(
f"{install_mode} run {run_number} failed\n"
f"{result.stdout}{result.stderr}"
),
)
expected_paths = (
"AGENTS.md",
"AGENT_RULES.md",
"AGENT_RULES.local.md",
"CLAUDE.md",
".gitignore",
".gitattributes",
"memory-bank/project-brief.md",
"memory-bank/system-patterns.md",
".agents/index.md",
".agents/tsl/index.md",
".agents/markdown/index.md",
".test-agents/skills/commit-message/SKILL.md",
".test-agents/skills/commit-message/references/commit_policy.json",
".test-agents/skills/commit-message/scripts/validate_commit_message.py",
".test-agents/skills/cook-it-through/SKILL.md",
".test-agents/skills/cook-it-through/rules/session-boundary.md",
".test-agents/skills/cook-it-through/scripts/main_loop.py",
".test-agents/skills/cook-it-through/scripts/main_loop_scheduler.py",
".test-agents/skills/cook-it-through/workflows/single-session.md",
".test-agents/skills/cook-it-through/workflows/feature-planning.md",
".test-agents/skills/cook-it-through/workflows/ticket-execution.md",
".test-agents/skills/cook-it-through/workflows/feature-integration.md",
)
missing = [
path
for path in expected_paths
if not (project_root / path).exists()
]
self.assertEqual(missing, [])
self.assertEqual(
(project_root / "memory-bank/custom.md").read_text(
encoding="utf-8"
),
"custom memory\n",
)
agents_text = (project_root / "AGENTS.md").read_text(
encoding="utf-8"
)
self.assertIn("Keep this agent note.", agents_text)
self.assertEqual(
agents_text.count("<!-- playbook:framework:start -->"), 1
)
claude_text = (project_root / "CLAUDE.md").read_text(
encoding="utf-8"
)
self.assertIn("Keep this Claude note.", claude_text)
self.assertIn("@AGENTS.md", claude_text)
self.assertEqual(
claude_text.count("<!-- playbook:claude:start -->"), 1
)
docs_prefix = f"{playbook_root.as_posix()}/docs"
agents_index = (project_root / ".agents/index.md").read_text(
encoding="utf-8"
)
self.assertIn(f"- {docs_prefix}", agents_index)
source_skill = ROOT / "skills" / "commit-message"
installed_commit_skill = (
project_root / ".test-agents/skills/commit-message"
)
self.assertEqual(
(installed_commit_skill / "SKILL.md").read_text(encoding="utf-8"),
(source_skill / "SKILL.md").read_text(encoding="utf-8"),
)
self.assertEqual(
(
installed_commit_skill / "references/commit_policy.json"
).read_text(encoding="utf-8"),
(source_skill / "references/commit_policy.json").read_text(
encoding="utf-8"
),
)
self.assertEqual(
(
installed_commit_skill
/ "scripts/validate_commit_message.py"
).read_text(encoding="utf-8"),
(
source_skill / "scripts/validate_commit_message.py"
).read_text(encoding="utf-8"),
)
source_main_loop_root = ROOT / "skills/cook-it-through"
installed_main_loop_root = (
project_root / ".test-agents/skills/cook-it-through"
)
for relative_path in (
"SKILL.md",
"rules/session-boundary.md",
"scripts/main_loop.py",
"scripts/main_loop_scheduler.py",
"workflows/single-session.md",
"workflows/feature-planning.md",
"workflows/ticket-execution.md",
"workflows/feature-integration.md",
):
self.assertEqual(
(installed_main_loop_root / relative_path).read_bytes(),
(source_main_loop_root / relative_path).read_bytes(),
)
installed_help = subprocess.run(
[
sys.executable,
str(installed_main_loop_root / "scripts/main_loop.py"),
"--help",
],
cwd=project_root,
capture_output=True,
text=True,
)
self.assertEqual(
installed_help.returncode, 0, msg=installed_help.stderr
)
self.assertIn("enqueue", installed_help.stdout)
rules_text = (project_root / "AGENT_RULES.md").read_text(
encoding="utf-8"
)
self.assertIn(
f"`{playbook_root.as_posix()}/` 是 Playbook 模板/供应商目录",
rules_text,
)
self.assertIn("`cook-it-through`", rules_text)
self.assertNotIn("**Blocked by:**", rules_text)
installed_main_loop_text = "\n".join(
(installed_main_loop_root / relative_path).read_text(
encoding="utf-8"
)
for relative_path in (
"SKILL.md",
"rules/session-boundary.md",
"workflows/single-session.md",
"workflows/feature-planning.md",
"workflows/ticket-execution.md",
"workflows/feature-integration.md",
)
)
for contract_fragment in (
"`TicketId`qualified `feature-slug/NN`",
"`FeatureIntegrationId``feature-slug@integrated`",
"**Blocked by:** None",
"**Blocked by:** feature-a/01; feature-b@integrated",
):
self.assertIn(contract_fragment, installed_main_loop_text)
gitignore_text = (project_root / ".gitignore").read_text(
encoding="utf-8"
)
self.assertIn("build/", gitignore_text)
self.assertEqual(
gitignore_text.count("# BEGIN playbook .scratch runtime"),
1,
)
self.assertEqual(
gitignore_text.count("# END playbook .scratch runtime"),
1,
)
self.assertIn("!/.scratch/**", gitignore_text)
self.assertIn("/.scratch/*.lock", gitignore_text)
self.assertIn("/.scratch/worktrees/", gitignore_text)
self.assertIn("/.scratch/**/*.tmp", gitignore_text)
deployed_playbook_root = project_root / playbook_root
deployed_main_loop_root = (
deployed_playbook_root / "skills/cook-it-through/scripts"
)
self.assertTrue((deployed_main_loop_root / "main_loop.py").is_file())
self.assertTrue(
(deployed_main_loop_root / "main_loop_scheduler.py").is_file()
)
self.assertFalse(
(deployed_playbook_root / "scripts/main_loop.py").exists()
)
self.assertFalse(
(deployed_playbook_root / "scripts/main_loop_scheduler.py").exists()
)
if install_mode == "snapshot":
snapshot_root = deployed_playbook_root
self.assertTrue((snapshot_root / "SOURCE.md").is_file())
self.assertTrue(
(snapshot_root / "scripts/playbook.py").is_file()
)
self.assertTrue(
(snapshot_root / "playbook.example.toml").is_file()
)
self.assertTrue(
(snapshot_root / "templates/gitignore.template").is_file()
)
self.assertFalse(
(snapshot_root / "playbook.toml.example").exists()
)
self.assertFalse(
(snapshot_root / "scripts/ticket_loop.py").exists()
)
self.assertTrue((snapshot_root / "skills").is_dir())
else:
self.assertFalse((source_root / "SOURCE.md").exists())
def test_sync_rules_gitignore_block_tracks_durable_scratch_only(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
seed_custom_files(project_root)
initialized = subprocess.run(
["git", "init", "-q"],
cwd=project_root,
capture_output=True,
text=True,
)
self.assertEqual(initialized.returncode, 0, msg=initialized.stderr)
playbook_root = MODE_ROOTS["snapshot"]
config = write_config(project_root, "snapshot", playbook_root)
result = run_playbook(SCRIPT, config, project_root)
self.assertEqual(result.returncode, 0, msg=f"{result.stdout}{result.stderr}")
paths = {
"queue": project_root / ".scratch/queue.md",
"spec": project_root / ".scratch/alpha/spec.md",
"evidence": project_root / ".scratch/alpha/evidence/check.json",
"lock": project_root / ".scratch/.main-loop.lock",
"other_lock": project_root / ".scratch/worker.lock",
"worktree": project_root / ".scratch/worktrees/alpha/file.txt",
"temp": project_root / ".scratch/alpha/.spec.md.deadbeef.tmp",
}
for path in paths.values():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("test\n", encoding="utf-8", newline="\n")
def is_ignored(path: Path) -> bool:
checked = subprocess.run(
[
"git",
"check-ignore",
"-q",
"--",
path.relative_to(project_root).as_posix(),
],
cwd=project_root,
capture_output=True,
text=True,
)
return checked.returncode == 0
for name in ("queue", "spec", "evidence"):
self.assertFalse(is_ignored(paths[name]), msg=f"{name} must be tracked")
for name in ("lock", "other_lock", "worktree", "temp"):
self.assertTrue(is_ignored(paths[name]), msg=f"{name} must be ignored")
def test_sync_without_standards_keeps_language_rules(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
playbook_root = MODE_ROOTS["snapshot"]
full_config = write_config(project_root, "snapshot", playbook_root)
result = run_playbook(SCRIPT, full_config, project_root)
self.assertEqual(
result.returncode,
0,
msg=f"seed run failed\n{result.stdout}{result.stderr}",
)
agents_md = project_root / "AGENTS.md"
seeded = agents_md.read_text(encoding="utf-8")
self.assertIn("`.agents/tsl/index.md`", seeded)
self.assertIn("`.agents/markdown/index.md`", seeded)
partial_config = project_root / "playbook-partial.toml"
partial_config.write_text(
f"""
[playbook]
project_root = "."
playbook_root = "{playbook_root.as_posix()}"
install_mode = "snapshot"
[sync_memory_bank]
project_name = "Demo"
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
result = run_playbook(SCRIPT, partial_config, project_root)
self.assertEqual(
result.returncode,
0,
msg=f"partial run failed\n{result.stdout}{result.stderr}",
)
after = agents_md.read_text(encoding="utf-8")
self.assertIn("`.agents/tsl/index.md`", after)
self.assertIn("`.agents/markdown/index.md`", after)
def test_resync_refreshes_the_rules_block_and_keeps_project_additions(self):
with tempfile.TemporaryDirectory() as tmp_dir:
project_root = Path(tmp_dir) / "project"
project_root.mkdir()
playbook_root = MODE_ROOTS["snapshot"]
config = write_config(project_root, "snapshot", playbook_root)
seed = run_playbook(SCRIPT, config, project_root)
self.assertEqual(
seed.returncode, 0, msg=f"{seed.stdout}{seed.stderr}"
)
rules_md = project_root / "AGENT_RULES.md"
seeded = rules_md.read_text(encoding="utf-8")
self.assertIn("<!-- playbook:rules:start -->", seeded)
self.assertIn("<!-- playbook:rules:end -->", seeded)
# A project appendix outside the block, and drift inside it.
appendix = "\n## 项目补充\n\n保留这段项目自己的说明。\n"
drifted = seeded.replace(
"## 工作流入口", "## 工作流入口(本地改过)"
) + appendix
rules_md.write_text(drifted, encoding="utf-8", newline="\n")
resync = run_playbook(SCRIPT, config, project_root)
self.assertEqual(
resync.returncode, 0, msg=f"{resync.stdout}{resync.stderr}"
)
after = rules_md.read_text(encoding="utf-8")
self.assertIn(
"保留这段项目自己的说明。",
after,
msg="content outside the block belongs to the project",
)
self.assertIn(
"## 工作流入口\n",
after,
msg="the process itself is playbook-owned and must be refreshed",
)
self.assertNotIn("## 工作流入口(本地改过)", after)
self.assertEqual(after.count("<!-- playbook:rules:start -->"), 1)
legacy = project_root / "legacy" / "AGENT_RULES.md"
legacy.parent.mkdir()
legacy.write_text(
"# AGENT_RULES\n\n没有 marker 的旧文件。\n",
encoding="utf-8",
newline="\n",
)
legacy_config = project_root / "playbook-legacy.toml"
legacy_config.write_text(
f"""
[playbook]
project_root = "legacy"
playbook_root = "{playbook_root.as_posix()}"
install_mode = "snapshot"
[sync_rules]
date = "2026-01-01"
no_backup = true
[install_skills]
agents_home = ".test-agents"
mode = "list"
skills = ["cook-it-through"]
no_backup = true
""".lstrip(),
encoding="utf-8",
newline="\n",
)
legacy_run = run_playbook(SCRIPT, legacy_config, project_root)
self.assertEqual(
legacy_run.returncode,
0,
msg=f"{legacy_run.stdout}{legacy_run.stderr}",
)
self.assertEqual(
legacy.read_text(encoding="utf-8"),
"# AGENT_RULES\n\n没有 marker 的旧文件。\n",
msg="a file predating the markers must not be rewritten silently",
)
if __name__ == "__main__":
unittest.main()