235 lines
8.6 KiB
Python
235 lines
8.6 KiB
Python
import importlib.util
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
TOOLS_SCRIPTS_DIR = REPO_ROOT / "tools" / "scripts"
|
|
if str(TOOLS_SCRIPTS_DIR) not in sys.path:
|
|
sys.path.insert(0, str(TOOLS_SCRIPTS_DIR))
|
|
|
|
|
|
def load_module(relative_path: str, module_name: str):
|
|
module_path = REPO_ROOT / relative_path
|
|
spec = importlib.util.spec_from_file_location(module_name, module_path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec.loader is not None
|
|
sys.modules[module_name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
generate_registry_report = load_module(
|
|
"tools/scripts/generate_registry_report.py", "generate_registry_report"
|
|
)
|
|
score_skills = load_module("tools/scripts/score_skills.py", "score_skills")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SKILL_TEMPLATE = """\
|
|
---
|
|
name: {name}
|
|
description: A test skill for registry report generation.
|
|
risk: {risk}
|
|
source: community
|
|
date_added: 2026-01-01
|
|
category: testing
|
|
---
|
|
|
|
# {name}
|
|
|
|
## When to Use
|
|
- Use in registry report tests.
|
|
|
|
## Examples
|
|
```bash
|
|
echo "test"
|
|
```
|
|
|
|
## Limitations
|
|
- Test fixture only.
|
|
"""
|
|
|
|
|
|
def _write_skill(skills_dir: Path, name: str, risk: str = "safe") -> Path:
|
|
skill_dir = skills_dir / name
|
|
skill_dir.mkdir(parents=True, exist_ok=True)
|
|
(skill_dir / "SKILL.md").write_text(
|
|
_SKILL_TEMPLATE.format(name=name, risk=risk), encoding="utf-8"
|
|
)
|
|
return skill_dir
|
|
|
|
|
|
def _make_score(
|
|
skill_id: str,
|
|
total: float = 75.0,
|
|
label: str = "good",
|
|
risk: str = "safe",
|
|
flag_severity: str | None = None,
|
|
) -> score_skills.SkillScore:
|
|
flags = []
|
|
if flag_severity:
|
|
flags = [{"code": "SEC001", "severity": flag_severity, "message": "test", "line": 1, "matched_text": "x"}]
|
|
return score_skills.SkillScore(
|
|
skill_id=skill_id,
|
|
risk=risk,
|
|
metadata_score=total,
|
|
documentation_score=total,
|
|
security_score=total,
|
|
total_score=total,
|
|
label=label,
|
|
flags=flags,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Report building
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ReportBuildingTests(unittest.TestCase):
|
|
def test_report_has_required_top_level_keys(self):
|
|
scores = [_make_score("skill-a"), _make_score("skill-b")]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
|
|
for key in ("schema_version", "generated_at", "skills_version", "summary", "skills"):
|
|
self.assertIn(key, report, f"Missing key: {key}")
|
|
|
|
def test_report_skills_version_matches_input(self):
|
|
scores = [_make_score("skill-a")]
|
|
report = generate_registry_report.build_report(scores, "99.9.9")
|
|
self.assertEqual(report["skills_version"], "99.9.9")
|
|
|
|
def test_report_summary_total_skills_count(self):
|
|
scores = [_make_score(f"skill-{i}") for i in range(10)]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
self.assertEqual(report["summary"]["total_skills"], 10)
|
|
|
|
def test_report_skills_list_length_matches_scores(self):
|
|
scores = [_make_score(f"s-{i}") for i in range(7)]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
self.assertEqual(len(report["skills"]), 7)
|
|
|
|
def test_report_skills_sorted_by_total_score_ascending(self):
|
|
scores = [
|
|
_make_score("high", 90.0, "excellent"),
|
|
_make_score("low", 30.0, "critical"),
|
|
_make_score("mid", 60.0, "good"),
|
|
]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
totals = [s["scores"]["total"] for s in report["skills"]]
|
|
self.assertEqual(totals, sorted(totals))
|
|
|
|
def test_report_security_flags_counted(self):
|
|
scores = [
|
|
_make_score("err-skill", flag_severity="error"),
|
|
_make_score("warn-skill", flag_severity="warning"),
|
|
_make_score("clean-skill"),
|
|
]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
sec = report["summary"]["security"]
|
|
self.assertEqual(sec["flag_errors"], 1)
|
|
self.assertEqual(sec["flag_warnings"], 1)
|
|
|
|
def test_report_risk_breakdown_structure(self):
|
|
scores = [
|
|
_make_score("a", risk="safe"),
|
|
_make_score("b", risk="safe"),
|
|
_make_score("c", risk="critical"),
|
|
]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
risk_list = report["summary"]["risk_breakdown"]
|
|
self.assertIsInstance(risk_list, list)
|
|
risk_map = {item["risk"]: item["count"] for item in risk_list}
|
|
self.assertEqual(risk_map.get("safe", 0), 2)
|
|
self.assertEqual(risk_map.get("critical", 0), 1)
|
|
|
|
def test_report_score_distribution_structure(self):
|
|
scores = [
|
|
_make_score("a", 90.0, "excellent"),
|
|
_make_score("b", 70.0, "good"),
|
|
_make_score("c", 50.0, "needs_improvement"),
|
|
_make_score("d", 20.0, "critical"),
|
|
]
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
dist_list = report["summary"]["score_distribution"]
|
|
dist_map = {item["label"]: item["count"] for item in dist_list}
|
|
self.assertEqual(dist_map["excellent"], 1)
|
|
self.assertEqual(dist_map["good"], 1)
|
|
self.assertEqual(dist_map["needs_improvement"], 1)
|
|
self.assertEqual(dist_map["critical"], 1)
|
|
|
|
def test_report_with_drift_summary_includes_drift_key(self):
|
|
scores = [_make_score("skill-a")]
|
|
drift = {"has_drift": True, "added": ["new-skill"], "removed": [], "drifted": [], "unchanged_count": 1}
|
|
report = generate_registry_report.build_report(scores, "12.7.0", drift_summary=drift)
|
|
self.assertIn("drift", report)
|
|
self.assertTrue(report["drift"]["has_drift"])
|
|
|
|
def test_report_without_drift_has_no_drift_key(self):
|
|
scores = [_make_score("skill-a")]
|
|
report = generate_registry_report.build_report(scores, "12.7.0", drift_summary=None)
|
|
self.assertNotIn("drift", report)
|
|
|
|
def test_report_schema_version_is_integer(self):
|
|
report = generate_registry_report.build_report([_make_score("x")], "1.0.0")
|
|
self.assertIsInstance(report["schema_version"], int)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# End-to-end (file system)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class RegistryReportEndToEndTests(unittest.TestCase):
|
|
def test_generated_report_is_valid_json(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
skills_dir = Path(tmp) / "skills"
|
|
output_path = Path(tmp) / "report.json"
|
|
|
|
for i in range(3):
|
|
_write_skill(skills_dir, f"skill-{i}")
|
|
|
|
scores = score_skills.score_all_skills(skills_dir)
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
|
|
output_path.write_text(
|
|
json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
|
|
reloaded = json.loads(output_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(reloaded["summary"]["total_skills"], 3)
|
|
|
|
def test_report_generated_at_is_iso_format(self):
|
|
from datetime import datetime
|
|
scores = [_make_score("x")]
|
|
report = generate_registry_report.build_report(scores, "1.0.0")
|
|
generated_at = report["generated_at"]
|
|
# Should parse without error
|
|
datetime.fromisoformat(generated_at.replace("Z", "+00:00"))
|
|
|
|
def test_empty_skills_directory_produces_valid_report(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
skills_dir = Path(tmp)
|
|
scores = score_skills.score_all_skills(skills_dir)
|
|
self.assertEqual(scores, [])
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
self.assertEqual(report["summary"].get("total_skills", 0), 0)
|
|
|
|
def test_report_skill_entries_have_scores_key(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
skills_dir = Path(tmp) / "skills"
|
|
_write_skill(skills_dir, "my-skill")
|
|
scores = score_skills.score_all_skills(skills_dir)
|
|
report = generate_registry_report.build_report(scores, "12.7.0")
|
|
for skill_entry in report["skills"]:
|
|
self.assertIn("scores", skill_entry)
|
|
self.assertIn("total", skill_entry["scores"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|