#!/usr/bin/env python3 """Deterministic before/after evidence for canonical skills changed by a PR.""" from __future__ import annotations import argparse import json import tempfile from collections import Counter, defaultdict from pathlib import Path from _project_paths import find_repo_root from audit_skills import build_skill_report from check_readme_credits import ( SOURCE_REPO_PATTERN, extract_credit_repos, normalize_repo_slug, parse_frontmatter, ) from git_change_records import ChangeRecord, list_tree, materialize_tree, read_change_records, read_path from security_scanner import scan_skill_file SCHEMA_VERSION = 1 RISK_RANK = {"unknown": -1, "none": 0, "safe": 1, "critical": 2, "offensive": 3} SEVERITY_RANK = {"info": 0, "warning": 1, "error": 2} def canonical_skill_roots(repo: Path, commit_oid: str) -> set[str]: roots: set[str] = set() for entry in list_tree(repo, commit_oid, "skills"): if entry.path.startswith("skills/") and entry.path.endswith("/SKILL.md"): roots.add(entry.path[len("skills/") : -len("/SKILL.md")]) return roots def canonical_skill_id(path: str | None, roots: set[str]) -> str | None: """Resolve a path to its nearest canonical SKILL.md ancestor.""" if path is None or not path.startswith("skills/"): return None relative = path[len("skills/") :] candidates = [ root for root in roots if relative == f"{root}/SKILL.md" or relative.startswith(f"{root}/") ] return max(candidates, key=lambda root: (root.count("/"), len(root))) if candidates else None def skill_pairs( records: list[ChangeRecord], old_roots: set[str], new_roots: set[str], ) -> list[tuple[str | None, str | None, list[ChangeRecord]]]: affected: set[str] = set() rename_pairs: dict[tuple[str, str], list[ChangeRecord]] = defaultdict(list) for record in records: old_id = canonical_skill_id(record.old_path, old_roots) new_id = canonical_skill_id(record.new_path, new_roots) if old_id: affected.add(old_id) if new_id: affected.add(new_id) if record.status in {"R", "C"} and old_id and new_id and old_id != new_id: rename_pairs[(old_id, new_id)].append(record) result: list[tuple[str | None, str | None, list[ChangeRecord]]] = [] consumed: set[str] = set() for (old_id, new_id), pair_records in sorted(rename_pairs.items()): if old_id in consumed or new_id in consumed: continue is_copy = any(record.status == "C" for record in pair_records) if is_copy: relevant = [ record for record in records if canonical_skill_id(record.new_path, new_roots) == new_id ] else: relevant = [ record for record in records if ( canonical_skill_id(record.new_path, new_roots) == new_id or ( canonical_skill_id(record.old_path, old_roots) == old_id and not ( record.status in {"R", "C"} and canonical_skill_id(record.new_path, new_roots) not in {old_id, new_id} ) ) ) ] result.append((old_id, new_id, relevant or pair_records)) consumed.add(new_id) if not is_copy: consumed.add(old_id) for skill_id in sorted(affected - consumed): relevant = [ record for record in records if ( canonical_skill_id(record.new_path, new_roots) == skill_id or ( canonical_skill_id(record.old_path, old_roots) == skill_id and not ( record.status in {"R", "C"} and canonical_skill_id(record.new_path, new_roots) != skill_id ) ) ) ] if relevant: result.append((skill_id, skill_id, relevant)) return result def _findings_by_severity_and_code(findings: list[dict[str, object]]) -> dict[str, dict[str, list[str]]]: grouped: dict[str, dict[str, list[str]]] = {} for finding in findings: severity = str(finding.get("severity", "unknown")) code = str(finding.get("code", "unknown")) grouped.setdefault(severity, {}).setdefault(code, []).append(str(finding.get("message", ""))) return { severity: { code: sorted(messages) for code, messages in sorted(codes.items()) } for severity, codes in sorted(grouped.items()) } def _metadata(skill_file: Path) -> dict[str, object]: try: return parse_frontmatter(skill_file.read_text(encoding="utf-8")) except (OSError, UnicodeError): return {} def _safe_metadata_value(value: object) -> object: if value is None or isinstance(value, (str, bool, int, float)): return value return {"invalid_type": type(value).__name__} def evaluate_snapshot(snapshot_root: Path, skill_id: str) -> dict[str, object]: skills_root = snapshot_root / "skills" skill_root = skills_root / skill_id audit = build_skill_report(skill_root, skills_root, snapshot_root=snapshot_root) try: security = scan_skill_file(skill_root) security_payload = security.to_dict() if security is not None else None except (OSError, UnicodeError, TypeError, ValueError): security_payload = { "skill_id": skill_id, "status": "error", "is_offensive": False, "error_count": 1, "warning_count": 0, "flags": [ { "code": "INVALID_SKILL_ENCODING", "severity": "error", "message": "SKILL.md is not valid UTF-8.", "line": 0, "matched_text": "", } ], } audit_findings = _findings_by_severity_and_code(audit["findings"]) audit_error_count = audit["error_count"] metadata = _metadata(skill_root / "SKILL.md") source_repo = normalize_repo_slug(metadata.get("source_repo")) raw_source_type = metadata.get("source_type") source_type = raw_source_type.strip().lower() if isinstance(raw_source_type, str) else None return { "audit": { "status": "error" if audit_error_count else audit["status"], "counts": { "error": audit_error_count, "warning": audit["warning_count"], "info": audit["info_count"], }, "findings": audit_findings, }, "security": security_payload, "risk": { "declared": _safe_metadata_value(metadata.get("risk")), }, "provenance": { "source": _safe_metadata_value(metadata.get("source")), "source_type": source_type, "source_repo": source_repo, }, } def _audit_severities(snapshot: dict[str, object] | None) -> dict[str, list[str]]: severities: dict[str, list[str]] = defaultdict(list) if not snapshot: return severities findings = snapshot["audit"]["findings"] for severity, by_code in findings.items(): for code, messages in by_code.items(): severities[code].extend([severity] * len(messages)) return severities def _security_severities(snapshot: dict[str, object] | None) -> dict[str, list[str]]: severities: dict[str, list[str]] = defaultdict(list) if not snapshot or snapshot.get("security") is None: return severities for flag in snapshot["security"]["flags"]: severities[str(flag["code"])].append(str(flag["severity"])) return severities def _worsened_codes( before: dict[str, list[str]], after: dict[str, list[str]], ) -> list[tuple[str, str, int]]: worsened: list[tuple[str, str, int]] = [] for code, after_values in sorted(after.items()): after_ranks = [SEVERITY_RANK.get(value, 0) for value in after_values] before_ranks = [SEVERITY_RANK.get(value, 0) for value in before.get(code, [])] if not after_ranks or max(after_ranks) == 0: continue maximum_increased = max(after_ranks) > max(before_ranks, default=0) weight_increased = sum(after_ranks) > sum(before_ranks) if maximum_increased or weight_increased: highest = max(after_ranks) severity = "error" if highest >= 2 else "warning" worsened.append((severity, code, max(1, sum(after_ranks) - sum(before_ranks)))) return worsened def regression_reasons( skill_id: str, change_type: str, before: dict[str, object] | None, after: dict[str, object] | None, ) -> list[str]: if change_type == "deleted" or after is None: return [] reasons: list[str] = [] for severity, code, increase in _worsened_codes(_audit_severities(before), _audit_severities(after)): reasons.append(f"{skill_id}:audit_{severity}_regression:{code}:+{increase}") for severity, code, increase in _worsened_codes( _security_severities(before), _security_severities(after) ): reasons.append(f"{skill_id}:security_{severity}_regression:{code}:+{increase}") before_declared = str(before["risk"].get("declared") or "unknown").lower() if before else "unknown" after_declared = str(after["risk"].get("declared") or "unknown").lower() if before and RISK_RANK.get(after_declared, -1) < RISK_RANK.get(before_declared, -1): reasons.append(f"{skill_id}:risk_downgrade:{before_declared}->{after_declared}") return reasons def provenance_reasons( skill_id: str, change_type: str, before: dict[str, object] | None, after: dict[str, object] | None, readme_credits: dict[str, set[str]], ) -> list[str]: if change_type == "deleted" or after is None: return [] provenance = after["provenance"] source = provenance.get("source") source_type = provenance.get("source_type") source_repo = provenance.get("source_repo") reasons: list[str] = [] source_is_self = isinstance(source, str) and source.strip().lower() == "self" before_provenance = before.get("provenance") if before else None before_source = before_provenance.get("source") if before_provenance else None before_is_self = ( isinstance(before_source, str) and before_source.strip().lower() == "self" ) if change_type in {"modified", "renamed"} and before_provenance: if not before_is_self or not source_is_self: for field in ("source", "source_type", "source_repo"): if before_provenance.get(field) != provenance.get(field): reasons.append(f"{skill_id}:provenance_identity_changed:{field}") needs_full_validation = change_type in {"added", "copied"} or ( before_provenance is not None and before_is_self and not source_is_self ) if not source_is_self and needs_full_validation: if source_type not in {"official", "community"}: reasons.append(f"{skill_id}:new_external_skill_invalid_source_type") if not isinstance(source_repo, str) or not SOURCE_REPO_PATTERN.fullmatch(source_repo): reasons.append(f"{skill_id}:new_external_skill_invalid_source_repo") elif source_type in readme_credits and source_repo not in readme_credits[source_type]: reasons.append( f"{skill_id}:new_external_skill_missing_readme_credit:{source_type}:{source_repo}" ) return reasons def _unsafe_counter(entries: list[dict[str, str]], skill_id: str | None) -> Counter[tuple[str, str, str]]: prefix = f"skills/{skill_id}/" if skill_id else "" return Counter( ( item["path"][len(prefix) :] if prefix and item["path"].startswith(prefix) else item["path"], item["mode"], item["oid"], ) for item in entries ) def build_report(repo: str | Path, base_ref: str, head_ref: str) -> dict[str, object]: root = Path(repo) base_oid, head_oid, records = read_change_records(root, base_ref, head_ref, merge_base=True) old_roots = canonical_skill_roots(root, base_oid) new_roots = canonical_skill_roots(root, head_oid) readme_bytes = read_path(root, head_oid, "README.md") or b"" readme_credits = extract_credit_repos(readme_bytes.decode("utf-8", "replace")) changes: list[dict[str, object]] = [] all_reasons: list[str] = [] with tempfile.TemporaryDirectory(prefix="changed-skill-evidence-") as temporary: temp_root = Path(temporary) for index, (old_id, new_id, pair_records) in enumerate(skill_pairs(records, old_roots, new_roots)): # Tree membership, rather than blob readability, distinguishes a # deleted skill from a SKILL.md replaced by a symlink or gitlink. base_exists = bool(old_id and old_id in old_roots) head_exists = bool(new_id and new_id in new_roots) if not base_exists and not head_exists: continue copied = any(record.status == "C" for record in pair_records) and old_id != new_id if copied: change_type = "copied" elif base_exists and head_exists and old_id != new_id: change_type = "renamed" elif base_exists and head_exists: change_type = "modified" elif head_exists: change_type = "added" else: change_type = "deleted" before = None after = None before_unsafe: list[dict[str, str]] = [] after_unsafe: list[dict[str, str]] = [] if base_exists and old_id: before_root = temp_root / f"{index}-before" before_unsafe = materialize_tree(root, base_oid, f"skills/{old_id}", before_root / "skills" / old_id) if (before_root / "skills" / old_id / "SKILL.md").is_file(): before = evaluate_snapshot(before_root, old_id) if head_exists and new_id: after_root = temp_root / f"{index}-after" after_unsafe = materialize_tree(root, head_oid, f"skills/{new_id}", after_root / "skills" / new_id) if (after_root / "skills" / new_id / "SKILL.md").is_file(): after = evaluate_snapshot(after_root, new_id) effective_id = new_id or old_id or "unknown" reasons: list[str] = [] if change_type != "deleted": before_modes = ( Counter() if change_type in {"added", "copied"} else _unsafe_counter(before_unsafe, old_id) ) after_modes = _unsafe_counter(after_unsafe, new_id) for (relative_path, mode, oid), count in sorted(after_modes.items()): increase = count - before_modes[(relative_path, mode, oid)] if increase > 0: reasons.append( f"{effective_id}:unsafe_snapshot_regression:{mode}:{relative_path}:{oid}:+{increase}" ) comparison_before = None if change_type in {"added", "copied"} else before reasons.extend(regression_reasons(effective_id, change_type, comparison_before, after)) reasons.extend( provenance_reasons( effective_id, change_type, comparison_before, after, readme_credits, ) ) reasons = sorted(set(reasons)) all_reasons.extend(reasons) changes.append( { "change_type": change_type, "old_skill_id": old_id if base_exists else None, "new_skill_id": new_id if head_exists else None, "records": [record.to_dict() for record in pair_records], "before": before, "after": after, "unsafe_entries": {"before": before_unsafe, "after": after_unsafe}, "blocking": bool(reasons), "reasons": reasons, } ) changes.sort(key=lambda item: (str(item["new_skill_id"] or item["old_skill_id"]), item["change_type"])) reasons = sorted(set(all_reasons)) return { "schema_version": SCHEMA_VERSION, "base_ref": base_ref, "head_ref": head_ref, "base_oid": base_oid, "head_oid": head_oid, "changes": changes, "blocking": bool(reasons), "reasons": reasons, } def stable_json(report: dict[str, object]) -> str: return json.dumps(report, indent=2, sort_keys=True, ensure_ascii=True) + "\n" def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Build changed-skill before/after evidence.") parser.add_argument("--base", required=True) parser.add_argument("--head", required=True) parser.add_argument("--output", required=True, type=Path) parser.add_argument( "--repo", type=Path, help="Repository whose immutable Git objects should be evaluated (defaults to the script checkout).", ) parser.add_argument("--json", action="store_true", help="Also print the report to stdout.") return parser.parse_args() def main() -> int: args = parse_args() root = args.repo.resolve() if args.repo else find_repo_root(__file__) report = build_report(root, args.base, args.head) payload = stable_json(report) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(payload, encoding="utf-8") if args.json: print(payload, end="") return 1 if report["blocking"] else 0 if __name__ == "__main__": raise SystemExit(main())