📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unit tests for metric math and relevance fixture validation."""
|
||||
|
||||
import importlib.util
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = next(parent for parent in Path(__file__).resolve().parents
|
||||
if (parent / "scripts/evaluate-relevance.py").exists())
|
||||
MODULE_PATH = ROOT / "scripts/evaluate-relevance.py"
|
||||
SPEC = importlib.util.spec_from_file_location("evaluate_relevance", MODULE_PATH)
|
||||
evaluator = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(evaluator)
|
||||
|
||||
|
||||
class TestMetricMath(unittest.TestCase):
|
||||
def test_precision_counts_missing_ranks_as_non_relevant(self):
|
||||
self.assertEqual(evaluator.precision_at_k([2], 1), 1.0)
|
||||
self.assertAlmostEqual(evaluator.precision_at_k([2], 3), 1 / 3)
|
||||
self.assertEqual(evaluator.precision_at_k([], 3), 0.0)
|
||||
self.assertEqual(evaluator.precision_at_k([2], 0), 0.0)
|
||||
|
||||
def test_reciprocal_rank_stops_at_k(self):
|
||||
self.assertEqual(evaluator.reciprocal_rank([0, 2, 0]), 0.5)
|
||||
self.assertEqual(evaluator.reciprocal_rank([0, 0, 0, 2]), 0.0)
|
||||
self.assertEqual(evaluator.reciprocal_rank([]), 0.0)
|
||||
|
||||
def test_ndcg_uses_graded_gain_and_handles_empty_ideal(self):
|
||||
self.assertEqual(evaluator.ndcg_at_k([2, 1], [2, 1]), 1.0)
|
||||
self.assertLess(evaluator.ndcg_at_k([1, 2], [2, 1]), 1.0)
|
||||
self.assertEqual(evaluator.ndcg_at_k([], [], 3), 0.0)
|
||||
|
||||
def test_result_grades_match_identity_subsets(self):
|
||||
results = [
|
||||
{"Category": "State", "Guideline": "Use useState", "Severity": "Medium"},
|
||||
{"Category": "State", "Guideline": "Use useReducer", "Severity": "Medium"},
|
||||
]
|
||||
judgments = [
|
||||
{"identity": {"Guideline": "Use useReducer"}, "grade": 2},
|
||||
{"identity": {"Category": "State"}, "grade": 1},
|
||||
]
|
||||
self.assertEqual(evaluator.grades_for_results(results, judgments), [1, 2])
|
||||
|
||||
|
||||
class TestFixtureValidation(unittest.TestCase):
|
||||
@staticmethod
|
||||
def valid_fixture():
|
||||
case = {
|
||||
"id": "domain-style-minimal",
|
||||
"split": "calibration",
|
||||
"mode": "domain",
|
||||
"domain": "style",
|
||||
"query": "minimal grid",
|
||||
"judgments": [{"identity": {"Style Category": "Minimalism"}, "grade": 2}],
|
||||
}
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"globalNegativeApplicability": {"domains": ["style"], "stacks": []},
|
||||
"cases": [dict(case, id=f"case-{index}") for index in range(60)],
|
||||
}
|
||||
|
||||
def test_valid_schema(self):
|
||||
self.assertEqual(evaluator.validate_fixture(self.valid_fixture(), {"style": {}}, []), [])
|
||||
|
||||
def test_rejects_bad_count_duplicate_id_and_grade(self):
|
||||
fixture = self.valid_fixture()
|
||||
fixture["cases"] = fixture["cases"][:2]
|
||||
fixture["cases"][1]["id"] = fixture["cases"][0]["id"]
|
||||
fixture["cases"][0]["judgments"][0]["grade"] = 3
|
||||
errors = "\n".join(evaluator.validate_fixture(fixture, {"style": {}}, []))
|
||||
self.assertIn("60-100", errors)
|
||||
self.assertIn("duplicate case id", errors)
|
||||
self.assertIn("grade 1 or 2", errors)
|
||||
|
||||
|
||||
class TestThresholdGate(unittest.TestCase):
|
||||
def test_runtime_fingerprint_binds_reasoning_contract(self):
|
||||
original = evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
runtime = root / "src/ui-ux-pro-max/scripts"
|
||||
data = root / "src/ui-ux-pro-max/data"
|
||||
runtime.mkdir(parents=True)
|
||||
data.mkdir(parents=True)
|
||||
for name in ("core.py", "design_system.py", "reasoning_contract.py"):
|
||||
(runtime / name).write_text(name, encoding="utf-8")
|
||||
(data / "styles.csv").write_text("No,Style\n1,Test\n", encoding="utf-8")
|
||||
evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR = root, runtime, data
|
||||
try:
|
||||
before = evaluator.runtime_fingerprint()
|
||||
(runtime / "reasoning_contract.py").write_text("changed", encoding="utf-8")
|
||||
self.assertNotEqual(before, evaluator.runtime_fingerprint())
|
||||
finally:
|
||||
evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR = original
|
||||
|
||||
def test_oracle_fingerprint_hashes_the_selected_cases_file(self):
|
||||
canonical = evaluator.FIXTURE_DIR / "relevance-cases.json"
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
selected = Path(tmp) / "cases.json"
|
||||
selected.write_bytes(canonical.read_bytes())
|
||||
self.assertEqual(
|
||||
evaluator.oracle_fingerprint(selected), evaluator.oracle_fingerprint(canonical))
|
||||
selected.write_bytes(canonical.read_bytes() + b" ")
|
||||
self.assertNotEqual(
|
||||
evaluator.oracle_fingerprint(selected), evaluator.oracle_fingerprint(canonical))
|
||||
|
||||
def test_metric_sample_and_locked_case_failures_are_actionable(self):
|
||||
report = {
|
||||
"metrics": {"precisionAt1": 0.5},
|
||||
"samples": {"retrieval": 1},
|
||||
"cases": [{"id": "locked", "grades": [0], "actual": [{"Style Category": "Wrong"}]}],
|
||||
}
|
||||
manifest = {
|
||||
"metrics": {"precisionAt1": {"floor": 0.8, "tolerance": 0.01}},
|
||||
"sampleMinimums": {"retrieval": 2},
|
||||
"lockedCases": {"locked": {"withinTop": 1, "minimumGrade": 2}},
|
||||
}
|
||||
manifest["splits"] = {"calibration": {"metrics": {}, "sampleMinimums": {}},
|
||||
"held_out": {"metrics": {}, "sampleMinimums": {}}}
|
||||
report["splits"] = {"calibration": {"metrics": {}, "samples": {}},
|
||||
"held_out": {"metrics": {}, "samples": {}}}
|
||||
failures = evaluator.check_thresholds(report, manifest)
|
||||
self.assertEqual(len(failures), 3)
|
||||
self.assertTrue(any("Wrong" in failure for failure in failures))
|
||||
|
||||
def test_manifest_rejects_missing_contract_sections(self):
|
||||
errors = evaluator.validate_manifest({}, "fingerprint")
|
||||
self.assertTrue(any("missing sections" in error for error in errors))
|
||||
self.assertTrue(any("missing metrics" in error for error in errors))
|
||||
|
||||
def test_manifest_rejects_non_finite_and_invalid_sample_values(self):
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"status": "approved",
|
||||
"approvingMaintainer": "maintainer",
|
||||
"units": "ratios",
|
||||
"splitPolicy": {},
|
||||
"runtimeFingerprint": "fingerprint",
|
||||
"oracleFingerprint": "oracle",
|
||||
"baselineRevision": "97eb2a2",
|
||||
"metrics": {name: {"floor": float("nan")} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": True},
|
||||
"lockedCases": {"case": {}},
|
||||
"splits": {
|
||||
split: {
|
||||
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": 1},
|
||||
} for split in ("calibration", "held_out")
|
||||
},
|
||||
}
|
||||
errors = evaluator.validate_manifest(manifest, "fingerprint", "oracle")
|
||||
self.assertTrue(any("finite" in error for error in errors))
|
||||
self.assertTrue(any("non-negative integer" in error for error in errors))
|
||||
|
||||
def test_manifest_binds_oracle_and_validates_baseline_revision(self):
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"status": "approved",
|
||||
"approvingMaintainer": "maintainer",
|
||||
"units": "ratios",
|
||||
"splitPolicy": {},
|
||||
"runtimeFingerprint": "runtime",
|
||||
"oracleFingerprint": "wrong",
|
||||
"baselineRevision": "not-a-revision",
|
||||
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": 1},
|
||||
"lockedCases": {"case": {}},
|
||||
"splits": {
|
||||
split: {
|
||||
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": 1},
|
||||
} for split in ("calibration", "held_out")
|
||||
},
|
||||
}
|
||||
errors = evaluator.validate_manifest(manifest, "runtime", "expected")
|
||||
self.assertTrue(any("oracleFingerprint" in error for error in errors))
|
||||
self.assertTrue(any("baselineRevision" in error for error in errors))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user