95 lines
6.5 KiB
JavaScript
95 lines
6.5 KiB
JavaScript
#!/usr/bin/env node
|
|
"use strict";
|
|
|
|
const assert = require("assert");
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const { buildParityManifest, duplicateGroups, EXPANSION_RULE, expandParityManifest, OVERLAY_MINIMUMS, validateParityExpansion, validateParityManifest } = require("../../local-skill-reviewer/parity-benchmark");
|
|
|
|
const repoRoot = path.resolve(__dirname, "../../..");
|
|
const options = {
|
|
repoRoot,
|
|
seed: "parity-benchmark-test-seed",
|
|
pluginChecksum: "1".repeat(64),
|
|
rubricChecksum: "2".repeat(64),
|
|
};
|
|
|
|
const first = buildParityManifest(options);
|
|
const second = buildParityManifest(options);
|
|
assert.deepStrictEqual(second, first, "same frozen Git index and seed must produce the same manifest");
|
|
assert.strictEqual(validateParityManifest(first), first);
|
|
|
|
assert.deepStrictEqual(first.sizes, { validation: 14, final_blind: 18 });
|
|
assert.strictEqual(first.splits.validation.length, 14);
|
|
assert.strictEqual(first.splits.final_blind.length, 18);
|
|
|
|
const splitIds = Object.fromEntries(Object.entries(first.splits).map(([split, items]) => [split, new Set(items.map((item) => item.id))]));
|
|
const allIds = Object.values(splitIds).flatMap((ids) => [...ids]);
|
|
assert.strictEqual(new Set(allIds).size, 32, "skill IDs must be unique and splits disjoint");
|
|
const calibration = JSON.parse(fs.readFileSync(path.join(repoRoot, "tools/config/local-skill-review-calibration.json"), "utf8"));
|
|
const reviewedIds = [...calibration.tuning, ...calibration.holdout].map((item) => item.id).sort();
|
|
assert.deepStrictEqual(first.excludedCalibrationIds, reviewedIds, "exclusions must come from the reviewed calibration manifest");
|
|
for (const id of reviewedIds) assert(!allIds.includes(id), `reviewed calibration skill leaked into parity cohort: ${id}`);
|
|
|
|
const splitGroups = Object.fromEntries(Object.entries(first.splits).map(([split, items]) => [split, new Set(items.map((item) => item.duplicateGroupSha256))]));
|
|
const allGroups = Object.values(splitGroups).flatMap((groups) => [...groups]);
|
|
assert.strictEqual(new Set(allGroups).size, 32, "no exact/near-duplicate group may cross or repeat within splits");
|
|
assert.deepStrictEqual(first.stabilityPanel, first.splits.validation.slice(0, 4).map((item) => item.id), "stability panel must be an ordered validation subset");
|
|
assert.strictEqual(new Set(first.reserveOrder).size, first.reserveOrder.length, "reserve order must be unique");
|
|
assert(first.reserveOrder.every((id) => !allIds.includes(id) && !reviewedIds.includes(id)), "reserve must exclude selected and reviewed skills");
|
|
|
|
const grouped = duplicateGroups([
|
|
{ id: "exact-a", contentHash: "a", tokenCount: 100, simhash: 0n },
|
|
{ id: "exact-b", contentHash: "a", tokenCount: 100, simhash: ~0n },
|
|
{ id: "near", contentHash: "b", tokenCount: 95, simhash: 3n },
|
|
{ id: "different", contentHash: "c", tokenCount: 100, simhash: 0xffffn },
|
|
]);
|
|
assert.strictEqual(grouped[0], grouped[1], "exact normalized content must group");
|
|
assert.strictEqual(grouped[0], grouped[2], "near content signature must group transitively");
|
|
assert.notStrictEqual(grouped[0], grouped[3], "materially different content must stay separate");
|
|
|
|
for (const split of Object.keys(first.splits)) {
|
|
const primaryTotal = Object.values(first.coverage[split].primaryStrata).reduce((sum, value) => sum + value, 0);
|
|
assert.strictEqual(primaryTotal, first.sizes[split], `${split} primary strata counts must cover its whole split`);
|
|
assert(Object.keys(first.coverage[split].primaryStrata).every((key) => /^length-s[1-6]:(?:bundle|no-bundle)$/.test(key)));
|
|
for (const [overlay, minimum] of Object.entries(OVERLAY_MINIMUMS[split])) {
|
|
assert(first.coverage[split].overlays[overlay] >= minimum, `${split} must cover ${overlay} at least ${minimum} time(s)`);
|
|
}
|
|
}
|
|
|
|
const tampered = JSON.parse(JSON.stringify(first));
|
|
tampered.unexpected = true;
|
|
assert.throws(() => validateParityManifest(tampered), /schema is not closed/);
|
|
const belowMinimum = JSON.parse(JSON.stringify(first));
|
|
belowMinimum.coverage.validation.overlays.nonEnglish = 0;
|
|
assert.throws(() => validateParityManifest(belowMinimum), /overlay coverage minimum not met/);
|
|
|
|
const expanded = expandParityManifest({ repoRoot, parentManifest: first });
|
|
const expandedAgain = expandParityManifest({ repoRoot, parentManifest: first });
|
|
assert.deepStrictEqual(expandedAgain, expanded, "v2 expansion must be deterministic from the frozen v1 reserve");
|
|
assert.strictEqual(validateParityManifest(expanded), expanded);
|
|
assert.strictEqual(validateParityExpansion(first, expanded), expanded);
|
|
assert.strictEqual(expanded.manifestVersion, 2);
|
|
assert.deepStrictEqual(expanded.sizes, { validation: 25, final_blind: 35 });
|
|
assert.deepStrictEqual(expanded.expansionRule, EXPANSION_RULE);
|
|
assert.strictEqual(expanded.parentSelectionSha256, first.integrity.selectionSha256);
|
|
assert.deepStrictEqual(expanded.splits.validation.slice(0, 14), first.splits.validation, "v2 must preserve the v1 validation prefix");
|
|
assert.deepStrictEqual(expanded.splits.validation.slice(14).map((item) => item.id), first.reserveOrder.slice(0, 11), "validation must consume the first 11 reserve IDs");
|
|
assert.deepStrictEqual(expanded.splits.final_blind.slice(0, 18), first.splits.final_blind, "v2 must preserve the v1 final-blind prefix");
|
|
assert.deepStrictEqual(expanded.splits.final_blind.slice(18).map((item) => item.id), first.reserveOrder.slice(11, 28), "final blind must consume the next 17 reserve IDs");
|
|
assert.deepStrictEqual(expanded.reserveOrder, first.reserveOrder.slice(28), "v2 reserve must remove exactly the consumed prefix");
|
|
assert.deepStrictEqual(expanded.stabilityPanel.slice(0, 4), first.stabilityPanel, "v2 stability panel must preserve the original four first");
|
|
assert.deepStrictEqual(expanded.stabilityPanel, expanded.splits.validation.slice(0, 15).map((item) => item.id));
|
|
assert.strictEqual(new Set(Object.values(expanded.splits).flat().map((item) => item.duplicateGroupSha256)).size, 60);
|
|
assert(/^[0-9a-f]{64}$/.test(expanded.integrity.parentManifestSha256));
|
|
assert(/^[0-9a-f]{64}$/.test(expanded.integrity.expansionSha256));
|
|
|
|
const prefixTamper = JSON.parse(JSON.stringify(expanded));
|
|
[prefixTamper.splits.validation[14], prefixTamper.splits.validation[15]] = [prefixTamper.splits.validation[15], prefixTamper.splits.validation[14]];
|
|
assert.throws(() => validateParityExpansion(first, prefixTamper), /invalid|integrity|prefix/i, "reordered reserve-prefix expansion must be rejected");
|
|
const expandedUnknownKey = JSON.parse(JSON.stringify(expanded));
|
|
expandedUnknownKey.expansionRule.unexpected = true;
|
|
assert.throws(() => validateParityManifest(expandedUnknownKey), /schema is not closed/);
|
|
|
|
process.stdout.write("parity benchmark tests passed\n");
|