📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.resolve(here, "..", "..");
|
||||
const modeArg = process.argv.indexOf("--mode");
|
||||
const mode = modeArg >= 0 ? process.argv[modeArg + 1] : "structure";
|
||||
|
||||
if (!["structure", "freeze-ready"].includes(mode)) {
|
||||
console.error(`Unknown mode: ${mode}`);
|
||||
process.exit(64);
|
||||
}
|
||||
|
||||
const failures = [];
|
||||
const pending = [];
|
||||
const assert = (condition, code, detail) => {
|
||||
if (!condition) failures.push({ code, detail });
|
||||
};
|
||||
const requireForFreeze = (condition, code, detail) => {
|
||||
if (!condition) pending.push({ code, detail });
|
||||
};
|
||||
const readJson = (relative) => JSON.parse(fs.readFileSync(path.join(root, relative), "utf8"));
|
||||
|
||||
const schemaDir = path.join(root, "schemas");
|
||||
const schemaFiles = fs.readdirSync(schemaDir).filter((name) => name.endsWith(".schema.json"));
|
||||
assert(schemaFiles.length >= 9, "AAS_BASELINE_SCHEMA_SET_INCOMPLETE", "At least nine public schemas are required.");
|
||||
for (const schemaFile of schemaFiles) {
|
||||
const schema = readJson(`schemas/${schemaFile}`);
|
||||
assert(schema.$schema === "https://json-schema.org/draft/2020-12/schema", "AAS_BASELINE_SCHEMA_DRAFT", schemaFile);
|
||||
assert(typeof schema.$id === "string" && schema.$id.length > 0, "AAS_BASELINE_SCHEMA_ID", schemaFile);
|
||||
}
|
||||
|
||||
const metrics = readJson("baseline/v1/metrics.json");
|
||||
assert(metrics.heldOutCasesPerIntent === 30, "AAS_BASELINE_METRIC_DENOMINATOR", "Expected 30 held-out cases per intent.");
|
||||
assert(metrics.thresholds.verifiedCoverage === 0.8, "AAS_BASELINE_METRIC_COVERAGE", "Coverage threshold must be 0.8.");
|
||||
assert(metrics.thresholds.inclusionPrecision === 0.9, "AAS_BASELINE_METRIC_PRECISION", "Precision threshold must be 0.9.");
|
||||
assert(metrics.thresholds.outOfCoverageAbstention === 1, "AAS_BASELINE_METRIC_ABSTENTION", "Abstention threshold must be 1.");
|
||||
assert(metrics.thresholds.hardPolicyViolations === 0, "AAS_BASELINE_METRIC_POLICY", "Hard-policy violations must be zero.");
|
||||
assert(metrics.canonicalComparisonExcludedFields.length === 4, "AAS_BASELINE_CANONICAL_EXCLUSIONS", "Canonical exclusions must be an explicit four-field list.");
|
||||
|
||||
const benchmark = readJson("baseline/v1/benchmark/manifest.json");
|
||||
const heldOut = readJson("baseline/v1/benchmark/held-out-index.json");
|
||||
const tuningManifest = readJson("baseline/v1/benchmark/tuning/manifest.json");
|
||||
const expectedIntents = [
|
||||
"web-application-delivery",
|
||||
"api-backend-delivery",
|
||||
"test-qa-automation",
|
||||
"security-review-hardening",
|
||||
"deployment-devops",
|
||||
"agent-mcp-development",
|
||||
];
|
||||
assert(JSON.stringify(benchmark.intents) === JSON.stringify(expectedIntents), "AAS_BASELINE_INTENTS", "The six frozen intents changed.");
|
||||
assert(benchmark.benchmarkVersion === "1.0.1", "AAS_BASELINE_BENCHMARK_VERSION", "The independently amended benchmark must be version 1.0.1.");
|
||||
assert(tuningManifest.tuningVersion === "1.0.1"
|
||||
&& tuningManifest.equivalenceAudit?.auditId === "aas-v1-tuning-gold-equivalence-1.0.1"
|
||||
&& tuningManifest.equivalenceAudit?.changedPairs === 7,
|
||||
"AAS_BASELINE_TUNING_AUDIT", "The tuning-gold equivalence amendment is missing or incomplete.");
|
||||
assert(heldOut.cases.length === 180, "AAS_BASELINE_HELDOUT_COUNT", `Expected 180 descriptors, found ${heldOut.cases.length}.`);
|
||||
assert(new Set(heldOut.cases.map((entry) => entry.caseId)).size === 180, "AAS_BASELINE_CASE_ID_DUPLICATE", "Held-out case IDs must be unique.");
|
||||
assert(new Set(heldOut.cases.map((entry) => entry.taskFamilyId)).size === 180, "AAS_BASELINE_TASK_FAMILY_DUPLICATE", "Task families must be unique.");
|
||||
for (const intent of expectedIntents) {
|
||||
const cases = heldOut.cases.filter((entry) => entry.intent === intent);
|
||||
const subIntents = heldOut.intentSubIntents[intent];
|
||||
assert(cases.length === 30, "AAS_BASELINE_INTENT_CASE_COUNT", `${intent}: ${cases.length}`);
|
||||
assert(Array.isArray(subIntents) && subIntents.length === 5, "AAS_BASELINE_SUBINTENT_COUNT", intent);
|
||||
assert(new Set(cases.map((entry) => entry.archetype)).size === 6, "AAS_BASELINE_ARCHETYPE_COUNT", intent);
|
||||
for (const subIntent of subIntents || []) {
|
||||
for (const archetype of heldOut.archetypes) {
|
||||
assert(cases.some((entry) => entry.subIntent === subIntent && entry.archetype === archetype), "AAS_BASELINE_DIVERSIFICATION_CELL", `${intent}/${subIntent}/${archetype}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(heldOut.cases.every((entry) => !("acceptedSolutions" in entry) && !("expectedStack" in entry)), "AAS_BASELINE_FABRICATED_LABEL", "Structural descriptors must not contain gold labels.");
|
||||
requireForFreeze(benchmark.labelsFrozen === true, "AAS_BASELINE_LABELS_PENDING", "Held-out labels are not frozen.");
|
||||
requireForFreeze(heldOut.cases.every((entry) => entry.inputPath && entry.goldPath && entry.provenance && entry.reviewStatus === "approved"), "AAS_BASELINE_CASES_PENDING", "Real case inputs, gold sets, provenance, or approvals are missing.");
|
||||
requireForFreeze(benchmark.abstention.labelsFrozen === true && benchmark.abstention.caseCount > 0, "AAS_BASELINE_ABSTENTION_PENDING", "The separate abstention set is not frozen.");
|
||||
requireForFreeze(benchmark.tuning.status === "frozen", "AAS_BASELINE_TUNING_PENDING", "The tuning set is not frozen separately.");
|
||||
|
||||
const budgets = readJson("baseline/v1/budgets.json");
|
||||
const sum = (strata) => strata.reduce((total, stratum) => total + stratum.executions, 0);
|
||||
assert(budgets.propertyAndGenerative.minimumExecutions === 100000, "AAS_BASELINE_PROPERTY_BUDGET", "Property/generative budget must be 100000.");
|
||||
assert(sum(budgets.propertyAndGenerative.strata) === 100000, "AAS_BASELINE_PROPERTY_DISTRIBUTION", "Property/generative strata must sum to 100000.");
|
||||
assert(budgets.parserAndMcpFuzz.minimumExecutions === 50000, "AAS_BASELINE_FUZZ_BUDGET", "Parser/MCP fuzz budget must be 50000.");
|
||||
assert(sum(budgets.parserAndMcpFuzz.strata) === 50000, "AAS_BASELINE_FUZZ_DISTRIBUTION", "Parser/MCP fuzz strata must sum to 50000.");
|
||||
assert(budgets.hardGate.budgetReductionAllowed === false, "AAS_BASELINE_BUDGET_REDUCTION", "Budget reduction must be forbidden.");
|
||||
requireForFreeze(budgets.status === "frozen" && budgets.prng.rootSeed && budgets.prng.algorithm !== "pending-independent-review", "AAS_BASELINE_SEEDS_PENDING", "Independent PRNG seed and derivation are pending.");
|
||||
|
||||
const hostile = readJson("baseline/v1/hostile/manifest.json");
|
||||
assert(hostile.classes.some((entry) => entry.surface === "archive"), "AAS_BASELINE_ARCHIVE_CORPUS", "Archive classes are missing.");
|
||||
assert(hostile.classes.some((entry) => entry.surface === "input"), "AAS_BASELINE_INPUT_CORPUS", "Input classes are missing.");
|
||||
assert(new Set(hostile.classes.map((entry) => entry.classId)).size === hostile.classes.length, "AAS_BASELINE_HOSTILE_DUPLICATE", "Hostile class IDs must be unique.");
|
||||
assert(hostile.classes.every((entry) => entry.exploit.expected === "reject" && entry.boundaryControl.expected === "accept"), "AAS_BASELINE_HOSTILE_PAIR", "Every hostile class requires reject/accept pairs.");
|
||||
requireForFreeze(hostile.status === "frozen" && hostile.classes.every((entry) => entry.status === "frozen" && entry.exploit.path && entry.exploit.sha256 && entry.boundaryControl.path && entry.boundaryControl.sha256), "AAS_BASELINE_HOSTILE_FIXTURES_PENDING", "Hostile exploit/control fixtures and hashes are pending.");
|
||||
|
||||
const runtime = readJson("baseline/v1/runtime-matrix.json");
|
||||
const expectedJobs = new Set(["linux-node-22", "linux-node-24", "macos-node-22", "macos-node-24", "windows-node-22", "windows-node-24"]);
|
||||
assert(runtime.jobs.length === 6 && runtime.jobs.every((job) => expectedJobs.has(job.id)), "AAS_BASELINE_RUNTIME_MATRIX", "Runtime matrix must contain exactly six OS/Node jobs.");
|
||||
assert(new Set(runtime.jobs.map((job) => job.id)).size === expectedJobs.size, "AAS_BASELINE_RUNTIME_MATRIX_DUPLICATE", "Runtime matrix job IDs must be unique.");
|
||||
assert([...expectedJobs].every((id) => runtime.jobs.some((job) => job.id === id)), "AAS_BASELINE_RUNTIME_MATRIX_EXHAUSTIVE", "Runtime matrix must contain every frozen job exactly once.");
|
||||
assert(new Set(runtime.jobs.map((job) => `${job.os}/${job.nodePatch}`)).size === expectedJobs.size, "AAS_BASELINE_RUNTIME_MATRIX_IDENTITY_DUPLICATE", "OS/Node identities must be unique.");
|
||||
assert(runtime.skipsAllowed === false && runtime.continueOnErrorAllowed === false, "AAS_BASELINE_MATRIX_FAILURE_POLICY", "Skips and allowed failures are forbidden.");
|
||||
requireForFreeze(runtime.status === "frozen" && runtime.jobs.every((job) => job.nodePatch && job.runnerImage && job.architecture && job.filesystem && job.observer && job.status === "frozen"), "AAS_BASELINE_RUNTIME_IDENTITIES_PENDING", "Exact Node, runner, architecture, filesystem, or observer identities are pending.");
|
||||
|
||||
const verifierManifest = readJson("baseline/v1/verifier-manifest.json");
|
||||
assert(verifierManifest.requiredSuites.length === 9 && new Set(verifierManifest.requiredSuites).size === 9, "AAS_BASELINE_PRODUCT_SUITE_SET", "The product verifier must require nine unique suites.");
|
||||
assert(verifierManifest.candidateIsolation.requiresProcessSeparation === true && verifierManifest.candidateIsolation.testHooksAllowed === false, "AAS_BASELINE_CANDIDATE_ISOLATION", "Candidate code requires an external process and forbids test hooks.");
|
||||
assert(verifierManifest.faultBoundaryClasses.length === 7 && verifierManifest.raceClasses.length === 6, "AAS_BASELINE_TRANSACTION_COVERAGE", "Fault/race class contracts changed.");
|
||||
|
||||
const legacy = readJson("baseline/v1/legacy/14.6.0/manifest.json");
|
||||
assert(legacy.baseline.version === "14.6.0", "AAS_BASELINE_LEGACY_VERSION", "Legacy baseline version changed.");
|
||||
assert(legacy.baseline.distIntegrity === "sha512-VTOb3O9PSYKCDO99i3h0vOn7vHQlGtO/+jSErR80g6OGaDJoBzg3q2GE9Nu890en1/Z54hBEYiVQj/1Rl95xEg==", "AAS_BASELINE_LEGACY_SRI", "Legacy npm SRI changed.");
|
||||
assert(legacy.baseline.sourceCommit === "ab5f6c205a548d2f4bec411728c79b9c156fc696", "AAS_BASELINE_LEGACY_COMMIT", "Legacy source commit changed.");
|
||||
const allArgs = legacy.cases.flatMap((entry) => entry.args);
|
||||
for (const flag of legacy.publicFlags) {
|
||||
assert(allArgs.includes(flag), "AAS_BASELINE_LEGACY_FLAG_COVERAGE", flag);
|
||||
}
|
||||
for (const target of legacy.targets) {
|
||||
assert(legacy.cases.some((entry) => entry.args.includes(`--${target}`)), "AAS_BASELINE_LEGACY_TARGET_COVERAGE", target);
|
||||
}
|
||||
assert(legacy.cases.some((entry) => entry.args.includes("install")), "AAS_BASELINE_LEGACY_INSTALL_COMMAND", "Literal install command is missing.");
|
||||
requireForFreeze(legacy.status === "frozen" && legacy.fixtureRepository.treeDigest && legacy.fixtureRepository.fakeGitTraceDigest && legacy.cases.every((entry) => entry.expectedSnapshot), "AAS_BASELINE_LEGACY_SNAPSHOTS_PENDING", "Legacy fixture digests or expected snapshots are pending.");
|
||||
|
||||
const ownership = readJson("ownership.v1.json");
|
||||
assert(ownership.minimumApprovals === 2 && ownership.requireNonScorerReviewer === true, "AAS_BASELINE_OWNERSHIP_POLICY", "Two approvals including a non-scorer reviewer are required.");
|
||||
const reviewerIdentities = new Set(ownership.reviewers.map((reviewer) => reviewer.identity));
|
||||
const reviewerReportsValid = ownership.reviewers.every((reviewer) => {
|
||||
if (!reviewer.report || !reviewer.reportSha256 || reviewer.reviewedPairs !== 270) return false;
|
||||
const reportPath = path.join(root, reviewer.report);
|
||||
if (!fs.existsSync(reportPath)) return false;
|
||||
const digest = crypto.createHash("sha256")
|
||||
.update(fs.readFileSync(reportPath))
|
||||
.digest("hex");
|
||||
return digest === reviewer.reportSha256;
|
||||
});
|
||||
const reviewAmendmentsValid = Array.isArray(ownership.reviewAmendments)
|
||||
&& ownership.reviewAmendments.length === 1
|
||||
&& ownership.reviewAmendments.every((amendment) => {
|
||||
if (amendment.auditId !== "aas-v1-tuning-gold-equivalence-1.0.1"
|
||||
|| amendment.claimsReviewed !== 19
|
||||
|| amendment.changedPairs !== 7
|
||||
|| amendment.minimumIndependentReviewsPerChangedPair !== 2
|
||||
|| !Array.isArray(amendment.reports)
|
||||
|| amendment.reports.length !== 4) return false;
|
||||
const auditPath = path.join(root, amendment.audit);
|
||||
if (!fs.existsSync(auditPath)) return false;
|
||||
const auditDigest = crypto.createHash("sha256").update(fs.readFileSync(auditPath)).digest("hex");
|
||||
if (auditDigest !== amendment.auditSha256) return false;
|
||||
const audit = JSON.parse(fs.readFileSync(auditPath, "utf8"));
|
||||
if (audit.auditId !== amendment.auditId || audit.scope?.changedPairCount !== amendment.changedPairs) return false;
|
||||
const identities = new Set();
|
||||
for (const reviewer of amendment.reports) {
|
||||
if (!reviewer.identity || reviewer.scorerImplementer !== false || reviewer.reviewedClaims < 1) return false;
|
||||
identities.add(reviewer.identity);
|
||||
const reportPath = path.join(root, reviewer.report);
|
||||
if (!fs.existsSync(reportPath)) return false;
|
||||
const bytes = fs.readFileSync(reportPath);
|
||||
const digest = crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
if (digest !== reviewer.reportSha256) return false;
|
||||
const report = JSON.parse(bytes.toString("utf8"));
|
||||
if (report.reviewer?.identity !== reviewer.identity
|
||||
|| report.reviewer?.scorerImplementer !== false
|
||||
|| report.decisions?.length !== reviewer.reviewedClaims) return false;
|
||||
}
|
||||
return identities.size === amendment.reports.length;
|
||||
});
|
||||
requireForFreeze(
|
||||
ownership.status === "frozen"
|
||||
&& reviewerIdentities.size >= 2
|
||||
&& ownership.reviewers.some((reviewer) => reviewer.scorerImplementer === false)
|
||||
&& reviewerReportsValid
|
||||
&& reviewAmendmentsValid
|
||||
&& ownership.repositorySettings.settingsVerified === true
|
||||
&& ownership.verificationEnvironment.requiredReviewersVerified === true
|
||||
&& ownership.verificationEnvironment.productPullRequestsMayModifyBaseline === false,
|
||||
"AAS_BASELINE_OWNERS_PENDING",
|
||||
"Named independent owners, report digests, or protected GitHub settings are pending.",
|
||||
);
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, mode, failures, pending }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (mode === "freeze-ready" && pending.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, mode, code: "AAS_BASELINE_NOT_FREEZE_READY", pending }, null, 2));
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ ok: true, mode, schemaCount: schemaFiles.length, heldOutDescriptors: heldOut.cases.length, hostileClasses: hostile.classes.length, legacyCases: legacy.cases.length, pendingCount: pending.length }, null, 2));
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const manifestPath = path.join(verificationRoot, "baseline", "v1", "freeze-manifest.json");
|
||||
const write = process.argv.includes("--write");
|
||||
const excludedPrefixes = [
|
||||
"verifier/node_modules/",
|
||||
"baseline/v1/legacy/14.6.0/_work/",
|
||||
];
|
||||
|
||||
function relativePath(target) {
|
||||
return path.relative(verificationRoot, target).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function isExcluded(target, { directory = false } = {}) {
|
||||
const relative = relativePath(target);
|
||||
const candidate = directory ? `${relative}/` : relative;
|
||||
return excludedPrefixes.some((prefix) => candidate.startsWith(prefix));
|
||||
}
|
||||
|
||||
function walk(directory) {
|
||||
return fs.readdirSync(directory, { withFileTypes: true })
|
||||
.flatMap((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (isExcluded(target, { directory: true })) return [];
|
||||
return walk(target);
|
||||
}
|
||||
if (!entry.isFile()) throw new Error(`Non-regular baseline entry: ${target}`);
|
||||
return [target];
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
function sha256(bytes) {
|
||||
return crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
const files = walk(verificationRoot)
|
||||
.filter((file) => file !== manifestPath)
|
||||
.filter((file) => !isExcluded(file))
|
||||
.map((file) => {
|
||||
const bytes = fs.readFileSync(file);
|
||||
return {
|
||||
path: relativePath(file),
|
||||
bytes: bytes.length,
|
||||
sha256: sha256(bytes),
|
||||
};
|
||||
});
|
||||
const rootDigest = sha256(Buffer.from(JSON.stringify(files)));
|
||||
const candidate = {
|
||||
schemaVersion: 1,
|
||||
baselineVersion: "1.0.1",
|
||||
status: "frozen",
|
||||
digestAlgorithm: "sha256",
|
||||
excludedPaths: ["baseline/v1/freeze-manifest.json", ...excludedPrefixes],
|
||||
rootDigest: `sha256-${rootDigest}`,
|
||||
fileCount: files.length,
|
||||
files,
|
||||
};
|
||||
|
||||
if (write) {
|
||||
fs.writeFileSync(manifestPath, `${JSON.stringify(candidate, null, 2)}\n`, { mode: 0o644 });
|
||||
console.log(JSON.stringify({ ok: true, wrote: path.relative(verificationRoot, manifestPath), fileCount: files.length, rootDigest: candidate.rootDigest }, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
console.error(JSON.stringify({ ok: false, code: "AAS_BASELINE_FREEZE_MANIFEST_MISSING" }, null, 2));
|
||||
process.exit(2);
|
||||
}
|
||||
const expected = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
||||
if (JSON.stringify(expected) !== JSON.stringify(candidate)) {
|
||||
const expectedByPath = new Map((expected.files || []).map((entry) => [entry.path, entry]));
|
||||
const actualByPath = new Map(files.map((entry) => [entry.path, entry]));
|
||||
const changed = [...new Set([...expectedByPath.keys(), ...actualByPath.keys()])]
|
||||
.filter((file) => JSON.stringify(expectedByPath.get(file)) !== JSON.stringify(actualByPath.get(file)))
|
||||
.sort();
|
||||
console.error(JSON.stringify({
|
||||
ok: false,
|
||||
code: "AAS_BASELINE_FREEZE_DIGEST_MISMATCH",
|
||||
expectedRootDigest: expected.rootDigest,
|
||||
actualRootDigest: candidate.rootDigest,
|
||||
changed,
|
||||
}, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(JSON.stringify({ ok: true, fileCount: files.length, rootDigest: candidate.rootDigest }, null, 2));
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { aggregateReceipts, writeBundle } from "../lib/aggregate.mjs";
|
||||
import { loadReceiptValidator } from "../lib/receipt.mjs";
|
||||
|
||||
const outIndex = process.argv.indexOf("--out");
|
||||
const out = outIndex >= 0 ? path.resolve(process.argv[outIndex + 1]) : null;
|
||||
const inputs = process.argv.slice(2).filter((value, index, all) => value !== "--out" && all[index - 1] !== "--out");
|
||||
if (!out || inputs.length === 0) throw new Error("Usage: merge-product-evidence --out <bundle.json> <receipt...>");
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const schema = path.resolve(here, "..", "..", "schemas", "product-verifier-receipt.schema.json");
|
||||
const validator = loadReceiptValidator(schema);
|
||||
const receipts = inputs.map((file) => JSON.parse(fs.readFileSync(file, "utf8")));
|
||||
const bundle = aggregateReceipts(receipts, validator);
|
||||
writeBundle(out, bundle);
|
||||
process.stdout.write(`${JSON.stringify({ ok: bundle.status === "passed", out, bundleDigest: bundle.bundleDigest, failures: bundle.failures })}\n`);
|
||||
if (bundle.status !== "passed") process.exit(1);
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { selfTestObserver } from "../lib/observer.mjs";
|
||||
import { isolatedZones } from "../lib/runtime.mjs";
|
||||
|
||||
const jobIndex = process.argv.indexOf("--job-id");
|
||||
const jobId = jobIndex >= 0 ? process.argv[jobIndex + 1] : "";
|
||||
const expected = jobId.startsWith("linux-") ? "linux-strace-process-tree"
|
||||
: jobId.startsWith("macos-") ? "macos-fs_usage-process"
|
||||
: jobId.startsWith("windows-") ? "windows-etw-kernel-process-tree" : null;
|
||||
if (!expected) throw new Error("--job-id must be a frozen runtime-matrix job");
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "aas-observer-self-test-"));
|
||||
try {
|
||||
const zones = isolatedZones(path.join(root, "zones"));
|
||||
const evidenceDir = path.join(root, "evidence");
|
||||
const result = await selfTestObserver({ cwd: zones.tmp, env: process.env, zones, evidenceDir });
|
||||
if (result.backend !== expected) throw new Error(`observer backend mismatch: ${result.backend}/${expected}`);
|
||||
if (result.observedNetworkSentinels < 1 || result.observedWriteSentinels < 1) throw new Error("observer sentinel was not detected");
|
||||
process.stdout.write(`${JSON.stringify({ ok: true, jobId, ...result })}\n`);
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const repositoryRoot = path.resolve(verificationRoot, "..", "..");
|
||||
const benchmarkRoot = path.join(verificationRoot, "baseline", "v1", "benchmark");
|
||||
const frozen = process.argv.includes("--require-approvals");
|
||||
const failures = [];
|
||||
|
||||
function fail(code, detail) {
|
||||
failures.push({ code, detail });
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
} catch (error) {
|
||||
fail("AAS_BENCHMARK_INVALID_JSON", `${path.relative(verificationRoot, file)}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function walkJson(directory) {
|
||||
if (!fs.existsSync(directory)) return [];
|
||||
return fs.readdirSync(directory, { withFileTypes: true })
|
||||
.flatMap((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) return walkJson(target);
|
||||
return entry.isFile() && entry.name.endsWith(".json") ? [target] : [];
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function pairDigest(caseData, goldData) {
|
||||
const reviewNeutralGold = { ...goldData, reviews: [] };
|
||||
const bytes = JSON.stringify(canonicalize({ case: caseData, gold: reviewNeutralGold }));
|
||||
return `sha256-${crypto.createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
|
||||
const index = readJson(path.join(benchmarkRoot, "held-out-index.json"));
|
||||
const catalog = readJson(path.join(repositoryRoot, "data", "skills_index.json"));
|
||||
if (!index || !catalog) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const descriptors = new Map(index.cases.map((entry) => [entry.caseId, entry]));
|
||||
const catalogById = new Map(catalog.map((entry) => [entry.id, entry]));
|
||||
const caseRoot = path.join(benchmarkRoot, "cases", "held-out");
|
||||
const goldRoot = path.join(benchmarkRoot, "gold", "held-out");
|
||||
const caseFiles = walkJson(caseRoot);
|
||||
const goldFiles = walkJson(goldRoot);
|
||||
const seenIds = new Set();
|
||||
const seenFingerprints = new Set();
|
||||
|
||||
if (caseFiles.length !== 180) fail("AAS_BENCHMARK_CASE_COUNT", `expected 180, found ${caseFiles.length}`);
|
||||
if (goldFiles.length !== 180) fail("AAS_BENCHMARK_GOLD_COUNT", `expected 180, found ${goldFiles.length}`);
|
||||
|
||||
for (const caseFile of caseFiles) {
|
||||
const relative = path.relative(caseRoot, caseFile);
|
||||
const goldFile = path.join(goldRoot, relative);
|
||||
const caseData = readJson(caseFile);
|
||||
const goldData = readJson(goldFile);
|
||||
if (!caseData || !goldData) continue;
|
||||
const descriptor = descriptors.get(caseData.caseId);
|
||||
|
||||
if (!descriptor) fail("AAS_BENCHMARK_UNKNOWN_CASE", caseData.caseId);
|
||||
if (seenIds.has(caseData.caseId)) fail("AAS_BENCHMARK_DUPLICATE_CASE", caseData.caseId);
|
||||
seenIds.add(caseData.caseId);
|
||||
if (seenFingerprints.has(caseData.taskFamilyFingerprint)) {
|
||||
fail("AAS_BENCHMARK_DUPLICATE_FINGERPRINT", caseData.taskFamilyFingerprint);
|
||||
}
|
||||
seenFingerprints.add(caseData.taskFamilyFingerprint);
|
||||
|
||||
if (!descriptor) continue;
|
||||
if (descriptor.inputPath && descriptor.inputPath !== path.relative(benchmarkRoot, caseFile)) {
|
||||
fail("AAS_BENCHMARK_PATH_MISMATCH", `${caseData.caseId}/input`);
|
||||
}
|
||||
if (descriptor.goldPath && descriptor.goldPath !== path.relative(benchmarkRoot, goldFile)) {
|
||||
fail("AAS_BENCHMARK_PATH_MISMATCH", `${caseData.caseId}/gold`);
|
||||
}
|
||||
if (caseData.intent !== descriptor.intent) fail("AAS_BENCHMARK_INTENT_MISMATCH", caseData.caseId);
|
||||
if (caseData.taskFamilyFingerprint !== descriptor.taskFamilyId) {
|
||||
fail("AAS_BENCHMARK_FINGERPRINT_MISMATCH", caseData.caseId);
|
||||
}
|
||||
if (goldData.caseId !== caseData.caseId) fail("AAS_BENCHMARK_PAIR_MISMATCH", caseData.caseId);
|
||||
if (!Array.isArray(caseData.criticalGoals) || caseData.criticalGoals.length === 0) {
|
||||
fail("AAS_BENCHMARK_CRITICAL_GOALS", caseData.caseId);
|
||||
}
|
||||
if ((caseData.minimumNonCriticalGoalCoverage ?? 0.8) < 0.8) {
|
||||
fail("AAS_BENCHMARK_NONCRITICAL_THRESHOLD", caseData.caseId);
|
||||
}
|
||||
if (caseData.requiresSkill !== true) fail("AAS_BENCHMARK_REQUIRES_SKILL", caseData.caseId);
|
||||
if (!Array.isArray(caseData.targets) || caseData.targets.length === 0) {
|
||||
fail("AAS_BENCHMARK_TARGETS", caseData.caseId);
|
||||
}
|
||||
if (!Array.isArray(caseData.policy?.allowedRisk)
|
||||
|| typeof caseData.policy?.requireKnownSource !== "boolean"
|
||||
|| typeof caseData.policy?.allowManualSetup !== "boolean") {
|
||||
fail("AAS_BENCHMARK_POLICY", caseData.caseId);
|
||||
}
|
||||
if (!caseData.provenance?.source || !caseData.provenance?.version || !caseData.provenance?.reviewedAt) {
|
||||
fail("AAS_BENCHMARK_PROVENANCE", caseData.caseId);
|
||||
}
|
||||
|
||||
for (const solution of goldData.acceptedSolutions || []) {
|
||||
const allowed = solution.allowedSkillIds || [];
|
||||
if (allowed.length === 0 || new Set(allowed).size !== allowed.length) {
|
||||
fail("AAS_BENCHMARK_ALLOWED_SKILLS", `${caseData.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
for (const skillId of allowed) {
|
||||
if (!catalogById.has(skillId)) fail("AAS_BENCHMARK_UNKNOWN_SKILL", `${caseData.caseId}/${skillId}`);
|
||||
}
|
||||
for (const group of solution.requiredGroups || []) {
|
||||
if (!Array.isArray(group) || group.length === 0 || group.some((skillId) => !allowed.includes(skillId))) {
|
||||
fail("AAS_BENCHMARK_REQUIRED_GROUP", `${caseData.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!goldData.provenance?.source || !goldData.provenance?.version || !goldData.provenance?.reviewedAt) {
|
||||
fail("AAS_BENCHMARK_GOLD_PROVENANCE", caseData.caseId);
|
||||
}
|
||||
const solutionFingerprints = new Set();
|
||||
for (const solution of goldData.acceptedSolutions || []) {
|
||||
const fingerprint = JSON.stringify(canonicalize({
|
||||
allowedSkillIds: solution.allowedSkillIds,
|
||||
requiredGroups: solution.requiredGroups,
|
||||
}));
|
||||
if (solutionFingerprints.has(fingerprint)) {
|
||||
fail("AAS_BENCHMARK_DUPLICATE_SOLUTION", `${caseData.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
solutionFingerprints.add(fingerprint);
|
||||
for (const skillId of solution.allowedSkillIds || []) {
|
||||
const skill = catalogById.get(skillId);
|
||||
if (!skill) continue;
|
||||
const explicitRisk = skill.risk;
|
||||
if (explicitRisk && explicitRisk !== "unknown" && !caseData.policy.allowedRisk.includes(explicitRisk)) {
|
||||
fail("AAS_BENCHMARK_GOLD_RISK_VIOLATION", `${caseData.caseId}/${skillId}/${explicitRisk}`);
|
||||
}
|
||||
if (skill.plugin?.setup?.type === "manual" && caseData.policy.allowManualSetup !== true) {
|
||||
fail("AAS_BENCHMARK_GOLD_SETUP_VIOLATION", `${caseData.caseId}/${skillId}`);
|
||||
}
|
||||
for (const target of caseData.targets) {
|
||||
if (skill.plugin?.targets?.[target.host] === "blocked") {
|
||||
fail("AAS_BENCHMARK_GOLD_HOST_VIOLATION", `${caseData.caseId}/${skillId}/${target.host}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const digest = pairDigest(caseData, goldData);
|
||||
const approvedReviews = (goldData.reviews || []).filter((review) => (
|
||||
review.decision === "approved" && review.reviewedDigest === digest
|
||||
));
|
||||
if (frozen) {
|
||||
const uniqueReviewers = new Set(approvedReviews.map((review) => review.reviewer));
|
||||
if (uniqueReviewers.size < 2) fail("AAS_BENCHMARK_REVIEW_COUNT", caseData.caseId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const descriptor of index.cases) {
|
||||
if (!seenIds.has(descriptor.caseId)) fail("AAS_BENCHMARK_MISSING_CASE", descriptor.caseId);
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, frozen, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
frozen,
|
||||
cases: caseFiles.length,
|
||||
gold: goldFiles.length,
|
||||
uniqueFingerprints: seenFingerprints.size,
|
||||
catalogSkills: catalogById.size,
|
||||
}, null, 2));
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import addFormats from "ajv-formats";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const baselineRoot = path.join(verificationRoot, "baseline", "v1");
|
||||
const schemasRoot = path.join(verificationRoot, "schemas");
|
||||
const failures = [];
|
||||
|
||||
function readJson(file) {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
}
|
||||
|
||||
function walkJson(directory) {
|
||||
return fs.readdirSync(directory, { withFileTypes: true })
|
||||
.flatMap((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
return entry.isDirectory() ? walkJson(target) : entry.name.endsWith(".json") ? [target] : [];
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
|
||||
addFormats(ajv);
|
||||
const compiled = new Map();
|
||||
for (const file of walkJson(schemasRoot)) {
|
||||
const schema = readJson(file);
|
||||
compiled.set(path.basename(file), ajv.compile(schema));
|
||||
}
|
||||
|
||||
function validate(schemaName, files) {
|
||||
const validator = compiled.get(schemaName);
|
||||
if (!validator) throw new Error(`Unknown schema: ${schemaName}`);
|
||||
for (const file of files) {
|
||||
const data = readJson(file);
|
||||
if (!validator(data)) {
|
||||
failures.push({
|
||||
schema: schemaName,
|
||||
file: path.relative(verificationRoot, file),
|
||||
errors: validator.errors,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const benchmarkRoot = path.join(baselineRoot, "benchmark");
|
||||
validate("benchmark-case.schema.json", [
|
||||
...walkJson(path.join(benchmarkRoot, "cases", "held-out")),
|
||||
...walkJson(path.join(benchmarkRoot, "tuning", "cases")),
|
||||
]);
|
||||
validate("benchmark-gold.schema.json", [
|
||||
...walkJson(path.join(benchmarkRoot, "gold", "held-out")),
|
||||
...walkJson(path.join(benchmarkRoot, "tuning", "gold")),
|
||||
]);
|
||||
validate("benchmark-manifest.schema.json", [path.join(benchmarkRoot, "manifest.json")]);
|
||||
validate("held-out-index.schema.json", [path.join(benchmarkRoot, "held-out-index.json")]);
|
||||
validate("budget-manifest.schema.json", [path.join(baselineRoot, "budgets.json")]);
|
||||
validate("hostile-corpus-manifest.schema.json", [path.join(baselineRoot, "hostile", "manifest.json")]);
|
||||
validate("legacy-command-corpus.schema.json", [path.join(baselineRoot, "legacy", "14.6.0", "manifest.json")]);
|
||||
validate("metric-definition.schema.json", [path.join(baselineRoot, "metrics.json")]);
|
||||
validate("runtime-matrix.schema.json", [path.join(baselineRoot, "runtime-matrix.json")]);
|
||||
validate("tuning-gold-equivalence-review.schema.json", [
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-alpha.json"),
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-beta.json"),
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-adjudicator.json"),
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-ci-tiebreak.json"),
|
||||
]);
|
||||
validate("tuning-gold-equivalence-audit.schema.json", [
|
||||
path.join(baselineRoot, "reviews", "tuning-gold-equivalence-audit.json"),
|
||||
]);
|
||||
validate("product-verifier-manifest.schema.json", [path.join(baselineRoot, "verifier-manifest.json")]);
|
||||
validate("host-adapter-fixtures.schema.json", [path.join(baselineRoot, "host-adapters", "manifest.json")]);
|
||||
|
||||
const abstentionRoot = path.join(benchmarkRoot, "abstention");
|
||||
const abstentionSchemas = path.join(abstentionRoot, "schemas");
|
||||
for (const name of ["abstention-case", "abstention-label", "abstention-index"]) {
|
||||
const schema = readJson(path.join(abstentionSchemas, `${name}.schema.json`));
|
||||
const validator = ajv.compile(schema);
|
||||
const files = name === "abstention-case"
|
||||
? walkJson(path.join(abstentionRoot, "cases"))
|
||||
: name === "abstention-label"
|
||||
? walkJson(path.join(abstentionRoot, "labels"))
|
||||
: [path.join(abstentionRoot, "index.json")];
|
||||
for (const file of files) {
|
||||
const data = readJson(file);
|
||||
if (!validator(data)) {
|
||||
failures.push({ schema: `${name}.schema.json`, file: path.relative(verificationRoot, file), errors: validator.errors });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
publicSchemas: compiled.size,
|
||||
heldOutCases: 180,
|
||||
heldOutGold: 180,
|
||||
tuningCases: 60,
|
||||
tuningGold: 60,
|
||||
abstentionCases: 30,
|
||||
abstentionLabels: 30,
|
||||
tuningEquivalenceReviews: 4,
|
||||
tuningEquivalenceAudits: 1,
|
||||
}, null, 2));
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const repositoryRoot = path.resolve(verificationRoot, "..", "..");
|
||||
const benchmarkRoot = path.join(verificationRoot, "baseline", "v1", "benchmark");
|
||||
const frozen = process.argv.includes("--require-approvals");
|
||||
const failures = [];
|
||||
const fail = (code, detail) => failures.push({ code, detail });
|
||||
const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function pairDigest(caseData, judgmentData, section) {
|
||||
const reviewNeutralJudgment = { ...judgmentData, reviews: [] };
|
||||
const payload = section === "abstention"
|
||||
? { case: caseData, label: reviewNeutralJudgment }
|
||||
: { case: caseData, gold: reviewNeutralJudgment };
|
||||
const bytes = JSON.stringify(canonicalize(payload));
|
||||
return `sha256-${crypto.createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
|
||||
function requireApprovals(caseId, caseData, judgmentData, section) {
|
||||
if (!frozen) return;
|
||||
const digest = pairDigest(caseData, judgmentData, section);
|
||||
const reviewers = new Set((judgmentData.reviews || [])
|
||||
.filter((review) => review.decision === "approved" && review.reviewedDigest === digest)
|
||||
.map((review) => review.reviewer));
|
||||
if (reviewers.size < 2) fail("AAS_SECONDARY_REVIEW_COUNT", caseId);
|
||||
}
|
||||
|
||||
function validateInputContract(caseData, expectedRequiresSkill) {
|
||||
if (!Array.isArray(caseData.targets) || caseData.targets.length === 0) {
|
||||
fail("AAS_SECONDARY_TARGETS", caseData.caseId);
|
||||
}
|
||||
if (!Array.isArray(caseData.policy?.allowedRisk)
|
||||
|| typeof caseData.policy?.requireKnownSource !== "boolean"
|
||||
|| typeof caseData.policy?.allowManualSetup !== "boolean") {
|
||||
fail("AAS_SECONDARY_POLICY", caseData.caseId);
|
||||
}
|
||||
if (caseData.requiresSkill !== expectedRequiresSkill) {
|
||||
fail("AAS_SECONDARY_REQUIRES_SKILL", caseData.caseId);
|
||||
}
|
||||
}
|
||||
|
||||
const catalog = readJson(path.join(repositoryRoot, "data", "skills_index.json"));
|
||||
const catalogById = new Map(catalog.map((entry) => [entry.id, entry]));
|
||||
const heldOutIndex = readJson(path.join(benchmarkRoot, "held-out-index.json"));
|
||||
const heldOutFingerprints = new Set(heldOutIndex.cases.map((entry) => entry.taskFamilyId));
|
||||
const supportedIntents = new Set(Object.keys(heldOutIndex.intentSubIntents));
|
||||
|
||||
const abstentionRoot = path.join(benchmarkRoot, "abstention");
|
||||
const abstentionIndex = readJson(path.join(abstentionRoot, "index.json"));
|
||||
const reasonRegistry = readJson(path.join(abstentionRoot, abstentionIndex.reasonCodeRegistry));
|
||||
const reasonCodes = new Set(reasonRegistry.codes.map((entry) => entry.code));
|
||||
const abstentionFingerprints = new Set();
|
||||
if (abstentionIndex.cases.length !== 30) fail("AAS_ABSTENTION_CASE_COUNT", abstentionIndex.cases.length);
|
||||
for (const entry of abstentionIndex.cases) {
|
||||
const caseData = readJson(path.join(abstentionRoot, entry.inputPath));
|
||||
const labelData = readJson(path.join(abstentionRoot, entry.labelPath));
|
||||
if (caseData.caseId !== entry.caseId || labelData.caseId !== entry.caseId) {
|
||||
fail("AAS_ABSTENTION_PAIR", entry.caseId);
|
||||
}
|
||||
validateInputContract(caseData, false);
|
||||
if (supportedIntents.has(caseData.intent)) fail("AAS_ABSTENTION_IN_SCOPE_INTENT", entry.caseId);
|
||||
if (labelData.expectedStatus !== "insufficientCoverage" || labelData.expectedProposedStack?.length !== 0) {
|
||||
fail("AAS_ABSTENTION_LABEL", entry.caseId);
|
||||
}
|
||||
if (!labelData.reasonCodes?.length || labelData.reasonCodes.some((code) => !reasonCodes.has(code))) {
|
||||
fail("AAS_ABSTENTION_REASON_CODE", entry.caseId);
|
||||
}
|
||||
if (caseData.taskFamilyFingerprint !== entry.taskFamilyFingerprint) {
|
||||
fail("AAS_ABSTENTION_FINGERPRINT", entry.caseId);
|
||||
}
|
||||
if (abstentionFingerprints.has(entry.taskFamilyFingerprint) || heldOutFingerprints.has(entry.taskFamilyFingerprint)) {
|
||||
fail("AAS_ABSTENTION_FINGERPRINT_COLLISION", entry.caseId);
|
||||
}
|
||||
abstentionFingerprints.add(entry.taskFamilyFingerprint);
|
||||
requireApprovals(entry.caseId, caseData, labelData, "abstention");
|
||||
}
|
||||
|
||||
const tuningRoot = path.join(benchmarkRoot, "tuning");
|
||||
const tuningManifest = readJson(path.join(tuningRoot, "manifest.json"));
|
||||
const tuningIndex = readJson(path.join(tuningRoot, tuningManifest.index));
|
||||
const tuningFingerprints = new Set();
|
||||
if (tuningIndex.cases.length !== 60) fail("AAS_TUNING_CASE_COUNT", tuningIndex.cases.length);
|
||||
for (const intent of supportedIntents) {
|
||||
const count = tuningIndex.cases.filter((entry) => entry.intent === intent).length;
|
||||
if (count !== 10) fail("AAS_TUNING_INTENT_COUNT", `${intent}/${count}`);
|
||||
}
|
||||
for (const entry of tuningIndex.cases) {
|
||||
const caseData = readJson(path.join(tuningRoot, entry.inputPath));
|
||||
const goldData = readJson(path.join(tuningRoot, entry.goldPath));
|
||||
if (caseData.caseId !== entry.caseId || goldData.caseId !== entry.caseId) {
|
||||
fail("AAS_TUNING_PAIR", entry.caseId);
|
||||
}
|
||||
validateInputContract(caseData, true);
|
||||
if (caseData.taskFamilyFingerprint !== entry.taskFamilyFingerprint) {
|
||||
fail("AAS_TUNING_FINGERPRINT", entry.caseId);
|
||||
}
|
||||
if (tuningFingerprints.has(entry.taskFamilyFingerprint)
|
||||
|| heldOutFingerprints.has(entry.taskFamilyFingerprint)
|
||||
|| abstentionFingerprints.has(entry.taskFamilyFingerprint)) {
|
||||
fail("AAS_TUNING_FINGERPRINT_COLLISION", entry.caseId);
|
||||
}
|
||||
tuningFingerprints.add(entry.taskFamilyFingerprint);
|
||||
const solutionFingerprints = new Set();
|
||||
for (const solution of goldData.acceptedSolutions || []) {
|
||||
const fingerprint = JSON.stringify(canonicalize({
|
||||
allowedSkillIds: solution.allowedSkillIds,
|
||||
requiredGroups: solution.requiredGroups,
|
||||
}));
|
||||
if (solutionFingerprints.has(fingerprint)) fail("AAS_TUNING_DUPLICATE_SOLUTION", `${entry.caseId}/${solution.solutionId}`);
|
||||
solutionFingerprints.add(fingerprint);
|
||||
const allowed = solution.allowedSkillIds || [];
|
||||
if (allowed.length === 0) fail("AAS_TUNING_EMPTY_SOLUTION", entry.caseId);
|
||||
for (const group of solution.requiredGroups || []) {
|
||||
if (!group.length || group.some((skillId) => !allowed.includes(skillId))) {
|
||||
fail("AAS_TUNING_REQUIRED_GROUP", `${entry.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
}
|
||||
for (const skillId of allowed) {
|
||||
const skill = catalogById.get(skillId);
|
||||
if (!skill) {
|
||||
fail("AAS_TUNING_UNKNOWN_SKILL", `${entry.caseId}/${skillId}`);
|
||||
continue;
|
||||
}
|
||||
if (skill.risk && skill.risk !== "unknown" && !caseData.policy.allowedRisk.includes(skill.risk)) {
|
||||
fail("AAS_TUNING_RISK_VIOLATION", `${entry.caseId}/${skillId}/${skill.risk}`);
|
||||
}
|
||||
if (skill.plugin?.setup?.type === "manual" && caseData.policy.allowManualSetup !== true) {
|
||||
fail("AAS_TUNING_SETUP_VIOLATION", `${entry.caseId}/${skillId}`);
|
||||
}
|
||||
for (const target of caseData.targets) {
|
||||
if (skill.plugin?.targets?.[target.host] === "blocked") {
|
||||
fail("AAS_TUNING_HOST_VIOLATION", `${entry.caseId}/${skillId}/${target.host}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
requireApprovals(entry.caseId, caseData, goldData, "tuning");
|
||||
}
|
||||
|
||||
if (failures.length) {
|
||||
console.error(JSON.stringify({ ok: false, frozen, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
frozen,
|
||||
abstentionCases: abstentionIndex.cases.length,
|
||||
tuningCases: tuningIndex.cases.length,
|
||||
disjointFingerprints: heldOutFingerprints.size + abstentionFingerprints.size + tuningFingerprints.size,
|
||||
}, null, 2));
|
||||
+381
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { caseInclusionAssessment, macroAverage } from "../lib/metrics.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const repositoryRoot = path.resolve(verificationRoot, "..", "..");
|
||||
const baselineRoot = path.join(verificationRoot, "baseline", "v1");
|
||||
const tuningRoot = path.join(baselineRoot, "benchmark", "tuning");
|
||||
const reviewsRoot = path.join(baselineRoot, "reviews");
|
||||
const auditPath = path.join(reviewsRoot, "tuning-gold-equivalence-audit.json");
|
||||
const failures = [];
|
||||
|
||||
function fail(code, detail) {
|
||||
failures.push({ code, detail });
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
} catch (error) {
|
||||
fail("AAS_TUNING_AUDIT_INVALID_JSON", `${path.relative(verificationRoot, file)}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function canonicalJson(value) {
|
||||
return JSON.stringify(canonicalize(value));
|
||||
}
|
||||
|
||||
function sha256(bytes) {
|
||||
return crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
function sha256File(file) {
|
||||
return sha256(fs.readFileSync(file));
|
||||
}
|
||||
|
||||
function pairDigest(caseData, goldData) {
|
||||
const reviewNeutralGold = { ...goldData, reviews: [] };
|
||||
return `sha256-${sha256(Buffer.from(canonicalJson({ case: caseData, gold: reviewNeutralGold })))}`;
|
||||
}
|
||||
|
||||
function decisionKey(caseId, skillId) {
|
||||
return `${caseId}\u0000${skillId}`;
|
||||
}
|
||||
|
||||
function decisionMap(report) {
|
||||
return new Map((report.decisions || []).map((entry) => [decisionKey(entry.caseId, entry.skillId), entry]));
|
||||
}
|
||||
|
||||
function mean(values) {
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
const audit = readJson(auditPath);
|
||||
if (!audit) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const diagnosticPath = path.join(verificationRoot, audit.diagnostic.path);
|
||||
if (!fs.existsSync(diagnosticPath)) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_MISSING", audit.diagnostic.path);
|
||||
}
|
||||
const diagnostic = fs.existsSync(diagnosticPath) ? readJson(diagnosticPath) : null;
|
||||
if (diagnostic && sha256File(diagnosticPath) !== audit.diagnostic.sha256) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_DIGEST", audit.diagnostic.path);
|
||||
}
|
||||
if (diagnostic) {
|
||||
if (diagnostic.reportType !== "aas-v1-tuning-diagnostic"
|
||||
|| diagnostic.benchmark?.tuningOnly !== true
|
||||
|| diagnostic.benchmark?.caseCount !== 60
|
||||
|| diagnostic.caseReports?.length !== 60
|
||||
|| diagnostic.caseReports.some((entry) => !entry.caseId?.startsWith("tuning."))) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_SCOPE", "The frozen input must contain exactly 60 tuning-only results.");
|
||||
}
|
||||
if ((diagnostic.benchmark?.roots || []).some((root) => !root.includes("/tuning/"))) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_ROOT", diagnostic.benchmark?.roots);
|
||||
}
|
||||
if (diagnostic.catalog?.digest !== audit.diagnostic.catalogDigest) {
|
||||
fail("AAS_TUNING_AUDIT_CATALOG_DIGEST", diagnostic.catalog?.digest);
|
||||
}
|
||||
}
|
||||
|
||||
const publicIndexPath = path.join(repositoryRoot, "data", "skills_index.json");
|
||||
if (sha256File(publicIndexPath) !== audit.baseline.publicSkillsIndexSha256) {
|
||||
fail("AAS_TUNING_AUDIT_PUBLIC_INDEX_DIGEST", "data/skills_index.json");
|
||||
}
|
||||
|
||||
const reports = new Map();
|
||||
for (const descriptor of audit.reviewReports || []) {
|
||||
const reportPath = path.join(verificationRoot, descriptor.path);
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_MISSING", descriptor.path);
|
||||
continue;
|
||||
}
|
||||
if (sha256File(reportPath) !== descriptor.sha256) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_DIGEST", descriptor.path);
|
||||
}
|
||||
const report = readJson(reportPath);
|
||||
if (!report) continue;
|
||||
if (report.reviewer?.identity !== descriptor.reviewer
|
||||
|| report.reviewer?.scorerImplementer !== false
|
||||
|| report.scope?.tuningOnly !== true
|
||||
|| report.scope?.heldOutContentsRead !== false
|
||||
|| report.scope?.abstentionLabelsRead !== false
|
||||
|| report.scope?.diagnosticSha256 !== audit.diagnostic.sha256
|
||||
|| report.scope?.assignedClaimCount !== report.decisions?.length
|
||||
|| descriptor.claimCount !== report.decisions?.length) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_SCOPE", descriptor.reviewer);
|
||||
}
|
||||
const counted = {
|
||||
ADD_TO_ALLOWED_EQUIVALENT: 0,
|
||||
REJECT_NOT_EQUIVALENT: 0,
|
||||
AMBIGUOUS_NEEDS_ADJUDICATION: 0,
|
||||
};
|
||||
const keys = new Set();
|
||||
for (const decision of report.decisions || []) {
|
||||
counted[decision.decision] += 1;
|
||||
const key = decisionKey(decision.caseId, decision.skillId);
|
||||
if (keys.has(key)) fail("AAS_TUNING_AUDIT_REVIEW_DUPLICATE_DECISION", `${descriptor.reviewer}/${key}`);
|
||||
keys.add(key);
|
||||
if (!decision.caseId.startsWith("tuning.")) fail("AAS_TUNING_AUDIT_REVIEW_NON_TUNING", key);
|
||||
if (decision.decision === "ADD_TO_ALLOWED_EQUIVALENT" && !decision.coherentSolution) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_ADD_WITHOUT_SOLUTION", key);
|
||||
}
|
||||
}
|
||||
if (canonicalJson(counted) !== canonicalJson(report.decisionCounts)) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_COUNTS", descriptor.reviewer);
|
||||
}
|
||||
for (const evidence of report.skillEvidence || []) {
|
||||
const evidencePath = path.resolve(repositoryRoot, evidence.path);
|
||||
const relative = path.relative(path.join(repositoryRoot, "skills"), evidencePath);
|
||||
if (relative.startsWith("..") || path.isAbsolute(relative) || !fs.existsSync(evidencePath)) {
|
||||
fail("AAS_TUNING_AUDIT_SKILL_EVIDENCE_PATH", evidence.path);
|
||||
continue;
|
||||
}
|
||||
if (sha256File(evidencePath) !== evidence.sha256) {
|
||||
fail("AAS_TUNING_AUDIT_SKILL_EVIDENCE_DIGEST", evidence.path);
|
||||
}
|
||||
}
|
||||
reports.set(descriptor.reviewer, report);
|
||||
}
|
||||
|
||||
const alpha = reports.get("gold-equivalence-independent-alpha");
|
||||
const beta = reports.get("gold-equivalence-independent-beta");
|
||||
const adjudicator = reports.get("gold-equivalence-independent-adjudicator");
|
||||
const tiebreak = reports.get("gold-equivalence-independent-ci-tiebreak");
|
||||
const alphaKeys = new Set((alpha?.decisions || []).map((entry) => decisionKey(entry.caseId, entry.skillId)));
|
||||
const betaKeys = new Set((beta?.decisions || []).map((entry) => decisionKey(entry.caseId, entry.skillId)));
|
||||
if ([...alphaKeys].some((key) => betaKeys.has(key))) {
|
||||
fail("AAS_TUNING_AUDIT_PRIMARY_ASSIGNMENTS_OVERLAP", "Primary assignments must be disjoint.");
|
||||
}
|
||||
|
||||
const claimMap = new Map();
|
||||
for (const claim of audit.claims || []) {
|
||||
const key = decisionKey(claim.caseId, claim.skillId);
|
||||
if (claimMap.has(key)) fail("AAS_TUNING_AUDIT_DUPLICATE_CLAIM", key);
|
||||
claimMap.set(key, claim);
|
||||
}
|
||||
if (claimMap.size !== 19 || audit.claims?.filter((entry) => entry.decision === "ADD_TO_ALLOWED_EQUIVALENT").length !== 8) {
|
||||
fail("AAS_TUNING_AUDIT_CLAIM_COUNTS", { total: claimMap.size });
|
||||
}
|
||||
const primaryUnion = new Set([...alphaKeys, ...betaKeys]);
|
||||
if (canonicalJson([...primaryUnion].sort()) !== canonicalJson([...claimMap.keys()].sort())) {
|
||||
fail("AAS_TUNING_AUDIT_PRIMARY_COVERAGE", "The disjoint primary assignments must cover every omission claim exactly once.");
|
||||
}
|
||||
const adjudicatorMap = adjudicator ? decisionMap(adjudicator) : new Map();
|
||||
const tiebreakMap = tiebreak ? decisionMap(tiebreak) : new Map();
|
||||
const primaryMap = new Map([
|
||||
...[...(alpha ? decisionMap(alpha) : new Map())],
|
||||
...[...(beta ? decisionMap(beta) : new Map())],
|
||||
]);
|
||||
for (const [key, claim] of claimMap) {
|
||||
const primary = primaryMap.get(key);
|
||||
const adjudicated = adjudicatorMap.get(key);
|
||||
if (!primary || !adjudicated) {
|
||||
fail("AAS_TUNING_AUDIT_CLAIM_REVIEW_MISSING", key);
|
||||
continue;
|
||||
}
|
||||
if (claim.resolution === "independent-agreement" || claim.resolution === "independent-agreement-on-conservative-exact-gold") {
|
||||
if (primary.decision !== claim.decision || adjudicated.decision !== claim.decision) {
|
||||
fail("AAS_TUNING_AUDIT_AGREEMENT_MISMATCH", key);
|
||||
}
|
||||
} else if (claim.resolution === "ambiguity-adjudicated-reject") {
|
||||
if (primary.decision !== "AMBIGUOUS_NEEDS_ADJUDICATION"
|
||||
|| adjudicated.decision !== "REJECT_NOT_EQUIVALENT"
|
||||
|| claim.decision !== "REJECT_NOT_EQUIVALENT") {
|
||||
fail("AAS_TUNING_AUDIT_AMBIGUITY_RESOLUTION", key);
|
||||
}
|
||||
} else if (claim.resolution === "independent-tiebreak-with-dissent-retained") {
|
||||
const tieDecision = tiebreakMap.get(key);
|
||||
if (primary.decision !== "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
|| adjudicated.decision !== "REJECT_NOT_EQUIVALENT"
|
||||
|| tieDecision?.decision !== "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
|| claim.decision !== "ADD_TO_ALLOWED_EQUIVALENT") {
|
||||
fail("AAS_TUNING_AUDIT_TIEBREAK_RESOLUTION", key);
|
||||
}
|
||||
} else {
|
||||
fail("AAS_TUNING_AUDIT_UNKNOWN_RESOLUTION", `${key}/${claim.resolution}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (diagnostic) {
|
||||
const diagnosticClaims = new Set();
|
||||
let omissionCases = 0;
|
||||
for (const report of diagnostic.caseReports) {
|
||||
if (!(report.inclusionCount > 0 && report.inclusionPrecision < 1)) continue;
|
||||
omissionCases += 1;
|
||||
const claims = (audit.claims || []).filter((entry) => entry.caseId === report.caseId);
|
||||
if (claims.length !== report.inclusionCount - report.acceptedInclusionCount
|
||||
|| claims.some((entry) => !report.includedSkillIds.includes(entry.skillId))) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_CLAIM_MAPPING", report.caseId);
|
||||
}
|
||||
for (const claim of claims) diagnosticClaims.add(decisionKey(claim.caseId, claim.skillId));
|
||||
}
|
||||
if (omissionCases !== 17 || diagnosticClaims.size !== 19) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_OMISSION_COUNTS", { omissionCases, claims: diagnosticClaims.size });
|
||||
}
|
||||
}
|
||||
|
||||
const tuningManifest = readJson(path.join(tuningRoot, "manifest.json"));
|
||||
const tuningIndex = tuningManifest ? readJson(path.join(tuningRoot, tuningManifest.index)) : null;
|
||||
const indexById = new Map((tuningIndex?.cases || []).map((entry) => [entry.caseId, entry]));
|
||||
const changedCaseIds = new Set();
|
||||
for (const changed of audit.changedPairs || []) {
|
||||
if (changedCaseIds.has(changed.caseId)) fail("AAS_TUNING_AUDIT_DUPLICATE_CHANGED_PAIR", changed.caseId);
|
||||
changedCaseIds.add(changed.caseId);
|
||||
const descriptor = indexById.get(changed.caseId);
|
||||
if (!descriptor) {
|
||||
fail("AAS_TUNING_AUDIT_CHANGED_PAIR_UNKNOWN", changed.caseId);
|
||||
continue;
|
||||
}
|
||||
const caseData = readJson(path.join(tuningRoot, descriptor.inputPath));
|
||||
const goldData = readJson(path.join(tuningRoot, descriptor.goldPath));
|
||||
if (!caseData || !goldData) continue;
|
||||
const digest = pairDigest(caseData, goldData);
|
||||
if (digest !== changed.newPairDigest) fail("AAS_TUNING_AUDIT_CHANGED_PAIR_DIGEST", changed.caseId);
|
||||
if (changed.priorPairDigest === changed.newPairDigest) fail("AAS_TUNING_AUDIT_CHANGED_PAIR_NO_CHANGE", changed.caseId);
|
||||
if (goldData.provenance?.source !== audit.provenance.source
|
||||
|| goldData.provenance?.version !== audit.provenance.version
|
||||
|| goldData.provenance?.reviewedAt !== audit.provenance.reviewedAt) {
|
||||
fail("AAS_TUNING_AUDIT_GOLD_PROVENANCE", changed.caseId);
|
||||
}
|
||||
const solution = (goldData.acceptedSolutions || []).find((entry) => entry.solutionId === changed.solutionId);
|
||||
if (!solution) fail("AAS_TUNING_AUDIT_SOLUTION_MISSING", `${changed.caseId}/${changed.solutionId}`);
|
||||
const approved = new Set((goldData.reviews || [])
|
||||
.filter((entry) => entry.decision === "approved" && entry.reviewedDigest === digest)
|
||||
.map((entry) => entry.reviewer));
|
||||
if (canonicalJson([...approved].sort()) !== canonicalJson([...changed.approvingReviewers].sort())) {
|
||||
fail("AAS_TUNING_AUDIT_PAIR_APPROVALS", changed.caseId);
|
||||
}
|
||||
const addedClaims = (audit.claims || []).filter((entry) => (
|
||||
entry.caseId === changed.caseId && entry.decision === "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
));
|
||||
if (!solution || addedClaims.some((entry) => !solution.allowedSkillIds.includes(entry.skillId))) {
|
||||
fail("AAS_TUNING_AUDIT_SOLUTION_CLAIMS", changed.caseId);
|
||||
}
|
||||
for (const reviewer of changed.approvingReviewers) {
|
||||
const report = reports.get(reviewer);
|
||||
for (const claim of addedClaims) {
|
||||
const decision = decisionMap(report || { decisions: [] }).get(decisionKey(claim.caseId, claim.skillId));
|
||||
if (decision?.decision !== "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
|| canonicalJson(decision.coherentSolution) !== canonicalJson(solution)) {
|
||||
fail("AAS_TUNING_AUDIT_EXACT_GOLD_ATTESTATION", `${changed.caseId}/${reviewer}/${claim.skillId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (changedCaseIds.size !== 7) fail("AAS_TUNING_AUDIT_CHANGED_PAIR_COUNT", changedCaseIds.size);
|
||||
for (const descriptor of tuningIndex?.cases || []) {
|
||||
const gold = readJson(path.join(tuningRoot, descriptor.goldPath));
|
||||
if (gold?.provenance?.source === audit.provenance.source && !changedCaseIds.has(descriptor.caseId)) {
|
||||
fail("AAS_TUNING_AUDIT_UNDECLARED_GOLD_CHANGE", descriptor.caseId);
|
||||
}
|
||||
}
|
||||
|
||||
if (diagnostic && tuningIndex) {
|
||||
const reportsAfter = diagnostic.caseReports.map((report) => {
|
||||
const descriptor = indexById.get(report.caseId);
|
||||
const gold = descriptor ? readJson(path.join(tuningRoot, descriptor.goldPath)) : null;
|
||||
if (!gold) return { ...report, acceptedInclusionCount: 0, inclusionPrecision: null };
|
||||
const inclusion = caseInclusionAssessment(report.includedSkillIds, gold.acceptedSolutions);
|
||||
return {
|
||||
...report,
|
||||
acceptedInclusionCount: inclusion.acceptedCount,
|
||||
inclusionPrecision: inclusion.precision,
|
||||
matchedSolutionId: inclusion.matchedSolutionId,
|
||||
};
|
||||
});
|
||||
const intents = [...new Set(reportsAfter.map((entry) => entry.intent))].sort();
|
||||
const perIntent = intents.map((intent) => {
|
||||
const entries = reportsAfter.filter((entry) => entry.intent === intent);
|
||||
const acceptedInclusions = entries.reduce((sum, entry) => sum + entry.acceptedInclusionCount, 0);
|
||||
const totalInclusions = entries.reduce((sum, entry) => sum + entry.inclusionCount, 0);
|
||||
return {
|
||||
intent,
|
||||
verifiedCoverage: entries.filter((entry) => entry.verified).length / entries.length,
|
||||
inclusionPrecision: acceptedInclusions / totalInclusions,
|
||||
criticalGoalCoverage: mean(entries.map((entry) => entry.criticalGoalCoverage)),
|
||||
nonCriticalGoalCoverage: mean(entries.map((entry) => entry.nonCriticalGoalCoverage)),
|
||||
acceptedInclusions,
|
||||
totalInclusions,
|
||||
hardPolicyViolations: entries.reduce((sum, entry) => sum + entry.hardPolicyViolationCount, 0),
|
||||
};
|
||||
});
|
||||
const afterMacro = {
|
||||
verifiedCoverage: macroAverage(perIntent, "verifiedCoverage"),
|
||||
inclusionPrecision: macroAverage(perIntent, "inclusionPrecision"),
|
||||
criticalGoalCoverage: macroAverage(perIntent, "criticalGoalCoverage"),
|
||||
nonCriticalGoalCoverage: macroAverage(perIntent, "nonCriticalGoalCoverage"),
|
||||
};
|
||||
const afterPerIntentPrecision = Object.fromEntries(perIntent.map((entry) => [entry.intent, entry.inclusionPrecision]));
|
||||
const beforePerIntentPrecision = Object.fromEntries(diagnostic.perIntent.map((entry) => [entry.intent, entry.inclusionPrecision]));
|
||||
if (canonicalJson(audit.metricImplications.before.macro) !== canonicalJson(diagnostic.macro)
|
||||
|| canonicalJson(audit.metricImplications.before.perIntentInclusionPrecision) !== canonicalJson(beforePerIntentPrecision)
|
||||
|| canonicalJson(audit.metricImplications.after.macro) !== canonicalJson(afterMacro)
|
||||
|| canonicalJson(audit.metricImplications.after.perIntentInclusionPrecision) !== canonicalJson(afterPerIntentPrecision)) {
|
||||
fail("AAS_TUNING_AUDIT_METRIC_IMPLICATIONS", { afterMacro, afterPerIntentPrecision });
|
||||
}
|
||||
const acceptedBefore = diagnostic.perIntent.reduce((sum, entry) => sum + entry.acceptedInclusions, 0);
|
||||
const acceptedAfter = perIntent.reduce((sum, entry) => sum + entry.acceptedInclusions, 0);
|
||||
const expectedDelta = {
|
||||
macroInclusionPrecision: afterMacro.inclusionPrecision - diagnostic.macro.inclusionPrecision,
|
||||
verifiedCoverage: afterMacro.verifiedCoverage - diagnostic.macro.verifiedCoverage,
|
||||
criticalGoalCoverage: afterMacro.criticalGoalCoverage - diagnostic.macro.criticalGoalCoverage,
|
||||
nonCriticalGoalCoverage: afterMacro.nonCriticalGoalCoverage - diagnostic.macro.nonCriticalGoalCoverage,
|
||||
acceptedInclusions: acceptedAfter - acceptedBefore,
|
||||
};
|
||||
if (canonicalJson(expectedDelta) !== canonicalJson(audit.metricImplications.delta)) {
|
||||
fail("AAS_TUNING_AUDIT_METRIC_DELTA", expectedDelta);
|
||||
}
|
||||
const hardPolicyViolations = perIntent.reduce((sum, entry) => sum + entry.hardPolicyViolations, 0);
|
||||
const postAuditGates = {
|
||||
hardPolicyViolations: hardPolicyViolations === 0,
|
||||
inclusionPrecision: afterMacro.inclusionPrecision >= diagnostic.thresholds.inclusionPrecision
|
||||
&& perIntent.every((entry) => entry.inclusionPrecision >= diagnostic.thresholds.inclusionPrecision),
|
||||
verifiedCoverage: afterMacro.verifiedCoverage >= diagnostic.thresholds.verifiedCoverage
|
||||
&& perIntent.every((entry) => entry.verifiedCoverage >= diagnostic.thresholds.verifiedCoverage),
|
||||
};
|
||||
if (canonicalJson(postAuditGates) !== canonicalJson(audit.metricImplications.postAuditGates)) {
|
||||
fail("AAS_TUNING_AUDIT_POST_GATES", postAuditGates);
|
||||
}
|
||||
const changedPrecisionCases = reportsAfter
|
||||
.filter((entry, index) => entry.inclusionPrecision !== diagnostic.caseReports[index].inclusionPrecision)
|
||||
.map((entry) => entry.caseId)
|
||||
.sort();
|
||||
if (canonicalJson(changedPrecisionCases) !== canonicalJson([...changedCaseIds].sort())) {
|
||||
fail("AAS_TUNING_AUDIT_METRIC_CHANGE_SCOPE", changedPrecisionCases);
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
auditId: audit.auditId,
|
||||
omissionCases: audit.scope.omissionCaseCount,
|
||||
claims: audit.scope.omittedInclusionCount,
|
||||
changedPairs: audit.scope.changedPairCount,
|
||||
independentReviewers: reports.size,
|
||||
macroInclusionPrecisionBefore: audit.metricImplications.before.macro.inclusionPrecision,
|
||||
macroInclusionPrecisionAfter: audit.metricImplications.after.macro.inclusionPrecision,
|
||||
postAuditGates: audit.metricImplications.postAuditGates,
|
||||
}, null, 2));
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { digestJson, sha256 } from "../lib/canonical.mjs";
|
||||
import { snapshotZones } from "../lib/fs-evidence.mjs";
|
||||
import { selfTestObserver } from "../lib/observer.mjs";
|
||||
import { executableDigest, loadReceiptValidator, SUITE_IDS, writeCanonicalReceipt } from "../lib/receipt.mjs";
|
||||
import { installCandidate, isolatedZones, systemIdentity } from "../lib/runtime.mjs";
|
||||
import {
|
||||
packageSuite, prepareRuntimeCache, suite, verifyAdapters, verifyEntrypoints,
|
||||
verifyFuzz, verifyHostile, verifyLegacy, verifyMcp, verifyProperty,
|
||||
} from "../lib/suites.mjs";
|
||||
import { inspectPackageTarball } from "../lib/tarball.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verifierRoot = path.resolve(here, "..");
|
||||
const verificationRoot = path.resolve(verifierRoot, "..");
|
||||
const baselineRoot = path.join(verificationRoot, "baseline", "v1");
|
||||
|
||||
function args(values) {
|
||||
const out = {};
|
||||
for (let i = 2; i < values.length; i += 2) {
|
||||
if (!values[i].startsWith("--") || values[i + 1] === undefined) throw new Error(`Invalid argument: ${values[i]}`);
|
||||
out[values[i].slice(2)] = values[i + 1];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function required(value, name, pattern = /./) {
|
||||
if (!value || !pattern.test(value)) throw new Error(`--${name} is required or invalid`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function readJson(file) { return JSON.parse(fs.readFileSync(file, "utf8")); }
|
||||
function failedSuite(id, error) {
|
||||
const evidence = { code: error.code || "AAS_VERIFIER_INTERNAL", detailDigest: digestJson({ name: error.name, message: error.message }) };
|
||||
return { id, status: "failed", executions: 0, failures: 1, evidenceSha256: digestJson(evidence), evidence };
|
||||
}
|
||||
|
||||
function transactionSuite(file, manifest, validator) {
|
||||
if (!file || !fs.existsSync(file)) throw Object.assign(new Error("OS-level transaction evidence is required"), { code: "AAS_VERIFIER_TRANSACTION_EVIDENCE_MISSING" });
|
||||
const value = readJson(file);
|
||||
if (!validator(value)) throw Object.assign(new Error("Transaction evidence schema failed"), { code: "AAS_VERIFIER_TRANSACTION_EVIDENCE_SCHEMA" });
|
||||
const fault = new Set(value.faultBoundaryClasses || []);
|
||||
const race = new Set(value.raceClasses || []);
|
||||
for (const item of manifest.faultBoundaryClasses) if (!fault.has(item)) throw Object.assign(new Error(`Fault boundary not covered: ${item}`), { code: "AAS_VERIFIER_FAULT_BOUNDARY_MISSING" });
|
||||
for (const item of manifest.raceClasses) if (!race.has(item)) throw Object.assign(new Error(`Race class not covered: ${item}`), { code: "AAS_VERIFIER_RACE_CLASS_MISSING" });
|
||||
if (value.testMode === true || value.mocked === true || value.productionBinary !== true) throw Object.assign(new Error("Transaction evidence is not production black-box evidence"), { code: "AAS_VERIFIER_TRANSACTION_NOT_BLACK_BOX" });
|
||||
if (value.partialStates !== 0 || value.unmanagedMutations !== 0 || value.hardPolicyViolations !== 0) throw Object.assign(new Error("Transaction safety invariant failed"), { code: "AAS_VERIFIER_TRANSACTION_INVARIANT" });
|
||||
return suite("transaction", value, value.executions || 0);
|
||||
}
|
||||
|
||||
const options = args(process.argv);
|
||||
const tarball = path.resolve(required(options.tarball, "tarball"));
|
||||
const candidateCommit = required(options["candidate-commit"], "candidate-commit", /^[a-f0-9]{40}$/);
|
||||
const verifierCommit = required(options["verifier-commit"], "verifier-commit", /^[a-f0-9]{40}$/);
|
||||
const jobId = required(options["job-id"], "job-id");
|
||||
const output = path.resolve(required(options.out, "out"));
|
||||
const workRoot = path.resolve(required(options["work-root"], "work-root"));
|
||||
const runtimeMatrix = readJson(path.join(baselineRoot, "runtime-matrix.json"));
|
||||
const job = runtimeMatrix.jobs.find((entry) => entry.id === jobId);
|
||||
if (!job) throw new Error(`Unknown job: ${jobId}`);
|
||||
if (process.platform !== job.os.replace("macos", "darwin").replace("windows", "win32")) throw new Error(`Job/platform mismatch: ${jobId}/${process.platform}`);
|
||||
if (process.arch !== job.architecture || process.version !== `v${job.nodePatch}`) throw new Error(`Frozen runtime identity mismatch: ${process.arch}/${process.version}`);
|
||||
|
||||
fs.mkdirSync(workRoot, { recursive: true, mode: 0o700 });
|
||||
const evidenceDir = path.join(workRoot, "observer-evidence");
|
||||
const zones = isolatedZones(path.join(workRoot, "zones"));
|
||||
const zoneBefore = snapshotZones(zones);
|
||||
const manifest = readJson(path.join(baselineRoot, "verifier-manifest.json"));
|
||||
const freeze = readJson(path.join(baselineRoot, "freeze-manifest.json"));
|
||||
const budgets = readJson(path.join(baselineRoot, "budgets.json"));
|
||||
const hostileManifest = readJson(path.join(baselineRoot, "hostile", "manifest.json"));
|
||||
const transactionValidator = loadReceiptValidator(path.join(verificationRoot, "schemas", "product-transaction-evidence.schema.json"));
|
||||
const inspection = inspectPackageTarball(tarball);
|
||||
const runtime = await installCandidate(tarball, path.join(workRoot, "candidate-install"));
|
||||
const tarballBytes = fs.readFileSync(tarball);
|
||||
const observer = await selfTestObserver({ cwd: zones.tmp, env: process.env, zones, evidenceDir });
|
||||
const runtimePromotion = await prepareRuntimeCache(runtime, tarballBytes, inspection.sha512, zones.cache);
|
||||
|
||||
const suites = [];
|
||||
const failures = [];
|
||||
const run = async (id, action) => {
|
||||
try { suites.push(await action()); }
|
||||
catch (error) {
|
||||
suites.push(failedSuite(id, error));
|
||||
failures.push({ code: /^AAS_VERIFIER_/.test(error.code || "") ? error.code : "AAS_VERIFIER_INTERNAL", suite: id, detailDigest: digestJson({ name: error.name, message: error.message }) });
|
||||
}
|
||||
};
|
||||
await run("package", () => packageSuite(inspection, runtime));
|
||||
await run("entrypoints", () => verifyEntrypoints(runtime, zones));
|
||||
await run("mcp", () => verifyMcp(runtime, zones, evidenceDir));
|
||||
await run("property", () => verifyProperty(runtime, budgets, runtimeMatrix.jobs.indexOf(job), verifierRoot));
|
||||
await run("fuzz", () => verifyFuzz(runtime, budgets, runtimeMatrix.jobs.indexOf(job), verifierRoot));
|
||||
await run("hostile", () => verifyHostile(runtime, zones, evidenceDir, hostileManifest, path.join(baselineRoot, "hostile"), verifierRoot));
|
||||
await run("legacy", () => verifyLegacy(runtime, zones, verifierRoot, path.join(baselineRoot, "legacy", "14.6.0")));
|
||||
await run("transaction", () => transactionSuite(options["transaction-evidence"], manifest, transactionValidator));
|
||||
await run("adapters", () => verifyAdapters(runtime, zones, path.join(baselineRoot, "host-adapters"), inspection.sha512, runtimePromotion.identity.closureDigest));
|
||||
for (const id of SUITE_IDS) if (!suites.some((entry) => entry.id === id)) suites.push(failedSuite(id, Object.assign(new Error("Suite did not execute"), { code: "AAS_VERIFIER_SUITE_MISSING" })));
|
||||
|
||||
const mcp = suites.find((entry) => entry.id === "mcp");
|
||||
const canonicalSha = mcp?.evidence?.canonicalResponseDigest || digestJson(null);
|
||||
const { jobId: _jobId, ...identity } = systemIdentity(job);
|
||||
const mcpBefore = mcp?.evidence?.before || Object.fromEntries(Object.keys(zones).map((name) => [name, zoneBefore[name].digest]));
|
||||
const mcpAfter = mcp?.evidence?.after || mcpBefore;
|
||||
const receipt = {
|
||||
schemaVersion: 1,
|
||||
receiptVersion: "1.0.0",
|
||||
status: failures.length ? "failed" : "passed",
|
||||
job: { id: job.id, workflowRunId: process.env.GITHUB_RUN_ID || "1", workflowRunAttempt: process.env.GITHUB_RUN_ATTEMPT || "1" },
|
||||
candidate: {
|
||||
commit: candidateCommit, package: runtime.manifest.name, version: runtime.manifest.version,
|
||||
tarballBytes: inspection.bytes, tarballSha256: inspection.sha256, tarballSha512: inspection.sha512,
|
||||
packManifestSha256: digestJson(inspection.entries), installTreeSha256: runtime.treeDigest,
|
||||
},
|
||||
verifier: { version: "1.0.0", commit: verifierCommit, rootDigest: freeze.rootDigest, contractDigest: digestJson(manifest), owner: "aas-v1-independent-verifier" },
|
||||
environment: { ...identity, nodeExecutableSha256: executableDigest() },
|
||||
observer: { contractVersion: observer.contractVersion, backend: observer.backend, selfTestDigest: observer.selfTestDigest, networkSentinels: observer.observedNetworkSentinels, writeSentinels: observer.observedWriteSentinels, overflow: false, ambiguousLineage: false },
|
||||
zones: Object.fromEntries(Object.keys(zones).map((name) => [name, { beforeSha256: mcpBefore[name], afterSha256: mcpAfter[name], persistentWriteCount: 0 }])),
|
||||
suites: suites.sort((a, b) => SUITE_IDS.indexOf(a.id) - SUITE_IDS.indexOf(b.id)),
|
||||
canonicalPayload: { sha256: canonicalSha, excludedFields: ["timestamp", "correlationId", "localizedMessage", "diagnostics"], sampleCount: 60 },
|
||||
failures,
|
||||
};
|
||||
writeCanonicalReceipt(output, receipt);
|
||||
process.stdout.write(`${JSON.stringify({ ok: failures.length === 0, output, failures })}\n`);
|
||||
if (failures.length) process.exit(1);
|
||||
Reference in New Issue
Block a user