📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.resolve(here, "..", "..");
|
||||
const modeArg = process.argv.indexOf("--mode");
|
||||
const mode = modeArg >= 0 ? process.argv[modeArg + 1] : "structure";
|
||||
|
||||
if (!["structure", "freeze-ready"].includes(mode)) {
|
||||
console.error(`Unknown mode: ${mode}`);
|
||||
process.exit(64);
|
||||
}
|
||||
|
||||
const failures = [];
|
||||
const pending = [];
|
||||
const assert = (condition, code, detail) => {
|
||||
if (!condition) failures.push({ code, detail });
|
||||
};
|
||||
const requireForFreeze = (condition, code, detail) => {
|
||||
if (!condition) pending.push({ code, detail });
|
||||
};
|
||||
const readJson = (relative) => JSON.parse(fs.readFileSync(path.join(root, relative), "utf8"));
|
||||
|
||||
const schemaDir = path.join(root, "schemas");
|
||||
const schemaFiles = fs.readdirSync(schemaDir).filter((name) => name.endsWith(".schema.json"));
|
||||
assert(schemaFiles.length >= 9, "AAS_BASELINE_SCHEMA_SET_INCOMPLETE", "At least nine public schemas are required.");
|
||||
for (const schemaFile of schemaFiles) {
|
||||
const schema = readJson(`schemas/${schemaFile}`);
|
||||
assert(schema.$schema === "https://json-schema.org/draft/2020-12/schema", "AAS_BASELINE_SCHEMA_DRAFT", schemaFile);
|
||||
assert(typeof schema.$id === "string" && schema.$id.length > 0, "AAS_BASELINE_SCHEMA_ID", schemaFile);
|
||||
}
|
||||
|
||||
const metrics = readJson("baseline/v1/metrics.json");
|
||||
assert(metrics.heldOutCasesPerIntent === 30, "AAS_BASELINE_METRIC_DENOMINATOR", "Expected 30 held-out cases per intent.");
|
||||
assert(metrics.thresholds.verifiedCoverage === 0.8, "AAS_BASELINE_METRIC_COVERAGE", "Coverage threshold must be 0.8.");
|
||||
assert(metrics.thresholds.inclusionPrecision === 0.9, "AAS_BASELINE_METRIC_PRECISION", "Precision threshold must be 0.9.");
|
||||
assert(metrics.thresholds.outOfCoverageAbstention === 1, "AAS_BASELINE_METRIC_ABSTENTION", "Abstention threshold must be 1.");
|
||||
assert(metrics.thresholds.hardPolicyViolations === 0, "AAS_BASELINE_METRIC_POLICY", "Hard-policy violations must be zero.");
|
||||
assert(metrics.canonicalComparisonExcludedFields.length === 4, "AAS_BASELINE_CANONICAL_EXCLUSIONS", "Canonical exclusions must be an explicit four-field list.");
|
||||
|
||||
const benchmark = readJson("baseline/v1/benchmark/manifest.json");
|
||||
const heldOut = readJson("baseline/v1/benchmark/held-out-index.json");
|
||||
const tuningManifest = readJson("baseline/v1/benchmark/tuning/manifest.json");
|
||||
const expectedIntents = [
|
||||
"web-application-delivery",
|
||||
"api-backend-delivery",
|
||||
"test-qa-automation",
|
||||
"security-review-hardening",
|
||||
"deployment-devops",
|
||||
"agent-mcp-development",
|
||||
];
|
||||
assert(JSON.stringify(benchmark.intents) === JSON.stringify(expectedIntents), "AAS_BASELINE_INTENTS", "The six frozen intents changed.");
|
||||
assert(benchmark.benchmarkVersion === "1.0.1", "AAS_BASELINE_BENCHMARK_VERSION", "The independently amended benchmark must be version 1.0.1.");
|
||||
assert(tuningManifest.tuningVersion === "1.0.1"
|
||||
&& tuningManifest.equivalenceAudit?.auditId === "aas-v1-tuning-gold-equivalence-1.0.1"
|
||||
&& tuningManifest.equivalenceAudit?.changedPairs === 7,
|
||||
"AAS_BASELINE_TUNING_AUDIT", "The tuning-gold equivalence amendment is missing or incomplete.");
|
||||
assert(heldOut.cases.length === 180, "AAS_BASELINE_HELDOUT_COUNT", `Expected 180 descriptors, found ${heldOut.cases.length}.`);
|
||||
assert(new Set(heldOut.cases.map((entry) => entry.caseId)).size === 180, "AAS_BASELINE_CASE_ID_DUPLICATE", "Held-out case IDs must be unique.");
|
||||
assert(new Set(heldOut.cases.map((entry) => entry.taskFamilyId)).size === 180, "AAS_BASELINE_TASK_FAMILY_DUPLICATE", "Task families must be unique.");
|
||||
for (const intent of expectedIntents) {
|
||||
const cases = heldOut.cases.filter((entry) => entry.intent === intent);
|
||||
const subIntents = heldOut.intentSubIntents[intent];
|
||||
assert(cases.length === 30, "AAS_BASELINE_INTENT_CASE_COUNT", `${intent}: ${cases.length}`);
|
||||
assert(Array.isArray(subIntents) && subIntents.length === 5, "AAS_BASELINE_SUBINTENT_COUNT", intent);
|
||||
assert(new Set(cases.map((entry) => entry.archetype)).size === 6, "AAS_BASELINE_ARCHETYPE_COUNT", intent);
|
||||
for (const subIntent of subIntents || []) {
|
||||
for (const archetype of heldOut.archetypes) {
|
||||
assert(cases.some((entry) => entry.subIntent === subIntent && entry.archetype === archetype), "AAS_BASELINE_DIVERSIFICATION_CELL", `${intent}/${subIntent}/${archetype}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(heldOut.cases.every((entry) => !("acceptedSolutions" in entry) && !("expectedStack" in entry)), "AAS_BASELINE_FABRICATED_LABEL", "Structural descriptors must not contain gold labels.");
|
||||
requireForFreeze(benchmark.labelsFrozen === true, "AAS_BASELINE_LABELS_PENDING", "Held-out labels are not frozen.");
|
||||
requireForFreeze(heldOut.cases.every((entry) => entry.inputPath && entry.goldPath && entry.provenance && entry.reviewStatus === "approved"), "AAS_BASELINE_CASES_PENDING", "Real case inputs, gold sets, provenance, or approvals are missing.");
|
||||
requireForFreeze(benchmark.abstention.labelsFrozen === true && benchmark.abstention.caseCount > 0, "AAS_BASELINE_ABSTENTION_PENDING", "The separate abstention set is not frozen.");
|
||||
requireForFreeze(benchmark.tuning.status === "frozen", "AAS_BASELINE_TUNING_PENDING", "The tuning set is not frozen separately.");
|
||||
|
||||
const budgets = readJson("baseline/v1/budgets.json");
|
||||
const sum = (strata) => strata.reduce((total, stratum) => total + stratum.executions, 0);
|
||||
assert(budgets.propertyAndGenerative.minimumExecutions === 100000, "AAS_BASELINE_PROPERTY_BUDGET", "Property/generative budget must be 100000.");
|
||||
assert(sum(budgets.propertyAndGenerative.strata) === 100000, "AAS_BASELINE_PROPERTY_DISTRIBUTION", "Property/generative strata must sum to 100000.");
|
||||
assert(budgets.parserAndMcpFuzz.minimumExecutions === 50000, "AAS_BASELINE_FUZZ_BUDGET", "Parser/MCP fuzz budget must be 50000.");
|
||||
assert(sum(budgets.parserAndMcpFuzz.strata) === 50000, "AAS_BASELINE_FUZZ_DISTRIBUTION", "Parser/MCP fuzz strata must sum to 50000.");
|
||||
assert(budgets.hardGate.budgetReductionAllowed === false, "AAS_BASELINE_BUDGET_REDUCTION", "Budget reduction must be forbidden.");
|
||||
requireForFreeze(budgets.status === "frozen" && budgets.prng.rootSeed && budgets.prng.algorithm !== "pending-independent-review", "AAS_BASELINE_SEEDS_PENDING", "Independent PRNG seed and derivation are pending.");
|
||||
|
||||
const hostile = readJson("baseline/v1/hostile/manifest.json");
|
||||
assert(hostile.classes.some((entry) => entry.surface === "archive"), "AAS_BASELINE_ARCHIVE_CORPUS", "Archive classes are missing.");
|
||||
assert(hostile.classes.some((entry) => entry.surface === "input"), "AAS_BASELINE_INPUT_CORPUS", "Input classes are missing.");
|
||||
assert(new Set(hostile.classes.map((entry) => entry.classId)).size === hostile.classes.length, "AAS_BASELINE_HOSTILE_DUPLICATE", "Hostile class IDs must be unique.");
|
||||
assert(hostile.classes.every((entry) => entry.exploit.expected === "reject" && entry.boundaryControl.expected === "accept"), "AAS_BASELINE_HOSTILE_PAIR", "Every hostile class requires reject/accept pairs.");
|
||||
requireForFreeze(hostile.status === "frozen" && hostile.classes.every((entry) => entry.status === "frozen" && entry.exploit.path && entry.exploit.sha256 && entry.boundaryControl.path && entry.boundaryControl.sha256), "AAS_BASELINE_HOSTILE_FIXTURES_PENDING", "Hostile exploit/control fixtures and hashes are pending.");
|
||||
|
||||
const runtime = readJson("baseline/v1/runtime-matrix.json");
|
||||
const expectedJobs = new Set(["linux-node-22", "linux-node-24", "macos-node-22", "macos-node-24", "windows-node-22", "windows-node-24"]);
|
||||
assert(runtime.jobs.length === 6 && runtime.jobs.every((job) => expectedJobs.has(job.id)), "AAS_BASELINE_RUNTIME_MATRIX", "Runtime matrix must contain exactly six OS/Node jobs.");
|
||||
assert(new Set(runtime.jobs.map((job) => job.id)).size === expectedJobs.size, "AAS_BASELINE_RUNTIME_MATRIX_DUPLICATE", "Runtime matrix job IDs must be unique.");
|
||||
assert([...expectedJobs].every((id) => runtime.jobs.some((job) => job.id === id)), "AAS_BASELINE_RUNTIME_MATRIX_EXHAUSTIVE", "Runtime matrix must contain every frozen job exactly once.");
|
||||
assert(new Set(runtime.jobs.map((job) => `${job.os}/${job.nodePatch}`)).size === expectedJobs.size, "AAS_BASELINE_RUNTIME_MATRIX_IDENTITY_DUPLICATE", "OS/Node identities must be unique.");
|
||||
assert(runtime.skipsAllowed === false && runtime.continueOnErrorAllowed === false, "AAS_BASELINE_MATRIX_FAILURE_POLICY", "Skips and allowed failures are forbidden.");
|
||||
requireForFreeze(runtime.status === "frozen" && runtime.jobs.every((job) => job.nodePatch && job.runnerImage && job.architecture && job.filesystem && job.observer && job.status === "frozen"), "AAS_BASELINE_RUNTIME_IDENTITIES_PENDING", "Exact Node, runner, architecture, filesystem, or observer identities are pending.");
|
||||
|
||||
const verifierManifest = readJson("baseline/v1/verifier-manifest.json");
|
||||
assert(verifierManifest.requiredSuites.length === 9 && new Set(verifierManifest.requiredSuites).size === 9, "AAS_BASELINE_PRODUCT_SUITE_SET", "The product verifier must require nine unique suites.");
|
||||
assert(verifierManifest.candidateIsolation.requiresProcessSeparation === true && verifierManifest.candidateIsolation.testHooksAllowed === false, "AAS_BASELINE_CANDIDATE_ISOLATION", "Candidate code requires an external process and forbids test hooks.");
|
||||
assert(verifierManifest.faultBoundaryClasses.length === 7 && verifierManifest.raceClasses.length === 6, "AAS_BASELINE_TRANSACTION_COVERAGE", "Fault/race class contracts changed.");
|
||||
|
||||
const legacy = readJson("baseline/v1/legacy/14.6.0/manifest.json");
|
||||
assert(legacy.baseline.version === "14.6.0", "AAS_BASELINE_LEGACY_VERSION", "Legacy baseline version changed.");
|
||||
assert(legacy.baseline.distIntegrity === "sha512-VTOb3O9PSYKCDO99i3h0vOn7vHQlGtO/+jSErR80g6OGaDJoBzg3q2GE9Nu890en1/Z54hBEYiVQj/1Rl95xEg==", "AAS_BASELINE_LEGACY_SRI", "Legacy npm SRI changed.");
|
||||
assert(legacy.baseline.sourceCommit === "ab5f6c205a548d2f4bec411728c79b9c156fc696", "AAS_BASELINE_LEGACY_COMMIT", "Legacy source commit changed.");
|
||||
const allArgs = legacy.cases.flatMap((entry) => entry.args);
|
||||
for (const flag of legacy.publicFlags) {
|
||||
assert(allArgs.includes(flag), "AAS_BASELINE_LEGACY_FLAG_COVERAGE", flag);
|
||||
}
|
||||
for (const target of legacy.targets) {
|
||||
assert(legacy.cases.some((entry) => entry.args.includes(`--${target}`)), "AAS_BASELINE_LEGACY_TARGET_COVERAGE", target);
|
||||
}
|
||||
assert(legacy.cases.some((entry) => entry.args.includes("install")), "AAS_BASELINE_LEGACY_INSTALL_COMMAND", "Literal install command is missing.");
|
||||
requireForFreeze(legacy.status === "frozen" && legacy.fixtureRepository.treeDigest && legacy.fixtureRepository.fakeGitTraceDigest && legacy.cases.every((entry) => entry.expectedSnapshot), "AAS_BASELINE_LEGACY_SNAPSHOTS_PENDING", "Legacy fixture digests or expected snapshots are pending.");
|
||||
|
||||
const ownership = readJson("ownership.v1.json");
|
||||
assert(ownership.minimumApprovals === 2 && ownership.requireNonScorerReviewer === true, "AAS_BASELINE_OWNERSHIP_POLICY", "Two approvals including a non-scorer reviewer are required.");
|
||||
const reviewerIdentities = new Set(ownership.reviewers.map((reviewer) => reviewer.identity));
|
||||
const reviewerReportsValid = ownership.reviewers.every((reviewer) => {
|
||||
if (!reviewer.report || !reviewer.reportSha256 || reviewer.reviewedPairs !== 270) return false;
|
||||
const reportPath = path.join(root, reviewer.report);
|
||||
if (!fs.existsSync(reportPath)) return false;
|
||||
const digest = crypto.createHash("sha256")
|
||||
.update(fs.readFileSync(reportPath))
|
||||
.digest("hex");
|
||||
return digest === reviewer.reportSha256;
|
||||
});
|
||||
const reviewAmendmentsValid = Array.isArray(ownership.reviewAmendments)
|
||||
&& ownership.reviewAmendments.length === 1
|
||||
&& ownership.reviewAmendments.every((amendment) => {
|
||||
if (amendment.auditId !== "aas-v1-tuning-gold-equivalence-1.0.1"
|
||||
|| amendment.claimsReviewed !== 19
|
||||
|| amendment.changedPairs !== 7
|
||||
|| amendment.minimumIndependentReviewsPerChangedPair !== 2
|
||||
|| !Array.isArray(amendment.reports)
|
||||
|| amendment.reports.length !== 4) return false;
|
||||
const auditPath = path.join(root, amendment.audit);
|
||||
if (!fs.existsSync(auditPath)) return false;
|
||||
const auditDigest = crypto.createHash("sha256").update(fs.readFileSync(auditPath)).digest("hex");
|
||||
if (auditDigest !== amendment.auditSha256) return false;
|
||||
const audit = JSON.parse(fs.readFileSync(auditPath, "utf8"));
|
||||
if (audit.auditId !== amendment.auditId || audit.scope?.changedPairCount !== amendment.changedPairs) return false;
|
||||
const identities = new Set();
|
||||
for (const reviewer of amendment.reports) {
|
||||
if (!reviewer.identity || reviewer.scorerImplementer !== false || reviewer.reviewedClaims < 1) return false;
|
||||
identities.add(reviewer.identity);
|
||||
const reportPath = path.join(root, reviewer.report);
|
||||
if (!fs.existsSync(reportPath)) return false;
|
||||
const bytes = fs.readFileSync(reportPath);
|
||||
const digest = crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
if (digest !== reviewer.reportSha256) return false;
|
||||
const report = JSON.parse(bytes.toString("utf8"));
|
||||
if (report.reviewer?.identity !== reviewer.identity
|
||||
|| report.reviewer?.scorerImplementer !== false
|
||||
|| report.decisions?.length !== reviewer.reviewedClaims) return false;
|
||||
}
|
||||
return identities.size === amendment.reports.length;
|
||||
});
|
||||
requireForFreeze(
|
||||
ownership.status === "frozen"
|
||||
&& reviewerIdentities.size >= 2
|
||||
&& ownership.reviewers.some((reviewer) => reviewer.scorerImplementer === false)
|
||||
&& reviewerReportsValid
|
||||
&& reviewAmendmentsValid
|
||||
&& ownership.repositorySettings.settingsVerified === true
|
||||
&& ownership.verificationEnvironment.requiredReviewersVerified === true
|
||||
&& ownership.verificationEnvironment.productPullRequestsMayModifyBaseline === false,
|
||||
"AAS_BASELINE_OWNERS_PENDING",
|
||||
"Named independent owners, report digests, or protected GitHub settings are pending.",
|
||||
);
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, mode, failures, pending }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (mode === "freeze-ready" && pending.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, mode, code: "AAS_BASELINE_NOT_FREEZE_READY", pending }, null, 2));
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ ok: true, mode, schemaCount: schemaFiles.length, heldOutDescriptors: heldOut.cases.length, hostileClasses: hostile.classes.length, legacyCases: legacy.cases.length, pendingCount: pending.length }, null, 2));
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const manifestPath = path.join(verificationRoot, "baseline", "v1", "freeze-manifest.json");
|
||||
const write = process.argv.includes("--write");
|
||||
const excludedPrefixes = [
|
||||
"verifier/node_modules/",
|
||||
"baseline/v1/legacy/14.6.0/_work/",
|
||||
];
|
||||
|
||||
function relativePath(target) {
|
||||
return path.relative(verificationRoot, target).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function isExcluded(target, { directory = false } = {}) {
|
||||
const relative = relativePath(target);
|
||||
const candidate = directory ? `${relative}/` : relative;
|
||||
return excludedPrefixes.some((prefix) => candidate.startsWith(prefix));
|
||||
}
|
||||
|
||||
function walk(directory) {
|
||||
return fs.readdirSync(directory, { withFileTypes: true })
|
||||
.flatMap((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (isExcluded(target, { directory: true })) return [];
|
||||
return walk(target);
|
||||
}
|
||||
if (!entry.isFile()) throw new Error(`Non-regular baseline entry: ${target}`);
|
||||
return [target];
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
function sha256(bytes) {
|
||||
return crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
const files = walk(verificationRoot)
|
||||
.filter((file) => file !== manifestPath)
|
||||
.filter((file) => !isExcluded(file))
|
||||
.map((file) => {
|
||||
const bytes = fs.readFileSync(file);
|
||||
return {
|
||||
path: relativePath(file),
|
||||
bytes: bytes.length,
|
||||
sha256: sha256(bytes),
|
||||
};
|
||||
});
|
||||
const rootDigest = sha256(Buffer.from(JSON.stringify(files)));
|
||||
const candidate = {
|
||||
schemaVersion: 1,
|
||||
baselineVersion: "1.0.1",
|
||||
status: "frozen",
|
||||
digestAlgorithm: "sha256",
|
||||
excludedPaths: ["baseline/v1/freeze-manifest.json", ...excludedPrefixes],
|
||||
rootDigest: `sha256-${rootDigest}`,
|
||||
fileCount: files.length,
|
||||
files,
|
||||
};
|
||||
|
||||
if (write) {
|
||||
fs.writeFileSync(manifestPath, `${JSON.stringify(candidate, null, 2)}\n`, { mode: 0o644 });
|
||||
console.log(JSON.stringify({ ok: true, wrote: path.relative(verificationRoot, manifestPath), fileCount: files.length, rootDigest: candidate.rootDigest }, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
console.error(JSON.stringify({ ok: false, code: "AAS_BASELINE_FREEZE_MANIFEST_MISSING" }, null, 2));
|
||||
process.exit(2);
|
||||
}
|
||||
const expected = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
||||
if (JSON.stringify(expected) !== JSON.stringify(candidate)) {
|
||||
const expectedByPath = new Map((expected.files || []).map((entry) => [entry.path, entry]));
|
||||
const actualByPath = new Map(files.map((entry) => [entry.path, entry]));
|
||||
const changed = [...new Set([...expectedByPath.keys(), ...actualByPath.keys()])]
|
||||
.filter((file) => JSON.stringify(expectedByPath.get(file)) !== JSON.stringify(actualByPath.get(file)))
|
||||
.sort();
|
||||
console.error(JSON.stringify({
|
||||
ok: false,
|
||||
code: "AAS_BASELINE_FREEZE_DIGEST_MISMATCH",
|
||||
expectedRootDigest: expected.rootDigest,
|
||||
actualRootDigest: candidate.rootDigest,
|
||||
changed,
|
||||
}, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(JSON.stringify({ ok: true, fileCount: files.length, rootDigest: candidate.rootDigest }, null, 2));
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { aggregateReceipts, writeBundle } from "../lib/aggregate.mjs";
|
||||
import { loadReceiptValidator } from "../lib/receipt.mjs";
|
||||
|
||||
const outIndex = process.argv.indexOf("--out");
|
||||
const out = outIndex >= 0 ? path.resolve(process.argv[outIndex + 1]) : null;
|
||||
const inputs = process.argv.slice(2).filter((value, index, all) => value !== "--out" && all[index - 1] !== "--out");
|
||||
if (!out || inputs.length === 0) throw new Error("Usage: merge-product-evidence --out <bundle.json> <receipt...>");
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const schema = path.resolve(here, "..", "..", "schemas", "product-verifier-receipt.schema.json");
|
||||
const validator = loadReceiptValidator(schema);
|
||||
const receipts = inputs.map((file) => JSON.parse(fs.readFileSync(file, "utf8")));
|
||||
const bundle = aggregateReceipts(receipts, validator);
|
||||
writeBundle(out, bundle);
|
||||
process.stdout.write(`${JSON.stringify({ ok: bundle.status === "passed", out, bundleDigest: bundle.bundleDigest, failures: bundle.failures })}\n`);
|
||||
if (bundle.status !== "passed") process.exit(1);
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { selfTestObserver } from "../lib/observer.mjs";
|
||||
import { isolatedZones } from "../lib/runtime.mjs";
|
||||
|
||||
const jobIndex = process.argv.indexOf("--job-id");
|
||||
const jobId = jobIndex >= 0 ? process.argv[jobIndex + 1] : "";
|
||||
const expected = jobId.startsWith("linux-") ? "linux-strace-process-tree"
|
||||
: jobId.startsWith("macos-") ? "macos-fs_usage-process"
|
||||
: jobId.startsWith("windows-") ? "windows-etw-kernel-process-tree" : null;
|
||||
if (!expected) throw new Error("--job-id must be a frozen runtime-matrix job");
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "aas-observer-self-test-"));
|
||||
try {
|
||||
const zones = isolatedZones(path.join(root, "zones"));
|
||||
const evidenceDir = path.join(root, "evidence");
|
||||
const result = await selfTestObserver({ cwd: zones.tmp, env: process.env, zones, evidenceDir });
|
||||
if (result.backend !== expected) throw new Error(`observer backend mismatch: ${result.backend}/${expected}`);
|
||||
if (result.observedNetworkSentinels < 1 || result.observedWriteSentinels < 1) throw new Error("observer sentinel was not detected");
|
||||
process.stdout.write(`${JSON.stringify({ ok: true, jobId, ...result })}\n`);
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const repositoryRoot = path.resolve(verificationRoot, "..", "..");
|
||||
const benchmarkRoot = path.join(verificationRoot, "baseline", "v1", "benchmark");
|
||||
const frozen = process.argv.includes("--require-approvals");
|
||||
const failures = [];
|
||||
|
||||
function fail(code, detail) {
|
||||
failures.push({ code, detail });
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
} catch (error) {
|
||||
fail("AAS_BENCHMARK_INVALID_JSON", `${path.relative(verificationRoot, file)}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function walkJson(directory) {
|
||||
if (!fs.existsSync(directory)) return [];
|
||||
return fs.readdirSync(directory, { withFileTypes: true })
|
||||
.flatMap((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) return walkJson(target);
|
||||
return entry.isFile() && entry.name.endsWith(".json") ? [target] : [];
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function pairDigest(caseData, goldData) {
|
||||
const reviewNeutralGold = { ...goldData, reviews: [] };
|
||||
const bytes = JSON.stringify(canonicalize({ case: caseData, gold: reviewNeutralGold }));
|
||||
return `sha256-${crypto.createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
|
||||
const index = readJson(path.join(benchmarkRoot, "held-out-index.json"));
|
||||
const catalog = readJson(path.join(repositoryRoot, "data", "skills_index.json"));
|
||||
if (!index || !catalog) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const descriptors = new Map(index.cases.map((entry) => [entry.caseId, entry]));
|
||||
const catalogById = new Map(catalog.map((entry) => [entry.id, entry]));
|
||||
const caseRoot = path.join(benchmarkRoot, "cases", "held-out");
|
||||
const goldRoot = path.join(benchmarkRoot, "gold", "held-out");
|
||||
const caseFiles = walkJson(caseRoot);
|
||||
const goldFiles = walkJson(goldRoot);
|
||||
const seenIds = new Set();
|
||||
const seenFingerprints = new Set();
|
||||
|
||||
if (caseFiles.length !== 180) fail("AAS_BENCHMARK_CASE_COUNT", `expected 180, found ${caseFiles.length}`);
|
||||
if (goldFiles.length !== 180) fail("AAS_BENCHMARK_GOLD_COUNT", `expected 180, found ${goldFiles.length}`);
|
||||
|
||||
for (const caseFile of caseFiles) {
|
||||
const relative = path.relative(caseRoot, caseFile);
|
||||
const goldFile = path.join(goldRoot, relative);
|
||||
const caseData = readJson(caseFile);
|
||||
const goldData = readJson(goldFile);
|
||||
if (!caseData || !goldData) continue;
|
||||
const descriptor = descriptors.get(caseData.caseId);
|
||||
|
||||
if (!descriptor) fail("AAS_BENCHMARK_UNKNOWN_CASE", caseData.caseId);
|
||||
if (seenIds.has(caseData.caseId)) fail("AAS_BENCHMARK_DUPLICATE_CASE", caseData.caseId);
|
||||
seenIds.add(caseData.caseId);
|
||||
if (seenFingerprints.has(caseData.taskFamilyFingerprint)) {
|
||||
fail("AAS_BENCHMARK_DUPLICATE_FINGERPRINT", caseData.taskFamilyFingerprint);
|
||||
}
|
||||
seenFingerprints.add(caseData.taskFamilyFingerprint);
|
||||
|
||||
if (!descriptor) continue;
|
||||
if (descriptor.inputPath && descriptor.inputPath !== path.relative(benchmarkRoot, caseFile)) {
|
||||
fail("AAS_BENCHMARK_PATH_MISMATCH", `${caseData.caseId}/input`);
|
||||
}
|
||||
if (descriptor.goldPath && descriptor.goldPath !== path.relative(benchmarkRoot, goldFile)) {
|
||||
fail("AAS_BENCHMARK_PATH_MISMATCH", `${caseData.caseId}/gold`);
|
||||
}
|
||||
if (caseData.intent !== descriptor.intent) fail("AAS_BENCHMARK_INTENT_MISMATCH", caseData.caseId);
|
||||
if (caseData.taskFamilyFingerprint !== descriptor.taskFamilyId) {
|
||||
fail("AAS_BENCHMARK_FINGERPRINT_MISMATCH", caseData.caseId);
|
||||
}
|
||||
if (goldData.caseId !== caseData.caseId) fail("AAS_BENCHMARK_PAIR_MISMATCH", caseData.caseId);
|
||||
if (!Array.isArray(caseData.criticalGoals) || caseData.criticalGoals.length === 0) {
|
||||
fail("AAS_BENCHMARK_CRITICAL_GOALS", caseData.caseId);
|
||||
}
|
||||
if ((caseData.minimumNonCriticalGoalCoverage ?? 0.8) < 0.8) {
|
||||
fail("AAS_BENCHMARK_NONCRITICAL_THRESHOLD", caseData.caseId);
|
||||
}
|
||||
if (caseData.requiresSkill !== true) fail("AAS_BENCHMARK_REQUIRES_SKILL", caseData.caseId);
|
||||
if (!Array.isArray(caseData.targets) || caseData.targets.length === 0) {
|
||||
fail("AAS_BENCHMARK_TARGETS", caseData.caseId);
|
||||
}
|
||||
if (!Array.isArray(caseData.policy?.allowedRisk)
|
||||
|| typeof caseData.policy?.requireKnownSource !== "boolean"
|
||||
|| typeof caseData.policy?.allowManualSetup !== "boolean") {
|
||||
fail("AAS_BENCHMARK_POLICY", caseData.caseId);
|
||||
}
|
||||
if (!caseData.provenance?.source || !caseData.provenance?.version || !caseData.provenance?.reviewedAt) {
|
||||
fail("AAS_BENCHMARK_PROVENANCE", caseData.caseId);
|
||||
}
|
||||
|
||||
for (const solution of goldData.acceptedSolutions || []) {
|
||||
const allowed = solution.allowedSkillIds || [];
|
||||
if (allowed.length === 0 || new Set(allowed).size !== allowed.length) {
|
||||
fail("AAS_BENCHMARK_ALLOWED_SKILLS", `${caseData.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
for (const skillId of allowed) {
|
||||
if (!catalogById.has(skillId)) fail("AAS_BENCHMARK_UNKNOWN_SKILL", `${caseData.caseId}/${skillId}`);
|
||||
}
|
||||
for (const group of solution.requiredGroups || []) {
|
||||
if (!Array.isArray(group) || group.length === 0 || group.some((skillId) => !allowed.includes(skillId))) {
|
||||
fail("AAS_BENCHMARK_REQUIRED_GROUP", `${caseData.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!goldData.provenance?.source || !goldData.provenance?.version || !goldData.provenance?.reviewedAt) {
|
||||
fail("AAS_BENCHMARK_GOLD_PROVENANCE", caseData.caseId);
|
||||
}
|
||||
const solutionFingerprints = new Set();
|
||||
for (const solution of goldData.acceptedSolutions || []) {
|
||||
const fingerprint = JSON.stringify(canonicalize({
|
||||
allowedSkillIds: solution.allowedSkillIds,
|
||||
requiredGroups: solution.requiredGroups,
|
||||
}));
|
||||
if (solutionFingerprints.has(fingerprint)) {
|
||||
fail("AAS_BENCHMARK_DUPLICATE_SOLUTION", `${caseData.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
solutionFingerprints.add(fingerprint);
|
||||
for (const skillId of solution.allowedSkillIds || []) {
|
||||
const skill = catalogById.get(skillId);
|
||||
if (!skill) continue;
|
||||
const explicitRisk = skill.risk;
|
||||
if (explicitRisk && explicitRisk !== "unknown" && !caseData.policy.allowedRisk.includes(explicitRisk)) {
|
||||
fail("AAS_BENCHMARK_GOLD_RISK_VIOLATION", `${caseData.caseId}/${skillId}/${explicitRisk}`);
|
||||
}
|
||||
if (skill.plugin?.setup?.type === "manual" && caseData.policy.allowManualSetup !== true) {
|
||||
fail("AAS_BENCHMARK_GOLD_SETUP_VIOLATION", `${caseData.caseId}/${skillId}`);
|
||||
}
|
||||
for (const target of caseData.targets) {
|
||||
if (skill.plugin?.targets?.[target.host] === "blocked") {
|
||||
fail("AAS_BENCHMARK_GOLD_HOST_VIOLATION", `${caseData.caseId}/${skillId}/${target.host}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const digest = pairDigest(caseData, goldData);
|
||||
const approvedReviews = (goldData.reviews || []).filter((review) => (
|
||||
review.decision === "approved" && review.reviewedDigest === digest
|
||||
));
|
||||
if (frozen) {
|
||||
const uniqueReviewers = new Set(approvedReviews.map((review) => review.reviewer));
|
||||
if (uniqueReviewers.size < 2) fail("AAS_BENCHMARK_REVIEW_COUNT", caseData.caseId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const descriptor of index.cases) {
|
||||
if (!seenIds.has(descriptor.caseId)) fail("AAS_BENCHMARK_MISSING_CASE", descriptor.caseId);
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, frozen, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
frozen,
|
||||
cases: caseFiles.length,
|
||||
gold: goldFiles.length,
|
||||
uniqueFingerprints: seenFingerprints.size,
|
||||
catalogSkills: catalogById.size,
|
||||
}, null, 2));
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import addFormats from "ajv-formats";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const baselineRoot = path.join(verificationRoot, "baseline", "v1");
|
||||
const schemasRoot = path.join(verificationRoot, "schemas");
|
||||
const failures = [];
|
||||
|
||||
function readJson(file) {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
}
|
||||
|
||||
function walkJson(directory) {
|
||||
return fs.readdirSync(directory, { withFileTypes: true })
|
||||
.flatMap((entry) => {
|
||||
const target = path.join(directory, entry.name);
|
||||
return entry.isDirectory() ? walkJson(target) : entry.name.endsWith(".json") ? [target] : [];
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
|
||||
addFormats(ajv);
|
||||
const compiled = new Map();
|
||||
for (const file of walkJson(schemasRoot)) {
|
||||
const schema = readJson(file);
|
||||
compiled.set(path.basename(file), ajv.compile(schema));
|
||||
}
|
||||
|
||||
function validate(schemaName, files) {
|
||||
const validator = compiled.get(schemaName);
|
||||
if (!validator) throw new Error(`Unknown schema: ${schemaName}`);
|
||||
for (const file of files) {
|
||||
const data = readJson(file);
|
||||
if (!validator(data)) {
|
||||
failures.push({
|
||||
schema: schemaName,
|
||||
file: path.relative(verificationRoot, file),
|
||||
errors: validator.errors,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const benchmarkRoot = path.join(baselineRoot, "benchmark");
|
||||
validate("benchmark-case.schema.json", [
|
||||
...walkJson(path.join(benchmarkRoot, "cases", "held-out")),
|
||||
...walkJson(path.join(benchmarkRoot, "tuning", "cases")),
|
||||
]);
|
||||
validate("benchmark-gold.schema.json", [
|
||||
...walkJson(path.join(benchmarkRoot, "gold", "held-out")),
|
||||
...walkJson(path.join(benchmarkRoot, "tuning", "gold")),
|
||||
]);
|
||||
validate("benchmark-manifest.schema.json", [path.join(benchmarkRoot, "manifest.json")]);
|
||||
validate("held-out-index.schema.json", [path.join(benchmarkRoot, "held-out-index.json")]);
|
||||
validate("budget-manifest.schema.json", [path.join(baselineRoot, "budgets.json")]);
|
||||
validate("hostile-corpus-manifest.schema.json", [path.join(baselineRoot, "hostile", "manifest.json")]);
|
||||
validate("legacy-command-corpus.schema.json", [path.join(baselineRoot, "legacy", "14.6.0", "manifest.json")]);
|
||||
validate("metric-definition.schema.json", [path.join(baselineRoot, "metrics.json")]);
|
||||
validate("runtime-matrix.schema.json", [path.join(baselineRoot, "runtime-matrix.json")]);
|
||||
validate("tuning-gold-equivalence-review.schema.json", [
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-alpha.json"),
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-beta.json"),
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-adjudicator.json"),
|
||||
path.join(baselineRoot, "reviews", "reviewer-tuning-equivalence-ci-tiebreak.json"),
|
||||
]);
|
||||
validate("tuning-gold-equivalence-audit.schema.json", [
|
||||
path.join(baselineRoot, "reviews", "tuning-gold-equivalence-audit.json"),
|
||||
]);
|
||||
validate("product-verifier-manifest.schema.json", [path.join(baselineRoot, "verifier-manifest.json")]);
|
||||
validate("host-adapter-fixtures.schema.json", [path.join(baselineRoot, "host-adapters", "manifest.json")]);
|
||||
|
||||
const abstentionRoot = path.join(benchmarkRoot, "abstention");
|
||||
const abstentionSchemas = path.join(abstentionRoot, "schemas");
|
||||
for (const name of ["abstention-case", "abstention-label", "abstention-index"]) {
|
||||
const schema = readJson(path.join(abstentionSchemas, `${name}.schema.json`));
|
||||
const validator = ajv.compile(schema);
|
||||
const files = name === "abstention-case"
|
||||
? walkJson(path.join(abstentionRoot, "cases"))
|
||||
: name === "abstention-label"
|
||||
? walkJson(path.join(abstentionRoot, "labels"))
|
||||
: [path.join(abstentionRoot, "index.json")];
|
||||
for (const file of files) {
|
||||
const data = readJson(file);
|
||||
if (!validator(data)) {
|
||||
failures.push({ schema: `${name}.schema.json`, file: path.relative(verificationRoot, file), errors: validator.errors });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
publicSchemas: compiled.size,
|
||||
heldOutCases: 180,
|
||||
heldOutGold: 180,
|
||||
tuningCases: 60,
|
||||
tuningGold: 60,
|
||||
abstentionCases: 30,
|
||||
abstentionLabels: 30,
|
||||
tuningEquivalenceReviews: 4,
|
||||
tuningEquivalenceAudits: 1,
|
||||
}, null, 2));
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const repositoryRoot = path.resolve(verificationRoot, "..", "..");
|
||||
const benchmarkRoot = path.join(verificationRoot, "baseline", "v1", "benchmark");
|
||||
const frozen = process.argv.includes("--require-approvals");
|
||||
const failures = [];
|
||||
const fail = (code, detail) => failures.push({ code, detail });
|
||||
const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function pairDigest(caseData, judgmentData, section) {
|
||||
const reviewNeutralJudgment = { ...judgmentData, reviews: [] };
|
||||
const payload = section === "abstention"
|
||||
? { case: caseData, label: reviewNeutralJudgment }
|
||||
: { case: caseData, gold: reviewNeutralJudgment };
|
||||
const bytes = JSON.stringify(canonicalize(payload));
|
||||
return `sha256-${crypto.createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
|
||||
function requireApprovals(caseId, caseData, judgmentData, section) {
|
||||
if (!frozen) return;
|
||||
const digest = pairDigest(caseData, judgmentData, section);
|
||||
const reviewers = new Set((judgmentData.reviews || [])
|
||||
.filter((review) => review.decision === "approved" && review.reviewedDigest === digest)
|
||||
.map((review) => review.reviewer));
|
||||
if (reviewers.size < 2) fail("AAS_SECONDARY_REVIEW_COUNT", caseId);
|
||||
}
|
||||
|
||||
function validateInputContract(caseData, expectedRequiresSkill) {
|
||||
if (!Array.isArray(caseData.targets) || caseData.targets.length === 0) {
|
||||
fail("AAS_SECONDARY_TARGETS", caseData.caseId);
|
||||
}
|
||||
if (!Array.isArray(caseData.policy?.allowedRisk)
|
||||
|| typeof caseData.policy?.requireKnownSource !== "boolean"
|
||||
|| typeof caseData.policy?.allowManualSetup !== "boolean") {
|
||||
fail("AAS_SECONDARY_POLICY", caseData.caseId);
|
||||
}
|
||||
if (caseData.requiresSkill !== expectedRequiresSkill) {
|
||||
fail("AAS_SECONDARY_REQUIRES_SKILL", caseData.caseId);
|
||||
}
|
||||
}
|
||||
|
||||
const catalog = readJson(path.join(repositoryRoot, "data", "skills_index.json"));
|
||||
const catalogById = new Map(catalog.map((entry) => [entry.id, entry]));
|
||||
const heldOutIndex = readJson(path.join(benchmarkRoot, "held-out-index.json"));
|
||||
const heldOutFingerprints = new Set(heldOutIndex.cases.map((entry) => entry.taskFamilyId));
|
||||
const supportedIntents = new Set(Object.keys(heldOutIndex.intentSubIntents));
|
||||
|
||||
const abstentionRoot = path.join(benchmarkRoot, "abstention");
|
||||
const abstentionIndex = readJson(path.join(abstentionRoot, "index.json"));
|
||||
const reasonRegistry = readJson(path.join(abstentionRoot, abstentionIndex.reasonCodeRegistry));
|
||||
const reasonCodes = new Set(reasonRegistry.codes.map((entry) => entry.code));
|
||||
const abstentionFingerprints = new Set();
|
||||
if (abstentionIndex.cases.length !== 30) fail("AAS_ABSTENTION_CASE_COUNT", abstentionIndex.cases.length);
|
||||
for (const entry of abstentionIndex.cases) {
|
||||
const caseData = readJson(path.join(abstentionRoot, entry.inputPath));
|
||||
const labelData = readJson(path.join(abstentionRoot, entry.labelPath));
|
||||
if (caseData.caseId !== entry.caseId || labelData.caseId !== entry.caseId) {
|
||||
fail("AAS_ABSTENTION_PAIR", entry.caseId);
|
||||
}
|
||||
validateInputContract(caseData, false);
|
||||
if (supportedIntents.has(caseData.intent)) fail("AAS_ABSTENTION_IN_SCOPE_INTENT", entry.caseId);
|
||||
if (labelData.expectedStatus !== "insufficientCoverage" || labelData.expectedProposedStack?.length !== 0) {
|
||||
fail("AAS_ABSTENTION_LABEL", entry.caseId);
|
||||
}
|
||||
if (!labelData.reasonCodes?.length || labelData.reasonCodes.some((code) => !reasonCodes.has(code))) {
|
||||
fail("AAS_ABSTENTION_REASON_CODE", entry.caseId);
|
||||
}
|
||||
if (caseData.taskFamilyFingerprint !== entry.taskFamilyFingerprint) {
|
||||
fail("AAS_ABSTENTION_FINGERPRINT", entry.caseId);
|
||||
}
|
||||
if (abstentionFingerprints.has(entry.taskFamilyFingerprint) || heldOutFingerprints.has(entry.taskFamilyFingerprint)) {
|
||||
fail("AAS_ABSTENTION_FINGERPRINT_COLLISION", entry.caseId);
|
||||
}
|
||||
abstentionFingerprints.add(entry.taskFamilyFingerprint);
|
||||
requireApprovals(entry.caseId, caseData, labelData, "abstention");
|
||||
}
|
||||
|
||||
const tuningRoot = path.join(benchmarkRoot, "tuning");
|
||||
const tuningManifest = readJson(path.join(tuningRoot, "manifest.json"));
|
||||
const tuningIndex = readJson(path.join(tuningRoot, tuningManifest.index));
|
||||
const tuningFingerprints = new Set();
|
||||
if (tuningIndex.cases.length !== 60) fail("AAS_TUNING_CASE_COUNT", tuningIndex.cases.length);
|
||||
for (const intent of supportedIntents) {
|
||||
const count = tuningIndex.cases.filter((entry) => entry.intent === intent).length;
|
||||
if (count !== 10) fail("AAS_TUNING_INTENT_COUNT", `${intent}/${count}`);
|
||||
}
|
||||
for (const entry of tuningIndex.cases) {
|
||||
const caseData = readJson(path.join(tuningRoot, entry.inputPath));
|
||||
const goldData = readJson(path.join(tuningRoot, entry.goldPath));
|
||||
if (caseData.caseId !== entry.caseId || goldData.caseId !== entry.caseId) {
|
||||
fail("AAS_TUNING_PAIR", entry.caseId);
|
||||
}
|
||||
validateInputContract(caseData, true);
|
||||
if (caseData.taskFamilyFingerprint !== entry.taskFamilyFingerprint) {
|
||||
fail("AAS_TUNING_FINGERPRINT", entry.caseId);
|
||||
}
|
||||
if (tuningFingerprints.has(entry.taskFamilyFingerprint)
|
||||
|| heldOutFingerprints.has(entry.taskFamilyFingerprint)
|
||||
|| abstentionFingerprints.has(entry.taskFamilyFingerprint)) {
|
||||
fail("AAS_TUNING_FINGERPRINT_COLLISION", entry.caseId);
|
||||
}
|
||||
tuningFingerprints.add(entry.taskFamilyFingerprint);
|
||||
const solutionFingerprints = new Set();
|
||||
for (const solution of goldData.acceptedSolutions || []) {
|
||||
const fingerprint = JSON.stringify(canonicalize({
|
||||
allowedSkillIds: solution.allowedSkillIds,
|
||||
requiredGroups: solution.requiredGroups,
|
||||
}));
|
||||
if (solutionFingerprints.has(fingerprint)) fail("AAS_TUNING_DUPLICATE_SOLUTION", `${entry.caseId}/${solution.solutionId}`);
|
||||
solutionFingerprints.add(fingerprint);
|
||||
const allowed = solution.allowedSkillIds || [];
|
||||
if (allowed.length === 0) fail("AAS_TUNING_EMPTY_SOLUTION", entry.caseId);
|
||||
for (const group of solution.requiredGroups || []) {
|
||||
if (!group.length || group.some((skillId) => !allowed.includes(skillId))) {
|
||||
fail("AAS_TUNING_REQUIRED_GROUP", `${entry.caseId}/${solution.solutionId}`);
|
||||
}
|
||||
}
|
||||
for (const skillId of allowed) {
|
||||
const skill = catalogById.get(skillId);
|
||||
if (!skill) {
|
||||
fail("AAS_TUNING_UNKNOWN_SKILL", `${entry.caseId}/${skillId}`);
|
||||
continue;
|
||||
}
|
||||
if (skill.risk && skill.risk !== "unknown" && !caseData.policy.allowedRisk.includes(skill.risk)) {
|
||||
fail("AAS_TUNING_RISK_VIOLATION", `${entry.caseId}/${skillId}/${skill.risk}`);
|
||||
}
|
||||
if (skill.plugin?.setup?.type === "manual" && caseData.policy.allowManualSetup !== true) {
|
||||
fail("AAS_TUNING_SETUP_VIOLATION", `${entry.caseId}/${skillId}`);
|
||||
}
|
||||
for (const target of caseData.targets) {
|
||||
if (skill.plugin?.targets?.[target.host] === "blocked") {
|
||||
fail("AAS_TUNING_HOST_VIOLATION", `${entry.caseId}/${skillId}/${target.host}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
requireApprovals(entry.caseId, caseData, goldData, "tuning");
|
||||
}
|
||||
|
||||
if (failures.length) {
|
||||
console.error(JSON.stringify({ ok: false, frozen, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
frozen,
|
||||
abstentionCases: abstentionIndex.cases.length,
|
||||
tuningCases: tuningIndex.cases.length,
|
||||
disjointFingerprints: heldOutFingerprints.size + abstentionFingerprints.size + tuningFingerprints.size,
|
||||
}, null, 2));
|
||||
+381
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { caseInclusionAssessment, macroAverage } from "../lib/metrics.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verificationRoot = path.resolve(here, "..", "..");
|
||||
const repositoryRoot = path.resolve(verificationRoot, "..", "..");
|
||||
const baselineRoot = path.join(verificationRoot, "baseline", "v1");
|
||||
const tuningRoot = path.join(baselineRoot, "benchmark", "tuning");
|
||||
const reviewsRoot = path.join(baselineRoot, "reviews");
|
||||
const auditPath = path.join(reviewsRoot, "tuning-gold-equivalence-audit.json");
|
||||
const failures = [];
|
||||
|
||||
function fail(code, detail) {
|
||||
failures.push({ code, detail });
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
} catch (error) {
|
||||
fail("AAS_TUNING_AUDIT_INVALID_JSON", `${path.relative(verificationRoot, file)}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function canonicalJson(value) {
|
||||
return JSON.stringify(canonicalize(value));
|
||||
}
|
||||
|
||||
function sha256(bytes) {
|
||||
return crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
function sha256File(file) {
|
||||
return sha256(fs.readFileSync(file));
|
||||
}
|
||||
|
||||
function pairDigest(caseData, goldData) {
|
||||
const reviewNeutralGold = { ...goldData, reviews: [] };
|
||||
return `sha256-${sha256(Buffer.from(canonicalJson({ case: caseData, gold: reviewNeutralGold })))}`;
|
||||
}
|
||||
|
||||
function decisionKey(caseId, skillId) {
|
||||
return `${caseId}\u0000${skillId}`;
|
||||
}
|
||||
|
||||
function decisionMap(report) {
|
||||
return new Map((report.decisions || []).map((entry) => [decisionKey(entry.caseId, entry.skillId), entry]));
|
||||
}
|
||||
|
||||
function mean(values) {
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
const audit = readJson(auditPath);
|
||||
if (!audit) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const diagnosticPath = path.join(verificationRoot, audit.diagnostic.path);
|
||||
if (!fs.existsSync(diagnosticPath)) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_MISSING", audit.diagnostic.path);
|
||||
}
|
||||
const diagnostic = fs.existsSync(diagnosticPath) ? readJson(diagnosticPath) : null;
|
||||
if (diagnostic && sha256File(diagnosticPath) !== audit.diagnostic.sha256) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_DIGEST", audit.diagnostic.path);
|
||||
}
|
||||
if (diagnostic) {
|
||||
if (diagnostic.reportType !== "aas-v1-tuning-diagnostic"
|
||||
|| diagnostic.benchmark?.tuningOnly !== true
|
||||
|| diagnostic.benchmark?.caseCount !== 60
|
||||
|| diagnostic.caseReports?.length !== 60
|
||||
|| diagnostic.caseReports.some((entry) => !entry.caseId?.startsWith("tuning."))) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_SCOPE", "The frozen input must contain exactly 60 tuning-only results.");
|
||||
}
|
||||
if ((diagnostic.benchmark?.roots || []).some((root) => !root.includes("/tuning/"))) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_ROOT", diagnostic.benchmark?.roots);
|
||||
}
|
||||
if (diagnostic.catalog?.digest !== audit.diagnostic.catalogDigest) {
|
||||
fail("AAS_TUNING_AUDIT_CATALOG_DIGEST", diagnostic.catalog?.digest);
|
||||
}
|
||||
}
|
||||
|
||||
const publicIndexPath = path.join(repositoryRoot, "data", "skills_index.json");
|
||||
if (sha256File(publicIndexPath) !== audit.baseline.publicSkillsIndexSha256) {
|
||||
fail("AAS_TUNING_AUDIT_PUBLIC_INDEX_DIGEST", "data/skills_index.json");
|
||||
}
|
||||
|
||||
const reports = new Map();
|
||||
for (const descriptor of audit.reviewReports || []) {
|
||||
const reportPath = path.join(verificationRoot, descriptor.path);
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_MISSING", descriptor.path);
|
||||
continue;
|
||||
}
|
||||
if (sha256File(reportPath) !== descriptor.sha256) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_DIGEST", descriptor.path);
|
||||
}
|
||||
const report = readJson(reportPath);
|
||||
if (!report) continue;
|
||||
if (report.reviewer?.identity !== descriptor.reviewer
|
||||
|| report.reviewer?.scorerImplementer !== false
|
||||
|| report.scope?.tuningOnly !== true
|
||||
|| report.scope?.heldOutContentsRead !== false
|
||||
|| report.scope?.abstentionLabelsRead !== false
|
||||
|| report.scope?.diagnosticSha256 !== audit.diagnostic.sha256
|
||||
|| report.scope?.assignedClaimCount !== report.decisions?.length
|
||||
|| descriptor.claimCount !== report.decisions?.length) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_SCOPE", descriptor.reviewer);
|
||||
}
|
||||
const counted = {
|
||||
ADD_TO_ALLOWED_EQUIVALENT: 0,
|
||||
REJECT_NOT_EQUIVALENT: 0,
|
||||
AMBIGUOUS_NEEDS_ADJUDICATION: 0,
|
||||
};
|
||||
const keys = new Set();
|
||||
for (const decision of report.decisions || []) {
|
||||
counted[decision.decision] += 1;
|
||||
const key = decisionKey(decision.caseId, decision.skillId);
|
||||
if (keys.has(key)) fail("AAS_TUNING_AUDIT_REVIEW_DUPLICATE_DECISION", `${descriptor.reviewer}/${key}`);
|
||||
keys.add(key);
|
||||
if (!decision.caseId.startsWith("tuning.")) fail("AAS_TUNING_AUDIT_REVIEW_NON_TUNING", key);
|
||||
if (decision.decision === "ADD_TO_ALLOWED_EQUIVALENT" && !decision.coherentSolution) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_ADD_WITHOUT_SOLUTION", key);
|
||||
}
|
||||
}
|
||||
if (canonicalJson(counted) !== canonicalJson(report.decisionCounts)) {
|
||||
fail("AAS_TUNING_AUDIT_REVIEW_COUNTS", descriptor.reviewer);
|
||||
}
|
||||
for (const evidence of report.skillEvidence || []) {
|
||||
const evidencePath = path.resolve(repositoryRoot, evidence.path);
|
||||
const relative = path.relative(path.join(repositoryRoot, "skills"), evidencePath);
|
||||
if (relative.startsWith("..") || path.isAbsolute(relative) || !fs.existsSync(evidencePath)) {
|
||||
fail("AAS_TUNING_AUDIT_SKILL_EVIDENCE_PATH", evidence.path);
|
||||
continue;
|
||||
}
|
||||
if (sha256File(evidencePath) !== evidence.sha256) {
|
||||
fail("AAS_TUNING_AUDIT_SKILL_EVIDENCE_DIGEST", evidence.path);
|
||||
}
|
||||
}
|
||||
reports.set(descriptor.reviewer, report);
|
||||
}
|
||||
|
||||
const alpha = reports.get("gold-equivalence-independent-alpha");
|
||||
const beta = reports.get("gold-equivalence-independent-beta");
|
||||
const adjudicator = reports.get("gold-equivalence-independent-adjudicator");
|
||||
const tiebreak = reports.get("gold-equivalence-independent-ci-tiebreak");
|
||||
const alphaKeys = new Set((alpha?.decisions || []).map((entry) => decisionKey(entry.caseId, entry.skillId)));
|
||||
const betaKeys = new Set((beta?.decisions || []).map((entry) => decisionKey(entry.caseId, entry.skillId)));
|
||||
if ([...alphaKeys].some((key) => betaKeys.has(key))) {
|
||||
fail("AAS_TUNING_AUDIT_PRIMARY_ASSIGNMENTS_OVERLAP", "Primary assignments must be disjoint.");
|
||||
}
|
||||
|
||||
const claimMap = new Map();
|
||||
for (const claim of audit.claims || []) {
|
||||
const key = decisionKey(claim.caseId, claim.skillId);
|
||||
if (claimMap.has(key)) fail("AAS_TUNING_AUDIT_DUPLICATE_CLAIM", key);
|
||||
claimMap.set(key, claim);
|
||||
}
|
||||
if (claimMap.size !== 19 || audit.claims?.filter((entry) => entry.decision === "ADD_TO_ALLOWED_EQUIVALENT").length !== 8) {
|
||||
fail("AAS_TUNING_AUDIT_CLAIM_COUNTS", { total: claimMap.size });
|
||||
}
|
||||
const primaryUnion = new Set([...alphaKeys, ...betaKeys]);
|
||||
if (canonicalJson([...primaryUnion].sort()) !== canonicalJson([...claimMap.keys()].sort())) {
|
||||
fail("AAS_TUNING_AUDIT_PRIMARY_COVERAGE", "The disjoint primary assignments must cover every omission claim exactly once.");
|
||||
}
|
||||
const adjudicatorMap = adjudicator ? decisionMap(adjudicator) : new Map();
|
||||
const tiebreakMap = tiebreak ? decisionMap(tiebreak) : new Map();
|
||||
const primaryMap = new Map([
|
||||
...[...(alpha ? decisionMap(alpha) : new Map())],
|
||||
...[...(beta ? decisionMap(beta) : new Map())],
|
||||
]);
|
||||
for (const [key, claim] of claimMap) {
|
||||
const primary = primaryMap.get(key);
|
||||
const adjudicated = adjudicatorMap.get(key);
|
||||
if (!primary || !adjudicated) {
|
||||
fail("AAS_TUNING_AUDIT_CLAIM_REVIEW_MISSING", key);
|
||||
continue;
|
||||
}
|
||||
if (claim.resolution === "independent-agreement" || claim.resolution === "independent-agreement-on-conservative-exact-gold") {
|
||||
if (primary.decision !== claim.decision || adjudicated.decision !== claim.decision) {
|
||||
fail("AAS_TUNING_AUDIT_AGREEMENT_MISMATCH", key);
|
||||
}
|
||||
} else if (claim.resolution === "ambiguity-adjudicated-reject") {
|
||||
if (primary.decision !== "AMBIGUOUS_NEEDS_ADJUDICATION"
|
||||
|| adjudicated.decision !== "REJECT_NOT_EQUIVALENT"
|
||||
|| claim.decision !== "REJECT_NOT_EQUIVALENT") {
|
||||
fail("AAS_TUNING_AUDIT_AMBIGUITY_RESOLUTION", key);
|
||||
}
|
||||
} else if (claim.resolution === "independent-tiebreak-with-dissent-retained") {
|
||||
const tieDecision = tiebreakMap.get(key);
|
||||
if (primary.decision !== "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
|| adjudicated.decision !== "REJECT_NOT_EQUIVALENT"
|
||||
|| tieDecision?.decision !== "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
|| claim.decision !== "ADD_TO_ALLOWED_EQUIVALENT") {
|
||||
fail("AAS_TUNING_AUDIT_TIEBREAK_RESOLUTION", key);
|
||||
}
|
||||
} else {
|
||||
fail("AAS_TUNING_AUDIT_UNKNOWN_RESOLUTION", `${key}/${claim.resolution}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (diagnostic) {
|
||||
const diagnosticClaims = new Set();
|
||||
let omissionCases = 0;
|
||||
for (const report of diagnostic.caseReports) {
|
||||
if (!(report.inclusionCount > 0 && report.inclusionPrecision < 1)) continue;
|
||||
omissionCases += 1;
|
||||
const claims = (audit.claims || []).filter((entry) => entry.caseId === report.caseId);
|
||||
if (claims.length !== report.inclusionCount - report.acceptedInclusionCount
|
||||
|| claims.some((entry) => !report.includedSkillIds.includes(entry.skillId))) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_CLAIM_MAPPING", report.caseId);
|
||||
}
|
||||
for (const claim of claims) diagnosticClaims.add(decisionKey(claim.caseId, claim.skillId));
|
||||
}
|
||||
if (omissionCases !== 17 || diagnosticClaims.size !== 19) {
|
||||
fail("AAS_TUNING_AUDIT_DIAGNOSTIC_OMISSION_COUNTS", { omissionCases, claims: diagnosticClaims.size });
|
||||
}
|
||||
}
|
||||
|
||||
const tuningManifest = readJson(path.join(tuningRoot, "manifest.json"));
|
||||
const tuningIndex = tuningManifest ? readJson(path.join(tuningRoot, tuningManifest.index)) : null;
|
||||
const indexById = new Map((tuningIndex?.cases || []).map((entry) => [entry.caseId, entry]));
|
||||
const changedCaseIds = new Set();
|
||||
for (const changed of audit.changedPairs || []) {
|
||||
if (changedCaseIds.has(changed.caseId)) fail("AAS_TUNING_AUDIT_DUPLICATE_CHANGED_PAIR", changed.caseId);
|
||||
changedCaseIds.add(changed.caseId);
|
||||
const descriptor = indexById.get(changed.caseId);
|
||||
if (!descriptor) {
|
||||
fail("AAS_TUNING_AUDIT_CHANGED_PAIR_UNKNOWN", changed.caseId);
|
||||
continue;
|
||||
}
|
||||
const caseData = readJson(path.join(tuningRoot, descriptor.inputPath));
|
||||
const goldData = readJson(path.join(tuningRoot, descriptor.goldPath));
|
||||
if (!caseData || !goldData) continue;
|
||||
const digest = pairDigest(caseData, goldData);
|
||||
if (digest !== changed.newPairDigest) fail("AAS_TUNING_AUDIT_CHANGED_PAIR_DIGEST", changed.caseId);
|
||||
if (changed.priorPairDigest === changed.newPairDigest) fail("AAS_TUNING_AUDIT_CHANGED_PAIR_NO_CHANGE", changed.caseId);
|
||||
if (goldData.provenance?.source !== audit.provenance.source
|
||||
|| goldData.provenance?.version !== audit.provenance.version
|
||||
|| goldData.provenance?.reviewedAt !== audit.provenance.reviewedAt) {
|
||||
fail("AAS_TUNING_AUDIT_GOLD_PROVENANCE", changed.caseId);
|
||||
}
|
||||
const solution = (goldData.acceptedSolutions || []).find((entry) => entry.solutionId === changed.solutionId);
|
||||
if (!solution) fail("AAS_TUNING_AUDIT_SOLUTION_MISSING", `${changed.caseId}/${changed.solutionId}`);
|
||||
const approved = new Set((goldData.reviews || [])
|
||||
.filter((entry) => entry.decision === "approved" && entry.reviewedDigest === digest)
|
||||
.map((entry) => entry.reviewer));
|
||||
if (canonicalJson([...approved].sort()) !== canonicalJson([...changed.approvingReviewers].sort())) {
|
||||
fail("AAS_TUNING_AUDIT_PAIR_APPROVALS", changed.caseId);
|
||||
}
|
||||
const addedClaims = (audit.claims || []).filter((entry) => (
|
||||
entry.caseId === changed.caseId && entry.decision === "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
));
|
||||
if (!solution || addedClaims.some((entry) => !solution.allowedSkillIds.includes(entry.skillId))) {
|
||||
fail("AAS_TUNING_AUDIT_SOLUTION_CLAIMS", changed.caseId);
|
||||
}
|
||||
for (const reviewer of changed.approvingReviewers) {
|
||||
const report = reports.get(reviewer);
|
||||
for (const claim of addedClaims) {
|
||||
const decision = decisionMap(report || { decisions: [] }).get(decisionKey(claim.caseId, claim.skillId));
|
||||
if (decision?.decision !== "ADD_TO_ALLOWED_EQUIVALENT"
|
||||
|| canonicalJson(decision.coherentSolution) !== canonicalJson(solution)) {
|
||||
fail("AAS_TUNING_AUDIT_EXACT_GOLD_ATTESTATION", `${changed.caseId}/${reviewer}/${claim.skillId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (changedCaseIds.size !== 7) fail("AAS_TUNING_AUDIT_CHANGED_PAIR_COUNT", changedCaseIds.size);
|
||||
for (const descriptor of tuningIndex?.cases || []) {
|
||||
const gold = readJson(path.join(tuningRoot, descriptor.goldPath));
|
||||
if (gold?.provenance?.source === audit.provenance.source && !changedCaseIds.has(descriptor.caseId)) {
|
||||
fail("AAS_TUNING_AUDIT_UNDECLARED_GOLD_CHANGE", descriptor.caseId);
|
||||
}
|
||||
}
|
||||
|
||||
if (diagnostic && tuningIndex) {
|
||||
const reportsAfter = diagnostic.caseReports.map((report) => {
|
||||
const descriptor = indexById.get(report.caseId);
|
||||
const gold = descriptor ? readJson(path.join(tuningRoot, descriptor.goldPath)) : null;
|
||||
if (!gold) return { ...report, acceptedInclusionCount: 0, inclusionPrecision: null };
|
||||
const inclusion = caseInclusionAssessment(report.includedSkillIds, gold.acceptedSolutions);
|
||||
return {
|
||||
...report,
|
||||
acceptedInclusionCount: inclusion.acceptedCount,
|
||||
inclusionPrecision: inclusion.precision,
|
||||
matchedSolutionId: inclusion.matchedSolutionId,
|
||||
};
|
||||
});
|
||||
const intents = [...new Set(reportsAfter.map((entry) => entry.intent))].sort();
|
||||
const perIntent = intents.map((intent) => {
|
||||
const entries = reportsAfter.filter((entry) => entry.intent === intent);
|
||||
const acceptedInclusions = entries.reduce((sum, entry) => sum + entry.acceptedInclusionCount, 0);
|
||||
const totalInclusions = entries.reduce((sum, entry) => sum + entry.inclusionCount, 0);
|
||||
return {
|
||||
intent,
|
||||
verifiedCoverage: entries.filter((entry) => entry.verified).length / entries.length,
|
||||
inclusionPrecision: acceptedInclusions / totalInclusions,
|
||||
criticalGoalCoverage: mean(entries.map((entry) => entry.criticalGoalCoverage)),
|
||||
nonCriticalGoalCoverage: mean(entries.map((entry) => entry.nonCriticalGoalCoverage)),
|
||||
acceptedInclusions,
|
||||
totalInclusions,
|
||||
hardPolicyViolations: entries.reduce((sum, entry) => sum + entry.hardPolicyViolationCount, 0),
|
||||
};
|
||||
});
|
||||
const afterMacro = {
|
||||
verifiedCoverage: macroAverage(perIntent, "verifiedCoverage"),
|
||||
inclusionPrecision: macroAverage(perIntent, "inclusionPrecision"),
|
||||
criticalGoalCoverage: macroAverage(perIntent, "criticalGoalCoverage"),
|
||||
nonCriticalGoalCoverage: macroAverage(perIntent, "nonCriticalGoalCoverage"),
|
||||
};
|
||||
const afterPerIntentPrecision = Object.fromEntries(perIntent.map((entry) => [entry.intent, entry.inclusionPrecision]));
|
||||
const beforePerIntentPrecision = Object.fromEntries(diagnostic.perIntent.map((entry) => [entry.intent, entry.inclusionPrecision]));
|
||||
if (canonicalJson(audit.metricImplications.before.macro) !== canonicalJson(diagnostic.macro)
|
||||
|| canonicalJson(audit.metricImplications.before.perIntentInclusionPrecision) !== canonicalJson(beforePerIntentPrecision)
|
||||
|| canonicalJson(audit.metricImplications.after.macro) !== canonicalJson(afterMacro)
|
||||
|| canonicalJson(audit.metricImplications.after.perIntentInclusionPrecision) !== canonicalJson(afterPerIntentPrecision)) {
|
||||
fail("AAS_TUNING_AUDIT_METRIC_IMPLICATIONS", { afterMacro, afterPerIntentPrecision });
|
||||
}
|
||||
const acceptedBefore = diagnostic.perIntent.reduce((sum, entry) => sum + entry.acceptedInclusions, 0);
|
||||
const acceptedAfter = perIntent.reduce((sum, entry) => sum + entry.acceptedInclusions, 0);
|
||||
const expectedDelta = {
|
||||
macroInclusionPrecision: afterMacro.inclusionPrecision - diagnostic.macro.inclusionPrecision,
|
||||
verifiedCoverage: afterMacro.verifiedCoverage - diagnostic.macro.verifiedCoverage,
|
||||
criticalGoalCoverage: afterMacro.criticalGoalCoverage - diagnostic.macro.criticalGoalCoverage,
|
||||
nonCriticalGoalCoverage: afterMacro.nonCriticalGoalCoverage - diagnostic.macro.nonCriticalGoalCoverage,
|
||||
acceptedInclusions: acceptedAfter - acceptedBefore,
|
||||
};
|
||||
if (canonicalJson(expectedDelta) !== canonicalJson(audit.metricImplications.delta)) {
|
||||
fail("AAS_TUNING_AUDIT_METRIC_DELTA", expectedDelta);
|
||||
}
|
||||
const hardPolicyViolations = perIntent.reduce((sum, entry) => sum + entry.hardPolicyViolations, 0);
|
||||
const postAuditGates = {
|
||||
hardPolicyViolations: hardPolicyViolations === 0,
|
||||
inclusionPrecision: afterMacro.inclusionPrecision >= diagnostic.thresholds.inclusionPrecision
|
||||
&& perIntent.every((entry) => entry.inclusionPrecision >= diagnostic.thresholds.inclusionPrecision),
|
||||
verifiedCoverage: afterMacro.verifiedCoverage >= diagnostic.thresholds.verifiedCoverage
|
||||
&& perIntent.every((entry) => entry.verifiedCoverage >= diagnostic.thresholds.verifiedCoverage),
|
||||
};
|
||||
if (canonicalJson(postAuditGates) !== canonicalJson(audit.metricImplications.postAuditGates)) {
|
||||
fail("AAS_TUNING_AUDIT_POST_GATES", postAuditGates);
|
||||
}
|
||||
const changedPrecisionCases = reportsAfter
|
||||
.filter((entry, index) => entry.inclusionPrecision !== diagnostic.caseReports[index].inclusionPrecision)
|
||||
.map((entry) => entry.caseId)
|
||||
.sort();
|
||||
if (canonicalJson(changedPrecisionCases) !== canonicalJson([...changedCaseIds].sort())) {
|
||||
fail("AAS_TUNING_AUDIT_METRIC_CHANGE_SCOPE", changedPrecisionCases);
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(JSON.stringify({ ok: false, failures }, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
auditId: audit.auditId,
|
||||
omissionCases: audit.scope.omissionCaseCount,
|
||||
claims: audit.scope.omittedInclusionCount,
|
||||
changedPairs: audit.scope.changedPairCount,
|
||||
independentReviewers: reports.size,
|
||||
macroInclusionPrecisionBefore: audit.metricImplications.before.macro.inclusionPrecision,
|
||||
macroInclusionPrecisionAfter: audit.metricImplications.after.macro.inclusionPrecision,
|
||||
postAuditGates: audit.metricImplications.postAuditGates,
|
||||
}, null, 2));
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { digestJson, sha256 } from "../lib/canonical.mjs";
|
||||
import { snapshotZones } from "../lib/fs-evidence.mjs";
|
||||
import { selfTestObserver } from "../lib/observer.mjs";
|
||||
import { executableDigest, loadReceiptValidator, SUITE_IDS, writeCanonicalReceipt } from "../lib/receipt.mjs";
|
||||
import { installCandidate, isolatedZones, systemIdentity } from "../lib/runtime.mjs";
|
||||
import {
|
||||
packageSuite, prepareRuntimeCache, suite, verifyAdapters, verifyEntrypoints,
|
||||
verifyFuzz, verifyHostile, verifyLegacy, verifyMcp, verifyProperty,
|
||||
} from "../lib/suites.mjs";
|
||||
import { inspectPackageTarball } from "../lib/tarball.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const verifierRoot = path.resolve(here, "..");
|
||||
const verificationRoot = path.resolve(verifierRoot, "..");
|
||||
const baselineRoot = path.join(verificationRoot, "baseline", "v1");
|
||||
|
||||
function args(values) {
|
||||
const out = {};
|
||||
for (let i = 2; i < values.length; i += 2) {
|
||||
if (!values[i].startsWith("--") || values[i + 1] === undefined) throw new Error(`Invalid argument: ${values[i]}`);
|
||||
out[values[i].slice(2)] = values[i + 1];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function required(value, name, pattern = /./) {
|
||||
if (!value || !pattern.test(value)) throw new Error(`--${name} is required or invalid`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function readJson(file) { return JSON.parse(fs.readFileSync(file, "utf8")); }
|
||||
function failedSuite(id, error) {
|
||||
const evidence = { code: error.code || "AAS_VERIFIER_INTERNAL", detailDigest: digestJson({ name: error.name, message: error.message }) };
|
||||
return { id, status: "failed", executions: 0, failures: 1, evidenceSha256: digestJson(evidence), evidence };
|
||||
}
|
||||
|
||||
function transactionSuite(file, manifest, validator) {
|
||||
if (!file || !fs.existsSync(file)) throw Object.assign(new Error("OS-level transaction evidence is required"), { code: "AAS_VERIFIER_TRANSACTION_EVIDENCE_MISSING" });
|
||||
const value = readJson(file);
|
||||
if (!validator(value)) throw Object.assign(new Error("Transaction evidence schema failed"), { code: "AAS_VERIFIER_TRANSACTION_EVIDENCE_SCHEMA" });
|
||||
const fault = new Set(value.faultBoundaryClasses || []);
|
||||
const race = new Set(value.raceClasses || []);
|
||||
for (const item of manifest.faultBoundaryClasses) if (!fault.has(item)) throw Object.assign(new Error(`Fault boundary not covered: ${item}`), { code: "AAS_VERIFIER_FAULT_BOUNDARY_MISSING" });
|
||||
for (const item of manifest.raceClasses) if (!race.has(item)) throw Object.assign(new Error(`Race class not covered: ${item}`), { code: "AAS_VERIFIER_RACE_CLASS_MISSING" });
|
||||
if (value.testMode === true || value.mocked === true || value.productionBinary !== true) throw Object.assign(new Error("Transaction evidence is not production black-box evidence"), { code: "AAS_VERIFIER_TRANSACTION_NOT_BLACK_BOX" });
|
||||
if (value.partialStates !== 0 || value.unmanagedMutations !== 0 || value.hardPolicyViolations !== 0) throw Object.assign(new Error("Transaction safety invariant failed"), { code: "AAS_VERIFIER_TRANSACTION_INVARIANT" });
|
||||
return suite("transaction", value, value.executions || 0);
|
||||
}
|
||||
|
||||
const options = args(process.argv);
|
||||
const tarball = path.resolve(required(options.tarball, "tarball"));
|
||||
const candidateCommit = required(options["candidate-commit"], "candidate-commit", /^[a-f0-9]{40}$/);
|
||||
const verifierCommit = required(options["verifier-commit"], "verifier-commit", /^[a-f0-9]{40}$/);
|
||||
const jobId = required(options["job-id"], "job-id");
|
||||
const output = path.resolve(required(options.out, "out"));
|
||||
const workRoot = path.resolve(required(options["work-root"], "work-root"));
|
||||
const runtimeMatrix = readJson(path.join(baselineRoot, "runtime-matrix.json"));
|
||||
const job = runtimeMatrix.jobs.find((entry) => entry.id === jobId);
|
||||
if (!job) throw new Error(`Unknown job: ${jobId}`);
|
||||
if (process.platform !== job.os.replace("macos", "darwin").replace("windows", "win32")) throw new Error(`Job/platform mismatch: ${jobId}/${process.platform}`);
|
||||
if (process.arch !== job.architecture || process.version !== `v${job.nodePatch}`) throw new Error(`Frozen runtime identity mismatch: ${process.arch}/${process.version}`);
|
||||
|
||||
fs.mkdirSync(workRoot, { recursive: true, mode: 0o700 });
|
||||
const evidenceDir = path.join(workRoot, "observer-evidence");
|
||||
const zones = isolatedZones(path.join(workRoot, "zones"));
|
||||
const zoneBefore = snapshotZones(zones);
|
||||
const manifest = readJson(path.join(baselineRoot, "verifier-manifest.json"));
|
||||
const freeze = readJson(path.join(baselineRoot, "freeze-manifest.json"));
|
||||
const budgets = readJson(path.join(baselineRoot, "budgets.json"));
|
||||
const hostileManifest = readJson(path.join(baselineRoot, "hostile", "manifest.json"));
|
||||
const transactionValidator = loadReceiptValidator(path.join(verificationRoot, "schemas", "product-transaction-evidence.schema.json"));
|
||||
const inspection = inspectPackageTarball(tarball);
|
||||
const runtime = await installCandidate(tarball, path.join(workRoot, "candidate-install"));
|
||||
const tarballBytes = fs.readFileSync(tarball);
|
||||
const observer = await selfTestObserver({ cwd: zones.tmp, env: process.env, zones, evidenceDir });
|
||||
const runtimePromotion = await prepareRuntimeCache(runtime, tarballBytes, inspection.sha512, zones.cache);
|
||||
|
||||
const suites = [];
|
||||
const failures = [];
|
||||
const run = async (id, action) => {
|
||||
try { suites.push(await action()); }
|
||||
catch (error) {
|
||||
suites.push(failedSuite(id, error));
|
||||
failures.push({ code: /^AAS_VERIFIER_/.test(error.code || "") ? error.code : "AAS_VERIFIER_INTERNAL", suite: id, detailDigest: digestJson({ name: error.name, message: error.message }) });
|
||||
}
|
||||
};
|
||||
await run("package", () => packageSuite(inspection, runtime));
|
||||
await run("entrypoints", () => verifyEntrypoints(runtime, zones));
|
||||
await run("mcp", () => verifyMcp(runtime, zones, evidenceDir));
|
||||
await run("property", () => verifyProperty(runtime, budgets, runtimeMatrix.jobs.indexOf(job), verifierRoot));
|
||||
await run("fuzz", () => verifyFuzz(runtime, budgets, runtimeMatrix.jobs.indexOf(job), verifierRoot));
|
||||
await run("hostile", () => verifyHostile(runtime, zones, evidenceDir, hostileManifest, path.join(baselineRoot, "hostile"), verifierRoot));
|
||||
await run("legacy", () => verifyLegacy(runtime, zones, verifierRoot, path.join(baselineRoot, "legacy", "14.6.0")));
|
||||
await run("transaction", () => transactionSuite(options["transaction-evidence"], manifest, transactionValidator));
|
||||
await run("adapters", () => verifyAdapters(runtime, zones, path.join(baselineRoot, "host-adapters"), inspection.sha512, runtimePromotion.identity.closureDigest));
|
||||
for (const id of SUITE_IDS) if (!suites.some((entry) => entry.id === id)) suites.push(failedSuite(id, Object.assign(new Error("Suite did not execute"), { code: "AAS_VERIFIER_SUITE_MISSING" })));
|
||||
|
||||
const mcp = suites.find((entry) => entry.id === "mcp");
|
||||
const canonicalSha = mcp?.evidence?.canonicalResponseDigest || digestJson(null);
|
||||
const { jobId: _jobId, ...identity } = systemIdentity(job);
|
||||
const mcpBefore = mcp?.evidence?.before || Object.fromEntries(Object.keys(zones).map((name) => [name, zoneBefore[name].digest]));
|
||||
const mcpAfter = mcp?.evidence?.after || mcpBefore;
|
||||
const receipt = {
|
||||
schemaVersion: 1,
|
||||
receiptVersion: "1.0.0",
|
||||
status: failures.length ? "failed" : "passed",
|
||||
job: { id: job.id, workflowRunId: process.env.GITHUB_RUN_ID || "1", workflowRunAttempt: process.env.GITHUB_RUN_ATTEMPT || "1" },
|
||||
candidate: {
|
||||
commit: candidateCommit, package: runtime.manifest.name, version: runtime.manifest.version,
|
||||
tarballBytes: inspection.bytes, tarballSha256: inspection.sha256, tarballSha512: inspection.sha512,
|
||||
packManifestSha256: digestJson(inspection.entries), installTreeSha256: runtime.treeDigest,
|
||||
},
|
||||
verifier: { version: "1.0.0", commit: verifierCommit, rootDigest: freeze.rootDigest, contractDigest: digestJson(manifest), owner: "aas-v1-independent-verifier" },
|
||||
environment: { ...identity, nodeExecutableSha256: executableDigest() },
|
||||
observer: { contractVersion: observer.contractVersion, backend: observer.backend, selfTestDigest: observer.selfTestDigest, networkSentinels: observer.observedNetworkSentinels, writeSentinels: observer.observedWriteSentinels, overflow: false, ambiguousLineage: false },
|
||||
zones: Object.fromEntries(Object.keys(zones).map((name) => [name, { beforeSha256: mcpBefore[name], afterSha256: mcpAfter[name], persistentWriteCount: 0 }])),
|
||||
suites: suites.sort((a, b) => SUITE_IDS.indexOf(a.id) - SUITE_IDS.indexOf(b.id)),
|
||||
canonicalPayload: { sha256: canonicalSha, excludedFields: ["timestamp", "correlationId", "localizedMessage", "diagnostics"], sampleCount: 60 },
|
||||
failures,
|
||||
};
|
||||
writeCanonicalReceipt(output, receipt);
|
||||
process.stdout.write(`${JSON.stringify({ ok: failures.length === 0, output, failures })}\n`);
|
||||
if (failures.length) process.exit(1);
|
||||
@@ -0,0 +1,128 @@
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const MASK = (1n << 64n) - 1n;
|
||||
const rotateLeft = (value, amount) => ((value << BigInt(amount)) | (value >> (64n - BigInt(amount)))) & MASK;
|
||||
function deriveState(rootSeedHex, namespace) {
|
||||
const key = Buffer.from(rootSeedHex, "hex");
|
||||
const bytes = crypto.createHmac("sha256", key).update(namespace).digest();
|
||||
return [0, 8, 16, 24].map((offset) => bytes.readBigUInt64BE(offset));
|
||||
}
|
||||
function nextUint64(state) {
|
||||
const result = (rotateLeft((state[1] * 5n) & MASK, 7) * 9n) & MASK;
|
||||
const temporary = (state[1] << 17n) & MASK;
|
||||
state[2] ^= state[0]; state[3] ^= state[1]; state[1] ^= state[2]; state[0] ^= state[3];
|
||||
state[2] ^= temporary; state[3] = rotateLeft(state[3], 45);
|
||||
return result;
|
||||
}
|
||||
const sample = (state, upper) => Number(nextUint64(state) % BigInt(upper));
|
||||
|
||||
async function main(input) {
|
||||
const core = require(path.join(input.packageRoot, "tools/lib/aas-v1"));
|
||||
const mcp = require(path.join(input.packageRoot, "tools/lib/aas-v1/mcp"));
|
||||
const { McpServer, parseStrictJsonLine } = mcp;
|
||||
const catalog = core.loadBundledCatalog({ root: input.packageRoot });
|
||||
const server = new McpServer({ root: input.packageRoot, catalog });
|
||||
await server.handle({ jsonrpc: "2.0", id: 0, method: "initialize", params: { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "fuzz-driver", version: "1" } } });
|
||||
await server.handle({ jsonrpc: "2.0", method: "notifications/initialized", params: {} });
|
||||
const budget = input.budget;
|
||||
const summary = {};
|
||||
let total = 0;
|
||||
let crashes = 0;
|
||||
let canaryLeaks = 0;
|
||||
|
||||
const expectParse = (bytes, accepted) => {
|
||||
let parsed = false;
|
||||
try { parseStrictJsonLine(bytes); parsed = true; } catch {}
|
||||
if (parsed !== accepted) throw new Error(`parser expectation mismatch: accepted=${accepted}`);
|
||||
};
|
||||
|
||||
for (const stratum of budget.parserAndMcpFuzz.strata) {
|
||||
let executions = 0;
|
||||
let accepted = 0;
|
||||
let rejected = 0;
|
||||
let serverExecutions = 0;
|
||||
for (let index = 0; index < stratum.executions; index += 1) {
|
||||
if (index % input.jobCount !== input.jobIndex) continue;
|
||||
const namespace = `aas.v1/parser-and-mcp-fuzz/${stratum.id}/execution/${index}`;
|
||||
const state = deriveState(budget.prng.rootSeed, namespace);
|
||||
const variant = sample(state, 6);
|
||||
try {
|
||||
if (stratum.id === "mcp-framing-json") {
|
||||
const cases = [
|
||||
[Buffer.from('{"jsonrpc":"2.0","id":1,"id":2}'), false],
|
||||
[Buffer.from("[1,2]"), false],
|
||||
[Buffer.from("null"), false],
|
||||
[Buffer.from('{"jsonrpc":"2.0"} trailing'), false],
|
||||
[Buffer.from([0xff, 0xfe]), false],
|
||||
[Buffer.from('{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'), true],
|
||||
];
|
||||
expectParse(...cases[variant]);
|
||||
if (cases[variant][1]) accepted += 1; else rejected += 1;
|
||||
} else if (stratum.id === "schema-depth-size") {
|
||||
const excessive = variant % 2 === 0;
|
||||
const bytes = excessive
|
||||
? Buffer.from(`{"x":${"[".repeat(17)}0${"]".repeat(17)}}`)
|
||||
: Buffer.from(`{"x":${"[".repeat(15)}0${"]".repeat(15)}}`);
|
||||
expectParse(bytes, !excessive);
|
||||
if (excessive) rejected += 1; else accepted += 1;
|
||||
} else if (stratum.id === "search-query-unicode") {
|
||||
const excessive = variant % 3 === 0;
|
||||
const query = `${"🧪".repeat(excessive ? 257 : 32)}${String.fromCodePoint(0x61 + variant)}`;
|
||||
const request = { jsonrpc: "2.0", id: index + 1, method: "tools/call", params: { name: "search_skills", arguments: { query, limit: 1 } } };
|
||||
expectParse(Buffer.from(JSON.stringify(request)), true);
|
||||
accepted += 1;
|
||||
if (index % 100 === 0) {
|
||||
const response = await server.handle(request);
|
||||
const serialized = JSON.stringify(response);
|
||||
if (serialized.includes("AAS_SECRET_CANARY_DO_NOT_EMIT_")) canaryLeaks += 1;
|
||||
serverExecutions += 1;
|
||||
}
|
||||
} else if (stratum.id === "resource-uri-get-skill") {
|
||||
const invalid = variant % 2 === 0;
|
||||
const request = { jsonrpc: "2.0", id: index + 1, method: "resources/read", params: { uri: invalid ? "aas://skills/%2e%2e%2fsecret" : "aas://skills/frontend-design" } };
|
||||
expectParse(Buffer.from(JSON.stringify(request)), true);
|
||||
accepted += 1;
|
||||
if (index % 100 === 0) {
|
||||
const response = await server.handle(request);
|
||||
if (invalid && !response.error) throw new Error("invalid resource URI accepted");
|
||||
serverExecutions += 1;
|
||||
}
|
||||
} else if (stratum.id === "manifest-plan-parsers") {
|
||||
const manifest = { schemaVersion: 1, name: "fuzz", catalog: { package: "agentic-awesome-skills", version: "14.6.0", integrity: `sha256-${"1".repeat(64)}` }, targets: [{ host: variant % 2 ? "codex" : "claude", scope: "project" }], intent: { goals: ["build"] }, policy: { allowedRisk: ["safe"], requireKnownSource: true, allowManualSetup: false }, skills: variant % 3 ? [{ id: "frontend-design" }] : [{ id: "../escape" }] };
|
||||
const validation = core.stack.validateManifest(manifest);
|
||||
if (variant % 3 === 0 && validation.ok) throw new Error("unsafe manifest accepted");
|
||||
if (variant % 3 !== 0 && !validation.ok) throw new Error("valid manifest rejected");
|
||||
if (validation.ok) accepted += 1; else rejected += 1;
|
||||
} else if (stratum.id === "timeout-result-limits") {
|
||||
const invalid = variant % 2 === 0;
|
||||
const request = { jsonrpc: "2.0", id: index + 1, method: "tools/call", params: { name: "search_skills", arguments: { query: "react", limit: invalid ? 51 : 1 } } };
|
||||
expectParse(Buffer.from(JSON.stringify(request)), true);
|
||||
accepted += 1;
|
||||
if (index % 100 === 0) {
|
||||
const response = await server.handle(request);
|
||||
const text = response.result?.content?.[0]?.text || "";
|
||||
if (invalid && !text.includes("AAS_INPUT_LIMIT_INVALID")) throw new Error("result limit was not enforced");
|
||||
serverExecutions += 1;
|
||||
}
|
||||
} else throw new Error(`unknown fuzz stratum: ${stratum.id}`);
|
||||
} catch (error) {
|
||||
crashes += 1;
|
||||
throw error;
|
||||
}
|
||||
executions += 1;
|
||||
total += 1;
|
||||
}
|
||||
summary[stratum.id] = { executions, accepted, rejected, serverExecutions };
|
||||
}
|
||||
return { schemaVersion: 1, ok: crashes === 0 && canaryLeaks === 0, total, crashes, canaryLeaks, summary };
|
||||
}
|
||||
|
||||
const input = JSON.parse(fs.readFileSync(0, "utf8"));
|
||||
main(input).then((result) => process.stdout.write(`${JSON.stringify(result)}\n`)).catch((error) => {
|
||||
process.stderr.write(`${error.stack || error.message}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
"use strict";
|
||||
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
async function main(input) {
|
||||
const core = require(path.join(input.packageRoot, "tools/lib/aas-v1"));
|
||||
const mcp = require(path.join(input.packageRoot, "tools/lib/aas-v1/mcp"));
|
||||
const manifest = input.manifest;
|
||||
const corpusRoot = input.corpusRoot;
|
||||
const catalog = core.loadBundledCatalog({ root: input.packageRoot });
|
||||
let archiveExploitRejected = 0;
|
||||
let archiveControlsAccepted = 0;
|
||||
let inputExploitRejected = 0;
|
||||
let inputControlsAccepted = 0;
|
||||
const parserBoundaryClasses = new Set(["malformed-mcp-framing", "malformed-json", "duplicate-json-key", "invalid-utf8", "request-byte-limit", "json-depth-limit"]);
|
||||
|
||||
for (const corpusCase of manifest.classes) {
|
||||
const pairs = [["exploit", corpusCase.exploit], ["boundaryControl", corpusCase.boundaryControl]];
|
||||
if (corpusCase.surface === "archive") {
|
||||
for (const [kind, fixture] of pairs) {
|
||||
const bytes = fs.readFileSync(path.resolve(corpusRoot, fixture.path));
|
||||
let accepted = false;
|
||||
try {
|
||||
core.cache.parsePackageArchive(bytes, { limits: manifest.fixtureContract.archive });
|
||||
accepted = true;
|
||||
} catch {}
|
||||
if (kind === "exploit" && accepted) throw new Error(`${corpusCase.classId}: archive exploit accepted`);
|
||||
if (kind === "boundaryControl" && !accepted) throw new Error(`${corpusCase.classId}: archive boundary control rejected`);
|
||||
if (kind === "exploit") archiveExploitRejected += 1;
|
||||
else archiveControlsAccepted += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const [kind, fixture] of pairs) {
|
||||
const bytes = fs.readFileSync(path.resolve(corpusRoot, fixture.path));
|
||||
let parsed;
|
||||
let rejected = false;
|
||||
try { parsed = mcp.parseStrictJsonLine(bytes); } catch { rejected = true; }
|
||||
if (!rejected && !parserBoundaryClasses.has(corpusCase.classId)) {
|
||||
const server = new mcp.McpServer({ root: input.packageRoot, catalog });
|
||||
await server.handle({ jsonrpc: "2.0", id: -1, method: "initialize", params: { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "hostile-driver", version: "1" } } });
|
||||
await server.handle({ jsonrpc: "2.0", method: "notifications/initialized", params: {} });
|
||||
const response = await server.handle(parsed);
|
||||
const payload = response?.result?.structuredContent;
|
||||
rejected = Boolean(response?.error || response?.result?.isError || payload?.ok === false);
|
||||
}
|
||||
if (kind === "exploit" && !rejected) throw new Error(`${corpusCase.classId}: input exploit accepted`);
|
||||
if (kind === "boundaryControl" && rejected) throw new Error(`${corpusCase.classId}: input boundary control rejected`);
|
||||
if (kind === "exploit") inputExploitRejected += 1;
|
||||
else inputControlsAccepted += 1;
|
||||
}
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
ok: true,
|
||||
executions: manifest.classes.length * 2,
|
||||
archiveExploitRejected,
|
||||
archiveControlsAccepted,
|
||||
inputExploitRejected,
|
||||
inputControlsAccepted,
|
||||
};
|
||||
}
|
||||
|
||||
const input = JSON.parse(fs.readFileSync(0, "utf8"));
|
||||
main(input).then((result) => process.stdout.write(`${JSON.stringify(result)}\n`)).catch((error) => {
|
||||
process.stderr.write(`${error.stack || error.message}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
canonicalize,
|
||||
normalizeText,
|
||||
parseTrace,
|
||||
treeDigest,
|
||||
treeEntries,
|
||||
} from "../../baseline/v1/legacy/14.6.0/corpus-lib.mjs";
|
||||
|
||||
function writeFixtureFile(file, content) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, content, { encoding: "utf8", mode: 0o600 });
|
||||
}
|
||||
|
||||
function managedState(target) {
|
||||
writeFixtureFile(path.join(target, "frontend-design", "SKILL.md"), "legacy managed bytes\n");
|
||||
writeFixtureFile(path.join(target, "removed-managed", "SKILL.md"), "stale managed bytes\n");
|
||||
writeFixtureFile(path.join(target, "unmanaged-sentinel", "KEEP.txt"), "unmanaged sentinel\n");
|
||||
writeFixtureFile(path.join(target, ".antigravity-install-manifest.json"), `${JSON.stringify({ schemaVersion: 1, updatedAt: "2026-01-01T00:00:00.000Z", entries: ["frontend-design", "removed-managed"] }, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function setupCase(input, caseData, caseRoot, fixtureDigest) {
|
||||
fs.rmSync(caseRoot, { recursive: true, force: true });
|
||||
const home = path.join(caseRoot, "home");
|
||||
const tmp = path.join(caseRoot, "tmp");
|
||||
const cwd = path.join(caseRoot, "workspace");
|
||||
const targets = path.join(caseRoot, "targets");
|
||||
const harness = path.join(caseRoot, "harness");
|
||||
const outside = path.join(caseRoot, "outside");
|
||||
for (const directory of [home, tmp, cwd, targets, harness, outside]) fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
const absoluteTarget = path.join(targets, "absolute");
|
||||
const missingTarget = path.join(targets, "missing");
|
||||
const symlinkTarget = path.join(targets, "symlink");
|
||||
if (["stale-managed-with-unmanaged-sentinel", "existing-managed-and-unmanaged"].includes(caseData.fixtureState)) managedState(absoluteTarget);
|
||||
if (caseData.fixtureState === "symlink-target") {
|
||||
const outsideTarget = path.join(outside, "target");
|
||||
fs.mkdirSync(outsideTarget, { recursive: true });
|
||||
fs.symlinkSync(outsideTarget, symlinkTarget, process.platform === "win32" ? "junction" : "dir");
|
||||
}
|
||||
// macOS canonicalizes /tmp to /private/tmp in process.cwd()/path.resolve.
|
||||
// Normalize both spellings before comparison so the frozen differential is
|
||||
// about candidate behavior, not the runner's symlinked temporary root.
|
||||
const roots = [
|
||||
[caseRoot, "<CASE_ROOT>"],
|
||||
[input.runtimeRoot, "<RUNTIME>"],
|
||||
[input.corpusRoot, "<CORPUS_ROOT>"],
|
||||
];
|
||||
const replacements = [...new Map(roots.flatMap(([root, token]) => {
|
||||
let canonical = root;
|
||||
try { canonical = fs.realpathSync(root); } catch {}
|
||||
return [[root, token], [canonical, token]];
|
||||
}).map((entry) => [entry[0], entry])).values()].sort((left, right) => right[0].length - left[0].length);
|
||||
const args = caseData.args.map((arg) => arg.replace("{{ABSOLUTE_TARGET}}", absoluteTarget).replace("{{MISSING_TARGET}}", missingTarget).replace("{{SYMLINK_TARGET}}", symlinkTarget));
|
||||
const trace = path.join(harness, "fake-git.jsonl");
|
||||
const networkTrace = path.join(harness, "network.jsonl");
|
||||
const env = {
|
||||
PATH: `${path.join(input.corpusRoot, "bin")}${path.delimiter}${process.env.PATH || ""}`,
|
||||
...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
|
||||
...(process.env.WINDIR ? { WINDIR: process.env.WINDIR } : {}),
|
||||
...(process.env.COMSPEC ? { COMSPEC: process.env.COMSPEC } : {}),
|
||||
...(process.env.PATHEXT ? { PATHEXT: process.env.PATHEXT } : {}),
|
||||
HOME: home, USERPROFILE: home, TMPDIR: tmp, TMP: tmp, TEMP: tmp,
|
||||
AAS_LEGACY_FIXTURE_REPO: path.join(input.corpusRoot, "fixture-repository"),
|
||||
AAS_LEGACY_FIXTURE_DIGEST: fixtureDigest,
|
||||
AAS_FAKE_GIT_TRACE: trace,
|
||||
AAS_FAKE_GIT_ALLOWED_ROOT: tmp,
|
||||
AAS_NETWORK_TRACE: networkTrace,
|
||||
NODE_OPTIONS: `--require=${path.join(input.corpusRoot, "network-observer.cjs")}`,
|
||||
NO_COLOR: "1",
|
||||
};
|
||||
if (caseData.fixtureState === "codex-home-override") env.CODEX_HOME = path.join(caseRoot, "codex-home");
|
||||
return { args, cwd, env, trace, networkTrace, replacements };
|
||||
}
|
||||
|
||||
function normalizedTree(caseRoot, replacements) {
|
||||
return treeEntries(caseRoot, { exclude: new Set(["harness"]) }).map((entry) => entry.type === "symlink" ? { ...entry, target: normalizeText(entry.target, replacements) } : entry);
|
||||
}
|
||||
|
||||
function normalizedOutput(text, replacements) {
|
||||
return normalizeText(text, replacements).replaceAll("<BASELINE_RUNTIME>", "<RUNTIME>");
|
||||
}
|
||||
|
||||
function normalizeTrace(trace, caseData, candidateVersion) {
|
||||
return trace.map((entry) => {
|
||||
const normalized = canonicalize(entry);
|
||||
const explicit = caseData.args.includes("--release") || caseData.args.includes("--tag");
|
||||
if (!explicit && normalized.branch === `v${candidateVersion}`) normalized.branch = "<CANDIDATE_DEFAULT_RELEASE>";
|
||||
return normalized;
|
||||
});
|
||||
}
|
||||
|
||||
const input = JSON.parse(fs.readFileSync(0, "utf8"));
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(input.corpusRoot, "manifest.json"), "utf8"));
|
||||
const fixtureEntries = treeEntries(path.join(input.corpusRoot, "fixture-repository"));
|
||||
const fixtureDigest = treeDigest(fixtureEntries);
|
||||
const installer = path.join(input.packageRoot, "tools", "bin", "install.js");
|
||||
const candidateManifest = JSON.parse(fs.readFileSync(path.join(input.packageRoot, "package.json"), "utf8"));
|
||||
const work = input.workRoot;
|
||||
fs.mkdirSync(path.join(work, "cases"), { recursive: true, mode: 0o700 });
|
||||
let passed = 0;
|
||||
const failures = [];
|
||||
for (const caseData of manifest.cases) {
|
||||
const caseRoot = path.join(work, "cases", caseData.id);
|
||||
const setup = setupCase(input, caseData, caseRoot, fixtureDigest);
|
||||
const result = spawnSync(process.execPath, [installer, ...setup.args], { cwd: setup.cwd, env: setup.env, encoding: "utf8", windowsHide: true, maxBuffer: 32 * 1024 * 1024 });
|
||||
const expected = JSON.parse(fs.readFileSync(path.join(input.corpusRoot, caseData.expectedSnapshot), "utf8"));
|
||||
const observedTrace = normalizeTrace(parseTrace(setup.trace), caseData, candidateManifest.version);
|
||||
const expectedTrace = normalizeTrace(expected.fakeGitTrace, caseData, manifest.baseline.version);
|
||||
const observedTree = normalizedTree(caseRoot, setup.replacements);
|
||||
const observedStdout = normalizedOutput(result.stdout || "", setup.replacements);
|
||||
const observedStderr = normalizedOutput(result.stderr || "", setup.replacements);
|
||||
const expectedStdout = normalizedOutput(expected.stdout || "", [["<BASELINE_RUNTIME>", "<RUNTIME>"]]);
|
||||
const expectedStderr = normalizedOutput(expected.stderr || "", [["<BASELINE_RUNTIME>", "<RUNTIME>"]]);
|
||||
const differences = [];
|
||||
if ((result.status ?? 128) !== expected.expectedExitCode) differences.push("exitCode");
|
||||
if (result.signal !== expected.signal) differences.push("signal");
|
||||
if (caseData.id !== "version" && observedStdout !== expectedStdout) differences.push("stdout");
|
||||
if (observedStderr !== expectedStderr) differences.push("stderr");
|
||||
if (JSON.stringify(observedTrace) !== JSON.stringify(expectedTrace)) differences.push("gitTrace");
|
||||
if (treeDigest(observedTree) !== expected.treeDigest) differences.push("filesystem");
|
||||
if (parseTrace(setup.networkTrace).length !== 0) differences.push("network");
|
||||
if (observedTree.some((entry) => /(^|\/)aas-stack\.json$/.test(entry.path))) differences.push("implicitStackState");
|
||||
if (differences.length) failures.push({
|
||||
caseId: caseData.id,
|
||||
differences,
|
||||
...(process.env.AAS_VERIFIER_DEBUG_FIXTURE === "1" ? {
|
||||
args: setup.args,
|
||||
cwd: normalizeText(setup.cwd, setup.replacements),
|
||||
exitCode: result.status ?? 128,
|
||||
expectedExitCode: expected.expectedExitCode,
|
||||
observedStdout,
|
||||
expectedStdout,
|
||||
observedTreeDigest: treeDigest(observedTree),
|
||||
expectedTreeDigest: expected.treeDigest,
|
||||
} : {}),
|
||||
});
|
||||
else passed += 1;
|
||||
}
|
||||
process.stdout.write(`${JSON.stringify({ schemaVersion: 1, ok: failures.length === 0, executions: manifest.cases.length, passed, failures })}\n`);
|
||||
if (failures.length) process.exit(1);
|
||||
@@ -0,0 +1,180 @@
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const MASK = (1n << 64n) - 1n;
|
||||
const rotateLeft = (value, amount) => ((value << BigInt(amount)) | (value >> (64n - BigInt(amount)))) & MASK;
|
||||
|
||||
function deriveState(rootSeedHex, namespace) {
|
||||
const key = Buffer.from(rootSeedHex, "hex");
|
||||
let material = namespace;
|
||||
for (let retry = 0; retry < 100; retry += 1) {
|
||||
const bytes = crypto.createHmac("sha256", key).update(material).digest();
|
||||
const state = [0, 8, 16, 24].map((offset) => bytes.readBigUInt64BE(offset));
|
||||
if (state.some((word) => word !== 0n)) return state;
|
||||
material = `${namespace}/retry/${retry + 1}`;
|
||||
}
|
||||
throw new Error("non-zero PRNG state unavailable");
|
||||
}
|
||||
|
||||
function nextUint64(state) {
|
||||
const result = (rotateLeft((state[1] * 5n) & MASK, 7) * 9n) & MASK;
|
||||
const temporary = (state[1] << 17n) & MASK;
|
||||
state[2] ^= state[0]; state[3] ^= state[1]; state[1] ^= state[2]; state[0] ^= state[3];
|
||||
state[2] ^= temporary; state[3] = rotateLeft(state[3], 45);
|
||||
return result;
|
||||
}
|
||||
|
||||
function sample(state, upper) {
|
||||
const bound = BigInt(upper);
|
||||
const limit = (1n << 64n) - ((1n << 64n) % bound);
|
||||
let value;
|
||||
do value = nextUint64(state); while (value >= limit);
|
||||
return Number(value % bound);
|
||||
}
|
||||
|
||||
function stateHex(state) {
|
||||
return state.map((word) => word.toString(16).padStart(16, "0")).join("");
|
||||
}
|
||||
|
||||
function main(input) {
|
||||
const core = require(path.join(input.packageRoot, "tools/lib/aas-v1"));
|
||||
const { judgment, notApplicable, recommendStack, canonicalJson, sha256, stack } = core;
|
||||
const budget = input.budget;
|
||||
const rootSeed = budget.prng.rootSeed;
|
||||
for (const vector of budget.prng.substreams.testVectors) {
|
||||
const state = deriveState(rootSeed, vector.namespace);
|
||||
if (stateHex(state) !== vector.stateHex || nextUint64(state).toString(16).padStart(16, "0") !== vector.firstUint64Hex) {
|
||||
throw new Error(`PRNG test vector failed: ${vector.namespace}`);
|
||||
}
|
||||
}
|
||||
|
||||
const known = (value) => judgment(value, [{ type: "independent-property-fixture" }]);
|
||||
const skill = (id, options = {}) => ({
|
||||
id,
|
||||
name: id,
|
||||
description: options.description || id,
|
||||
category: "fixture",
|
||||
tags: [],
|
||||
triggers: [],
|
||||
searchTokens: options.tokens || [id],
|
||||
recommendationTokens: options.tokens || [id],
|
||||
metadata: {
|
||||
capabilities: known(options.capabilities || ["goal-a"]),
|
||||
risk: options.risk === null ? judgment(null) : known(options.risk || "safe"),
|
||||
source: options.source === null ? judgment(null) : known(options.source || { repository: "fixture" }),
|
||||
license: notApplicable(),
|
||||
targets: {
|
||||
codex: known(options.codex || "supported"),
|
||||
claude: known(options.claude || "supported"),
|
||||
},
|
||||
setup: options.setup === null ? judgment(null) : known(options.setup || "none"),
|
||||
dependencies: known(options.dependencies || []),
|
||||
conflicts: known(options.conflicts || []),
|
||||
validation: known({ catalogWideSelection: true }),
|
||||
tests: notApplicable(),
|
||||
reviews: known([{ reviewer: "independent-property-driver" }]),
|
||||
},
|
||||
});
|
||||
const catalog = (skills) => ({ schemaVersion: 1, package: "property-fixture", version: "1.0.0", digest: sha256(canonicalJson({ skills })), skills });
|
||||
const baseInput = (overrides = {}) => ({
|
||||
intent: "web-application-delivery",
|
||||
profile: { request: "goal-a" },
|
||||
targets: [{ host: "codex", scope: "project" }],
|
||||
criticalGoals: ["goal-a"],
|
||||
nonCriticalGoals: [],
|
||||
minimumNonCriticalGoalCoverage: 0.8,
|
||||
policy: { allowedRisk: ["safe"], requireKnownSource: false, allowManualSetup: false },
|
||||
maxSkills: 4,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const summary = {};
|
||||
let total = 0;
|
||||
let hardPolicyViolations = 0;
|
||||
const fail = (condition, code) => {
|
||||
if (!condition) {
|
||||
hardPolicyViolations += 1;
|
||||
throw new Error(code);
|
||||
}
|
||||
};
|
||||
const remap = (value, mapping) => {
|
||||
if (typeof value === "string") return mapping[value] || value;
|
||||
if (Array.isArray(value)) return value.map((entry) => remap(entry, mapping));
|
||||
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value)
|
||||
.filter(([key]) => key !== "canonicalJson")
|
||||
.map(([key, entry]) => [key, remap(entry, mapping)]));
|
||||
return value;
|
||||
};
|
||||
|
||||
for (const stratum of budget.propertyAndGenerative.strata) {
|
||||
let executions = 0;
|
||||
let accumulator = 0n;
|
||||
for (let index = 0; index < stratum.executions; index += 1) {
|
||||
if (index % input.jobCount !== input.jobIndex) continue;
|
||||
const namespace = `aas.v1/property-and-generative/${stratum.id}/execution/${index}`;
|
||||
const state = deriveState(rootSeed, namespace);
|
||||
const variant = sample(state, 4);
|
||||
if (stratum.id === "hard-policy-risk") {
|
||||
const result = recommendStack(catalog([skill("safe", { tokens: ["goal-a"] }), skill("blocked", { risk: "offensive", tokens: ["goal-a", "goal-a"] })]), baseInput());
|
||||
fail(!result.proposedStack.includes("blocked"), "risk policy violation");
|
||||
} else if (stratum.id === "hard-policy-provenance") {
|
||||
const result = recommendStack(catalog([skill("known"), skill("unknown", { source: null, tokens: ["goal-a", "goal-a"] })]), baseInput({ policy: { allowedRisk: ["safe"], requireKnownSource: true, allowManualSetup: false } }));
|
||||
fail(!result.proposedStack.includes("unknown"), "source policy violation");
|
||||
} else if (stratum.id === "hard-policy-compatibility") {
|
||||
const host = variant % 2 ? "claude" : "codex";
|
||||
const blocked = host === "codex" ? skill("blocked", { codex: "blocked" }) : skill("blocked", { claude: "blocked" });
|
||||
const result = recommendStack(catalog([skill("supported"), blocked]), baseInput({ targets: [{ host, scope: "project" }] }));
|
||||
fail(!result.proposedStack.includes("blocked"), "compatibility policy violation");
|
||||
} else if (stratum.id === "unknown-eligibility") {
|
||||
const result = recommendStack(catalog([skill("unknown", { risk: null }), skill("known")]), baseInput());
|
||||
fail(!result.proposedStack.includes("unknown") && result.discoveryCandidates.some((entry) => entry.id === "unknown"), "unknown eligibility was hidden or promoted");
|
||||
} else if (stratum.id === "dependency-conflict") {
|
||||
const skills = [
|
||||
skill("root", { dependencies: ["dep"], capabilities: ["goal-a"] }),
|
||||
skill("dep", { capabilities: ["goal-b"] }),
|
||||
skill("conflict", { conflicts: ["root"], capabilities: ["goal-a"] }),
|
||||
];
|
||||
const result = recommendStack(catalog(skills), baseInput());
|
||||
if (result.proposedStack.includes("root")) fail(result.proposedStack.includes("dep"), "dependency omitted");
|
||||
fail(!(result.proposedStack.includes("root") && result.proposedStack.includes("conflict")), "conflict co-selected");
|
||||
} else if (stratum.id === "catalog-order-metamorphic") {
|
||||
const skills = [skill("a", { tokens: ["goal-a", "goal-a"] }), skill("b", { tokens: ["goal-a"] }), skill("c", { capabilities: ["goal-b"] })];
|
||||
const identity = catalog(skills);
|
||||
const left = recommendStack(identity, baseInput());
|
||||
const right = recommendStack({ ...identity, skills: [...skills].reverse() }, baseInput());
|
||||
fail(left.canonicalJson === right.canonicalJson, "catalog-order metamorphic mismatch");
|
||||
} else if (stratum.id === "consistent-id-permutation") {
|
||||
const skills = [skill("alpha", { tokens: ["goal-a", "goal-a"] }), skill("beta", { tokens: ["goal-a"] })];
|
||||
const firstCatalog = catalog(skills);
|
||||
const mapping = { alpha: "renamed-a", beta: "renamed-b" };
|
||||
const inverse = { "renamed-a": "alpha", "renamed-b": "beta" };
|
||||
const renamedSkills = skills.map((entry) => ({ ...entry, id: mapping[entry.id], name: mapping[entry.id] }));
|
||||
const left = recommendStack(firstCatalog, baseInput());
|
||||
const right = recommendStack({ ...firstCatalog, skills: renamedSkills }, baseInput());
|
||||
fail(canonicalJson(remap(left, {})) === canonicalJson(remap(right, inverse)), "consistent-ID metamorphic mismatch");
|
||||
} else if (stratum.id === "plan-policy-invariants") {
|
||||
const digest = `sha256-${"1".repeat(64)}`;
|
||||
const emptyStateDigest = sha256(canonicalJson({ schemaVersion: 1, entries: [] }));
|
||||
const nextStateDigest = sha256(canonicalJson({ schemaVersion: 1, entries: [{ skillId: "skill-a", treeDigest: digest, catalogIntegrity: digest }] }));
|
||||
const manifest = { schemaVersion: 1, name: "property", catalog: { package: "property-fixture", version: "1.0.0", integrity: digest }, targets: [{ host: "codex", scope: "project" }], intent: { goals: ["goal-a"] }, policy: { allowedRisk: ["safe"], requireKnownSource: true, allowManualSetup: false }, skills: [{ id: "skill-a" }] };
|
||||
const plan = stack.buildPlanEnvelope({ manifest, handshake: { protocolVersion: core.protocolVersion, coreVersion: core.coreVersion, metadataSchemaVersion: core.metadataSchemaVersion, scorerVersion: core.scorerVersion }, catalog: manifest.catalog, runtime: { package: "property-fixture", version: "1.0.0", integrity: digest, closureDigest: digest }, target: { host: "codex", scope: "project", adapterVersion: "1.0.0", identityDigest: digest }, installedState: { digest: emptyStateDigest, entries: [] }, operations: [{ kind: "install", skillId: "skill-a", sourceTreeDigest: digest, expectedTreeDigest: null, resultTreeDigest: digest, backupRequired: false }], overrides: [], stateCommit: { previousDigest: emptyStateDigest, nextDigest: nextStateDigest, position: "final" } });
|
||||
const tampered = JSON.parse(JSON.stringify(plan));
|
||||
tampered.payload.policy.allowedRisk = [variant % 2 ? "offensive" : "none"];
|
||||
let rejected = false;
|
||||
try { stack.validatePlanEnvelope(tampered); } catch { rejected = true; }
|
||||
fail(rejected, "tampered immutable plan accepted");
|
||||
} else throw new Error(`unknown property stratum: ${stratum.id}`);
|
||||
accumulator ^= nextUint64(state);
|
||||
executions += 1;
|
||||
total += 1;
|
||||
}
|
||||
summary[stratum.id] = { executions, accumulator: accumulator.toString(16).padStart(16, "0") };
|
||||
}
|
||||
return { schemaVersion: 1, ok: true, total, hardPolicyViolations, summary };
|
||||
}
|
||||
|
||||
const input = JSON.parse(fs.readFileSync(0, "utf8"));
|
||||
process.stdout.write(`${JSON.stringify(main(input))}\n`);
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$JobSource,
|
||||
[Parameter(Mandatory = $true)][int]$ParentProcessId,
|
||||
[Parameter(Mandatory = $true)][string]$ReadyCanary,
|
||||
[Parameter(Mandatory = $true)][string]$AfterParentCanary
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -Path $JobSource
|
||||
[IO.File]::WriteAllText($ReadyCanary, "ready", (New-Object Text.UTF8Encoding($false)))
|
||||
if (![AasVerifier.JobProcess]::WaitForProcessExit($ParentProcessId, 6000)) { exit 66 }
|
||||
[IO.File]::WriteAllText($AfterParentCanary, "child", (New-Object Text.UTF8Encoding($false)))
|
||||
while ($true) { Start-Sleep -Seconds 1 }
|
||||
@@ -0,0 +1,48 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$Powershell,
|
||||
[Parameter(Mandatory = $true)][string]$ChildDriver,
|
||||
[Parameter(Mandatory = $true)][string]$JobSource,
|
||||
[Parameter(Mandatory = $true)][string]$ReadyCanary,
|
||||
[Parameter(Mandatory = $true)][string]$RootAckCanary,
|
||||
[Parameter(Mandatory = $true)][string]$AfterParentCanary
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
function Quote-NativeArgument([string]$Value) {
|
||||
return '"' + $Value.Replace('"', '\"') + '"'
|
||||
}
|
||||
|
||||
$childArguments = @(
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
(Quote-NativeArgument $ChildDriver),
|
||||
"-JobSource",
|
||||
(Quote-NativeArgument $JobSource),
|
||||
"-ParentProcessId",
|
||||
"$PID",
|
||||
"-ReadyCanary",
|
||||
(Quote-NativeArgument $ReadyCanary),
|
||||
"-AfterParentCanary",
|
||||
(Quote-NativeArgument $AfterParentCanary)
|
||||
)
|
||||
$child = Start-Process -FilePath $Powershell -ArgumentList $childArguments -NoNewWindow -PassThru
|
||||
[Console]::Out.Write("$($child.Id)")
|
||||
|
||||
$deadline = [DateTimeOffset]::UtcNow.AddSeconds(6)
|
||||
while ([DateTimeOffset]::UtcNow -lt $deadline) {
|
||||
if (Test-Path -LiteralPath $ReadyCanary -PathType Leaf) {
|
||||
[IO.File]::WriteAllText($RootAckCanary, "ack", (New-Object Text.UTF8Encoding($false)))
|
||||
exit 0
|
||||
}
|
||||
if ($child.HasExited) {
|
||||
[Console]::Error.Write("child-exited-before-readiness:$($child.ExitCode)")
|
||||
exit 67
|
||||
}
|
||||
Start-Sleep -Milliseconds 25
|
||||
}
|
||||
[Console]::Error.Write("child-readiness-timeout")
|
||||
exit 65
|
||||
@@ -0,0 +1,63 @@
|
||||
import fs from "node:fs";
|
||||
import { canonicalJson, digestJson } from "./canonical.mjs";
|
||||
import { JOB_IDS, validateReceipt } from "./receipt.mjs";
|
||||
|
||||
function fail(code, detail = {}) { return { code, detail }; }
|
||||
|
||||
export function aggregateReceipts(receipts, validator) {
|
||||
const failures = [];
|
||||
if (receipts.length !== JOB_IDS.length) failures.push(fail("AAS_VERIFIER_MATRIX_RECEIPT_COUNT", { actual: receipts.length }));
|
||||
const byJob = new Map();
|
||||
for (const receipt of receipts) {
|
||||
for (const entry of validateReceipt(receipt, validator)) failures.push(fail(entry.code, entry));
|
||||
if (byJob.has(receipt.job?.id)) failures.push(fail("AAS_VERIFIER_MATRIX_DUPLICATE_JOB", { job: receipt.job?.id }));
|
||||
byJob.set(receipt.job?.id, receipt);
|
||||
}
|
||||
for (const job of JOB_IDS) if (!byJob.has(job)) failures.push(fail("AAS_VERIFIER_MATRIX_MISSING_JOB", { job }));
|
||||
|
||||
for (const field of ["commit", "tarballSha256", "tarballSha512", "packManifestSha256"]) {
|
||||
if (new Set(receipts.map((entry) => entry.candidate?.[field])).size !== 1) failures.push(fail("AAS_VERIFIER_MATRIX_CANDIDATE_MISMATCH", { field }));
|
||||
}
|
||||
for (const field of ["commit", "rootDigest", "contractDigest"]) {
|
||||
if (new Set(receipts.map((entry) => entry.verifier?.[field])).size !== 1) failures.push(fail("AAS_VERIFIER_MATRIX_VERIFIER_MISMATCH", { field }));
|
||||
}
|
||||
if (receipts.some((entry) => entry.status !== "passed")) failures.push(fail("AAS_VERIFIER_MATRIX_JOB_FAILED"));
|
||||
|
||||
const evidence = (id) => receipts.map((receipt) => receipt.suites?.find((suite) => suite.id === id)?.evidence);
|
||||
const propertyTotal = evidence("property").reduce((sum, value) => sum + (value?.total || 0), 0);
|
||||
const fuzzTotal = evidence("fuzz").reduce((sum, value) => sum + (value?.total || 0), 0);
|
||||
if (propertyTotal !== 100_000) failures.push(fail("AAS_VERIFIER_PROPERTY_BUDGET", { propertyTotal }));
|
||||
if (fuzzTotal !== 50_000) failures.push(fail("AAS_VERIFIER_FUZZ_BUDGET", { fuzzTotal }));
|
||||
if (evidence("property").some((value) => value?.hardPolicyViolations !== 0)) failures.push(fail("AAS_VERIFIER_HARD_POLICY_VIOLATION"));
|
||||
if (evidence("hostile").some((value) => value?.executions !== 64)) failures.push(fail("AAS_VERIFIER_HOSTILE_DENOMINATOR"));
|
||||
if (evidence("legacy").some((value) => value?.executions !== 41)) failures.push(fail("AAS_VERIFIER_LEGACY_DENOMINATOR"));
|
||||
|
||||
const canonical = new Set(receipts.map((entry) => entry.canonicalPayload?.sha256));
|
||||
if (canonical.size !== 1) failures.push(fail("AAS_VERIFIER_CANONICAL_CROSS_MATRIX_MISMATCH"));
|
||||
|
||||
const faultClasses = new Set(evidence("transaction").flatMap((value) => value?.faultBoundaryClasses || []));
|
||||
const raceClasses = new Set(evidence("transaction").flatMap((value) => value?.raceClasses || []));
|
||||
for (const value of ["lock", "journal", "backup", "write", "fsync", "rename", "commit"]) {
|
||||
if (!faultClasses.has(value)) failures.push(fail("AAS_VERIFIER_FAULT_CLASS_MISSING", { value }));
|
||||
}
|
||||
for (const value of ["concurrency", "drift", "symlink-swap", "target-swap", "corrupt-journal", "recovery-race"]) {
|
||||
if (!raceClasses.has(value)) failures.push(fail("AAS_VERIFIER_RACE_CLASS_MISSING", { value }));
|
||||
}
|
||||
|
||||
const bundle = {
|
||||
schemaVersion: 1,
|
||||
status: failures.length ? "failed" : "passed",
|
||||
candidate: receipts[0]?.candidate || null,
|
||||
verifier: receipts[0]?.verifier || null,
|
||||
jobs: receipts.map((entry) => ({ id: entry.job?.id, receiptDigest: entry.receiptDigest, status: entry.status })).sort((a, b) => a.id.localeCompare(b.id)),
|
||||
denominators: { property: propertyTotal, fuzz: fuzzTotal, hostilePerJob: 64, legacyPerJob: 41 },
|
||||
canonicalPayloadSha256: receipts[0]?.canonicalPayload?.sha256 || null,
|
||||
failures,
|
||||
};
|
||||
return { ...bundle, bundleDigest: digestJson(bundle) };
|
||||
}
|
||||
|
||||
export function writeBundle(file, bundle) {
|
||||
fs.writeFileSync(file, `${canonicalJson(bundle)}\n`, { mode: 0o600, flag: "wx" });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
function canonicalNumber(value) {
|
||||
if (!Number.isFinite(value)) throw new TypeError("Canonical JSON forbids non-finite numbers");
|
||||
if (Object.is(value, -0)) return "0";
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function canonicalJson(value) {
|
||||
if (value === null) return "null";
|
||||
if (typeof value === "boolean") return value ? "true" : "false";
|
||||
if (typeof value === "number") return canonicalNumber(value);
|
||||
if (typeof value === "string") return JSON.stringify(value);
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
if (typeof value === "object") {
|
||||
// RFC 8785 orders names by UTF-16 code units, matching ECMAScript's
|
||||
// default string sort. UTF-8 byte order differs for some non-ASCII keys.
|
||||
const keys = Object.keys(value).sort();
|
||||
return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
|
||||
}
|
||||
throw new TypeError(`Canonical JSON cannot encode ${typeof value}`);
|
||||
}
|
||||
|
||||
export function sha256(value) {
|
||||
return `sha256-${crypto.createHash("sha256").update(value).digest("hex")}`;
|
||||
}
|
||||
|
||||
export function sha512(value) {
|
||||
return `sha512-${crypto.createHash("sha512").update(value).digest("base64")}`;
|
||||
}
|
||||
|
||||
export function digestJson(value) {
|
||||
return sha256(Buffer.from(canonicalJson(value), "utf8"));
|
||||
}
|
||||
|
||||
export function parseCanonicalJson(text) {
|
||||
const value = JSON.parse(text);
|
||||
if (canonicalJson(value) !== text.trim()) throw new Error("Input is not canonical JSON");
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { digestJson, sha256 } from "./canonical.mjs";
|
||||
|
||||
function normalizeMode(mode) {
|
||||
return mode & 0o7777;
|
||||
}
|
||||
|
||||
export function snapshotTree(root) {
|
||||
const entries = [];
|
||||
if (!fs.existsSync(root)) return { entries, digest: digestJson(entries) };
|
||||
const walk = (directory) => {
|
||||
for (const name of fs.readdirSync(directory).sort()) {
|
||||
const absolute = path.join(directory, name);
|
||||
const relative = path.relative(root, absolute).split(path.sep).join("/");
|
||||
const stat = fs.lstatSync(absolute, { bigint: true });
|
||||
const common = {
|
||||
path: relative,
|
||||
mode: normalizeMode(Number(stat.mode)),
|
||||
size: Number(stat.size),
|
||||
};
|
||||
if (stat.isSymbolicLink()) {
|
||||
entries.push({ ...common, type: "symlink", target: fs.readlinkSync(absolute) });
|
||||
} else if (stat.isDirectory()) {
|
||||
entries.push({ ...common, type: "directory" });
|
||||
walk(absolute);
|
||||
} else if (stat.isFile()) {
|
||||
entries.push({ ...common, type: "file", sha256: sha256(fs.readFileSync(absolute)) });
|
||||
} else {
|
||||
entries.push({ ...common, type: "special" });
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(root);
|
||||
return { entries, digest: digestJson(entries) };
|
||||
}
|
||||
|
||||
export function snapshotZones(zones) {
|
||||
return Object.fromEntries(Object.entries(zones).map(([name, root]) => [name, snapshotTree(root)]));
|
||||
}
|
||||
|
||||
export function assertNoZoneDrift(before, after) {
|
||||
const changed = Object.keys(before).filter((name) => before[name]?.digest !== after[name]?.digest);
|
||||
if (changed.length) {
|
||||
const error = new Error(`Observed persistent filesystem drift in: ${changed.join(", ")}`);
|
||||
error.code = "AAS_VERIFIER_PERSISTENT_WRITE";
|
||||
error.changedZones = changed;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
export class McpClient {
|
||||
constructor(executable, args, options = {}) {
|
||||
this.executable = executable;
|
||||
this.args = args;
|
||||
this.options = options;
|
||||
this.nextId = 1;
|
||||
this.pending = new Map();
|
||||
this.stdoutBuffer = "";
|
||||
this.stderr = "";
|
||||
this.protocolNoise = [];
|
||||
}
|
||||
|
||||
async start() {
|
||||
this.child = spawn(this.executable, this.args, {
|
||||
cwd: this.options.cwd,
|
||||
env: this.options.env,
|
||||
windowsHide: true,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
this.child.stdout.setEncoding("utf8");
|
||||
this.child.stderr.setEncoding("utf8");
|
||||
this.child.stdout.on("data", (chunk) => this.#receive(chunk));
|
||||
this.child.stderr.on("data", (chunk) => {
|
||||
this.stderr += chunk;
|
||||
if (Buffer.byteLength(this.stderr) > (this.options.maxStderrBytes ?? 1024 * 1024)) this.child.kill("SIGKILL");
|
||||
});
|
||||
this.child.on("close", (code, signal) => {
|
||||
const error = new Error(`MCP process closed (${code ?? signal})`);
|
||||
for (const { reject } of this.pending.values()) reject(error);
|
||||
this.pending.clear();
|
||||
});
|
||||
this.child.on("error", (error) => {
|
||||
for (const { reject } of this.pending.values()) reject(error);
|
||||
this.pending.clear();
|
||||
});
|
||||
}
|
||||
|
||||
#receive(chunk) {
|
||||
this.stdoutBuffer += chunk;
|
||||
if (Buffer.byteLength(this.stdoutBuffer) > (this.options.maxStdoutBufferBytes ?? 4 * 1024 * 1024)) {
|
||||
this.child.kill("SIGKILL");
|
||||
return;
|
||||
}
|
||||
while (this.stdoutBuffer.includes("\n")) {
|
||||
const newline = this.stdoutBuffer.indexOf("\n");
|
||||
const line = this.stdoutBuffer.slice(0, newline).replace(/\r$/, "");
|
||||
this.stdoutBuffer = this.stdoutBuffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
let message;
|
||||
try {
|
||||
message = JSON.parse(line);
|
||||
} catch {
|
||||
this.protocolNoise.push(line.slice(0, 200));
|
||||
continue;
|
||||
}
|
||||
if (message.id !== undefined && this.pending.has(message.id)) {
|
||||
const pending = this.pending.get(message.id);
|
||||
this.pending.delete(message.id);
|
||||
clearTimeout(pending.timer);
|
||||
pending.resolve(message);
|
||||
} else if (!message.method) {
|
||||
this.protocolNoise.push(line.slice(0, 200));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
request(method, params = {}, timeoutMs = 10_000) {
|
||||
const id = this.nextId;
|
||||
this.nextId += 1;
|
||||
const payload = JSON.stringify({ jsonrpc: "2.0", id, method, params });
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id);
|
||||
reject(new Error(`MCP request timed out: ${method}`));
|
||||
}, timeoutMs);
|
||||
this.pending.set(id, { resolve, reject, timer });
|
||||
this.child.stdin.write(`${payload}\n`);
|
||||
});
|
||||
}
|
||||
|
||||
notify(method, params = {}) {
|
||||
this.child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (!this.child || this.child.exitCode !== null) return;
|
||||
this.child.stdin.end();
|
||||
await new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.child.kill("SIGKILL");
|
||||
resolve();
|
||||
}, 2_000);
|
||||
this.child.once("close", () => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
function ratio(numerator, denominator, emptyValue = 0) {
|
||||
if (denominator === 0) return emptyValue;
|
||||
return numerator / denominator;
|
||||
}
|
||||
|
||||
function solutionSatisfiesRequiredGroups(includedSkillIds, solution) {
|
||||
const included = new Set(includedSkillIds);
|
||||
return (solution.requiredGroups || []).every((group) => group.some((skillId) => included.has(skillId)));
|
||||
}
|
||||
|
||||
export function evaluateCase(result, goldCase, acceptedSolutions = []) {
|
||||
const criticalTotal = goldCase.criticalGoals.length;
|
||||
const nonCriticalTotal = goldCase.nonCriticalGoals.length;
|
||||
if (criticalTotal === 0) {
|
||||
throw new Error("Every in-scope held-out case must declare at least one critical goal.");
|
||||
}
|
||||
|
||||
const covered = new Set(result.coveredGoals || []);
|
||||
const criticalCovered = goldCase.criticalGoals.filter((goal) => covered.has(goal)).length;
|
||||
const nonCriticalCovered = goldCase.nonCriticalGoals.filter((goal) => covered.has(goal)).length;
|
||||
const criticalGoalCoverage = ratio(criticalCovered, criticalTotal);
|
||||
const nonCriticalGoalCoverage = ratio(nonCriticalCovered, nonCriticalTotal, 1);
|
||||
const minimumNonCriticalGoalCoverage = Math.max(
|
||||
0.8,
|
||||
goldCase.minimumNonCriticalGoalCoverage ?? 0.8,
|
||||
);
|
||||
const includedSkillIds = Array.isArray(result.includedSkillIds) ? result.includedSkillIds : [];
|
||||
const discoveryPromotions = Array.isArray(result.discoveryPromotions)
|
||||
? result.discoveryPromotions
|
||||
: [];
|
||||
const everyDiscoveryPromotionHasVisibleOverride = discoveryPromotions.every(
|
||||
(promotion) => promotion && promotion.visibleOverride === true,
|
||||
);
|
||||
const hardPolicyViolations = Array.isArray(result.hardPolicyViolations)
|
||||
? result.hardPolicyViolations.length
|
||||
: Number(result.hardPolicyViolations || 0);
|
||||
const terminal = result.terminal === true
|
||||
&& result.crashed !== true
|
||||
&& result.timedOut !== true
|
||||
&& result.missing !== true;
|
||||
|
||||
const independentlyAcceptedStack = acceptedSolutions.length > 0
|
||||
&& acceptedSolutions.some((solution) => (
|
||||
includedSkillIds.every((skillId) => (solution.allowedSkillIds || []).includes(skillId))
|
||||
&& solutionSatisfiesRequiredGroups(includedSkillIds, solution)
|
||||
));
|
||||
|
||||
const verified = terminal
|
||||
&& result.schemaValid === true
|
||||
&& hardPolicyViolations === 0
|
||||
&& criticalGoalCoverage === 1
|
||||
&& nonCriticalGoalCoverage >= minimumNonCriticalGoalCoverage
|
||||
&& (!goldCase.requiresSkill || includedSkillIds.length >= 1)
|
||||
&& everyDiscoveryPromotionHasVisibleOverride
|
||||
&& independentlyAcceptedStack;
|
||||
|
||||
return {
|
||||
verified,
|
||||
terminal,
|
||||
hardPolicyViolations,
|
||||
criticalGoalCoverage,
|
||||
nonCriticalGoalCoverage,
|
||||
everyDiscoveryPromotionHasVisibleOverride,
|
||||
independentlyAcceptedStack,
|
||||
includedSkillIds,
|
||||
};
|
||||
}
|
||||
|
||||
export function caseInclusionAssessment(includedSkillIds, acceptedSolutions) {
|
||||
if (!Array.isArray(includedSkillIds) || includedSkillIds.length === 0) {
|
||||
return {
|
||||
acceptedCount: 0,
|
||||
inclusionCount: 0,
|
||||
precision: null,
|
||||
matchedSolutionId: null,
|
||||
};
|
||||
}
|
||||
if (!Array.isArray(acceptedSolutions) || acceptedSolutions.length === 0) {
|
||||
throw new Error("At least one coherent accepted-equivalent solution is required.");
|
||||
}
|
||||
|
||||
const assessments = acceptedSolutions.map((solution) => {
|
||||
const allowed = new Set(solution.allowedSkillIds || []);
|
||||
const acceptedCount = includedSkillIds.filter((id) => allowed.has(id)).length;
|
||||
return {
|
||||
acceptedCount,
|
||||
inclusionCount: includedSkillIds.length,
|
||||
precision: acceptedCount / includedSkillIds.length,
|
||||
matchedSolutionId: solution.solutionId,
|
||||
};
|
||||
});
|
||||
assessments.sort((left, right) => (
|
||||
right.precision - left.precision
|
||||
|| right.acceptedCount - left.acceptedCount
|
||||
|| String(left.matchedSolutionId).localeCompare(String(right.matchedSolutionId))
|
||||
));
|
||||
return assessments[0];
|
||||
}
|
||||
|
||||
export function caseInclusionPrecision(includedSkillIds, acceptedSolutions) {
|
||||
return caseInclusionAssessment(includedSkillIds, acceptedSolutions).precision;
|
||||
}
|
||||
|
||||
export function aggregateIntent(caseReports, frozenDenominator = 30) {
|
||||
if (frozenDenominator !== 30) {
|
||||
throw new Error("The v1 held-out denominator is frozen at exactly 30 cases per intent.");
|
||||
}
|
||||
const verifiedCount = caseReports.filter((report) => report.verified === true).length;
|
||||
const acceptedInclusions = caseReports.reduce(
|
||||
(total, report) => total + Number(report.acceptedInclusionCount || 0),
|
||||
0,
|
||||
);
|
||||
const totalInclusions = caseReports.reduce(
|
||||
(total, report) => total + Number(report.inclusionCount || 0),
|
||||
0,
|
||||
);
|
||||
const perStackPrecisions = caseReports
|
||||
.filter((report) => Number(report.inclusionCount || 0) > 0)
|
||||
.map((report) => {
|
||||
if (typeof report.inclusionPrecision === "number") return report.inclusionPrecision;
|
||||
return Number(report.acceptedInclusionCount || 0) / Number(report.inclusionCount);
|
||||
});
|
||||
const nonEmptyStackCount = caseReports.filter((report) => Number(report.inclusionCount || 0) > 0).length;
|
||||
const emptyStackCount = caseReports.filter((report) => Number(report.inclusionCount || 0) === 0).length
|
||||
+ Math.max(0, frozenDenominator - caseReports.length);
|
||||
|
||||
return {
|
||||
frozenDenominator,
|
||||
observedResultCount: caseReports.length,
|
||||
verifiedCount,
|
||||
verifiedCoverage: verifiedCount / frozenDenominator,
|
||||
inclusionPrecision: perStackPrecisions.length === 0
|
||||
? null
|
||||
: perStackPrecisions.reduce((sum, value) => sum + value, 0) / perStackPrecisions.length,
|
||||
acceptedInclusions,
|
||||
totalInclusions,
|
||||
nonEmptyStackCount,
|
||||
emptyStackCount,
|
||||
};
|
||||
}
|
||||
|
||||
export function macroAverage(perIntent, field) {
|
||||
if (!Array.isArray(perIntent) || perIntent.length !== 6) {
|
||||
throw new Error("The v1 macro average requires exactly six intent values.");
|
||||
}
|
||||
const values = perIntent.map((entry) => entry[field]);
|
||||
if (values.some((value) => typeof value !== "number" || !Number.isFinite(value))) {
|
||||
return null;
|
||||
}
|
||||
return values.reduce((sum, value) => sum + value, 0) / 6;
|
||||
}
|
||||
|
||||
export function isCorrectAbstention(result) {
|
||||
return result?.ok === true
|
||||
&& result?.status === "insufficientCoverage"
|
||||
&& Array.isArray(result?.proposedStack)
|
||||
&& result.proposedStack.length === 0;
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { digestJson, sha256 } from "./canonical.mjs";
|
||||
import { runProcess } from "./process.mjs";
|
||||
|
||||
const NETWORK_CALL = /\b(?:socket|socketpair|connect|bind|listen|accept|accept4|sendto|sendmsg|recvfrom|recvmsg|getaddrinfo|GetAddrInfoW)\s*\(/;
|
||||
const MUTATION_CALL = /\b(?:creat|mkdir|mkdirat|rmdir|unlink|unlinkat|rename|renameat|renameat2|link|linkat|symlink|symlinkat|truncate|ftruncate|chmod|fchmod|fchmodat|chown|fchown|fchownat|utime|utimes|futimes|futimens|fsync|fdatasync)\s*\(/;
|
||||
const OPEN_WRITE = /\b(?:open|openat|openat2)\s*\([^\n]*\b(?:O_WRONLY|O_RDWR|O_CREAT|O_TRUNC|O_APPEND)\b/;
|
||||
const FD_WRITE = /\b(?:write|writev|pwrite|pwritev)\s*\((\d+)(?:<[^>]*>)?,/;
|
||||
const PROCESS_CALL = /\b(?:execve|execveat|posix_spawn)\s*\(/;
|
||||
|
||||
function resolveCommandPath(command) {
|
||||
const probe = process.platform === "win32" ? "where.exe" : "which";
|
||||
const result = spawnSync(probe, [command], { encoding: "utf8", windowsHide: true });
|
||||
if (result.status !== 0) return null;
|
||||
return result.stdout.split(/\r?\n/).map((line) => line.trim()).find((line) => path.isAbsolute(line)) ?? null;
|
||||
}
|
||||
|
||||
function commandExists(command) {
|
||||
return resolveCommandPath(command) !== null;
|
||||
}
|
||||
|
||||
function redactToken(token, zones) {
|
||||
let normalized = token;
|
||||
for (const [name, root] of Object.entries(zones)) {
|
||||
if (root && normalized.includes(root)) normalized = normalized.split(root).join(`<${name.toUpperCase()}>`);
|
||||
}
|
||||
return sha256(Buffer.from(normalized, "utf8"));
|
||||
}
|
||||
|
||||
export function parseLinuxStrace(text, zones = {}) {
|
||||
const events = [];
|
||||
let rootExecSeen = false;
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line || line.startsWith("+++") || line.startsWith("---")) continue;
|
||||
let kind = null;
|
||||
if (NETWORK_CALL.test(line)) kind = "network";
|
||||
else if (MUTATION_CALL.test(line) || OPEN_WRITE.test(line)) kind = "write";
|
||||
else {
|
||||
const fdWrite = line.match(FD_WRITE);
|
||||
if (fdWrite && !["1", "2"].includes(fdWrite[1])) kind = "write";
|
||||
else if (PROCESS_CALL.test(line)) {
|
||||
if (!rootExecSeen) rootExecSeen = true;
|
||||
else kind = "process";
|
||||
}
|
||||
}
|
||||
if (kind) events.push({ kind, targetDigest: redactToken(line, zones) });
|
||||
}
|
||||
return summarizeEvents(events);
|
||||
}
|
||||
|
||||
export function parseDelimitedObserver(text, zones = {}) {
|
||||
const events = [];
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line.trim()) continue;
|
||||
const [kind, ...rest] = line.split("|");
|
||||
if (!["network", "write", "process"].includes(kind)) continue;
|
||||
events.push({ kind, targetDigest: redactToken(rest.join("|"), zones) });
|
||||
}
|
||||
return summarizeEvents(events);
|
||||
}
|
||||
|
||||
function shellQuote(value) {
|
||||
if (/[\0\r\n]/.test(value)) throw new Error("Observer command contains forbidden control characters");
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function summarizeEvents(events) {
|
||||
const byKind = (kind) => events.filter((entry) => entry.kind === kind);
|
||||
return {
|
||||
networkAttempts: byKind("network").length,
|
||||
writeAttempts: byKind("write").length,
|
||||
childProcesses: byKind("process").length,
|
||||
events,
|
||||
eventDigest: digestJson(events),
|
||||
};
|
||||
}
|
||||
|
||||
async function linuxObserved(executable, args, options) {
|
||||
if (!commandExists("strace")) throw Object.assign(new Error("strace is required"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
const tracePrefix = path.join(options.evidenceDir, `strace-${process.pid}`);
|
||||
const traceArgs = [
|
||||
"-ff", "-qq", "-s", "256", "-yy",
|
||||
"-e", "trace=%network,%file,%process,write,writev,pwrite64,pwritev,fsync,fdatasync,ftruncate",
|
||||
"-o", tracePrefix,
|
||||
executable,
|
||||
...args,
|
||||
];
|
||||
const result = await runProcess("strace", traceArgs, options);
|
||||
const traceFiles = fs.readdirSync(options.evidenceDir)
|
||||
.filter((name) => name.startsWith(path.basename(tracePrefix)))
|
||||
.sort();
|
||||
if (!traceFiles.length) throw Object.assign(new Error("strace produced no trace"), { code: "AAS_OBSERVER_EMPTY" });
|
||||
const raw = traceFiles.map((name) => fs.readFileSync(path.join(options.evidenceDir, name), "utf8")).join("\n");
|
||||
for (const name of traceFiles) fs.rmSync(path.join(options.evidenceDir, name), { force: true });
|
||||
return { result, observation: parseLinuxStrace(raw, options.zones), backend: "linux-strace-process-tree" };
|
||||
}
|
||||
|
||||
function fsUsageLines(text) {
|
||||
return text.split(/\r?\n/).map((line) => line.trim()).filter((line) => /^\d{2}:\d{2}:\d{2}\.\d+/.test(line));
|
||||
}
|
||||
|
||||
export function parseMacFsUsage(filesystemText, networkText, execText, zones = {}) {
|
||||
const events = [];
|
||||
for (const line of fsUsageLines(networkText)) events.push({ kind: "network", targetDigest: redactToken(line, zones) });
|
||||
for (const line of fsUsageLines(filesystemText)) {
|
||||
if (/\b(?:WrData|WrMeta|write|pwrite|rename|unlink|mkdir|rmdir|truncate|chmod|chown|fsync|fdatasync|setattr|setxattr)\b/i.test(line)) {
|
||||
events.push({ kind: "write", targetDigest: redactToken(line, zones) });
|
||||
}
|
||||
}
|
||||
const execLines = fsUsageLines(execText);
|
||||
for (const line of execLines.slice(1)) events.push({ kind: "process", targetDigest: redactToken(line, zones) });
|
||||
return summarizeEvents(events);
|
||||
}
|
||||
|
||||
export function parseMacCombinedFsUsage(text, zones = {}, readinessToken = "", candidateToken = "") {
|
||||
const classified = [];
|
||||
let readinessIndex = -1;
|
||||
let candidateIndex = -1;
|
||||
const lines = fsUsageLines(text);
|
||||
for (const [index, line] of lines.entries()) {
|
||||
if (/\b(?:socket|connect|bind|listen|accept|sendto|sendmsg|recvfrom|recvmsg|getaddrinfo)\b/i.test(line)) {
|
||||
classified.push({ index, kind: "network", line });
|
||||
} else if (/\b(?:WrData|WrMeta|write|pwrite|rename|unlink|mkdir|rmdir|truncate|chmod|chown|fsync|fdatasync|setattr|setxattr)\b/i.test(line)) {
|
||||
if (readinessToken && line.includes(readinessToken)) readinessIndex = index;
|
||||
else if (candidateToken && line.includes(candidateToken)) candidateIndex = index;
|
||||
else classified.push({ index, kind: "write", line });
|
||||
} else if (/\b(?:execve|posix_spawn|exec|spawn)\b/i.test(line)) {
|
||||
classified.push({ index, kind: "process", line });
|
||||
}
|
||||
}
|
||||
const diagnostic = () => JSON.stringify({
|
||||
eventLines: lines.length,
|
||||
readinessTokenAnywhere: readinessToken ? text.includes(readinessToken) : null,
|
||||
candidateTokenAnywhere: candidateToken ? text.includes(candidateToken) : null,
|
||||
callNames: [...new Set(lines.map((line) => line.match(/^\d{2}:\d{2}:\d{2}\.\d+\s+(\S+)/)?.[1]).filter(Boolean))].slice(0, 24),
|
||||
});
|
||||
if (readinessToken && readinessIndex < 0) {
|
||||
throw Object.assign(new Error(`fs_usage missed the observer readiness canary: ${diagnostic()}`), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
if (candidateToken && candidateIndex < 0) {
|
||||
throw Object.assign(new Error(`fs_usage missed the candidate start canary: ${diagnostic()}`), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
if (candidateToken && candidateIndex <= readinessIndex) {
|
||||
throw Object.assign(new Error("fs_usage canary ordering is ambiguous"), { code: "AAS_OBSERVER_AMBIGUOUS_LINEAGE" });
|
||||
}
|
||||
const boundary = candidateToken ? candidateIndex : -1;
|
||||
const events = classified
|
||||
.filter((entry) => entry.index > boundary)
|
||||
.map(({ kind, line }) => ({ kind, targetDigest: redactToken(line, zones) }));
|
||||
return summarizeEvents(events);
|
||||
}
|
||||
|
||||
async function macObserved(executable, args, options) {
|
||||
if (!commandExists("fs_usage")) throw Object.assign(new Error("fs_usage is required"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
if (path.resolve(executable) !== path.resolve(process.execPath)) {
|
||||
throw Object.assign(new Error("macOS verifier supports only the pinned Node executable"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
const budgets = macObserverBudgets(options.timeoutMs);
|
||||
const sequence = `${process.pid.toString(36)}${Date.now().toString(36).slice(-5)}`.slice(-8);
|
||||
const observedName = `aasobs${sequence}`;
|
||||
const observedExecutable = path.join(options.evidenceDir, observedName);
|
||||
const readinessCanary = path.join(options.evidenceDir, `aas-ready-${sequence}`);
|
||||
const candidateCanary = path.join(options.evidenceDir, `aas-start-${sequence}`);
|
||||
const readinessToken = path.basename(readinessCanary);
|
||||
const candidateToken = path.basename(candidateCanary);
|
||||
const launcher = path.join(options.evidenceDir, `observer-${process.pid}.command`);
|
||||
fs.copyFileSync(executable, observedExecutable, fs.constants.COPYFILE_FICLONE);
|
||||
fs.chmodSync(observedExecutable, 0o700);
|
||||
const encodedArgs = Buffer.from(JSON.stringify(args), "utf8").toString("base64");
|
||||
fs.writeFileSync(launcher, [
|
||||
`process.title = ${JSON.stringify(observedName)};`,
|
||||
"const fs = require('node:fs');",
|
||||
`const fd = fs.openSync(${JSON.stringify(candidateCanary)}, 'w', 0o600);`,
|
||||
"fs.writeSync(fd, 'start'); fs.fsyncSync(fd); fs.closeSync(fd);",
|
||||
`const args = JSON.parse(Buffer.from(${JSON.stringify(encodedArgs)}, 'base64').toString('utf8'));`,
|
||||
"if (args[0] === '-e') { process.argv = [process.execPath, ...args.slice(2)]; eval(args[1]); }",
|
||||
"else { process.argv = [process.execPath, ...args]; require('node:module').runMain(); }",
|
||||
"",
|
||||
].join("\n"), { mode: 0o600 });
|
||||
let observerPid = 0;
|
||||
let observerOutcome = null;
|
||||
let observerPromise = null;
|
||||
let observerStopStarted = false;
|
||||
let observerCleanupFailed = false;
|
||||
let liveObserverOutput = "";
|
||||
let captureLiveOutput = true;
|
||||
const captureObserverOutput = (callback) => (chunk) => {
|
||||
if (captureLiveOutput) liveObserverOutput = `${liveObserverOutput}${chunk.toString("utf8")}`.slice(-1024 * 1024);
|
||||
if (typeof callback === "function") callback(chunk);
|
||||
};
|
||||
let readinessAttempts = 0;
|
||||
let readinessObservedLive = false;
|
||||
const stopObserver = async () => {
|
||||
if (observerStopStarted) return;
|
||||
observerStopStarted = true;
|
||||
if (!observerPid) return;
|
||||
const group = `-${observerPid}`;
|
||||
const probe = async () => runProcess("sudo", ["-n", "/bin/kill", "-0", group], { timeoutMs: 5_000 }).catch(() => null);
|
||||
const initialProbe = await probe();
|
||||
if (!initialProbe) {
|
||||
observerCleanupFailed = true;
|
||||
return;
|
||||
}
|
||||
if (initialProbe.code !== 0) return;
|
||||
await runProcess("sudo", ["-n", "/bin/kill", "-INT", group], { timeoutMs: 5_000 }).catch(() => null);
|
||||
await Promise.race([observerPromise, new Promise((resolve) => setTimeout(resolve, 1_000))]);
|
||||
const afterInterrupt = await probe();
|
||||
if (afterInterrupt?.code === 0) {
|
||||
await runProcess("sudo", ["-n", "/bin/kill", "-KILL", group], { timeoutMs: 5_000 }).catch(() => null);
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
const finalProbe = await probe();
|
||||
observerCleanupFailed = !finalProbe || finalProbe.code === 0;
|
||||
};
|
||||
const assertObserverActive = (stage) => {
|
||||
if (!observerOutcome) return;
|
||||
const result = observerOutcome.result;
|
||||
const diagnostic = JSON.stringify({
|
||||
stage,
|
||||
code: result?.code ?? null,
|
||||
signal: result?.signal ?? null,
|
||||
timedOut: result?.timedOut ?? null,
|
||||
outputLimitExceeded: result?.outputLimitExceeded ?? null,
|
||||
error: observerOutcome.error?.message ?? null,
|
||||
});
|
||||
throw Object.assign(new Error(`fs_usage exited before observation completed: ${diagnostic}`), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
};
|
||||
try {
|
||||
observerPromise = runProcess("sudo", [
|
||||
"-n", "/usr/bin/fs_usage", "-w", "-t", String(Math.ceil(budgets.observerTimeoutMs / 1000)),
|
||||
observedName,
|
||||
], {
|
||||
...options,
|
||||
detached: true,
|
||||
timeoutMs: budgets.observerTimeoutMs,
|
||||
maxOutputBytes: 8 * 1024 * 1024,
|
||||
onSpawn(child) { observerPid = child.pid; },
|
||||
onStdoutData: captureObserverOutput(options.onStdoutData),
|
||||
onStderrData: captureObserverOutput(options.onStderrData),
|
||||
}).then(
|
||||
(result) => (observerOutcome = { result }),
|
||||
(error) => (observerOutcome = { error }),
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, budgets.startupMs));
|
||||
assertObserverActive("startup");
|
||||
const readinessProgram = `process.title=${JSON.stringify(observedName)};const fs=require('node:fs');const target=${JSON.stringify(readinessCanary)};const deadline=Date.now()+4000;function beat(){const fd=fs.openSync(target,'w',0o600);fs.writeSync(fd,'ready');fs.fsyncSync(fd);fs.closeSync(fd);if(Date.now()<deadline)setTimeout(beat,200);}beat();`;
|
||||
for (let attempt = 0; attempt < budgets.readinessMaxAttempts; attempt += 1) {
|
||||
assertObserverActive("readiness-before-probe");
|
||||
readinessAttempts += 1;
|
||||
const readinessResult = await runProcess(observedExecutable, ["-e", readinessProgram], {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
timeoutMs: budgets.readinessProcessTimeoutMs,
|
||||
});
|
||||
if (readinessResult.code !== 0 || readinessResult.timedOut) {
|
||||
const diagnostic = JSON.stringify({
|
||||
code: readinessResult.code,
|
||||
signal: readinessResult.signal,
|
||||
timedOut: readinessResult.timedOut,
|
||||
outputLimitExceeded: readinessResult.outputLimitExceeded,
|
||||
stderr: readinessResult.stderr.slice(0, 240),
|
||||
});
|
||||
throw Object.assign(new Error(`macOS readiness process failed: ${diagnostic}`), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, budgets.readinessDelayMs));
|
||||
assertObserverActive("readiness-after-probe");
|
||||
if (liveObserverOutput.includes(readinessToken)) {
|
||||
readinessObservedLive = true;
|
||||
captureLiveOutput = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!readinessObservedLive) {
|
||||
throw Object.assign(new Error("fs_usage did not confirm readiness before the candidate deadline"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
assertObserverActive("candidate-start");
|
||||
const result = await runProcess(observedExecutable, [launcher], options);
|
||||
await new Promise((resolve) => setTimeout(resolve, budgets.drainMs));
|
||||
assertObserverActive("candidate-drain");
|
||||
await stopObserver();
|
||||
if (observerCleanupFailed) {
|
||||
throw Object.assign(new Error("fs_usage process group survived bounded cleanup"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
const outcome = await observerPromise;
|
||||
if (outcome.error) throw Object.assign(new Error("fs_usage observer process failed to start"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
const trace = outcome.result;
|
||||
if (trace.outputLimitExceeded) throw Object.assign(new Error("fs_usage trace exceeded the observer limit"), { code: "AAS_OBSERVER_OVERFLOW" });
|
||||
if (trace.timedOut) throw Object.assign(new Error("fs_usage exceeded its derived lifecycle budget"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
const raw = `${trace.stdout}\n${trace.stderr}`;
|
||||
return {
|
||||
result,
|
||||
observation: parseMacCombinedFsUsage(raw, options.zones, readinessToken, candidateToken),
|
||||
backend: "macos-fs_usage-process",
|
||||
diagnostics: {
|
||||
bytes: Buffer.byteLength(raw),
|
||||
eventLines: fsUsageLines(raw).length,
|
||||
readinessCanaryObserved: raw.includes(readinessToken),
|
||||
candidateCanaryObserved: raw.includes(candidateToken),
|
||||
readinessAttempts,
|
||||
readinessObservedLive,
|
||||
preview: fsUsageLines(raw).length ? null : raw.trim().slice(0, 160),
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
captureLiveOutput = false;
|
||||
await stopObserver();
|
||||
if (observerPromise) await observerPromise.catch(() => null);
|
||||
fs.rmSync(readinessCanary, { force: true });
|
||||
fs.rmSync(candidateCanary, { force: true });
|
||||
fs.rmSync(launcher, { force: true });
|
||||
fs.rmSync(observedExecutable, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function macObserverBudgets(candidateTimeoutMs = 30_000) {
|
||||
if (!Number.isSafeInteger(candidateTimeoutMs) || candidateTimeoutMs < 1 || candidateTimeoutMs > 15 * 60_000) {
|
||||
throw Object.assign(new Error("macOS observer timeout must be an integer from 1 to 900000 milliseconds"), {
|
||||
code: "AAS_OBSERVER_INVALID_TIMEOUT",
|
||||
});
|
||||
}
|
||||
const startupMs = 1_500;
|
||||
const readinessMaxAttempts = 2;
|
||||
const readinessProcessTimeoutMs = 10_000;
|
||||
const readinessDelayMs = 250;
|
||||
const drainMs = 1_000;
|
||||
return {
|
||||
startupMs,
|
||||
readinessMaxAttempts,
|
||||
readinessProcessTimeoutMs,
|
||||
readinessDelayMs,
|
||||
drainMs,
|
||||
observerTimeoutMs: startupMs
|
||||
+ readinessMaxAttempts * (readinessProcessTimeoutMs + readinessDelayMs)
|
||||
+ candidateTimeoutMs
|
||||
+ drainMs
|
||||
+ 5_000,
|
||||
};
|
||||
}
|
||||
|
||||
async function windowsObserved(executable, args, options) {
|
||||
const script = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "observers", "windows-etw.ps1");
|
||||
const powershell = resolveCommandPath("pwsh.exe") ?? resolveCommandPath("powershell.exe");
|
||||
if (!fs.existsSync(script) || !powershell) {
|
||||
throw Object.assign(new Error("Windows ETW observer is unavailable"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
const trace = path.join(options.evidenceDir, `windows-etw-${process.pid}.jsonl`);
|
||||
const encodedArgs = Buffer.from(JSON.stringify(args), "utf8").toString("base64");
|
||||
const resultFile = path.join(options.evidenceDir, `windows-result-${process.pid}.json`);
|
||||
const sessionName = `AASVerifier-${process.pid}-${randomUUID().replaceAll("-", "")}`;
|
||||
const { candidateTimeoutMs, wrapperTimeoutMs } = windowsObserverBudgets(options.timeoutMs);
|
||||
const wrapper = await runProcess(powershell, [
|
||||
"-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
||||
"-File", script,
|
||||
"-Executable", executable,
|
||||
"-ArgumentsBase64", encodedArgs,
|
||||
"-TraceOutput", trace,
|
||||
"-ResultOutput", resultFile,
|
||||
"-SessionName", sessionName,
|
||||
"-CandidateTimeoutMilliseconds", String(candidateTimeoutMs),
|
||||
], { ...options, timeoutMs: wrapperTimeoutMs });
|
||||
if (wrapper.code !== 0 || !fs.existsSync(trace) || !fs.existsSync(resultFile)) {
|
||||
const cleanup = await runProcess("logman.exe", ["stop", sessionName, "-ets"], {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
timeoutMs: 5_000,
|
||||
maxOutputBytes: 64 * 1024,
|
||||
});
|
||||
fs.rmSync(path.join(options.evidenceDir, `${sessionName}.etl`), { force: true });
|
||||
fs.rmSync(path.join(options.evidenceDir, `${sessionName}.csv`), { force: true });
|
||||
const traceExists = fs.existsSync(trace);
|
||||
const resultExists = fs.existsSync(resultFile);
|
||||
const diagnostic = JSON.stringify({
|
||||
wrapperCode: wrapper.code,
|
||||
signal: wrapper.signal,
|
||||
timedOut: wrapper.timedOut,
|
||||
outputLimitExceeded: wrapper.outputLimitExceeded,
|
||||
traceExists,
|
||||
resultExists,
|
||||
cleanupCode: cleanup.code,
|
||||
stderr: wrapper.stderr.slice(0, 500),
|
||||
stdout: wrapper.stdout.slice(0, 500),
|
||||
});
|
||||
fs.rmSync(trace, { force: true });
|
||||
fs.rmSync(resultFile, { force: true });
|
||||
fs.rmSync(`${resultFile}.stdout`, { force: true });
|
||||
fs.rmSync(`${resultFile}.stderr`, { force: true });
|
||||
throw Object.assign(new Error(`ETW observer failed closed: ${diagnostic}`), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
const raw = fs.readFileSync(trace, "utf8");
|
||||
const result = JSON.parse(fs.readFileSync(resultFile, "utf8"));
|
||||
fs.rmSync(trace, { force: true });
|
||||
fs.rmSync(resultFile, { force: true });
|
||||
return {
|
||||
result,
|
||||
observation: parseDelimitedObserver(raw, options.zones),
|
||||
backend: "windows-etw-kernel-process-tree",
|
||||
diagnostics: result.observerDiagnostics ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function windowsObserverBudgets(timeoutMs = 30_000) {
|
||||
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 15 * 60_000) {
|
||||
throw Object.assign(new Error("Windows observer timeout must be an integer from 1 to 900000 milliseconds"), {
|
||||
code: "AAS_OBSERVER_INVALID_TIMEOUT",
|
||||
});
|
||||
}
|
||||
return {
|
||||
candidateTimeoutMs: timeoutMs,
|
||||
wrapperTimeoutMs: timeoutMs + 60_000,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runObserved(executable, args, options) {
|
||||
fs.mkdirSync(options.evidenceDir, { recursive: true, mode: 0o700 });
|
||||
if (process.platform === "linux") return linuxObserved(executable, args, options);
|
||||
if (process.platform === "darwin") return macObserved(executable, args, options);
|
||||
if (process.platform === "win32") return windowsObserved(executable, args, options);
|
||||
throw Object.assign(new Error(`Unsupported observer platform: ${process.platform}`), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
}
|
||||
|
||||
export async function selfTestObserver(options) {
|
||||
const sentinel = path.join(options.evidenceDir, `sentinel-${process.pid}.txt`);
|
||||
const program = `const fs=require('node:fs'),net=require('node:net');const fd=fs.openSync(${JSON.stringify(sentinel)},'w',0o600);fs.writeSync(fd,'sentinel');fs.fsyncSync(fd);fs.closeSync(fd);const server=net.createServer(s=>s.end());server.listen(0,'127.0.0.1',()=>{const s=net.connect({host:'127.0.0.1',port:server.address().port},()=>s.end());s.on('close',()=>server.close(()=>setTimeout(()=>process.exit(0),500)));s.on('error',()=>process.exit(2));});setTimeout(()=>process.exit(1),3000);`;
|
||||
const observed = await runObserved(process.execPath, ["-e", program], { ...options, timeoutMs: 10_000 });
|
||||
fs.rmSync(sentinel, { force: true });
|
||||
if (observed.result.code !== 0 || observed.result.timedOut) {
|
||||
throw Object.assign(new Error("Observer sentinel process did not complete within its candidate budget"), {
|
||||
code: "AAS_OBSERVER_SELF_TEST_FAILED",
|
||||
});
|
||||
}
|
||||
if (observed.observation.networkAttempts < 1 || observed.observation.writeAttempts < 1) {
|
||||
const diagnostic = JSON.stringify({
|
||||
backend: observed.backend,
|
||||
networkAttempts: observed.observation.networkAttempts,
|
||||
writeAttempts: observed.observation.writeAttempts,
|
||||
diagnostics: observed.diagnostics ?? null,
|
||||
});
|
||||
throw Object.assign(new Error(`Observer missed sentinel network/write attempts: ${diagnostic}`), { code: "AAS_OBSERVER_SELF_TEST_FAILED" });
|
||||
}
|
||||
if (process.platform === "win32") await selfTestWindowsProcessTree(options);
|
||||
return {
|
||||
backend: observed.backend,
|
||||
contractVersion: "1.0.0",
|
||||
selfTestDigest: observed.observation.eventDigest,
|
||||
observedNetworkSentinels: observed.observation.networkAttempts,
|
||||
observedWriteSentinels: observed.observation.writeAttempts,
|
||||
host: { platform: os.platform(), release: os.release(), architecture: os.arch() },
|
||||
};
|
||||
}
|
||||
|
||||
async function selfTestWindowsProcessTree(options) {
|
||||
const drivers = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "drivers");
|
||||
const rootDriver = path.join(drivers, "windows-tree-root.ps1");
|
||||
const childDriver = path.join(drivers, "windows-tree-child.ps1");
|
||||
const jobSource = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "observers", "windows-job.cs");
|
||||
const powershell = resolveCommandPath("pwsh.exe") ?? resolveCommandPath("powershell.exe");
|
||||
if (!powershell) throw Object.assign(new Error("Windows PowerShell runtime is unavailable"), { code: "AAS_OBSERVER_UNAVAILABLE" });
|
||||
const readyCanary = path.join(options.evidenceDir, `child-ready-${process.pid}.txt`);
|
||||
const rootAckCanary = path.join(options.evidenceDir, `root-ack-${process.pid}.txt`);
|
||||
const childCanary = path.join(options.evidenceDir, `child-after-parent-${process.pid}.txt`);
|
||||
const observed = await runObserved(powershell, [
|
||||
"-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
||||
"-File", rootDriver,
|
||||
"-Powershell", powershell,
|
||||
"-ChildDriver", childDriver,
|
||||
"-JobSource", jobSource,
|
||||
"-ReadyCanary", readyCanary,
|
||||
"-RootAckCanary", rootAckCanary,
|
||||
"-AfterParentCanary", childCanary,
|
||||
], {
|
||||
...options,
|
||||
timeoutMs: 10_000,
|
||||
});
|
||||
const childPid = Number.parseInt(observed.result.stdout.trim(), 10);
|
||||
const sessionName = observed.diagnostics?.sessionName;
|
||||
let childAlive = false;
|
||||
if (Number.isSafeInteger(childPid) && childPid > 0) {
|
||||
try {
|
||||
process.kill(childPid, 0);
|
||||
childAlive = true;
|
||||
} catch {
|
||||
childAlive = false;
|
||||
}
|
||||
}
|
||||
const sessionProbe = sessionName
|
||||
? spawnSync("logman.exe", ["query", sessionName, "-ets"], { encoding: "utf8", windowsHide: true })
|
||||
: { status: 0 };
|
||||
const leftovers = fs.readdirSync(options.evidenceDir).filter((name) =>
|
||||
/^(?:AASVerifier-|windows-etw-|windows-result-)/.test(name) || /\.(?:etl|csv|stdout|stderr)$/.test(name));
|
||||
const readyCanaryWritten = fs.existsSync(readyCanary) && fs.readFileSync(readyCanary, "utf8") === "ready";
|
||||
const rootAckCanaryWritten = fs.existsSync(rootAckCanary) && fs.readFileSync(rootAckCanary, "utf8") === "ack";
|
||||
const childCanaryWritten = fs.existsSync(childCanary) && fs.readFileSync(childCanary, "utf8") === "child";
|
||||
fs.rmSync(readyCanary, { force: true });
|
||||
fs.rmSync(rootAckCanary, { force: true });
|
||||
fs.rmSync(childCanary, { force: true });
|
||||
const valid = observed.result.code === 124
|
||||
&& observed.result.timedOut === true
|
||||
&& observed.observation.childProcesses >= 1
|
||||
&& readyCanaryWritten
|
||||
&& rootAckCanaryWritten
|
||||
&& childCanaryWritten
|
||||
&& Number.isSafeInteger(childPid)
|
||||
&& !childAlive
|
||||
&& typeof sessionName === "string"
|
||||
&& sessionProbe.status !== 0
|
||||
&& leftovers.length === 0;
|
||||
if (!valid) {
|
||||
const diagnostic = JSON.stringify({
|
||||
code: observed.result.code,
|
||||
timedOut: observed.result.timedOut,
|
||||
childProcesses: observed.observation.childProcesses,
|
||||
childPid,
|
||||
childAlive,
|
||||
readyCanaryWritten,
|
||||
rootAckCanaryWritten,
|
||||
childCanaryWritten,
|
||||
totalRows: observed.diagnostics?.totalRows ?? null,
|
||||
processStartRows: observed.diagnostics?.processStartRows ?? null,
|
||||
jobTotalProcesses: observed.diagnostics?.jobTotalProcesses ?? null,
|
||||
rootStopRows: observed.diagnostics?.rootStopRows ?? null,
|
||||
postRootDescendantWriteRows: observed.diagnostics?.postRootDescendantWriteRows ?? null,
|
||||
rootEventSamples: observed.diagnostics?.rootEventSamples ?? null,
|
||||
stderr: observed.result.stderr.slice(0, 500),
|
||||
sessionName: sessionName ?? null,
|
||||
sessionStillExists: sessionProbe.status === 0,
|
||||
leftovers,
|
||||
});
|
||||
throw Object.assign(new Error(`Windows Job Object self-test failed: ${diagnostic}`), { code: "AAS_OBSERVER_SELF_TEST_FAILED" });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
const MASK = (1n << 64n) - 1n;
|
||||
|
||||
function rotateLeft(value, amount) {
|
||||
const shift = BigInt(amount);
|
||||
return ((value << shift) | (value >> (64n - shift))) & MASK;
|
||||
}
|
||||
|
||||
export function deriveState(rootSeedHex, namespace) {
|
||||
const key = Buffer.from(rootSeedHex, "hex");
|
||||
if (key.length !== 32) throw new Error("Root seed must be exactly 256 bits");
|
||||
let attempt = namespace;
|
||||
for (let retry = 0; retry < 100; retry += 1) {
|
||||
const bytes = crypto.createHmac("sha256", key).update(attempt, "utf8").digest();
|
||||
const state = [0, 8, 16, 24].map((offset) => bytes.readBigUInt64BE(offset));
|
||||
if (state.some((word) => word !== 0n)) return state;
|
||||
attempt = `${namespace}/retry/${retry + 1}`;
|
||||
}
|
||||
throw new Error("Could not derive a non-zero xoshiro256** state");
|
||||
}
|
||||
|
||||
export function nextUint64(state) {
|
||||
const result = (rotateLeft((state[1] * 5n) & MASK, 7) * 9n) & MASK;
|
||||
const temporary = (state[1] << 17n) & MASK;
|
||||
state[2] ^= state[0];
|
||||
state[3] ^= state[1];
|
||||
state[1] ^= state[2];
|
||||
state[0] ^= state[3];
|
||||
state[2] ^= temporary;
|
||||
state[3] = rotateLeft(state[3], 45);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function sampleInteger(state, upperExclusive) {
|
||||
if (!Number.isSafeInteger(upperExclusive) || upperExclusive <= 0) {
|
||||
throw new TypeError("upperExclusive must be a positive safe integer");
|
||||
}
|
||||
const bound = BigInt(upperExclusive);
|
||||
const limit = (1n << 64n) - ((1n << 64n) % bound);
|
||||
let value;
|
||||
do value = nextUint64(state); while (value >= limit);
|
||||
return Number(value % bound);
|
||||
}
|
||||
|
||||
export function stateHex(state) {
|
||||
return state.map((word) => word.toString(16).padStart(16, "0")).join("");
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
export function runProcess(executable, args, options = {}) {
|
||||
const timeoutMs = options.timeoutMs ?? 30_000;
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(executable, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
detached: options.detached === true,
|
||||
windowsHide: true,
|
||||
stdio: [options.stdin === undefined ? "ignore" : "pipe", "pipe", "pipe"],
|
||||
});
|
||||
if (typeof options.onSpawn === "function") options.onSpawn(child);
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
let stdoutBytes = 0;
|
||||
let stderrBytes = 0;
|
||||
const maxOutputBytes = options.maxOutputBytes ?? 4 * 1024 * 1024;
|
||||
let killedForOutput = false;
|
||||
const collect = (chunks, kind) => (chunk) => {
|
||||
const callback = kind === "stdout" ? options.onStdoutData : options.onStderrData;
|
||||
if (typeof callback === "function") callback(chunk);
|
||||
if (kind === "stdout") stdoutBytes += chunk.length;
|
||||
else stderrBytes += chunk.length;
|
||||
if (stdoutBytes + stderrBytes > maxOutputBytes) {
|
||||
killedForOutput = true;
|
||||
child.kill("SIGKILL");
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
};
|
||||
child.stdout.on("data", collect(stdout, "stdout"));
|
||||
child.stderr.on("data", collect(stderr, "stderr"));
|
||||
child.once("error", reject);
|
||||
if (options.stdin !== undefined) {
|
||||
child.stdin.end(options.stdin);
|
||||
}
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
||||
child.once("close", (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
resolve({
|
||||
code: code ?? 128,
|
||||
signal,
|
||||
stdout: Buffer.concat(stdout).toString("utf8"),
|
||||
stderr: Buffer.concat(stderr).toString("utf8"),
|
||||
timedOut: signal === "SIGKILL" && !killedForOutput,
|
||||
outputLimitExceeded: killedForOutput,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import addFormats from "ajv-formats";
|
||||
import { canonicalJson, digestJson, sha256 } from "./canonical.mjs";
|
||||
|
||||
export const JOB_IDS = Object.freeze([
|
||||
"linux-node-22", "linux-node-24", "macos-node-22", "macos-node-24",
|
||||
"windows-node-22", "windows-node-24",
|
||||
]);
|
||||
|
||||
export const SUITE_IDS = Object.freeze([
|
||||
"package", "entrypoints", "mcp", "property", "fuzz", "hostile",
|
||||
"legacy", "transaction", "adapters",
|
||||
]);
|
||||
|
||||
export function receiptDigest(receipt) {
|
||||
const { receiptDigest: _omitted, ...payload } = receipt;
|
||||
return digestJson(payload);
|
||||
}
|
||||
|
||||
export function finalizeReceipt(receipt) {
|
||||
return { ...receipt, receiptDigest: receiptDigest(receipt) };
|
||||
}
|
||||
|
||||
export function writeCanonicalReceipt(file, receipt) {
|
||||
const finalized = finalizeReceipt(receipt);
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(file, `${canonicalJson(finalized)}\n`, { mode: 0o600, flag: "wx" });
|
||||
return finalized;
|
||||
}
|
||||
|
||||
export function loadReceiptValidator(schemaFile) {
|
||||
const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
|
||||
addFormats(ajv);
|
||||
return ajv.compile(JSON.parse(fs.readFileSync(schemaFile, "utf8")));
|
||||
}
|
||||
|
||||
export function validateReceipt(receipt, validator) {
|
||||
const failures = [];
|
||||
if (!validator(receipt)) failures.push({ code: "AAS_VERIFIER_RECEIPT_SCHEMA", errors: validator.errors });
|
||||
if (receipt.receiptDigest !== receiptDigest(receipt)) failures.push({ code: "AAS_VERIFIER_RECEIPT_DIGEST" });
|
||||
const ids = (receipt.suites || []).map((entry) => entry.id);
|
||||
if (new Set(ids).size !== SUITE_IDS.length || SUITE_IDS.some((id) => !ids.includes(id))) {
|
||||
failures.push({ code: "AAS_VERIFIER_RECEIPT_SUITE_SET", ids });
|
||||
}
|
||||
for (const suite of receipt.suites || []) {
|
||||
if (suite.evidenceSha256 !== digestJson(suite.evidence)) failures.push({ code: "AAS_VERIFIER_SUITE_EVIDENCE_DIGEST", suite: suite.id });
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
export function executableDigest() {
|
||||
return sha256(fs.readFileSync(process.execPath));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { digestJson } from "./canonical.mjs";
|
||||
import { snapshotTree } from "./fs-evidence.mjs";
|
||||
import { runProcess } from "./process.mjs";
|
||||
|
||||
function npmExecutable() {
|
||||
return process.platform === "win32" ? "npm.cmd" : "npm";
|
||||
}
|
||||
|
||||
export async function installCandidate(tarball, root) {
|
||||
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(path.join(root, "package.json"), `${JSON.stringify({ private: true }, null, 2)}\n`, { mode: 0o600 });
|
||||
const npmCache = path.join(root, ".npm-cache");
|
||||
const result = await runProcess(npmExecutable(), [
|
||||
"install", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock", "--save=false", path.resolve(tarball),
|
||||
], {
|
||||
cwd: root,
|
||||
env: { ...process.env, npm_config_cache: npmCache, npm_config_ignore_scripts: "true" },
|
||||
timeoutMs: 180_000,
|
||||
maxOutputBytes: 8 * 1024 * 1024,
|
||||
});
|
||||
if (result.code !== 0) {
|
||||
const error = new Error(`Clean candidate install failed (${result.code})`);
|
||||
error.code = "AAS_VERIFIER_INSTALL_FAILED";
|
||||
error.detail = result.stderr.slice(0, 1000);
|
||||
throw error;
|
||||
}
|
||||
const packageRoot = path.join(root, "node_modules", "agentic-awesome-skills");
|
||||
const manifestPath = path.join(packageRoot, "package.json");
|
||||
if (!fs.existsSync(manifestPath)) throw new Error("Installed candidate package is missing");
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
||||
return {
|
||||
root,
|
||||
packageRoot,
|
||||
manifest,
|
||||
treeDigest: snapshotTree(packageRoot).digest,
|
||||
bins: Object.fromEntries(Object.entries(manifest.bin || {}).map(([name, relative]) => [name, path.join(packageRoot, relative)])),
|
||||
installReceiptDigest: digestJson({ manifest, treeDigest: snapshotTree(packageRoot).digest }),
|
||||
};
|
||||
}
|
||||
|
||||
export function isolatedZones(root) {
|
||||
const zones = Object.fromEntries(["home", "project", "cache", "tmp"].map((name) => [name, path.join(root, name)]));
|
||||
for (const directory of Object.values(zones)) fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
return zones;
|
||||
}
|
||||
|
||||
export function candidateEnvironment(zones, extra = {}) {
|
||||
const allowed = {};
|
||||
for (const key of ["PATH", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT"]) {
|
||||
if (process.env[key]) allowed[key] = process.env[key];
|
||||
}
|
||||
return {
|
||||
...allowed,
|
||||
HOME: zones.home,
|
||||
USERPROFILE: zones.home,
|
||||
TMPDIR: zones.tmp,
|
||||
TMP: zones.tmp,
|
||||
TEMP: zones.tmp,
|
||||
AAS_CACHE_ROOT: zones.cache,
|
||||
AAS_CACHE_DIR: zones.cache,
|
||||
NO_COLOR: "1",
|
||||
NODE_NO_WARNINGS: "1",
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseJsonLines(text) {
|
||||
return text.split(/\r?\n/).filter((line) => line.trim()).map((line) => JSON.parse(line));
|
||||
}
|
||||
|
||||
export function systemIdentity(job) {
|
||||
return {
|
||||
platform: process.platform,
|
||||
osVersion: os.version(),
|
||||
kernelVersion: os.release(),
|
||||
architecture: process.arch,
|
||||
nodeVersion: process.version,
|
||||
runnerImageLabel: process.env.AAS_VERIFIER_RUNNER_LABEL || "local-untrusted",
|
||||
runnerImageVersion: process.env.ImageVersion || process.env.AAS_VERIFIER_RUNNER_VERSION || "unknown",
|
||||
filesystemType: process.env.AAS_VERIFIER_FILESYSTEM_TYPE || "unprobed",
|
||||
filesystemCaseSensitivity: process.env.AAS_VERIFIER_FILESYSTEM_CASE || "unprobed",
|
||||
jobId: job.id,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import { digestJson } from "./canonical.mjs";
|
||||
import { snapshotZones, assertNoZoneDrift } from "./fs-evidence.mjs";
|
||||
import { runObserved } from "./observer.mjs";
|
||||
import { runProcess } from "./process.mjs";
|
||||
import { candidateEnvironment, parseJsonLines } from "./runtime.mjs";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const VERSION_FIELDS = {
|
||||
protocolVersion: "2025-06-18",
|
||||
coreVersion: "1.0.0",
|
||||
metadataSchemaVersion: "1.0.0",
|
||||
scorerVersion: "1.0.0",
|
||||
};
|
||||
|
||||
function suite(id, evidence, executions = 1) {
|
||||
return { id, status: "passed", executions, failures: 0, evidenceSha256: digestJson(evidence), evidence };
|
||||
}
|
||||
|
||||
function decodeToolPayload(response) {
|
||||
if (response?.error) return { rpcError: response.error };
|
||||
const text = response?.result?.content?.find?.((entry) => entry.type === "text")?.text;
|
||||
if (typeof text === "string") {
|
||||
try { return JSON.parse(text); } catch { return { text }; }
|
||||
}
|
||||
return response?.result;
|
||||
}
|
||||
|
||||
function assert(condition, code, message) {
|
||||
if (!condition) throw Object.assign(new Error(message), { code });
|
||||
}
|
||||
|
||||
function assertVersions(value, prefix = "response") {
|
||||
for (const [key, expected] of Object.entries(VERSION_FIELDS)) {
|
||||
assert(value?.[key] === expected, "AAS_VERIFIER_VERSION_CONTRACT", `${prefix}.${key} differs`);
|
||||
}
|
||||
}
|
||||
|
||||
function request(id, method, params = {}) {
|
||||
return { jsonrpc: "2.0", id, method, params };
|
||||
}
|
||||
|
||||
function parseMcpOutput(stdout) {
|
||||
const lines = stdout.split(/\r?\n/).filter((line) => line.trim());
|
||||
assert(lines.every((line) => Buffer.byteLength(line) <= 256 * 1024), "AAS_VERIFIER_MCP_RESULT_LIMIT", "MCP response exceeded 256 KiB");
|
||||
const responses = lines.map((line) => JSON.parse(line));
|
||||
assert(responses.every((entry) => entry.jsonrpc === "2.0"), "AAS_VERIFIER_MCP_STDOUT_NOISE", "MCP stdout contains non-protocol output");
|
||||
return responses;
|
||||
}
|
||||
|
||||
export async function verifyEntrypoints(runtime, zones) {
|
||||
const env = candidateEnvironment(zones);
|
||||
const help = await runProcess(process.execPath, [runtime.bins.aas, "--help"], { cwd: zones.project, env });
|
||||
assert(help.code === 0 && !help.signal, "AAS_VERIFIER_AAS_HELP", "aas --help failed");
|
||||
const values = parseJsonLines(help.stdout);
|
||||
assert(values.length === 1 && values[0].ok === true && values[0].status === "help", "AAS_VERIFIER_AAS_HELP_ENVELOPE", "aas --help envelope differs");
|
||||
assertVersions(values[0], "aas-help");
|
||||
const signatures = values[0].commands || [];
|
||||
for (const prefix of ["catalog status", "catalog update", "mcp configure", "mcp backups cleanup", "stack init", "stack recommend", "stack validate", "stack plan", "stack apply", "stack doctor", "stack recover"]) {
|
||||
assert(signatures.some((entry) => entry.startsWith(prefix)), "AAS_VERIFIER_COMMAND_MISSING", `Missing command signature: ${prefix}`);
|
||||
}
|
||||
const legacyHelp = await runProcess(process.execPath, [runtime.bins["agentic-awesome-skills"], "--help"], { cwd: zones.project, env });
|
||||
const legacyVersion = await runProcess(process.execPath, [runtime.bins["agentic-awesome-skills"], "--version"], { cwd: zones.project, env });
|
||||
assert(legacyHelp.code === 0 && legacyVersion.code === 0, "AAS_VERIFIER_LEGACY_ENTRYPOINT", "Legacy alias smoke failed");
|
||||
assert(!fs.existsSync(path.join(zones.project, "aas-stack.json")), "AAS_VERIFIER_LEGACY_STACK_STATE", "Legacy smoke created stack state");
|
||||
return suite("entrypoints", {
|
||||
helpEnvelope: values[0],
|
||||
legacyHelpDigest: digestJson({ stdout: legacyHelp.stdout, stderr: legacyHelp.stderr, code: legacyHelp.code }),
|
||||
legacyVersionDigest: digestJson({ stdout: legacyVersion.stdout, stderr: legacyVersion.stderr, code: legacyVersion.code }),
|
||||
}, 3);
|
||||
}
|
||||
|
||||
export async function verifyMcp(runtime, zones, evidenceDir) {
|
||||
const before = snapshotZones(zones);
|
||||
const requests = [
|
||||
request(1, "initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "aas-v1-verifier", version: "1" } }),
|
||||
{ jsonrpc: "2.0", method: "notifications/initialized", params: {} },
|
||||
request(2, "tools/list"),
|
||||
request(3, "resources/templates/list"),
|
||||
request(4, "tools/call", { name: "search_skills", arguments: { query: "react testing", target: "codex", limit: 5 } }),
|
||||
request(5, "tools/call", { name: "get_skill", arguments: { id: "frontend-design", includeContent: true } }),
|
||||
request(6, "tools/call", { name: "recommend_stack", arguments: {
|
||||
intent: "web-application-delivery",
|
||||
profile: { languages: ["typescript"], frameworks: ["react"] },
|
||||
targets: [{ host: "codex", scope: "project" }],
|
||||
criticalGoals: ["build", "test"],
|
||||
nonCriticalGoals: ["deploy"],
|
||||
policy: { allowedRisk: ["none", "safe"], requireKnownSource: true, allowManualSetup: false },
|
||||
maxSkills: 6,
|
||||
} }),
|
||||
];
|
||||
const input = `${requests.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
|
||||
const observed = await runObserved(process.execPath, [runtime.bins["aas-mcp"]], {
|
||||
cwd: zones.project,
|
||||
env: candidateEnvironment(zones),
|
||||
stdin: input,
|
||||
timeoutMs: 30_000,
|
||||
maxOutputBytes: 2 * 1024 * 1024,
|
||||
zones,
|
||||
evidenceDir,
|
||||
});
|
||||
assert(observed.result.code === 0, "AAS_VERIFIER_MCP_EXIT", "MCP smoke exited non-zero");
|
||||
assert(observed.observation.networkAttempts === 0, "AAS_VERIFIER_MCP_NETWORK_ATTEMPT", "MCP attempted network access");
|
||||
assert(observed.observation.writeAttempts === 0, "AAS_VERIFIER_MCP_WRITE_ATTEMPT", "MCP attempted persistent writes");
|
||||
assert(observed.observation.childProcesses === 0, "AAS_VERIFIER_MCP_CHILD_PROCESS", "MCP launched a child process");
|
||||
const after = snapshotZones(zones);
|
||||
assertNoZoneDrift(before, after);
|
||||
const responses = parseMcpOutput(observed.result.stdout);
|
||||
const byId = new Map(responses.filter((entry) => entry.id !== undefined).map((entry) => [entry.id, entry]));
|
||||
const initialize = byId.get(1)?.result;
|
||||
assert(initialize?.protocolVersion === "2025-06-18", "AAS_VERIFIER_MCP_PROTOCOL", "MCP initialize version differs");
|
||||
assertVersions(initialize?._meta, "mcp-initialize-meta");
|
||||
const toolNames = (byId.get(2)?.result?.tools || []).map((entry) => entry.name).sort();
|
||||
assert(JSON.stringify(toolNames) === JSON.stringify(["diff_stack", "get_skill", "inspect_stack", "recommend_stack", "search_skills"]), "AAS_VERIFIER_MCP_TOOL_SURFACE", "MCP tool surface differs");
|
||||
const templates = byId.get(3)?.result?.resourceTemplates || [];
|
||||
assert(templates.length === 1 && templates[0].uriTemplate === "aas://skills/{id}", "AAS_VERIFIER_MCP_RESOURCE_SURFACE", "MCP resource template differs");
|
||||
const getSkillText = JSON.stringify(byId.get(5));
|
||||
assert(getSkillText.includes("untrustedContent") && !getSkillText.includes("AAS_SECRET_CANARY_DO_NOT_EMIT_8f37ec75"), "AAS_VERIFIER_UNTRUSTED_CONTENT_BOUNDARY", "Skill content trust boundary missing or leaked canary");
|
||||
return suite("mcp", {
|
||||
requests: requests.length,
|
||||
responseDigest: digestJson(responses),
|
||||
canonicalResponseDigest: digestJson(decodeToolPayload(byId.get(6))),
|
||||
observation: observed.observation,
|
||||
before: Object.fromEntries(Object.entries(before).map(([name, value]) => [name, value.digest])),
|
||||
after: Object.fromEntries(Object.entries(after).map(([name, value]) => [name, value.digest])),
|
||||
stderrDigest: digestJson(observed.result.stderr),
|
||||
}, requests.length);
|
||||
}
|
||||
|
||||
export function packageSuite(inspection, runtime) {
|
||||
assert(inspection.failures.length === 0, "AAS_VERIFIER_PACKAGE_CONTENT", "Tarball content contract failed");
|
||||
assert(runtime.manifest.name === "agentic-awesome-skills", "AAS_VERIFIER_PACKAGE_NAME", "Package name differs");
|
||||
return suite("package", {
|
||||
tarballSha256: inspection.sha256,
|
||||
tarballSha512: inspection.sha512,
|
||||
entries: inspection.entries,
|
||||
installTreeSha256: runtime.treeDigest,
|
||||
installReceiptDigest: runtime.installReceiptDigest,
|
||||
}, inspection.entries.length);
|
||||
}
|
||||
|
||||
async function runDriver(id, driver, runtime, budget, jobIndex) {
|
||||
const result = await runProcess(process.execPath, [driver], {
|
||||
cwd: runtime.root,
|
||||
env: candidateEnvironment({ home: runtime.root, project: runtime.root, cache: runtime.root, tmp: runtime.root }),
|
||||
stdin: JSON.stringify({ packageRoot: runtime.packageRoot, budget, jobIndex, jobCount: 6 }),
|
||||
timeoutMs: 20 * 60_000,
|
||||
maxOutputBytes: 16 * 1024 * 1024,
|
||||
});
|
||||
assert(result.code === 0 && !result.timedOut && !result.outputLimitExceeded, `AAS_VERIFIER_${id.toUpperCase()}_DRIVER`, `${id} driver failed: ${result.stderr.slice(0, 500)}`);
|
||||
const values = parseJsonLines(result.stdout);
|
||||
assert(values.length === 1 && values[0].ok === true, `AAS_VERIFIER_${id.toUpperCase()}_RESULT`, `${id} result failed`);
|
||||
return suite(id, values[0], values[0].total);
|
||||
}
|
||||
|
||||
export function verifyProperty(runtime, budget, jobIndex, verifierRoot) {
|
||||
return runDriver("property", path.join(verifierRoot, "drivers", "property.cjs"), runtime, budget, jobIndex);
|
||||
}
|
||||
|
||||
export function verifyFuzz(runtime, budget, jobIndex, verifierRoot) {
|
||||
return runDriver("fuzz", path.join(verifierRoot, "drivers", "fuzz.cjs"), runtime, budget, jobIndex);
|
||||
}
|
||||
|
||||
export async function verifyHostile(runtime, zones, evidenceDir, hostileManifest, hostileRoot, verifierRoot) {
|
||||
const before = snapshotZones(zones);
|
||||
const observed = await runObserved(process.execPath, [path.join(verifierRoot, "drivers", "hostile.cjs")], {
|
||||
cwd: runtime.root,
|
||||
env: candidateEnvironment(zones),
|
||||
stdin: JSON.stringify({ packageRoot: runtime.packageRoot, manifest: hostileManifest, corpusRoot: hostileRoot }),
|
||||
timeoutMs: 120_000,
|
||||
maxOutputBytes: 4 * 1024 * 1024,
|
||||
zones,
|
||||
evidenceDir,
|
||||
});
|
||||
assert(observed.result.code === 0, "AAS_VERIFIER_HOSTILE_DRIVER", `Hostile driver failed: ${observed.result.stderr.slice(0, 500)}`);
|
||||
assert(observed.observation.networkAttempts === 0, "AAS_VERIFIER_HOSTILE_NETWORK", "Hostile suite attempted network access");
|
||||
assert(observed.observation.writeAttempts === 0, "AAS_VERIFIER_HOSTILE_WRITE", "Hostile suite attempted writes");
|
||||
assert(observed.observation.childProcesses === 0, "AAS_VERIFIER_HOSTILE_CHILD", "Hostile suite launched child code");
|
||||
const after = snapshotZones(zones);
|
||||
assertNoZoneDrift(before, after);
|
||||
const values = parseJsonLines(observed.result.stdout);
|
||||
assert(values.length === 1 && values[0].ok === true && values[0].executions === hostileManifest.classes.length * 2, "AAS_VERIFIER_HOSTILE_RESULT", "Hostile result failed");
|
||||
return suite("hostile", { ...values[0], observation: observed.observation }, values[0].executions);
|
||||
}
|
||||
|
||||
export async function prepareRuntimeCache(runtime, tarballBytes, integrity, cacheRoot) {
|
||||
const core = require(path.join(runtime.packageRoot, "tools/lib/aas-v1"));
|
||||
const parsed = core.cache.parsePackageArchive(tarballBytes, { limits: core.cache.RUNTIME_ARCHIVE_LIMITS });
|
||||
return core.cache.promoteRuntime({
|
||||
cacheRoot,
|
||||
release: {
|
||||
version: runtime.manifest.version,
|
||||
integrity,
|
||||
provenance: { registryOrigin: "https://registry.npmjs.org", signaturesPresent: false, attestationsPresent: false },
|
||||
},
|
||||
parsed,
|
||||
});
|
||||
}
|
||||
|
||||
function allOutput(result) {
|
||||
return `${result.stdout || ""}\n${result.stderr || ""}`;
|
||||
}
|
||||
|
||||
export async function verifyAdapters(runtime, zones, fixtureRoot, runtimeIntegrity, runtimeClosureDigest) {
|
||||
const hosts = [
|
||||
{ host: "codex", fixture: "codex-config.toml", config: "codex.toml", sentinel: "unknown_fixture_key" },
|
||||
{ host: "claude", fixture: "claude-config.json", config: "claude.json", sentinel: "unknownFixtureKey" },
|
||||
];
|
||||
const cases = [];
|
||||
for (const host of hosts) {
|
||||
const root = path.join(zones.project, `adapter-${host.host}`);
|
||||
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
||||
const config = path.join(root, host.config);
|
||||
const backupDir = path.join(root, "backups");
|
||||
fs.copyFileSync(path.join(fixtureRoot, host.fixture), config);
|
||||
if (process.platform !== "win32") fs.chmodSync(config, 0o600);
|
||||
const beforeStat = fs.statSync(config);
|
||||
const beforeBytes = fs.readFileSync(config);
|
||||
const common = ["mcp", "configure", "--host", host.host, "--scope", "project", "--config", config, "--cache-root", zones.cache, "--version", runtime.manifest.version, "--runtime-integrity", runtimeIntegrity, "--runtime-closure-digest", runtimeClosureDigest, "--backup-dir", backupDir];
|
||||
const preview = await runProcess(process.execPath, [runtime.bins.aas, ...common], { cwd: root, env: candidateEnvironment(zones), timeoutMs: 30_000 });
|
||||
assert(preview.code === 0, "AAS_VERIFIER_ADAPTER_PREVIEW", `${host.host} preview failed`);
|
||||
const previewValue = parseJsonLines(preview.stdout)[0];
|
||||
assert(previewValue?.status === "approvalRequired" && /^sha256-[a-f0-9]{64}$/.test(previewValue.approvalDigest), "AAS_VERIFIER_ADAPTER_APPROVAL", `${host.host} approval digest missing`);
|
||||
assert(fs.readFileSync(config).equals(beforeBytes), "AAS_VERIFIER_ADAPTER_PREVIEW_WRITE", `${host.host} preview changed config`);
|
||||
assert(!allOutput(preview).includes("AAS_SECRET_CANARY_DO_NOT_EMIT_"), "AAS_VERIFIER_ADAPTER_SECRET_LEAK", `${host.host} preview leaked canary`);
|
||||
assert(previewValue.runtime?.integrity === runtimeIntegrity, "AAS_VERIFIER_RUNTIME_INTEGRITY_BINDING", `${host.host} preview did not bind the exact candidate runtime SRI`);
|
||||
const apply = await runProcess(process.execPath, [runtime.bins.aas, ...common, "--approve", previewValue.approvalDigest], { cwd: root, env: candidateEnvironment(zones), timeoutMs: 30_000 });
|
||||
assert(apply.code === 0, "AAS_VERIFIER_ADAPTER_APPLY", `${host.host} apply failed: ${apply.stderr}`);
|
||||
assert(!allOutput(apply).includes("AAS_SECRET_CANARY_DO_NOT_EMIT_"), "AAS_VERIFIER_ADAPTER_SECRET_LEAK", `${host.host} apply leaked canary`);
|
||||
const afterBytes = fs.readFileSync(config);
|
||||
const afterText = afterBytes.toString("utf8");
|
||||
assert(afterText.includes(host.sentinel) && afterText.includes("preserve-me") && afterText.includes("existing-server"), "AAS_VERIFIER_ADAPTER_UNKNOWN_FIELD", `${host.host} did not preserve unknown/existing fields`);
|
||||
assert(afterText.includes("aas-mcp.js"), "AAS_VERIFIER_ADAPTER_MCP_ENTRY", `${host.host} did not configure AAS MCP`);
|
||||
const afterStat = fs.statSync(config);
|
||||
if (process.platform !== "win32") {
|
||||
assert((afterStat.mode & 0o777) === (beforeStat.mode & 0o777), "AAS_VERIFIER_ADAPTER_MODE", `${host.host} mode changed`);
|
||||
assert(afterStat.uid === beforeStat.uid && afterStat.gid === beforeStat.gid, "AAS_VERIFIER_ADAPTER_OWNER", `${host.host} owner changed`);
|
||||
}
|
||||
const backupEntries = fs.readdirSync(backupDir).filter((name) => !name.startsWith(".")).sort();
|
||||
const backups = backupEntries.filter((name) => name.endsWith(".bak"));
|
||||
const metadata = backupEntries.filter((name) => name.endsWith(".json"));
|
||||
assert(backups.length === 1 && metadata.length === 1, "AAS_VERIFIER_ADAPTER_BACKUP", `${host.host} backup pair count differs`);
|
||||
if (process.platform !== "win32") {
|
||||
for (const name of backupEntries) {
|
||||
const backupStat = fs.statSync(path.join(backupDir, name));
|
||||
assert((backupStat.mode & 0o077) === 0, "AAS_VERIFIER_ADAPTER_BACKUP_MODE", `${host.host} backup entry is not user-only`);
|
||||
}
|
||||
}
|
||||
const cleanupBase = ["mcp", "backups", "cleanup", "--config", config, "--backup-dir", backupDir, "--keep", "0"];
|
||||
const cleanupPreview = await runProcess(process.execPath, [runtime.bins.aas, ...cleanupBase], { cwd: root, env: candidateEnvironment(zones) });
|
||||
const cleanupValue = parseJsonLines(cleanupPreview.stdout)[0];
|
||||
assert(cleanupPreview.code === 0 && /^sha256-[a-f0-9]{64}$/.test(cleanupValue?.approvalDigest), "AAS_VERIFIER_ADAPTER_CLEANUP_PREVIEW", `${host.host} backup cleanup preview failed`);
|
||||
const cleanup = await runProcess(process.execPath, [runtime.bins.aas, ...cleanupBase, "--approve", cleanupValue.approvalDigest], { cwd: root, env: candidateEnvironment(zones) });
|
||||
assert(cleanup.code === 0 && fs.readdirSync(backupDir).filter((name) => !name.startsWith(".")).length === 0, "AAS_VERIFIER_ADAPTER_CLEANUP", `${host.host} backup cleanup failed`);
|
||||
|
||||
const unsafe = path.join(root, `unsafe-${host.config}`);
|
||||
const outside = path.join(root, `outside-${host.config}`);
|
||||
fs.copyFileSync(path.join(fixtureRoot, host.fixture), outside);
|
||||
try {
|
||||
fs.symlinkSync(outside, unsafe, process.platform === "win32" ? "file" : undefined);
|
||||
const unsafeResult = await runProcess(process.execPath, [runtime.bins.aas, "mcp", "configure", "--host", host.host, "--scope", "project", "--config", unsafe, "--cache-root", zones.cache, "--version", runtime.manifest.version], { cwd: root, env: candidateEnvironment(zones) });
|
||||
assert(unsafeResult.code !== 0, "AAS_VERIFIER_ADAPTER_SYMLINK", `${host.host} accepted symlink config`);
|
||||
} finally { fs.rmSync(unsafe, { force: true }); }
|
||||
cases.push({ host: host.host, previewDigest: digestJson(previewValue), applyDigest: digestJson(parseJsonLines(apply.stdout)[0]), backupCount: backups.length, cleanupDigest: digestJson(cleanupValue) });
|
||||
}
|
||||
return suite("adapters", { runtimeIntegrity, runtimeClosureDigest, cases, invariants: ["unknown-fields-preserved", "minimal-patch", "mode-owner-preserved", "atomic-write", "user-only-backup", "secret-redacted", "unsafe-file-rejected"] }, cases.length * 4);
|
||||
}
|
||||
|
||||
export async function verifyLegacy(runtime, zones, verifierRoot, corpusRoot) {
|
||||
const driver = path.resolve(verifierRoot, "drivers", "legacy.mjs");
|
||||
const absoluteCorpusRoot = path.resolve(corpusRoot);
|
||||
const workRoot = path.join(zones.tmp, "legacy");
|
||||
const result = await runProcess(process.execPath, [driver], {
|
||||
cwd: zones.project,
|
||||
env: candidateEnvironment(zones),
|
||||
stdin: JSON.stringify({ packageRoot: runtime.packageRoot, runtimeRoot: runtime.root, corpusRoot: absoluteCorpusRoot, workRoot }),
|
||||
timeoutMs: 10 * 60_000,
|
||||
maxOutputBytes: 8 * 1024 * 1024,
|
||||
});
|
||||
const values = parseJsonLines(result.stdout);
|
||||
assert(result.code === 0 && values.length === 1 && values[0].ok === true && values[0].executions === 41, "AAS_VERIFIER_LEGACY_DIFFERENTIAL", `Legacy differential failed: ${result.stderr.slice(0, 500)} ${result.stdout.slice(0, 1000)}`);
|
||||
return suite("legacy", values[0], values[0].executions);
|
||||
}
|
||||
|
||||
export { suite };
|
||||
@@ -0,0 +1,117 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import zlib from "node:zlib";
|
||||
import { sha256, sha512 } from "./canonical.mjs";
|
||||
|
||||
function textField(block, offset, length) {
|
||||
return block.subarray(offset, offset + length).toString("utf8").replace(/\0.*$/s, "");
|
||||
}
|
||||
|
||||
function octalField(block, offset, length) {
|
||||
const value = textField(block, offset, length).trim();
|
||||
if (!/^[0-7]*$/.test(value)) throw new Error(`Invalid tar octal field: ${JSON.stringify(value)}`);
|
||||
return Number.parseInt(value || "0", 8);
|
||||
}
|
||||
|
||||
export function parseTarGzip(bytes) {
|
||||
const expanded = zlib.gunzipSync(bytes, { maxOutputLength: 512 * 1024 * 1024 });
|
||||
if (expanded.length % 512 !== 0) throw new Error("Tarball is not block aligned");
|
||||
const entries = [];
|
||||
let offset = 0;
|
||||
let zeroBlocks = 0;
|
||||
while (offset + 512 <= expanded.length) {
|
||||
const header = expanded.subarray(offset, offset + 512);
|
||||
offset += 512;
|
||||
if (header.every((byte) => byte === 0)) {
|
||||
zeroBlocks += 1;
|
||||
if (zeroBlocks === 2) break;
|
||||
continue;
|
||||
}
|
||||
zeroBlocks = 0;
|
||||
const checksumHeader = Buffer.from(header);
|
||||
checksumHeader.fill(0x20, 148, 156);
|
||||
const actualChecksum = [...checksumHeader].reduce((total, byte) => total + byte, 0);
|
||||
if (actualChecksum !== octalField(header, 148, 8)) throw new Error("Tar header checksum mismatch");
|
||||
const name = textField(header, 0, 100);
|
||||
const prefix = textField(header, 345, 155);
|
||||
const entryPath = prefix ? `${prefix}/${name}` : name;
|
||||
const size = octalField(header, 124, 12);
|
||||
const type = textField(header, 156, 1) || "0";
|
||||
if (offset + size > expanded.length) throw new Error(`${entryPath}: tar entry exceeds archive`);
|
||||
const content = expanded.subarray(offset, offset + size);
|
||||
entries.push({
|
||||
path: entryPath,
|
||||
type,
|
||||
size,
|
||||
mode: octalField(header, 100, 8),
|
||||
linkName: textField(header, 157, 100),
|
||||
sha256: type === "0" || type === "\0" ? sha256(content) : null,
|
||||
content,
|
||||
});
|
||||
offset += size + ((512 - (size % 512)) % 512);
|
||||
}
|
||||
if (zeroBlocks !== 2) throw new Error("Tarball lacks two terminal zero blocks");
|
||||
return entries;
|
||||
}
|
||||
|
||||
function portablePath(value) {
|
||||
if (!value || value.includes("\\") || value.startsWith("/") || /^[A-Za-z]:/.test(value)) return false;
|
||||
const segments = value.split("/");
|
||||
return segments.every((segment) => segment && segment !== "." && segment !== "..");
|
||||
}
|
||||
|
||||
const ALLOWED_PACKAGE_PATHS = [
|
||||
/^package\/(?:LICENSE(?:\.[^/]+)?|README(?:\.[^/]+)?|package\.json)$/i,
|
||||
/^package\/tools\/(?:bin|lib)\/[A-Za-z0-9._/-]+$/,
|
||||
/^package\/(?:data|schemas)\/[A-Za-z0-9._/-]+$/,
|
||||
/^package\/skills\/.+$/,
|
||||
/^package\/skills_index\.json$/,
|
||||
/^package\/node_modules\/(?:ajv|fast-deep-equal|fast-uri|json-schema-traverse|require-from-string|sanitize-filename|truncate-utf8-bytes|utf8-byte-length|yaml)\/[A-Za-z0-9@._+/-]+$/,
|
||||
];
|
||||
|
||||
const FORBIDDEN_NAMES = /(?:^|\/)(?:\.git|verification|coverage|\.env(?:\.(?!example$|sample$|template$)[^/]*)?|[^/]*\.(?:pem|key|p12|pfx|log))(?:\/|$)/i;
|
||||
|
||||
export function inspectPackageTarball(tarballPath) {
|
||||
const bytes = fs.readFileSync(tarballPath);
|
||||
const entries = parseTarGzip(bytes);
|
||||
const failures = [];
|
||||
const normalized = new Set();
|
||||
for (const entry of entries) {
|
||||
if (!portablePath(entry.path)) failures.push({ code: "PACKAGE_UNSAFE_PATH", path: entry.path });
|
||||
if (!["0", "\0", "5"].includes(entry.type)) failures.push({ code: "PACKAGE_NON_REGULAR_ENTRY", path: entry.path, type: entry.type });
|
||||
const collisionKey = entry.path.normalize("NFC").toLowerCase();
|
||||
if (normalized.has(collisionKey)) failures.push({ code: "PACKAGE_PATH_COLLISION", path: entry.path });
|
||||
normalized.add(collisionKey);
|
||||
if (entry.type !== "5" && !ALLOWED_PACKAGE_PATHS.some((pattern) => pattern.test(entry.path))) {
|
||||
failures.push({ code: "PACKAGE_PATH_NOT_ALLOWLISTED", path: entry.path });
|
||||
}
|
||||
if (FORBIDDEN_NAMES.test(entry.path)) failures.push({ code: "PACKAGE_SENSITIVE_OR_CHECKOUT_PATH", path: entry.path });
|
||||
if ((entry.mode & 0o7000) !== 0 || (entry.mode & 0o002) !== 0) failures.push({ code: "PACKAGE_UNSAFE_MODE", path: entry.path, mode: entry.mode });
|
||||
}
|
||||
const packageEntry = entries.find((entry) => entry.path === "package/package.json" && entry.type === "0");
|
||||
if (!packageEntry) failures.push({ code: "PACKAGE_JSON_MISSING" });
|
||||
let manifest = null;
|
||||
if (packageEntry) {
|
||||
manifest = JSON.parse(packageEntry.content.toString("utf8"));
|
||||
const bins = manifest.bin || {};
|
||||
for (const [name, expected] of Object.entries({
|
||||
aas: "tools/bin/aas.js",
|
||||
"aas-mcp": "tools/bin/aas-mcp.js",
|
||||
"agentic-awesome-skills": "tools/bin/install.js",
|
||||
})) {
|
||||
if (bins[name] !== expected) failures.push({ code: "PACKAGE_BIN_CONTRACT", name, expected, actual: bins[name] });
|
||||
}
|
||||
for (const target of Object.values(bins)) {
|
||||
if (!entries.some((entry) => entry.path === `package/${target}` && entry.type === "0")) failures.push({ code: "PACKAGE_BIN_TARGET_MISSING", target });
|
||||
}
|
||||
}
|
||||
return {
|
||||
tarballPath: path.resolve(tarballPath),
|
||||
bytes: bytes.length,
|
||||
sha256: sha256(bytes),
|
||||
sha512: sha512(bytes),
|
||||
entries: entries.map(({ content, ...entry }) => entry),
|
||||
manifest,
|
||||
failures,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$Executable,
|
||||
[Parameter(Mandatory = $true)][string]$ArgumentsBase64,
|
||||
[Parameter(Mandatory = $true)][string]$TraceOutput,
|
||||
[Parameter(Mandatory = $true)][string]$ResultOutput,
|
||||
[Parameter(Mandatory = $true)][ValidatePattern('^AASVerifier-[A-Za-z0-9-]+$')][string]$SessionName,
|
||||
[Parameter(Mandatory = $true)][ValidateRange(1, 900000)][int]$CandidateTimeoutMilliseconds
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$jobSource = Join-Path $PSScriptRoot "windows-job.cs"
|
||||
if (!(Test-Path -LiteralPath $jobSource -PathType Leaf)) { throw "Windows Job Object helper is unavailable" }
|
||||
Add-Type -Path $jobSource
|
||||
$etl = Join-Path ([IO.Path]::GetDirectoryName($TraceOutput)) "$SessionName.etl"
|
||||
$csv = Join-Path ([IO.Path]::GetDirectoryName($TraceOutput)) "$SessionName.csv"
|
||||
$arguments = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($ArgumentsBase64)) | ConvertFrom-Json
|
||||
$providers = @(
|
||||
"Microsoft-Windows-Kernel-Process",
|
||||
"Microsoft-Windows-Kernel-File",
|
||||
"Microsoft-Windows-Winsock-AFD",
|
||||
"Microsoft-Windows-DNS-Client"
|
||||
)
|
||||
$jobProcess = $null
|
||||
|
||||
try {
|
||||
& logman.exe create trace $SessionName -o $etl -ets | Out-Null
|
||||
foreach ($provider in $providers) {
|
||||
& logman.exe update trace $SessionName -p $provider 0xffffffffffffffff 0xff -ets | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw "Unable to enable ETW provider $provider" }
|
||||
}
|
||||
$started = [DateTimeOffset]::UtcNow
|
||||
$jobProcess = [AasVerifier.JobProcess]::Start(
|
||||
$Executable,
|
||||
[string[]]$arguments,
|
||||
"$ResultOutput.stdout",
|
||||
"$ResultOutput.stderr"
|
||||
)
|
||||
$waitResult = [AasVerifier.JobProcess]::Wait($jobProcess, $CandidateTimeoutMilliseconds)
|
||||
$timedOut = $waitResult -eq [AasVerifier.JobProcess]::WaitTimeout
|
||||
if ($timedOut) {
|
||||
[AasVerifier.JobProcess]::Terminate($jobProcess, 124)
|
||||
$waitResult = [AasVerifier.JobProcess]::Wait($jobProcess, 5000)
|
||||
}
|
||||
if ($waitResult -ne [AasVerifier.JobProcess]::WaitObject0) {
|
||||
throw "Windows Job Object did not reach an empty process-tree state"
|
||||
}
|
||||
$rootExitCode = [AasVerifier.JobProcess]::ExitCode($jobProcess)
|
||||
$jobTotalProcesses = [AasVerifier.JobProcess]::TotalProcesses($jobProcess)
|
||||
$ended = [DateTimeOffset]::UtcNow
|
||||
& logman.exe stop $SessionName -ets | Out-Null
|
||||
& tracerpt.exe $etl -of CSV -o $csv -y | Out-Null
|
||||
if ($LASTEXITCODE -ne 0 -or !(Test-Path -LiteralPath $csv -PathType Leaf)) { throw "ETW trace export failed" }
|
||||
$rootPid = $jobProcess.ProcessId
|
||||
$lines = New-Object System.Collections.Generic.List[string]
|
||||
for ($processIndex = 1; $processIndex -lt $jobTotalProcesses; $processIndex++) {
|
||||
$lines.Add("process|job-object|index=$processIndex")
|
||||
}
|
||||
$childPids = New-Object System.Collections.Generic.HashSet[int]
|
||||
$childPids.Add($rootPid) | Out-Null
|
||||
function Get-IntegerField($row, [string[]]$patterns) {
|
||||
foreach ($property in $row.PSObject.Properties) {
|
||||
foreach ($pattern in $patterns) {
|
||||
if ($property.Name -match $pattern) {
|
||||
$parsed = 0
|
||||
$raw = ([string]$property.Value).Trim()
|
||||
if ([int]::TryParse($raw, [ref]$parsed)) { return $parsed }
|
||||
if ($raw -match '^0[xX][0-9a-fA-F]+$') {
|
||||
try { return [Convert]::ToInt32($raw.Substring(2), 16) } catch { }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
function Convert-ObservedInteger([string]$raw) {
|
||||
$value = $raw.Trim().Trim('"').Trim("'")
|
||||
$parsed = 0
|
||||
if ([int]::TryParse($value, [ref]$parsed)) { return $parsed }
|
||||
if ($value -match '^0[xX][0-9a-fA-F]+$') {
|
||||
try { return [Convert]::ToInt32($value.Substring(2), 16) } catch { }
|
||||
}
|
||||
return 0
|
||||
}
|
||||
function Get-PayloadInteger($row, [string]$name) {
|
||||
foreach ($property in $row.PSObject.Properties) {
|
||||
if (([string]$property.Name).Trim() -ieq $name) {
|
||||
$direct = Convert-ObservedInteger ([string]$property.Value)
|
||||
if ($direct -gt 0) { return $direct }
|
||||
}
|
||||
}
|
||||
$payloadPattern = '(?i)(?:^|[;,{\s])["'']?' + [regex]::Escape($name) + '["'']?\s*[:=]\s*["'']?(0[xX][0-9a-fA-F]+|[0-9]+)["'']?'
|
||||
foreach ($property in $row.PSObject.Properties) {
|
||||
$text = [string]$property.Value
|
||||
$match = [regex]::Match($text, $payloadPattern)
|
||||
if ($match.Success) { return Convert-ObservedInteger $match.Groups[1].Value }
|
||||
}
|
||||
return 0
|
||||
}
|
||||
$totalRows = 0
|
||||
$rootRows = 0
|
||||
$networkRows = 0
|
||||
$writeRows = 0
|
||||
$winsockCreateRows = 0
|
||||
$winsockDecodedPids = New-Object System.Collections.Generic.List[int]
|
||||
$processStartRows = 0
|
||||
$rootStopRows = 0
|
||||
$rootExitObserved = $false
|
||||
$postRootDescendantWriteRows = 0
|
||||
$rootEventSamples = New-Object System.Collections.Generic.List[string]
|
||||
foreach ($row in (Import-Csv -LiteralPath $csv)) {
|
||||
$totalRows++
|
||||
$serialized = $row | ConvertTo-Json -Compress
|
||||
$eventName = (($row.PSObject.Properties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join " ")
|
||||
$providerName = ([string]$row.'Event Name').Trim()
|
||||
$eventType = ([string]$row.Type).Trim()
|
||||
$eventId = Get-IntegerField $row @("(?i)^Event ID$")
|
||||
$opcode = Get-IntegerField $row @("(?i)^Opcode$")
|
||||
if ($providerName -eq 'Microsoft-Windows-Kernel-Process' -and $eventType -eq 'Start' -and $opcode -eq 1) {
|
||||
$processStartRows++
|
||||
# Process Start events are emitted in the creating process context, so the
|
||||
# ETW header PID is the parent. The payload ProcessID is the new process.
|
||||
$parentPid = Get-IntegerField $row @("(?i)^PID$")
|
||||
if ($parentPid -eq 0) { $parentPid = Get-PayloadInteger $row 'ParentProcessID' }
|
||||
$newPid = Get-PayloadInteger $row 'ProcessID'
|
||||
if ($childPids.Contains($parentPid) -and $newPid -gt 0 -and !$childPids.Contains($newPid)) {
|
||||
$childPids.Add($newPid) | Out-Null
|
||||
$lines.Add("process|$newPid|parent=$parentPid")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ($providerName -eq 'Microsoft-Windows-Kernel-Process' -and $eventType -eq 'Stop' -and $opcode -eq 2) {
|
||||
$stoppedPid = Get-PayloadInteger $row 'ProcessID'
|
||||
if ($stoppedPid -eq 0) { $stoppedPid = Get-IntegerField $row @("(?i)^PID$") }
|
||||
if ($stoppedPid -eq $rootPid) {
|
||||
$rootStopRows++
|
||||
$rootExitObserved = $true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ($providerName -like 'Microsoft-Windows-Winsock*' -and $eventId -eq 1000) {
|
||||
$winsockCreateRows++
|
||||
$userModePid = Get-PayloadInteger $row 'UserModePid'
|
||||
if ($userModePid -gt 0 -and $winsockDecodedPids.Count -lt 8 -and !$winsockDecodedPids.Contains($userModePid)) {
|
||||
$winsockDecodedPids.Add($userModePid)
|
||||
}
|
||||
$networkPid = Get-IntegerField $row @("(?i)^Process.*Id$", "(?i)^PID$")
|
||||
if ($childPids.Contains($networkPid) -or $childPids.Contains($userModePid)) {
|
||||
$networkRows++
|
||||
$lines.Add("network|$networkPid|provider=$providerName;event=$eventId")
|
||||
}
|
||||
continue
|
||||
}
|
||||
$pidValue = Get-IntegerField $row @("(?i)^Process.*Id$", "(?i)^PID$")
|
||||
if (!$childPids.Contains($pidValue)) { continue }
|
||||
$rootRows++
|
||||
if ($rootEventSamples.Count -lt 12) {
|
||||
$safeIdentity = (($row.PSObject.Properties | Where-Object {
|
||||
$_.Name -match '(?i)^(Event Name|Type|Event ID|Opcode|Task|Keyword|PID|Provider Name|Provider Guid)$'
|
||||
} | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ';')
|
||||
if ($safeIdentity -and !$rootEventSamples.Contains($safeIdentity)) { $rootEventSamples.Add($safeIdentity) }
|
||||
}
|
||||
if ($providerName -eq 'Microsoft-Windows-DNS-Client') {
|
||||
$networkRows++
|
||||
$lines.Add("network|$pidValue|provider=$providerName;event=$eventId")
|
||||
}
|
||||
elseif ($providerName -eq 'Microsoft-Windows-Kernel-File' -and $eventId -in @(16,17,18,19)) {
|
||||
$writeRows++
|
||||
if ($rootExitObserved -and $pidValue -ne $rootPid) { $postRootDescendantWriteRows++ }
|
||||
$lines.Add("write|$pidValue|provider=$providerName;event=$eventId")
|
||||
}
|
||||
}
|
||||
[IO.File]::WriteAllLines($TraceOutput, $lines, (New-Object Text.UTF8Encoding($false)))
|
||||
$receipt = [ordered]@{
|
||||
code = $(if ($timedOut) { 124 } else { $rootExitCode })
|
||||
signal = $null
|
||||
stdout = [IO.File]::ReadAllText("$ResultOutput.stdout")
|
||||
stderr = [IO.File]::ReadAllText("$ResultOutput.stderr")
|
||||
timedOut = $timedOut
|
||||
outputLimitExceeded = $false
|
||||
startedAt = $started.ToString("o")
|
||||
endedAt = $ended.ToString("o")
|
||||
observerDiagnostics = [ordered]@{
|
||||
totalRows = $totalRows
|
||||
rootRows = $rootRows
|
||||
networkRows = $networkRows
|
||||
writeRows = $writeRows
|
||||
winsockCreateRows = $winsockCreateRows
|
||||
winsockDecodedPids = @($winsockDecodedPids)
|
||||
processStartRows = $processStartRows
|
||||
rootStopRows = $rootStopRows
|
||||
postRootDescendantWriteRows = $postRootDescendantWriteRows
|
||||
rootEventSamples = @($rootEventSamples)
|
||||
sessionName = $SessionName
|
||||
processTreeTimedOut = $timedOut
|
||||
processTreeEmpty = $waitResult -eq [AasVerifier.JobProcess]::WaitObject0
|
||||
jobTotalProcesses = $jobTotalProcesses
|
||||
}
|
||||
}
|
||||
[IO.File]::WriteAllText($ResultOutput, ($receipt | ConvertTo-Json -Compress), (New-Object Text.UTF8Encoding($false)))
|
||||
}
|
||||
finally {
|
||||
& logman.exe stop $SessionName -ets 2>$null | Out-Null
|
||||
[AasVerifier.JobProcess]::Close($jobProcess)
|
||||
Remove-Item -LiteralPath $etl,$csv,"$ResultOutput.stdout","$ResultOutput.stderr" -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace AasVerifier
|
||||
{
|
||||
public sealed class JobProcessHandles
|
||||
{
|
||||
public IntPtr JobHandle;
|
||||
public IntPtr ProcessHandle;
|
||||
public int ProcessId;
|
||||
}
|
||||
|
||||
public static class JobProcess
|
||||
{
|
||||
public const uint WaitObject0 = 0x00000000;
|
||||
public const uint WaitTimeout = 0x00000102;
|
||||
public const uint WaitFailed = 0xffffffff;
|
||||
|
||||
private const uint GenericRead = 0x80000000;
|
||||
private const uint GenericWrite = 0x40000000;
|
||||
private const uint FileShareRead = 0x00000001;
|
||||
private const uint FileShareWrite = 0x00000002;
|
||||
private const uint CreateAlways = 2;
|
||||
private const uint OpenExisting = 3;
|
||||
private const uint FileAttributeNormal = 0x00000080;
|
||||
private const uint CreateSuspended = 0x00000004;
|
||||
private const uint CreateNoWindow = 0x08000000;
|
||||
private const uint StartfUseStdHandles = 0x00000100;
|
||||
private const uint JobObjectLimitKillOnJobClose = 0x00002000;
|
||||
private const uint Synchronize = 0x00100000;
|
||||
private const int JobObjectBasicAccountingInformationClass = 1;
|
||||
private const int JobObjectExtendedLimitInformationClass = 9;
|
||||
private static readonly IntPtr InvalidHandleValue = new IntPtr(-1);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct SecurityAttributes
|
||||
{
|
||||
public int Length;
|
||||
public IntPtr SecurityDescriptor;
|
||||
[MarshalAs(UnmanagedType.Bool)] public bool InheritHandle;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
|
||||
private struct StartupInfo
|
||||
{
|
||||
public int Size;
|
||||
public string Reserved;
|
||||
public string Desktop;
|
||||
public string Title;
|
||||
public int X;
|
||||
public int Y;
|
||||
public int XSize;
|
||||
public int YSize;
|
||||
public int XCountChars;
|
||||
public int YCountChars;
|
||||
public int FillAttribute;
|
||||
public int Flags;
|
||||
public short ShowWindow;
|
||||
public short Reserved2;
|
||||
public IntPtr Reserved2Pointer;
|
||||
public IntPtr StandardInput;
|
||||
public IntPtr StandardOutput;
|
||||
public IntPtr StandardError;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct ProcessInformation
|
||||
{
|
||||
public IntPtr Process;
|
||||
public IntPtr Thread;
|
||||
public int ProcessId;
|
||||
public int ThreadId;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct JobObjectBasicLimitInformation
|
||||
{
|
||||
public long PerProcessUserTimeLimit;
|
||||
public long PerJobUserTimeLimit;
|
||||
public uint LimitFlags;
|
||||
public UIntPtr MinimumWorkingSetSize;
|
||||
public UIntPtr MaximumWorkingSetSize;
|
||||
public uint ActiveProcessLimit;
|
||||
public UIntPtr Affinity;
|
||||
public uint PriorityClass;
|
||||
public uint SchedulingClass;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct JobObjectBasicAccountingInformation
|
||||
{
|
||||
public long TotalUserTime;
|
||||
public long TotalKernelTime;
|
||||
public long ThisPeriodTotalUserTime;
|
||||
public long ThisPeriodTotalKernelTime;
|
||||
public uint TotalPageFaultCount;
|
||||
public uint TotalProcesses;
|
||||
public uint ActiveProcesses;
|
||||
public uint TotalTerminatedProcesses;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct IoCounters
|
||||
{
|
||||
public ulong ReadOperationCount;
|
||||
public ulong WriteOperationCount;
|
||||
public ulong OtherOperationCount;
|
||||
public ulong ReadTransferCount;
|
||||
public ulong WriteTransferCount;
|
||||
public ulong OtherTransferCount;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct JobObjectExtendedLimitInformation
|
||||
{
|
||||
public JobObjectBasicLimitInformation BasicLimitInformation;
|
||||
public IoCounters IoInfo;
|
||||
public UIntPtr ProcessMemoryLimit;
|
||||
public UIntPtr JobMemoryLimit;
|
||||
public UIntPtr PeakProcessMemoryUsed;
|
||||
public UIntPtr PeakJobMemoryUsed;
|
||||
}
|
||||
|
||||
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
|
||||
private static extern IntPtr CreateJobObjectW(IntPtr jobAttributes, string name);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool SetInformationJobObject(
|
||||
IntPtr job,
|
||||
int informationClass,
|
||||
ref JobObjectExtendedLimitInformation information,
|
||||
int informationLength);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool TerminateJobObject(IntPtr job, uint exitCode);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool QueryInformationJobObject(
|
||||
IntPtr job,
|
||||
int informationClass,
|
||||
ref JobObjectBasicAccountingInformation information,
|
||||
int informationLength,
|
||||
IntPtr returnLength);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool GetExitCodeProcess(IntPtr process, out uint exitCode);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
private static extern uint ResumeThread(IntPtr thread);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool TerminateProcess(IntPtr process, uint exitCode);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
private static extern IntPtr OpenProcess(uint desiredAccess, [MarshalAs(UnmanagedType.Bool)] bool inheritHandle, int processId);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
private static extern uint WaitForSingleObject(IntPtr handle, uint milliseconds);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool CloseHandle(IntPtr handle);
|
||||
|
||||
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
|
||||
private static extern IntPtr CreateFileW(
|
||||
string fileName,
|
||||
uint desiredAccess,
|
||||
uint shareMode,
|
||||
ref SecurityAttributes securityAttributes,
|
||||
uint creationDisposition,
|
||||
uint flagsAndAttributes,
|
||||
IntPtr templateFile);
|
||||
|
||||
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool CreateProcessW(
|
||||
string applicationName,
|
||||
StringBuilder commandLine,
|
||||
IntPtr processAttributes,
|
||||
IntPtr threadAttributes,
|
||||
[MarshalAs(UnmanagedType.Bool)] bool inheritHandles,
|
||||
uint creationFlags,
|
||||
IntPtr environment,
|
||||
string currentDirectory,
|
||||
ref StartupInfo startupInfo,
|
||||
out ProcessInformation processInformation);
|
||||
|
||||
private static void ThrowLastError(string operation)
|
||||
{
|
||||
throw new Win32Exception(Marshal.GetLastWin32Error(), operation);
|
||||
}
|
||||
|
||||
private static string QuoteArgument(string argument)
|
||||
{
|
||||
if (argument == null) throw new ArgumentNullException("argument");
|
||||
if (argument.Length > 0 && argument.IndexOfAny(new[] { ' ', '\t', '\n', '\v', '"' }) < 0) return argument;
|
||||
StringBuilder output = new StringBuilder();
|
||||
output.Append('"');
|
||||
int backslashes = 0;
|
||||
foreach (char character in argument)
|
||||
{
|
||||
if (character == '\\')
|
||||
{
|
||||
backslashes++;
|
||||
}
|
||||
else if (character == '"')
|
||||
{
|
||||
output.Append('\\', backslashes * 2 + 1);
|
||||
output.Append('"');
|
||||
backslashes = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
output.Append('\\', backslashes);
|
||||
output.Append(character);
|
||||
backslashes = 0;
|
||||
}
|
||||
}
|
||||
output.Append('\\', backslashes * 2);
|
||||
output.Append('"');
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
private static StringBuilder BuildCommandLine(string executable, string[] arguments)
|
||||
{
|
||||
List<string> tokens = new List<string>();
|
||||
tokens.Add(QuoteArgument(executable));
|
||||
foreach (string argument in arguments) tokens.Add(QuoteArgument(argument));
|
||||
return new StringBuilder(string.Join(" ", tokens.ToArray()));
|
||||
}
|
||||
|
||||
public static JobProcessHandles Start(
|
||||
string executable,
|
||||
string[] arguments,
|
||||
string stdoutPath,
|
||||
string stderrPath)
|
||||
{
|
||||
IntPtr job = IntPtr.Zero;
|
||||
IntPtr standardInput = InvalidHandleValue;
|
||||
IntPtr standardOutput = InvalidHandleValue;
|
||||
IntPtr standardError = InvalidHandleValue;
|
||||
ProcessInformation process = new ProcessInformation();
|
||||
bool processCreated = false;
|
||||
bool processAssigned = false;
|
||||
try
|
||||
{
|
||||
job = CreateJobObjectW(IntPtr.Zero, null);
|
||||
if (job == IntPtr.Zero) ThrowLastError("CreateJobObjectW failed");
|
||||
JobObjectExtendedLimitInformation limits = new JobObjectExtendedLimitInformation();
|
||||
limits.BasicLimitInformation.LimitFlags = JobObjectLimitKillOnJobClose;
|
||||
if (!SetInformationJobObject(
|
||||
job,
|
||||
JobObjectExtendedLimitInformationClass,
|
||||
ref limits,
|
||||
Marshal.SizeOf(typeof(JobObjectExtendedLimitInformation))))
|
||||
{
|
||||
ThrowLastError("SetInformationJobObject failed");
|
||||
}
|
||||
|
||||
SecurityAttributes inheritable = new SecurityAttributes();
|
||||
inheritable.Length = Marshal.SizeOf(typeof(SecurityAttributes));
|
||||
inheritable.InheritHandle = true;
|
||||
standardInput = CreateFileW("NUL", GenericRead, FileShareRead | FileShareWrite, ref inheritable, OpenExisting, FileAttributeNormal, IntPtr.Zero);
|
||||
standardOutput = CreateFileW(stdoutPath, GenericWrite, FileShareRead | FileShareWrite, ref inheritable, CreateAlways, FileAttributeNormal, IntPtr.Zero);
|
||||
standardError = CreateFileW(stderrPath, GenericWrite, FileShareRead | FileShareWrite, ref inheritable, CreateAlways, FileAttributeNormal, IntPtr.Zero);
|
||||
if (standardInput == InvalidHandleValue || standardOutput == InvalidHandleValue || standardError == InvalidHandleValue)
|
||||
{
|
||||
ThrowLastError("CreateFileW for redirected streams failed");
|
||||
}
|
||||
|
||||
StartupInfo startup = new StartupInfo();
|
||||
startup.Size = Marshal.SizeOf(typeof(StartupInfo));
|
||||
startup.Flags = (int)StartfUseStdHandles;
|
||||
startup.StandardInput = standardInput;
|
||||
startup.StandardOutput = standardOutput;
|
||||
startup.StandardError = standardError;
|
||||
if (!CreateProcessW(
|
||||
executable,
|
||||
BuildCommandLine(executable, arguments),
|
||||
IntPtr.Zero,
|
||||
IntPtr.Zero,
|
||||
true,
|
||||
CreateSuspended | CreateNoWindow,
|
||||
IntPtr.Zero,
|
||||
null,
|
||||
ref startup,
|
||||
out process))
|
||||
{
|
||||
ThrowLastError("CreateProcessW failed");
|
||||
}
|
||||
processCreated = true;
|
||||
if (!AssignProcessToJobObject(job, process.Process)) ThrowLastError("AssignProcessToJobObject failed");
|
||||
processAssigned = true;
|
||||
if (ResumeThread(process.Thread) == WaitFailed) ThrowLastError("ResumeThread failed");
|
||||
CloseHandle(process.Thread);
|
||||
process.Thread = IntPtr.Zero;
|
||||
return new JobProcessHandles { JobHandle = job, ProcessHandle = process.Process, ProcessId = process.ProcessId };
|
||||
}
|
||||
catch (Exception startError)
|
||||
{
|
||||
Exception cleanupError = null;
|
||||
if (processCreated && !processAssigned && process.Process != IntPtr.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!TerminateProcess(process.Process, 125)) ThrowLastError("TerminateProcess for unassigned root failed");
|
||||
if (WaitForSingleObject(process.Process, 5000) != WaitObject0)
|
||||
{
|
||||
throw new InvalidOperationException("Unassigned suspended root did not terminate within the cleanup budget");
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
cleanupError = error;
|
||||
}
|
||||
}
|
||||
if (job != IntPtr.Zero) CloseHandle(job);
|
||||
if (process.Thread != IntPtr.Zero) CloseHandle(process.Thread);
|
||||
if (process.Process != IntPtr.Zero) CloseHandle(process.Process);
|
||||
if (cleanupError != null)
|
||||
{
|
||||
throw new AggregateException("Candidate start failed and cleanup of the unassigned root also failed", startError, cleanupError);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (standardInput != InvalidHandleValue) CloseHandle(standardInput);
|
||||
if (standardOutput != InvalidHandleValue) CloseHandle(standardOutput);
|
||||
if (standardError != InvalidHandleValue) CloseHandle(standardError);
|
||||
}
|
||||
}
|
||||
|
||||
public static uint Wait(JobProcessHandles handles, int milliseconds)
|
||||
{
|
||||
Stopwatch stopwatch = Stopwatch.StartNew();
|
||||
while (true)
|
||||
{
|
||||
JobObjectBasicAccountingInformation accounting = new JobObjectBasicAccountingInformation();
|
||||
if (!QueryInformationJobObject(
|
||||
handles.JobHandle,
|
||||
JobObjectBasicAccountingInformationClass,
|
||||
ref accounting,
|
||||
Marshal.SizeOf(typeof(JobObjectBasicAccountingInformation)),
|
||||
IntPtr.Zero))
|
||||
{
|
||||
ThrowLastError("QueryInformationJobObject failed");
|
||||
}
|
||||
if (accounting.ActiveProcesses == 0) return WaitObject0;
|
||||
if (stopwatch.ElapsedMilliseconds >= milliseconds) return WaitTimeout;
|
||||
Thread.Sleep(Math.Min(25, Math.Max(1, milliseconds - (int)stopwatch.ElapsedMilliseconds)));
|
||||
}
|
||||
}
|
||||
|
||||
public static uint TotalProcesses(JobProcessHandles handles)
|
||||
{
|
||||
JobObjectBasicAccountingInformation accounting = new JobObjectBasicAccountingInformation();
|
||||
if (!QueryInformationJobObject(
|
||||
handles.JobHandle,
|
||||
JobObjectBasicAccountingInformationClass,
|
||||
ref accounting,
|
||||
Marshal.SizeOf(typeof(JobObjectBasicAccountingInformation)),
|
||||
IntPtr.Zero))
|
||||
{
|
||||
ThrowLastError("QueryInformationJobObject failed");
|
||||
}
|
||||
return accounting.TotalProcesses;
|
||||
}
|
||||
|
||||
public static void Terminate(JobProcessHandles handles, uint exitCode)
|
||||
{
|
||||
if (!TerminateJobObject(handles.JobHandle, exitCode)) ThrowLastError("TerminateJobObject failed");
|
||||
}
|
||||
|
||||
public static uint ExitCode(JobProcessHandles handles)
|
||||
{
|
||||
uint exitCode;
|
||||
if (!GetExitCodeProcess(handles.ProcessHandle, out exitCode)) ThrowLastError("GetExitCodeProcess failed");
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
public static bool WaitForProcessExit(int processId, int milliseconds)
|
||||
{
|
||||
if (processId <= 0) throw new ArgumentOutOfRangeException("processId");
|
||||
if (milliseconds < 1 || milliseconds > 900000) throw new ArgumentOutOfRangeException("milliseconds");
|
||||
IntPtr process = OpenProcess(Synchronize, false, processId);
|
||||
if (process == IntPtr.Zero) ThrowLastError("OpenProcess for parent synchronization failed");
|
||||
try
|
||||
{
|
||||
uint result = WaitForSingleObject(process, (uint)milliseconds);
|
||||
if (result == WaitObject0) return true;
|
||||
if (result == WaitTimeout) return false;
|
||||
ThrowLastError("WaitForSingleObject for parent synchronization failed");
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
CloseHandle(process);
|
||||
}
|
||||
}
|
||||
|
||||
public static void Close(JobProcessHandles handles)
|
||||
{
|
||||
if (handles == null) return;
|
||||
if (handles.JobHandle != IntPtr.Zero)
|
||||
{
|
||||
CloseHandle(handles.JobHandle);
|
||||
handles.JobHandle = IntPtr.Zero;
|
||||
}
|
||||
if (handles.ProcessHandle != IntPtr.Zero)
|
||||
{
|
||||
CloseHandle(handles.ProcessHandle);
|
||||
handles.ProcessHandle = IntPtr.Zero;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"name": "aas-v1-independent-verifier",
|
||||
"version": "1.0.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "aas-v1-independent-verifier",
|
||||
"version": "1.0.1",
|
||||
"dependencies": {
|
||||
"ajv": "8.17.1",
|
||||
"ajv-formats": "3.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "8.17.1",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
|
||||
"integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
"json-schema-traverse": "^1.0.0",
|
||||
"require-from-string": "^2.0.2"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/epoberezkin"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv-formats": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz",
|
||||
"integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ajv": "^8.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"ajv": "^8.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"ajv": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz",
|
||||
"integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/json-schema-traverse": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
|
||||
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/require-from-string": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
|
||||
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "aas-v1-independent-verifier",
|
||||
"version": "1.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"check:structure": "node bin/check-baseline.mjs --mode structure",
|
||||
"check:freeze-ready": "node bin/check-baseline.mjs --mode freeze-ready",
|
||||
"check:schemas": "node bin/validate-schemas.mjs",
|
||||
"check:benchmark": "node bin/validate-benchmark.mjs",
|
||||
"check:benchmark:frozen": "node bin/validate-benchmark.mjs --require-approvals",
|
||||
"check:secondary": "node bin/validate-secondary-corpora.mjs",
|
||||
"check:secondary:frozen": "node bin/validate-secondary-corpora.mjs --require-approvals && node bin/validate-tuning-gold-equivalence-audit.mjs",
|
||||
"check:tuning-gold-audit": "node bin/validate-tuning-gold-equivalence-audit.mjs",
|
||||
"freeze:write": "node bin/freeze-baseline.mjs --write",
|
||||
"freeze:check": "node bin/freeze-baseline.mjs",
|
||||
"verify:product": "node bin/verify-product.mjs",
|
||||
"verify:merge": "node bin/merge-product-evidence.mjs",
|
||||
"test:platform": "node bin/self-test-platform.mjs",
|
||||
"test": "node --test tests/*.test.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": "8.17.1",
|
||||
"ajv-formats": "3.0.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { aggregateReceipts } from "../lib/aggregate.mjs";
|
||||
import { digestJson } from "../lib/canonical.mjs";
|
||||
import { finalizeReceipt, JOB_IDS, loadReceiptValidator, SUITE_IDS } from "../lib/receipt.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const validator = loadReceiptValidator(path.resolve(here, "..", "..", "schemas", "product-verifier-receipt.schema.json"));
|
||||
const d = `sha256-${"a".repeat(64)}`;
|
||||
const sri = `sha512-${Buffer.alloc(64).toString("base64")}`;
|
||||
|
||||
function receipt(job, index) {
|
||||
const totals = { property: index < 4 ? 16667 : 16666, fuzz: index < 2 ? 8334 : 8333 };
|
||||
const suites = SUITE_IDS.map((id) => {
|
||||
const evidence = id === "property" ? { total: totals.property, hardPolicyViolations: 0 }
|
||||
: id === "fuzz" ? { total: totals.fuzz }
|
||||
: id === "hostile" ? { executions: 64 }
|
||||
: id === "legacy" ? { executions: 41 }
|
||||
: id === "transaction" ? { faultBoundaryClasses: ["lock", "journal", "backup", "write", "fsync", "rename", "commit"], raceClasses: ["concurrency", "drift", "symlink-swap", "target-swap", "corrupt-journal", "recovery-race"] }
|
||||
: {};
|
||||
return { id, status: "passed", executions: evidence.total || evidence.executions || 1, failures: 0, evidenceSha256: digestJson(evidence), evidence };
|
||||
});
|
||||
return finalizeReceipt({ schemaVersion: 1, receiptVersion: "1.0.0", status: "passed",
|
||||
job: { id: job, workflowRunId: "1", workflowRunAttempt: "1" },
|
||||
candidate: { commit: "1".repeat(40), package: "agentic-awesome-skills", version: "14.6.0", tarballBytes: 1, tarballSha256: d, tarballSha512: sri, packManifestSha256: d, installTreeSha256: d },
|
||||
verifier: { version: "1.0.0", commit: "2".repeat(40), rootDigest: d, contractDigest: d, owner: "aas-v1-independent-verifier" },
|
||||
environment: { platform: job.startsWith("linux") ? "linux" : job.startsWith("macos") ? "darwin" : "win32", osVersion: "test", kernelVersion: "test", architecture: "x64", nodeVersion: job.endsWith("22") ? "v22.23.1" : "v24.18.0", nodeExecutableSha256: d, runnerImageLabel: "test", runnerImageVersion: "test", filesystemType: job.startsWith("linux") ? "ext4" : job.startsWith("macos") ? "apfs" : "ntfs", filesystemCaseSensitivity: job.startsWith("linux") ? "sensitive" : "insensitive-preserving" },
|
||||
observer: { contractVersion: "1.0.0", backend: job.startsWith("linux") ? "linux-strace-process-tree" : job.startsWith("macos") ? "macos-fs_usage-process" : "windows-etw-kernel-process-tree", selfTestDigest: d, networkSentinels: 1, writeSentinels: 1, overflow: false, ambiguousLineage: false },
|
||||
zones: Object.fromEntries(["home", "project", "cache", "tmp"].map((name) => [name, { beforeSha256: d, afterSha256: d, persistentWriteCount: 0 }])),
|
||||
suites, canonicalPayload: { sha256: d, excludedFields: ["timestamp", "correlationId", "localizedMessage", "diagnostics"], sampleCount: 60 }, failures: [] });
|
||||
}
|
||||
|
||||
test("aggregator accepts only the exact complete matrix", () => {
|
||||
const receipts = JOB_IDS.map(receipt);
|
||||
assert.equal(aggregateReceipts(receipts, validator).status, "passed");
|
||||
const missing = aggregateReceipts(receipts.slice(1), validator);
|
||||
assert.equal(missing.status, "failed");
|
||||
assert.ok(missing.failures.some((entry) => entry.code === "AAS_VERIFIER_MATRIX_MISSING_JOB"));
|
||||
});
|
||||
|
||||
test("aggregator catches receipt tampering and duplicates", () => {
|
||||
const receipts = JOB_IDS.map(receipt);
|
||||
receipts[0].candidate.tarballBytes = 2;
|
||||
const result = aggregateReceipts([...receipts.slice(0, 5), receipts[0]], validator);
|
||||
assert.equal(result.status, "failed");
|
||||
assert.ok(result.failures.some((entry) => entry.code === "AAS_VERIFIER_RECEIPT_DIGEST"));
|
||||
assert.ok(result.failures.some((entry) => entry.code === "AAS_VERIFIER_MATRIX_DUPLICATE_JOB"));
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const checker = path.resolve(here, "..", "bin", "check-baseline.mjs");
|
||||
|
||||
test("frozen structure-only baseline is internally valid", () => {
|
||||
const run = spawnSync(process.execPath, [checker, "--mode", "structure"], { encoding: "utf8" });
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const report = JSON.parse(run.stdout);
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.heldOutDescriptors, 180);
|
||||
assert.equal(report.pendingCount, 0);
|
||||
});
|
||||
|
||||
test("freeze-ready mode never reports an incomplete baseline as complete", () => {
|
||||
const run = spawnSync(process.execPath, [checker, "--mode", "freeze-ready"], { encoding: "utf8" });
|
||||
if (run.status === 0) {
|
||||
const report = JSON.parse(run.stdout);
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.pendingCount, 0);
|
||||
return;
|
||||
}
|
||||
assert.equal(run.status, 2, run.stdout || run.stderr);
|
||||
const report = JSON.parse(run.stderr);
|
||||
assert.equal(report.code, "AAS_BASELINE_NOT_FREEZE_READY");
|
||||
assert.ok(report.pending.length > 0);
|
||||
assert.ok(report.pending.every((entry) => entry.code.endsWith("_PENDING")));
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { canonicalJson, digestJson, parseCanonicalJson } from "../lib/canonical.mjs";
|
||||
|
||||
test("canonical JSON uses deterministic UTF-16 property ordering", () => {
|
||||
const value = { "\udfff": 1, "\ue000": 2, a: 3 };
|
||||
assert.equal(canonicalJson(value), '{"a":3,"\\udfff":1,"\ue000":2}');
|
||||
assert.equal(parseCanonicalJson(canonicalJson(value)).a, 3);
|
||||
assert.match(digestJson(value), /^sha256-[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
test("canonical JSON rejects values outside the JSON data model", () => {
|
||||
assert.throws(() => canonicalJson({ value: undefined }), /cannot encode/);
|
||||
assert.throws(() => canonicalJson(Number.NaN), /non-finite/);
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
aggregateIntent,
|
||||
caseInclusionAssessment,
|
||||
caseInclusionPrecision,
|
||||
evaluateCase,
|
||||
isCorrectAbstention,
|
||||
macroAverage,
|
||||
} from "../lib/metrics.mjs";
|
||||
|
||||
const gold = {
|
||||
criticalGoals: ["critical-a", "critical-b"],
|
||||
nonCriticalGoals: ["optional-a", "optional-b", "optional-c", "optional-d", "optional-e"],
|
||||
minimumNonCriticalGoalCoverage: 0.8,
|
||||
requiresSkill: true,
|
||||
};
|
||||
|
||||
const acceptedSolutions = [{
|
||||
solutionId: "solution-a",
|
||||
allowedSkillIds: ["skill-a", "skill-helper"],
|
||||
requiredGroups: [["skill-a"]],
|
||||
}];
|
||||
|
||||
const validResult = {
|
||||
terminal: true,
|
||||
schemaValid: true,
|
||||
hardPolicyViolations: [],
|
||||
coveredGoals: ["critical-a", "critical-b", "optional-a", "optional-b", "optional-c", "optional-d"],
|
||||
includedSkillIds: ["skill-a"],
|
||||
discoveryPromotions: [],
|
||||
};
|
||||
|
||||
test("verified case requires all critical and at least 80 percent non-critical coverage", () => {
|
||||
assert.equal(evaluateCase(validResult, gold, acceptedSolutions).verified, true);
|
||||
assert.equal(evaluateCase({ ...validResult, coveredGoals: validResult.coveredGoals.slice(0, -1) }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, coveredGoals: validResult.coveredGoals.slice(1) }, gold, acceptedSolutions).verified, false);
|
||||
});
|
||||
|
||||
test("hard policy, timeout, empty required stack, and hidden discovery override fail verification", () => {
|
||||
assert.equal(evaluateCase({ ...validResult, hardPolicyViolations: ["risk"] }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, timedOut: true }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, includedSkillIds: [] }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, discoveryPromotions: [{ id: "skill-a", visibleOverride: false }] }, gold, acceptedSolutions).verified, false);
|
||||
});
|
||||
|
||||
test("candidate coverage self-claims cannot replace independently accepted required groups", () => {
|
||||
const lying = { ...validResult, includedSkillIds: ["skill-helper"] };
|
||||
const report = evaluateCase(lying, gold, acceptedSolutions);
|
||||
assert.equal(report.criticalGoalCoverage, 1);
|
||||
assert.equal(report.independentlyAcceptedStack, false);
|
||||
assert.equal(report.verified, false);
|
||||
});
|
||||
|
||||
test("precision scores one coherent alternative rather than a union", () => {
|
||||
const solutions = [
|
||||
{ allowedSkillIds: ["a", "a-helper"] },
|
||||
{ allowedSkillIds: ["b", "b-helper"] },
|
||||
];
|
||||
assert.equal(caseInclusionPrecision(["a", "a-helper"], solutions), 1);
|
||||
assert.equal(caseInclusionPrecision(["a", "b"], solutions), 0.5);
|
||||
assert.equal(caseInclusionPrecision([], solutions), null);
|
||||
assert.deepEqual(caseInclusionAssessment(["a", "a-helper"], solutions), {
|
||||
acceptedCount: 2,
|
||||
inclusionCount: 2,
|
||||
precision: 1,
|
||||
matchedSolutionId: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
test("coverage denominator stays 30 when results are missing", () => {
|
||||
const reports = Array.from({ length: 24 }, () => ({
|
||||
verified: true,
|
||||
acceptedInclusionCount: 9,
|
||||
inclusionCount: 10,
|
||||
}));
|
||||
const aggregate = aggregateIntent(reports);
|
||||
assert.equal(aggregate.verifiedCoverage, 0.8);
|
||||
assert.equal(aggregate.emptyStackCount, 6);
|
||||
assert.ok(Math.abs(aggregate.inclusionPrecision - 0.9) < Number.EPSILON * 24);
|
||||
assert.equal(aggregateIntent(reports.slice(0, 23)).verifiedCoverage < 0.8, true);
|
||||
});
|
||||
|
||||
test("intent precision is the mean of per-stack precision rather than pooled inclusions", () => {
|
||||
const aggregate = aggregateIntent([
|
||||
{ verified: true, acceptedInclusionCount: 1, inclusionCount: 1 },
|
||||
{ verified: true, acceptedInclusionCount: 1, inclusionCount: 9 },
|
||||
]);
|
||||
assert.equal(aggregate.inclusionPrecision, (1 + (1 / 9)) / 2);
|
||||
assert.equal(aggregate.acceptedInclusions, 2);
|
||||
assert.equal(aggregate.totalInclusions, 10);
|
||||
});
|
||||
|
||||
test("macro average requires six intents and is not a micro average", () => {
|
||||
assert.equal(macroAverage([0.8, 0.8, 0.8, 0.8, 0.8, 1].map((value) => ({ value })), "value"), 5 / 6);
|
||||
assert.throws(() => macroAverage([{ value: 1 }], "value"), /exactly six/i);
|
||||
});
|
||||
|
||||
test("correct abstention is successful, insufficient, and empty", () => {
|
||||
assert.equal(isCorrectAbstention({ ok: true, status: "insufficientCoverage", proposedStack: [] }), true);
|
||||
assert.equal(isCorrectAbstention({ ok: false, status: "insufficientCoverage", proposedStack: [] }), false);
|
||||
assert.equal(isCorrectAbstention({ ok: true, status: "insufficientCoverage", proposedStack: ["weak"] }), false);
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { macObserverBudgets, parseDelimitedObserver, parseLinuxStrace, parseMacCombinedFsUsage, parseMacFsUsage, windowsObserverBudgets } from "../lib/observer.mjs";
|
||||
|
||||
test("strace parser counts failed network attempts and non-stream writes", () => {
|
||||
const result = parseLinuxStrace([
|
||||
'execve("/usr/bin/node", ["node"], 0x0) = 0',
|
||||
'connect(7<TCP:[1]>, {sa_family=AF_INET}, 16) = -1 ECONNREFUSED',
|
||||
'openat(AT_FDCWD, "/tmp/x", O_WRONLY|O_CREAT, 0600) = 8',
|
||||
'write(8</tmp/x>, "x", 1) = 1',
|
||||
'write(1</dev/pts/1>, "ok", 2) = 2',
|
||||
'execve("/bin/true", ["true"], 0x0) = 0',
|
||||
].join("\n"), { tmp: "/tmp" });
|
||||
assert.equal(result.networkAttempts, 1);
|
||||
assert.equal(result.writeAttempts, 2);
|
||||
assert.equal(result.childProcesses, 1);
|
||||
assert.ok(result.events.every((entry) => !JSON.stringify(entry).includes("/tmp/x")));
|
||||
});
|
||||
|
||||
test("delimited observers ignore malformed and unknown records", () => {
|
||||
const result = parseDelimitedObserver("network|connect|127.0.0.1\nwrite|open|/tmp/x\nnoise|secret\n");
|
||||
assert.deepEqual([result.networkAttempts, result.writeAttempts, result.childProcesses], [1, 1, 0]);
|
||||
});
|
||||
|
||||
test("Windows observer separates the candidate limit from ETW finalization grace", () => {
|
||||
assert.deepEqual(windowsObserverBudgets(10_000), {
|
||||
candidateTimeoutMs: 10_000,
|
||||
wrapperTimeoutMs: 70_000,
|
||||
});
|
||||
assert.throws(() => windowsObserverBudgets(0), (error) => error.code === "AAS_OBSERVER_INVALID_TIMEOUT");
|
||||
assert.throws(() => windowsObserverBudgets(900_001), (error) => error.code === "AAS_OBSERVER_INVALID_TIMEOUT");
|
||||
});
|
||||
|
||||
test("macOS observer lifetime covers readiness, candidate, drain, and stop margin", () => {
|
||||
assert.deepEqual(macObserverBudgets(10_000), {
|
||||
startupMs: 1_500,
|
||||
readinessMaxAttempts: 2,
|
||||
readinessProcessTimeoutMs: 10_000,
|
||||
readinessDelayMs: 250,
|
||||
drainMs: 1_000,
|
||||
observerTimeoutMs: 38_000,
|
||||
});
|
||||
assert.throws(() => macObserverBudgets(0), (error) => error.code === "AAS_OBSERVER_INVALID_TIMEOUT");
|
||||
});
|
||||
|
||||
test("macOS fs_usage parser separates network, writes, and child execs", () => {
|
||||
const result = parseMacFsUsage(
|
||||
"12:00:00.100 WrData[A] F=3 /tmp/canary node.1\n12:00:00.200 read F=4 /tmp/input node.1\n",
|
||||
"12:00:00.300 connect 127.0.0.1:9 node.1\n",
|
||||
"12:00:00.400 exec node node.1\n12:00:00.500 exec child node.2\n",
|
||||
);
|
||||
assert.equal(result.networkAttempts, 1);
|
||||
assert.equal(result.writeAttempts, 1);
|
||||
assert.equal(result.childProcesses, 1);
|
||||
});
|
||||
|
||||
test("combined macOS fs_usage parser enforces canary ordering and classifies candidate calls", () => {
|
||||
const result = parseMacCombinedFsUsage([
|
||||
"12:00:00.005 WrData[A] F=3 /tmp/aas-ready-1 aasobs.1",
|
||||
"12:00:00.010 WrData[A] F=3 /tmp/aas-start-1 aasobs.1",
|
||||
"12:00:00.020 WrData[A] F=3 /tmp/canary node.1",
|
||||
"12:00:00.030 connect 127.0.0.1:9 node.1",
|
||||
"12:00:00.040 posix_spawn child node.1",
|
||||
].join("\n"), {}, "aas-ready-1", "aas-start-1");
|
||||
assert.deepEqual([result.networkAttempts, result.writeAttempts, result.childProcesses], [1, 1, 1]);
|
||||
assert.throws(() => parseMacCombinedFsUsage("12:00:00.010 WrData[A] F=3 /tmp/aas-start-1 aasobs.1\n", {}, "aas-ready-1", "aas-start-1"), /readiness canary/);
|
||||
assert.throws(() => parseMacCombinedFsUsage("12:00:00.010 WrData[A] F=3 /tmp/aas-ready-1 aasobs.1\n", {}, "aas-ready-1", "aas-start-1"), /candidate start canary/);
|
||||
});
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { loadReceiptValidator } from "../lib/receipt.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const validate = loadReceiptValidator(path.resolve(here, "..", "..", "schemas", "product-transaction-evidence.schema.json"));
|
||||
const digest = `sha256-${"a".repeat(64)}`;
|
||||
const fault = ["lock", "journal", "backup", "write", "fsync", "rename", "commit"];
|
||||
const race = ["concurrency", "drift", "symlink-swap", "target-swap", "corrupt-journal", "recovery-race"];
|
||||
|
||||
function evidence() {
|
||||
return {
|
||||
schemaVersion: 1, status: "passed", productionBinary: true, testMode: false, mocked: false,
|
||||
observer: { backend: "linux-strace-process-tree", eventDigest: digest, overflow: false, ambiguousLineage: false },
|
||||
faultBoundaryClasses: fault, raceClasses: race, executions: 13, killExecutions: 7,
|
||||
swapExecutions: 2, recoveryExecutions: 2, partialStates: 0, unmanagedMutations: 0,
|
||||
hardPolicyViolations: 0,
|
||||
boundaryEvidence: [...fault, ...race].map((value, index) => ({
|
||||
class: value, observedOperation: `event-${index}`,
|
||||
injectionAction: index < fault.length ? "kill" : value === "concurrency" ? "concurrent" : value,
|
||||
beforeDigest: digest, afterDigest: digest, finalState: index % 2 ? "new" : "previous", noPartialState: true,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
test("transaction evidence requires full external black-box coverage", () => {
|
||||
assert.equal(validate(evidence()), true, JSON.stringify(validate.errors));
|
||||
const mocked = evidence();
|
||||
mocked.mocked = true;
|
||||
assert.equal(validate(mocked), false);
|
||||
const incomplete = evidence();
|
||||
incomplete.faultBoundaryClasses.pop();
|
||||
assert.equal(validate(incomplete), false);
|
||||
});
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const validator = path.resolve(here, "..", "bin", "validate-tuning-gold-equivalence-audit.mjs");
|
||||
|
||||
test("tuning gold equivalence audit is complete, independently reviewed, and metric-bound", () => {
|
||||
const run = spawnSync(process.execPath, [validator], { encoding: "utf8" });
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const report = JSON.parse(run.stdout);
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.omissionCases, 17);
|
||||
assert.equal(report.claims, 19);
|
||||
assert.equal(report.changedPairs, 7);
|
||||
assert.equal(report.independentReviewers, 4);
|
||||
assert.equal(report.macroInclusionPrecisionBefore, 0.5638888888888889);
|
||||
assert.equal(report.macroInclusionPrecisionAfter, 0.7222222222222223);
|
||||
assert.deepEqual(report.postAuditGates, {
|
||||
hardPolicyViolations: true,
|
||||
inclusionPrecision: false,
|
||||
verifiedCoverage: false,
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user