📦 deps(thirdparty): update snapshots
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { aggregateReceipts } from "../lib/aggregate.mjs";
|
||||
import { digestJson } from "../lib/canonical.mjs";
|
||||
import { finalizeReceipt, JOB_IDS, loadReceiptValidator, SUITE_IDS } from "../lib/receipt.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const validator = loadReceiptValidator(path.resolve(here, "..", "..", "schemas", "product-verifier-receipt.schema.json"));
|
||||
const d = `sha256-${"a".repeat(64)}`;
|
||||
const sri = `sha512-${Buffer.alloc(64).toString("base64")}`;
|
||||
|
||||
function receipt(job, index) {
|
||||
const totals = { property: index < 4 ? 16667 : 16666, fuzz: index < 2 ? 8334 : 8333 };
|
||||
const suites = SUITE_IDS.map((id) => {
|
||||
const evidence = id === "property" ? { total: totals.property, hardPolicyViolations: 0 }
|
||||
: id === "fuzz" ? { total: totals.fuzz }
|
||||
: id === "hostile" ? { executions: 64 }
|
||||
: id === "legacy" ? { executions: 41 }
|
||||
: id === "transaction" ? { faultBoundaryClasses: ["lock", "journal", "backup", "write", "fsync", "rename", "commit"], raceClasses: ["concurrency", "drift", "symlink-swap", "target-swap", "corrupt-journal", "recovery-race"] }
|
||||
: {};
|
||||
return { id, status: "passed", executions: evidence.total || evidence.executions || 1, failures: 0, evidenceSha256: digestJson(evidence), evidence };
|
||||
});
|
||||
return finalizeReceipt({ schemaVersion: 1, receiptVersion: "1.0.0", status: "passed",
|
||||
job: { id: job, workflowRunId: "1", workflowRunAttempt: "1" },
|
||||
candidate: { commit: "1".repeat(40), package: "agentic-awesome-skills", version: "14.6.0", tarballBytes: 1, tarballSha256: d, tarballSha512: sri, packManifestSha256: d, installTreeSha256: d },
|
||||
verifier: { version: "1.0.0", commit: "2".repeat(40), rootDigest: d, contractDigest: d, owner: "aas-v1-independent-verifier" },
|
||||
environment: { platform: job.startsWith("linux") ? "linux" : job.startsWith("macos") ? "darwin" : "win32", osVersion: "test", kernelVersion: "test", architecture: "x64", nodeVersion: job.endsWith("22") ? "v22.23.1" : "v24.18.0", nodeExecutableSha256: d, runnerImageLabel: "test", runnerImageVersion: "test", filesystemType: job.startsWith("linux") ? "ext4" : job.startsWith("macos") ? "apfs" : "ntfs", filesystemCaseSensitivity: job.startsWith("linux") ? "sensitive" : "insensitive-preserving" },
|
||||
observer: { contractVersion: "1.0.0", backend: job.startsWith("linux") ? "linux-strace-process-tree" : job.startsWith("macos") ? "macos-fs_usage-process" : "windows-etw-kernel-process-tree", selfTestDigest: d, networkSentinels: 1, writeSentinels: 1, overflow: false, ambiguousLineage: false },
|
||||
zones: Object.fromEntries(["home", "project", "cache", "tmp"].map((name) => [name, { beforeSha256: d, afterSha256: d, persistentWriteCount: 0 }])),
|
||||
suites, canonicalPayload: { sha256: d, excludedFields: ["timestamp", "correlationId", "localizedMessage", "diagnostics"], sampleCount: 60 }, failures: [] });
|
||||
}
|
||||
|
||||
test("aggregator accepts only the exact complete matrix", () => {
|
||||
const receipts = JOB_IDS.map(receipt);
|
||||
assert.equal(aggregateReceipts(receipts, validator).status, "passed");
|
||||
const missing = aggregateReceipts(receipts.slice(1), validator);
|
||||
assert.equal(missing.status, "failed");
|
||||
assert.ok(missing.failures.some((entry) => entry.code === "AAS_VERIFIER_MATRIX_MISSING_JOB"));
|
||||
});
|
||||
|
||||
test("aggregator catches receipt tampering and duplicates", () => {
|
||||
const receipts = JOB_IDS.map(receipt);
|
||||
receipts[0].candidate.tarballBytes = 2;
|
||||
const result = aggregateReceipts([...receipts.slice(0, 5), receipts[0]], validator);
|
||||
assert.equal(result.status, "failed");
|
||||
assert.ok(result.failures.some((entry) => entry.code === "AAS_VERIFIER_RECEIPT_DIGEST"));
|
||||
assert.ok(result.failures.some((entry) => entry.code === "AAS_VERIFIER_MATRIX_DUPLICATE_JOB"));
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const checker = path.resolve(here, "..", "bin", "check-baseline.mjs");
|
||||
|
||||
test("frozen structure-only baseline is internally valid", () => {
|
||||
const run = spawnSync(process.execPath, [checker, "--mode", "structure"], { encoding: "utf8" });
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const report = JSON.parse(run.stdout);
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.heldOutDescriptors, 180);
|
||||
assert.equal(report.pendingCount, 0);
|
||||
});
|
||||
|
||||
test("freeze-ready mode never reports an incomplete baseline as complete", () => {
|
||||
const run = spawnSync(process.execPath, [checker, "--mode", "freeze-ready"], { encoding: "utf8" });
|
||||
if (run.status === 0) {
|
||||
const report = JSON.parse(run.stdout);
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.pendingCount, 0);
|
||||
return;
|
||||
}
|
||||
assert.equal(run.status, 2, run.stdout || run.stderr);
|
||||
const report = JSON.parse(run.stderr);
|
||||
assert.equal(report.code, "AAS_BASELINE_NOT_FREEZE_READY");
|
||||
assert.ok(report.pending.length > 0);
|
||||
assert.ok(report.pending.every((entry) => entry.code.endsWith("_PENDING")));
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { canonicalJson, digestJson, parseCanonicalJson } from "../lib/canonical.mjs";
|
||||
|
||||
test("canonical JSON uses deterministic UTF-16 property ordering", () => {
|
||||
const value = { "\udfff": 1, "\ue000": 2, a: 3 };
|
||||
assert.equal(canonicalJson(value), '{"a":3,"\\udfff":1,"\ue000":2}');
|
||||
assert.equal(parseCanonicalJson(canonicalJson(value)).a, 3);
|
||||
assert.match(digestJson(value), /^sha256-[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
test("canonical JSON rejects values outside the JSON data model", () => {
|
||||
assert.throws(() => canonicalJson({ value: undefined }), /cannot encode/);
|
||||
assert.throws(() => canonicalJson(Number.NaN), /non-finite/);
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
aggregateIntent,
|
||||
caseInclusionAssessment,
|
||||
caseInclusionPrecision,
|
||||
evaluateCase,
|
||||
isCorrectAbstention,
|
||||
macroAverage,
|
||||
} from "../lib/metrics.mjs";
|
||||
|
||||
const gold = {
|
||||
criticalGoals: ["critical-a", "critical-b"],
|
||||
nonCriticalGoals: ["optional-a", "optional-b", "optional-c", "optional-d", "optional-e"],
|
||||
minimumNonCriticalGoalCoverage: 0.8,
|
||||
requiresSkill: true,
|
||||
};
|
||||
|
||||
const acceptedSolutions = [{
|
||||
solutionId: "solution-a",
|
||||
allowedSkillIds: ["skill-a", "skill-helper"],
|
||||
requiredGroups: [["skill-a"]],
|
||||
}];
|
||||
|
||||
const validResult = {
|
||||
terminal: true,
|
||||
schemaValid: true,
|
||||
hardPolicyViolations: [],
|
||||
coveredGoals: ["critical-a", "critical-b", "optional-a", "optional-b", "optional-c", "optional-d"],
|
||||
includedSkillIds: ["skill-a"],
|
||||
discoveryPromotions: [],
|
||||
};
|
||||
|
||||
test("verified case requires all critical and at least 80 percent non-critical coverage", () => {
|
||||
assert.equal(evaluateCase(validResult, gold, acceptedSolutions).verified, true);
|
||||
assert.equal(evaluateCase({ ...validResult, coveredGoals: validResult.coveredGoals.slice(0, -1) }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, coveredGoals: validResult.coveredGoals.slice(1) }, gold, acceptedSolutions).verified, false);
|
||||
});
|
||||
|
||||
test("hard policy, timeout, empty required stack, and hidden discovery override fail verification", () => {
|
||||
assert.equal(evaluateCase({ ...validResult, hardPolicyViolations: ["risk"] }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, timedOut: true }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, includedSkillIds: [] }, gold, acceptedSolutions).verified, false);
|
||||
assert.equal(evaluateCase({ ...validResult, discoveryPromotions: [{ id: "skill-a", visibleOverride: false }] }, gold, acceptedSolutions).verified, false);
|
||||
});
|
||||
|
||||
test("candidate coverage self-claims cannot replace independently accepted required groups", () => {
|
||||
const lying = { ...validResult, includedSkillIds: ["skill-helper"] };
|
||||
const report = evaluateCase(lying, gold, acceptedSolutions);
|
||||
assert.equal(report.criticalGoalCoverage, 1);
|
||||
assert.equal(report.independentlyAcceptedStack, false);
|
||||
assert.equal(report.verified, false);
|
||||
});
|
||||
|
||||
test("precision scores one coherent alternative rather than a union", () => {
|
||||
const solutions = [
|
||||
{ allowedSkillIds: ["a", "a-helper"] },
|
||||
{ allowedSkillIds: ["b", "b-helper"] },
|
||||
];
|
||||
assert.equal(caseInclusionPrecision(["a", "a-helper"], solutions), 1);
|
||||
assert.equal(caseInclusionPrecision(["a", "b"], solutions), 0.5);
|
||||
assert.equal(caseInclusionPrecision([], solutions), null);
|
||||
assert.deepEqual(caseInclusionAssessment(["a", "a-helper"], solutions), {
|
||||
acceptedCount: 2,
|
||||
inclusionCount: 2,
|
||||
precision: 1,
|
||||
matchedSolutionId: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
test("coverage denominator stays 30 when results are missing", () => {
|
||||
const reports = Array.from({ length: 24 }, () => ({
|
||||
verified: true,
|
||||
acceptedInclusionCount: 9,
|
||||
inclusionCount: 10,
|
||||
}));
|
||||
const aggregate = aggregateIntent(reports);
|
||||
assert.equal(aggregate.verifiedCoverage, 0.8);
|
||||
assert.equal(aggregate.emptyStackCount, 6);
|
||||
assert.ok(Math.abs(aggregate.inclusionPrecision - 0.9) < Number.EPSILON * 24);
|
||||
assert.equal(aggregateIntent(reports.slice(0, 23)).verifiedCoverage < 0.8, true);
|
||||
});
|
||||
|
||||
test("intent precision is the mean of per-stack precision rather than pooled inclusions", () => {
|
||||
const aggregate = aggregateIntent([
|
||||
{ verified: true, acceptedInclusionCount: 1, inclusionCount: 1 },
|
||||
{ verified: true, acceptedInclusionCount: 1, inclusionCount: 9 },
|
||||
]);
|
||||
assert.equal(aggregate.inclusionPrecision, (1 + (1 / 9)) / 2);
|
||||
assert.equal(aggregate.acceptedInclusions, 2);
|
||||
assert.equal(aggregate.totalInclusions, 10);
|
||||
});
|
||||
|
||||
test("macro average requires six intents and is not a micro average", () => {
|
||||
assert.equal(macroAverage([0.8, 0.8, 0.8, 0.8, 0.8, 1].map((value) => ({ value })), "value"), 5 / 6);
|
||||
assert.throws(() => macroAverage([{ value: 1 }], "value"), /exactly six/i);
|
||||
});
|
||||
|
||||
test("correct abstention is successful, insufficient, and empty", () => {
|
||||
assert.equal(isCorrectAbstention({ ok: true, status: "insufficientCoverage", proposedStack: [] }), true);
|
||||
assert.equal(isCorrectAbstention({ ok: false, status: "insufficientCoverage", proposedStack: [] }), false);
|
||||
assert.equal(isCorrectAbstention({ ok: true, status: "insufficientCoverage", proposedStack: ["weak"] }), false);
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { macObserverBudgets, parseDelimitedObserver, parseLinuxStrace, parseMacCombinedFsUsage, parseMacFsUsage, windowsObserverBudgets } from "../lib/observer.mjs";
|
||||
|
||||
test("strace parser counts failed network attempts and non-stream writes", () => {
|
||||
const result = parseLinuxStrace([
|
||||
'execve("/usr/bin/node", ["node"], 0x0) = 0',
|
||||
'connect(7<TCP:[1]>, {sa_family=AF_INET}, 16) = -1 ECONNREFUSED',
|
||||
'openat(AT_FDCWD, "/tmp/x", O_WRONLY|O_CREAT, 0600) = 8',
|
||||
'write(8</tmp/x>, "x", 1) = 1',
|
||||
'write(1</dev/pts/1>, "ok", 2) = 2',
|
||||
'execve("/bin/true", ["true"], 0x0) = 0',
|
||||
].join("\n"), { tmp: "/tmp" });
|
||||
assert.equal(result.networkAttempts, 1);
|
||||
assert.equal(result.writeAttempts, 2);
|
||||
assert.equal(result.childProcesses, 1);
|
||||
assert.ok(result.events.every((entry) => !JSON.stringify(entry).includes("/tmp/x")));
|
||||
});
|
||||
|
||||
test("delimited observers ignore malformed and unknown records", () => {
|
||||
const result = parseDelimitedObserver("network|connect|127.0.0.1\nwrite|open|/tmp/x\nnoise|secret\n");
|
||||
assert.deepEqual([result.networkAttempts, result.writeAttempts, result.childProcesses], [1, 1, 0]);
|
||||
});
|
||||
|
||||
test("Windows observer separates the candidate limit from ETW finalization grace", () => {
|
||||
assert.deepEqual(windowsObserverBudgets(10_000), {
|
||||
candidateTimeoutMs: 10_000,
|
||||
wrapperTimeoutMs: 70_000,
|
||||
});
|
||||
assert.throws(() => windowsObserverBudgets(0), (error) => error.code === "AAS_OBSERVER_INVALID_TIMEOUT");
|
||||
assert.throws(() => windowsObserverBudgets(900_001), (error) => error.code === "AAS_OBSERVER_INVALID_TIMEOUT");
|
||||
});
|
||||
|
||||
test("macOS observer lifetime covers readiness, candidate, drain, and stop margin", () => {
|
||||
assert.deepEqual(macObserverBudgets(10_000), {
|
||||
startupMs: 1_500,
|
||||
readinessMaxAttempts: 2,
|
||||
readinessProcessTimeoutMs: 10_000,
|
||||
readinessDelayMs: 250,
|
||||
drainMs: 1_000,
|
||||
observerTimeoutMs: 38_000,
|
||||
});
|
||||
assert.throws(() => macObserverBudgets(0), (error) => error.code === "AAS_OBSERVER_INVALID_TIMEOUT");
|
||||
});
|
||||
|
||||
test("macOS fs_usage parser separates network, writes, and child execs", () => {
|
||||
const result = parseMacFsUsage(
|
||||
"12:00:00.100 WrData[A] F=3 /tmp/canary node.1\n12:00:00.200 read F=4 /tmp/input node.1\n",
|
||||
"12:00:00.300 connect 127.0.0.1:9 node.1\n",
|
||||
"12:00:00.400 exec node node.1\n12:00:00.500 exec child node.2\n",
|
||||
);
|
||||
assert.equal(result.networkAttempts, 1);
|
||||
assert.equal(result.writeAttempts, 1);
|
||||
assert.equal(result.childProcesses, 1);
|
||||
});
|
||||
|
||||
test("combined macOS fs_usage parser enforces canary ordering and classifies candidate calls", () => {
|
||||
const result = parseMacCombinedFsUsage([
|
||||
"12:00:00.005 WrData[A] F=3 /tmp/aas-ready-1 aasobs.1",
|
||||
"12:00:00.010 WrData[A] F=3 /tmp/aas-start-1 aasobs.1",
|
||||
"12:00:00.020 WrData[A] F=3 /tmp/canary node.1",
|
||||
"12:00:00.030 connect 127.0.0.1:9 node.1",
|
||||
"12:00:00.040 posix_spawn child node.1",
|
||||
].join("\n"), {}, "aas-ready-1", "aas-start-1");
|
||||
assert.deepEqual([result.networkAttempts, result.writeAttempts, result.childProcesses], [1, 1, 1]);
|
||||
assert.throws(() => parseMacCombinedFsUsage("12:00:00.010 WrData[A] F=3 /tmp/aas-start-1 aasobs.1\n", {}, "aas-ready-1", "aas-start-1"), /readiness canary/);
|
||||
assert.throws(() => parseMacCombinedFsUsage("12:00:00.010 WrData[A] F=3 /tmp/aas-ready-1 aasobs.1\n", {}, "aas-ready-1", "aas-start-1"), /candidate start canary/);
|
||||
});
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { loadReceiptValidator } from "../lib/receipt.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const validate = loadReceiptValidator(path.resolve(here, "..", "..", "schemas", "product-transaction-evidence.schema.json"));
|
||||
const digest = `sha256-${"a".repeat(64)}`;
|
||||
const fault = ["lock", "journal", "backup", "write", "fsync", "rename", "commit"];
|
||||
const race = ["concurrency", "drift", "symlink-swap", "target-swap", "corrupt-journal", "recovery-race"];
|
||||
|
||||
function evidence() {
|
||||
return {
|
||||
schemaVersion: 1, status: "passed", productionBinary: true, testMode: false, mocked: false,
|
||||
observer: { backend: "linux-strace-process-tree", eventDigest: digest, overflow: false, ambiguousLineage: false },
|
||||
faultBoundaryClasses: fault, raceClasses: race, executions: 13, killExecutions: 7,
|
||||
swapExecutions: 2, recoveryExecutions: 2, partialStates: 0, unmanagedMutations: 0,
|
||||
hardPolicyViolations: 0,
|
||||
boundaryEvidence: [...fault, ...race].map((value, index) => ({
|
||||
class: value, observedOperation: `event-${index}`,
|
||||
injectionAction: index < fault.length ? "kill" : value === "concurrency" ? "concurrent" : value,
|
||||
beforeDigest: digest, afterDigest: digest, finalState: index % 2 ? "new" : "previous", noPartialState: true,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
test("transaction evidence requires full external black-box coverage", () => {
|
||||
assert.equal(validate(evidence()), true, JSON.stringify(validate.errors));
|
||||
const mocked = evidence();
|
||||
mocked.mocked = true;
|
||||
assert.equal(validate(mocked), false);
|
||||
const incomplete = evidence();
|
||||
incomplete.faultBoundaryClasses.pop();
|
||||
assert.equal(validate(incomplete), false);
|
||||
});
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const validator = path.resolve(here, "..", "bin", "validate-tuning-gold-equivalence-audit.mjs");
|
||||
|
||||
test("tuning gold equivalence audit is complete, independently reviewed, and metric-bound", () => {
|
||||
const run = spawnSync(process.execPath, [validator], { encoding: "utf8" });
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const report = JSON.parse(run.stdout);
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.omissionCases, 17);
|
||||
assert.equal(report.claims, 19);
|
||||
assert.equal(report.changedPairs, 7);
|
||||
assert.equal(report.independentReviewers, 4);
|
||||
assert.equal(report.macroInclusionPrecisionBefore, 0.5638888888888889);
|
||||
assert.equal(report.macroInclusionPrecisionAfter, 0.7222222222222223);
|
||||
assert.deepEqual(report.postAuditGates, {
|
||||
hardPolicyViolations: true,
|
||||
inclusionPrecision: false,
|
||||
verifiedCoverage: false,
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user