289 lines
18 KiB
JavaScript
289 lines
18 KiB
JavaScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { createRequire } from "node:module";
|
|
import { digestJson } from "./canonical.mjs";
|
|
import { snapshotZones, assertNoZoneDrift } from "./fs-evidence.mjs";
|
|
import { runObserved } from "./observer.mjs";
|
|
import { runProcess } from "./process.mjs";
|
|
import { candidateEnvironment, parseJsonLines } from "./runtime.mjs";
|
|
|
|
const require = createRequire(import.meta.url);
|
|
|
|
const VERSION_FIELDS = {
|
|
protocolVersion: "2025-06-18",
|
|
coreVersion: "1.0.0",
|
|
metadataSchemaVersion: "1.0.0",
|
|
scorerVersion: "1.0.0",
|
|
};
|
|
|
|
function suite(id, evidence, executions = 1) {
|
|
return { id, status: "passed", executions, failures: 0, evidenceSha256: digestJson(evidence), evidence };
|
|
}
|
|
|
|
function decodeToolPayload(response) {
|
|
if (response?.error) return { rpcError: response.error };
|
|
const text = response?.result?.content?.find?.((entry) => entry.type === "text")?.text;
|
|
if (typeof text === "string") {
|
|
try { return JSON.parse(text); } catch { return { text }; }
|
|
}
|
|
return response?.result;
|
|
}
|
|
|
|
function assert(condition, code, message) {
|
|
if (!condition) throw Object.assign(new Error(message), { code });
|
|
}
|
|
|
|
function assertVersions(value, prefix = "response") {
|
|
for (const [key, expected] of Object.entries(VERSION_FIELDS)) {
|
|
assert(value?.[key] === expected, "AAS_VERIFIER_VERSION_CONTRACT", `${prefix}.${key} differs`);
|
|
}
|
|
}
|
|
|
|
function request(id, method, params = {}) {
|
|
return { jsonrpc: "2.0", id, method, params };
|
|
}
|
|
|
|
function parseMcpOutput(stdout) {
|
|
const lines = stdout.split(/\r?\n/).filter((line) => line.trim());
|
|
assert(lines.every((line) => Buffer.byteLength(line) <= 256 * 1024), "AAS_VERIFIER_MCP_RESULT_LIMIT", "MCP response exceeded 256 KiB");
|
|
const responses = lines.map((line) => JSON.parse(line));
|
|
assert(responses.every((entry) => entry.jsonrpc === "2.0"), "AAS_VERIFIER_MCP_STDOUT_NOISE", "MCP stdout contains non-protocol output");
|
|
return responses;
|
|
}
|
|
|
|
export async function verifyEntrypoints(runtime, zones) {
|
|
const env = candidateEnvironment(zones);
|
|
const help = await runProcess(process.execPath, [runtime.bins.aas, "--help"], { cwd: zones.project, env });
|
|
assert(help.code === 0 && !help.signal, "AAS_VERIFIER_AAS_HELP", "aas --help failed");
|
|
const values = parseJsonLines(help.stdout);
|
|
assert(values.length === 1 && values[0].ok === true && values[0].status === "help", "AAS_VERIFIER_AAS_HELP_ENVELOPE", "aas --help envelope differs");
|
|
assertVersions(values[0], "aas-help");
|
|
const signatures = values[0].commands || [];
|
|
for (const prefix of ["catalog status", "catalog update", "mcp configure", "mcp backups cleanup", "stack init", "stack recommend", "stack validate", "stack plan", "stack apply", "stack doctor", "stack recover"]) {
|
|
assert(signatures.some((entry) => entry.startsWith(prefix)), "AAS_VERIFIER_COMMAND_MISSING", `Missing command signature: ${prefix}`);
|
|
}
|
|
const legacyHelp = await runProcess(process.execPath, [runtime.bins["agentic-awesome-skills"], "--help"], { cwd: zones.project, env });
|
|
const legacyVersion = await runProcess(process.execPath, [runtime.bins["agentic-awesome-skills"], "--version"], { cwd: zones.project, env });
|
|
assert(legacyHelp.code === 0 && legacyVersion.code === 0, "AAS_VERIFIER_LEGACY_ENTRYPOINT", "Legacy alias smoke failed");
|
|
assert(!fs.existsSync(path.join(zones.project, "aas-stack.json")), "AAS_VERIFIER_LEGACY_STACK_STATE", "Legacy smoke created stack state");
|
|
return suite("entrypoints", {
|
|
helpEnvelope: values[0],
|
|
legacyHelpDigest: digestJson({ stdout: legacyHelp.stdout, stderr: legacyHelp.stderr, code: legacyHelp.code }),
|
|
legacyVersionDigest: digestJson({ stdout: legacyVersion.stdout, stderr: legacyVersion.stderr, code: legacyVersion.code }),
|
|
}, 3);
|
|
}
|
|
|
|
export async function verifyMcp(runtime, zones, evidenceDir) {
|
|
const before = snapshotZones(zones);
|
|
const requests = [
|
|
request(1, "initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "aas-v1-verifier", version: "1" } }),
|
|
{ jsonrpc: "2.0", method: "notifications/initialized", params: {} },
|
|
request(2, "tools/list"),
|
|
request(3, "resources/templates/list"),
|
|
request(4, "tools/call", { name: "search_skills", arguments: { query: "react testing", target: "codex", limit: 5 } }),
|
|
request(5, "tools/call", { name: "get_skill", arguments: { id: "frontend-design", includeContent: true } }),
|
|
request(6, "tools/call", { name: "recommend_stack", arguments: {
|
|
intent: "web-application-delivery",
|
|
profile: { languages: ["typescript"], frameworks: ["react"] },
|
|
targets: [{ host: "codex", scope: "project" }],
|
|
criticalGoals: ["build", "test"],
|
|
nonCriticalGoals: ["deploy"],
|
|
policy: { allowedRisk: ["none", "safe"], requireKnownSource: true, allowManualSetup: false },
|
|
maxSkills: 6,
|
|
} }),
|
|
];
|
|
const input = `${requests.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
|
|
const observed = await runObserved(process.execPath, [runtime.bins["aas-mcp"]], {
|
|
cwd: zones.project,
|
|
env: candidateEnvironment(zones),
|
|
stdin: input,
|
|
timeoutMs: 30_000,
|
|
maxOutputBytes: 2 * 1024 * 1024,
|
|
zones,
|
|
evidenceDir,
|
|
});
|
|
assert(observed.result.code === 0, "AAS_VERIFIER_MCP_EXIT", "MCP smoke exited non-zero");
|
|
assert(observed.observation.networkAttempts === 0, "AAS_VERIFIER_MCP_NETWORK_ATTEMPT", "MCP attempted network access");
|
|
assert(observed.observation.writeAttempts === 0, "AAS_VERIFIER_MCP_WRITE_ATTEMPT", "MCP attempted persistent writes");
|
|
assert(observed.observation.childProcesses === 0, "AAS_VERIFIER_MCP_CHILD_PROCESS", "MCP launched a child process");
|
|
const after = snapshotZones(zones);
|
|
assertNoZoneDrift(before, after);
|
|
const responses = parseMcpOutput(observed.result.stdout);
|
|
const byId = new Map(responses.filter((entry) => entry.id !== undefined).map((entry) => [entry.id, entry]));
|
|
const initialize = byId.get(1)?.result;
|
|
assert(initialize?.protocolVersion === "2025-06-18", "AAS_VERIFIER_MCP_PROTOCOL", "MCP initialize version differs");
|
|
assertVersions(initialize?._meta, "mcp-initialize-meta");
|
|
const toolNames = (byId.get(2)?.result?.tools || []).map((entry) => entry.name).sort();
|
|
assert(JSON.stringify(toolNames) === JSON.stringify(["diff_stack", "get_skill", "inspect_stack", "recommend_stack", "search_skills"]), "AAS_VERIFIER_MCP_TOOL_SURFACE", "MCP tool surface differs");
|
|
const templates = byId.get(3)?.result?.resourceTemplates || [];
|
|
assert(templates.length === 1 && templates[0].uriTemplate === "aas://skills/{id}", "AAS_VERIFIER_MCP_RESOURCE_SURFACE", "MCP resource template differs");
|
|
const getSkillText = JSON.stringify(byId.get(5));
|
|
assert(getSkillText.includes("untrustedContent") && !getSkillText.includes("AAS_SECRET_CANARY_DO_NOT_EMIT_8f37ec75"), "AAS_VERIFIER_UNTRUSTED_CONTENT_BOUNDARY", "Skill content trust boundary missing or leaked canary");
|
|
return suite("mcp", {
|
|
requests: requests.length,
|
|
responseDigest: digestJson(responses),
|
|
canonicalResponseDigest: digestJson(decodeToolPayload(byId.get(6))),
|
|
observation: observed.observation,
|
|
before: Object.fromEntries(Object.entries(before).map(([name, value]) => [name, value.digest])),
|
|
after: Object.fromEntries(Object.entries(after).map(([name, value]) => [name, value.digest])),
|
|
stderrDigest: digestJson(observed.result.stderr),
|
|
}, requests.length);
|
|
}
|
|
|
|
export function packageSuite(inspection, runtime) {
|
|
assert(inspection.failures.length === 0, "AAS_VERIFIER_PACKAGE_CONTENT", "Tarball content contract failed");
|
|
assert(runtime.manifest.name === "agentic-awesome-skills", "AAS_VERIFIER_PACKAGE_NAME", "Package name differs");
|
|
return suite("package", {
|
|
tarballSha256: inspection.sha256,
|
|
tarballSha512: inspection.sha512,
|
|
entries: inspection.entries,
|
|
installTreeSha256: runtime.treeDigest,
|
|
installReceiptDigest: runtime.installReceiptDigest,
|
|
}, inspection.entries.length);
|
|
}
|
|
|
|
async function runDriver(id, driver, runtime, budget, jobIndex) {
|
|
const result = await runProcess(process.execPath, [driver], {
|
|
cwd: runtime.root,
|
|
env: candidateEnvironment({ home: runtime.root, project: runtime.root, cache: runtime.root, tmp: runtime.root }),
|
|
stdin: JSON.stringify({ packageRoot: runtime.packageRoot, budget, jobIndex, jobCount: 6 }),
|
|
timeoutMs: 20 * 60_000,
|
|
maxOutputBytes: 16 * 1024 * 1024,
|
|
});
|
|
assert(result.code === 0 && !result.timedOut && !result.outputLimitExceeded, `AAS_VERIFIER_${id.toUpperCase()}_DRIVER`, `${id} driver failed: ${result.stderr.slice(0, 500)}`);
|
|
const values = parseJsonLines(result.stdout);
|
|
assert(values.length === 1 && values[0].ok === true, `AAS_VERIFIER_${id.toUpperCase()}_RESULT`, `${id} result failed`);
|
|
return suite(id, values[0], values[0].total);
|
|
}
|
|
|
|
export function verifyProperty(runtime, budget, jobIndex, verifierRoot) {
|
|
return runDriver("property", path.join(verifierRoot, "drivers", "property.cjs"), runtime, budget, jobIndex);
|
|
}
|
|
|
|
export function verifyFuzz(runtime, budget, jobIndex, verifierRoot) {
|
|
return runDriver("fuzz", path.join(verifierRoot, "drivers", "fuzz.cjs"), runtime, budget, jobIndex);
|
|
}
|
|
|
|
export async function verifyHostile(runtime, zones, evidenceDir, hostileManifest, hostileRoot, verifierRoot) {
|
|
const before = snapshotZones(zones);
|
|
const observed = await runObserved(process.execPath, [path.join(verifierRoot, "drivers", "hostile.cjs")], {
|
|
cwd: runtime.root,
|
|
env: candidateEnvironment(zones),
|
|
stdin: JSON.stringify({ packageRoot: runtime.packageRoot, manifest: hostileManifest, corpusRoot: hostileRoot }),
|
|
timeoutMs: 120_000,
|
|
maxOutputBytes: 4 * 1024 * 1024,
|
|
zones,
|
|
evidenceDir,
|
|
});
|
|
assert(observed.result.code === 0, "AAS_VERIFIER_HOSTILE_DRIVER", `Hostile driver failed: ${observed.result.stderr.slice(0, 500)}`);
|
|
assert(observed.observation.networkAttempts === 0, "AAS_VERIFIER_HOSTILE_NETWORK", "Hostile suite attempted network access");
|
|
assert(observed.observation.writeAttempts === 0, "AAS_VERIFIER_HOSTILE_WRITE", "Hostile suite attempted writes");
|
|
assert(observed.observation.childProcesses === 0, "AAS_VERIFIER_HOSTILE_CHILD", "Hostile suite launched child code");
|
|
const after = snapshotZones(zones);
|
|
assertNoZoneDrift(before, after);
|
|
const values = parseJsonLines(observed.result.stdout);
|
|
assert(values.length === 1 && values[0].ok === true && values[0].executions === hostileManifest.classes.length * 2, "AAS_VERIFIER_HOSTILE_RESULT", "Hostile result failed");
|
|
return suite("hostile", { ...values[0], observation: observed.observation }, values[0].executions);
|
|
}
|
|
|
|
export async function prepareRuntimeCache(runtime, tarballBytes, integrity, cacheRoot) {
|
|
const core = require(path.join(runtime.packageRoot, "tools/lib/aas-v1"));
|
|
const parsed = core.cache.parsePackageArchive(tarballBytes, { limits: core.cache.RUNTIME_ARCHIVE_LIMITS });
|
|
return core.cache.promoteRuntime({
|
|
cacheRoot,
|
|
release: {
|
|
version: runtime.manifest.version,
|
|
integrity,
|
|
provenance: { registryOrigin: "https://registry.npmjs.org", signaturesPresent: false, attestationsPresent: false },
|
|
},
|
|
parsed,
|
|
});
|
|
}
|
|
|
|
function allOutput(result) {
|
|
return `${result.stdout || ""}\n${result.stderr || ""}`;
|
|
}
|
|
|
|
export async function verifyAdapters(runtime, zones, fixtureRoot, runtimeIntegrity, runtimeClosureDigest) {
|
|
const hosts = [
|
|
{ host: "codex", fixture: "codex-config.toml", config: "codex.toml", sentinel: "unknown_fixture_key" },
|
|
{ host: "claude", fixture: "claude-config.json", config: "claude.json", sentinel: "unknownFixtureKey" },
|
|
];
|
|
const cases = [];
|
|
for (const host of hosts) {
|
|
const root = path.join(zones.project, `adapter-${host.host}`);
|
|
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
const config = path.join(root, host.config);
|
|
const backupDir = path.join(root, "backups");
|
|
fs.copyFileSync(path.join(fixtureRoot, host.fixture), config);
|
|
if (process.platform !== "win32") fs.chmodSync(config, 0o600);
|
|
const beforeStat = fs.statSync(config);
|
|
const beforeBytes = fs.readFileSync(config);
|
|
const common = ["mcp", "configure", "--host", host.host, "--scope", "project", "--config", config, "--cache-root", zones.cache, "--version", runtime.manifest.version, "--runtime-integrity", runtimeIntegrity, "--runtime-closure-digest", runtimeClosureDigest, "--backup-dir", backupDir];
|
|
const preview = await runProcess(process.execPath, [runtime.bins.aas, ...common], { cwd: root, env: candidateEnvironment(zones), timeoutMs: 30_000 });
|
|
assert(preview.code === 0, "AAS_VERIFIER_ADAPTER_PREVIEW", `${host.host} preview failed`);
|
|
const previewValue = parseJsonLines(preview.stdout)[0];
|
|
assert(previewValue?.status === "approvalRequired" && /^sha256-[a-f0-9]{64}$/.test(previewValue.approvalDigest), "AAS_VERIFIER_ADAPTER_APPROVAL", `${host.host} approval digest missing`);
|
|
assert(fs.readFileSync(config).equals(beforeBytes), "AAS_VERIFIER_ADAPTER_PREVIEW_WRITE", `${host.host} preview changed config`);
|
|
assert(!allOutput(preview).includes("AAS_SECRET_CANARY_DO_NOT_EMIT_"), "AAS_VERIFIER_ADAPTER_SECRET_LEAK", `${host.host} preview leaked canary`);
|
|
assert(previewValue.runtime?.integrity === runtimeIntegrity, "AAS_VERIFIER_RUNTIME_INTEGRITY_BINDING", `${host.host} preview did not bind the exact candidate runtime SRI`);
|
|
const apply = await runProcess(process.execPath, [runtime.bins.aas, ...common, "--approve", previewValue.approvalDigest], { cwd: root, env: candidateEnvironment(zones), timeoutMs: 30_000 });
|
|
assert(apply.code === 0, "AAS_VERIFIER_ADAPTER_APPLY", `${host.host} apply failed: ${apply.stderr}`);
|
|
assert(!allOutput(apply).includes("AAS_SECRET_CANARY_DO_NOT_EMIT_"), "AAS_VERIFIER_ADAPTER_SECRET_LEAK", `${host.host} apply leaked canary`);
|
|
const afterBytes = fs.readFileSync(config);
|
|
const afterText = afterBytes.toString("utf8");
|
|
assert(afterText.includes(host.sentinel) && afterText.includes("preserve-me") && afterText.includes("existing-server"), "AAS_VERIFIER_ADAPTER_UNKNOWN_FIELD", `${host.host} did not preserve unknown/existing fields`);
|
|
assert(afterText.includes("aas-mcp.js"), "AAS_VERIFIER_ADAPTER_MCP_ENTRY", `${host.host} did not configure AAS MCP`);
|
|
const afterStat = fs.statSync(config);
|
|
if (process.platform !== "win32") {
|
|
assert((afterStat.mode & 0o777) === (beforeStat.mode & 0o777), "AAS_VERIFIER_ADAPTER_MODE", `${host.host} mode changed`);
|
|
assert(afterStat.uid === beforeStat.uid && afterStat.gid === beforeStat.gid, "AAS_VERIFIER_ADAPTER_OWNER", `${host.host} owner changed`);
|
|
}
|
|
const backupEntries = fs.readdirSync(backupDir).filter((name) => !name.startsWith(".")).sort();
|
|
const backups = backupEntries.filter((name) => name.endsWith(".bak"));
|
|
const metadata = backupEntries.filter((name) => name.endsWith(".json"));
|
|
assert(backups.length === 1 && metadata.length === 1, "AAS_VERIFIER_ADAPTER_BACKUP", `${host.host} backup pair count differs`);
|
|
if (process.platform !== "win32") {
|
|
for (const name of backupEntries) {
|
|
const backupStat = fs.statSync(path.join(backupDir, name));
|
|
assert((backupStat.mode & 0o077) === 0, "AAS_VERIFIER_ADAPTER_BACKUP_MODE", `${host.host} backup entry is not user-only`);
|
|
}
|
|
}
|
|
const cleanupBase = ["mcp", "backups", "cleanup", "--config", config, "--backup-dir", backupDir, "--keep", "0"];
|
|
const cleanupPreview = await runProcess(process.execPath, [runtime.bins.aas, ...cleanupBase], { cwd: root, env: candidateEnvironment(zones) });
|
|
const cleanupValue = parseJsonLines(cleanupPreview.stdout)[0];
|
|
assert(cleanupPreview.code === 0 && /^sha256-[a-f0-9]{64}$/.test(cleanupValue?.approvalDigest), "AAS_VERIFIER_ADAPTER_CLEANUP_PREVIEW", `${host.host} backup cleanup preview failed`);
|
|
const cleanup = await runProcess(process.execPath, [runtime.bins.aas, ...cleanupBase, "--approve", cleanupValue.approvalDigest], { cwd: root, env: candidateEnvironment(zones) });
|
|
assert(cleanup.code === 0 && fs.readdirSync(backupDir).filter((name) => !name.startsWith(".")).length === 0, "AAS_VERIFIER_ADAPTER_CLEANUP", `${host.host} backup cleanup failed`);
|
|
|
|
const unsafe = path.join(root, `unsafe-${host.config}`);
|
|
const outside = path.join(root, `outside-${host.config}`);
|
|
fs.copyFileSync(path.join(fixtureRoot, host.fixture), outside);
|
|
try {
|
|
fs.symlinkSync(outside, unsafe, process.platform === "win32" ? "file" : undefined);
|
|
const unsafeResult = await runProcess(process.execPath, [runtime.bins.aas, "mcp", "configure", "--host", host.host, "--scope", "project", "--config", unsafe, "--cache-root", zones.cache, "--version", runtime.manifest.version], { cwd: root, env: candidateEnvironment(zones) });
|
|
assert(unsafeResult.code !== 0, "AAS_VERIFIER_ADAPTER_SYMLINK", `${host.host} accepted symlink config`);
|
|
} finally { fs.rmSync(unsafe, { force: true }); }
|
|
cases.push({ host: host.host, previewDigest: digestJson(previewValue), applyDigest: digestJson(parseJsonLines(apply.stdout)[0]), backupCount: backups.length, cleanupDigest: digestJson(cleanupValue) });
|
|
}
|
|
return suite("adapters", { runtimeIntegrity, runtimeClosureDigest, cases, invariants: ["unknown-fields-preserved", "minimal-patch", "mode-owner-preserved", "atomic-write", "user-only-backup", "secret-redacted", "unsafe-file-rejected"] }, cases.length * 4);
|
|
}
|
|
|
|
export async function verifyLegacy(runtime, zones, verifierRoot, corpusRoot) {
|
|
const driver = path.resolve(verifierRoot, "drivers", "legacy.mjs");
|
|
const absoluteCorpusRoot = path.resolve(corpusRoot);
|
|
const workRoot = path.join(zones.tmp, "legacy");
|
|
const result = await runProcess(process.execPath, [driver], {
|
|
cwd: zones.project,
|
|
env: candidateEnvironment(zones),
|
|
stdin: JSON.stringify({ packageRoot: runtime.packageRoot, runtimeRoot: runtime.root, corpusRoot: absoluteCorpusRoot, workRoot }),
|
|
timeoutMs: 10 * 60_000,
|
|
maxOutputBytes: 8 * 1024 * 1024,
|
|
});
|
|
const values = parseJsonLines(result.stdout);
|
|
assert(result.code === 0 && values.length === 1 && values[0].ok === true && values[0].executions === 41, "AAS_VERIFIER_LEGACY_DIFFERENTIAL", `Legacy differential failed: ${result.stderr.slice(0, 500)} ${result.stdout.slice(0, 1000)}`);
|
|
return suite("legacy", values[0], values[0].executions);
|
|
}
|
|
|
|
export { suite };
|