📦 deps(thirdparty): update snapshots

2026-07-18 00:02:59 +00:00
parent 82f7c6e56a
commit 47ce7f78dc
1446 changed files with 141041 additions and 6442 deletions
@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const benchmarkRoot = path.resolve(here, "..");
+const baselineRoot = path.resolve(benchmarkRoot, "..");
+const repositoryRoot = path.resolve(baselineRoot, "..", "..", "..", "..");
+const reportPaths = [
+  path.join(baselineRoot, "reviews", "reviewer-alpha.json"),
+  path.join(baselineRoot, "reviews", "reviewer-beta.json"),
+];
+
+function sha256File(file) {
+  return `sha256:${crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex")}`;
+}
+
+function canonicalize(value) {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
+  }
+  return value;
+}
+
+function pairDigest(caseData, judgmentData, section) {
+  const reviewNeutralJudgment = { ...judgmentData, reviews: [] };
+  const payload = section === "abstention"
+    ? { case: caseData, label: reviewNeutralJudgment }
+    : { case: caseData, gold: reviewNeutralJudgment };
+  const bytes = JSON.stringify(canonicalize(payload));
+  return `sha256-${crypto.createHash("sha256").update(bytes).digest("hex")}`;
+}
+
+function pairManifestRoot(entries) {
+  const sorted = entries.sort((left, right) => (
+    left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0
+  ));
+  return `sha256-${crypto.createHash("sha256").update(JSON.stringify(sorted)).digest("hex")}`;
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+function writeJson(file, data) {
+  fs.writeFileSync(file, `${JSON.stringify(data, null, 2)}\n`);
+}
+
+const reports = reportPaths.map((file) => {
+  if (!fs.existsSync(file)) throw new Error(`Missing independent review report: ${file}`);
+  const report = readJson(file);
+  if ((report.overallDecision || report.overall?.decision) !== "approved") {
+    throw new Error(`Review is not approved: ${file}`);
+  }
+  return report;
+});
+const reviewerId = (report) => report.reviewer.id || report.reviewer.identity;
+if (new Set(reports.map(reviewerId)).size !== 2) {
+  throw new Error("Exactly two distinct independent reviewer identities are required.");
+}
+
+const pendingWrites = new Map();
+const reviewedCaseIds = new Map(reports.map((report) => [reviewerId(report), new Set()]));
+
+const pairPaths = new Map();
+const heldOutIndexForPaths = readJson(path.join(benchmarkRoot, "held-out-index.json"));
+for (const entry of heldOutIndexForPaths.cases) {
+  pairPaths.set(entry.caseId, {
+    caseFile: path.join(benchmarkRoot, entry.inputPath),
+    judgmentFile: path.join(benchmarkRoot, entry.goldPath),
+    section: "heldOut",
+  });
+}
+const tuningRoot = path.join(benchmarkRoot, "tuning");
+const tuningIndexForPaths = readJson(path.join(tuningRoot, "index.json"));
+for (const entry of tuningIndexForPaths.cases) {
+  pairPaths.set(entry.caseId, {
+    caseFile: path.join(tuningRoot, entry.inputPath),
+    judgmentFile: path.join(tuningRoot, entry.goldPath),
+    section: "tuning",
+  });
+}
+const abstentionRoot = path.join(benchmarkRoot, "abstention");
+const abstentionIndexForPaths = readJson(path.join(abstentionRoot, "index.json"));
+for (const entry of abstentionIndexForPaths.cases) {
+  pairPaths.set(entry.caseId, {
+    caseFile: path.join(abstentionRoot, entry.inputPath),
+    judgmentFile: path.join(abstentionRoot, entry.labelPath),
+    section: "abstention",
+  });
+}
+
+function addReview(report, caseId, caseFile, judgmentFile, section) {
+  const identity = reviewerId(report);
+  const caseData = readJson(caseFile);
+  const judgmentData = readJson(judgmentFile);
+  const digest = pairDigest(caseData, judgmentData, section);
+  const pending = pendingWrites.get(judgmentFile) || { data: judgmentData, reviews: [] };
+  pending.reviews.push({ reviewer: identity, decision: "approved", reviewedDigest: digest });
+  pendingWrites.set(judgmentFile, pending);
+  reviewedCaseIds.get(identity).add(caseId);
+  return digest;
+}
+
+for (const report of reports) {
+  if (report.scope) {
+    for (const section of ["heldOut", "tuning", "abstention"]) {
+      const items = report.scope?.[section]?.items;
+      if (!Array.isArray(items)) throw new Error(`${reviewerId(report)} missing ${section} items`);
+      for (const item of items) {
+        if (item.decision !== "approved") throw new Error(`${reviewerId(report)} rejected ${item.caseId}`);
+        const caseFile = path.join(repositoryRoot, item.casePath);
+        const judgmentFile = path.join(repositoryRoot, item.judgmentPath);
+        if (sha256File(caseFile) !== item.caseSha256) throw new Error(`Case digest mismatch: ${item.caseId}`);
+        if (sha256File(judgmentFile) !== item.judgmentSha256) throw new Error(`Judgment digest mismatch: ${item.caseId}`);
+        addReview(report, item.caseId, caseFile, judgmentFile, section);
+      }
+    }
+    continue;
+  }
+  if (report.splits) {
+    const globalEntries = [];
+    for (const section of ["heldOut", "tuning", "abstention"]) {
+      const splitEntries = [];
+      for (const [caseId, pair] of pairPaths) {
+        if (pair.section !== section) continue;
+        const digest = addReview(report, caseId, pair.caseFile, pair.judgmentFile, section);
+        splitEntries.push([caseId, digest]);
+        globalEntries.push([caseId, digest]);
+      }
+      if (splitEntries.length !== report.splits[section]?.pairCount) {
+        throw new Error(`${reviewerId(report)} ${section} pair count mismatch`);
+      }
+      if (pairManifestRoot(splitEntries) !== report.splits[section]?.pairManifestRootDigest) {
+        throw new Error(`${reviewerId(report)} ${section} root digest mismatch`);
+      }
+    }
+    if (globalEntries.length !== report.pairCount || pairManifestRoot(globalEntries) !== report.pairManifestRootDigest) {
+      throw new Error(`${reviewerId(report)} global root digest mismatch`);
+    }
+    continue;
+  }
+  for (const section of ["heldOut", "tuning", "abstention"]) {
+    const digestMap = report[section]?.pairSha256ByCaseId;
+    if (!digestMap || typeof digestMap !== "object") throw new Error(`${reviewerId(report)} missing ${section} digest map`);
+    for (const [caseId, expectedDigest] of Object.entries(digestMap)) {
+      const pair = pairPaths.get(caseId);
+      if (!pair || pair.section !== section) throw new Error(`Unknown ${section} review pair: ${caseId}`);
+      const actualDigest = addReview(report, caseId, pair.caseFile, pair.judgmentFile, section);
+      if (actualDigest !== expectedDigest) throw new Error(`Pair digest mismatch: ${caseId}`);
+    }
+  }
+}
+
+for (const [reviewer, ids] of reviewedCaseIds) {
+  if (ids.size !== 270) throw new Error(`${reviewer} approved ${ids.size} unique pairs, expected 270`);
+}
+if (pendingWrites.size !== 270) throw new Error(`Expected 270 judgments, found ${pendingWrites.size}`);
+for (const [file, pending] of pendingWrites) {
+  pending.data.reviews = pending.reviews.sort((left, right) => left.reviewer.localeCompare(right.reviewer));
+  writeJson(file, pending.data);
+}
+
+const heldOutIndexPath = path.join(benchmarkRoot, "held-out-index.json");
+const heldOutIndex = readJson(heldOutIndexPath);
+heldOutIndex.status = "frozen";
+heldOutIndex.indexVersion = "1.0.0";
+heldOutIndex.cases = heldOutIndex.cases.map((entry) => ({ ...entry, reviewStatus: "approved" }));
+writeJson(heldOutIndexPath, heldOutIndex);
+
+const tuningIndexPath = path.join(benchmarkRoot, "tuning", "index.json");
+const tuningIndex = readJson(tuningIndexPath);
+tuningIndex.status = "frozen";
+tuningIndex.cases = tuningIndex.cases.map((entry) => ({ ...entry, reviewStatus: "approved" }));
+writeJson(tuningIndexPath, tuningIndex);
+
+const tuningManifestPath = path.join(benchmarkRoot, "tuning", "manifest.json");
+const tuningManifest = readJson(tuningManifestPath);
+tuningManifest.status = "frozen";
+tuningManifest.reviews = reports.map((report) => ({ reviewer: report.reviewer.id, decision: "approved" }));
+writeJson(tuningManifestPath, tuningManifest);
+
+const abstentionIndexPath = path.join(benchmarkRoot, "abstention", "index.json");
+const abstentionIndex = readJson(abstentionIndexPath);
+abstentionIndex.status = "frozen";
+abstentionIndex.cases = abstentionIndex.cases.map((entry) => ({ ...entry, reviewStatus: "approved" }));
+writeJson(abstentionIndexPath, abstentionIndex);
+
+const manifestPath = path.join(benchmarkRoot, "manifest.json");
+const manifest = readJson(manifestPath);
+manifest.benchmarkVersion = "1.0.0";
+manifest.status = "frozen";
+manifest.heldOut.actualInputsPresent = true;
+manifest.heldOut.goldLabelsPresent = true;
+manifest.abstention = {
+  index: "abstention/index.json",
+  caseCount: 30,
+  labelsFrozen: true,
+  status: "frozen"
+};
+manifest.tuning = {
+  manifest: "tuning/manifest.json",
+  separatedFromHeldOut: true,
+  status: "frozen"
+};
+manifest.labelsFrozen = true;
+writeJson(manifestPath, manifest);
+
+console.log(JSON.stringify({
+  ok: true,
+  reviewers: reports.map(reviewerId),
+  reviewedPairs: pendingWrites.size,
+  heldOut: 180,
+  tuning: 60,
+  abstention: 30,
+}, null, 2));
@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const benchmarkRoot = path.resolve(here, "..");
+const caseRoot = path.join(benchmarkRoot, "cases", "held-out");
+const goldRoot = path.join(benchmarkRoot, "gold", "held-out");
+const indexPath = path.join(benchmarkRoot, "held-out-index.json");
+
+function walkJson(directory) {
+  return fs.readdirSync(directory, { withFileTypes: true })
+    .flatMap((entry) => {
+      const target = path.join(directory, entry.name);
+      return entry.isDirectory() ? walkJson(target) : entry.name.endsWith(".json") ? [target] : [];
+    })
+    .sort();
+}
+
+function hostsForArchetype(archetype) {
+  if (["small-greenfield", "mature-legacy-migration"].includes(archetype)) return ["codex"];
+  if (["production-greenfield", "constrained-offline"].includes(archetype)) return ["claude"];
+  return ["codex", "claude"];
+}
+
+const index = JSON.parse(fs.readFileSync(indexPath, "utf8"));
+const descriptors = new Map(index.cases.map((entry) => [entry.caseId, entry]));
+const casePaths = new Map();
+const goldPaths = new Map();
+
+for (const file of walkJson(caseRoot)) {
+  const data = JSON.parse(fs.readFileSync(file, "utf8"));
+  const descriptor = descriptors.get(data.caseId);
+  if (!descriptor) throw new Error(`Unknown held-out case: ${data.caseId}`);
+  const normalized = {
+    schemaVersion: 1,
+    caseId: data.caseId,
+    intent: data.intent,
+    targets: hostsForArchetype(descriptor.archetype).map((host) => ({ host, scope: "project" })),
+    profile: data.profile,
+    criticalGoals: data.criticalGoals,
+    nonCriticalGoals: data.nonCriticalGoals,
+    minimumNonCriticalGoalCoverage: Math.max(0.8, data.minimumNonCriticalGoalCoverage ?? 0.8),
+    requiresSkill: true,
+    policy: {
+      allowedRisk: ["none", "safe"],
+      requireKnownSource: false,
+      allowManualSetup: false
+    },
+    provenance: data.provenance,
+    taskFamilyFingerprint: descriptor.taskFamilyId
+  };
+  fs.writeFileSync(file, `${JSON.stringify(normalized, null, 2)}\n`);
+  casePaths.set(data.caseId, path.relative(benchmarkRoot, file));
+}
+
+for (const file of walkJson(goldRoot)) {
+  const data = JSON.parse(fs.readFileSync(file, "utf8"));
+  const normalized = {
+    schemaVersion: 1,
+    caseId: data.caseId,
+    acceptedSolutions: data.acceptedSolutions,
+    ambiguous: data.ambiguous,
+    provenance: {
+      source: data.provenance?.source || "independent-synthetic-held-out-gold",
+      version: data.provenance?.version || "1.0.0",
+      reviewedAt: data.provenance?.reviewedAt || data.provenance?.labeledAt || "2026-07-17"
+    },
+    reviews: []
+  };
+  fs.writeFileSync(file, `${JSON.stringify(normalized, null, 2)}\n`);
+  goldPaths.set(data.caseId, path.relative(benchmarkRoot, file));
+}
+
+index.cases = index.cases.map((entry) => ({
+  ...entry,
+  inputPath: casePaths.get(entry.caseId),
+  goldPath: goldPaths.get(entry.caseId),
+  provenance: "independent-synthetic-held-out@1.0.0",
+  reviewStatus: "pendingIndependentReview"
+}));
+fs.writeFileSync(indexPath, `${JSON.stringify(index, null, 2)}\n`);
+
+console.log(JSON.stringify({ normalizedCases: casePaths.size, normalizedGold: goldPaths.size }, null, 2));