📦 deps(thirdparty): update snapshots

2026-06-18 16:04:36 +00:00
parent 71b421806e
commit dd4c084042
416 changed files with 35467 additions and 3065 deletions
@@ -0,0 +1,119 @@
+/**
+ * Parser-fidelity benchmark for brooks-lint.
+ *
+ * Reads evals/benchmark-corpus.json — a FROZEN corpus of real, model-generated
+ * brooks-lint reports, each paired with an independently graded ground-truth
+ * finding inventory. Runs the shipped report-parse.mjs / sarif.mjs against every
+ * report and measures how faithfully the parser reproduces what the report says.
+ *
+ * Because the parser is deterministic and the corpus is frozen, the numbers are
+ * exactly reproducible: anyone can re-run `npm run benchmark` and get the same
+ * result. This benchmarks the PARSER (the SARIF/CI-gate plumbing), not the model
+ * — model quality is measured separately by the 57-scenario suite (npm run evals:live).
+ *
+ * Exit code: 0 if every report is parsed faithfully and emits valid SARIF; 1 otherwise.
+ */
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseFindings, countFindings } from "./report-parse.mjs";
+import { reportToSarif } from "./sarif.mjs";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const root = path.resolve(__dirname, "..");
+const VALID_LEVELS = new Set(["error", "warning", "note"]);
+
+/** Keep only valid R1–R6 / T1–T6 codes (duplicates preserved), uppercased. */
+function validCodes(codes) {
+  return (codes ?? [])
+    .map((c) => String(c).toUpperCase().trim())
+    .filter((c) => /^[RT][1-6]$/.test(c));
+}
+
+/** Count occurrences of each code → { code: n }. */
+function multiset(codes) {
+  const m = {};
+  for (const c of codes) m[c] = (m[c] ?? 0) + 1;
+  return m;
+}
+
+/**
+ * Score one corpus sample: compare the parser's output against the graded truth.
+ * Returns severity-count match, SARIF validity, and risk-code tp/fp/fn.
+ */
+export function scoreReport(sample) {
+  const pf = parseFindings(sample.report);
+  const pc = countFindings(sample.report);
+  const t = sample.truth;
+  const countMatch = pc.critical === t.critical && pc.warning === t.warning && pc.suggestion === t.suggestion;
+
+  // Compare codes per-finding (multiset), so a dropped duplicate-code finding
+  // is caught, not masked by set-level de-duplication.
+  const pCodes = validCodes(pf.map((f) => f.riskCode));
+  const tCodes = validCodes(t.codes);
+  const pm = multiset(pCodes), tm = multiset(tCodes);
+  let tp = 0, fp = 0, fn = 0;
+  for (const code of new Set([...Object.keys(pm), ...Object.keys(tm)])) {
+    const p = pm[code] ?? 0, q = tm[code] ?? 0;
+    tp += Math.min(p, q);
+    fp += Math.max(0, p - q);
+    fn += Math.max(0, q - p);
+  }
+
+  const sarif = reportToSarif(sample.report, { mode: sample.mode, toolVersion: "bench" });
+  const ruleIds = new Set(sarif.runs[0].tool.driver.rules.map((r) => r.id));
+  const results = sarif.runs[0].results;
+  const sarifValid = sarif.version === "2.1.0"
+    && results.length === pf.length
+    && results.every((r) => VALID_LEVELS.has(r.level))
+    && results.every((r) => ruleIds.has(r.ruleId));
+
+  return { id: sample.id, mode: sample.mode, isFP: sample.isFP, countMatch, sarifValid, tp, fp, fn,
+    truth: `${t.critical}/${t.warning}/${t.suggestion}`, parser: `${pc.critical}/${pc.warning}/${pc.suggestion}`,
+    truthCodes: [...new Set(tCodes)].sort(), parserCodes: [...new Set(pCodes)].sort() };
+}
+
+/**
+ * Score every sample in a corpus and aggregate corpus-wide totals:
+ * exact severity-count matches, SARIF validity, and code precision/recall.
+ */
+export function summarize(corpus) {
+  const rows = corpus.samples.map(scoreReport);
+  const n = rows.length;
+  const exact = rows.filter((r) => r.countMatch).length;
+  const sarifOk = rows.filter((r) => r.sarifValid).length;
+  const tp = rows.reduce((s, r) => s + r.tp, 0);
+  const fp = rows.reduce((s, r) => s + r.fp, 0);
+  const fn = rows.reduce((s, r) => s + r.fn, 0);
+  return {
+    rows, n, exact, sarifOk, tp, fp, fn,
+    precision: tp / (tp + fp || 1),
+    recall: tp / (tp + fn || 1),
+  };
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  const corpus = JSON.parse(readFileSync(path.join(root, "evals/benchmark-corpus.json"), "utf8"));
+  const s = summarize(corpus);
+
+  console.log("\nBrooks-Lint Parser-Fidelity Benchmark");
+  console.log("=====================================");
+  console.log(`Corpus: ${s.n} real model-generated reports (frozen) across ${new Set(s.rows.map((r) => r.mode)).size} modes`);
+  console.table(s.rows.map((r) => ({
+    id: r.id, mode: r.mode, FP: r.isFP ? "Y" : "",
+    truth: r.truth, parser: r.parser, countMatch: r.countMatch,
+    codes: r.parserCodes.join(",") || "-", sarif: r.sarifValid ? "ok" : "BAD",
+  })));
+  console.log(`Exact severity-count match : ${s.exact}/${s.n} (${(100 * s.exact / s.n).toFixed(1)}%)`);
+  console.log(`Risk-code precision        : ${(100 * s.precision).toFixed(1)}%   recall: ${(100 * s.recall).toFixed(1)}%   (tp=${s.tp} fp=${s.fp} fn=${s.fn})`);
+  console.log(`SARIF 2.1.0 validity       : ${s.sarifOk}/${s.n}`);
+
+  if (corpus.strictness?.length) {
+    console.log("\nStrictness preset scoring (recorded single-run, fixed 2C/3W/1S findings):");
+    console.table(corpus.strictness.map((x) => ({ preset: x.preset, expected: x.expected, modelScore: x.score, match: x.score === x.expected, leadsWithTopFixes: x.leadsWithTopFixes })));
+  }
+
+  const ok = s.exact === s.n && s.sarifOk === s.n;
+  console.log(`\n${ok ? "PASS" : "FAIL"} — parser fidelity ${ok ? "100%" : "below threshold"} on the frozen corpus.`);
+  if (!ok) process.exit(1);
+}