📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Parser-fidelity benchmark for brooks-lint.
|
||||
*
|
||||
* Reads evals/benchmark-corpus.json — a FROZEN corpus of real, model-generated
|
||||
* brooks-lint reports, each paired with an independently graded ground-truth
|
||||
* finding inventory. Runs the shipped report-parse.mjs / sarif.mjs against every
|
||||
* report and measures how faithfully the parser reproduces what the report says.
|
||||
*
|
||||
* Because the parser is deterministic and the corpus is frozen, the numbers are
|
||||
* exactly reproducible: anyone can re-run `npm run benchmark` and get the same
|
||||
* result. This benchmarks the PARSER (the SARIF/CI-gate plumbing), not the model
|
||||
* — model quality is measured separately by the 57-scenario suite (npm run evals:live).
|
||||
*
|
||||
* Exit code: 0 if every report is parsed faithfully and emits valid SARIF; 1 otherwise.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseFindings, countFindings } from "./report-parse.mjs";
|
||||
import { reportToSarif } from "./sarif.mjs";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.resolve(__dirname, "..");
|
||||
const VALID_LEVELS = new Set(["error", "warning", "note"]);
|
||||
|
||||
/** Keep only valid R1–R6 / T1–T6 codes (duplicates preserved), uppercased. */
|
||||
function validCodes(codes) {
|
||||
return (codes ?? [])
|
||||
.map((c) => String(c).toUpperCase().trim())
|
||||
.filter((c) => /^[RT][1-6]$/.test(c));
|
||||
}
|
||||
|
||||
/** Count occurrences of each code → { code: n }. */
|
||||
function multiset(codes) {
|
||||
const m = {};
|
||||
for (const c of codes) m[c] = (m[c] ?? 0) + 1;
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score one corpus sample: compare the parser's output against the graded truth.
|
||||
* Returns severity-count match, SARIF validity, and risk-code tp/fp/fn.
|
||||
*/
|
||||
export function scoreReport(sample) {
|
||||
const pf = parseFindings(sample.report);
|
||||
const pc = countFindings(sample.report);
|
||||
const t = sample.truth;
|
||||
const countMatch = pc.critical === t.critical && pc.warning === t.warning && pc.suggestion === t.suggestion;
|
||||
|
||||
// Compare codes per-finding (multiset), so a dropped duplicate-code finding
|
||||
// is caught, not masked by set-level de-duplication.
|
||||
const pCodes = validCodes(pf.map((f) => f.riskCode));
|
||||
const tCodes = validCodes(t.codes);
|
||||
const pm = multiset(pCodes), tm = multiset(tCodes);
|
||||
let tp = 0, fp = 0, fn = 0;
|
||||
for (const code of new Set([...Object.keys(pm), ...Object.keys(tm)])) {
|
||||
const p = pm[code] ?? 0, q = tm[code] ?? 0;
|
||||
tp += Math.min(p, q);
|
||||
fp += Math.max(0, p - q);
|
||||
fn += Math.max(0, q - p);
|
||||
}
|
||||
|
||||
const sarif = reportToSarif(sample.report, { mode: sample.mode, toolVersion: "bench" });
|
||||
const ruleIds = new Set(sarif.runs[0].tool.driver.rules.map((r) => r.id));
|
||||
const results = sarif.runs[0].results;
|
||||
const sarifValid = sarif.version === "2.1.0"
|
||||
&& results.length === pf.length
|
||||
&& results.every((r) => VALID_LEVELS.has(r.level))
|
||||
&& results.every((r) => ruleIds.has(r.ruleId));
|
||||
|
||||
return { id: sample.id, mode: sample.mode, isFP: sample.isFP, countMatch, sarifValid, tp, fp, fn,
|
||||
truth: `${t.critical}/${t.warning}/${t.suggestion}`, parser: `${pc.critical}/${pc.warning}/${pc.suggestion}`,
|
||||
truthCodes: [...new Set(tCodes)].sort(), parserCodes: [...new Set(pCodes)].sort() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Score every sample in a corpus and aggregate corpus-wide totals:
|
||||
* exact severity-count matches, SARIF validity, and code precision/recall.
|
||||
*/
|
||||
export function summarize(corpus) {
|
||||
const rows = corpus.samples.map(scoreReport);
|
||||
const n = rows.length;
|
||||
const exact = rows.filter((r) => r.countMatch).length;
|
||||
const sarifOk = rows.filter((r) => r.sarifValid).length;
|
||||
const tp = rows.reduce((s, r) => s + r.tp, 0);
|
||||
const fp = rows.reduce((s, r) => s + r.fp, 0);
|
||||
const fn = rows.reduce((s, r) => s + r.fn, 0);
|
||||
return {
|
||||
rows, n, exact, sarifOk, tp, fp, fn,
|
||||
precision: tp / (tp + fp || 1),
|
||||
recall: tp / (tp + fn || 1),
|
||||
};
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const corpus = JSON.parse(readFileSync(path.join(root, "evals/benchmark-corpus.json"), "utf8"));
|
||||
const s = summarize(corpus);
|
||||
|
||||
console.log("\nBrooks-Lint Parser-Fidelity Benchmark");
|
||||
console.log("=====================================");
|
||||
console.log(`Corpus: ${s.n} real model-generated reports (frozen) across ${new Set(s.rows.map((r) => r.mode)).size} modes`);
|
||||
console.table(s.rows.map((r) => ({
|
||||
id: r.id, mode: r.mode, FP: r.isFP ? "Y" : "",
|
||||
truth: r.truth, parser: r.parser, countMatch: r.countMatch,
|
||||
codes: r.parserCodes.join(",") || "-", sarif: r.sarifValid ? "ok" : "BAD",
|
||||
})));
|
||||
console.log(`Exact severity-count match : ${s.exact}/${s.n} (${(100 * s.exact / s.n).toFixed(1)}%)`);
|
||||
console.log(`Risk-code precision : ${(100 * s.precision).toFixed(1)}% recall: ${(100 * s.recall).toFixed(1)}% (tp=${s.tp} fp=${s.fp} fn=${s.fn})`);
|
||||
console.log(`SARIF 2.1.0 validity : ${s.sarifOk}/${s.n}`);
|
||||
|
||||
if (corpus.strictness?.length) {
|
||||
console.log("\nStrictness preset scoring (recorded single-run, fixed 2C/3W/1S findings):");
|
||||
console.table(corpus.strictness.map((x) => ({ preset: x.preset, expected: x.expected, modelScore: x.score, match: x.score === x.expected, leadsWithTopFixes: x.leadsWithTopFixes })));
|
||||
}
|
||||
|
||||
const ok = s.exact === s.n && s.sarifOk === s.n;
|
||||
console.log(`\n${ok ? "PASS" : "FAIL"} — parser fidelity ${ok ? "100%" : "below threshold"} on the frozen corpus.`);
|
||||
if (!ok) process.exit(1);
|
||||
}
|
||||
@@ -28,9 +28,11 @@ for (const { rel, update } of manifests) {
|
||||
console.log(` ✓ ${rel}`);
|
||||
}
|
||||
|
||||
let readme = readFileSync(path.join(root, "README.md"), "utf8");
|
||||
readme = readme.replace(/version-[\d.]+?-blue\.svg/g, `version-${version}-blue.svg`);
|
||||
writeFileSync(path.join(root, "README.md"), readme, "utf8");
|
||||
console.log(" ✓ README.md badge");
|
||||
for (const readmeRel of ["README.md", "README.zh-CN.md"]) {
|
||||
let readme = readFileSync(path.join(root, readmeRel), "utf8");
|
||||
readme = readme.replace(/version-[\d.]+?-blue\.svg/g, `version-${version}-blue.svg`);
|
||||
writeFileSync(path.join(root, readmeRel), readme, "utf8");
|
||||
console.log(` ✓ ${readmeRel} badge`);
|
||||
}
|
||||
|
||||
console.log(`\nAll manifests updated to ${version}. Run npm run validate to confirm.`);
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* CI quality gates for the GitHub Action.
|
||||
*
|
||||
* Pure decision helpers (severityBreached / isRegression) plus a CLI that reads
|
||||
* the JSON report emitted by ci-review.mjs and exits non-zero when a gate is
|
||||
* breached. Kept out of action.yml so the logic is unit-testable instead of
|
||||
* living as an untested inline `node -e` block.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/ci-gate.mjs --report brooks-lint-report.json \
|
||||
* --fail-on critical --fail-on-regression true
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "./cli-utils.mjs";
|
||||
|
||||
/**
|
||||
* True when findings at/above the `failOn` severity exist.
|
||||
* @param {{critical?: number, warning?: number, suggestion?: number}} findings
|
||||
* @param {"none"|"warning"|"critical"} failOn
|
||||
*/
|
||||
export function severityBreached(findings, failOn) {
|
||||
const critical = findings?.critical ?? 0;
|
||||
const warning = findings?.warning ?? 0;
|
||||
if (failOn === "critical") return critical > 0;
|
||||
if (failOn === "warning") return critical + warning > 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True when the Health Score regressed (delta is a negative number). */
|
||||
export function isRegression(delta) {
|
||||
return typeof delta === "number" && delta < 0;
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const report = JSON.parse(readFileSync(args.report, "utf8"));
|
||||
const failOn = args["fail-on"] ?? "none";
|
||||
// parseArgs yields boolean true for a bare flag and a string otherwise.
|
||||
const failOnRegression = String(args["fail-on-regression"]).toLowerCase() === "true";
|
||||
|
||||
let failed = false;
|
||||
if (failOn !== "none" && severityBreached(report.findings, failOn)) {
|
||||
console.error(`Severity gate failed (fail-on=${failOn}): ${JSON.stringify(report.findings)}`);
|
||||
failed = true;
|
||||
}
|
||||
if (failOnRegression && report.delta == null) {
|
||||
console.log("Regression gate inactive: no prior history (commit .brooks-lint-history.json to enable it).");
|
||||
} else if (failOnRegression && isRegression(report.delta)) {
|
||||
console.error(`Regression gate failed: ${report.previousScore} → ${report.score} (${report.delta})`);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (failed) process.exit(1);
|
||||
console.log("brooks-lint quality gates passed.");
|
||||
}
|
||||
@@ -3,25 +3,32 @@
|
||||
* Shared prompt assembly with run-evals-live.mjs (via assemble-prompt.mjs).
|
||||
*
|
||||
* Reads git diff from the project, assembles the system prompt for the mode,
|
||||
* calls Claude API, and outputs JSON { report, score, mode } to stdout.
|
||||
* calls Claude API, and outputs JSON { report, score, mode, scope, trend,
|
||||
* findings, previousScore, delta } to stdout. With --format sarif it emits a
|
||||
* SARIF 2.1.0 log instead (for GitHub Code Scanning).
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/ci-review.mjs \
|
||||
* --mode review \
|
||||
* --model claude-sonnet-4-6 \
|
||||
* --skills-dir ./skills \
|
||||
* --project-dir /path/to/project
|
||||
* --project-dir /path/to/project \
|
||||
* [--format json|sarif] \
|
||||
* [--sarif-out brooks-lint.sarif]
|
||||
*
|
||||
* Environment:
|
||||
* ANTHROPIC_API_KEY required
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Anthropic from "@anthropic-ai/sdk";
|
||||
import { assembleSystemPrompt, VALID_MODES } from "./assemble-prompt.mjs";
|
||||
import { readHistory, getTrend } from "./history.mjs";
|
||||
import { countFindings } from "./report-parse.mjs";
|
||||
import { reportToSarif } from "./sarif.mjs";
|
||||
import { parseArgs } from "./cli-utils.mjs";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
@@ -30,14 +37,23 @@ const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
const mode = args.mode ?? "review";
|
||||
const model = args.model ?? "claude-sonnet-4-6";
|
||||
const format = args.format ?? "json";
|
||||
const skillsDir = path.resolve(args["skills-dir"] ?? path.join(__dirname, "..", "skills"));
|
||||
const projectDir = path.resolve(args["project-dir"] ?? process.cwd());
|
||||
const toolVersion = JSON.parse(
|
||||
readFileSync(path.join(__dirname, "..", "package.json"), "utf8"),
|
||||
).version;
|
||||
|
||||
if (!VALID_MODES.includes(mode)) {
|
||||
console.error(`Unknown mode: ${mode}. Valid modes: ${VALID_MODES.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!["json", "sarif"].includes(format)) {
|
||||
console.error(`Unknown format: ${format}. Valid formats: json, sarif`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Read git diff ─────────────────────────────────────────────────────────────
|
||||
|
||||
function getGitDiff(projectRoot) {
|
||||
@@ -90,17 +106,41 @@ const report = message.content[0]?.text ?? "";
|
||||
const scoreMatch = report.match(/Health\s+Score[:\s]+(\d+)/i);
|
||||
const score = scoreMatch ? parseInt(scoreMatch[1], 10) : null;
|
||||
|
||||
const findings = countFindings(report);
|
||||
|
||||
const trend = getTrend(readHistory(projectDir), mode);
|
||||
const previousScore = trend ? trend.lastScore : null;
|
||||
const delta = trend && score !== null ? score - previousScore : null;
|
||||
|
||||
let trendNote;
|
||||
if (!trend) {
|
||||
trendNote = "First CI run — no trend data";
|
||||
} else if (score === null) {
|
||||
trendNote = "Score unavailable — cannot compute trend";
|
||||
} else {
|
||||
const delta = score - trend.lastScore;
|
||||
trendNote = delta === 0
|
||||
? `Stable at ${score} over last ${trend.runCount} runs`
|
||||
: `${trend.lastScore} → ${score} (${delta > 0 ? "+" : ""}${delta}) over last ${trend.runCount} runs`;
|
||||
: `${previousScore} → ${score} (${delta > 0 ? "+" : ""}${delta}) over last ${trend.runCount} runs`;
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ report, score, mode, scope, trend: trendNote }, null, 2));
|
||||
// SARIF is needed if either the stdout format is sarif or --sarif-out is set.
|
||||
const needsSarif = format === "sarif" || args["sarif-out"];
|
||||
const sarif = needsSarif
|
||||
? JSON.stringify(reportToSarif(report, { mode, toolVersion }), null, 2)
|
||||
: null;
|
||||
|
||||
// --sarif-out writes a SARIF file regardless of the stdout format, so the Action
|
||||
// can keep emitting JSON (for the PR comment + gates) and still upload SARIF.
|
||||
if (args["sarif-out"]) {
|
||||
writeFileSync(path.resolve(args["sarif-out"]), sarif + "\n");
|
||||
}
|
||||
|
||||
if (format === "sarif") {
|
||||
console.log(sarif);
|
||||
} else {
|
||||
console.log(JSON.stringify(
|
||||
{ report, score, mode, scope, trend: trendNote, findings, previousScore, delta },
|
||||
null,
|
||||
2,
|
||||
));
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
* Used by run-evals-live.mjs (runtime) and validate-repo.test.mjs (tests).
|
||||
*/
|
||||
|
||||
const RISK_CODE_RE = /\b([RT]\d+)\b/g;
|
||||
// Only R1–R6 / T1–T6 are valid codes; \d+ would also match typos like R10 or
|
||||
// stray text like "R20", polluting true/false-positive classification.
|
||||
const RISK_CODE_RE = /\b([RT][1-6])\b/g;
|
||||
|
||||
export function extractRiskCodes(text) {
|
||||
return new Set(text.match(RISK_CODE_RE) ?? []);
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
* Reads and writes .brooks-lint-history.json in the project root.
|
||||
* Each record: { date, mode, score, findings: { critical, warning, suggestion }, scope }
|
||||
*
|
||||
* Run: node scripts/history.mjs [projectRoot]
|
||||
* Prints the history for projectRoot (default: cwd).
|
||||
* Run: node scripts/history.mjs [projectRoot] # readable trend view
|
||||
* node scripts/history.mjs [projectRoot] --json # raw JSON for tooling
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
@@ -14,6 +14,27 @@ import path from "node:path";
|
||||
|
||||
const HISTORY_FILE = ".brooks-lint-history.json";
|
||||
|
||||
// Map the human-facing mode names the report template uses (common.md) onto the
|
||||
// canonical CLI mode names. History records are written by the model and may use
|
||||
// either form ("PR Review" vs "review"); ci-review.mjs queries with the canonical
|
||||
// name. Normalizing both sides keeps getTrend from silently missing every record.
|
||||
const MODE_ALIASES = {
|
||||
"pr review": "review",
|
||||
"architecture audit": "audit",
|
||||
"tech debt": "debt",
|
||||
"tech debt assessment": "debt",
|
||||
"test quality": "test",
|
||||
"test quality review": "test",
|
||||
"health dashboard": "health",
|
||||
"full sweep": "sweep",
|
||||
};
|
||||
|
||||
export function normalizeMode(mode) {
|
||||
if (typeof mode !== "string") return mode;
|
||||
const key = mode.trim().toLowerCase();
|
||||
return MODE_ALIASES[key] ?? key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read history from .brooks-lint-history.json.
|
||||
* Returns empty array if the file does not exist or contains invalid JSON.
|
||||
@@ -40,11 +61,14 @@ export function appendHistory(projectRoot, record) {
|
||||
|
||||
/**
|
||||
* Get trend info for a mode from a history array (not including the current run).
|
||||
* Mode matching is alias-tolerant (see normalizeMode), so a canonical query like
|
||||
* "review" still matches records stored as "PR Review".
|
||||
* Returns null if no prior records exist for the mode.
|
||||
* Returns { lastScore, runCount } where lastScore is the most recent prior score.
|
||||
*/
|
||||
export function getTrend(history, mode) {
|
||||
const modeHistory = history.filter(r => r.mode === mode);
|
||||
const target = normalizeMode(mode);
|
||||
const modeHistory = history.filter(r => normalizeMode(r.mode) === target);
|
||||
if (modeHistory.length === 0) return null;
|
||||
return {
|
||||
lastScore: modeHistory[modeHistory.length - 1].score,
|
||||
@@ -52,12 +76,50 @@ export function getTrend(history, mode) {
|
||||
};
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const projectRoot = process.argv[2] ?? process.cwd();
|
||||
const history = readHistory(projectRoot);
|
||||
if (history.length === 0) {
|
||||
console.log("No history found.");
|
||||
} else {
|
||||
console.log(JSON.stringify(history, null, 2));
|
||||
}
|
||||
/**
|
||||
* Render a sequence of 0–100 scores as a unicode sparkline.
|
||||
*/
|
||||
export function sparkline(scores) {
|
||||
const bars = "▁▂▃▄▅▆▇█";
|
||||
return scores
|
||||
.map(s => {
|
||||
const clamped = Math.max(0, Math.min(100, s));
|
||||
return bars[Math.round((clamped / 100) * (bars.length - 1))];
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the whole history as a per-mode trend summary (one line per mode).
|
||||
*/
|
||||
export function renderHistory(history) {
|
||||
if (history.length === 0) return "No history found.";
|
||||
|
||||
const byMode = new Map();
|
||||
for (const r of history) {
|
||||
const m = normalizeMode(r.mode);
|
||||
if (!byMode.has(m)) byMode.set(m, []);
|
||||
byMode.get(m).push(r);
|
||||
}
|
||||
|
||||
const lines = [`Brooks-Lint Health History — ${history.length} record(s)`, ""];
|
||||
for (const [mode, records] of byMode) {
|
||||
const scores = records.map(r => r.score).filter(s => typeof s === "number");
|
||||
if (scores.length === 0) continue;
|
||||
const latest = scores[scores.length - 1];
|
||||
const delta = latest - scores[0];
|
||||
const trend = scores.length > 1
|
||||
? `${delta >= 0 ? "+" : ""}${delta} over ${scores.length} runs`
|
||||
: "1 run";
|
||||
lines.push(`${mode.padEnd(8)} ${sparkline(scores)} latest ${latest}/100 (${trend})`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const cliArgs = process.argv.slice(2);
|
||||
const asJson = cliArgs.includes("--json");
|
||||
const projectRoot = cliArgs.find(a => !a.startsWith("--")) ?? process.cwd();
|
||||
const history = readHistory(projectRoot);
|
||||
console.log(asJson ? JSON.stringify(history, null, 2) : renderHistory(history));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* Parse a brooks-lint Markdown report into structured findings.
|
||||
*
|
||||
* The report format is defined by the Report Template in skills/_shared/common.md:
|
||||
* findings live under severity sub-headers (### 🔴 Critical / 🟡 Warning /
|
||||
* 🟢 Suggestion), each finding is a bold title line `**Risk Name — title**`
|
||||
* followed by Symptom / Source / Consequence / Remedy fields.
|
||||
*
|
||||
* Consumed by sarif.mjs (SARIF export) and ci-review.mjs (severity gates).
|
||||
* Best-effort: the report is LLM-authored, so the parser tolerates bracket
|
||||
* placeholders, an explicit `(R2)` code, and inline severity emoji.
|
||||
*/
|
||||
|
||||
/** Canonical risk code → display name (decay-risks.md + test-decay-risks.md). */
|
||||
export const RISK_CATALOG = {
|
||||
R1: "Cognitive Overload",
|
||||
R2: "Change Propagation",
|
||||
R3: "Knowledge Duplication",
|
||||
R4: "Accidental Complexity",
|
||||
R5: "Dependency Disorder",
|
||||
R6: "Domain Model Distortion",
|
||||
T1: "Test Obscurity",
|
||||
T2: "Test Brittleness",
|
||||
T3: "Test Duplication",
|
||||
T4: "Mock Abuse",
|
||||
T5: "Coverage Illusion",
|
||||
T6: "Architecture Mismatch",
|
||||
};
|
||||
|
||||
const NAME_TO_CODE = Object.fromEntries(
|
||||
Object.entries(RISK_CATALOG).map(([code, name]) => [name.toLowerCase(), code]),
|
||||
);
|
||||
|
||||
// The template prescribes a bare `### 🔴 Critical`, but LLM output drifts —
|
||||
// tolerate a plural and a trailing "Issues"/"Findings"/"Items" qualifier while
|
||||
// still anchoring on the line so section headers like `## Findings` never match.
|
||||
const SEVERITY_HEADER_RE =
|
||||
/^#{2,6}\s*(?:🔴|🟡|🟢|⚠️?|❗)?\s*(Critical|Warning|Suggestion)s?(?:\s+(?:Issues?|Findings?|Items?))?\s*:?\s*$/i;
|
||||
const SECTION_HEADER_RE = /^#{1,6}\s/;
|
||||
const BOLD_TITLE_RE = /^\s*(?:🔴|🟡|🟢)?\s*\*\*(.+?)\*\*\s*$/;
|
||||
const FIELD_RE = /^\s*(Symptom|Source|Consequence|Remedy)\s*[:：]\s*(.*)$/i;
|
||||
const EMOJI_SEVERITY = { "🔴": "critical", "🟡": "warning", "🟢": "suggestion" };
|
||||
|
||||
// A path with a directory separator, or a bare filename with a known source
|
||||
// extension — optionally followed by `:line`. The extension allowlist keeps
|
||||
// prose like "e.g." or "i.e." from being mistaken for a file reference.
|
||||
const LOCATION_RE =
|
||||
/([\w.-]*\/[\w./-]*\.\w+|[\w.-]+\.(?:ts|tsx|js|jsx|mjs|cjs|py|java|go|rb|rs|cc|cpp|cxx|c|h|hpp|cs|php|kt|kts|swift|scala|vue|sql|rsx|m|mm))(?::(\d+))?/;
|
||||
|
||||
function splitTitle(bold) {
|
||||
// Dash is the template separator; a colon is a common LLM variant. `.match`
|
||||
// returns the leftmost hit, so a dash still wins when both are present.
|
||||
const sep = bold.match(/\s*[—–]\s*|\s+--\s+|\s+-\s+|\s*:\s*/);
|
||||
if (!sep) return { namePart: bold.trim(), title: "" };
|
||||
return {
|
||||
namePart: bold.slice(0, sep.index).trim(),
|
||||
title: bold.slice(sep.index + sep[0].length).trim(),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCode(namePart) {
|
||||
const explicit = namePart.match(/\b([RT][1-6])\b/);
|
||||
if (explicit) return explicit[1].toUpperCase();
|
||||
const cleaned = namePart
|
||||
.replace(/\(([RT][1-6])\)/i, "")
|
||||
.replace(/[[\]]/g, "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (NAME_TO_CODE[cleaned]) return NAME_TO_CODE[cleaned];
|
||||
// Fallback: a missed separator can leave trailing words on the name, so match
|
||||
// the longest known risk name the cleaned string starts with.
|
||||
const prefix = Object.keys(NAME_TO_CODE)
|
||||
.filter((name) => cleaned.startsWith(name))
|
||||
.sort((a, b) => b.length - a.length)[0];
|
||||
return prefix ? NAME_TO_CODE[prefix] : null;
|
||||
}
|
||||
|
||||
/** Extract `{ file, line }` from text, or `{ file: null, line: null }`. */
|
||||
export function extractLocation(text) {
|
||||
const m = (text ?? "").match(LOCATION_RE);
|
||||
if (!m) return { file: null, line: null };
|
||||
// Group 1 is the path; group 2 is the optional `:line` (the extension list is
|
||||
// a non-capturing group, so the line digits are m[2], not m[3]).
|
||||
return { file: m[1], line: m[2] ? parseInt(m[2], 10) : null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a report into an array of findings.
|
||||
* @returns {Array<{severity, riskCode, riskName, title, symptom, source,
|
||||
* consequence, remedy, file, line}>}
|
||||
*/
|
||||
export function parseFindings(report) {
|
||||
const lines = (report ?? "").split(/\r?\n/);
|
||||
const findings = [];
|
||||
let severity = null;
|
||||
let current = null;
|
||||
let field = null;
|
||||
|
||||
const commit = () => {
|
||||
if (!current) return;
|
||||
// Keep only blocks that look like real findings (a known risk or a symptom).
|
||||
if (current.riskCode || current.symptom) findings.push(current);
|
||||
current = null;
|
||||
field = null;
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const sevHeader = line.match(SEVERITY_HEADER_RE);
|
||||
if (sevHeader) {
|
||||
commit();
|
||||
severity = sevHeader[1].toLowerCase();
|
||||
continue;
|
||||
}
|
||||
if (SECTION_HEADER_RE.test(line)) {
|
||||
// A non-severity header ends the current Findings group (e.g. ## Summary).
|
||||
commit();
|
||||
severity = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
const bold = severity && line.match(BOLD_TITLE_RE);
|
||||
if (bold) {
|
||||
commit();
|
||||
const emoji = line.match(/^\s*(🔴|🟡|🟢)/);
|
||||
const { namePart, title } = splitTitle(bold[1]);
|
||||
const riskCode = resolveCode(namePart);
|
||||
current = {
|
||||
severity: emoji ? EMOJI_SEVERITY[emoji[1]] : severity,
|
||||
riskCode,
|
||||
riskName: riskCode ? RISK_CATALOG[riskCode] : namePart.replace(/[[\]]/g, "").trim(),
|
||||
title,
|
||||
symptom: "",
|
||||
source: "",
|
||||
consequence: "",
|
||||
remedy: "",
|
||||
file: null,
|
||||
line: null,
|
||||
};
|
||||
field = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!current) continue;
|
||||
|
||||
const fieldMatch = line.match(FIELD_RE);
|
||||
if (fieldMatch) {
|
||||
field = fieldMatch[1].toLowerCase();
|
||||
current[field] = fieldMatch[2].trim();
|
||||
continue;
|
||||
}
|
||||
// Continuation line for the field in progress.
|
||||
if (field && line.trim()) {
|
||||
current[field] = `${current[field]} ${line.trim()}`.trim();
|
||||
}
|
||||
}
|
||||
commit();
|
||||
|
||||
for (const f of findings) {
|
||||
// Location belongs in the Symptom, but fall back to Source/Consequence when
|
||||
// it's absent. Remedy is excluded — it often names a destination, not the
|
||||
// site of the finding.
|
||||
const fromSymptom = extractLocation(f.symptom);
|
||||
const loc = fromSymptom.file ? fromSymptom : extractLocation(`${f.source} ${f.consequence}`);
|
||||
f.file = loc.file;
|
||||
f.line = loc.line;
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/** Count findings by severity. @returns {{critical, warning, suggestion}} */
|
||||
export function countFindings(report) {
|
||||
const counts = { critical: 0, warning: 0, suggestion: 0 };
|
||||
for (const f of parseFindings(report)) {
|
||||
if (counts[f.severity] !== undefined) counts[f.severity] += 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
@@ -48,6 +48,14 @@ for (let i = 0; i < evals.length; i++) {
|
||||
}
|
||||
}
|
||||
|
||||
// Explicit duplicate-id guard (the sequential check only catches dups that also
|
||||
// break the running count; a deliberate re-use of the same id would not).
|
||||
const idCounts = new Map();
|
||||
for (const ev of evals) idCounts.set(ev.id, (idCounts.get(ev.id) ?? 0) + 1);
|
||||
for (const [id, count] of idCounts) {
|
||||
if (count > 1) errors.push(`Duplicate eval id ${JSON.stringify(id)} appears ${count} times`);
|
||||
}
|
||||
|
||||
// ── Per-eval field and content checks ─────────────────────────────────────
|
||||
|
||||
for (const ev of evals) {
|
||||
@@ -71,13 +79,34 @@ for (const ev of evals) {
|
||||
errors.push(`${label}: 'mode' must be one of ${VALID_MODES.join(", ")} (got '${ev.mode}')`);
|
||||
}
|
||||
|
||||
if ("files" in ev && !Array.isArray(ev.files)) {
|
||||
errors.push(`${label}: 'files' must be an array when present (got ${typeof ev.files})`);
|
||||
}
|
||||
|
||||
// expected_output should reference at least one risk code so reviewers know
|
||||
// which risk the scenario is testing
|
||||
// which risk the scenario is testing. False-positive (no_risk_codes) and
|
||||
// health-score-suppression (no_health_score) scenarios are code-free by
|
||||
// design — warning on them is noise, so the check skips boundary scenarios.
|
||||
if (typeof ev.expected_output === "string") {
|
||||
const referencedCodes = RISK_CODES.filter((code) => ev.expected_output.includes(code));
|
||||
if (referencedCodes.length === 0) {
|
||||
const isBoundaryScenario = ev.no_risk_codes || ev.no_health_score;
|
||||
if (referencedCodes.length === 0 && !isBoundaryScenario) {
|
||||
warnings.push(`${label}: expected_output does not reference any risk code (${RISK_CODES.join(", ")})`);
|
||||
}
|
||||
|
||||
// mode ↔ risk-code compatibility: assemble-prompt.mjs only loads the risk
|
||||
// definitions for that mode (test→T-codes, review/audit/debt→R-codes,
|
||||
// health/sweep→both). A code outside the loaded set is a dead reference —
|
||||
// the model is never given its definition, so the scenario cannot pass live.
|
||||
// RISK_CODES is R/T-prefixed by construction, so c[0] fully partitions it.
|
||||
const refsR = referencedCodes.filter((c) => c[0] === "R");
|
||||
const refsT = referencedCodes.filter((c) => c[0] === "T");
|
||||
if (ev.mode === "test" && refsR.length > 0) {
|
||||
errors.push(`${label}: mode 'test' loads only T-codes but expected_output references ${refsR.join(", ")}`);
|
||||
}
|
||||
if (["review", "audit", "debt"].includes(ev.mode) && refsT.length > 0) {
|
||||
errors.push(`${label}: mode '${ev.mode}' loads only R-codes but expected_output references ${refsT.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
// no_risk_codes and no_health_score are optional flags that put the live
|
||||
@@ -95,18 +124,40 @@ for (const ev of evals) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Reverse coverage ───────────────────────────────────────────────────────
|
||||
// Every risk code must have at least one positive happy-path scenario. Skip the
|
||||
// false-positive (no_risk_codes) and health-score-suppression (no_health_score)
|
||||
// boundary scenarios — neither is a clean positive demonstration of a code.
|
||||
// CLAUDE.md requires "every new risk code gets paired coverage"; this enforces it
|
||||
// so a new code can never ship without a happy-path eval.
|
||||
|
||||
const coveredCodes = new Set();
|
||||
for (const ev of evals) {
|
||||
if (ev.no_risk_codes || ev.no_health_score) continue;
|
||||
if (typeof ev.expected_output !== "string") continue;
|
||||
for (const code of RISK_CODES) {
|
||||
if (ev.expected_output.includes(code)) coveredCodes.add(code);
|
||||
}
|
||||
}
|
||||
const uncoveredCodes = RISK_CODES.filter((code) => !coveredCodes.has(code));
|
||||
if (uncoveredCodes.length > 0) {
|
||||
errors.push(`Risk codes with no positive eval scenario: ${uncoveredCodes.join(", ")}`);
|
||||
}
|
||||
|
||||
// ── Report ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const idCheckPass = !errors.some((e) => e.includes("expected id"));
|
||||
const fieldCheckPass = !errors.some((e) => e.includes("missing required field") || e.includes("is empty"));
|
||||
const idCheckPass = !errors.some((e) => e.includes("expected id") || e.includes("Duplicate eval id"));
|
||||
const fieldCheckPass = !errors.some((e) => e.includes("missing required field") || e.includes("is empty") || e.includes("'files' must"));
|
||||
const coherencePass = !errors.some((e) => e.includes("loads only") || e.includes("no positive eval scenario"));
|
||||
const riskCodePass = warnings.length === 0;
|
||||
|
||||
console.log("\nEval Suite Structural Validation");
|
||||
console.log("=================================");
|
||||
console.log(`Total scenarios : ${evals.length}`);
|
||||
console.log(`Sequential IDs : ${idCheckPass ? "PASS" : "FAIL"}`);
|
||||
console.log(`Required fields : ${fieldCheckPass ? "PASS" : "FAIL"}`);
|
||||
console.log(`Risk code refs : ${riskCodePass ? "PASS" : `${warnings.length} warning(s)`}`);
|
||||
console.log(`Total scenarios : ${evals.length}`);
|
||||
console.log(`Sequential IDs : ${idCheckPass ? "PASS" : "FAIL"}`);
|
||||
console.log(`Required fields : ${fieldCheckPass ? "PASS" : "FAIL"}`);
|
||||
console.log(`Mode/risk & cover : ${coherencePass ? "PASS" : "FAIL"}`);
|
||||
console.log(`Risk code refs : ${riskCodePass ? "PASS" : `${warnings.length} warning(s)`}`);
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.error("\nErrors:");
|
||||
|
||||
Binary file not shown.
@@ -99,6 +99,8 @@ const CANONICAL_INSTALL_CMD = "/plugin marketplace add hyhmrright/brooks-lint";
|
||||
function checkReadmeIntegrity() {
|
||||
const readme = readText("README.md");
|
||||
check(readme.includes(`version-${version}-blue.svg`), `README.md badge does not reference version ${version}`);
|
||||
const readmeZh = readText("README.zh-CN.md");
|
||||
check(readmeZh.includes(`version-${version}-blue.svg`), `README.zh-CN.md badge does not reference version ${version} (run npm run bump)`);
|
||||
check(readme.includes(CANONICAL_INSTALL_CMD), `README.md should contain canonical install command`);
|
||||
check(
|
||||
readme.includes(`grounded in ${sourceWord} classic engineering books`),
|
||||
|
||||
@@ -8,12 +8,12 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { writeFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { assembleSystemPrompt, VALID_MODES } from "./assemble-prompt.mjs";
|
||||
import { readHistory, appendHistory, getTrend } from "./history.mjs";
|
||||
import { readHistory, appendHistory, getTrend, normalizeMode, sparkline, renderHistory } from "./history.mjs";
|
||||
import {
|
||||
parseFrontmatterBooks,
|
||||
countBookSections,
|
||||
@@ -23,6 +23,10 @@ import {
|
||||
extractGuideStepLabels,
|
||||
} from "./frontmatter.mjs";
|
||||
import { extractRiskCodes, classify } from "./eval-utils.mjs";
|
||||
import { parseFindings, countFindings, extractLocation } from "./report-parse.mjs";
|
||||
import { reportToSarif } from "./sarif.mjs";
|
||||
import { severityBreached, isRegression } from "./ci-gate.mjs";
|
||||
import { summarize } from "./benchmark.mjs";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
@@ -371,6 +375,76 @@ test("ignores records for other modes", () => {
|
||||
assert.equal(trend.runCount, 2);
|
||||
});
|
||||
|
||||
test("matches a canonical query against display-name records", () => {
|
||||
// Regression: ci-review.mjs queries with the canonical mode ("review") while
|
||||
// records written by the model are stored as display names ("PR Review").
|
||||
const history = [{ mode: "PR Review", score: 88 }];
|
||||
const trend = getTrend(history, "review");
|
||||
assert.equal(trend.lastScore, 88);
|
||||
assert.equal(trend.runCount, 1);
|
||||
});
|
||||
|
||||
// ── normalizeMode ────────────────────────────────────────────────────────────
|
||||
|
||||
console.log("\nnormalizeMode");
|
||||
|
||||
test("maps display names to canonical modes", () => {
|
||||
assert.equal(normalizeMode("PR Review"), "review");
|
||||
assert.equal(normalizeMode("Architecture Audit"), "audit");
|
||||
assert.equal(normalizeMode("Tech Debt Assessment"), "debt");
|
||||
assert.equal(normalizeMode("Full Sweep"), "sweep");
|
||||
});
|
||||
|
||||
test("passes canonical names through unchanged", () => {
|
||||
assert.equal(normalizeMode("review"), "review");
|
||||
assert.equal(normalizeMode("health"), "health");
|
||||
});
|
||||
|
||||
test("is case- and whitespace-insensitive", () => {
|
||||
assert.equal(normalizeMode(" pr review "), "review");
|
||||
});
|
||||
|
||||
test("passes non-string input through unchanged", () => {
|
||||
assert.equal(normalizeMode(undefined), undefined);
|
||||
});
|
||||
|
||||
// ── sparkline ────────────────────────────────────────────────────────────────
|
||||
|
||||
console.log("\nsparkline");
|
||||
|
||||
test("maps score extremes to the lowest and highest bars", () => {
|
||||
assert.equal(sparkline([0]), "▁");
|
||||
assert.equal(sparkline([100]), "█");
|
||||
});
|
||||
|
||||
test("renders one bar per score and clamps out-of-range values", () => {
|
||||
assert.equal(sparkline([0, 50, 100]).length, 3);
|
||||
assert.equal(sparkline([150]), "█");
|
||||
assert.equal(sparkline([-10]), "▁");
|
||||
});
|
||||
|
||||
// ── renderHistory ────────────────────────────────────────────────────────────
|
||||
|
||||
console.log("\nrenderHistory");
|
||||
|
||||
test("reports no history for an empty array", () => {
|
||||
assert.equal(renderHistory([]), "No history found.");
|
||||
});
|
||||
|
||||
test("summarizes a single record as one run", () => {
|
||||
const out = renderHistory([{ mode: "PR Review", score: 88 }]);
|
||||
assert.match(out, /review/);
|
||||
assert.match(out, /1 run/);
|
||||
});
|
||||
|
||||
test("collapses display-name and canonical records into one mode line", () => {
|
||||
const out = renderHistory([
|
||||
{ mode: "PR Review", score: 70 },
|
||||
{ mode: "review", score: 90 },
|
||||
]);
|
||||
assert.match(out, /\+20 over 2 runs/);
|
||||
});
|
||||
|
||||
// ── extractRiskCodes ───────────────────────────────────────────────────────
|
||||
|
||||
console.log("\nextractRiskCodes");
|
||||
@@ -441,6 +515,294 @@ test("returns 'fail' when codes found but Iron Law terms absent", () => {
|
||||
assert.equal(classify(scenario, aiText), "fail");
|
||||
});
|
||||
|
||||
// ── report-parse: parseFindings / countFindings / extractLocation ──────────
|
||||
|
||||
const SAMPLE_REPORT = [
|
||||
"# Brooks-Lint Review",
|
||||
"",
|
||||
"**Health Score:** 62/100",
|
||||
"",
|
||||
"## Findings",
|
||||
"",
|
||||
"### 🔴 Critical",
|
||||
"",
|
||||
"**Change Propagation — Divergent change**",
|
||||
"Symptom: src/services/UserService.ts:42 handles auth, email, and billing.",
|
||||
"Source: Refactoring — Divergent Change",
|
||||
"Consequence: Every feature touches the same class.",
|
||||
"Remedy: Split into focused collaborators.",
|
||||
"",
|
||||
"### 🟡 Warning",
|
||||
"",
|
||||
"**Cognitive Overload (R1) — God method**",
|
||||
"Symptom: generate() in report_gen.py takes nine positional parameters.",
|
||||
"Source: A Philosophy of Software Design — shallow modules",
|
||||
"Consequence: Callers must understand the whole signature.",
|
||||
"Remedy: Introduce a ReportOptions object.",
|
||||
"",
|
||||
"### 🟢 Suggestion",
|
||||
"",
|
||||
"**Knowledge Duplication — Shipping rule copied**",
|
||||
"Symptom: the free-shipping threshold appears in cart.js and checkout.js.",
|
||||
"Source: The Pragmatic Programmer — DRY",
|
||||
"Consequence: A policy change must be made in two places.",
|
||||
"Remedy: Extract a single shippingPolicy module.",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
"**Bold prose, not a finding** — should be ignored.",
|
||||
].join("\n");
|
||||
|
||||
console.log("\nparseFindings");
|
||||
|
||||
test("parses one finding per severity group", () => {
|
||||
assert.equal(parseFindings(SAMPLE_REPORT).length, 3);
|
||||
});
|
||||
|
||||
test("maps risk name to code and keeps severity", () => {
|
||||
const [crit, warn, sug] = parseFindings(SAMPLE_REPORT);
|
||||
assert.deepEqual([crit.riskCode, crit.severity], ["R2", "critical"]);
|
||||
assert.deepEqual([warn.riskCode, warn.severity], ["R1", "warning"]);
|
||||
assert.deepEqual([sug.riskCode, sug.severity], ["R3", "suggestion"]);
|
||||
});
|
||||
|
||||
test("resolves an explicit (R1) code in the title", () => {
|
||||
const warn = parseFindings(SAMPLE_REPORT)[1];
|
||||
assert.equal(warn.riskName, "Cognitive Overload");
|
||||
assert.equal(warn.title, "God method");
|
||||
});
|
||||
|
||||
test("extracts file and line from the Symptom", () => {
|
||||
const crit = parseFindings(SAMPLE_REPORT)[0];
|
||||
assert.equal(crit.file, "src/services/UserService.ts");
|
||||
assert.equal(crit.line, 42);
|
||||
});
|
||||
|
||||
test("ignores bold text outside any severity group", () => {
|
||||
// The Summary's bold line must not be counted as a finding.
|
||||
assert.ok(parseFindings(SAMPLE_REPORT).every((f) => f.title !== ""));
|
||||
assert.equal(parseFindings(SAMPLE_REPORT).length, 3);
|
||||
});
|
||||
|
||||
test("empty report yields no findings", () => {
|
||||
assert.deepEqual(parseFindings(""), []);
|
||||
assert.deepEqual(parseFindings(null), []);
|
||||
});
|
||||
|
||||
const VARIANT_REPORT = [
|
||||
"## Findings",
|
||||
"",
|
||||
"### 🔴 Critical Issues",
|
||||
"",
|
||||
"**Dependency Disorder: models import services**",
|
||||
"Symptom: a cyclic import exists.",
|
||||
"Source: Clean Architecture — the Dependency Rule",
|
||||
"Consequence: the build in app/core/wiring.ts breaks.",
|
||||
"Remedy: invert the dependency toward an interface.",
|
||||
"",
|
||||
"### 🟡 Warnings",
|
||||
"",
|
||||
"**Coverage Illusion — green but hollow**",
|
||||
"Symptom: the suite asserts nothing meaningful.",
|
||||
"Source: How Google Tests Software — coverage signal",
|
||||
"Consequence: regressions slip through unnoticed.",
|
||||
"Remedy: assert on observable outcomes.",
|
||||
].join("\n");
|
||||
|
||||
test("tolerates plural / qualified severity headers", () => {
|
||||
// `### 🔴 Critical Issues` and `### 🟡 Warnings` must still register as groups.
|
||||
const f = parseFindings(VARIANT_REPORT);
|
||||
assert.equal(f.length, 2);
|
||||
assert.deepEqual([f[0].severity, f[1].severity], ["critical", "warning"]);
|
||||
});
|
||||
|
||||
test("splits a colon-separated title and resolves its code", () => {
|
||||
const first = parseFindings(VARIANT_REPORT)[0];
|
||||
assert.equal(first.riskCode, "R5");
|
||||
assert.equal(first.riskName, "Dependency Disorder");
|
||||
assert.equal(first.title, "models import services");
|
||||
});
|
||||
|
||||
test("falls back to Consequence for the location when Symptom has none", () => {
|
||||
const first = parseFindings(VARIANT_REPORT)[0];
|
||||
assert.equal(first.file, "app/core/wiring.ts");
|
||||
});
|
||||
|
||||
console.log("\ncountFindings");
|
||||
|
||||
test("counts findings by severity", () => {
|
||||
assert.deepEqual(countFindings(SAMPLE_REPORT), { critical: 1, warning: 1, suggestion: 1 });
|
||||
});
|
||||
|
||||
test("empty report counts all zero", () => {
|
||||
assert.deepEqual(countFindings(""), { critical: 0, warning: 0, suggestion: 0 });
|
||||
});
|
||||
|
||||
console.log("\nextractLocation");
|
||||
|
||||
test("captures path with line number", () => {
|
||||
assert.deepEqual(extractLocation("see app/models/order.rb:128 only"), {
|
||||
file: "app/models/order.rb",
|
||||
line: 128,
|
||||
});
|
||||
});
|
||||
|
||||
test("captures bare filename without a line", () => {
|
||||
assert.deepEqual(extractLocation("generate() in report_gen.py"), {
|
||||
file: "report_gen.py",
|
||||
line: null,
|
||||
});
|
||||
});
|
||||
|
||||
test("does not mistake prose for a file reference", () => {
|
||||
assert.deepEqual(extractLocation("nothing here, e.g. no path"), { file: null, line: null });
|
||||
assert.deepEqual(extractLocation("see line 3 (i.e. nowhere)"), { file: null, line: null });
|
||||
});
|
||||
|
||||
// ── sarif: reportToSarif ───────────────────────────────────────────────────
|
||||
|
||||
console.log("\nreportToSarif");
|
||||
|
||||
test("emits a SARIF 2.1.0 envelope", () => {
|
||||
const log = reportToSarif(SAMPLE_REPORT, { mode: "review", toolVersion: "1.3.0" });
|
||||
assert.equal(log.version, "2.1.0");
|
||||
assert.ok(log.$schema.includes("sarif-2.1.0"));
|
||||
assert.equal(log.runs[0].tool.driver.name, "brooks-lint");
|
||||
assert.equal(log.runs[0].tool.driver.version, "1.3.0");
|
||||
});
|
||||
|
||||
test("declares a deduped, PascalCased rule per risk code", () => {
|
||||
const rules = reportToSarif(SAMPLE_REPORT).runs[0].tool.driver.rules;
|
||||
assert.deepEqual(rules.map((r) => r.id), ["R2", "R1", "R3"]);
|
||||
assert.equal(rules[0].name, "ChangePropagation");
|
||||
});
|
||||
|
||||
test("maps severities to SARIF levels", () => {
|
||||
const results = reportToSarif(SAMPLE_REPORT).runs[0].results;
|
||||
assert.deepEqual(results.map((r) => r.level), ["error", "warning", "note"]);
|
||||
});
|
||||
|
||||
test("attaches a physical location when a file is known", () => {
|
||||
const first = reportToSarif(SAMPLE_REPORT).runs[0].results[0];
|
||||
const loc = first.locations[0].physicalLocation;
|
||||
assert.equal(loc.artifactLocation.uri, "src/services/UserService.ts");
|
||||
assert.equal(loc.region.startLine, 42);
|
||||
assert.ok(first.message.text.includes("Remedy:"));
|
||||
});
|
||||
|
||||
test("fingerprints are stable across runs", () => {
|
||||
const a = reportToSarif(SAMPLE_REPORT).runs[0].results[0].partialFingerprints.brooksLint;
|
||||
const b = reportToSarif(SAMPLE_REPORT).runs[0].results[0].partialFingerprints.brooksLint;
|
||||
assert.equal(a, b);
|
||||
});
|
||||
|
||||
test("empty report yields no rules or results", () => {
|
||||
const log = reportToSarif("");
|
||||
assert.deepEqual(log.runs[0].tool.driver.rules, []);
|
||||
assert.deepEqual(log.runs[0].results, []);
|
||||
});
|
||||
|
||||
test("routes T-code helpUri off the guide (no #t anchor) and R-code onto it", () => {
|
||||
const rules = reportToSarif(VARIANT_REPORT).runs[0].tool.driver.rules;
|
||||
const r5 = rules.find((r) => r.id === "R5");
|
||||
const t5 = rules.find((r) => r.id === "T5");
|
||||
assert.ok(r5.helpUri.endsWith("guide.html#r5"));
|
||||
assert.ok(t5.helpUri.includes("test-decay-risks.md"));
|
||||
assert.ok(!t5.helpUri.includes("#t5"));
|
||||
});
|
||||
|
||||
test("declares a BL000 rule when a finding is unmapped", () => {
|
||||
const unmapped = [
|
||||
"## Findings",
|
||||
"",
|
||||
"### 🔴 Critical",
|
||||
"",
|
||||
"**Some Unknown Smell — mystery**",
|
||||
"Symptom: something odd in foo.ts.",
|
||||
"Consequence: unclear impact.",
|
||||
"Remedy: investigate.",
|
||||
].join("\n");
|
||||
const run = reportToSarif(unmapped).runs[0];
|
||||
assert.equal(run.results[0].ruleId, "BL000");
|
||||
assert.ok(run.tool.driver.rules.some((r) => r.id === "BL000"));
|
||||
});
|
||||
|
||||
// ── ci-gate: severityBreached / isRegression ───────────────────────────────
|
||||
|
||||
console.log("\nseverityBreached");
|
||||
|
||||
test("fail-on critical trips only on a critical finding", () => {
|
||||
assert.equal(severityBreached({ critical: 1, warning: 0, suggestion: 0 }, "critical"), true);
|
||||
assert.equal(severityBreached({ critical: 0, warning: 5, suggestion: 9 }, "critical"), false);
|
||||
});
|
||||
|
||||
test("fail-on warning trips on critical or warning", () => {
|
||||
assert.equal(severityBreached({ critical: 0, warning: 1, suggestion: 0 }, "warning"), true);
|
||||
assert.equal(severityBreached({ critical: 2, warning: 0, suggestion: 0 }, "warning"), true);
|
||||
assert.equal(severityBreached({ critical: 0, warning: 0, suggestion: 3 }, "warning"), false);
|
||||
});
|
||||
|
||||
test("fail-on none never trips", () => {
|
||||
assert.equal(severityBreached({ critical: 9, warning: 9, suggestion: 9 }, "none"), false);
|
||||
});
|
||||
|
||||
test("missing or partial findings are treated as zero", () => {
|
||||
assert.equal(severityBreached(undefined, "critical"), false);
|
||||
assert.equal(severityBreached({}, "warning"), false);
|
||||
});
|
||||
|
||||
console.log("\nisRegression");
|
||||
|
||||
test("only a negative numeric delta is a regression", () => {
|
||||
assert.equal(isRegression(-1), true);
|
||||
assert.equal(isRegression(0), false);
|
||||
assert.equal(isRegression(5), false);
|
||||
assert.equal(isRegression(null), false);
|
||||
assert.equal(isRegression(undefined), false);
|
||||
});
|
||||
|
||||
// ── Parser-fidelity benchmark on the FROZEN real-report corpus ─────────────
|
||||
// Deterministic regression guard: the shipped parser must reproduce the
|
||||
// independently-graded finding inventory of 30 real model-generated reports.
|
||||
// This is the non-circular counterpart to the synthetic SAMPLE_REPORT tests
|
||||
// above — the reports here are real model output, the truth was graded by a
|
||||
// separate pass and spot-checked by hand. See scripts/benchmark.mjs.
|
||||
|
||||
console.log("\nparser-fidelity benchmark (frozen real-report corpus)");
|
||||
|
||||
const CORPUS = JSON.parse(readFileSync(path.join(__dirname, "..", "evals", "benchmark-corpus.json"), "utf8"));
|
||||
const BENCH = summarize(CORPUS);
|
||||
|
||||
test("corpus has >= 30 real reports spanning all six modes", () => {
|
||||
assert.ok(CORPUS.samples.length >= 30, `expected >=30 samples, got ${CORPUS.samples.length}`);
|
||||
const modes = new Set(CORPUS.samples.map((s) => s.mode));
|
||||
for (const m of VALID_MODES) assert.ok(modes.has(m), `corpus is missing mode ${m}`);
|
||||
});
|
||||
|
||||
test("corpus composition matches the documented numbers (30 total, 9 false-positive)", () => {
|
||||
// These exact counts are published in the README "Reproducible benchmarks"
|
||||
// section — fail loudly if a corpus regen changes them without a docs update.
|
||||
assert.equal(CORPUS.samples.length, 30);
|
||||
assert.equal(CORPUS.samples.filter((s) => s.isFP).length, 9);
|
||||
});
|
||||
|
||||
test("parser reproduces the graded severity counts on every report", () => {
|
||||
const bad = BENCH.rows.filter((r) => !r.countMatch).map((r) => `${r.id}: truth ${r.truth} vs parser ${r.parser}`);
|
||||
assert.equal(bad.length, 0, `count mismatches: ${bad.join("; ")}`);
|
||||
});
|
||||
|
||||
test("every report emits valid SARIF 2.1.0", () => {
|
||||
const bad = BENCH.rows.filter((r) => !r.sarifValid).map((r) => r.id);
|
||||
assert.equal(bad.length, 0, `invalid SARIF for: ${bad.join(", ")}`);
|
||||
});
|
||||
|
||||
test("risk-code extraction has zero false positives / negatives on the corpus", () => {
|
||||
assert.equal(BENCH.fp, 0, `${BENCH.fp} false-positive code(s)`);
|
||||
assert.equal(BENCH.fn, 0, `${BENCH.fn} false-negative code(s)`);
|
||||
assert.equal(BENCH.precision, 1);
|
||||
assert.equal(BENCH.recall, 1);
|
||||
});
|
||||
|
||||
// ── Integration: validate-repo.mjs passes against current repo ─────────────
|
||||
|
||||
console.log("\nvalidate-repo integration");
|
||||
|
||||
Reference in New Issue
Block a user