#!/usr/bin/env node // Verify → grade → emit regenPlan. Does NOT spawn sub-agents — the // orchestrator reads regenPlan, re-spawns one sub-agent per targeted candidate // with topFailures injected, then re-runs this script. Thresholds are tuned // below (REGEN_*, QUALITY_FLOOR) — read those constants for the live values. import { readFile, writeFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { mkdir } from 'node:fs/promises'; import { verifyClaim } from '../lib/verify-claim.mjs'; import { extractClaims, summarizeClaimResults } from '../lib/extract-claims.mjs'; import { gradeRecommendation, applyQualityFloor } from '../lib/grade-recommendation.mjs'; import { deriveProjectFacts } from '../lib/project-facts.mjs'; import { resolveRepoRoot } from '../lib/repo-root.mjs'; import { applySanitizers } from '../lib/sanitizers/index.mjs'; const SCHEMA_VERSION = '1.0'; const REGEN_PASS_RATE_THRESHOLD = 0.8; // 1/1 failed is as broken as 1/5; below 2 claims is below the noise floor. const REGEN_MIN_CLAIMS = 2; // The Poor/Fair grade boundary — Poor recs erode trust faster than recall helps. const QUALITY_FLOOR = 0.55; const log = (...a) => console.error('[verify-and-regen]', ...a); async function main() { const args = parseArgs(process.argv.slice(2)); if (!args.recsPath) { console.error('usage: node scripts/verify-and-regen.mjs [--signals merged.json] [--repo-root DIR] [--out FILE]'); process.exit(1); } const recs = JSON.parse(await readFile(args.recsPath, 'utf-8')); if (!Array.isArray(recs)) { console.error('[verify-and-regen] FATAL: recommendations.json must be an array of rec objects'); process.exit(2); } let framework, version, cacheComponents, knownFindings = [], projectFacts = [], signals = null; if (args.signalsPath) { signals = JSON.parse(await readFile(args.signalsPath, 'utf-8')); const stack = signals.stack ?? signals.codebase?.stack ?? {}; framework = stack.framework; version = stack.frameworkVersion; cacheComponents = stack.cacheComponents; knownFindings = (signals.codebase?.findings ?? signals.findings ?? []) .filter((f) => f.file && (f.line != null)) .map((f) => ({ file: f.file, line: f.line })); projectFacts = deriveProjectFacts(signals); if (projectFacts.length > 0) { log(`project facts in play: ${projectFacts.map((f) => f.id).join(', ')}`); } } // Repo-root priority: (1) signals.project.rootDirectory from Vercel API // (authoritative — returns "apps/" so cwd can be back-mapped without // filesystem probing), (2) supplied --repo-root, (3) walk-up from cwd. const rootResult = await resolveRepoRoot(recs, args.repoRoot, process.cwd(), signals); const repoRoot = rootResult.root; if (rootResult.source === 'api') { log(`repo-root from Vercel API: '${repoRoot}' (rootDirectory='${rootResult.apiOffset}')`); } else if (rootResult.source === 'auto-detected') { log(`repo-root auto-detected: '${repoRoot}' (probe: ${rootResult.probe})`); } else if (rootResult.source === 'corrected') { log(`repo-root auto-corrected: '${args.repoRoot}' → '${repoRoot}' (sub-agent paths resolve there)`); } log(`verifying ${recs.length} rec(s) — framework=${framework ?? '?'}@${version ?? '?'} repoRoot=${repoRoot}`); // knownFindings MUST combine scanner findings + sub-agent's verified // findingRefs — scanner-only grounding would miss every metric-gate rec. // Abstentions are first-class outputs ({abstain:true, candidateRef, reason}) // and MUST NOT be graded; the abstention IS the answer. const recsGraded = []; const abstentions = []; const observations = []; const sanitizerDropped = []; for (let i = 0; i < recs.length; i++) { const rec = recs[i]; if (rec?.abstain === true) { abstentions.push({ index: i, candidateRef: rec.candidateRef ?? null, reason: rec.reason ?? '(no reason recorded)', }); // Observation: real non-perf signal worth surfacing (regression, error storm). if (rec.observation && typeof rec.observation === 'object' && rec.observation.summary) { observations.push({ index: i, candidateRef: rec.candidateRef ?? null, summary: String(rec.observation.summary), evidence: rec.observation.evidence ?? null, suggestedAction: rec.observation.suggestedAction ?? null, kind: rec.observation.kind ?? 'other', }); } continue; } const baseClaimCtx = { framework, version, repoRoot, projectFacts, projectRootDirectory: signals?.project?.rootDirectory ?? null, cacheComponents, signals, }; const initialClaims = extractClaims(rec, baseClaimCtx); const initialVerifyResults = await Promise.all(initialClaims.map((c) => verifyClaim(c))); const initialClaimsWithResults = initialVerifyResults.map((r, j) => ({ ...r, type: initialClaims[j]?.type, claimType: initialClaims[j]?.type, claim: initialClaims[j], })); const sanitizerResult = await applySanitizers(rec, { framework, version, signals, verifyResults: initialClaimsWithResults, }); if (!sanitizerResult.kept) { sanitizerDropped.push({ index: i, candidateRef: rec.candidateRef ?? null, what: rec.what ?? null, reason: sanitizerResult.dropReason ?? 'automated-check', }); continue; } const sanitizedRec = sanitizerResult.rec; const claims = extractClaims(sanitizedRec, baseClaimCtx); const verifyResults = await Promise.all(claims.map((c) => verifyClaim(c))); const verification = summarizeClaimResults(verifyResults); // A findingRef whose file_exists claim verified counts as grounding evidence. const verifiedRefs = []; for (let j = 0; j < claims.length; j++) { const c = claims[j]; const r = verifyResults[j]; if (r?.disposition !== 'verified') continue; if (c.sourceField === 'findingRefs' && c.type === 'file_exists') { const ref = (rec.findingRefs ?? []).find((x) => String(x).startsWith(c.file + ':')); if (ref) { const m = String(ref).match(/^(.+?):(\d+)$/); if (m) verifiedRefs.push({ file: m[1], line: Number(m[2]) }); } } } const recKnownFindings = [...knownFindings, ...verifiedRefs]; const quality = gradeRecommendation(sanitizedRec, { knownFindings: recKnownFindings }); recsGraded.push({ index: i, rec: { ...sanitizedRec, verification, verifyResults, quality }, claims, verifyResults, verification, quality, }); } // Project-config contradictions are a HARD trigger: a "turn on Fluid" rec on // a project where Fluid is already on passes 8/9 claims but is the wrong rec. // passRate alone won't catch this. const regenPlan = []; for (const g of recsGraded) { const { passRate, verifiable } = g.verification; const claimsWithResults = g.verifyResults.map((r, j) => ({ ...r, claim: g.claims[j] })); const contradictions = claimsWithResults.filter( (r) => r.disposition === 'failed' && r.claim?.type === 'does_not_contradict_project_config' ); const triggeredByPassRate = verifiable >= REGEN_MIN_CLAIMS && passRate < REGEN_PASS_RATE_THRESHOLD; const cacheSafetyFailures = claimsWithResults.filter( (r) => r.disposition === 'failed' && ( r.claim?.type === 'cache_vary_matches_dynamic_inputs' || r.claim?.type === 'cache_vary_cardinality_safe' ) ); const semanticSafetyFailures = claimsWithResults.filter( (r) => r.disposition === 'failed' && ( r.claim?.type === 'next_cached_not_found_causal_support' || r.claim?.type === 'next_stable_cache_api_for_version' || r.claim?.type === 'next_runtime_cache_api_for_version' || r.claim?.type === 'next_cache_life_single_execution' || r.claim?.type === 'next_cache_lifetime_freshness_supported' || r.claim?.type === 'next_cache_components_route_chain_file' || r.claim?.type === 'next_cache_life_cdn_header_semantics' || r.claim?.type === 'image_response_headers_citation' || r.claim?.type === 'next_image_priority_api_for_version' || r.claim?.type === 'next_cache_components_route_segment_config' || r.claim?.type === 'next_route_revalidate_static_prereq' || r.claim?.type === 'next_cache_tag_invalidation_supported' || r.claim?.type === 'cache_rec_not_error_dominated_or_acknowledged' || r.claim?.type === 'cache_control_header_syntax' || r.claim?.type === 'cache_control_headers_citation' || r.claim?.type === 'cache_404_long_ttl_safety' || r.claim?.type === 'route_error_not_found_status_and_scope' || r.claim?.type === 'immutable_dynamic_route_safety' || r.claim?.type === 'auth_guard_parallelization_safety' || r.claim?.type === 'parallelization_impact_not_overclaimed' || r.claim?.type === 'parallelization_not_cpu_bound_work' || r.claim?.type === 'runtime_error_cause_supported' || r.claim?.type === 'vercel_ignore_command_project_state' ) ); const triggeredByContradiction = contradictions.length > 0; const triggeredByCacheSafety = cacheSafetyFailures.length > 0; const triggeredBySemanticSafety = semanticSafetyFailures.length > 0; if (!triggeredByPassRate && !triggeredByContradiction && !triggeredByCacheSafety && !triggeredBySemanticSafety) continue; const failures = claimsWithResults .filter((r) => r.disposition === 'failed') .slice(0, 5); regenPlan.push({ index: g.index, candidateRef: g.rec.candidateRef ?? null, what: g.rec.what ?? null, verifiableClaimCount: verifiable, passRate, regenTrigger: triggeredByContradiction ? 'project_config_contradiction' : triggeredByCacheSafety ? 'cache_vary_safety' : triggeredBySemanticSafety ? 'semantic_safety' : 'pass_rate_below_threshold', topFailures: failures.map((f) => ({ claimType: f.claim?.type, field: f.claim?.sourceField, url: f.claim?.url, file: f.claim?.file, pattern: f.claim?.pattern, reason: f.reason, })), regenBriefHint: triggeredByContradiction ? 'Sub-agent recommended toggling on a project setting that is already enabled. Re-spawn with the project-config Strengths block highlighted; the rec must drop the contradictory step and keep only the actionable parts.' : triggeredByCacheSafety ? 'Sub-agent recommended CDN caching with unsafe or missing Vary behavior. Re-spawn with the cache safety failure highlighted; the rec must use a low-cardinality Vary header that matches the dynamic inputs, or abstain.' : triggeredBySemanticSafety ? 'Sub-agent made a framework-semantic claim that failed deterministic checks. Re-spawn with the failure highlighted; the rec must either add version-correct code/citations/runtime evidence or abstain.' : 'Re-spawn the sub-agent with this rec\'s topFailures injected as feedback. Re-emit the rec only if regenPassRate >= originalPassRate AND citation count not gutted.', }); } const qualityCheck = applyQualityFloor(recsGraded.map((g) => g.rec), QUALITY_FLOOR); const hardRegenIndexes = new Set(regenPlan.map((p) => p.index)); const qualityDroppedIndexes = new Set( qualityCheck.dropped .map((d) => recsGraded.findIndex((g) => g.rec === d.rec)) .filter((i) => i >= 0) ); const needsReviewIndexes = new Set( recsGraded .filter((g) => g.rec.needsReview === true) .map((g) => g.index) ); const verifiedRecommendations = recsGraded .filter((g) => !hardRegenIndexes.has(g.index) && !qualityDroppedIndexes.has(g.index) && !needsReviewIndexes.has(g.index)) .map((g) => g.rec); const withheldRecommendations = recsGraded .filter((g) => hardRegenIndexes.has(g.index) || qualityDroppedIndexes.has(g.index) || needsReviewIndexes.has(g.index)) .map((g) => ({ index: g.index, candidateRef: g.rec.candidateRef ?? null, what: g.rec.what ?? null, reason: hardRegenIndexes.has(g.index) ? (regenPlan.find((p) => p.index === g.index)?.regenTrigger ?? 'verification') : qualityDroppedIndexes.has(g.index) ? 'quality_floor' : 'needs_review', })); const summary = { totalRecs: recs.length, abstentions: abstentions.length, observations: observations.length, sanitizerDropped: sanitizerDropped.length, needsRegen: regenPlan.length, qualityDropped: qualityCheck.dropped.length, needsReview: needsReviewIndexes.size, verifiedRecommendations: verifiedRecommendations.length, withheldRecommendations: withheldRecommendations.length, averagePassRate: recsGraded.length > 0 ? round4(recsGraded.reduce((s, g) => s + g.verification.passRate, 0) / recsGraded.length) : null, averageQuality: recsGraded.length > 0 ? round4(recsGraded.reduce((s, g) => s + g.quality.overall, 0) / recsGraded.length) : null, }; const output = { schemaVersion: SCHEMA_VERSION, summary, recsGraded: recsGraded.map((g) => g.rec), verifiedRecommendations, renderableRecommendations: verifiedRecommendations, withheldRecommendations, abstentions, observations, sanitizerDropped, regenPlan, qualityDropped: qualityCheck.dropped.map((d) => ({ index: recsGraded.findIndex((g) => g.rec === d.rec), candidateRef: d.rec.candidateRef ?? null, quality: d.rec.quality, reason: d.reason, })), }; const serialized = JSON.stringify(output, null, 2) + '\n'; if (args.outPath) { await mkdir(dirname(args.outPath), { recursive: true }); await writeFile(args.outPath, serialized, 'utf-8'); log(`wrote ${serialized.length}B → ${args.outPath}`); } else { process.stdout.write(serialized); } log(`done: ${summary.totalRecs} records checked; ${summary.verifiedRecommendations} ready, ${summary.withheldRecommendations} held back, ${summary.abstentions} found no supported change, ${summary.sanitizerDropped} dropped by safety checks`); } function parseArgs(argv) { const out = { positional: [] }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--signals') out.signalsPath = argv[++i]; else if (a.startsWith('--signals=')) out.signalsPath = a.slice('--signals='.length); else if (a === '--repo-root') out.repoRoot = argv[++i]; else if (a.startsWith('--repo-root=')) out.repoRoot = a.slice('--repo-root='.length); else if (a === '--out') out.outPath = resolve(argv[++i]); else if (a.startsWith('--out=')) out.outPath = resolve(a.slice('--out='.length)); else out.positional.push(a); } out.recsPath = out.positional[0]; return out; } function round4(n) { return Math.round(n * 10000) / 10000; } main().catch((err) => { console.error('[verify-and-regen] FAILED:', err.message); console.error(err.stack); process.exit(1); });