#!/usr/bin/env node "use strict"; const crypto = require("crypto"); const fs = require("fs"); const os = require("os"); const path = require("path"); const { execFile } = require("child_process"); const { promisify } = require("util"); const { atomicWriteJson, canonicalJson } = require("./cache"); const { ensureOutputRoot, resolveOutputPath } = require("./output"); const { validateParityManifest } = require("./parity-benchmark"); const { artifactName, readBoundedRegular } = require("./safe-io"); const { discoverBundle, disposeSnapshot, materializeBundle, sha256, trackedFiles } = require("./safe-bundle"); const execFileAsync = promisify(execFile); const WORKSPACE = "019f1c57-2d90-72cd-b477-f452fd852e62"; const ORG = "019f1c56-d131-702d-8f36-69ae1aa5555c"; const PLUGIN = "tessl/default-skill-review@0.1.0"; const EXPECTED_AGENT = "claude"; const EXPECTED_MODEL = "glm-5.2"; const EXPECTED_CLI_VERSION = "0.91.0"; const REVIEW_CREDIT_COST = 10; const DIMENSIONS = Object.freeze({ description: ["specificity", "trigger_term_quality", "completeness", "distinctiveness_conflict_risk"], content: ["conciseness", "actionability", "workflow_clarity", "progressive_disclosure"], }); const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; function exactKeys(value, keys, label) { if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).sort().join("\0") !== [...keys].sort().join("\0")) throw new Error(`${label} schema is not closed`); } function parseJson(text, label) { if (typeof text !== "string" || Buffer.byteLength(text) > 16 * 1024 * 1024) throw new Error(`${label} JSON exceeds boundary`); try { return JSON.parse(text); } catch { throw new Error(`${label} did not return valid JSON`); } } function readJson(filePath, label, max = 16 * 1024 * 1024) { return JSON.parse(readBoundedRegular(filePath, max, label).toString("utf8")); } function validatePredictions(value, manifest, expectedHash, bytes) { if (!/^[0-9a-f]{64}$/.test(expectedHash || "")) throw new Error("final_blind requires --predictions-sha256"); if (sha256(bytes) !== expectedHash) throw new Error("Pre-reveal predictions hash mismatch"); exactKeys(value, ["schemaVersion", "kind", "manifestSelectionSha256", "split", "predictions"], "Pre-reveal predictions"); if (value.schemaVersion !== 1 || value.kind !== "aas-tessl-parity-pre-reveal-predictions" || value.split !== "final_blind" || value.manifestSelectionSha256 !== manifest.integrity.selectionSha256) throw new Error("Pre-reveal predictions provenance mismatch"); exactKeys(value.predictions, manifest.splits.final_blind.map((item) => item.id), "Pre-reveal prediction IDs"); for (const [id, prediction] of Object.entries(value.predictions)) { exactKeys(prediction, ["score", "description", "content"], `Prediction ${id}`); if (!Number.isInteger(prediction.score) || prediction.score < 0 || prediction.score > 100) throw new Error(`Prediction score invalid: ${id}`); for (const kind of Object.keys(DIMENSIONS)) if (!Array.isArray(prediction[kind]) || prediction[kind].length !== 4 || prediction[kind].some((score) => !Number.isInteger(score) || score < 1 || score > 3)) throw new Error(`Prediction labels invalid: ${id}/${kind}`); } return value; } function validateCredits(raw) { if (!raw || typeof raw !== "object" || Array.isArray(raw) || !raw.data || typeof raw.data !== "object" || !raw.data.attributes || typeof raw.data.attributes !== "object") throw new Error("Credit response schema unsupported"); const credits = raw.data.attributes.credits; exactKeys(credits, ["state", "limit", "used", "remaining", "windowStart", "overLimit", "blocked", "overageAllowed"], "Credits"); if (typeof credits.state !== "string" || !Number.isFinite(credits.limit) || credits.limit < 0 || !Number.isFinite(credits.used) || credits.used < 0 || !Number.isFinite(credits.remaining) || credits.remaining < 0 || typeof credits.windowStart !== "string" || !Number.isFinite(Date.parse(credits.windowStart)) || [credits.overLimit, credits.blocked, credits.overageAllowed].some((item) => typeof item !== "boolean")) throw new Error("Credit values invalid"); return credits; } function extractRunId(raw) { const id = raw?.reviewRunId || raw?.data?.id; if (!UUID.test(id || "")) throw new Error(`Submission JSON has unsupported keys: ${Object.keys(raw || {}).sort().join(",")}`); return id; } function statusOf(raw) { return raw?.status || raw?.attributes?.status || raw?.review?.status || raw?.data?.attributes?.status || null; } function metadataEntry(raw, runId) { const rows = Array.isArray(raw) ? raw : Array.isArray(raw?.data) ? raw.data : null; if (!rows) throw new Error("Review list JSON schema unsupported"); const entry = rows.find((item) => (item?.reviewRunId || item?.id || item?.data?.id) === runId); if (!entry) throw new Error(`Review list is missing run ${runId}`); return entry; } function firstDefined(...values) { return values.find((value) => value !== undefined); } function verifyMetadata(entry, { runId, label }) { const attrs = entry?.attributes || entry?.data?.attributes || entry; const config = attrs?.config || attrs?.reviewConfig || {}; const metadata = attrs?.metadata || {}; const reusedFrom = firstDefined(attrs?.reusedFromReviewRunId, metadata?.reusedFromReviewRunId); const observed = { runId: firstDefined(entry?.reviewRunId, entry?.id, entry?.data?.id), label: firstDefined(attrs?.label, metadata?.label, metadata?.pathLabel), status: firstDefined(attrs?.status, metadata?.status), agent: firstDefined(config?.agent, attrs?.agent, metadata?.agent), model: firstDefined(config?.model, attrs?.model, metadata?.model), plugin: firstDefined(config?.reviewPlugin, config?.["review-plugin"], config?.pluginRef, config?.reviewPluginName, attrs?.reviewPlugin, attrs?.["review-plugin"], metadata?.reviewPlugin), reusedFrom: reusedFrom === undefined ? null : reusedFrom, }; if (observed.runId !== runId || observed.label !== label || observed.status !== "completed" || observed.agent !== EXPECTED_AGENT || observed.model !== EXPECTED_MODEL || observed.plugin !== PLUGIN || observed.reusedFrom !== null) throw new Error(`Review metadata mismatch: ${canonicalJson(observed)}`); return observed; } function normalize(raw, context, observed, listEntry) { const review = raw?.review || raw?.data?.attributes?.review; const judges = raw?.judges || review?.judges; const validation = raw?.validation || review?.validation; const reviewRunId = raw?.reviewRunId || raw?.data?.id; if (reviewRunId !== context.runId || raw?.["review-plugin"] !== PLUGIN || !review || !judges || !validation) throw new Error("Completed review JSON schema unsupported"); const score = review.reviewScore; if (!Number.isInteger(score) || score < 0 || score > 100) throw new Error("Review score invalid"); const labels = {}; for (const [kind, dimensions] of Object.entries(DIMENSIONS)) { const judge = judges[kind]; if (!judge || judge.success !== true || !judge.evaluation?.scores) throw new Error(`Judge missing or failed: ${kind}`); exactKeys(judge.evaluation.scores, dimensions, `${kind} dimension scores`); labels[kind] = dimensions.map((dimension) => { const item = judge.evaluation.scores[dimension]; if (!item || !Number.isInteger(item.score) || item.score < 1 || item.score > 3) throw new Error(`Dimension score invalid: ${kind}/${dimension}`); return item.score; }); } const warnings = Array.isArray(validation.warnings) ? validation.warnings.length : firstDefined(validation.warningCount, validation.warnings, 0); const errors = Array.isArray(validation.errors) ? validation.errors.length : firstDefined(validation.errorCount, validation.errors, 0); const totalChecks = validation.checks?.length; if (!Number.isInteger(warnings) || warnings < 0 || !Number.isInteger(errors) || errors < 0 || !Number.isInteger(totalChecks) || totalChecks < 1 || warnings + errors > totalChecks) throw new Error("Validation counts invalid"); const attrs = listEntry?.attributes || listEntry?.data?.attributes || listEntry; const component = attrs?.results?.scoring?.components?.find((item) => item?.id === "validation"); const validationNormalized = (totalChecks - errors - (0.5 * warnings)) / totalChecks; if (typeof component?.normalized !== "number" || Math.abs(component.normalized - validationNormalized) > 1e-12) throw new Error("Validation normalized score mismatch"); return { schemaVersion: 1, kind: "aas-tessl-parity-labels", split: context.split, skillId: context.skill.id, bundleHash: context.skill.bundleHash, repeat: context.repeat, reviewRunId, label: context.label, score, validationNormalized, warnings, errors, totalChecks, description: labels.description, content: labels.content, agent: observed.agent, model: observed.model, reviewPlugin: observed.plugin, reusedFrom: observed.reusedFrom, }; } function validateNormalized(value, context) { exactKeys(value, ["schemaVersion", "kind", "split", "skillId", "bundleHash", "repeat", "reviewRunId", "label", "score", "validationNormalized", "warnings", "errors", "totalChecks", "description", "content", "agent", "model", "reviewPlugin", "reusedFrom"], "Normalized labels"); if (value.schemaVersion !== 1 || value.kind !== "aas-tessl-parity-labels" || value.split !== context.split || value.skillId !== context.skill.id || value.bundleHash !== context.skill.bundleHash || value.repeat !== context.repeat || value.label !== context.label || !UUID.test(value.reviewRunId) || value.agent !== EXPECTED_AGENT || value.model !== EXPECTED_MODEL || value.reviewPlugin !== PLUGIN || value.reusedFrom !== null) throw new Error("Normalized label binding mismatch"); if (!Number.isInteger(value.score) || value.score < 0 || value.score > 100 || typeof value.validationNormalized !== "number" || value.validationNormalized < 0 || value.validationNormalized > 1 || !Number.isInteger(value.warnings) || value.warnings < 0 || !Number.isInteger(value.errors) || value.errors < 0 || !Number.isInteger(value.totalChecks) || value.totalChecks < 1 || value.warnings + value.errors > value.totalChecks || Math.abs(value.validationNormalized - ((value.totalChecks - value.errors - (0.5 * value.warnings)) / value.totalChecks)) > 1e-12) throw new Error("Normalized scalar invalid"); for (const kind of Object.keys(DIMENSIONS)) if (!Array.isArray(value[kind]) || value[kind].length !== 4 || value[kind].some((score) => !Number.isInteger(score) || score < 1 || score > 3)) throw new Error("Normalized dimensions invalid"); return value; } async function defaultCommand(tesslPath, args) { const result = await execFileAsync(tesslPath, args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: 10 * 60 * 1000 }); return parseJson(result.stdout, `tessl ${args.slice(0, 3).join(" ")}`); } async function defaultVersionCommand(tesslPath) { const result = await execFileAsync(tesslPath, ["--version"], { encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 60 * 1000 }); return result.stdout.trim(); } function makeLabel(split, id, repeat) { return repeat === 0 ? `aas-parity-v1:${split}:${id}` : `aas-parity-v1:${split}:${id}:repeat-${repeat}`; } function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function collectParity({ repoRoot, manifestPath, split, resultDir, predictionsPath, predictionsSha256, adoptRuns = {}, concurrency = 2, stabilityRepeats = 0, pollMs = 3000, tesslPath = "tessl", command = defaultCommand, versionCommand = defaultVersionCommand }) { if (!repoRoot || !manifestPath || !resultDir) throw new Error("repoRoot, manifestPath, and resultDir are required"); if (!Object.hasOwn({ validation: true, final_blind: true }, split)) throw new Error("Split must be validation or final_blind"); if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 4 || !Number.isInteger(stabilityRepeats) || stabilityRepeats < 0 || stabilityRepeats > 3 || !Number.isInteger(pollMs) || pollMs < 0 || pollMs > 60000) throw new Error("Collector limits invalid"); if (split === "final_blind" && stabilityRepeats !== 0) throw new Error("Stability repeats are validation-only"); const manifest = validateParityManifest(readJson(path.resolve(manifestPath), "Parity manifest")); if (split === "final_blind") { if (!predictionsPath) throw new Error("final_blind requires --predictions-file"); const bytes = readBoundedRegular(path.resolve(predictionsPath), 1024 * 1024, "Pre-reveal predictions"); validatePredictions(JSON.parse(bytes.toString("utf8")), manifest, predictionsSha256, bytes); } const tracked = trackedFiles(repoRoot); const selected = manifest.splits[split]; if (!adoptRuns || typeof adoptRuns !== "object" || Array.isArray(adoptRuns)) throw new Error("Adopted runs must be a skill-to-run map"); const selectedIds = new Set(selected.map((item) => item.id)); for (const [skillId, runId] of Object.entries(adoptRuns)) if (!selectedIds.has(skillId) || !UUID.test(runId || "")) throw new Error(`Adopted run is outside the selected split or invalid: ${skillId}`); for (const item of selected) { const bundle = discoverBundle(repoRoot, item.id, tracked); if (bundle.bundleHash !== item.bundleHash || bundle.files[0].sha256 !== item.skillSha256) throw new Error(`Frozen bundle hash mismatch: ${item.id}`); } const outputRoot = ensureOutputRoot(path.resolve(resultDir), repoRoot); const calls = (args) => command(tesslPath, args); const credits = () => calls(["api", `/v1/orgs/${ORG}/credits`]).then(validateCredits); let exhausted = false; async function one(skill, repeat) { const label = makeLabel(split, skill.id, repeat); const context = { split, skill, repeat, label }; const suffix = repeat === 0 ? "unique" : `repeat-${repeat}`; const normalizedPath = `normalized/${artifactName(skill.id)}.${suffix}.json`; const submissionPath = `submissions/${artifactName(skill.id)}.${suffix}.json`; const intentPath = `submissions/${artifactName(skill.id)}.${suffix}.intent.json`; const rawPath = `raw/${artifactName(skill.id)}.${suffix}.json`; try { const existing = readJson(resolveOutputPath(outputRoot, normalizedPath), "Existing normalized labels", 1024 * 1024); return validateNormalized(existing, context); } catch (error) { if (fs.existsSync(resolveOutputPath(outputRoot, normalizedPath))) throw error; } try { const completed = readJson(resolveOutputPath(outputRoot, rawPath), "Completed raw review"); exactKeys(completed, ["origin", "submission", "view", "listEntry"], "Completed raw review"); if (!["submitted", "adopted"].includes(completed.origin) || (completed.origin === "submitted" && !completed.submission) || (completed.origin === "adopted" && completed.submission !== null)) throw new Error("Completed raw review origin invalid"); const runId = extractRunId(completed.view); const observed = verifyMetadata(completed.listEntry, { runId, label }); const normalized = normalize(completed.view, { ...context, runId }, observed, completed.listEntry); atomicWriteJson(outputRoot, normalizedPath, normalized); return normalized; } catch (error) { if (fs.existsSync(resolveOutputPath(outputRoot, rawPath))) throw error; } const adoptedRunId = repeat === 0 ? adoptRuns[skill.id] : undefined; if (adoptedRunId) { const list = await calls(["review", "list", "--json", "--limit", "100", "--workspace", WORKSPACE]); const entry = metadataEntry(list, adoptedRunId); if (statusOf(entry) !== "completed") throw new Error(`Adopted review is not completed: ${skill.id}`); const view = await calls(["review", "view", "--json", adoptedRunId]); atomicWriteJson(outputRoot, rawPath, { origin: "adopted", submission: null, view, listEntry: entry }); const observed = verifyMetadata(entry, { runId: adoptedRunId, label }); const normalized = normalize(view, { ...context, runId: adoptedRunId }, observed, entry); atomicWriteJson(outputRoot, normalizedPath, normalized); return normalized; } const bundle = discoverBundle(repoRoot, skill.id, tracked); let submitted; try { const raw = readJson(resolveOutputPath(outputRoot, submissionPath), "Stored submission", 1024 * 1024); submitted = { raw, runId: extractRunId(raw) }; } catch (error) { if (fs.existsSync(resolveOutputPath(outputRoot, submissionPath))) throw error; if (fs.existsSync(resolveOutputPath(outputRoot, intentPath))) throw new Error(`Ambiguous prior submission intent; refusing paid retry: ${skill.id}/${suffix}`); let snapshot; try { snapshot = materializeBundle(bundle); submitted = await (async () => { atomicWriteJson(outputRoot, intentPath, { schemaVersion: 1, skillId: skill.id, bundleHash: skill.bundleHash, split, repeat, label }); // The free plan rejects an explicit --review-plugin as a custom-plugin // request. Tessl's implicit quality-review default is verified against // PLUGIN in the completed run metadata before labels are accepted. const raw = await calls(["review", "run", "quality", path.join(snapshot.root, ...skill.skillPath.split("/").slice(0, -1)), "--json", "--workspace", WORKSPACE, "--threshold", "1", "--force", "--label", label]); atomicWriteJson(outputRoot, submissionPath, raw); return { raw, runId: extractRunId(raw) }; })(); } finally { if (snapshot) disposeSnapshot(snapshot); } } let entry; for (;;) { const list = await calls(["review", "list", "--json", "--limit", "100", "--workspace", WORKSPACE]); entry = metadataEntry(list, submitted.runId); const status = statusOf(entry); if (status === "completed") break; if (status === "failed" || status === "cancelled") throw new Error(`Tessl review ended with ${status}`); if (status !== "pending" && status !== "in_progress" && status !== "running") throw new Error(`Unsupported Tessl review status: ${status}`); await sleep(pollMs); } const view = submitted.raw?.judges && submitted.raw?.review ? submitted.raw : await calls(["review", "view", "--json", submitted.runId]); atomicWriteJson(outputRoot, rawPath, { origin: "submitted", submission: submitted.raw, view, listEntry: entry }); const observed = verifyMetadata(entry, { runId: submitted.runId, label }); const normalized = normalize(view, { ...context, runId: submitted.runId }, observed, entry); atomicWriteJson(outputRoot, normalizedPath, normalized); return normalized; } function requiresNewSubmission({ skill, repeat }) { const suffix = repeat === 0 ? "unique" : `repeat-${repeat}`; const stem = artifactName(skill.id); if (fs.existsSync(resolveOutputPath(outputRoot, `normalized/${stem}.${suffix}.json`)) || fs.existsSync(resolveOutputPath(outputRoot, `raw/${stem}.${suffix}.json`)) || fs.existsSync(resolveOutputPath(outputRoot, `submissions/${stem}.${suffix}.json`))) return false; if (repeat === 0 && adoptRuns[skill.id]) return false; if (fs.existsSync(resolveOutputPath(outputRoot, `submissions/${stem}.${suffix}.intent.json`))) throw new Error(`Ambiguous prior submission intent; refusing paid retry: ${skill.id}/${suffix}`); return true; } async function wave(items) { const newSubmissions = items.filter(requiresNewSubmission).length; let before; if (newSubmissions) { before = await credits(); if (before.blocked || before.overLimit || before.remaining < newSubmissions * REVIEW_CREDIT_COST) throw new Error(`Insufficient Tessl credits for wave: need ${newSubmissions * REVIEW_CREDIT_COST}, have ${before.remaining}`); } const settled = await Promise.allSettled(items.map((item) => one(item.skill, item.repeat))); if (newSubmissions) { const after = await credits(); atomicWriteJson(outputRoot, `credits/wave-${Date.now()}-${crypto.randomBytes(4).toString("hex")}.json`, { newSubmissions, expectedCost: newSubmissions * REVIEW_CREDIT_COST, before, after }); const delta = before.remaining - after.remaining; if (delta !== newSubmissions * REVIEW_CREDIT_COST) throw new Error(`Tessl credit cost mismatch: expected ${newSubmissions * REVIEW_CREDIT_COST}, observed ${delta}`); if (after.remaining <= 0 || after.blocked || after.overLimit) exhausted = true; } const failed = settled.find((item) => item.status === "rejected"); if (failed) throw failed.reason; return settled.map((item) => item.value); } async function batch(items) { const results = []; for (let index = 0; index < items.length; index += concurrency) results.push(...await wave(items.slice(index, index + concurrency))); return results; } const uniquePlan = selected.map((skill) => ({ skill, repeat: 0 })); const byId = new Map(selected.map((skill) => [skill.id, skill])); const repeatPlan = []; if (split === "validation" && stabilityRepeats) for (let repeat = 1; repeat <= stabilityRepeats; repeat += 1) repeatPlan.push(...manifest.stabilityPanel.map((id) => ({ skill: byId.get(id), repeat }))); const initiallyMissing = [...uniquePlan, ...repeatPlan].filter(requiresNewSubmission).length; if (initiallyMissing) { const cliVersion = await versionCommand(tesslPath); if (cliVersion !== EXPECTED_CLI_VERSION) throw new Error(`Tessl CLI version mismatch: expected ${EXPECTED_CLI_VERSION}, observed ${cliVersion || ""}`); const preflight = await credits(); const required = initiallyMissing * REVIEW_CREDIT_COST; if (preflight.blocked || preflight.overLimit || preflight.remaining < required) throw new Error(`Insufficient Tessl credits for batch: need ${required}, have ${preflight.remaining}`); atomicWriteJson(outputRoot, "credits/preflight.json", { missing: initiallyMissing, unitCost: REVIEW_CREDIT_COST, required, credits: preflight }); } const unique = await batch(uniquePlan); const repeats = []; if (repeatPlan.length) repeats.push(...await batch(repeatPlan)); const goldItems = unique.slice().sort((a, b) => a.skillId.localeCompare(b.skillId)).map((item) => ({ skillId: item.skillId, reviewRunId: item.reviewRunId, score: item.score, validation: { normalized: item.validationNormalized, warnings: item.warnings, errors: item.errors, totalChecks: item.totalChecks }, description: Object.fromEntries(DIMENSIONS.description.map((id, index) => [id, item.description[index]])), content: Object.fromEntries(DIMENSIONS.content.map((id, index) => [id, item.content[index]])), })); const gold = { schemaVersion: 1, kind: "aas-tessl-parity-gold", split, oracle: { plugin: PLUGIN, agent: EXPECTED_AGENT, model: EXPECTED_MODEL }, items: goldItems }; atomicWriteJson(outputRoot, "gold.json", gold); const summary = { schemaVersion: 1, kind: "aas-tessl-parity-collection", split, unique: unique.length, repeats: repeats.length, exhausted, results: [...unique, ...repeats].map((item) => ({ skillId: item.skillId, repeat: item.repeat, reviewRunId: item.reviewRunId, score: item.score })) }; atomicWriteJson(outputRoot, "summary.json", summary); return summary; } function option(args, name, fallback) { const index = args.indexOf(name); if (index < 0) return fallback; if (!args[index + 1] || args[index + 1].startsWith("--")) throw new Error(`${name} requires a value`); return args[index + 1]; } function mappingOptions(args, name) { const result = {}; for (let index = 0; index < args.length; index += 1) { if (args[index] !== name) continue; const value = args[index + 1]; if (!value || value.startsWith("--") || value.indexOf("=") < 1) throw new Error(`${name} requires skillId=runId`); const separator = value.indexOf("="); const skillId = value.slice(0, separator); const runId = value.slice(separator + 1); if (Object.hasOwn(result, skillId)) throw new Error(`Duplicate ${name} skill: ${skillId}`); result[skillId] = runId; } return result; } async function main() { const args = process.argv.slice(2); const repoRoot = path.resolve(__dirname, "../.."); const value = await collectParity({ repoRoot, manifestPath: option(args, "--manifest", path.join(repoRoot, "tools/config/local-skill-review-parity-benchmark.json")), split: option(args, "--split"), resultDir: option(args, "--result-dir", path.join(os.tmpdir(), "aas-tessl-parity")), predictionsPath: option(args, "--predictions-file"), predictionsSha256: option(args, "--predictions-sha256"), adoptRuns: mappingOptions(args, "--adopt-run"), concurrency: Number(option(args, "--concurrency", "2")), stabilityRepeats: Number(option(args, "--stability-repeats", "0")), pollMs: Number(option(args, "--poll-ms", "3000")), tesslPath: option(args, "--tessl-path", "tessl"), }); process.stdout.write(`${JSON.stringify(value, null, 2)}\n`); } if (require.main === module) main().catch((error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }); module.exports = { DIMENSIONS, EXPECTED_CLI_VERSION, ORG, PLUGIN, REVIEW_CREDIT_COST, WORKSPACE, collectParity, defaultVersionCommand, extractRunId, makeLabel, mappingOptions, normalize, validateCredits, validateNormalized, validatePredictions, verifyMetadata };