paid-report.ts
826 lines · scripts/lib/paid-report.ts at 3f79d81
1/**
2 * Paid-lane local diagnosis and the report: JUnit parsing, panel verdicts, history records and the human readout. Moved from scripts/test-paid-shards.ts.
3 */
4import * as fs from 'node:fs';
5import * as path from 'node:path';
6import { normalizeRelativePath } from './shard-engine';
7import { EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
8import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
9import {
10 isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
11 type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
12 trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
13} from '../../test/helpers/eval-store';
14import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
15import { publishedFence, sanitizeFixedFenceLine } from './published-text';
16import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
17import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
18import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
19import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
20import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts';
21import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
22import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, writePaidTestDurations } from './paid-plan';
23import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
24import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
25import type { RunShardsOptions } from '../test-paid-shards';
26import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map';
27import { checkoutExercisedConsumers } from '../pr-dependencies';
28import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile';
29import { FLAKE_RECORDS_FILE, flakeRecords, flakyShards, formatFlakyLine, gateProblems, rerunSpend, rerunState } from './paid-rerun';
31// ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
33/** How --case runs exactly one case: trials selected by its Bun test name, or its whole file when that file registers no other case. */
34export interface CaseSelection { file: string; mode: 'name' | 'file'; reason: string }
36const escapeRe = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
38/**
39 * The paid file that registers `id` and how a trial selects it. A file whose
40 * registration is complete wins over one that also lists the id; a test
41 * registered literally under the id (or its CASE_TEST_NAMES label) is
42 * selected by name; a loop that registers ids at runtime is selected by the
43 * id it names its test; a file registering only this case runs whole (its
44 * test names are not the id). Anything else throws before any process starts.
45 */
46export function caseSelection(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): CaseSelection {
47 const files = discovered.map(file => {
48 const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
49 return { file, source, ...fileCaseRegistration(file, source) };
50 }).filter(f => f.registered.includes(id));
51 const owners = files.filter(f => f.known).length ? files.filter(f => f.known) : files;
52 if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.map(f => f.file).join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`);
53 const owner = owners[0]!;
54 const name = escapeRe(CASE_TEST_NAMES[id] ?? id);
55 if (owner.registered.length === 1) return { file: owner.file, mode: 'file', reason: 'the whole file (it registers no other case)' };
56 if (new RegExp(`\\b(?:test|it|test\\.\\w+|test(?:Concurrent)?IfSelected)\\s*\\(\\s*(['"\`])${name}\\1|testName\\s*:\\s*(['"\`])${name}\\2`).test(owner.source)) {
57 return { file: owner.file, mode: 'name', reason: `its Bun test is named ${CASE_TEST_NAMES[id] ?? id}` };
58 }
59 if (owner.computed) return { file: owner.file, mode: 'name', reason: `its loop names the Bun test ${id} at runtime` };
60 throw new Error(`--case ${id}: ${owner.file} registers several cases and none of its tests is named ${id}; add its test name to CASE_TEST_NAMES`);
61}
63/** The one paid file that registers `id`, else a thrown reason. */
64export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): string {
65 return caseSelection(id, rootDir, discovered).file;
66}
68/**
69 * Run `trials` independent trials of one case exactly as CI runs a panel
70 * (trial shards, TRIAL_ENV identity, name-pattern isolation), then print its
71 * panelVerdict(). `trials` defaults to the case's policy panel; CI never
72 * reads it. k keeps the kind's meaning (all trials for rule, the policy
73 * majority for behavior, capped at n).
74 */
75export async function runCaseDiagnosis(id: string, options: {
76 trials?: number; jobs?: number; withinShardConcurrency?: number; timeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv;
77 evalDirBase?: string; commandFor?: RunShardsOptions['commandFor']; log?: (line: string) => void; file?: string;
78 /** The CLI passes its own runner: importing the entry module during its top-level await never resolves. */
79 runShards?: typeof import('../test-paid-shards').runPaidShards;
80} = {}): Promise<PanelVerdict> {
81 const rootDir = options.rootDir ?? ROOT;
82 const log = options.log ?? ((line: string) => console.log(line));
83 const selection = options.file ? { file: options.file, mode: 'name' as const } : caseSelection(id, rootDir);
84 const file = selection.file;
85 const policy = caseTrialPlan(id);
86 const n = options.trials ?? policy.panel.n;
87 const plan: CaseTrialPlan = { ...policy, panel: { n, k: policy.panel.k === policy.panel.n ? n : Math.min(policy.panel.k, n) } };
88 const keys = Array.from({ length: n }, (_, i) => trialShardKey(file, id, i + 1));
89 const tier = E2E_TIERS[id] as PaidTier;
90 log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`);
91 // The runner is the CLI module; load it lazily so this library never imports it statically (no cycle).
92 const runPaidShards = options.runShards ?? (await import('../test-paid-shards')).runPaidShards;
93 const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor,
94 env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1',
95 ...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } };
96 if (selection.mode === 'file') {
97 // The whole file is the case: one file shard per trial, each with its own eval dir, judged by shard status as CI does.
98 const fileTrials = [];
99 for (let trial = 1; trial <= n; trial++) {
100 const [outcome] = (await runPaidShards([[file]], { ...shardOptions, jobs: 1,
101 ...(options.evalDirBase ? { evalDirBase: path.join(options.evalDirBase, `t${trial}`) } : {}) })).outcomes;
102 if (!outcome || outcome.runnerError !== undefined || isAllSkippedPass(outcome) || !['passed', 'failed', 'timed-out'].includes(outcome.status)) continue;
103 fileTrials.push({ trial, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const,
104 ...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) });
105 }
106 const verdict = panelVerdict({ case: id, kind: plan.kind, panel: plan.panel, trials: fileTrials, quarantined: plan.quarantined });
107 log(formatPanelLine({ ...verdict, file, slices: {} }, tier));
108 return verdict;
109 }
110 const summary = await runPaidShards(keys.map(key => [key]), { ...shardOptions,
111 jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), evalDirBase: options.evalDirBase,
112 trials: Object.fromEntries(keys.map(key => [key, plan])) });
113 const trials = summary.outcomes.flatMap(outcome => outcome.trial && outcome.trial.outcome !== null ? [{
114 trial: outcome.trial.trial, outcome: outcome.trial.outcome, ...(outcome.trial.failure_class ? { failure_class: outcome.trial.failure_class } : {}),
115 ...(outcome.trial.exit_reason ? { exit_reason: outcome.trial.exit_reason } : {}), ...(outcome.trial.error ? { error: outcome.trial.error } : {}),
116 ...(outcome.trial.timeout_at_turn !== undefined ? { timeout_at_turn: outcome.trial.timeout_at_turn } : {}) }] : []);
117 const verdict = panelVerdict({ case: id, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined });
118 log(formatPanelLine({ ...verdict, file, slices: {} }, tier));
119 for (const outcome of summary.outcomes.filter(o => o.trial?.outcome === null)) log(` t${outcome.trial!.trial}: no trial record (${outcome.trial!.harness})`);
120 return verdict;
121}
123// ─── Report: verdicts, history records and the human readout ───────────────
125/** One Bun JUnit testcase (`--reporter=junit`); `line` is absent on Bun's hook placeholders. */
126export interface JUnitCase { name: string; classname: string; outcome: TrialOutcome; timeMs: number; line?: number; failureType?: string; message?: string }
128const xmlUnescape = (text: string) => text.replace(/&(lt|gt|quot|apos|amp|#(\d+)|#x([0-9a-f]+));/gi, (_, name: string, dec?: string, hex?: string) =>
129 dec ? String.fromCodePoint(Number(dec)) : hex ? String.fromCodePoint(parseInt(hex, 16))
130 : ({ lt: '<', gt: '>', quot: '"', apos: "'", amp: '&' } as Record<string, string>)[name.toLowerCase()]!);
132function xmlAttributes(tag: string): Record<string, string> {
133 return Object.fromEntries([...tag.matchAll(/([\w:-]+)="([^"]*)"/g)].map(match => [match[1]!, xmlUnescape(match[2]!)]));
134}
136/** Per-test outcomes from a Bun JUnit report; unparseable input yields []. */
137export function parseJUnitCases(xml: string): JUnitCase[] {
138 const cases: JUnitCase[] = [];
139 for (const match of xml.matchAll(/<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) {
140 const attrs = xmlAttributes(match[1]!);
141 const body = match[3] ?? '';
142 const failure = /<(failure|error)\b([^>]*?)(?:\/>|>)/.exec(body);
143 const failureAttrs = failure ? xmlAttributes(failure[2]!) : {};
144 cases.push({
145 name: attrs.name ?? '', classname: attrs.classname ?? '',
146 outcome: failure ? 'failed' : /<skipped\b/.test(body) ? 'skipped' : 'passed',
147 timeMs: Math.round(Number(attrs.time ?? 0) * 1000) || 0,
148 ...(attrs.line !== undefined && Number.isSafeInteger(Number(attrs.line)) ? { line: Number(attrs.line) } : {}),
149 ...(failure ? { failureType: failureAttrs.type ?? failure[1]!, message: failureAttrs.message } : {}),
150 });
151 }
152 return cases;
153}
155/** Registry id of a Bun test name: the id itself or its CASE_TEST_NAMES label, else null (unattributed). */
156export function caseIdForTestName(name: string): string | null {
157 if (Object.hasOwn(E2E_TIERS, name) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, name)) return name;
158 return Object.keys(CASE_TEST_NAMES).find(id => CASE_TEST_NAMES[id] === name) ?? null;
159}
161/** Where one JUnit testcase lands in the census: a case result, a zero-credit deselection with its reason, or unattributed. */
162export type JUnitAttribution =
163 | { kind: 'case'; id: string }
164 | { kind: 'deselected'; reason: string }
165 | { kind: 'unattributed' };
167export interface JUnitShardContext {
168 /** Shard key: `<file>` or `<file>#<case id>`. */
169 key: string;
170 tier: PaidTier;
171 /** The run's case selection; a null list selects every case of the tier. */
172 selection?: PaidCaseSelection;
173 /** Ids this file shard leaves to their isolated trial shards. */
174 excludeCases?: readonly string[];
175 /** The run's agent host (manifest.host); absent means claude. */
176 host?: EvalHost;
177}
179export const DESELECTION_REASONS = {
180 hook: 'Bun hook placeholder of a skipped describe block (not a test)',
181 sibling: 'sibling case of a case shard (its own shard runs it)',
182 isolated: 'isolated case (its trial shards run it)',
183 selection: "outside the run's case selection",
184} as const;
185export const RUNTIME_SKIP_REASON = 'selected in this lane but skipped at runtime by the test itself (test.skip, describe.skip or an unmet prerequisite)';
187/**
188 * Attribute one testcase of a non-trial shard. A case shard's own case is
189 * its id or label; elsewhere a name is its registry id or CASE_TEST_NAMES
190 * label, else the only id the file registers, else the file itself when it
191 * registers none (one keyless case). A skipped testcase of another tier, a
192 * case-shard sibling, an isolated or unselected case, or a Bun hook
193 * placeholder (`(unnamed)` with no line) is deselected: zero credit, never
194 * SKIPPED. Anything else is unattributed and reported by name.
195 */
196export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext,
197 touchfiles: Record<string, string[]> = E2E_TOUCHFILES, tiers: Record<string, string> = E2E_TIERS): JUnitAttribution {
198 if (tc.outcome === 'skipped' && tc.name === '(unnamed)' && tc.line === undefined) return { kind: 'deselected', reason: DESELECTION_REASONS.hook };
199 const file = shardFile(ctx.key);
200 const shardCase = shardCaseId(ctx.key);
201 const named = caseIdForTestName(tc.name);
202 if (shardCase !== null && named !== shardCase) {
203 return tc.outcome === 'skipped' ? { kind: 'deselected', reason: DESELECTION_REASONS.sibling } : { kind: 'unattributed' };
204 }
205 const registered = Object.keys(touchfiles).filter(id => touchfiles[id]!.includes(file));
206 const id = named ?? (registered.length === 1 ? registered[0]! : registered.length === 0 ? file : null);
207 if (id === null) return { kind: 'unattributed' };
208 if (tc.outcome !== 'skipped') return { kind: 'case', id };
209 const tier = tiers[id];
210 if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` };
211 if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated };
212 const hostReason = hostSkipReason(id, ctx.host ?? 'claude');
213 if (hostReason) return { kind: 'deselected', reason: hostReason };
214 const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null;
215 if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection };
216 return { kind: 'case', id };
217}
219export interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; key: string; slice: number; failureClass?: TrialFailureClass; error?: string; costUsd: number; costKnown: boolean; line?: string }
221/** JUnit evidence of the non-trial shards of one attempt. */
222export interface JUnitCensus {
223 /** One result per attributed case per shard. */
224 ruleCases: RuleCase[];
225 history: TrialOutcomeRecord[];
226 failedShards: Set<string>;
227 /** One line per selected in-lane case that skipped, with its reason. */
228 skipped: string[];
229 /** Zero-credit deselections: reason -> testcase count. */
230 deselected: Map<string, number>;
231 /** Per shard key: skip reason -> skipped testcase count. */
232 shardSkipReasons: Map<string, Map<string, number>>;
233 /** `<shard key> :: <test name> (<outcome>)` for every testcase no rule attributes. */
234 unattributed: string[];
235}
237const bump = (counts: Map<string, number>, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1);
239/** Ledger rows of one JUnit case: its own id or test name, else the shard's only case. */
240function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): SessionLedgerRow[] {
241 return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
242}
244/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
245/** The red-census guide the report and every eval:pass-rates view link. */
246export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
248export function sliceArtifactName(root: string, slice: number, attempt: number): string {
249 const name = path.basename(root);
250 return /^(?:paid-slice|paid-rerun|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
251}
253export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
254 base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
255 const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
256 const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry]));
257 for (const { root, result } of artifacts) {
258 for (const outcome of result.outcomes) {
259 const key = normalizeRelativePath(outcome.files[0] ?? '');
260 if (outcome.trial || outcome.files.length !== 1) continue;
261 let xml = '';
262 try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; }
263 const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases, host: manifest.host };
264 const byCase = new Map<string, JUnitCase[]>();
265 const reasons = new Map<string, number>();
266 for (const tc of parseJUnitCases(xml)) {
267 const attributed = attributeJUnitCase(tc, ctx);
268 if (attributed.kind === 'case') byCase.set(attributed.id, [...(byCase.get(attributed.id) ?? []), tc]);
269 else if (attributed.kind === 'deselected') { bump(census.deselected, attributed.reason); bump(reasons, attributed.reason); }
270 else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`);
271 }
272 const records = recordsByShard.get(key) ?? [];
273 const ledger = readSessionLedger(path.join(root, 'shards', shardSlug([key])));
274 for (const [id, cases] of byCase) {
275 const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind;
276 const failed = cases.find(tc => tc.outcome === 'failed');
277 const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped';
278 // A record is named by its case id or carries case_id; an E2E record named by its display label ("/qa fix loop") is
279 // the shard's only case's when the shard ran one case, else it stays unattributed and the trial's cost unknown.
280 const mine = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
281 const failedRecord = mine.find((r: any) => r.passed === false);
282 const recorded: TrialFailureClass | undefined = !failed ? undefined
283 : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
284 const sessions = caseSessions(ledger, id, byCase.size === 1);
285 const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
286 const failureClass = credentials ? 'infra' : recorded;
287 const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
288 const error = sanitizeTrialError(rawError);
289 // A case's JUnit times sum its concurrent tests (plan-mode-no-op: 346 s in a 176 s shard), so its wall is capped by the shard's.
290 const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
291 error: rawError, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
292 if (failed) census.failedShards.add(key);
293 if (caseOutcome === 'skipped') {
294 bump(reasons, RUNTIME_SKIP_REASON);
295 census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
296 }
297 census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}),
298 costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false,
299 ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
300 + `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
301 + `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
302 + ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(key))}` } : {}) });
303 census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
304 ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
305 ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
306 cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions),
307 ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
308 quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
309 }
310 census.shardSkipReasons.set(key, reasons);
311 }
312 }
313 return census;
314}
316/** The census block of the readout: every skipped case with its reason, deselections by reason, every unattributed testcase by name. */
317export function formatJUnitCensus(census: JUnitCensus): string[] {
318 const deselected = [...census.deselected.entries()].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1));
319 return [
320 ...(census.skipped.length ? [`SKIPPED cases (${census.skipped.length}):`, ...census.skipped.map(line => ` ${line}`)] : []),
321 ...(deselected.length ? [`deselected testcases, zero credit (${deselected.reduce((sum, [, n]) => sum + n, 0)}):`,
322 ...deselected.map(([reason, n]) => ` ${String(n).padStart(4)} ${reason}`)] : []),
323 ...(census.unattributed.length ? [`UNATTRIBUTED testcases (${census.unattributed.length}; no registry id, label, single registered id or keyless file):`,
324 ...census.unattributed.map(line => ` ? ${line}`)] : []),
325 ];
326}
328export interface ReportArtifact { root: string; result: SliceResult }
330/**
331 * Every slice result under the report dir: flat (merged) or one directory per
332 * attempt-scoped artifact. Rule-16 rerun results (scripts/lib/paid-rerun.ts)
333 * are returned only with `{ reruns: true }`, and then only they are.
334 */
335export function loadSliceArtifacts(reportDir: string, options: { reruns?: boolean } = {}): ReportArtifact[] {
336 const found: ReportArtifact[] = [];
337 for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) {
338 const rel = normalizeRelativePath(name);
339 if (!/^slice-\d+\.json$/.test(path.basename(rel)) || rel.split('/').includes('shards') || rel.split('/').includes('receipts')) continue;
340 const result = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf8')) as SliceResult;
341 if ((result.rerun === true) === (options.reruns === true)) found.push({ root: path.join(reportDir, path.dirname(rel)), result });
342 }
343 return found.sort((a, b) => (a.result.attempt ?? 1) - (b.result.attempt ?? 1) || a.result.sliceIndex - b.result.sliceIndex);
344}
346export interface PanelReport extends PanelVerdict {
347 file: string;
348 /** Slice per trial index (trial n -> slice), for the rerun/artifact pointer. */
349 slices: Record<number, number>;
350}
352/** Panel verdicts of one run attempt: exactly the planned trials, each from its reported record. */
353export function panelReports(manifest: PaidRunManifest, results: SliceResult[], attempt: number): PanelReport[] {
354 const reported = new Map<string, { slice: number; outcome: SliceResult['outcomes'][number] }>();
355 for (const result of results.filter(r => (r.attempt ?? 1) === attempt)) {
356 for (const outcome of result.outcomes) reported.set(normalizeRelativePath(outcome.files[0] ?? ''), { slice: result.sliceIndex, outcome });
357 }
358 const panels = new Map<string, ManifestEntry[]>();
359 for (const entry of manifest.entries.filter(e => e.status === 'planned' && e.trial)) {
360 const key = trialPanelKey(entry.file)!;
361 panels.set(key, [...(panels.get(key) ?? []), entry]);
362 }
363 return [...panels.entries()].sort(([a], [b]) => (a < b ? -1 : 1)).map(([key, entries]) => {
364 const plan = entries[0]!.trial!;
365 const slices: Record<number, number> = {};
366 const trials = entries.flatMap(entry => {
367 const got = reported.get(normalizeRelativePath(entry.file));
368 const t = got?.outcome.trial;
369 if (!got || !t || t.outcome === null) return [];
370 slices[t.trial] = got.slice;
371 return [{ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
372 ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
373 ...(got.outcome.reused ? { execution: 'reused' as const } : {}),
374 ...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}),
375 ...(t.failure_cause ? { failure_cause: t.failure_cause } : {}), ...(t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
376 ...(t.failure_detail ? { failure_detail: t.failure_detail } : {}) }];
377 });
378 const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined });
379 // Reuse is whole-panel only: every trial reused from one run, or none.
380 const sources = entries.map(entry => reported.get(normalizeRelativePath(entry.file))?.outcome.reused?.runId ?? null);
381 if (sources.some(source => source !== null) && (sources.some(source => source === null) || new Set(sources).size > 1)) {
382 return { ...verdict, status: 'INCOMPLETE', split: false, failsLane: true, coverage: false, redClass: 'INCOMPLETE',
383 reason: 'partial panel reuse (every trial must come from one reused panel, or none)', file: shardFile(key), slices };
384 }
385 return { ...verdict, file: shardFile(key), slices };
386 });
387}
389type TrialDiagnosis = { failure_cause?: string; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail };
391/** Cause (when it says more than the class), its evidence and the failure detail, appended to a red line. */
392function diagnosisText(d: TrialDiagnosis | undefined, error?: string): string {
393 if (!d?.failure_cause) return '';
394 const evidence = d.failure_cause_evidence && d.failure_cause_evidence !== error ? `: ${d.failure_cause_evidence}` : '';
395 const detail = !d.failure_detail ? ''
396 : 'judge' in d.failure_detail ? ` ${d.failure_detail.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
397 : ` Expected: ${d.failure_detail.expected} · Received: ${d.failure_detail.received}`;
398 return ` · cause ${d.failure_cause}${evidence}${detail}`;
399}
401/** Where a red's transcript lives and how to fetch it. */
402function evidencePointer(artifact: string, key: string, runId: string | undefined, id: string): string {
403 const arg = /^[\w./-]+$/.test(id) ? id : `'${id.replace(/'/g, `'\\''`)}'`;
404 return ` evidence: ${artifact}/shards/${shardSlug([key])}${runId ? ` (fetch: bun run eval:pass-rates --run ${runId} --case ${arg})` : ''}`;
405}
407/** Why one failed trial failed, in one line: a case timeout names its turn. */
408function trialCause(trial: PanelVerdict['trials'][number] & TrialDiagnosis & { timeout_at_turn?: number }): string {
409 if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`;
410 const cls = trial.failure_class ?? 'assertion';
411 const head = cls === 'timeout' || trial.exit_reason === 'timeout'
412 ? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}`
413 : cls;
414 const exit = trial.exit_reason === 'success' ? ' (session completed; check failed)'
415 : trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : '';
416 return `t${trial.trial}: ${head}${exit}${trial.error ? ` — ${trial.error}` : ''}${diagnosisText(trial, trial.error)}`;
417}
419/**
420 * A paid run of exactly one case, for after a repair (never on unchanged
421 * inputs): E2E cases through --case, judges through their selection and an
422 * exact test-name filter, a keyless file (its own case) by file.
423 */
424export function afterRepairCommand(tier: PaidTier, id: string, file: string, trials = 1): string {
425 if (Object.hasOwn(E2E_TIERS, id) && resolvesAlone(id)) return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
426 if (Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) {
427 return `EVALS=1 EVALS_TIER=${tier} EVALS_JUDGE_SELECTION_JSON='${JSON.stringify({ version: 1, selected: [id], reason: 'after a repair' })}' bun test ${file} -t '${caseTestNamePattern([id])}'`;
428 }
429 return `EVALS=1 EVALS_TIER=${tier} bun test ${file}`;
430}
432const resolved = new Map<string, boolean>();
433/** Whether --case can select this E2E id (a case several files register runs by its shard file instead). */
434function resolvesAlone(id: string): boolean {
435 if (!resolved.has(id)) { try { caseSelection(id); resolved.set(id, true); } catch { resolved.set(id, false); } }
436 return resolved.get(id)!;
437}
439/** One line per non-PASS or split panel verdict. `artifacts` names each slice's artifact (slice -> name). */
440export function formatPanelLine(panel: PanelReport, tier: PaidTier, ctx: { artifacts?: Map<number, string>; runId?: string } = {}): string {
441 const mark = panel.status === 'PASS' ? '⚠' : panel.failsLane ? '✗' : '◌';
442 const label = panel.status === 'PASS' ? `PASS ${panel.passed}/${panel.panel.n}` : `${panel.status} ${panel.passed}/${panel.panel.n}`;
443 const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any));
444 const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', ');
445 const red = panel.trials.find(t => t.outcome === 'failed');
446 const slice = red ? panel.slices[red.trial] : undefined;
447 const evidence = red && slice !== undefined
448 ? evidencePointer(ctx.artifacts?.get(slice) ?? `slice ${slice} attempt ${panel.attempt}`, trialShardKey(panel.file, panel.case, red.trial), ctx.runId, panel.case) : '';
449 return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})`
450 + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}`
451 + `${where ? ` [${where}, attempt ${panel.attempt}]` : ''}${evidence} after a repair: ${afterRepairCommand(tier, panel.case, panel.file, panel.panel.n)}`;
452}
454export interface ReportHeadline {
455 lane: string;
456 verdict: 'GREEN' | 'RED';
457 attempt: number;
458 counts: {
459 rule: { passed: number; total: number };
460 behavior: { passed: number; total: number; split: number };
461 judge: { passed: number; total: number };
462 quarantined: { total: number; failingLane: number };
463 skipped: number;
464 infra: number;
465 incomplete: number;
466 unattributed: number;
467 /** Cases that failed once and passed on their one rule-16 rerun (scripts/lib/paid-rerun.ts). */
468 flaky?: number;
469 };
470 /** Re-executions of once-red cases (rule-16 reruns and later attempts) and their cost, when any ran. */
471 reruns?: { executions: number; usd: number; unknown: number };
472 actionRequired: number;
473 wallMs: number | null;
474 costUsd: number;
475 redispatchEligible: boolean;
476}
478/** Known cost, never a bare total when any trial's cost is unknown, and no total when none is known. */
479export function formatCost(knownUsd: number, unknownTrials: number, trials: number): string {
480 if (trials > 0 && unknownTrials === trials) return `cost unknown (no billing captured for any of ${trials} trial(s))`;
481 return `cost $${knownUsd.toFixed(2)}${unknownTrials ? ` known + ${unknownTrials} of ${trials} trial(s) cost unknown` : ''}`;
482}
484export function formatHeadline(h: ReportHeadline, cost: { unknownTrials: number; trials: number } = { unknownTrials: 0, trials: 0 }): string[] {
485 const c = h.counts;
486 const minutes = h.wallMs === null ? 'unknown' : `${Math.floor(h.wallMs / 60_000)}m${String(Math.round((h.wallMs % 60_000) / 1000)).padStart(2, '0')}s`;
487 return [
488 `[test:paid] VERDICT ${h.verdict} — lane ${h.lane}, attempt ${h.attempt}`,
489 ` rule ${c.rule.passed}/${c.rule.total} · behavior ${c.behavior.passed}/${c.behavior.total}${c.behavior.split ? ` (${c.behavior.split} split)` : ''}`
490 + ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`,
491 ` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`,
492 ` wall ${minutes} · ${formatCost(h.costUsd, cost.unknownTrials, cost.trials)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`,
493 ...(c.flaky || h.reruns ? [` FLAKY ${c.flaky ?? 0} (failed once, passed on its one rule-16 rerun) · ${h.reruns?.executions ?? 0} rerun execution(s), `
494 + `${formatCost(h.reruns?.usd ?? 0, h.reruns?.unknown ?? 0, h.reruns?.executions ?? 0)}`] : []),
495 ];
496}
498/** Problems a runner loss or an API/CLI failure before grading produces; nothing else qualifies for re-dispatch. */
499const INFRA_PROBLEMS = [
500 /^slice \d+\/\d+ reported NO result/,
501 /^planned .* was never reported$/,
502 /: never-started$/,
503 /: no trial record \((?:never started|runner error: .*|no test summary)\)$/,
504 /^PANEL \S+ INCOMPLETE /,
505 /^PANEL \S+ FAIL \(INFRA\)/,
506];
507export function infraOnly(problems: readonly string[]): boolean {
508 return problems.length > 0 && problems.every(problem => INFRA_PROBLEMS.some(re => re.test(problem)));
509}
511/**
512 * Shadow gap detection (docs/TESTING_INTERNALS.md#exercised-path-map): a
513 * command an executed case ran outside its static-plus-observed exercised
514 * closure prints a MAP_GAP line (formatMapGap: non-blocking for this PR, the
515 * next census refresh widens the map) and lands in map-gaps.txt. It is its own
516 * result: never a case verdict, a lane problem or a re-dispatch input.
517 */
518export function reportMapGaps(reportDir: string, executedCases: ReadonlySet<string>, rootDir = ROOT): string[] {
519 const out = path.join(reportDir, 'map-gaps.txt');
520 try {
521 const commands = new Map([...attributeCommands(readShardCommands(reportDir))].filter(([id, list]) => executedCases.has(id) && list.length > 0));
522 const lines = commands.size === 0 ? [] : mapGaps(commands, checkoutExercisedConsumers(PR_PROFILE_MAPS, rootDir, PR_FULL_GATE_FILES)).map(formatMapGap);
523 fs.writeFileSync(out, lines.length ? `${lines.join('\n')}\n` : '');
524 console.log(`[test:paid] exercised-path map (shadow, non-blocking): ${lines.length} MAP_GAP line(s) over ${commands.size} executed case(s) with recorded commands`);
525 for (const line of lines) console.log(` ${line}`);
526 return lines;
527 } catch (error) {
528 const reason = error instanceof Error ? error.message : String(error);
529 fs.writeFileSync(out, `MAP_GAP unavailable: ${reason}\n`);
530 console.log(`[test:paid] exercised-path map (shadow, non-blocking): MAP_GAP unavailable: ${reason}`);
531 return [];
532 }
533}
535/**
536 * Report mode: reconcile slice artifacts against the manifest (fail-closed),
537 * compute every panel verdict with panelVerdict(), write collector-outcomes
538 * v2, trial-outcomes.jsonl and report-summary.md, and exit non-zero when the
539 * lane is red. Only the earliest run attempt decides the lane; later attempts
540 * are reported beside it and never replace it.
541 */
542export function runPaidReport(reportDir: string, options: { writeDurations?: boolean; env?: NodeJS.ProcessEnv; rootDir?: string } = {}): number {
543 const env = options.env ?? process.env;
544 const rootDir = options.rootDir ?? ROOT;
545 const summaryPath = path.join(reportDir, 'collector-outcomes.json');
546 const summaryMdPath = path.join(reportDir, 'report-summary.md');
547 const trialOutcomesPath = path.join(reportDir, TRIAL_OUTCOMES_FILE);
548 for (const file of [summaryPath, summaryMdPath, trialOutcomesPath, path.join(reportDir, FLAKE_RECORDS_FILE)]) fs.rmSync(file, { force: true });
549 const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf-8'));
550 const artifacts = loadSliceArtifacts(reportDir);
551 const attempts = [...new Set(artifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b);
552 const primary = attempts[0] ?? 1;
553 const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result);
554 // Rule 16 (scripts/lib/paid-rerun.ts): every attempt and each attempt's one rerun form one history per case; a flaky case's own red is no problem.
555 const rerun = rerunState(reportDir, manifest, artifacts);
556 const flakyFirst = flakyShards(rerun.observations.filter(o => o.attempt === primary && o.phase === 'first'), rerun.flaky);
557 const verdict = { problems: gateProblems(rerun.sets, rerun.observations, rerun.flaky) };
558 const planned = manifest.entries.filter((e) => e.status === 'planned').length;
559 const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`;
560 console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it); a later attempt resolves a red only by the rule-16 flaky rule and never hides a new red` : ''}`);
561 for (const line of rerun.ignored) console.log(`[test:paid] rule-16 rerun ignored: ${line}`);
562 for (const line of formatProfileCoverage(manifest)) console.log(line);
563 for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) {
564 for (const outcome of result.outcomes) {
565 const shown = outcome.reused ? `reused (run ${outcome.reused.runId})` : outcome.trial
566 ? `trial ${outcome.trial.outcome ?? 'NO RECORD'}` : outcome.status;
567 console.log(` slice ${result.sliceIndex} ${shown.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ${outcome.files.join(' ')}`);
568 }
569 }
570 if (options.writeDurations) {
571 const durations = mergePaidTestDurations(loadPaidTestDurations(rootDir, manifest.tier), results);
572 writePaidTestDurations(manifest.tier, durations, rootDir);
573 console.log(`[test:paid] wrote ${Object.keys(durations).length} ${manifest.tier} durations to ${PAID_TEST_DURATIONS_FILE}`);
574 }
576 // Which artifact root (attempt) and which shard (isolated or not) each file belongs to.
577 const roots = artifacts.map(a => ({ root: path.resolve(a.root), attempt: a.result.attempt ?? 1 }))
578 .sort((a, b) => b.root.length - a.root.length);
579 const attemptOf = (abs: string) => roots.find(r => abs === r.root || abs.startsWith(r.root + path.sep))?.attempt ?? primary;
580 const rerunRoots = loadSliceArtifacts(reportDir, { reruns: true }).map(a => path.resolve(a.root));
581 const inRerun = (abs: string) => rerunRoots.some(root => abs === root || abs.startsWith(root + path.sep));
582 const entryBySlug = new Map(manifest.entries.map(entry => [shardSlug([entry.file]), entry]));
583 const shardOf = (rel: string) => {
584 const parts = normalizeRelativePath(rel).split('/');
585 const at = parts.lastIndexOf('shards');
586 return at >= 0 && parts[at + 1] ? entryBySlug.get(parts[at + 1]!) ?? null : null;
587 };
589 const flaky: Array<{ name: string; attempts: number; file: string }> = [];
590 const collectors: Parameters<typeof collectorOutcomeCounts>[0] = [];
591 const files: Array<{ file: string; tier: string; shard: string | number; cost: number;
592 flaky: number; total: number; executed: number; reused: number; passed: number;
593 failed: number; manual_accepted: number; attempts: number }> = [];
594 const manualProblems: string[] = [];
595 const manualClaims = new Map<string, string>();
596 const recordsByShard = new Map<string, any[]>();
597 let costUsd = 0;
598 let flakyCollectorFailures = 0;
599 for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) {
600 const rel = normalizeRelativePath(name);
601 if (!isFinalizedEvalResultFile(rel) || rel.split('/').includes('receipts') || rel === 'collector-outcomes.json') continue;
602 if (inRerun(path.resolve(reportDir, rel)) || attemptOf(path.resolve(reportDir, rel)) !== primary) continue;
603 const shard = shardOf(rel);
604 try {
605 const parsed = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf-8'));
606 if (!Array.isArray(parsed.tests)) {
607 if (Object.hasOwn(parsed, 'tests') || parsed.total_tests !== undefined || parsed.manual_review !== undefined) {
608 manualProblems.push(`${rel}: malformed collector tests[]`);
609 }
610 continue;
611 }
612 costUsd += Number(parsed.total_cost_usd) || 0;
613 if (shard) recordsByShard.set(shard.file, [...(recordsByShard.get(shard.file) ?? []), ...parsed.tests]);
614 // Trial records are verdict input for panelVerdict(), never collector gates.
615 if (shard?.trial) continue;
616 const seen = new Map<string, number>();
617 for (const [index, entry] of parsed.tests.entries()) {
618 if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string' || !entry.name
619 || typeof entry.passed !== 'boolean') {
620 manualProblems.push(`${rel}: attempt ${index + 1}: malformed collector entry (name/passed required)`);
621 continue;
622 }
623 const key = `${entry.suite ?? ''}\0${entry.name}`;
624 const occurrence = (seen.get(key) ?? 0) + 1;
625 seen.set(key, occurrence);
626 if (Object.hasOwn(entry, 'manual_review') && occurrence !== 1) {
627 manualProblems.push(`${rel}: attempt ${index + 1}: manual review is only valid on the first case attempt`);
628 }
629 if (Object.hasOwn(entry, 'manual_review')) {
630 const previous = manualClaims.get(key);
631 if (previous && previous !== rel) manualProblems.push(`${rel}: duplicate manual-review claim for ${entry.name} (also in ${previous})`);
632 else manualClaims.set(key, rel);
633 }
634 const problem = manualReviewProblem(entry, rootDir);
635 if (problem) manualProblems.push(`${rel}: attempt ${index + 1}: ${problem}`);
636 }
637 collectors.push(parsed);
638 const counts = collectorOutcomeCounts([parsed]);
639 if (shard && flakyFirst.has(normalizeRelativePath(shard.file))) flakyCollectorFailures += counts.failed;
640 files.push({ file: rel, tier: parsed.tier ?? 'unknown', shard: parsed.shard ?? '-',
641 cost: parsed.total_cost_usd ?? 0, flaky: parsed.flaky_retries?.length ?? 0,
642 total: counts.passed + counts.failed + counts.manual_accepted, ...counts });
643 for (const f of parsed.flaky_retries ?? []) flaky.push({ ...f, file: rel });
644 } catch (error) {
645 manualProblems.push(`${rel}: malformed collector JSON (${error instanceof Error ? error.message : String(error)})`);
646 }
647 }
648 const evidence = collectorOutcomeCounts(collectors);
649 console.log(`[test:paid] collector final outcomes: ${evidence.executed} executed, ${evidence.reused} reused; ${evidence.passed} passed, ${evidence.failed} failed, ${evidence.manual_accepted} manual accepted (unscored; no score-cache credit) (${evidence.attempts} attempt records from ${collectors.length} collectors; every record counts)`);
650 if (flaky.length > 0) {
651 console.log(`[test:paid] report: ⚠ ${flaky.length} cases with multiple attempts this run: (paid evals never retry; each record counts)`);
652 for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`);
653 }
654 if (manualProblems.length) verdict.problems.push(...manualProblems);
655 if (evidence.failed > flakyCollectorFailures) verdict.problems.push(`${evidence.failed - flakyCollectorFailures} unapproved final collector failure(s)`);
656 if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted
657 || files.reduce((sum, file) => sum + file.executed + file.reused, 0) !== evidence.executed + evidence.reused) {
658 verdict.problems.push('Collector summary totals are inconsistent');
659 }
661 // Panel verdicts: one function, computed here only.
662 const panels = panelReports(manifest, results, primary);
663 const laterPanels = attempts.slice(1).flatMap(attempt => panelReports(manifest, artifacts.map(a => a.result), attempt)
664 .filter(panel => panel.trials.length > 0));
666 // PR-lane receipts from verdicts (the planner ships them to the next run):
667 // a whole fresh PASS panel with one input identity becomes a panel receipt;
668 // a FAIL panel or a failed rule shard becomes a negative receipt that
669 // blocks reuse of any older PASS for the same identity.
670 const revision = env.GITHUB_SHA ?? '';
671 if (env.GITHUB_RUN_ID && /^[a-f0-9]{40}$/.test(revision)) {
672 const receiptsDir = path.join(reportDir, 'report-receipts');
673 const source = { runId: `${env.GITHUB_RUN_ID}/${primary}`, revision, completedAt: Date.now() };
674 const outcomes = results.flatMap(r => r.outcomes);
675 for (const panel of panels) {
676 const keys = outcomes.filter(o => o.trial?.case === panel.case && shardFile(o.files[0] ?? '') === panel.file).map(o => o.reused ? null : o.inputKey ?? null);
677 if (keys.length !== panel.panel.n || keys.some(k => k === null) || new Set(keys).size !== 1) continue;
678 if (panel.status === 'PASS') {
679 writePanelReceipt(receiptsDir, { schema: 1, key: keys[0]!, case: panel.case, kind: panel.kind, panel: panel.panel, source,
680 trials: panel.trials.map(({ trial, outcome, failure_class, exit_reason, error }) => ({ trial, outcome,
681 ...(failure_class ? { failure_class } : {}), ...(exit_reason ? { exit_reason } : {}), ...(error ? { error } : {}) })) });
682 } else if (panel.status === 'FAIL') writeNegativeReceipt(receiptsDir, { schema: 1, key: keys[0]!, source });
683 }
684 for (const outcome of outcomes.filter(o => !o.trial && o.inputKey && !o.reused && o.status !== 'passed')) {
685 writeNegativeReceipt(receiptsDir, { schema: 1, key: outcome.inputKey!, source });
686 }
687 }
689 // Quarantine cap and expiry are the weekly pass-rates gate's (eval-flake-rank --gate).
691 // History: one trial-outcomes line per isolated trial and per JUnit rule/judge case.
692 const runId = env.GITHUB_RUN_ID;
693 const sha = env.GITHUB_SHA;
694 const history: TrialOutcomeRecord[] = [];
695 const common = (attempt: number) => ({ schema: TRIAL_OUTCOME_SCHEMA, tier: manifest.tier, attempt, policy_version: EVAL_POLICY.version,
696 ...(runId ? { run_id: runId } : {}), ...(sha ? { sha } : {}), lane, recorded_at: new Date().toISOString() });
697 for (const { result } of artifacts) {
698 const attempt = result.attempt ?? 1;
699 for (const outcome of result.outcomes) {
700 const t = outcome.trial;
701 if (!t || t.outcome === null) continue;
702 history.push({ ...common(attempt), case: t.case, file: shardFile(outcome.files[0]!), kind: t.kind, trial: t.trial, panel: t.panel,
703 outcome: t.outcome, ...(t.outcome === 'failed' ? { failure_class: t.failure_class ?? 'assertion' } : {}),
704 ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
705 ...(t.outcome === 'failed' && t.failure_cause ? { failure_cause: t.failure_cause } : {}),
706 ...(t.outcome === 'failed' && t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
707 ...(t.outcome === 'failed' && t.failure_detail ? { failure_detail: t.failure_detail } : {}), ...(t.sessions ? { sessions: t.sessions } : {}),
708 duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.cost_known === false ? { cost_known: false } : {}), ...(t.model ? { model: t.model } : {}),
709 ...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}),
710 quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord);
711 }
712 }
713 const sliceArtifacts = new Map(artifacts.filter(a => (a.result.attempt ?? 1) === primary)
714 .map(a => [a.result.sliceIndex, sliceArtifactName(a.root, a.result.sliceIndex, primary)] as const));
715 const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard,
716 common(primary), env.GSTACK_CLAUDE_CLI_VERSION);
717 history.push(...census.history);
718 const { ruleCases } = census;
719 const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass));
720 const hollowWithoutJUnit = allSkipped.filter(outcome => !census.shardSkipReasons.has(normalizeRelativePath(outcome.files[0] ?? '')));
721 if (allSkipped.length > 0) {
722 console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`);
723 for (const outcome of allSkipped) {
724 const reasons = census.shardSkipReasons.get(normalizeRelativePath(outcome.files[0] ?? ''));
725 console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — ${reasons?.size
726 ? [...reasons.entries()].map(([reason, n]) => `${n} ${reason}`).join('; ') : 'no JUnit report, reason unknown'})`);
727 }
728 }
729 const censusLines = formatJUnitCensus(census);
730 for (const line of censusLines) console.log(`[test:paid] ${line}`);
731 // series_identity is stamped afterwards by scripts/eval-trial-series.ts (the report job's next step).
732 fs.writeFileSync(trialOutcomesPath, formatTrialOutcomes(history));
734 // Headline and failure block (A4): one formatter for the log, the PR comment and the weekly issue.
735 const ruleShardFailures = manifest.entries.filter(entry => entry.status === 'planned' && !entry.trial).flatMap(entry => {
736 const got = results.flatMap(r => r.outcomes.map(o => ({ o, slice: r.sliceIndex }))).find(({ o }) => normalizeRelativePath(o.files[0] ?? '') === normalizeRelativePath(entry.file));
737 if (got && got.o.status === 'passed') return [];
738 if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return [];
739 const id = shardCaseId(entry.file);
740 return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(entry.file))}` : ''}`];
741 });
742 const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior');
743 const lanePanels = panels.filter(p => !p.quarantined);
744 const count = (kind: EvalCaseKind) => ({
745 passed: ruleCases.filter(c => c.kind === kind && (c.outcome === 'passed' || (c.outcome === 'failed' && rerun.flaky.has(c.id)))).length
746 + lanePanels.filter(p => p.kind === kind && (p.status === 'PASS' || rerun.flaky.has(p.case))).length,
747 total: ruleCases.filter(c => c.kind === kind).length + lanePanels.filter(p => p.kind === kind).length,
748 });
749 const primaryTrials = results.flatMap(r => r.outcomes.map(o => o.trial)).filter((t): t is ShardTrialRecord => !!t);
750 const wall = results.filter(r => Number.isSafeInteger(r.startedAt) && Number.isSafeInteger(r.finishedAt));
751 const failureLines = [
752 ...ruleShardFailures,
753 ...ruleCases.filter(c => c.line && !rerun.flaky.has(c.id)).map(c => c.line!),
754 ...panels.filter(p => (p.status !== 'PASS' || p.split) && !rerun.flaky.has(p.case)).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })),
755 ...[...rerun.resolutions.values()].filter(r => r.status === 'red' && r.next)
756 .map(r => `✗ ${r.case} ${r.kind} FAILED TWICE (rule 16: a real red): ${r.first.where}; again in ${r.next!.where}${r.next!.symptom ? ` — ${r.next!.symptom}` : ''}`),
757 ];
758 const flakes = flakeRecords(rerun.resolutions.values(), { lane, env });
759 const flakyLines = flakes.map(formatFlakyLine);
760 const spend = rerunSpend(rerun.observations);
761 const red = verdict.problems.length > 0;
762 const headline: ReportHeadline = {
763 lane, verdict: red ? 'RED' : 'GREEN', attempt: primary,
764 counts: {
765 rule: count('rule'),
766 behavior: { ...count('behavior'), split: behaviorPanels.filter(p => p.split).length },
767 judge: count('judge'),
768 quarantined: { total: panels.filter(p => p.quarantined).length, failingLane: panels.filter(p => p.quarantined && p.failsLane).length },
769 skipped: hollowWithoutJUnit.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length,
770 infra: primaryTrials.filter(t => t.failure_class === 'infra').length + ruleCases.filter(c => c.outcome === 'failed' && c.failureClass === 'infra').length
771 + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length,
772 incomplete: panels.filter(p => p.status === 'INCOMPLETE').length,
773 unattributed: census.unattributed.length,
774 ...(flakes.length ? { flaky: flakes.length } : {}),
775 },
776 ...(spend.executions ? { reruns: spend } : {}),
777 actionRequired: verdict.problems.length,
778 wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null,
779 costUsd: Math.round(costUsd * 100) / 100,
780 redispatchEligible: red && infraOnly(verdict.problems),
781 };
782 const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped');
783 const headlineLines = formatHeadline(headline, { unknownTrials: executed.filter(r => r.cost_known === false).length, trials: executed.length });
784 for (const line of headlineLines) console.log(line);
785 if (failureLines.length) {
786 console.log('[test:paid] failures and split verdicts:');
787 for (const line of failureLines) console.log(` ${line}`);
788 }
789 if (flakyLines.length) {
790 console.log('[test:paid] flaky (rule 16: failed once, passed on its one rerun; green with this annotation):');
791 for (const line of flakyLines) console.log(` ${line}`);
792 }
793 for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; resolves a red only by the rule-16 flaky rule): ${formatPanelLine(panel, manifest.tier)}`);
794 if (flakes.length) fs.writeFileSync(path.join(reportDir, FLAKE_RECORDS_FILE), flakes.map(record => JSON.stringify(record)).join('\n') + '\n');
795 const fence = (lines: string[]) => publishedFence(lines);
796 fs.writeFileSync(summaryMdPath, [
797 ...fence(headlineLines),
798 ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []),
799 ...(flakyLines.length ? ['', '**Flaky: failed once, passed on its one rule-16 rerun**', '', ...fence(flakyLines)] : []),
800 ...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []),
801 ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []),
802 ].join('\n') + '\n');
803 if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: {
804 ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted,
805 flaky: files.reduce((sum, file) => sum + file.flaky, 0),
806 }, verdict: headline, headline: headlineLines,
807 census: { skipped: census.skipped, deselected: Object.fromEntries(census.deselected), unattributed: census.unattributed },
808 panels: panels.map(p => ({ case: p.case, kind: p.kind, status: p.status, passed: p.passed, n: p.panel.n, k: p.panel.k,
809 marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason,
810 trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
811 ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
812 // evals.yml wraps these in a fixed fence for the PR comment.
813 failures: failureLines.map(line => sanitizeFixedFenceLine(line)),
814 flaky: flakes, flakyLines: flakyLines.map(line => sanitizeFixedFenceLine(line)), flakyCollectorFailures }, null, 2) + '\n');
815 reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir);
816 if (verdict.problems.length) {
817 console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
818 for (const problem of verdict.problems) console.error(` ✗ ${problem}`);
819 if (headline.redispatchEligible) console.error('[test:paid] report: INFRA-ONLY RED — one re-dispatch as a new run is allowed; report both runs');
820 return 1;
821 }
822 console.log(evidence.manual_accepted
823 ? `[test:paid] report: every planned shard accounted; ${evidence.manual_accepted} manual acceptance(s), no automated-score credit`
824 : '[test:paid] report: every planned shard accounted and passed');
825 return 0;
826}