paid-rerun.ts
396 lines · scripts/lib/paid-rerun.ts at 3f79d81
1/**
2 * Rule-16 flaky reruns for the paid gate (CREW rule 16, Garry 2026-10-19:
3 * "you can just run the failed test again and note that it is flaky... You
4 * don't have to do the whole test pass again for one failed test.").
5 *
6 * Every executed case leaves an ordered history of observations: each run
7 * attempt's first execution, then that attempt's one in-run rerun. A case is
8 * - FLAKY (green, annotated) when it failed exactly once, that failure was a
9 * case verdict (a red rule or judge case, or a behavior panel FAIL whose
10 * red class is VERDICT), and the very next observation of it passed;
11 * - RED when it failed twice, or its failure was harness, INFRA,
12 * INCOMPLETE or unattributed, or nothing has re-executed it yet.
13 * A case that passed is never rerun and leaves no flake record.
14 *
15 * The rerun planner (evals.yml `rerun-plan`) reruns, once and in the same
16 * run attempt, exactly the cases whose only failure is a case verdict that
17 * nothing has re-executed yet, and only when their passing would turn the
18 * gate green. On the gate lane a completed rule shard's red case no longer
19 * fails its slice job (sliceExitCode caseVerdicts), so the run's conclusion
20 * follows this report and a flaky green concludes success. The
21 * report (runPaidReport) applies the same history to every attempt, so a
22 * manual `gh run rerun --failed` attempt that passes every earlier red is
23 * green with the same annotation, while a new red on any attempt stays red
24 * unless its own next observation passes. Trials, kinds, thresholds and
25 * panels never change: a behavior case reruns its whole planned panel, a
26 * judge case its judge's samples, a rule case its one trial.
27 */
28import * as fs from 'node:fs';
29import * as path from 'node:path';
30import { normalizeRelativePath } from './shard-engine';
31import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
32import { isFinalizedEvalResultFile, sanitizeTrialError, type EvalCaseKind } from '../../test/helpers/eval-store';
33import { shardTrial } from './paid-cases';
34import { type PaidRunManifest, type SliceResult, codexSlicesOf, isCodexShard, loadPaidTestDurations, packBySliceBudget, parseRunManifest, trialPanelKey, verifySliceResults } from './paid-plan';
35import { type ReportArtifact, junitCensus, loadSliceArtifacts, panelReports, sliceArtifactName } from './paid-report';
36import { expectedPrCaseCount, shardSlug } from './paid-select';
37import { ROOT } from './paid-types';
39export const FLAKE_RECORDS_FILE = 'flake-records.jsonl';
40/** Each attempt's rerun manifest lands in the report dir as `<RERUN_PLAN_ARTIFACT>-a<attempt>/manifest.json` (one level deeper when several attempts' plans download). */
41export const RERUN_PLAN_ARTIFACT = 'rerun-plan';
43export type Phase = 'first' | 'rerun';
45export interface CaseObservation {
46 case: string;
47 kind: EvalCaseKind;
48 attempt: number;
49 phase: Phase;
50 outcome: 'passed' | 'failed';
51 /** A failed case verdict the one rerun may resolve; harness, INFRA, INCOMPLETE and unattributed failures never are. */
52 eligible: boolean;
53 /** Why a failure is not eligible. */
54 reason?: string;
55 /** The manifest keys that executed it: its non-trial shard, or its panel's trial shards. */
56 keys: string[];
57 /** Where it ran: attempt, phase and slice artifacts. */
58 where: string;
59 symptom?: string;
60 costUsd: number;
61 /** False when some execution captured no billing (PTY sessions): costUsd is then a lower bound. */
62 costKnown: boolean;
63}
65export interface CaseResolution {
66 case: string;
67 kind: EvalCaseKind;
68 status: 'flaky' | 'red';
69 first: CaseObservation;
70 /** flaky: the passing rerun; red: the second failure, when there is one. */
71 next?: CaseObservation;
72 reason: string;
73}
75export interface ObservationSet {
76 attempt: number;
77 phase: Phase;
78 manifest: PaidRunManifest;
79 artifacts: ReportArtifact[];
80 /** A later attempt or a rerun that ran only some slices: panels count only when all their trials ran in it. */
81 partial: boolean;
82}
84const phaseOrder = (phase: Phase) => (phase === 'first' ? 0 : 1);
85const registered = (id: string) => Object.hasOwn(E2E_TOUCHFILES, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id);
86function where(set: ObservationSet, slices: number[]): string {
87 const names = [...new Set(slices)].sort((a, b) => a - b)
88 .map(slice => sliceArtifactName(set.artifacts.find(a => a.result.sliceIndex === slice)?.root ?? '', slice, set.attempt));
89 return `attempt ${set.attempt}${set.phase === 'rerun' ? ' rerun' : ''} (${names.join(', ')})`;
90}
92/** Collector records under each reported shard's own eval dir, keyed by shard. */
93function shardRecords(artifacts: ReportArtifact[]): Map<string, any[]> {
94 const records = new Map<string, any[]>();
95 for (const { root, result } of artifacts) {
96 for (const outcome of result.outcomes) {
97 const key = normalizeRelativePath(outcome.files[0] ?? '');
98 const dir = path.join(root, 'shards', shardSlug([key]));
99 if (!fs.existsSync(dir)) continue;
100 for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) {
101 if (!isFinalizedEvalResultFile(name)) continue;
102 try {
103 const parsed = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));
104 if (Array.isArray(parsed.tests)) records.set(key, [...(records.get(key) ?? []), ...parsed.tests]);
105 } catch { /* a malformed collector is the report's problem, not an observation */ }
106 }
107 }
108 }
109 return records;
110}
112type Outcome = SliceResult['outcomes'][number];
114/** Why a non-trial shard's case results cannot count as a verdict observation, else null. */
115export function shardProblem(manifest: PaidRunManifest, key: string, got: { outcome: Outcome; checkpoint: boolean } | undefined): string | null {
116 const entry = manifest.entries.find(e => normalizeRelativePath(e.file) === key);
117 if (!entry || entry.status !== 'planned') return 'not a planned shard';
118 if (!got) return 'never reported';
119 const o = got.outcome;
120 if (got.checkpoint) return 'its slice ended before its final result';
121 if (o.reused) return 'a reused result';
122 if (o.runnerError !== undefined) return 'runner error';
123 if (o.sliceDeadline) return `slice deadline (${o.sliceDeadline})`;
124 if (o.status !== 'passed' && o.status !== 'failed') return `shard ${o.status}`;
125 if (o.executedTests === null || o.skippedTests === null) return 'no test summary';
126 if (manifest.prCoverage?.mode === 'pr' && o.executedTests - o.skippedTests !== expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)) {
127 return 'executed a different case count than planned';
128 }
129 return null;
130}
132/** Every case verdict one set of slice results holds: JUnit rule and judge cases, and complete trial panels. */
133export function observeCases(set: ObservationSet): CaseObservation[] {
134 const { manifest, artifacts, attempt, phase } = set;
135 const results = artifacts.map(a => a.result);
136 const reported = new Map<string, { outcome: Outcome; checkpoint: boolean }>();
137 for (const result of results) {
138 for (const outcome of result.outcomes) reported.set(normalizeRelativePath(outcome.files[0] ?? ''), { outcome, checkpoint: !!result.checkpoint });
139 }
140 const census = junitCensus(manifest, artifacts, shardRecords(artifacts), {});
141 const unattributed = new Set(census.unattributed.map(line => line.split(' :: ')[0]));
142 const observations: CaseObservation[] = [];
143 for (const c of census.ruleCases) {
144 if (c.outcome === 'skipped') continue;
145 const unsound = shardProblem(manifest, c.key, reported.get(c.key)) ?? (unattributed.has(c.key) ? 'an unattributed testcase in its shard' : null);
146 if (c.outcome === 'passed' && unsound) continue;
147 const reason = c.outcome !== 'failed' ? undefined : unsound ?? (c.failureClass === 'infra' ? 'INFRA failure class'
148 : !registered(c.id) ? 'no registry case id (unattributed)' : undefined);
149 observations.push({ case: c.id, kind: c.kind, attempt, phase, outcome: c.outcome, eligible: c.outcome === 'failed' && !reason, ...(reason ? { reason } : {}),
150 keys: [c.key], where: where(set, [c.slice]),
151 ...(c.outcome === 'failed' ? { symptom: sanitizeTrialError(`${c.failureClass ?? 'assertion'}${c.error ? `: ${c.error}` : ''}`) } : {}),
152 costUsd: c.costUsd, costKnown: c.costKnown });
153 }
154 const slices = new Set(results.map(r => r.sliceIndex));
155 for (const panel of panelReports(manifest, results, attempt)) {
156 const entries = manifest.entries.filter(e => e.status === 'planned' && e.trial && trialPanelKey(e.file) === `${panel.file}#${panel.case}`);
157 if (set.partial && !entries.every(e => slices.has(e.slice))) continue;
158 if (panel.status === 'SKIPPED' || (panel.status === 'FAIL' && !panel.failsLane)) continue;
159 const outcomes = entries.map(e => reported.get(normalizeRelativePath(e.file))?.outcome);
160 if (phase === 'rerun' && outcomes.some(o => o?.reused)) continue;
161 const passed = panel.status === 'PASS';
162 const reason = passed ? undefined : panel.status === 'INCOMPLETE' ? `INCOMPLETE: ${panel.reason}`
163 : panel.redClass === 'INFRA' ? 'INFRA: every failed trial is infra-class' : panel.redClass !== 'VERDICT' ? `red class ${panel.redClass}` : undefined;
164 const failed = panel.trials.filter(t => t.outcome === 'failed').map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}${t.error ? ` ${t.error}` : ''}`);
165 observations.push({ case: panel.case, kind: panel.kind, attempt, phase, outcome: passed ? 'passed' : 'failed', eligible: !passed && !reason, ...(reason ? { reason } : {}),
166 keys: entries.map(e => normalizeRelativePath(e.file)), where: where(set, Object.values(panel.slices)),
167 ...(passed ? {} : { symptom: sanitizeTrialError(`${panel.status} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${failed.join('; ')}`) }),
168 costUsd: Math.round(outcomes.reduce((sum, o) => sum + (o?.trial?.cost_usd ?? 0), 0) * 100) / 100, costKnown: outcomes.every(o => o?.trial && o.trial.cost_known !== false) });
169 }
170 return observations;
171}
173/**
174 * The observation sets of a report dir: each attempt's first execution
175 * (attempt 1 complete, later attempts partial) and each attempt's rerun,
176 * bound to its own rerun manifest. A rerun result that does not match its
177 * manifest is set aside and named.
178 */
179export function observationSets(reportDir: string, manifest: PaidRunManifest, firstArtifacts: ReportArtifact[]): { sets: ObservationSet[]; ignored: string[] } {
180 const reported = [...new Set(firstArtifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b);
181 const attempts = reported.length ? reported : [1];
182 const sets: ObservationSet[] = attempts.map((attempt, i) => ({ attempt, phase: 'first' as const, manifest, partial: i > 0,
183 artifacts: firstArtifacts.filter(a => (a.result.attempt ?? 1) === attempt) }));
184 const ignored: string[] = [];
185 const reruns = loadSliceArtifacts(reportDir, { reruns: true });
186 for (const attempt of [...new Set(reruns.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b)) {
187 let rerunManifest: PaidRunManifest;
188 // download-artifact unpacks a lone pattern match into the report dir and each of several into its own directory.
189 const dir = path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`);
190 const file = [path.join(dir, 'manifest.json'), path.join(dir, path.basename(dir), 'manifest.json')].find(candidate => fs.existsSync(candidate)) ?? path.join(dir, 'manifest.json');
191 try { rerunManifest = parseRunManifest(fs.readFileSync(file, 'utf8')); } catch (error) {
192 ignored.push(`attempt ${attempt}: rerun results without a readable rerun manifest (${error instanceof Error ? error.message : String(error)})`);
193 continue;
194 }
195 const mine = reruns.filter(a => (a.result.attempt ?? 1) === attempt);
196 const bound = mine.filter(a => rerunManifest.rerun?.attempt === attempt && a.result.sliceCount === rerunManifest.sliceCount
197 && a.result.tier === rerunManifest.tier && JSON.stringify(a.result.selection ?? null) === JSON.stringify(rerunManifest.selection ?? null));
198 if (bound.length !== mine.length) ignored.push(`attempt ${attempt}: ${mine.length - bound.length} rerun result(s) do not match the rerun manifest`);
199 sets.push({ attempt, phase: 'rerun', manifest: rerunManifest, partial: true, artifacts: bound });
200 }
201 return { sets, ignored };
202}
204/** Every observation in order: attempt, then first execution before its rerun. A rerun counts only for the cases it was planned for. */
205export function caseHistory(sets: ObservationSet[]): CaseObservation[] {
206 return sets.flatMap(set => observeCases(set).filter(o => set.phase === 'first' || (set.manifest.rerun?.cases ?? []).includes(o.case)))
207 .map((o, index) => ({ o, index }))
208 .sort((a, b) => a.o.attempt - b.o.attempt || phaseOrder(a.o.phase) - phaseOrder(b.o.phase) || a.index - b.index)
209 .map(({ o }) => o);
210}
212/** The rule-16 status of every case that failed at least once. */
213export function resolveCases(history: CaseObservation[]): Map<string, CaseResolution> {
214 const resolved = new Map<string, CaseResolution>();
215 const byCase = new Map<string, CaseObservation[]>();
216 for (const o of history) byCase.set(o.case, [...(byCase.get(o.case) ?? []), o]);
217 for (const [id, list] of byCase) {
218 const fails = list.filter(o => o.outcome === 'failed');
219 if (!fails.length) continue;
220 const first = fails[0]!;
221 const next = list[list.indexOf(first) + 1];
222 if (fails.length === 1 && first.eligible && next?.outcome === 'passed') {
223 resolved.set(id, { case: id, kind: first.kind, status: 'flaky', first, next, reason: `failed once in ${first.where}, passed on rerun in ${next.where}` });
224 continue;
225 }
226 const reason = fails.length > 1 ? `failed twice (${first.where}; ${fails[1]!.where})`
227 : !first.eligible ? `never rerun: ${first.reason}` : 'no rerun verdict';
228 resolved.set(id, { case: id, kind: first.kind, status: 'red', first, ...(fails[1] ? { next: fails[1] } : {}), reason });
229 }
230 return resolved;
231}
233/**
234 * The cases to rerun once now: their only failure is a case verdict and
235 * nothing has executed them since. Usually this attempt's own reds; on a
236 * `gh run rerun --failed` attempt also an earlier attempt's red that its
237 * in-run rerun never reached (another problem blocked it then).
238 */
239export function rerunCandidates(history: CaseObservation[]): CaseObservation[] {
240 return [...resolveCases(history).values()].filter(r => r.status === 'red' && !r.next && r.first.eligible
241 && history.filter(o => o.case === r.case).at(-1) === r.first).map(r => r.first);
242}
244/** Non-trial shards whose every failed case in `observations` is flaky: their non-zero exit and failed status are explained. */
245export function flakyShards(observations: CaseObservation[], flaky: ReadonlySet<string>): Set<string> {
246 const failed = observations.filter(o => o.outcome === 'failed' && o.keys.length === 1 && shardTrial(o.keys[0]!) === null);
247 const keys = new Set(failed.map(o => o.keys[0]!));
248 return new Set([...keys].filter(key => failed.filter(o => o.keys[0] === key).every(o => flaky.has(o.case))));
249}
251/**
252 * The rerun manifest: the candidates' own shards from the plan (a behavior
253 * case's whole panel, a rule or judge case's own shard, narrowed to it in PR
254 * mode; otherwise the file reruns and only the planned cases count), packed
255 * with the plan's budget. It runs fresh: the rerun job
256 * gets no receipts and the executor never reuses for a rerun manifest.
257 */
258export function buildRerunManifest(manifest: PaidRunManifest, candidates: CaseObservation[], attempt: number,
259 durations: Record<string, number> = loadPaidTestDurations(ROOT, manifest.tier)): PaidRunManifest {
260 const ids = [...new Set(candidates.map(c => c.case))].sort();
261 const keys = [...new Set(candidates.flatMap(c => c.keys))].sort();
262 const panels = new Set(candidates.filter(c => c.keys.some(key => shardTrial(key) !== null)).map(c => c.case));
263 const selection = { e2e: ids.filter(id => Object.hasOwn(E2E_TOUCHFILES, id)),
264 judges: ids.filter(id => !Object.hasOwn(E2E_TOUCHFILES, id) && Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) };
265 const budget = manifest.plan ?? { sliceBudgetMs: 420_000, jobs: 2 };
266 const packed = packBySliceBudget(keys, budget.sliceBudgetMs, budget.jobs, durations, undefined, file => isCodexShard(file, manifest.tier));
267 const entries = packed.slices.flatMap((files, index) => files.map(file => {
268 const source = manifest.entries.find(e => normalizeRelativePath(e.file) === file)!;
269 const exclude = (source.excludeCases ?? []).filter(id => panels.has(id));
270 return { file: source.file, slice: index + 1, status: 'planned' as const, estimatedMs: packed.estimates[file]!,
271 ...(source.budget ? { budget: source.budget } : {}), ...(source.trial ? { trial: source.trial } : {}), ...(exclude.length ? { excludeCases: exclude } : {}) };
272 })).sort((a, b) => (a.file < b.file ? -1 : 1));
273 return parseRunManifest(JSON.stringify({
274 version: 1, tier: manifest.tier, evalsAll: false, sliceCount: packed.slices.length,
275 selectionReason: `rule-16 rerun of attempt ${attempt}: ${ids.join(', ')}`, ...(manifest.host ? { host: manifest.host } : {}),
276 profile: manifest.profile ?? 'full', selection,
277 ...(manifest.prCoverage ? { prCoverage: { ...manifest.prCoverage, e2e: selection.e2e, judges: selection.judges } } : {}),
278 plan: { sliceBudgetMs: budget.sliceBudgetMs, jobs: budget.jobs, estimatedSliceMs: packed.estimatedSliceMs, ciTimeoutMinutes: packed.ciTimeoutMinutes,
279 sliceCiTimeoutMinutes: packed.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(manifest.tier, entries) },
280 rerun: { attempt, cases: ids }, entries,
281 } satisfies PaidRunManifest));
282}
284export interface FlakeRecord {
285 schema: 1;
286 kind: 'gate-flake';
287 case: string;
288 case_kind: EvalCaseKind;
289 repository: string | null;
290 pr: number | null;
291 head_sha: string | null;
292 tested_sha: string | null;
293 lane: string;
294 run_id: string | null;
295 first: { attempt: number; phase: Phase; where: string };
296 rerun: { attempt: number; phase: Phase; where: string };
297 symptom: string;
298 /** Null when the rerun captured no billing (PTY sessions). */
299 rerun_cost_usd: number | null;
300 recorded_at: string;
301}
303/** One JSONL record per flaky case, read by the daily flaky-fix pass from the run's `flake-records-a<attempt>` artifact. */
304export function flakeRecords(resolutions: Iterable<CaseResolution>, ctx: { lane: string; env: NodeJS.ProcessEnv; now?: Date }): FlakeRecord[] {
305 const { env } = ctx;
306 const pr = Number(env.GSTACK_GATE_PR);
307 const sha = (value: string | undefined) => (/^[0-9a-f]{40}$/.test(value ?? '') ? value! : null);
308 return [...resolutions].filter(r => r.status === 'flaky').sort((a, b) => (a.case < b.case ? -1 : 1)).map(r => ({
309 schema: 1, kind: 'gate-flake', case: r.case, case_kind: r.kind, repository: env.GITHUB_REPOSITORY ?? null,
310 pr: Number.isSafeInteger(pr) && pr > 0 ? pr : null, head_sha: sha(env.GSTACK_GATE_HEAD_SHA) ?? sha(env.GITHUB_SHA), tested_sha: sha(env.GITHUB_SHA),
311 lane: ctx.lane, run_id: env.GITHUB_RUN_ID ?? null,
312 first: { attempt: r.first.attempt, phase: r.first.phase, where: r.first.where },
313 rerun: { attempt: r.next!.attempt, phase: r.next!.phase, where: r.next!.where },
314 symptom: r.first.symptom ?? 'failed', rerun_cost_usd: r.next!.costKnown ? r.next!.costUsd : null, recorded_at: (ctx.now ?? new Date()).toISOString(),
315 }));
316}
318/** The report and PR comment line of one flaky case (the run id stays in the record: the fence sanitizer redacts long digit runs). */
319export function formatFlakyLine(record: FlakeRecord): string {
320 return `⚑ ${record.case} ${record.case_kind} FLAKY: failed in ${record.first.where}, passed on its one rerun in ${record.rerun.where}; `
321 + `rerun cost ${record.rerun_cost_usd === null ? 'unknown (no billing captured)' : `$${record.rerun_cost_usd.toFixed(2)}`}; first failure: ${record.symptom}`;
322}
324/** What re-executing once-red cases cost: every observation after a case's first failure, in-run reruns and later attempts alike. */
325export function rerunSpend(history: CaseObservation[]): { executions: number; usd: number; unknown: number } {
326 const firstFail = new Map<string, number>();
327 history.forEach((o, index) => { if (o.outcome === 'failed' && !firstFail.has(o.case)) firstFail.set(o.case, index); });
328 const after = history.filter((o, index) => firstFail.has(o.case) && index > firstFail.get(o.case)!);
329 return { executions: after.length, usd: Math.round(after.reduce((sum, o) => sum + o.costUsd, 0) * 100) / 100, unknown: after.filter(o => !o.costKnown).length };
330}
332/** A panel whose every planned trial ran in this set (a later attempt or rerun may hold only some). */
333function completePanel(set: ObservationSet, file: string, id: string): boolean {
334 const slices = new Set(set.artifacts.map(a => a.result.sliceIndex));
335 return set.manifest.entries.filter(e => e.status === 'planned' && e.trial && trialPanelKey(e.file) === `${file}#${id}`).every(e => slices.has(e.slice));
336}
338/**
339 * The slice and panel problems of every first-execution set, with each flaky
340 * case's own red taken out: its shard's non-zero exit and failed status and
341 * its panel's FAIL. A later attempt's problems are prefixed with it, so a new
342 * red there is never hidden. Collector and manual-review checks stay in the
343 * report.
344 */
345export function gateProblems(sets: ObservationSet[], observations: CaseObservation[], flaky: ReadonlySet<string>): string[] {
346 const problems: string[] = [];
347 for (const set of sets.filter(s => s.phase === 'first')) {
348 const results = set.artifacts.map(a => a.result);
349 const prefix = set.partial ? `attempt ${set.attempt}: ` : '';
350 const shards = flakyShards(observations.filter(o => o.attempt === set.attempt && o.phase === 'first'), flaky);
351 problems.push(...verifySliceResults(set.manifest, results, { flaky: shards, partial: set.partial }).problems.map(problem => prefix + problem));
352 for (const panel of panelReports(set.manifest, results, set.attempt)) {
353 if (!panel.failsLane || flaky.has(panel.case) || (set.partial && !completePanel(set, panel.file, panel.case))) continue;
354 problems.push(`${prefix}PANEL ${panel.case} ${panel.status}${panel.redClass === 'INFRA' ? ' (INFRA)' : ''} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${panel.reason}`);
355 }
356 }
357 return problems;
358}
360/** Everything a report dir proves about rule-16 reruns: the ordered history, each failed case's resolution and the flaky set. */
361export function rerunState(reportDir: string, manifest: PaidRunManifest, firstArtifacts: ReportArtifact[]) {
362 const { sets, ignored } = observationSets(reportDir, manifest, firstArtifacts);
363 const observations = caseHistory(sets);
364 const resolutions = resolveCases(observations);
365 const flaky = new Set([...resolutions.values()].filter(r => r.status === 'flaky').map(r => r.case));
366 return { sets, ignored, observations, resolutions, flaky };
367}
369/**
370 * evals.yml `rerun-plan`: write this attempt's rerun manifest to
371 * `<out>/manifest.json` when some case verdict failed once in this attempt
372 * and those cases passing would leave the gate with no other problem. A
373 * harness, INFRA, INCOMPLETE or unattributed red, a case already red twice,
374 * or any other problem plans nothing: the gate keeps its red.
375 */
376export function planRerun(reportDir: string, attempt: number, out: string, durations?: Record<string, number>): { manifest: PaidRunManifest | null; lines: string[] } {
377 const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf8'));
378 const state = rerunState(reportDir, manifest, loadSliceArtifacts(reportDir));
379 const candidates = rerunCandidates(state.observations);
380 const reds = [...state.resolutions.values()].filter(r => r.status === 'red' && !candidates.includes(r.first));
381 const lines = [...state.ignored.map(line => `ignored: ${line}`), ...reds.map(r => `not rerun: ${r.case} (${r.reason})`)];
382 if (!candidates.length) return { manifest: null, lines: [`no rule-16 rerun: no case verdict failed once with no rerun yet`, ...lines] };
383 const remaining = gateProblems(state.sets, state.observations, new Set([...state.flaky, ...candidates.map(c => c.case)]));
384 if (remaining.length) {
385 return { manifest: null, lines: [`no rule-16 rerun: rerunning ${candidates.map(c => c.case).join(', ')} cannot turn this gate green; ${remaining.length} other problem(s), first: ${remaining[0]}`, ...lines] };
386 }
387 let rerun: PaidRunManifest;
388 try { rerun = buildRerunManifest(manifest, candidates, attempt, durations); } catch (error) {
389 return { manifest: null, lines: [`no rule-16 rerun: the rerun manifest is invalid (${error instanceof Error ? error.message : String(error)})`, ...lines] };
390 }
391 fs.mkdirSync(out, { recursive: true });
392 fs.writeFileSync(path.join(out, 'manifest.json'), `${JSON.stringify(rerun, null, 2)}\n`);
393 return { manifest: rerun, lines: [
394 `rule-16 rerun of attempt ${attempt}: ${candidates.length} case(s) once, on ${rerun.sliceCount} slice(s), same head, fresh (no reuse)`,
395 ...candidates.map(c => ` ${c.case} ${c.kind} failed in ${c.where}: ${c.symptom ?? 'failed'}; reruns ${c.keys.join(' ')}`), ...lines] };
396}