paid-plan.ts
892 lines · scripts/lib/paid-plan.ts at 3f79d81
1/**
2 * Paid-lane planner: duration seeds, slice packing, the run manifest and its verification. Moved from scripts/test-paid-shards.ts.
3 */
4import * as fs from 'node:fs';
5import * as path from 'node:path';
6import { normalizeRelativePath } from './shard-engine';
7import { CASE_QUARANTINE } from '../../test/helpers/periodic-exclude-data';
8import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
9import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-store';
10import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
11import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile';
12import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
13import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, partitionHostExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases';
14import { EVAL_HOSTS, type EvalHost, resolveEvalHost } from '../../test/helpers/e2e-hosts';
15import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types';
16import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select';
18// ─── Planner / executor / report (the CI re-platform surface) ──────────────
19// One PLANNER computes selection and the slice plan ONCE; K executor jobs
20// consume it; a REPORT reconciles results against the plan. This kills two
21// classes at the root: per-slice selector divergence (one slice failing
22// merge-base resolution and running a different partition than its siblings)
23// and hollow lanes (a missing/failed slice that artifact-presence aggregation
24// would read as green). CI wiring: evals.yml planner job → K-way matrix of
25// `--plan manifest.json --slice i` → report job running `--report <dir>`.
27export interface ManifestEntry {
28 file: string;
29 /** 1-based executor slice for planned entries; 0 for skipped/excluded. */
30 slice: number;
31 status: 'planned' | 'skipped-by-diff' | 'excluded';
32 reason?: string;
33 /** Required when a registered retry-budget file is planned. */
34 budget?: PaidShardBudget;
35 /** Budget-mode packing weight (recorded wall, or the whole budget when unknown). */
36 estimatedMs?: number;
37 /** Isolated trial shard (`<file>#<id>~t<N>`): the case's kind, fixed panel and quarantine at plan time. */
38 trial?: CaseTrialPlan;
39 /** File shard whose isolated cases run as trial shards: their ids, excluded by name here. */
40 excludeCases?: string[];
41}
43/** Budget-mode plan: per-executor estimate and the CI job timeout it needs. */
44export interface PaidSlicePlan {
45 sliceBudgetMs: number;
46 jobs: number;
47 estimatedSliceMs: number[];
48 /** The largest per-slice ceiling: one number for executors that use a single job timeout. */
49 ciTimeoutMinutes: number;
50 /** Per-slice CI job ceilings (sliceCiTimeoutMinutes); absent in plans written before W2c. */
51 sliceCiTimeoutMinutes?: number[];
52 /** Slices that run only Codex shards; CI executes them on the host-run Codex job, where bubblewrap can start. */
53 codexSlices?: number[];
54}
56export interface PaidRunManifest {
57 version: 1;
58 tier: PaidTier;
59 evalsAll: boolean;
60 sliceCount: number;
61 selectionReason: string;
62 /** The agent host every session runs on (EVALS_HOST); absent means claude. */
63 host?: EvalHost;
64 /** Legacy v1 manifests omit these; new plans bind case-level execution. */
65 profile?: PaidProfile;
66 selection?: PaidCaseSelection;
67 prCoverage?: PrProfileSelection;
68 plan?: PaidSlicePlan;
69 /** A rule-16 rerun manifest (scripts/lib/paid-rerun.ts): the run attempt whose reds it reruns once, and those cases. */
70 rerun?: { attempt: number; cases: string[] };
71 entries: ManifestEntry[];
72}
74/**
75 * Bun never retries a paid case inside a shard (approved 2026-09-29): trials
76 * are fixed by kind before the run (EVAL_POLICY). The one rule-16 rerun of a
77 * red case (CREW rule 16, scripts/lib/paid-rerun.ts) is a separate,
78 * reported execution, never a Bun retry. The function stays the single
79 * statement of that policy for the Bun arguments and the reuse identity.
80 */
81export function retriesForFiles(_files: string[]): number {
82 return 0;
83}
85export const PAID_TEST_DURATIONS_FILE = 'scripts/paid-test-durations.json';
87/**
88 * Recorded per-file paid-shard wall times (ms) per tier from real CI slice
89 * reports (a file's gate and periodic cases differ), refreshed with
90 * `--report <dir> --write-durations`. A packing hint only: a missing or
91 * corrupt seed keeps the supervision-budget allocation.
92 */
93export function loadPaidTestDurations(rootDir = ROOT, tier: PaidTier = 'gate'): Record<string, number> {
94 try {
95 const parsed = JSON.parse(fs.readFileSync(path.join(rootDir, PAID_TEST_DURATIONS_FILE), 'utf8')) as { version?: unknown; tiers?: Record<string, Record<string, unknown>> };
96 if (parsed.version !== 2) return {};
97 return Object.fromEntries(Object.entries(parsed.tiers?.[tier] ?? {})
98 .filter((entry): entry is [string, number] => typeof entry[1] === 'number' && Number.isFinite(entry[1]) && entry[1] > 0));
99 } catch {
100 return {};
101 }
102}
104/** Rewrite one tier of the committed seed, keeping the other tiers. */
105export function writePaidTestDurations(tier: PaidTier, durations: Record<string, number>, rootDir = ROOT): void {
106 const target = path.join(rootDir, PAID_TEST_DURATIONS_FILE);
107 const tiers = Object.fromEntries(PAID_TIERS.map(name => [name, loadPaidTestDurations(rootDir, name)])
108 .filter(([name, recorded]) => name === tier || Object.keys(recorded as object).length > 0));
109 tiers[tier] = durations;
110 const temporary = `${target}.tmp-${process.pid}`;
111 fs.writeFileSync(temporary, `${JSON.stringify({ version: 2, recordedAt: new Date().toISOString(), tiers }, null, 2)}\n`);
112 fs.renameSync(temporary, target);
113}
115/** Seed key of a shard: trials of one case share their case key (`<file>#<id>`). */
116function durationKey(key: string): string {
117 const rel = normalizeRelativePath(key);
118 return shardTrial(rel) === null ? rel : `${shardFile(rel)}${CASE_KEY_SEPARATOR}${shardCaseId(rel)}`;
119}
121/** Recorded wall of a shard key; an unrecorded trial falls back to its whole file's wall. */
122export function recordedShardMs(recorded: Record<string, number>, key: string): number | undefined {
123 const rel = normalizeRelativePath(key);
124 return recorded[durationKey(rel)] ?? (shardTrial(rel) === null ? undefined : recorded[shardFile(rel)]);
125}
127/** Longest recorded wall a PR-lane (gate) shard may have: the lane's ~10-minute case target. */
128export const PR_LANE_SHARD_LIMIT_MS = 600_000;
130/** One line per planned shard whose recorded wall exceeds `limitMs` (split or shorten it; never raise the limit). */
131export function shardDurationViolations(keys: readonly string[], recorded: Record<string, number>, limitMs = PR_LANE_SHARD_LIMIT_MS): string[] {
132 return keys.flatMap(key => {
133 const ms = recorded[durationKey(key)];
134 return ms !== undefined && ms > limitMs ? [`${key}: recorded ${Math.round(ms / 1000)}s > ${limitMs / 1000}s`] : [];
135 });
136}
138/**
139 * Merge a report's executed single-file outcomes into the seed; all-skipped
140 * shards carry no cost signal. Trials of one case record their longest wall
141 * under the case key.
142 */
143export function mergePaidTestDurations(seed: Record<string, number>, results: SliceResult[]): Record<string, number> {
144 const merged = { ...seed };
145 const fresh = new Map<string, number>();
146 for (const result of results) {
147 for (const outcome of result.outcomes) {
148 if (outcome.files.length !== 1 || outcome.elapsedMs < 1_000 || isAllSkippedPass(outcome) || outcome.reused) continue;
149 const key = durationKey(outcome.files[0]!);
150 fresh.set(key, Math.max(fresh.get(key) ?? 0, outcome.elapsedMs));
151 }
152 }
153 for (const [key, ms] of fresh) merged[key] = ms;
154 return Object.fromEntries(Object.entries(merged).sort(([a], [b]) => (a < b ? -1 : 1)));
155}
157/** True when a shard key's file sees the CI Codex install in `tier` (scripts/lib/paid-cases.ts codexShardAccess). */
158export function isCodexShard(key: string, tier: PaidTier): boolean {
159 return codexShardAccess([shardFile(key)], tier) !== 'none';
160}
162/** Sorted slices holding at least one planned Codex shard. */
163export function codexSlicesOf(tier: PaidTier, entries: readonly ManifestEntry[]): number[] {
164 return [...new Set(entries.filter(entry => entry.status === 'planned' && isCodexShard(entry.file, tier)).map(entry => entry.slice))].sort((a, b) => a - b);
165}
167/** Panel identity of a trial shard key (`<file>#<id>`), else null. */
168export function trialPanelKey(key: string): string | null {
169 return shardTrial(key) === null ? null : durationKey(key);
170}
172/** True when `key` is a trial whose panel already has a trial in `planned`. */
173function sharesPanel(planned: readonly string[], key: string): boolean {
174 const panel = trialPanelKey(key);
175 return panel !== null && planned.some(other => other !== key && trialPanelKey(other) === panel);
176}
178/** Setup, image pull and artifact upload allowance on top of a slice's supervised wall. */
179export const CI_SETUP_ALLOWANCE_MINUTES = 20;
181/**
182 * CI job ceiling of one slice (W2c, ENG-2): its envelope is the larger of twice
183 * the slice budget, the longest single shard's supervised wall, and, for the
184 * overlay slice, the serialized overlay group; plus the setup allowance. A hang
185 * fails within this bound instead of holding a runner for the sum of every
186 * shard's worst case; the executor's in-process deadline (sliceDeadlineMs)
187 * stops starting work 5 minutes earlier so results still upload. Same cases run.
188 */
189export function sliceCiTimeoutMinutes(files: string[], budgetMs: number, jobs: number, timeoutMs?: number): number {
190 const longestShardMs = Math.max(0, ...files.map(file => resolvePaidShardTimeoutMs([file], timeoutMs)));
191 const overlayEnvelopeMs = files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, jobs, timeoutMs) : 0;
192 return Math.ceil(Math.max(2 * budgetMs, longestShardMs, overlayEnvelopeMs) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES;
193}
195/**
196 * The executor's in-process deadline: job start (the workflow's first step
197 * records it in GSTACK_SLICE_JOB_STARTED_AT, epoch seconds) plus the slice's
198 * job ceiling, minus the upload reserve. Without a recorded job start (local
199 * runs) the executor's own start stands in.
200 */
201export function sliceDeadlineMs(ceilingMinutes: number, jobStartedAtSeconds: string | undefined, fallbackStartMs: number): number {
202 const recorded = Number(jobStartedAtSeconds);
203 const start = Number.isSafeInteger(recorded) && recorded > 0 ? recorded * 1000 : fallbackStartMs;
204 return start + ceilingMinutes * 60_000 - SLICE_UPLOAD_RESERVE_MS;
205}
207/** Estimated wall of one executor running `files` in order on `jobs` FIFO workers. */
208export function estimatedSliceMs(files: string[], weight: (file: string) => number, jobs: number): number {
209 const workers = Array<number>(Math.max(1, jobs)).fill(0);
210 for (const file of files) {
211 const next = workers.indexOf(Math.min(...workers));
212 workers[next] += weight(file);
213 }
214 return Math.max(...workers);
215}
217/** Executor order within a slice: longest recorded work first, then by path. */
218export function sliceExecutionOrder<T extends { file: string; estimatedMs?: number }>(entries: T[]): T[] {
219 return [...entries].sort((a, b) => (b.estimatedMs ?? 0) - (a.estimatedMs ?? 0) || (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
220}
222/** Supervised worst case of one slice in execution order, overlays at their own admission limit. */
223export function sliceSupervisedWallMs(files: string[], jobs: number, overrideMs?: number): number {
224 return paidShardWallUpperBoundMs(files.filter(file => !isOverlayTestFile(file)), jobs, overrideMs)
225 + paidShardWallUpperBoundMs(files.filter(isOverlayTestFile), Math.min(jobs, OVERLAY_MAX_ACTIVE_SHARDS), overrideMs);
226}
228/**
229 * Budget packing: one runner per file or per tightly packed group. Files go
230 * longest-recorded-first into the fullest slice whose estimated wall stays
231 * within the budget (best fit), else into a new slice. A file with no recorded
232 * wall weighs the whole budget, so unknown cost gets a runner of its own. A
233 * file longer than the budget runs alone. Overlay wrappers keep one shared
234 * final slice (one wrapper at a time). Codex shards never share a slice with
235 * other shards. Each slice gets its own CI job ceiling (sliceCiTimeoutMinutes).
236 */
237export function packBySliceBudget(files: string[], budgetMs: number, jobs: number,
238 recorded: Record<string, number>, timeoutMs?: number, codexShard: (file: string) => boolean = () => false): {
239 slices: string[][]; estimates: Record<string, number>; estimatedSliceMs: number[]; ciTimeoutMinutes: number; sliceCiTimeoutMinutes: number[];
240} {
241 const estimates = Object.fromEntries(files.map(file => [file, recordedShardMs(recorded, file) ?? budgetMs]));
242 const weight = (file: string) => estimates[file]!;
243 const slices: string[][] = [];
244 for (const file of sliceExecutionOrder(files.filter(file => !isOverlayTestFile(file)).map(file => ({ file, estimatedMs: weight(file) }))).map(entry => entry.file)) {
245 let best = -1, bestMs = -1;
246 slices.forEach((planned, index) => {
247 // Trials of one case never share a runner: independent machines, and
248 // the panel's wall stays one trial long.
249 if (sharesPanel(planned, file)) return;
250 // Codex shards run on the host-run Codex job; other shards keep the default container sandbox.
251 if (codexShard(planned[0]!) !== codexShard(file)) return;
252 const ms = estimatedSliceMs([...planned, file], weight, jobs);
253 if (ms <= budgetMs && ms > bestMs) { best = index; bestMs = ms; }
254 });
255 if (best < 0) slices.push([file]);
256 else slices[best]!.push(file);
257 }
258 const overlays = files.filter(isOverlayTestFile).sort();
259 if (overlays.length) slices.push(sliceExecutionOrder(overlays.map(file => ({ file, estimatedMs: weight(file) }))).map(entry => entry.file));
260 if (!slices.length) slices.push([]);
261 const estimatedSliceMsList = slices.map(planned => planned.some(isOverlayTestFile)
262 ? estimatedSliceMs(planned, weight, Math.min(jobs, OVERLAY_MAX_ACTIVE_SHARDS)) : estimatedSliceMs(planned, weight, jobs));
263 const ceilings = slices.map(planned => sliceCiTimeoutMinutes(planned, budgetMs, jobs, timeoutMs));
264 return { slices, estimates, estimatedSliceMs: estimatedSliceMsList, ciTimeoutMinutes: Math.max(...ceilings), sliceCiTimeoutMinutes: ceilings };
265}
267/** Worker counts whose worst-case slice wall duration packing may never worsen. */
268export const SUPERVISED_WORKER_COUNTS = [1, 2, 3, 4] as const;
270/**
271 * Allocate the RUNNABLE shard plan across K slices — deterministic. Registered
272 * long files are spread by supervision budget and the rest round-robin; that
273 * baseline fixes each slice's worst-case wall. With a duration seed, files are
274 * then re-packed longest-recorded-first onto the lightest slice, accepting a
275 * placement only if no slice's worst-case wall exceeds the baseline's maximum
276 * for any supervised worker count. If any file cannot be placed, the baseline
277 * stands.
278 */
279export function buildRunManifest(opts: {
280 tier: PaidTier;
281 profile?: PaidProfile;
282 /** Fixed slice count; exclusive with sliceBudgetMs. */
283 sliceCount?: number;
284 /** Budget mode: pack recorded work so each executor's estimated wall stays
285 * within this budget; the slice count follows from the plan. */
286 sliceBudgetMs?: number;
287 /** Shard workers per executor (EVALS_JOBS) that budget mode plans for. */
288 jobs?: number;
289 evalsAll: boolean;
290 timeoutMs?: number;
291 discovered?: string[];
292 env?: NodeJS.ProcessEnv;
293 rootDir?: string;
294 changedFiles?: string[];
295 /** Recorded per-file durations; defaults to the committed seed under rootDir. */
296 durations?: Record<string, number>;
297 /** Weekly gate census only: LLM judges already run in the periodic census and PR gate lanes. */
298 skipJudges?: boolean;
299 /** Injectable registries (default: E2E_KINDS and CASE_QUARANTINE). */
300 kinds?: Record<string, EvalCaseKind>;
301 quarantine?: Record<string, unknown>;
302}): PaidRunManifest {
303 const budgetMode = opts.sliceBudgetMs !== undefined;
304 if (budgetMode === (opts.sliceCount !== undefined)) throw new Error('Plan with exactly one of --slices or --slice-budget');
305 if (budgetMode && (!Number.isSafeInteger(opts.sliceBudgetMs) || opts.sliceBudgetMs! <= 0 || !Number.isSafeInteger(opts.jobs) || opts.jobs! <= 0)) {
306 throw new Error('--slice-budget needs a positive budget and an explicit positive --jobs');
307 }
308 if (!budgetMode && (!Number.isInteger(opts.sliceCount) || opts.sliceCount! <= 0)) {
309 throw new Error(`--slices needs a positive integer. Received: ${opts.sliceCount}`);
310 }
311 const rootDir = opts.rootDir ?? ROOT;
312 const env = opts.env ?? process.env;
313 const profile = opts.profile ?? validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE');
314 if (profile === 'pr' && opts.tier !== 'gate') throw new Error('PR profile requires gate tier; use --profile full for periodic coverage');
315 const discovered = opts.discovered ?? collectPaidTestFiles(rootDir);
316 const tierSelection = selectPaidTestFiles(discovered, opts.tier, rootDir);
317 const judge = (file: string) => /^test\/skill-llm-eval[^/]*\.test\.ts$/.test(normalizeRelativePath(file));
318 const selected = opts.skipJudges ? tierSelection.selected.filter(file => !judge(file)) : tierSelection.selected;
319 const kinds = opts.kinds ?? E2E_KINDS;
320 const quarantine = opts.quarantine ?? CASE_QUARANTINE;
321 const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id));
322 if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`);
323 const host = resolveEvalHost(env);
324 const ciKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir));
325 const caseKeys = partitionHostExclusions(ciKeys.runnable, opts.tier, host, rootDir);
326 const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge)
327 .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...ciKeys.excluded, ...caseKeys.excluded];
328 const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine });
329 const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? [];
330 const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 });
331 const cases = computePaidCaseSelection({ profile, env, rootDir, changedFiles: opts.changedFiles });
332 const fast = cases.coverage?.mode === 'pr';
333 const profileShards = fast ? shards.filter(files => prProfileFileSelected(files[0], cases.selection, excludeOf(files[0]!))) : shards;
334 const { runnable, skipped } = partitionShardsByDiffSelection(profileShards,
335 cases.selection.e2e === null ? null : new Set(cases.selection.e2e), { rootDir, excludeCases: expansion.excludeCases });
336 if (fast) for (const files of shards) {
337 if (!prProfileFileSelected(files[0], cases.selection, excludeOf(files[0]!))) skipped.push({ files, reason: 'Outside the fast PR profile; retained in broad gate/periodic coverage' });
338 }
339 const extras = (key: string): Pick<ManifestEntry, 'trial' | 'excludeCases'> => {
340 const trial = expansion.trials[normalizeRelativePath(key)];
341 const exclude = expansion.excludeCases[normalizeRelativePath(key)];
342 return { ...(trial ? { trial } : {}), ...(exclude ? { excludeCases: exclude } : {}) };
343 };
345 const entries: ManifestEntry[] = [];
346 if (budgetMode) {
347 const plan = packBySliceBudget(runnable.map(files => files[0]!), opts.sliceBudgetMs!, opts.jobs!,
348 opts.durations ?? loadPaidTestDurations(rootDir, opts.tier), opts.timeoutMs, file => isCodexShard(file, opts.tier));
349 plan.slices.forEach((files, index) => files.forEach(file => entries.push({ file, slice: index + 1, status: 'planned',
350 estimatedMs: plan.estimates[file]!, ...extras(file),
351 ...(FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file)) ? { budget: resolvePaidShardBudget([file], opts.timeoutMs) } : {}) })));
352 for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason, ...extras(s.files[0]!) });
353 for (const e of excluded) entries.push({ file: e.file, slice: 0, status: 'excluded', reason: e.reason });
354 entries.sort((a, b) => (a.file < b.file ? -1 : 1));
355 return parseRunManifest(JSON.stringify({
356 version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length,
357 selectionReason: cases.reason, ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection,
358 ...(cases.coverage ? { prCoverage: cases.coverage } : {}),
359 plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes,
360 sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) },
361 entries,
362 } satisfies PaidRunManifest));
363 }
364 const sliceCount = opts.sliceCount!;
365 const overlaySlice = sliceCount;
366 const reserveOverlaySlice = overlaySlice > 1 && runnable.some(files => files.some(isOverlayTestFile));
367 const ordinarySlices = overlaySlice - Number(reserveOverlaySlice);
368 // Spread registered long files by supervised load. Keep one ordinary-only
369 // lane when possible, so every lane does not inherit a long-workflow tail.
370 // The reserved overlay slice retains its ownership.
371 const ordinary = runnable.filter(files => !files.some(isOverlayTestFile));
372 const registered = ordinary.filter(files => FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(files[0]!)));
373 const allocations = new Map<string, number>();
374 if (registered.length && ordinarySlices > 1) {
375 const loads = Array<number>(ordinarySlices).fill(0);
376 const longLanes = ordinarySlices - Number(registered.length < ordinary.length);
377 const registeredFiles = new Set(registered.map(files => files[0]));
378 const byWall = (a: string[], b: string[]) =>
379 resolvePaidShardTimeoutMs(b, opts.timeoutMs) - resolvePaidShardTimeoutMs(a, opts.timeoutMs) ||
380 (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0);
381 for (const files of [...registered].sort(byWall).concat(
382 ordinary.filter(files => !registeredFiles.has(files[0])))) {
383 const lanes = registeredFiles.has(files[0]) ? longLanes : ordinarySlices;
384 const laneKeys = (index: number) => [...allocations].filter(([, lane]) => lane === index + 1).map(([key]) => key);
385 // A trial whose siblings already hold every long lane may use any
386 // ordinary lane: independent runners outrank long-lane ownership.
387 const pick = (limit: number) => {
388 let best = -1;
389 for (let index = 0; index < limit; index++) {
390 if (sharesPanel(laneKeys(index), files[0]!)) continue;
391 if (best < 0 || loads[index] < loads[best]) best = index;
392 }
393 return best;
394 };
395 let lane = pick(lanes);
396 if (lane < 0) lane = pick(ordinarySlices);
397 if (lane < 0) lane = loads.slice(0, lanes).indexOf(Math.min(...loads.slice(0, lanes)));
398 allocations.set(files[0], lane + 1);
399 loads[lane] += resolvePaidShardTimeoutMs(files, opts.timeoutMs);
400 }
401 }
402 let ordinaryIndex = 0;
403 for (const files of ordinary) {
404 if (allocations.has(files[0])) continue;
405 // Round-robin, skipping a lane that already holds a trial of the same panel.
406 let lane = ordinaryIndex % ordinarySlices;
407 for (let step = 0; step < ordinarySlices; step++) {
408 const candidate = (ordinaryIndex + step) % ordinarySlices;
409 if (!sharesPanel([...allocations].filter(([, l]) => l === candidate + 1).map(([key]) => key), files[0]!)) { lane = candidate; break; }
410 }
411 ordinaryIndex++;
412 allocations.set(files[0], lane + 1);
413 }
414 const packed = packByRecordedDuration();
415 function packByRecordedDuration(): Map<string, number> | null {
416 const recorded = opts.durations ?? loadPaidTestDurations(rootDir, opts.tier);
417 if (ordinarySlices < 2 || ordinary.length === 0 || Object.keys(recorded).length === 0) return null;
418 const bound = (files: string[], jobs: number) => paidShardWallUpperBoundMs([...files].sort(), jobs, opts.timeoutMs);
419 const lanes = Array.from({ length: ordinarySlices }, (_, lane) =>
420 ordinary.filter(files => allocations.get(files[0]) === lane + 1).map(files => files[0]));
421 const caps = SUPERVISED_WORKER_COUNTS.map(jobs => Math.max(...lanes.map(files => bound(files, jobs))));
422 const fits = (files: string[]) => SUPERVISED_WORKER_COUNTS.every((jobs, k) => bound(files, jobs) <= caps[k]);
423 const known = ordinary.map(files => recordedShardMs(recorded, files[0]!))
424 .filter((ms): ms is number => ms !== undefined).sort((a, b) => a - b);
425 const fallback = known.length ? known[Math.min(known.length - 1, Math.floor(known.length * 0.75))] : 1;
426 const weight = (file: string) => recordedShardMs(recorded, file) ?? fallback;
427 const load = (files: string[]) => files.reduce((sum, file) => sum + weight(file), 0);
428 const registeredFiles = new Set(registered.map(files => files[0]));
429 // Local search from the supervised baseline: move or swap a file out of
430 // the heaviest slice whenever that lowers its recorded load without
431 // making the other slice the new maximum or breaching any worst-case cap.
432 // Registered files only trade places with registered files, so the long
433 // lanes keep their ownership.
434 for (let step = 0; step < 10 * ordinary.length; step++) {
435 const loads = lanes.map(load);
436 const heavy = loads.indexOf(Math.max(...loads));
437 let best: { gain: number; apply: () => void } | null = null;
438 for (let other = 0; other < lanes.length; other++) {
439 if (other === heavy) continue;
440 for (const a of lanes[heavy]) {
441 const moves: Array<string | null> = registeredFiles.has(a) ? lanes[other].filter(b => registeredFiles.has(b)) : [null, ...lanes[other].filter(b => !registeredFiles.has(b))];
442 for (const b of moves) {
443 const delta = weight(a) - (b === null ? 0 : weight(b));
444 if (delta <= 0 || loads[other] + delta >= loads[heavy]) continue;
445 const gain = Math.min(delta, loads[heavy] - loads[other] - delta);
446 if (best && gain <= best.gain) continue;
447 const heavyAfter = lanes[heavy].filter(file => file !== a).concat(b === null ? [] : [b]);
448 const otherAfter = lanes[other].filter(file => file !== b).concat([a]);
449 if (!fits(heavyAfter) || !fits(otherAfter)) continue;
450 if (sharesPanel(otherAfter, a) || (b !== null && sharesPanel(heavyAfter, b))) continue;
451 const [h, o] = [heavy, other];
452 best = { gain, apply: () => { lanes[h] = heavyAfter; lanes[o] = otherAfter; } };
453 }
454 }
455 }
456 if (!best) break;
457 best.apply();
458 }
459 return new Map(lanes.flatMap((files, lane) => files.map(file => [file, lane + 1] as const)));
460 }
461 runnable.forEach((files) => {
462 const slice = files.some(isOverlayTestFile) ? overlaySlice : (packed ?? allocations).get(files[0])!;
463 entries.push({ file: files[0], slice, status: 'planned', ...extras(files[0]!),
464 ...(FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(files[0]!))
465 ? { budget: resolvePaidShardBudget(files, opts.timeoutMs) } : {}) });
466 });
467 for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason, ...extras(s.files[0]!) });
468 for (const e of excluded) entries.push({ file: e.file, slice: 0, status: 'excluded', reason: e.reason });
469 entries.sort((a, b) => (a.file < b.file ? -1 : 1));
471 const manifest: PaidRunManifest = {
472 version: 1,
473 tier: opts.tier,
474 evalsAll: opts.evalsAll,
475 sliceCount,
476 selectionReason: cases.reason,
477 ...(host !== 'claude' ? { host } : {}),
478 profile,
479 selection: cases.selection,
480 ...(cases.coverage ? { prCoverage: cases.coverage } : {}),
481 entries,
482 };
483 return parseRunManifest(JSON.stringify(manifest));
484}
486export function parseRunManifest(raw: string): PaidRunManifest {
487 const parsed = JSON.parse(raw) as PaidRunManifest;
488 if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`);
489 if (!PAID_TIERS.includes(parsed.tier)) throw new Error(`manifest tier invalid: ${parsed.tier}`);
490 if (parsed.profile !== undefined && parsed.profile !== 'pr' && parsed.profile !== 'full') throw new Error('manifest profile invalid');
491 if (parsed.host !== undefined && (parsed.host === 'claude' || !EVAL_HOSTS.includes(parsed.host))) throw new Error(`manifest host invalid: ${parsed.host}`);
492 if (parsed.selection !== undefined) {
493 for (const [key, inventory] of [['e2e', E2E_TOUCHFILES], ['judges', LLM_JUDGE_TOUCHFILES]] as const) {
494 const ids = parsed.selection?.[key];
495 if (ids !== null && (!Array.isArray(ids) || ids.some(id => typeof id !== 'string' || !Object.hasOwn(inventory, id)) || new Set(ids).size !== ids.length)) {
496 throw new Error(`manifest ${key} selection invalid`);
497 }
498 }
499 }
500 if (parsed.profile === 'pr') {
501 const coverage = parsed.prCoverage;
502 if (parsed.tier !== 'gate' || !parsed.selection || !coverage ||
503 !['pr', 'dependents', 'full-fallback'].includes(coverage.mode) || !Array.isArray(coverage.deferred) ||
504 !Array.isArray(coverage.unknownFiles) || !Array.isArray(coverage.missingCoverage) ||
505 !Array.isArray(coverage.deferredPromptFiles) || coverage.deferredPromptFiles.some(file => typeof file !== 'string') ||
506 !Array.isArray(coverage.e2e) || !Array.isArray(coverage.judges) ||
507 coverage.unknownFiles.some(file => typeof file !== 'string') ||
508 coverage.needsFullValidation !== false || coverage.missingCoverage.length !== 0 ||
509 JSON.stringify(parsed.selection.e2e) !== JSON.stringify(coverage.e2e) ||
510 JSON.stringify(parsed.selection.judges) !== JSON.stringify(coverage.judges)) {
511 throw new Error('manifest PR coverage/selection invalid or requires full validation');
512 }
513 if (coverage.mode === 'pr' && coverage.e2e.some(id => !prProfileCaseAllowed(id, coverage.directCases ?? []))) {
514 throw new Error('manifest PR selection contains a broad-only case');
515 }
516 if (coverage.deferred.some(item => !Object.hasOwn(E2E_TOUCHFILES, item.id) || E2E_TIERS[item.id] !== item.tier || typeof item.reason !== 'string')) {
517 throw new Error('manifest deferred case is outside the broad census');
518 }
519 }
520 if (parsed.rerun !== undefined && (!Number.isSafeInteger(parsed.rerun?.attempt) || parsed.rerun.attempt < 1 || !Array.isArray(parsed.rerun.cases)
521 || parsed.rerun.cases.length === 0 || parsed.rerun.cases.some(id => typeof id !== 'string') || new Set(parsed.rerun.cases).size !== parsed.rerun.cases.length)) {
522 throw new Error('manifest rerun block invalid');
523 }
524 if (!Number.isInteger(parsed.sliceCount) || parsed.sliceCount <= 0) throw new Error('manifest sliceCount invalid');
525 if (!Array.isArray(parsed.entries)) throw new Error('manifest entries missing');
526 for (const entry of parsed.entries) {
527 if (typeof entry.file !== 'string' || !Number.isInteger(entry.slice)) throw new Error('manifest entry malformed');
528 if (!['planned', 'skipped-by-diff', 'excluded'].includes(entry.status)) throw new Error(`manifest entry status invalid: ${entry.status}`);
529 if (entry.status === 'planned' && (entry.slice < 1 || entry.slice > parsed.sliceCount)) {
530 throw new Error(`planned entry ${entry.file} has out-of-range slice ${entry.slice}`);
531 }
532 if (entry.estimatedMs !== undefined && (entry.status !== 'planned' || !Number.isSafeInteger(entry.estimatedMs) || entry.estimatedMs < 0)) {
533 throw new Error(`manifest entry ${entry.file} has an invalid estimate`);
534 }
535 if (entry.status === 'planned' && parsed.prCoverage?.mode === 'pr' && !prProfileFileSelected(entry.file, parsed.selection!, entry.excludeCases ?? [])) {
536 throw new Error(`manifest file is outside its PR case selection: ${entry.file}`);
537 }
538 }
539 for (const entry of parsed.entries) {
540 const caseId = shardCaseId(entry.file);
541 const trial = shardTrial(entry.file);
542 if (trial !== null) {
543 const plan = entry.trial;
544 const expected = plan && ['rule', 'behavior', 'judge'].includes(plan.kind) && typeof plan.quarantined === 'boolean'
545 ? caseTrialPlan(caseId!, { [caseId!]: plan.kind }, plan.quarantined ? { [caseId!]: true } : {}) : undefined;
546 if (!Object.hasOwn(E2E_TOUCHFILES, caseId!) || !E2E_TOUCHFILES[caseId!]!.includes(shardFile(entry.file))
547 || !plan || !expected || !isIsolatedCase(expected) || !sameTrialPlan(plan, expected) || trial > plan.panel.n) {
548 throw new Error(`Trial shard must name a registered isolated case of its file with its fixed policy panel: ${entry.file}`);
549 }
550 } else if (entry.trial !== undefined) {
551 throw new Error(`Only trial shards carry a trial plan: ${entry.file}`);
552 } else if (caseId === null ? entry.status === 'planned' && CASE_SHARDED_FILES.includes(shardFile(entry.file))
553 : !CASE_SHARDED_FILES.includes(shardFile(entry.file)) || !(caseId in E2E_TOUCHFILES)) {
554 throw new Error(`Case-sharded files plan one registered case per shard: ${entry.file}`);
555 }
556 if (entry.excludeCases !== undefined && (caseId !== null || !Array.isArray(entry.excludeCases) || entry.excludeCases.length === 0
557 || entry.excludeCases.some(id => !parsed.entries.some(other => shardTrial(other.file) !== null
558 && shardFile(other.file) === shardFile(entry.file) && shardCaseId(other.file) === id)))) {
559 throw new Error(`A file shard may exclude only cases that run as its trial shards: ${entry.file}`);
560 }
561 if (caseId !== null && entry.status === 'planned' && parsed.selection?.e2e && !parsed.selection.e2e.includes(caseId)) {
562 throw new Error(`Planned case shard is outside the manifest selection: ${entry.file}`);
563 }
564 }
565 // Panels are whole: exactly n trial entries per isolated case with one plan
566 // and one status, and planned trials on distinct slices when the ordinary
567 // slices allow it.
568 const panels = new Map<string, ManifestEntry[]>();
569 for (const entry of parsed.entries) {
570 const panel = trialPanelKey(entry.file);
571 if (panel !== null) panels.set(panel, [...(panels.get(panel) ?? []), entry]);
572 }
573 const reservedOverlay = parsed.sliceCount > 1 && parsed.entries.some(entry => entry.status === 'planned' && isOverlayTestFile(entry.file));
574 const ordinarySliceCount = parsed.sliceCount - Number(reservedOverlay);
575 for (const [panel, trials] of panels) {
576 const n = trials[0]!.trial!.panel.n;
577 const indices = trials.map(entry => shardTrial(entry.file)!).sort((a, b) => a - b);
578 if (indices.length !== n || indices.some((index, i) => index !== i + 1)
579 || trials.some(entry => !sameTrialPlan(entry.trial, trials[0]!.trial) || entry.status !== trials[0]!.status)
580 || parsed.entries.some(entry => normalizeRelativePath(entry.file) === panel)) {
581 throw new Error(`Isolated case ${panel} must plan exactly its ${n} trials together`);
582 }
583 const slices = trials.filter(entry => entry.status === 'planned').map(entry => entry.slice);
584 if (ordinarySliceCount >= n && new Set(slices).size !== slices.length) {
585 throw new Error(`Trials of ${panel} share a slice; each trial needs its own runner`);
586 }
587 }
588 if (parsed.prCoverage?.mode === 'pr') {
589 const planned = parsed.entries.filter(entry => entry.status === 'planned').map(entry => normalizeRelativePath(entry.file));
590 const required: string[][] = Object.entries(prProfileFileMap(parsed.selection!.e2e)).flatMap(([file, ids]) => {
591 const selected = ids.filter(id => parsed.selection!.e2e!.includes(id));
592 if (!selected.length) return [];
593 const owners = new Set(selected.flatMap(id => {
594 const trials = parsed.entries.filter(entry => shardTrial(entry.file) !== null && shardFile(entry.file) === file && shardCaseId(entry.file) === id);
595 if (trials.length) return trials.map(entry => normalizeRelativePath(entry.file));
596 return [CASE_SHARDED_FILES.includes(file) ? `${file}#${id}` : file];
597 }));
598 return [[...owners]];
599 });
600 if (parsed.selection!.judges!.length) required.push(['test/skill-llm-eval.test.ts']);
601 for (const owners of required) {
602 if (owners.some(owner => planned.filter(key => key === owner).length !== 1)
603 || planned.filter(key => shardFile(key) === shardFile(owners[0]!)).length !== owners.length) {
604 throw new Error(`PR selected cases require exactly one planned owning file: ${owners.join(', ')}`);
605 }
606 }
607 }
608 if (parsed.plan !== undefined) {
609 const plan = parsed.plan;
610 const count = (value: unknown) => Number.isSafeInteger(value) && Number(value) > 0;
611 if (!plan || typeof plan !== 'object' || !count(plan.sliceBudgetMs) || !count(plan.jobs) || !count(plan.ciTimeoutMinutes)
612 || !Array.isArray(plan.estimatedSliceMs) || plan.estimatedSliceMs.length !== parsed.sliceCount
613 || !plan.estimatedSliceMs.every(ms => Number.isSafeInteger(ms) && ms >= 0)
614 || parsed.entries.some(entry => entry.status === 'planned' && entry.estimatedMs === undefined)
615 || (plan.sliceCiTimeoutMinutes !== undefined && (!Array.isArray(plan.sliceCiTimeoutMinutes)
616 || plan.sliceCiTimeoutMinutes.length !== parsed.sliceCount || !plan.sliceCiTimeoutMinutes.every(count)
617 || Math.max(...plan.sliceCiTimeoutMinutes) !== plan.ciTimeoutMinutes))) {
618 throw new Error('manifest slice plan malformed');
619 }
620 const codexSlices = codexSlicesOf(parsed.tier, parsed.entries);
621 if (JSON.stringify(plan.codexSlices ?? []) !== JSON.stringify(codexSlices)) {
622 throw new Error(`manifest Codex slices must be exactly the slices holding Codex shards: ${JSON.stringify(codexSlices)}`);
623 }
624 const mixed = parsed.entries.find(entry => entry.status === 'planned' && codexSlices.includes(entry.slice) && !isCodexShard(entry.file, parsed.tier));
625 if (mixed) throw new Error(`Codex slice ${mixed.slice} also holds ${mixed.file}; only Codex shards run on the host-run Codex job`);
626 }
627 const keys = parsed.entries.map(entry => normalizeRelativePath(entry.file));
628 if (new Set(keys).size !== keys.length) throw new Error('Duplicate manifest entry');
629 // Unique result slugs: shard artifacts merge by path, so a shared slug would
630 // let one trial's records overwrite another's.
631 const slugs = new Map<string, string>();
632 for (const entry of parsed.entries) {
633 const slug = shardSlug([entry.file]);
634 if (slugs.has(slug)) throw new Error(`Shards ${slugs.get(slug)} and ${entry.file} share the result slug ${slug}`);
635 slugs.set(slug, entry.file);
636 }
637 const overlaySlice = parsed.sliceCount;
638 const plannedOverlays = parsed.entries.filter(entry => entry.status === 'planned' && isOverlayTestFile(entry.file));
639 if (plannedOverlays.some(entry => entry.slice !== overlaySlice)) {
640 throw new Error('Overlay manifest entries must share the final ordinary slice to preserve one-process API admission');
641 }
642 if (plannedOverlays.length && overlaySlice > 1 && parsed.entries.some(entry =>
643 entry.status === 'planned' && !isOverlayTestFile(entry.file) && entry.slice === overlaySlice)) {
644 throw new Error('The final ordinary manifest slice is reserved for overlay files');
645 }
646 for (const budget of FILE_RETRY_BUDGETS) {
647 const entries = parsed.entries.filter(entry => shardFile(entry.file) === budget.file);
648 const fileKeys = entries.filter(entry => shardCaseId(entry.file) === null).length;
649 const caseKeys = entries.filter(entry => shardCaseId(entry.file) !== null && shardTrial(entry.file) === null).length;
650 if (fileKeys > 1 || (fileKeys === 1 && caseKeys > 0)) throw new Error(`Duplicate registered manifest entry: ${budget.file}`);
651 for (const entry of entries.filter(entry => entry.status === 'planned')) {
652 if (!entry.budget) throw new Error(`Registered manifest needs an explicit budget record: ${budget.file}`);
653 const expected = resolvePaidShardBudget([entry.file], entry.budget.source === 'explicit' ? entry.budget.timeoutMs : undefined);
654 if (!sameBudget(entry.budget, expected)) throw new Error(`Registered manifest budget differs from declared policy: ${budget.file}`);
655 }
656 }
657 return parsed;
658}
660export interface SliceResult {
661 version: 1;
662 tier: PaidTier;
663 profile?: PaidProfile;
664 selection?: PaidCaseSelection;
665 sliceIndex: number;
666 sliceCount: number;
667 timeoutOverrideMs?: number;
668 /** CI run attempt (github.run_attempt) that produced this slice; absent means 1. */
669 attempt?: number;
670 /** Epoch ms bounds of the slice's shard execution (lane wall time). */
671 startedAt?: number;
672 finishedAt?: number;
673 /** A progress checkpoint the executor never replaced: the job ended (ceiling or cancellation) before its final result. */
674 checkpoint?: true;
675 /** Produced from a rule-16 rerun manifest (`manifest.rerun`), never a first-attempt slice. */
676 rerun?: true;
677 outcomes: Array<Pick<ShardOutcome, 'files' | 'status' | 'exitCode' | 'elapsedMs' | 'executedTests' | 'skippedTests' | 'budget' | 'reused' | 'runnerError' | 'trial' | 'inputKey' | 'sliceDeadline'>>;
678}
680/**
681 * Slice exit = execution completeness, never the semantic verdict. A rule
682 * shard that did not pass fails the slice (unchanged fail-closed rule); an
683 * isolated trial shard fails it only when the harness produced no trial
684 * record. Failed, timed-out or crashed trials are verdict input for the
685 * report's panelVerdict(), so a 2/3 PASS panel never reds its runner.
686 * `caseVerdicts` (the gate lane's CI slices, rule 16): a rule shard that ran
687 * to completion and reported executed cases is verdict input too, so its red
688 * cases stay in the report (which still fails closed on them) and the run's
689 * conclusion follows the report, where a flaky case can resolve green.
690 */
691export function sliceExitCode(outcomes: ReadonlyArray<Pick<ShardOutcome, 'status' | 'trial'> & Partial<Pick<ShardOutcome, 'exitCode' | 'executedTests' | 'skippedTests' | 'runnerError' | 'sliceDeadline'>>>,
692 options: { caseVerdicts?: boolean } = {}): number {
693 const completedRed = (o: (typeof outcomes)[number]) => !!options.caseVerdicts && o.status === 'failed' && o.exitCode !== null && o.exitCode !== undefined
694 && typeof o.executedTests === 'number' && o.executedTests - (o.skippedTests ?? 0) > 0 && o.runnerError === undefined && !o.sliceDeadline;
695 return outcomes.every(outcome => outcome.trial !== undefined
696 ? outcome.trial.outcome !== null
697 : outcome.status === 'passed' || outcome.status === 'skipped-by-diff' || completedRed(outcome)) ? 0 : 1;
698}
700/** A hollow-guarded trial shard has no trial record: the guard's verdict is a harness problem. */
701export function guardTrialRecords<T extends Pick<ShardOutcome, 'status' | 'trial'>>(outcomes: T[]): T[] {
702 return outcomes.map(outcome => outcome.trial && outcome.status === 'passed-empty' && outcome.trial.outcome !== null
703 ? { ...outcome, trial: { ...outcome.trial, outcome: null, harness: 'hollow: executed no case' } } : outcome);
704}
706/**
707 * Reconcile slice results against the manifest — the fail-closed aggregation.
708 * Problems (any → non-zero): a slice index missing entirely (a cancelled or
709 * crashed executor whose artifact never landed), a planned entry no slice
710 * reported, an entry reported by the wrong/duplicate slice, or any reported
711 * outcome that is not a pass.
712 */
713export function verifySliceResults(
714 manifest: PaidRunManifest,
715 results: SliceResult[],
716 /**
717 * `flaky`: non-trial shards whose every failed case passed its one rule-16
718 * rerun (scripts/lib/paid-rerun.ts); their non-zero exit and failed status
719 * are not problems. `partial`: a later run attempt that re-ran only some
720 * slices, so a slice it did not re-run is no problem.
721 */
722 options: { flaky?: ReadonlySet<string>; partial?: boolean } = {},
723): { ok: boolean; problems: string[] } {
724 const problems: string[] = [];
725 const exitOk = (file: string, exitCode: number | null) => exitCode === 0 || !!options.flaky?.has(file);
726 try { parseRunManifest(JSON.stringify(manifest)); }
727 catch (error) { problems.push(`Invalid run manifest: ${error instanceof Error ? error.message : String(error)}`); }
728 const byIndex = new Map<number, SliceResult>();
729 for (const result of results) {
730 if (result.version !== 1) { problems.push(`slice result with unsupported version: ${String(result.version)}`); continue; }
731 if (result.tier !== manifest.tier) problems.push(`slice ${result.sliceIndex} ran tier ${result.tier}, manifest says ${manifest.tier}`);
732 if (manifest.profile === 'pr' && (result.profile !== 'pr' || JSON.stringify(result.selection) !== JSON.stringify(manifest.selection))) {
733 problems.push(`slice ${result.sliceIndex} did not bind the manifest PR case selection`);
734 }
735 if (byIndex.has(result.sliceIndex)) problems.push(`duplicate result for slice ${result.sliceIndex}`);
736 if (result.checkpoint) problems.push(`slice ${result.sliceIndex}/${result.sliceCount} ended before its final result (job ceiling or cancellation); its outcomes come from the last checkpoint: TIMEOUT`);
737 byIndex.set(result.sliceIndex, result);
738 }
739 for (let index = 1; index <= manifest.sliceCount; index += 1) {
740 if (!byIndex.has(index) && !options.partial) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`);
741 }
743 const reported = new Map<string, { slice: number; status: ShardStatus; trial?: ShardTrialRecord; sliceDeadline?: ShardOutcome['sliceDeadline'] }>();
744 const planned = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry]));
745 for (const result of results) {
746 for (const outcome of result.outcomes) {
747 if (outcome.files.some(file => FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file)) || shardCaseId(file) !== null) && outcome.files.length !== 1) {
748 problems.push('Registered result must report its own shard');
749 }
750 const file = normalizeRelativePath(outcome.files[0] ?? '');
751 const trialPlan = planned.get(file)?.trial;
752 if (manifest.prCoverage?.mode === 'pr') {
753 const expected = expectedPrCaseCount(file, manifest.selection!, planned.get(file)?.excludeCases);
754 const executed = outcome.executedTests === null || outcome.skippedTests === null
755 ? -1 : outcome.executedTests - outcome.skippedTests;
756 // A trial's exit status is its verdict (panelVerdict decides); its
757 // completeness is the trial record checked below.
758 if ((!trialPlan && !exitOk(file, outcome.exitCode)) || expected < 1 || (!trialPlan && executed !== expected)) {
759 problems.push(`PR profile expected ${expected} executed cases in ${file}, received ${executed}`);
760 }
761 }
762 if (trialPlan) {
763 const t = outcome.trial;
764 if (!t || t.case !== shardCaseId(file) || t.trial !== shardTrial(file) || !sameTrialPlan(t, trialPlan)
765 || !(t.outcome === null || ['passed', 'failed', 'skipped'].includes(t.outcome))
766 || (t.outcome === 'failed') !== (t.failure_class !== undefined)) {
767 problems.push(`${file}: trial record missing or does not match its planned trial`);
768 }
769 } else if (outcome.trial !== undefined) {
770 problems.push(`${file}: an unplanned trial record`);
771 }
772 if (shardCaseId(file) !== null && outcome.status === 'passed'
773 && (outcome.executedTests === null || outcome.skippedTests === null || outcome.executedTests - outcome.skippedTests !== 1)) {
774 problems.push(`Case shard must execute exactly its one case: ${file}`);
775 }
776 if (outcome.reused !== undefined) {
777 const r = outcome.reused;
778 if (manifest.prCoverage?.mode !== 'pr') problems.push(`${file}: only the fast PR profile may reuse results; this lane executes fresh`);
779 const reusedVerdictOk = outcome.trial ? outcome.trial.outcome !== null : outcome.status === 'passed' && outcome.exitCode === 0;
780 if (!reusedVerdictOk || !/^[a-f0-9]{64}$/.test(r?.inputKey ?? '')
781 || !/^[\w./-]{1,160}$/.test(r?.runId ?? '') || !/^[a-f0-9]{40}$/.test(r?.revision ?? '') || !Number.isSafeInteger(r?.completedAt) || r.completedAt <= 0) {
782 problems.push(`${file}: malformed reused result`);
783 }
784 }
785 if (outcome.inputKey !== undefined && !/^[a-f0-9]{64}$/.test(outcome.inputKey)) problems.push(`${file}: malformed input identity`);
786 if (reported.has(file)) problems.push(`${file} reported by two slices`);
787 reported.set(file, { slice: result.sliceIndex, status: outcome.status, ...(outcome.trial ? { trial: outcome.trial } : {}),
788 ...(outcome.sliceDeadline ? { sliceDeadline: outcome.sliceDeadline } : {}) });
789 const registered = FILE_RETRY_BUDGETS.find(budget => budget.file === shardFile(file));
790 const finding = STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === file);
791 if (finding) {
792 // Full-census runs must account for every registered case. A manifest
793 // explicitly marked selective may report its executed subset.
794 if (!exitOk(file, outcome.exitCode) || !Number.isInteger(outcome.executedTests) ||
795 outcome.executedTests! < 1 || outcome.executedTests! > finding.cases ||
796 (manifest.evalsAll !== false && outcome.executedTests !== finding.cases) || outcome.skippedTests !== 0) {
797 problems.push(`Finding workflow must execute real unskipped cases with exit zero: ${file}`);
798 }
799 }
800 if (registered) {
801 try {
802 const planned = manifest.entries.find(entry => normalizeRelativePath(entry.file) === file)?.budget;
803 const expected = resolvePaidShardBudget([file], result.timeoutOverrideMs ??
804 (planned?.source === 'explicit' ? planned.timeoutMs : undefined));
805 if (!sameBudget(outcome.budget, expected)) problems.push(`Registered effective result budget differs from its planned/explicit allocation: ${file}`);
806 } catch { problems.push(`Invalid registered effective result budget: ${file}`); }
807 }
808 }
809 }
810 for (const entry of manifest.entries) {
811 if (entry.status !== 'planned') continue;
812 const got = reported.get(normalizeRelativePath(entry.file));
813 if (!got) {
814 if (byIndex.has(entry.slice)) problems.push(`planned ${entry.file} (slice ${entry.slice}) was never reported`);
815 continue; // the missing-slice problem above already covers it
816 }
817 if (got.slice !== entry.slice) problems.push(`${entry.file} planned for slice ${entry.slice} but reported by slice ${got.slice}`);
818 // Isolated trial shards: harness health only; the panel verdict gates.
819 if (entry.trial) {
820 if (got.trial?.outcome === null) problems.push(`${entry.file}: no trial record (${got.trial.harness ?? 'unknown'})`);
821 } else if (got.status !== 'passed' && !(got.status === 'failed' && options.flaky?.has(normalizeRelativePath(entry.file)))) {
822 // The slice deadline distinguishes work that never started (INFRA) from work killed in flight (TIMEOUT).
823 const why = got.sliceDeadline === 'not_run' ? ' (not_run: the slice deadline passed before it started)'
824 : got.sliceDeadline === 'hung' ? ' (hung: killed at the slice deadline)' : '';
825 problems.push(`${entry.file}${why}: ${got.status}`);
826 }
827 }
828 return { ok: problems.length === 0, problems };
829}
831/** Budget-mode plan lines: every slice with its estimate, files and retries. */
832export function formatSlicePlan(manifest: PaidRunManifest): string[] {
833 const plan = manifest.plan;
834 if (!plan) return [];
835 const minutes = (ms: number) => (ms / 60_000).toFixed(1);
836 const lines = [`[test:paid] slice plan: ${manifest.sliceCount} slice(s) x ${plan.jobs} worker(s), budget ${minutes(plan.sliceBudgetMs)}m per slice, `
837 + `longest estimate ${minutes(Math.max(0, ...plan.estimatedSliceMs))}m, CI job timeout ${plan.ciTimeoutMinutes}m (largest per-slice ceiling)`];
838 for (let slice = 1; slice <= manifest.sliceCount; slice++) {
839 const mine = sliceExecutionOrder(manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === slice));
840 const over = plan.estimatedSliceMs[slice - 1]! > plan.sliceBudgetMs ? ' [over budget: longer than one runner allows]' : '';
841 const ceiling = plan.sliceCiTimeoutMinutes?.[slice - 1];
842 lines.push(` slice ${slice}: ~${minutes(plan.estimatedSliceMs[slice - 1]!)}m${ceiling !== undefined ? `, job ceiling ${ceiling}m` : ''}${over}`);
843 for (const entry of mine) lines.push(` ${entry.file} ~${minutes(entry.estimatedMs ?? 0)}m retries=${retriesForFiles([entry.file])}`);
844 }
845 return lines;
846}
848/**
849 * Capacity preflight (E-A10): what the plan asks of the runner pool and the
850 * API. Sessions are planned shard processes; at most jobs of them run per
851 * slice at once. Waves > 1 mean slices queue behind the matrix cap and the
852 * lane wall grows by whole slices.
853 */
854export function formatCapacityPreflight(manifest: PaidRunManifest, maxParallel?: number): string[] {
855 const planned = manifest.entries.filter(entry => entry.status === 'planned');
856 const trials = planned.filter(entry => entry.trial);
857 const jobs = manifest.plan?.jobs ?? DEFAULT_JOBS;
858 const longest = [...trials].sort((a, b) => (b.estimatedMs ?? 0) - (a.estimatedMs ?? 0))[0];
859 const waves = maxParallel ? Math.ceil(manifest.sliceCount / maxParallel) : null;
860 return [
861 `[test:paid] capacity: ${manifest.sliceCount} slice(s), ${planned.length} planned shard(s) (${planned.length - trials.length} rule/judge, ${trials.length} trial shard(s) in ${new Set(trials.map(entry => trialPanelKey(entry.file))).size} panel(s)); `
862 + `peak ${Math.min(manifest.sliceCount, maxParallel ?? manifest.sliceCount) * jobs} concurrent shard process(es)`
863 + (waves !== null ? `; wave(s) at max-parallel ${maxParallel}: ${waves}` : ''),
864 ...(longest ? [`[test:paid] capacity: longest indivisible trial ~${((longest.estimatedMs ?? 0) / 60_000).toFixed(1)}m (${longest.file})`] : []),
865 ...(waves !== null && waves > 1 ? [`[test:paid] capacity: ⚠ ${manifest.sliceCount} slices exceed max-parallel ${maxParallel}; later slices queue for a second wave`] : []),
866 ];
867}
869export function formatProfileCoverage(manifest: PaidRunManifest): string[] {
870 const coverage = manifest.prCoverage;
871 return [
872 `[test:paid] coverage: profile=${manifest.profile ?? 'full'} mode=${coverage?.mode ?? 'full'}; selected E2E=${manifest.selection?.e2e?.length ?? 'all'}, judges=${manifest.selection?.judges?.length ?? 'all'}`,
873 ...(coverage ? [`[test:paid] deferred: ${coverage.deferred.length} broad behaviors, ${coverage.deferredPromptFiles.length} changed prompts without quick live coverage; these are not PR passes`] : []),
874 ];
875}
877/** Every collector record counts: paid evals never retry, so a later record never replaces an earlier one. */
878export function collectorOutcomeCounts(results: Array<{ tests?: Array<{
879 name: string; suite?: string; passed: boolean; execution?: string; manual_review?: unknown;
880}> }>): { executed: number; reused: number; passed: number; failed: number; manual_accepted: number; attempts: number } {
881 const counts = { executed: 0, reused: 0, passed: 0, failed: 0, manual_accepted: 0, attempts: 0 };
882 for (const result of results) {
883 for (const entry of result.tests ?? []) {
884 if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string' || typeof entry.passed !== 'boolean') continue;
885 counts.attempts++;
886 counts[entry.execution === 'reused' ? 'reused' : 'executed']++;
887 const outcome = evalEntryOutcome(entry);
888 counts[outcome === 'manual-review' ? 'manual_accepted' : outcome]++;
889 }
890 }
891 return counts;
892}