test-paid-shards.ts
1179 lines · scripts/test-paid-shards.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * test-paid-shards — enumerate, shard, plan and run the paid (gate/periodic/marathon) tiers.
4 *
5 * Every paid test file (or case, or trial) runs in its own Bun process with an
6 * EXTERNAL wall-clock timeout that kills the shard's process GROUP, so a wedged
7 * file or a surviving `claude`/`codex` PTY grandchild never takes a run down or
8 * outlives it. Each shard gets its own GSTACK_EVAL_DIR, TMPDIR and Chromium
9 * profile. The aggregate distinguishes passed, failed, timed-out, never-started
10 * and skipped-by-diff, so partial execution can never look like a pass.
11 *
12 * Modes:
13 * --emit-plan PLANNER: select once (tier, diff/PR profile, exclusions) and
14 * pack shards into slices by recorded duration
15 * (scripts/paid-test-durations.json); each slice gets its own CI
16 * job ceiling (scripts/lib/paid-plan.ts sliceCiTimeoutMinutes).
17 * --plan/--slice EXECUTOR: run one slice of that manifest, stop starting work at
18 * the slice deadline (job ceiling minus a 5-minute upload
19 * reserve: not_run), kill in-flight shards there (hung), and
20 * checkpoint the slice result after every shard.
21 * --report REPORT: reconcile slice results against the manifest,
22 * fail-closed (a missing slice is a failure, not an absence).
23 * (none) local run of a whole tier; --list previews the plan.
24 * --case local diagnosis of one case through the CI panel runner.
25 *
26 * Background local runs and CI dispatch go through scripts/eval-bg.ts
27 * (`bun run eval:bg:<lane>`), which caps a local run at
28 * ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min, at most 4 h.
29 *
30 * Env contract: EVALS_JOBS = how many shard PROCESSES run at once.
31 * EVALS_CONCURRENCY = bun's --max-concurrency WITHIN a shard. Exporting 15 as
32 * the shard count would start 15 claude-spawning processes (the 429 storm).
33 *
34 * Enumeration uses test/helpers/paid-test-set.ts and honors EVALS_TIER against
35 * E2E_TIERS (test/helpers/touchfiles.ts). Spawn, kill, sandbox, logs, seeds and
36 * output classification come from scripts/lib/shard-engine.ts; selection and
37 * planning live in scripts/lib/paid-{types,select,cases,plan,report}.ts; this
38 * file keeps the runner and the CLI.
39 *
40 * Usage: bun run scripts/test-paid-shards.ts --help
41 */
43import * as fs from 'node:fs';
44import * as os from 'node:os';
45import * as path from 'node:path';
46import { createBootstrapRetentionScope } from '../test/helpers/bootstrap-retention';
47import {
48 BunTestOutputClassifier,
49 createShardSandbox,
50 exactTestFileSelectors,
51 forwardAndClassify,
52 isTerminationRequested,
53 nextShardLogPath,
54 normalizeRelativePath,
55 openShardLog,
56 parseCliFlags,
57 removeShardSandbox,
58 runShardChild,
59 strictShardStatus,
60 zeroExecutionVerdict,
61 type ShardChildResult,
62 type ShardLog,
63} from './lib/shard-engine';
64import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
65import { EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
66import {
67 getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict,
68 sanitizeTrialError, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, type TrialFailureClass,
69 trialCostKnown, trialFailureFields, trialSessions,
70} from '../test/helpers/eval-store';
71import { STALL_WINDOW_MS } from '../test/helpers/eval-budgets';
72import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/session-ledger';
73import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
74import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse';
75import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles';
76import { resolveEvalHost } from '../test/helpers/e2e-hosts';
77import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases';
78import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan';
79import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
80import { planRerun } from './lib/paid-rerun';
81import {
82 DEFAULT_JOBS, DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_TIER, DEFAULT_WITHIN_SHARD_CONCURRENCY, OVERLAY_MAX_ACTIVE_SHARDS,
83 PAID_LANE_POLICY, PAID_TIERS, ROOT, SLICE_JOB_STARTED_AT_ENV, SLICE_UPLOAD_RESERVE_MS, isOverlayTestFile,
84 type PaidProfile, type PaidShardBudget, type PaidTier, type ShardOutcome, type ShardStatus, type ShardTrialRecord,
85} from './lib/paid-types';
86import {
87 collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidSelectionEnv, partitionShardsByDiffSelection,
88 planPaidShards, prProfileFileSelected, prProfileShardIds, prProfileTestNamePattern, resolvePaidShardBudget, resolvePaidShardTimeoutMs,
89 selectPaidTestFiles, shardSlug, buildPaidShardArgs, validatedProfile,
90} from './lib/paid-select';
92type E2EShardReuse = NonNullable<ReturnType<typeof prepareE2EShardReuse>>;
94export { PAID_TEST_GLOBS, isPaidTestFile };
95export { PERIODIC_CI_EXCLUDE };
96export * from './lib/paid-types';
97export * from './lib/paid-select';
98export * from './lib/paid-cases';
99export * from './lib/paid-plan';
100export * from './lib/paid-report';
101export * from './lib/paid-rerun';
103type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] };
105/** Records, contract evidence and session-ledger rows an isolated shard left in its eval dir. */
106export function readTrialEvidence(evalDir: string | undefined): TrialEvidence {
107 if (!evalDir || !fs.existsSync(evalDir)) return { records: [], contract: null };
108 const names = fs.readdirSync(evalDir);
109 const parse = (name: string) => { try { return JSON.parse(fs.readFileSync(path.join(evalDir, name), 'utf8')); } catch { return null; } };
110 const finalized = names.filter(name => isFinalizedEvalResultFile(name) && !name.startsWith('e2e-reused-')).map(parse).filter(Boolean);
111 const source = finalized.length ? finalized : names.filter(name => name.startsWith('_partial') && name.endsWith('.json')).map(parse).filter(Boolean);
112 const records = source.flatMap((result: any) => Array.isArray(result?.tests) ? result.tests.filter((t: any) => t && typeof t === 'object') : []);
113 let contract: string | null = records.find((t: any) => t.failure_class === 'contract')?.error ?? null;
114 if (records.some((t: any) => t.failure_class === 'contract') && contract === null) contract = 'contract violation';
115 try {
116 const line = fs.readFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), 'utf8').split('\n').find(l => l.trim());
117 if (line) contract = String(JSON.parse(line).message ?? 'contract violation');
118 } catch { /* no sidecar */ }
119 return { records, contract, sessions: readSessionLedger(evalDir) };
120}
122/** Classify one isolated trial shard. Contract evidence always fails the trial. */
123export function classifyTrialShard(
124 outcome: Pick<ShardOutcome, 'status' | 'executedTests' | 'skippedTests' | 'elapsedMs' | 'runnerError'>,
125 caseId: string, trial: number, plan: CaseTrialPlan,
126 evidence: TrialEvidence,
127): ShardTrialRecord {
128 const failedRecord = evidence.records.find(record => record.passed === false) ?? evidence.records[0];
129 const base: ShardTrialRecord = {
130 case: caseId, trial, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined, outcome: null,
131 cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100,
132 duration_ms: outcome.elapsedMs,
133 ...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), ...trialSessions(evidence.sessions ?? []),
134 ...trialCostKnown(evidence.records, evidence.sessions),
135 };
136 const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => {
137 const credentials = evidence.contract === null ? credentialRejection({ error: failedRecord?.error ?? error, sessions: evidence.sessions }) : undefined;
138 const cls = evidence.contract !== null ? 'contract' : credentials ? 'infra' : failureClass;
139 const raw = evidence.contract ?? (credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? error);
140 return { ...base, outcome: 'failed', failure_class: cls,
141 ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
142 ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
143 ...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}),
144 ...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }, STALL_WINDOW_MS) };
145 };
146 if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` };
147 if (outcome.status === 'never-started') return { ...base, harness: 'never started' };
148 if (outcome.status === 'passed-empty') return { ...base, harness: 'hollow: executed no case' };
149 if (outcome.status === 'skipped-by-diff') return { ...base, harness: 'skipped by diff' };
150 const known = outcome.executedTests !== null && outcome.skippedTests !== null;
151 const ran = known ? outcome.executedTests! - outcome.skippedTests! : null;
152 if (outcome.status === 'timed-out') {
153 if (ran !== null && ran > 1) return { ...base, harness: `isolation broken: ${ran} cases ran` };
154 return failed('timeout', 'shard wall reached');
155 }
156 if (ran === null) return outcome.status === 'failed' ? failed('infra', 'crashed without a test summary') : { ...base, harness: 'no test summary' };
157 if (ran > 1) return { ...base, harness: `isolation broken: ${ran} cases ran` };
158 if (ran === 0) {
159 if (outcome.status === 'passed' && outcome.skippedTests! > 0 && evidence.contract === null) return { ...base, outcome: 'skipped' };
160 if (outcome.status === 'failed') return failed('infra', 'the case never ran (load or setup failure)');
161 return { ...base, harness: 'hollow: executed no case' };
162 }
163 if (outcome.status === 'passed') return evidence.contract !== null ? failed('contract') : { ...base, outcome: 'passed' };
164 return failed(failedRecord ? failureClassOf(failedRecord) : 'assertion');
165}
168export interface ShardCommand {
169 command: string;
170 args: string[];
171}
174export interface RunShardsOptions {
175 timeoutMs?: number;
176 registeredBudgets?: Record<string, PaidShardBudget>;
177 jobs?: number;
178 /** bun --max-concurrency inside each shard (EVALS_CONCURRENCY). */
179 withinShardConcurrency?: number;
180 rootDir?: string;
181 env?: NodeJS.ProcessEnv;
182 /** When set, each shard child gets GSTACK_EVAL_DIR=<evalDirBase>/shards/<slug>/. */
183 evalDirBase?: string;
184 /** Directory for the per-shard full-stream log files (default os.tmpdir()). Tests inject. */
185 logDir?: string;
186 /** Override the spawned command. Tests inject fake slow/spinning commands. */
187 commandFor?: (files: string[]) => ShardCommand;
188 log?: (line: string) => void;
189 /** Fast-profile census: selected real cases per file, excluding Bun skips. */
190 expectedCases?: Record<string, number>;
191 casePatterns?: Record<string, string>;
192 /** The selected case ids per shard key (reported for reused shards). */
193 expectedCaseIds?: Record<string, string[]>;
194 /** PR lane only: verified reuse for one shard's exact child environment and wall. */
195 reuseFor?: (files: string[], env: NodeJS.ProcessEnv, budget: PaidShardBudget) => E2EShardReuse | null;
196 /** Isolated trial shards: key -> the case's fixed trial plan. */
197 trials?: Record<string, CaseTrialPlan>;
198 /** Epoch ms after which no shard starts and in-flight shards are killed (ENG-2 slice deadline). */
199 sliceDeadlineMs?: number;
200 /** Called after every shard start and finish with checkpoint outcomes: as if the job ended now,
201 * in-flight shards read as hung and unstarted ones as not_run (executor checkpoints). */
202 onProgress?: (checkpoint: ShardOutcome[]) => void;
203}
205/** On-failure console excerpt budget: the last N bytes of the shard's log. */
206export const FAILURE_TAIL_BYTES = 64 * 1024;
208/** Read back only the tail of a shard log (never the whole 30-min stream). */
209function readLogTail(logPath: string, maxBytes = FAILURE_TAIL_BYTES): string {
210 try {
211 const size = fs.statSync(logPath).size;
212 const start = Math.max(0, size - maxBytes);
213 const fd = fs.openSync(logPath, 'r');
214 try {
215 const buffer = Buffer.alloc(size - start);
216 fs.readSync(fd, buffer, 0, buffer.length, start);
217 return buffer.toString('utf8');
218 } finally {
219 fs.closeSync(fd);
220 }
221 } catch {
222 return ''; // a lost tail must never turn a real verdict into an exception
223 }
224}
226function paidShardCommand(files: string[], rootDir: string, timeoutMs: number, options: RunShardsOptions,
227 casePattern: string | undefined, evalDir: string | undefined): ShardCommand {
228 return {
229 command: process.execPath,
230 args: [...buildPaidShardArgs(
231 exactTestFileSelectors(files.map(shardFile), rootDir),
232 timeoutMs,
233 options.withinShardConcurrency ?? DEFAULT_WITHIN_SHARD_CONCURRENCY,
234 retriesForFiles(files),
235 ), ...(casePattern !== undefined ? ['--test-name-pattern', casePattern] : []),
236 // Per-test outcomes for pass-rate history, keyed by Bun test name.
237 ...(evalDir ? ['--reporter=junit', '--reporter-outfile', path.join(evalDir, 'junit.xml')] : [])],
238 };
239}
241/** Print the last FAILURE_TAIL_BYTES of a failed shard's log to stdout. */
242function printLogTail(label: string, logPath: string): void {
243 const tail = readLogTail(logPath);
244 if (tail.length === 0) return;
245 process.stdout.write(`${label} last ${Math.min(tail.length, FAILURE_TAIL_BYTES)} bytes of ${logPath}:\n`);
246 process.stdout.write(tail.endsWith('\n') ? tail : `${tail}\n`);
247}
249/** Acknowledge bootstrap dependency retention; an unconfirmed scope keeps the shard state. */
250async function settleBootstrapRetention(
251 scope: NonNullable<ReturnType<typeof createBootstrapRetentionScope>>,
252 deadlineMs: number,
253 label: string,
254 log: (line: string) => void,
255): Promise<{ failed: boolean; removable: boolean }> {
256 try {
257 const retained = await scope.cleanup(deadlineMs);
258 if (!retained.complete) log(`${label} bootstrap retention incomplete; qualification failed`);
259 return { failed: !retained.complete, removable: retained.removable };
260 } catch {
261 log(`${label} bootstrap retention acknowledgment failed; preserving shard state`);
262 return { failed: true, removable: false };
263 }
264}
266/**
267 * End the shard's log spool within the shard's own deadline. An error, a
268 * premature close or the deadline marks the spool failed (and logs once);
269 * returns true when the deadline expired first.
270 */
271function settleShardSpool(spool: ShardLog, deadlineMs: number, onIncomplete: () => void): Promise<boolean> {
272 const logStream = spool.stream;
273 return new Promise<boolean>((resolve) => {
274 let settled = false;
275 let expired = false;
276 let timer: ReturnType<typeof setTimeout> | undefined;
277 const finish = (complete: boolean) => {
278 if (settled) return;
279 settled = true;
280 clearTimeout(timer);
281 logStream.off('error', onError);
282 logStream.off('close', onClose);
283 if (!complete) {
284 spool.failed = true;
285 logStream.destroy();
286 onIncomplete();
287 }
288 resolve(expired);
289 };
290 const onError = () => finish(false);
291 const onClose = () => finish(logStream.writableFinished && !spool.failed);
292 const expire = () => { expired = true; finish(false); };
293 logStream.once('error', onError);
294 logStream.once('close', onClose);
295 if (Date.now() >= deadlineMs) { expire(); return; }
296 if (spool.failed || logStream.destroyed) { finish(false); return; }
297 timer = setTimeout(expire, deadlineMs - Date.now());
298 try { logStream.end(() => finish(logStream.writableFinished && !spool.failed)); }
299 catch { finish(false); }
300 });
301}
303export async function runPaidShard(
304 files: string[],
305 shardNumber: number,
306 totalShards: number,
307 options: RunShardsOptions = {},
308): Promise<ShardOutcome> {
309 if (files.length === 0) throw new Error('Cannot run an empty paid-test shard.');
310 const rootDir = options.rootDir ?? ROOT;
311 const planned = options.registeredBudgets?.[normalizeRelativePath(files[0]!)];
312 const budget = resolvePaidShardBudget(files, options.timeoutMs ??
313 (planned?.source === 'explicit' ? planned.timeoutMs : undefined));
314 const timeoutMs = budget.timeoutMs;
315 const streamLive = (options.jobs ?? DEFAULT_JOBS) === 1;
316 const log = options.log ?? ((line: string) => console.log(line));
317 const label = `[test:paid] shard ${shardNumber}/${totalShards}`;
319 // A case shard runs exactly its one case; PR patterns narrow further.
320 const caseId = files.length === 1 ? shardCaseId(files[0]!) : null;
321 const casePattern = options.casePatterns?.[files[0]!] ?? (caseId !== null ? caseTestNamePattern([caseId]) : undefined);
322 const expectedCases = options.expectedCases ?? (caseId !== null ? { [files[0]!]: 1 } : undefined);
324 const baseEnv = { ...(options.env ?? process.env) };
325 scopeCodexAccess(baseEnv, files.map(shardFile));
326 if (options.evalDirBase) {
327 baseEnv.GSTACK_EVAL_DIR = path.join(options.evalDirBase, 'shards', shardSlug(files));
328 }
329 const trialPlan = files.length === 1 ? options.trials?.[normalizeRelativePath(files[0]!)] : undefined;
330 const trialIndex = files.length === 1 ? shardTrial(files[0]!) : null;
331 if (trialPlan && trialIndex !== null && caseId !== null) {
332 // One case, one trial: the selection binds the child to exactly this id,
333 // and eval-store stamps every record with the trial identity.
334 Object.assign(baseEnv, {
335 [TRIAL_ENV.caseId]: caseId, [TRIAL_ENV.kind]: trialPlan.kind, [TRIAL_ENV.trial]: String(trialIndex),
336 [TRIAL_ENV.panelN]: String(trialPlan.panel.n), [TRIAL_ENV.panelK]: String(trialPlan.panel.k),
337 [TRIAL_ENV.policyVersion]: String(EVAL_POLICY.version),
338 EVALS_SELECTION_JSON: JSON.stringify({ version: 1, selected: [caseId], reason: `trial ${trialIndex}/${trialPlan.panel.n} of ${caseId}` }),
339 });
340 } else {
341 for (const name of Object.values(TRIAL_ENV)) delete baseEnv[name];
342 }
343 const withTrial = (outcome: ShardOutcome): ShardOutcome => trialPlan && trialIndex !== null && caseId !== null
344 ? { ...outcome, trial: classifyTrialShard(outcome, caseId, trialIndex, trialPlan, readTrialEvidence(baseEnv.GSTACK_EVAL_DIR)) }
345 : outcome;
346 // Resolve `claude --version` ONCE in the parent (cached across shards) and
347 // hand it to every child: eval-store's fallback is a synchronous spawn on
348 // the same thread that polls PTY sessions, so children must never pay it.
349 if (!baseEnv.GSTACK_CLAUDE_CLI_VERSION) {
350 baseEnv.GSTACK_CLAUDE_CLI_VERSION = getClaudeCliVersion();
351 }
352 // Verified first-attempt reuse (PR lane only; scripts/e2e-shard-reuse.ts):
353 // identical consumed inputs to a fresh pass in this PR replace execution
354 // with an explicitly reported reused result.
355 // Bootstrap-retention qualification binds per-run state, so that file's shards (and case shards) stay fresh.
356 const reuse = files.some(file => normalizeRelativePath(file).startsWith('test/skill-e2e-qa-workflow.test.ts'))
357 ? null : options.reuseFor?.(files, baseEnv, budget) ?? null;
358 // A trial reuses only its record from a whole PASS panel receipt the
359 // planner shipped; a single trial never has a pass receipt of its own.
360 const panelHit = trialPlan && trialIndex !== null ? reuse?.lookupPanelTrial(trialIndex) ?? null : null;
361 if (panelHit && trialPlan && trialIndex !== null && caseId !== null) {
362 const reusedFrom = { inputKey: panelHit.hit.key, runId: panelHit.hit.source.runId, revision: panelHit.hit.source.revision, completedAt: panelHit.hit.source.completedAt };
363 const passedTrial = panelHit.trial.outcome === 'passed';
364 log(`${label} REUSED trial ${trialIndex}/${trialPlan.panel.n} of ${caseId} (${panelHit.trial.outcome}) from the whole PASS panel of run ${reusedFrom.runId}`);
365 return { shard: shardNumber, files, status: passedTrial ? 'passed' : 'failed', exitCode: passedTrial ? 0 : 1, elapsedMs: 0, groupPid: null,
366 executedTests: 1, skippedTests: 0, budget, reused: reusedFrom,
367 trial: { case: caseId, trial: trialIndex, kind: trialPlan.kind, panel: trialPlan.panel, quarantined: trialPlan.quarantined,
368 outcome: panelHit.trial.outcome, cost_usd: 0, duration_ms: 0,
369 ...(panelHit.trial.failure_class ? { failure_class: panelHit.trial.failure_class } : {}),
370 ...(panelHit.trial.exit_reason ? { exit_reason: panelHit.trial.exit_reason } : {}),
371 ...(panelHit.trial.error ? { error: panelHit.trial.error } : {}) } };
372 }
373 const reused = trialPlan ? null : reuse?.lookup() ?? null;
374 if (reused) {
375 const reusedFrom = { input_key: reused.key, run_id: reused.source.runId, revision: reused.source.revision,
376 completed_at: new Date(reused.source.completedAt).toISOString() };
377 const caseIds = options.expectedCaseIds?.[files[0]!] ?? [];
378 if (baseEnv.GSTACK_EVAL_DIR) {
379 fs.mkdirSync(baseEnv.GSTACK_EVAL_DIR, { recursive: true });
380 fs.writeFileSync(path.join(baseEnv.GSTACK_EVAL_DIR, `e2e-reused-${shardSlug(files)}.json`), `${JSON.stringify({
381 schema_version: 1, tier: 'e2e', shard: shardSlug(files), total_tests: caseIds.length, executed_tests: 0,
382 reused_tests: caseIds.length, passed: caseIds.length, failed: 0, total_cost_usd: 0, total_duration_ms: 0,
383 tests: caseIds.map(name => ({ name, suite: shardSlug(files), tier: 'e2e', passed: true, duration_ms: 0, cost_usd: 0,
384 execution: 'reused', reused_from: reusedFrom, attempt: 1 })),
385 }, null, 2)}\n`);
386 }
387 log(`${label} REUSED ${files.join(' ')} — identical inputs passed in run ${reused.source.runId} at ${reusedFrom.completed_at}`);
388 return withTrial({ shard: shardNumber, files, status: 'passed', exitCode: 0, elapsedMs: 0, groupPid: null,
389 executedTests: caseIds.length, skippedTests: 0, budget,
390 reused: { inputKey: reused.key, runId: reused.source.runId, revision: reused.source.revision, completedAt: reused.source.completedAt } });
391 }
392 const { command, args } = options.commandFor ? options.commandFor(files)
393 : paidShardCommand(files, rootDir, timeoutMs, options, casePattern, baseEnv.GSTACK_EVAL_DIR);
394 if (baseEnv.GSTACK_EVAL_DIR) fs.mkdirSync(baseEnv.GSTACK_EVAL_DIR, { recursive: true });
395 // Per-shard temp + Chromium-profile isolation — the free runner treats
396 // this as mandatory (test-free-shards.ts: two concurrent shards on one
397 // profile dir kill each other's browser; shared tmp cross-contaminates),
398 // and the paid lane had NONE of it. Doubly load-bearing here: when a
399 // shard hits its 30-min wall the group-SIGKILL means per-test afterAll
400 // cleanup never runs — the rmSync backstop below is the only thing
401 // stopping wedged runs from accumulating full git-repo workspaces in the
402 // shared tmpdir forever. Prerequisite for raising EVALS_JOBS (more
403 // concurrency on shared state amplifies exactly the opus-47 race class).
404 const sandbox = createShardSandbox('gstack-paid-shard-', baseEnv);
405 const { stateDir, tmp: childTmp, env } = sandbox;
406 const bootstrapFile = files.some(file => normalizeRelativePath(file).startsWith('test/skill-e2e-qa-workflow.test.ts'));
407 delete env.GSTACK_BOOTSTRAP_RETENTION;
408 if (bootstrapFile && process.platform !== 'linux') log(`${label} bootstrap dependency retention unavailable on ${process.platform}; native behavior still runs without retained-dependency qualification`);
409 const bootstrapRetention = bootstrapFile && process.platform === 'linux'
410 ? createBootstrapRetentionScope(childTmp, path.join(env.GSTACK_EVAL_DIR || getProjectEvalDir(), 'bootstrap-retention'), env.EVALS_RUN_ID ||= `bootstrap-${Date.now()}-${process.pid}`)
411 : undefined;
412 if (bootstrapRetention) Object.assign(env, bootstrapRetention.env);
413 let retentionFailed = false;
415 const startedAt = Date.now();
416 log(`${label} START ${files.join(' ')} (timeout ${Math.round(timeoutMs / 1000)}s, ${budget.source}${budget.policyId ? `: ${budget.policyId}` : ''})`);
418 // Full-stream spool: EVERY child byte lands on disk (the free runner's
419 // model), never in a whole-run Buffer[] — non-live shards used to hold
420 // their entire 30-min stream-json stdout+stderr in RAM, × concurrent jobs.
421 // Printed at START so a wedged shard is inspectable live, mid-run.
422 const logPath = nextShardLogPath(options.logDir ?? os.tmpdir(), `gstack-paid-shard-${shardSlug(files)}`);
423 const spool = openShardLog(logPath, label);
424 log(`${label} full log: ${logPath}`);
426 const classifier = new BunTestOutputClassifier();
427 // Tee: the spool always gets the chunk; live mode (jobs=1) also forwards to
428 // the console. forwardAndClassify feeds the classifier FIRST, so the strict
429 // verdict path is unchanged by where the bytes land afterwards.
430 const sink = (destination: NodeJS.WriteStream): NodeJS.WriteStream => ({
431 write: (chunk: Buffer | string): boolean => {
432 spool.write(chunk);
433 if (streamLive) destination.write(chunk);
434 return true;
435 },
436 } as unknown as NodeJS.WriteStream);
438 let exitCode: number | null = null;
439 let timedOut = false;
440 let groupPid: number | null = null;
441 let incompleteCapture: ShardChildResult['incompleteCapture'];
442 const ownDeadline = Date.now() + timeoutMs;
443 const shardDeadline = Math.min(ownDeadline, options.sliceDeadlineMs ?? Infinity);
444 try {
445 // Shared spawn/detached/group-kill/wall-timer/reap lifecycle.
446 const result = await runShardChild({
447 command,
448 args,
449 cwd: rootDir,
450 env,
451 timeoutMs,
452 deadlineMs: shardDeadline,
453 hookStreams: (child) => {
454 const streams: Array<Promise<void>> = [];
455 if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier, 'stdout'));
456 if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier, 'stderr'));
457 return streams;
458 },
459 });
460 exitCode = result.exitCode;
461 timedOut = result.timedOut;
462 groupPid = result.groupPid;
463 incompleteCapture = result.incompleteCapture;
464 } catch (error) {
465 const result = (error as { shardResult?: ShardChildResult } | null)?.shardResult;
466 if (result) {
467 exitCode = result.exitCode;
468 timedOut = result.timedOut;
469 groupPid = result.groupPid;
470 incompleteCapture = result.incompleteCapture;
471 }
472 throw error;
473 } finally {
474 if (incompleteCapture) log(`${label} incomplete child capture: ${JSON.stringify(incompleteCapture)}; retained log prefix: ${logPath}`);
475 if (await settleShardSpool(spool, shardDeadline, () => log(`${label} incomplete log capture; retained prefix: ${logPath}`))) timedOut = true;
476 let retentionRemovable = true;
477 if (bootstrapRetention) {
478 ({ failed: retentionFailed, removable: retentionRemovable } = await settleBootstrapRetention(bootstrapRetention, shardDeadline, label, log));
479 }
480 // Async, best-effort backstop: group-SIGKILLed tests never clean up.
481 if (retentionRemovable) await removeShardSandbox(stateDir);
482 }
484 const summary = classifier.end();
486 // expectedFiles: a shard whose bun child ran fewer files than planned with
487 // exit 0 is NOT 'passed' (bun counts self-skipped files, so M = planned).
488 // Fake commandFor children must print a synthetic `Ran N tests across M files`.
489 const expectedFiles = files.length;
490 let status: ShardStatus = strictShardStatus({
491 timedOut, exitCode, summary, expectedFiles,
492 evidenceComplete: !retentionFailed && !spool.failed && !incompleteCapture,
493 });
494 if (status === 'passed' && expectedCases) {
495 const expected = files.reduce((count, file) => count + (expectedCases[file] ?? 0), 0);
496 const actual = summary.terminalTestCounts.reduce((count, value) => count + value, 0) - summary.skippedTests;
497 if (expected < 1 || actual !== expected) {
498 status = 'failed';
499 log(`${label} expected ${expected} selected cases, executed ${actual}; refusing incomplete case coverage`);
500 }
501 }
502 const elapsedMs = Date.now() - startedAt;
503 if (reuse && !trialPlan && !isTerminationRequested()) { if (status === 'passed') reuse.publish(); else reuse.publishFailure(); }
504 const inputKey = reuse?.unchanged() ? reuse.inputKey : undefined;
506 // Failure debuggability without the RAM cost: read back only the log's
507 // tail. Live mode already streamed everything, so no re-print there.
508 if (status !== 'passed' && !streamLive) printLogTail(label, logPath);
509 const logSuffix = status === 'passed' ? '' : ` — full log: ${logPath}`;
510 log(`${label} ${status.toUpperCase()} in ${Math.round(elapsedMs / 1000)}s (exit ${exitCode ?? 'signal'})${logSuffix}`);
512 const executedTests = summary.terminalTestCounts.length > 0
513 ? summary.terminalTestCounts.reduce((a, b) => a + b, 0)
514 : null;
515 const skippedTests = summary.terminalTestCounts.length > 0 ? summary.skippedTests : null;
516 const hung = status === 'timed-out' && shardDeadline < ownDeadline;
517 if (hung) log(`${label} HUNG: killed at the slice deadline before its own ${Math.round(timeoutMs / 1000)}s wall`);
518 return withTrial({ shard: shardNumber, files, status, exitCode, elapsedMs, groupPid, executedTests, skippedTests, budget,
519 ...(inputKey ? { inputKey } : {}), ...(hung ? { sliceDeadline: 'hung' as const } : {}) });
520}
522export interface RunSummary {
523 total: number;
524 executed: number;
525 passed: number;
526 failed: number;
527 timedOut: number;
528 neverStarted: number;
529 /** Shards the parent skipped via diff selection — successes, never conflated with never-started. */
530 skippedByDiff: number;
531 outcomes: ShardOutcome[];
532}
534export function summarize(outcomes: ShardOutcome[]): RunSummary {
535 const count = (status: ShardStatus) => outcomes.filter((o) => o.status === status).length;
536 return {
537 total: outcomes.length,
538 executed: outcomes.length - count('never-started') - count('skipped-by-diff'),
539 passed: count('passed'),
540 failed: count('failed') + count('passed-empty'),
541 timedOut: count('timed-out'),
542 neverStarted: count('never-started'),
543 skippedByDiff: count('skipped-by-diff'),
544 outcomes,
545 };
546}
548/**
549 * Hollow-shard guard. Under EVALS_ALL (the run promised EVERY test), a
550 * passed shard whose bun summary reported 0 executed tests is not a pass —
551 * it is the zero-execution class one layer down (file selected, every test
552 * inside self-skipped, exit 0). Selective runs keep those shards 'passed'
553 * (in-file diff/tier self-skips are legitimate) and only warn.
554 */
555export function applyHollowShardGuard(
556 outcomes: ShardOutcome[],
557 opts: { evalsAll: boolean; requireExecuted?: boolean; warn?: (line: string) => void },
558): ShardOutcome[] {
559 const warn = opts.warn ?? ((line: string) => console.error(line));
560 return outcomes.map((outcome) => {
561 if (opts.requireExecuted && outcome.status === 'passed' &&
562 (outcome.executedTests === null || outcome.executedTests === 0 || isAllSkippedPass(outcome))) {
563 return { ...outcome, status: 'passed-empty' };
564 }
565 if (outcome.status !== 'passed') return outcome;
566 const verdict = zeroExecutionVerdict(outcome.executedTests, PAID_LANE_POLICY, { promisedAll: opts.evalsAll });
567 if (verdict === 'passed-with-warning') {
568 warn(`[test:paid] WARNING: shard ${outcome.shard} passed with 0 executed tests (${outcome.files.join(' ')}) — legitimate under selection, hollow under EVALS_ALL`);
569 return outcome;
570 }
571 return verdict === 'passed-empty' ? { ...outcome, status: 'passed-empty' } : outcome;
572 });
573}
575/**
576 * Exit code for a finished run: skipped-by-diff shards are successes (the
577 * parent proved none of their tests were selected); everything else must
578 * have passed.
579 */
580export function summaryExitCode(summary: RunSummary): number {
581 return summary.passed + summary.skippedByDiff === summary.total ? 0 : 1;
582}
584/** Run every shard in its own process. A timeout or failure never aborts the run. */
585export async function runPaidShards(
586 shards: string[][],
587 options: RunShardsOptions = {},
588): Promise<RunSummary> {
589 const jobs = Math.max(1, options.jobs ?? DEFAULT_JOBS);
590 const outcomes: ShardOutcome[] = shards.map((files, index) => ({
591 shard: index + 1,
592 files,
593 status: 'never-started',
594 exitCode: null,
595 elapsedMs: 0,
596 groupPid: null,
597 executedTests: null,
598 skippedTests: null,
599 }));
601 // Validate the whole batch before any child can spend or create artifacts.
602 for (const files of shards) resolvePaidShardTimeoutMs(files, options.timeoutMs);
603 const pending = shards.map((_, index) => index);
604 const running = new Set<number>();
605 let activeOverlayShards = 0;
606 const waiters = new Set<() => void>();
607 const wakeWorkers = () => {
608 for (const resolve of waiters) resolve();
609 waiters.clear();
610 };
611 // A synthesized outcome keeps a trial shard's record (harness reason) so the report reconciles it.
612 const withPlannedTrial = (index: number, outcome: ShardOutcome): ShardOutcome => {
613 const key = shards[index].length === 1 ? normalizeRelativePath(shards[index][0]!) : '';
614 const plan = options.trials?.[key];
615 return plan && shardCaseId(key) !== null && shardTrial(key) !== null
616 ? { ...outcome, trial: classifyTrialShard(outcome, shardCaseId(key)!, shardTrial(key)!, plan, { records: [], contract: null }) }
617 : outcome;
618 };
619 const progress = () => options.onProgress?.(outcomes.map((outcome, index) =>
620 running.has(index) ? withPlannedTrial(index, { ...outcome, status: 'timed-out', sliceDeadline: 'hung' })
621 : outcome.status === 'never-started' && !outcome.sliceDeadline ? withPlannedTrial(index, { ...outcome, sliceDeadline: 'not_run' }) : outcome));
622 const worker = async (): Promise<void> => {
623 while (true) {
624 // Cancellation (SIGINT/SIGTERM) must stop the RUN: the signal
625 // forwarders kill in-flight children, and this guard stops the pool
626 // from launching replacement shards that would keep burning API spend.
627 if (isTerminationRequested()) return;
628 if (pending.length === 0) return;
629 // ENG-2: past the slice deadline nothing starts; each remaining shard is an explicit not_run (INFRA).
630 if (options.sliceDeadlineMs !== undefined && Date.now() >= options.sliceDeadlineMs) {
631 for (const index of pending.splice(0)) {
632 outcomes[index] = withPlannedTrial(index, { ...outcomes[index]!, sliceDeadline: 'not_run' });
633 console.error(`[test:paid] shard ${index + 1} NOT RUN: the slice deadline passed before it started (${shards[index].join(' ')})`);
634 }
635 wakeWorkers();
636 progress();
637 return;
638 }
639 const position = pending.findIndex(index => !shards[index].some(isOverlayTestFile)
640 || activeOverlayShards < OVERLAY_MAX_ACTIVE_SHARDS);
641 if (position < 0) {
642 await new Promise<void>(resolve => waiters.add(resolve));
643 continue;
644 }
645 const [index] = pending.splice(position, 1);
646 const overlay = shards[index].some(isOverlayTestFile);
647 if (overlay) activeOverlayShards++;
648 running.add(index);
649 progress();
650 try {
651 outcomes[index] = await runPaidShard(shards[index], index + 1, shards.length, { ...options, jobs });
652 } catch (error) {
653 const runnerError = error instanceof Error ? error.message : String(error);
654 const failed: ShardOutcome = {
655 shard: index + 1,
656 files: shards[index],
657 status: 'failed',
658 exitCode: null,
659 elapsedMs: 0,
660 groupPid: null,
661 executedTests: null,
662 skippedTests: null,
663 runnerError,
664 };
665 outcomes[index] = withPlannedTrial(index, failed);
666 console.error(`[test:paid] shard ${index + 1} could not run: ${error instanceof Error ? error.message : String(error)}`);
667 } finally {
668 if (overlay) activeOverlayShards--;
669 running.delete(index);
670 wakeWorkers();
671 progress();
672 }
673 }
674 };
676 await Promise.all(Array.from({ length: Math.min(jobs, shards.length) }, worker));
677 return summarize(outcomes);
678}
680export function formatSummary(summary: RunSummary): string[] {
681 const lines = [
682 '',
683 `[test:paid] ${summary.executed}/${summary.total} shards executed — `
684 + `${summary.passed} passed, ${summary.failed} failed, `
685 + `${summary.timedOut} timed out, ${summary.neverStarted} never started, `
686 + `${summary.skippedByDiff} skipped by diff`,
687 ];
688 for (const outcome of summary.outcomes) {
689 // A pass whose every test skipped is labeled distinctly: it exited 0 but
690 // verified NOTHING (codex/gemini files on hosts without the binary).
691 // Status stays 'passed' — availability of an external service is not a
692 // repo regression — but the census must never read it as coverage.
693 const allSkipped = isAllSkippedPass(outcome) ? ` ⚠ all ${outcome.executedTests} tests SKIPPED — verified nothing` : '';
694 lines.push(
695 ` ${outcome.status.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s `
696 + outcome.files.join(' ') + allSkipped,
697 );
698 }
699 return lines;
700}
702type CliOptions = {
703 tier: PaidTier;
704 profile: PaidProfile;
705 profileExplicit: boolean;
706 listOnly: boolean;
707 skipJudges: boolean;
708 timeoutMs: number;
709 timeoutExplicit: boolean;
710 jobs: number;
711 withinShardConcurrency: number;
712 maxFilesPerShard: number;
713 /** Planner mode: write the run manifest here and exit. */
714 emitPlanPath: string | null;
715 /** Slice count for --emit-plan. */
716 slices: number;
717 /** Budget mode for --emit-plan / --list: per-executor estimated wall. */
718 sliceBudgetMs: number | null;
719 jobsExplicit: boolean;
720 /** Executor mode: consume this manifest... */
721 planPath: string | null;
722 /** ...running only this 1-based slice. */
723 sliceIndex: number | null;
724 /** Report mode: reconcile manifest.json + slice-*.json under this dir. */
725 reportDir: string | null;
726 /** Report mode: merge executed shard wall times into the duration seed. */
727 writeDurations: boolean;
728 /** With --report: plan this attempt's rule-16 rerun into this directory instead of reporting. */
729 rerunPlanDir: string | null;
730 /** Planner: the workflow matrix cap, for the capacity preflight's wave count. */
731 maxParallel: number | null;
732 /** Local diagnosis: run one case through the panel runner CI uses (never read by CI). */
733 caseId: string | null;
734 trials: number | null;
735};
737function parsePositiveInt(value: string | undefined, flag: string): number {
738 const parsed = Number.parseInt(value ?? '', 10);
739 if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} needs a positive integer. Received: ${value}`);
740 return parsed;
741}
743function validatedTier(value: string | undefined, source: string): PaidTier {
744 if (value === undefined || value === '') return DEFAULT_TIER;
745 // A typo'd EVALS_TIER (e.g. 'e2e', the tier string eval-store uses) would
746 // otherwise cast through unchecked, match nothing in the runtime E2E_TIERS
747 // filter, self-skip every test, and exit 0 with all shards 'passed' — the
748 // exact 0%-execution-looks-like-a-pass class this runner exists to kill.
749 if (!PAID_TIERS.includes(value as PaidTier)) {
750 throw new Error(`${source} must be gate, periodic or marathon. Received: ${value}`);
751 }
752 return value as PaidTier;
753}
756const PAID_USAGE = `Usage: bun run scripts/test-paid-shards.ts [flags]
758Local runs (paid; needs ANTHROPIC_API_KEY):
759 --tier gate|periodic|marathon tier to run (default: EVALS_TIER or gate)
760 --profile pr|full pr = diff-selected PR gate; full = the tier census
761 --list print the shard plan and exit (free)
762 --slice-budget SECS --jobs N with --list: preview the CI slice plan (free)
763 --jobs N shard processes at once (EVALS_JOBS; default ${DEFAULT_JOBS})
764 --timeout SECS explicit per-shard wall (default: registered or 1800)
765 --files-per-shard N files per shard (full profile only)
766 --case ID [--trials N] run one case through the CI panel runner (add --list to preview)
768CI modes (the eval workflows):
769 --emit-plan PATH (--slices K | --slice-budget SECS --jobs N) [--skip-judges] [--max-parallel N]
770 --plan PATH --slice I execute one slice of a manifest
771 --report DIR [--write-durations] reconcile slice results against the manifest
772 --report DIR --rerun-plan OUT plan this attempt's one rule-16 rerun of its failed cases (scripts/lib/paid-rerun.ts)
774Background runs and CI dispatch: bun run eval:bg:<pr|gate|periodic|release> (scripts/eval-bg.ts --help).`;
776export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process.env): CliOptions {
777 const options: CliOptions = {
778 tier: validatedTier(env.EVALS_TIER, 'EVALS_TIER'),
779 profile: validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE'),
780 profileExplicit: !!env.EVALS_PROFILE,
781 listOnly: false,
782 skipJudges: false,
783 timeoutExplicit: !!env.EVALS_SHARD_TIMEOUT_MS,
784 timeoutMs: env.EVALS_SHARD_TIMEOUT_MS
785 ? parsePositiveInt(env.EVALS_SHARD_TIMEOUT_MS, 'EVALS_SHARD_TIMEOUT_MS')
786 : DEFAULT_SHARD_TIMEOUT_MS,
787 // EVALS_JOBS = shard process count. EVALS_CONCURRENCY deliberately does
788 // NOT set jobs anymore — it's bun's within-shard --max-concurrency (its
789 // legacy meaning). Conflating them turned "EVALS_CONCURRENCY=15" into 15
790 // parallel Bun processes each spawning claude.
791 jobs: env.EVALS_JOBS ? parsePositiveInt(env.EVALS_JOBS, 'EVALS_JOBS') : DEFAULT_JOBS,
792 withinShardConcurrency: env.EVALS_CONCURRENCY
793 ? parsePositiveInt(env.EVALS_CONCURRENCY, 'EVALS_CONCURRENCY')
794 : DEFAULT_WITHIN_SHARD_CONCURRENCY,
795 maxFilesPerShard: DEFAULT_MAX_FILES_PER_SHARD,
796 emitPlanPath: null,
797 slices: 1,
798 sliceBudgetMs: null,
799 jobsExplicit: !!env.EVALS_JOBS,
800 planPath: null,
801 sliceIndex: null,
802 reportDir: null,
803 writeDurations: false,
804 rerunPlanDir: null,
805 maxParallel: null,
806 caseId: null,
807 trials: null,
808 };
810 const pathValue = (message: string, assign: (value: string) => void) => (next: () => string | undefined) => {
811 const value = next();
812 if (!value) throw new Error(message);
813 assign(value);
814 };
815 parseCliFlags(argv, {
816 '--list': () => { options.listOnly = true; },
817 '--tier': (next) => { options.tier = validatedTier(next() ?? '-', '--tier'); },
818 '--profile': pathValue('--profile needs pr or full', (value) => {
819 options.profile = validatedProfile(value, '--profile'); options.profileExplicit = true;
820 }),
821 '--timeout': (next) => { options.timeoutMs = parsePositiveInt(next(), '--timeout') * 1000; options.timeoutExplicit = true; },
822 '--jobs': (next) => { options.jobs = parsePositiveInt(next(), '--jobs'); options.jobsExplicit = true; },
823 '--slice-budget': (next) => { options.sliceBudgetMs = parsePositiveInt(next(), '--slice-budget') * 1000; },
824 '--files-per-shard': (next) => { options.maxFilesPerShard = parsePositiveInt(next(), '--files-per-shard'); },
825 '--emit-plan': pathValue('--emit-plan needs a file path', (value) => { options.emitPlanPath = value; }),
826 '--skip-judges': () => { options.skipJudges = true; },
827 '--slices': (next) => { options.slices = parsePositiveInt(next(), '--slices'); },
828 '--plan': pathValue('--plan needs a manifest path', (value) => { options.planPath = value; }),
829 '--slice': (next) => { options.sliceIndex = parsePositiveInt(next(), '--slice'); },
830 '--report': pathValue('--report needs a directory', (value) => { options.reportDir = value; }),
831 '--write-durations': () => { options.writeDurations = true; },
832 '--rerun-plan': pathValue('--rerun-plan needs a directory', (value) => { options.rerunPlanDir = value; }),
833 '--max-parallel': (next) => { options.maxParallel = parsePositiveInt(next(), '--max-parallel'); },
834 '--case': (next) => {
835 const value = next();
836 if (!value || !Object.hasOwn(E2E_TIERS, value)) throw new Error(`--case needs a live E2E case id. Received: ${value}`);
837 options.caseId = value;
838 },
839 '--trials': (next) => { options.trials = parsePositiveInt(next(), '--trials'); },
840 }, PAID_USAGE);
841 if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report');
842 if (options.rerunPlanDir && (!options.reportDir || options.writeDurations)) throw new Error('--rerun-plan requires --report and writes no durations');
843 if (options.trials !== null && options.caseId === null) throw new Error('--trials requires --case');
844 if (options.caseId !== null && (options.emitPlanPath || options.planPath || options.reportDir || options.sliceIndex !== null)) {
845 throw new Error('--case is local diagnosis; it cannot combine with --emit-plan, --plan/--slice or --report');
846 }
847 if (options.sliceBudgetMs !== null && argv.includes('--slices')) throw new Error('Plan with exactly one of --slices or --slice-budget');
848 if (options.sliceBudgetMs !== null && !options.jobsExplicit) throw new Error('--slice-budget needs explicit --jobs (or EVALS_JOBS): the plan packs and supervises for that worker count');
849 if (options.skipJudges && (!options.emitPlanPath || options.tier !== 'gate')) throw new Error('--skip-judges applies only to an emitted gate census plan');
850 if (options.profile === 'pr' && options.tier !== 'gate') throw new Error('PR profile requires gate tier');
851 if (options.profile === 'pr' && options.maxFilesPerShard !== 1) throw new Error('PR profile requires one file per shard to preserve case accounting');
852 return options;
853}
855async function main(): Promise<number> {
856 const options = parseCliOptions(process.argv.slice(2));
857 const timeoutOverride = options.timeoutExplicit ? options.timeoutMs : undefined;
859 // ── Planner mode: compute selection + the slice plan ONCE, write it, exit.
860 if (options.emitPlanPath) {
861 const manifest = buildRunManifest({
862 tier: options.tier,
863 profile: options.profile,
864 ...(options.sliceBudgetMs !== null ? { sliceBudgetMs: options.sliceBudgetMs, jobs: options.jobs } : { sliceCount: options.slices }),
865 timeoutMs: options.timeoutExplicit ? options.timeoutMs : undefined,
866 evalsAll: process.env.EVALS_ALL === '1',
867 skipJudges: options.skipJudges,
868 });
869 fs.mkdirSync(path.dirname(path.resolve(options.emitPlanPath)), { recursive: true });
870 fs.writeFileSync(options.emitPlanPath, `${JSON.stringify(manifest, null, 2)}\n`);
871 const planned = manifest.entries.filter((e) => e.status === 'planned').length;
872 const skipped = manifest.entries.filter((e) => e.status === 'skipped-by-diff').length;
873 const excludedCount = manifest.entries.filter((e) => e.status === 'excluded').length;
874 console.log(
875 `[test:paid] plan: tier=${manifest.tier} profile=${manifest.profile ?? 'full'} evalsAll=${manifest.evalsAll} — `
876 + `${planned} planned across ${manifest.sliceCount} slice(s), ${skipped} skipped by diff, `
877 + `${excludedCount} excluded (${manifest.selectionReason})`,
878 );
879 for (const line of formatSlicePlan(manifest)) console.log(line);
880 for (const line of formatCapacityPreflight(manifest, options.maxParallel ?? undefined)) console.log(line);
881 // Planner-side reuse: ship ONE filtered receipt set with the plan, so
882 // every trial of a panel (on any slice) sees the same receipts.
883 if (manifest.profile === 'pr' && process.env.EVALS_CACHE_DIR) {
884 const shipped = selectPlanReceipts(process.env.EVALS_CACHE_DIR, path.join(path.dirname(path.resolve(options.emitPlanPath)), 'receipts'));
885 console.log(`[test:paid] reuse: shipped ${shipped.shipped} receipt(s) with the plan; blocked ${shipped.blocked.length} (newer FAIL or partial panel)`);
886 }
887 return 0;
888 }
890 // ── Report mode: reconcile slice artifacts against the manifest. Fail-closed:
891 // a slice whose artifact never landed is a FAILURE, not an absence.
892 if (options.reportDir && options.rerunPlanDir) {
893 const attempt = Number(process.env.GITHUB_RUN_ATTEMPT);
894 const { lines } = planRerun(options.reportDir, Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1, options.rerunPlanDir);
895 for (const line of lines) console.log(`- ${line}`);
896 return 0;
897 }
898 if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations });
900 if (options.caseId && options.listOnly) {
901 const { file, mode, reason } = caseSelection(options.caseId);
902 const plan = caseTrialPlan(options.caseId);
903 const n = options.trials ?? plan.panel.n;
904 console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), selects ${reason}, list only`);
905 for (let trial = 1; trial <= n; trial++) console.log(` ${mode === 'file' ? `${file} (trial ${trial})` : trialShardKey(file, options.caseId, trial)}`);
906 return 0;
907 }
908 if (options.caseId) {
909 preflightAnthropicApi(process.env);
910 const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, runShards: runPaidShards,
911 withinShardConcurrency: options.withinShardConcurrency, timeoutMs: timeoutOverride,
912 evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir() });
913 return verdict.status === 'PASS' ? 0 : 1;
914 }
916 const discovered = collectPaidTestFiles();
917 if (discovered.length === 0) throw new Error('No paid test files were discovered.');
919 // ── Executor mode: consume the planner's manifest; never self-select.
920 if (options.planPath || options.sliceIndex !== null) {
921 if (!options.planPath || options.sliceIndex === null) {
922 throw new Error('--plan and --slice must be used together');
923 }
924 const manifest = parseRunManifest(fs.readFileSync(options.planPath, 'utf-8'));
925 if (manifest.tier !== options.tier) {
926 throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`);
927 }
928 if ((manifest.host ?? 'claude') !== resolveEvalHost()) {
929 throw new Error(`manifest host ${manifest.host ?? 'claude'} != EVALS_HOST ${resolveEvalHost()} — refusing a cross-host run`);
930 }
931 const profile = manifest.profile ?? 'full';
932 if (options.profileExplicit && options.profile !== profile) throw new Error(`manifest profile ${profile} != requested profile ${options.profile}`);
933 if (options.sliceIndex > manifest.sliceCount) {
934 throw new Error(`--slice ${options.sliceIndex} exceeds manifest sliceCount ${manifest.sliceCount}`);
935 }
936 if (manifest.plan && options.jobs !== manifest.plan.jobs) {
937 throw new Error(`manifest was packed for ${manifest.plan.jobs} worker(s) per slice; EVALS_JOBS=${options.jobs} would break its supervision bound`);
938 }
939 const mine = sliceExecutionOrder(manifest.entries.filter((e) => e.status === 'planned' && e.slice === options.sliceIndex));
940 const shards = mine.map((e) => [e.file]);
941 for (const files of shards) resolvePaidShardTimeoutMs(files, timeoutOverride);
942 console.log(`[test:paid] slice ${options.sliceIndex}/${manifest.sliceCount}: ${shards.length} shard(s), tier=${manifest.tier}, evalsAll=${manifest.evalsAll}`);
944 if (options.listOnly) {
945 for (const [index, files] of shards.entries()) {
946 const budget = resolvePaidShardBudget(files, timeoutOverride);
947 console.log(` shard ${index + 1}/${shards.length}: ${files.join(' ')} wall=${budget.timeoutMs}ms source=${budget.source} policy=${budget.policyId ?? 'none'} retries=${retriesForFiles(files)}`);
948 }
949 return 0;
950 }
952 const evalDirBase = process.env.GSTACK_EVAL_DIR || getProjectEvalDir();
953 const trials = Object.fromEntries(mine.filter(entry => entry.trial).map(entry => [normalizeRelativePath(entry.file), entry.trial!]));
954 const exclusionPatterns = Object.fromEntries(mine.filter(entry => entry.excludeCases).map(entry => [entry.file,
955 manifest.prCoverage?.mode === 'pr' ? prProfileTestNamePattern(entry.file, manifest.selection!, entry.excludeCases)
956 : excludedCasesNamePattern(entry.excludeCases!)]));
957 const startedAt = Date.now();
958 const attempt = Number(process.env.GITHUB_RUN_ATTEMPT);
959 const ceilingMinutes = manifest.plan?.sliceCiTimeoutMinutes?.[options.sliceIndex - 1] ?? manifest.plan?.ciTimeoutMinutes;
960 const deadlineMs = ceilingMinutes === undefined ? undefined
961 : sliceDeadlineMs(ceilingMinutes, process.env[SLICE_JOB_STARTED_AT_ENV], startedAt);
962 if (deadlineMs !== undefined) console.log(`[test:paid] slice deadline ${new Date(deadlineMs).toISOString()} (job ceiling ${ceilingMinutes}m minus ${SLICE_UPLOAD_RESERVE_MS / 60_000}m upload reserve)`);
963 fs.mkdirSync(evalDirBase, { recursive: true });
964 const sliceResultPath = path.join(evalDirBase, `slice-${options.sliceIndex}.json`);
965 const writeSliceResult = (outcomes: ShardOutcome[], checkpoint: boolean) => {
966 const sliceResult: SliceResult = {
967 version: 1,
968 tier: manifest.tier,
969 profile,
970 ...(manifest.selection ? { selection: manifest.selection } : {}),
971 sliceIndex: options.sliceIndex!,
972 sliceCount: manifest.sliceCount,
973 ...(options.timeoutExplicit ? { timeoutOverrideMs: options.timeoutMs } : {}),
974 attempt: Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1,
975 startedAt,
976 finishedAt: Date.now(),
977 ...(checkpoint ? { checkpoint: true as const } : {}),
978 ...(manifest.rerun ? { rerun: true as const } : {}),
979 outcomes: outcomes.map(({ files, status, exitCode, elapsedMs, executedTests, skippedTests, budget, reused, runnerError, trial, inputKey, sliceDeadline }) =>
980 ({ files, status, exitCode, elapsedMs, executedTests, skippedTests, ...(budget ? { budget } : {}), ...(reused ? { reused } : {}),
981 ...(runnerError !== undefined ? { runnerError } : {}), ...(trial ? { trial } : {}), ...(inputKey ? { inputKey } : {}),
982 ...(sliceDeadline ? { sliceDeadline } : {}) })),
983 };
984 const temporary = `${sliceResultPath}.tmp-${process.pid}`;
985 fs.writeFileSync(temporary, `${JSON.stringify(sliceResult, null, 2)}\n`);
986 fs.renameSync(temporary, sliceResultPath);
987 };
988 // Checkpoint after every shard start/finish: if the job ceiling or a
989 // cancellation ends the job, the always() upload still carries finished
990 // outcomes; in-flight shards read as hung, unstarted ones as not_run.
991 const checkpoint = (outcomes: ShardOutcome[]) => writeSliceResult(outcomes, true);
992 let summary: RunSummary;
993 if (shards.length === 0) {
994 summary = summarize([]);
995 } else {
996 preflightAnthropicApi(process.env);
997 summary = await runPaidShards(shards, {
998 trials,
999 casePatterns: exclusionPatterns,
1000 timeoutMs: options.timeoutExplicit ? options.timeoutMs : undefined,
1001 jobs: options.jobs,
1002 withinShardConcurrency: options.withinShardConcurrency,
1003 registeredBudgets: Object.fromEntries(mine.filter(entry => entry.budget).map(entry => [normalizeRelativePath(entry.file), entry.budget!])),
1004 ...(manifest.prCoverage?.mode === 'pr' ? {
1005 expectedCases: Object.fromEntries(mine.map(entry => [entry.file, expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)])),
1006 casePatterns: Object.fromEntries(mine.map(entry => [entry.file, prProfileTestNamePattern(entry.file, manifest.selection!, entry.excludeCases)])),
1007 expectedCaseIds: Object.fromEntries(mine.map(entry => [entry.file, prProfileShardIds(entry.file, manifest.selection!, entry.excludeCases)])),
1008 // A rule-16 rerun executes fresh: a receipt would replay the verdict it is rerunning.
1009 reuseFor: manifest.rerun || e2eReuseLaneProblem(process.env, manifest.prCoverage.mode) !== null ? undefined : (files, env, budget) => {
1010 const key = files[0]!;
1011 const file = shardFile(key);
1012 if (files.length !== 1 || !/^test\/skill-e2e-/.test(file)) return null;
1013 const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(ROOT, file), 'utf8'));
1014 const exclude = mine.find(entry => entry.file === key)?.excludeCases;
1015 return prepareE2EShardReuse({ root: ROOT, key, file, caseIds: prProfileShardIds(key, manifest.selection!, exclude),
1016 ...(trials[normalizeRelativePath(key)] ? { panel: trials[normalizeRelativePath(key)] } : {}),
1017 registeredIds: registered, registrationKnown: known,
1018 casePattern: prProfileTestNamePattern(key, manifest.selection!, exclude), expectedCases: expectedPrCaseCount(key, manifest.selection!, exclude),
1019 retries: retriesForFiles(files), timeoutMs: budget.timeoutMs, withinShardConcurrency: options.withinShardConcurrency,
1020 tier: manifest.tier, profile, env });
1021 },
1022 } : {}),
1023 env: {
1024 ...process.env,
1025 EVALS: '1',
1026 EVALS_TIER: options.tier,
1027 EVALS_ALL: manifest.evalsAll ? '1' : '',
1028 EVALS_PREFLIGHT_OK: '1',
1029 // The manifest IS the selection: children must not re-derive a
1030 // possibly-different one from their own git view.
1031 ...paidSelectionEnv(profile, manifest.selection ?? { e2e: null, judges: null }, `manifest slice ${options.sliceIndex}: ${manifest.selectionReason}`),
1032 },
1033 evalDirBase,
1034 sliceDeadlineMs: deadlineMs,
1035 onProgress: checkpoint,
1036 });
1037 }
1038 const guarded = guardTrialRecords(applyHollowShardGuard(summary.outcomes, { evalsAll: manifest.evalsAll, requireExecuted: manifest.prCoverage?.mode === 'pr' }));
1039 summary = summarize(guarded);
1040 writeSliceResult(guarded, false);
1041 console.log(`[test:paid] slice result: ${sliceResultPath}`);
1042 for (const line of formatSummary(summary)) console.log(line);
1043 for (const outcome of guarded.filter(outcome => outcome.trial)) {
1044 const t = outcome.trial!;
1045 console.log(` trial ${t.case} t${t.trial}/${t.panel.n}: ${t.outcome ?? `NO RECORD (${t.harness})`}${t.failure_class ? ` [${t.failure_class}]` : ''}`);
1046 }
1047 return sliceExitCode(guarded, { caseVerdicts: manifest.tier === 'gate' });
1048 }
1050 if (options.listOnly && options.sliceBudgetMs !== null) {
1051 const manifest = buildRunManifest({ tier: options.tier, profile: options.profile, sliceBudgetMs: options.sliceBudgetMs,
1052 jobs: options.jobs, evalsAll: process.env.EVALS_ALL === '1', timeoutMs: timeoutOverride });
1053 console.log(`[test:paid] slice plan preview: tier=${manifest.tier} profile=${manifest.profile ?? 'full'} (${manifest.selectionReason})`);
1054 for (const line of formatSlicePlan(manifest)) console.log(line);
1055 return 0;
1056 }
1058 const tierSelection = selectPaidTestFiles(discovered, options.tier);
1059 const caseKeys = partitionCaseExclusions(expandCaseShards(tierSelection.selected, options.tier));
1060 const selected = tierSelection.selected;
1061 const excluded = [...tierSelection.excluded, ...caseKeys.excluded];
1062 // Same panels as CI: isolated cases run as trial shards, their file shard excludes them.
1063 const expansion = expandTrialShards(caseKeys.runnable, options.tier);
1064 const shards = planPaidShards(expansion.keys, { maxFilesPerShard: options.maxFilesPerShard,
1065 ownShard: new Set(Object.keys(expansion.excludeCases)) });
1067 // Parent-side diff selection (D9): skip whole shards whose mapped tests are
1068 // all unselected. Fail-open everywhere — the child's self-skip stays
1069 // authoritative for anything the mapper can't attribute.
1070 const cases = computePaidCaseSelection({ profile: options.profile });
1071 const fast = cases.coverage?.mode === 'pr';
1072 const excludeOf = (file: string) => expansion.excludeCases[normalizeRelativePath(file)] ?? [];
1073 const profileShards = fast ? shards.filter(files => files.some(file => prProfileFileSelected(file, cases.selection, excludeOf(file)))) : shards;
1074 const { runnable, skipped } = partitionShardsByDiffSelection(profileShards,
1075 cases.selection.e2e === null ? null : new Set(cases.selection.e2e), { excludeCases: expansion.excludeCases });
1076 if (fast) for (const files of shards) {
1077 if (!files.some(file => prProfileFileSelected(file, cases.selection, excludeOf(file)))) skipped.push({ files, reason: 'Outside the fast PR profile; retained in broad coverage' });
1078 }
1079 const selectedCount = cases.selection.e2e?.length ?? Object.keys(E2E_TOUCHFILES).length;
1080 console.log(
1081 `[test:paid] selection: profile=${options.profile} selected ${selectedCount} of ${Object.keys(E2E_TOUCHFILES).length} tests -> `
1082 + `running ${runnable.length} of ${shards.length} shards, reason: ${cases.reason}`,
1083 );
1084 console.log(
1085 `[test:paid] tier=${options.tier}: ${selected.length}/${discovered.length} files, `
1086 + `${shards.length} shards, jobs=${options.jobs}, ${options.timeoutExplicit ? 'explicit' : 'ordinary default'} wall=${Math.round(options.timeoutMs / 1000)}s; per-shard policies below`,
1087 );
1089 if (options.listOnly) {
1090 const skipReasons = new Map(skipped.map((s) => [s.files.join(' '), s.reason]));
1091 for (let index = 0; index < shards.length; index += 1) {
1092 const key = shards[index].join(' ');
1093 const note = skipReasons.has(key) ? ` [would skip: ${skipReasons.get(key)}]` : '';
1094 const budget = resolvePaidShardBudget(shards[index], options.timeoutExplicit ? options.timeoutMs : undefined);
1095 console.log(` shard ${index + 1}/${shards.length}: ${key} wall=${budget.timeoutMs}ms source=${budget.source} policy=${budget.policyId ?? 'none'} retries=${retriesForFiles(shards[index])}${note}`);
1096 }
1097 if (excluded.length > 0) {
1098 console.log(`\nExcluded (${excluded.length}):`);
1099 for (const { file, reason } of excluded) console.log(` - ${file} [${reason}]`);
1100 }
1101 return 0;
1102 }
1104 // One preflight ping in the parent; children skip theirs via the env flag.
1105 // Before this, every shard's e2e-helpers module load re-pinged the API —
1106 // ~30 paid claude -p calls (30s timeout each) per full run for one bit of
1107 // information. A dead API now fails here, before any shard spawns.
1108 // Nothing runnable → nothing to ping.
1109 for (const files of runnable) resolvePaidShardTimeoutMs(files, timeoutOverride);
1110 if (runnable.length > 0) preflightAnthropicApi(process.env);
1112 const runSummary = await runPaidShards(runnable, {
1113 // Tier reaches the children only via EVALS_TIER below; the runtime
1114 // E2E_TIERS filter inside each child is the real selection mechanism.
1115 timeoutMs: options.timeoutExplicit ? options.timeoutMs : undefined,
1116 jobs: options.jobs,
1117 withinShardConcurrency: options.withinShardConcurrency,
1118 trials: expansion.trials,
1119 casePatterns: Object.fromEntries(Object.entries(expansion.excludeCases).map(([file, ids]) => [file, excludedCasesNamePattern(ids)])),
1120 ...(fast ? {
1121 expectedCases: Object.fromEntries(runnable.flat().map(file => [file, expectedPrCaseCount(file, cases.selection, excludeOf(file))])),
1122 casePatterns: Object.fromEntries(runnable.flat().map(file => [file, prProfileTestNamePattern(file, cases.selection, excludeOf(file))])),
1123 } : {}),
1124 env: {
1125 ...process.env,
1126 EVALS: '1',
1127 EVALS_TIER: options.tier,
1128 EVALS_PREFLIGHT_OK: '1',
1129 // The parent's selection, computed once above — children's e2e-helpers
1130 // module load adopts it instead of re-deriving per shard (which spawned
1131 // a bun subprocess per child on the touchfiles-data map-diff path).
1132 // Children fall back to local derivation on any parse failure.
1133 ...paidSelectionEnv(options.profile, cases.selection, cases.reason),
1134 },
1135 evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
1136 });
1137 const skippedOutcomes: ShardOutcome[] = skipped.map((s, index) => ({
1138 shard: runnable.length + index + 1,
1139 files: s.files,
1140 status: 'skipped-by-diff',
1141 exitCode: null,
1142 elapsedMs: 0,
1143 groupPid: null,
1144 executedTests: null,
1145 skippedTests: null,
1146 }));
1147 const guardedOutcomes = guardTrialRecords(applyHollowShardGuard(runSummary.outcomes, {
1148 evalsAll: process.env.EVALS_ALL === '1',
1149 requireExecuted: fast,
1150 }));
1151 const summary = summarize([...guardedOutcomes, ...skippedOutcomes]);
1152 for (const line of formatSummary(summary)) console.log(line);
1153 // The same verdict rule as the CI report: one panelVerdict() per isolated case.
1154 const panels = new Map<string, ShardTrialRecord[]>();
1155 for (const outcome of guardedOutcomes) {
1156 const panel = outcome.trial ? trialPanelKey(outcome.files[0]!) : null;
1157 if (panel) panels.set(panel, [...(panels.get(panel) ?? []), outcome.trial!]);
1158 }
1159 let panelRed = false;
1160 for (const [key, records] of panels) {
1161 const plan = expansion.trials[`${key}~t1`]!;
1162 const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined,
1163 trials: records.filter(r => r.outcome !== null).map(r => ({ trial: r.trial, outcome: r.outcome!,
1164 ...(r.failure_class ? { failure_class: r.failure_class } : {}), ...(r.exit_reason ? { exit_reason: r.exit_reason } : {}),
1165 ...(r.error ? { error: r.error } : {}) })) });
1166 if (verdict.status !== 'PASS' || verdict.split) console.log(` ${formatPanelLine({ ...verdict, file: shardFile(key), slices: {} }, options.tier)}`);
1167 panelRed ||= verdict.failsLane;
1168 }
1169 return sliceExitCode([...guardedOutcomes, ...skippedOutcomes]) || (panelRed ? 1 : 0);
1170}
1172if (import.meta.main) {
1173 try {
1174 process.exitCode = await main();
1175 } catch (error) {
1176 console.error(`[test:paid] ${error instanceof Error ? error.message : String(error)}`);
1177 process.exitCode = 1;
1178 }
1179}