shard-engine.ts
753 lines · scripts/lib/shard-engine.ts at 3f79d81
1/**
2 * Shard engine: the one owner of how a test shard runs — process-group spawn,
3 * wall timeout + group kill, signal forwarding, strict Bun output verdicts,
4 * per-shard tmp/Chromium sandbox, log files, duration seeds, CLI flag loop.
5 * Lanes (scripts/test-free-shards.ts, scripts/test-paid-shards.ts) keep only
6 * policy: selection, budgets, manifests, retries, and a LanePolicy, e.g.
7 * const LANE: LanePolicy = { acceptsSeedDuration: ms => ms > 0, zeroExecution: () => 'passed' };
8 * const result = await runShardChild({ command, args, cwd, env, timeoutMs, hookStreams });
9 * Enforced by ratchet (c) (test/module-size-ratchet.test.ts). Moved from
10 * scripts/test-strict-output.ts, which re-exports this module.
11 */
13import { spawn, type ChildProcess } from 'node:child_process';
14import { StringDecoder } from 'node:string_decoder';
15import * as fs from 'node:fs';
16import * as os from 'node:os';
17import * as path from 'node:path';
19const ROOT = path.resolve(import.meta.dir, '..', '..');
20// Strict Bun-test output classification works around a Bun test runner bug
21// where failures can be printed even though the child exits successfully:
22// output is forwarded byte-for-byte as it arrives, only complete Bun result
23// lines and terminal summaries are classified, and `strictTestExitCode`
24// refuses a zero exit when the output shows failures or fewer files ran.
25const ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
26const BUN_FAIL_RESULT = /^(?:\(fail\)|✗) (.+) \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
27const BUN_BETWEEN_TESTS_ERROR = '# Unhandled error between tests';
28const BUN_TERMINAL_SUMMARY = /^Ran (\d+) tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
29// The counts block bun prints just before the terminal summary (" 1 pass",
30// " 2 skip", " 0 fail"). "Ran N tests" COUNTS skipped tests, so N alone
31// cannot distinguish a shard that verified work from one whose every test
32// self-skipped (external-service binary missing, tier mismatch) — the
33// green-by-skip class. Anchored to whole-line matches; nested bun-test
34// children can still contribute counts (same known limit as the terminal
35// summary — see the last-summary-anchoring TODO in the audit).
36const BUN_SKIP_COUNT = /^\s*(\d+) skip$/;
37const BUN_PASS_COUNT = /^\s*(\d+) pass$/;
38const BUN_FAIL_COUNT = /^\s*(\d+) fail$/;
40export type BunTestOutputFinding = 'failed-test' | 'unhandled-between-tests';
42export interface BunTestOutputSummary {
43 failedTests: number;
44 unhandledBetweenTests: number;
45 terminalFileCounts: number[];
46 /** Test counts from the same terminal lines — feeds the hollow-shard guard. */
47 terminalTestCounts: number[];
48 /** Sum of bun's " N skip" count lines. "Ran N tests" includes skips, so
49 * this is what separates verified work from green-by-skip. */
50 skippedTests: number;
51 /** Sum of bun's " N pass" count lines. */
52 passedTests: number;
53}
55export type ForwardedTerminationSignal = 'SIGINT' | 'SIGTERM';
57export interface TerminationSignalSource {
58 on(event: string, listener: () => void): unknown;
59 off(event: string, listener: () => void): unknown;
60}
62export interface TerminationTimerApi {
63 schedule(callback: () => void, delayMs: number): unknown;
64 cancel(handle: unknown): void;
65}
67export interface ChildSignalForwarding {
68 readonly receivedSignal: ForwardedTerminationSignal | null;
69 dispose(): void;
70}
72const DEFAULT_TERMINATION_TIMER: TerminationTimerApi = {
73 schedule: (callback, delayMs) => setTimeout(callback, delayMs),
74 cancel: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
75};
77/**
78 * Per-source termination bookkeeping, shared across every forwarder bound to
79 * the same source. Installing ANY signal listener suppresses Node's default
80 * terminate-on-SIGINT/SIGTERM, so without this the parent runner survived
81 * cancellation: it killed the current child, then kept LAUNCHING new shards
82 * (observed: paid runs continuing to burn API spend after Ctrl-C). The first
83 * signal now also schedules the parent's own exit after the children's
84 * SIGKILL grace, and runners consult isTerminationRequested() before
85 * launching more work.
86 */
87interface SourceTerminationState {
88 requested: boolean;
89 exitScheduled: boolean;
90}
91const SOURCE_TERMINATION_STATE = new WeakMap<TerminationSignalSource, SourceTerminationState>();
92function terminationStateFor(source: TerminationSignalSource): SourceTerminationState {
93 let state = SOURCE_TERMINATION_STATE.get(source);
94 if (!state) {
95 state = { requested: false, exitScheduled: false };
96 SOURCE_TERMINATION_STATE.set(source, state);
97 }
98 return state;
99}
100export function isTerminationRequested(source: TerminationSignalSource = process): boolean {
101 return SOURCE_TERMINATION_STATE.get(source)?.requested ?? false;
102}
103const signalExitCode = (signal: ForwardedTerminationSignal): number =>
104 128 + (signal === 'SIGINT' ? 2 : 15);
106/**
107 * Bind one active child to the parent's termination lifecycle. SIGINT and
108 * SIGTERM get a grace period so Bun can clean up; a repeated signal, timeout,
109 * or synchronous parent exit uses SIGKILL so the child cannot be orphaned.
110 * The parent itself exits shortly after the grace window (or immediately on
111 * a repeated signal) — cancellation must terminate the RUN, not just the
112 * currently-running children.
113 */
114export function installChildSignalForwarding(
115 child: Pick<ChildProcess, 'kill'>,
116 source: TerminationSignalSource = process,
117 timer: TerminationTimerApi = DEFAULT_TERMINATION_TIMER,
118 graceMs = 5_000,
119 exitImpl: (code: number) => void = (code) => process.exit(code),
120): ChildSignalForwarding {
121 let receivedSignal: ForwardedTerminationSignal | null = null;
122 let forceTimer: unknown = null;
123 let disposed = false;
125 const scheduleParentExit = (signal: ForwardedTerminationSignal, delayMs: number): void => {
126 const state = terminationStateFor(source);
127 state.requested = true;
128 if (state.exitScheduled) return;
129 state.exitScheduled = true;
130 // Never cancelled by dispose(): once cancellation is requested, the run
131 // is going down even if this particular shard finishes cleanly first.
132 timer.schedule(() => exitImpl(signalExitCode(signal)), delayMs);
133 };
135 const forward = (signal: ForwardedTerminationSignal): void => {
136 if (disposed) return;
137 if (receivedSignal !== null) {
138 child.kill('SIGKILL');
139 scheduleParentExit(signal, 0);
140 return;
141 }
142 receivedSignal = signal;
143 child.kill(signal);
144 forceTimer = timer.schedule(() => {
145 forceTimer = null;
146 child.kill('SIGKILL');
147 }, graceMs);
148 // Exit AFTER the children's SIGKILL grace so the group kills land first.
149 scheduleParentExit(signal, graceMs + 1_000);
150 };
151 const onSigint = () => forward('SIGINT');
152 const onSigterm = () => forward('SIGTERM');
153 const onExit = () => { child.kill('SIGKILL'); };
155 source.on('SIGINT', onSigint);
156 source.on('SIGTERM', onSigterm);
157 source.on('exit', onExit);
159 return {
160 get receivedSignal() {
161 return receivedSignal;
162 },
163 dispose() {
164 if (disposed) return;
165 disposed = true;
166 source.off('SIGINT', onSigint);
167 source.off('SIGTERM', onSigterm);
168 source.off('exit', onExit);
169 if (forceTimer !== null) timer.cancel(forceTimer);
170 forceTimer = null;
171 },
172 };
173}
175/**
176 * SIGKILL the shard's whole process group. Orphaned grandchildren (browsers,
177 * claude sessions) are how a stalled run once burned a core for 15.7 hours.
178 */
179export function killProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void {
180 if (process.platform === 'win32' || typeof child.pid !== 'number') {
181 child.kill(signal);
182 return;
183 }
184 try {
185 process.kill(-child.pid, signal);
186 } catch (err) {
187 const code = (err as NodeJS.ErrnoException).code;
188 if (code === 'ESRCH') return; // group already gone
189 if (code !== 'EPERM') throw err;
190 // Observed on macOS after a SIGKILLed group is reaped: signalling the
191 // now-empty group id returns EPERM, not ESRCH. Throwing here loses the
192 // shard's real outcome (a timeout gets recorded as a failure) and, from
193 // the timeout timer, leaves the shard promise unsettled — a hang, which
194 // is the exact failure class this runner exists to kill. Fall back to the
195 // direct pid so a genuinely-live child is still signalled.
196 try {
197 child.kill(signal);
198 } catch {
199 // Best-effort reap: nothing actionable is left if this fails too.
200 }
201 }
202}
204/**
205 * Strip ANSI escapes and a trailing CR from one output line. Every line
206 * matcher (here and in the free runner's console filter / failure
207 * attribution) MUST match against this form — a prior grep for `(fail)`
208 * lines missed real failures because color codes sat inside the line.
209 */
210export function stripAnsiLine(rawLine: string): string {
211 return rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
212}
214export function classifyBunTestOutputLine(rawLine: string): BunTestOutputFinding | null {
215 const line = stripAnsiLine(rawLine);
216 if (parseBunFailureResult(line) !== null) return 'failed-test';
217 if (line === BUN_BETWEEN_TESTS_ERROR) return 'unhandled-between-tests';
218 return null;
219}
221export function parseBunFailureResult(rawLine: string): string | null {
222 return BUN_FAIL_RESULT.exec(stripAnsiLine(rawLine))?.[1] ?? null;
223}
225export function parseBunTerminalSummaryLine(rawLine: string): number | null {
226 return parseBunTerminalSummary(rawLine)?.files ?? null;
227}
229export function parseBunTerminalSummary(rawLine: string): { tests: number; files: number } | null {
230 const line = stripAnsiLine(rawLine);
231 const match = BUN_TERMINAL_SUMMARY.exec(line);
232 return match
233 ? { tests: Number.parseInt(match[1], 10), files: Number.parseInt(match[2], 10) }
234 : null;
235}
237/**
238 * Incrementally classifies output without assuming process chunks align to
239 * lines. Buffers are PER ORIGIN: stdout and stderr are independent pipes, so
240 * a chunk from one can arrive between two halves of a line from the other.
241 * A single shared buffer would glue those fragments into garbled lines — a
242 * sheared `(fail)` line goes uncounted and a sheared terminal summary reads
243 * as truncation. Counters are shared; only line assembly is per-stream.
244 */
245export type ClassifierOrigin = 'stdout' | 'stderr';
247export class BunFailureSummaryParser {
248 private readonly pending: Partial<Record<ClassifierOrigin, { failures: number | null }>> = {};
250 consume(rawLine: string, origin: ClassifierOrigin): number | null {
251 const line = stripAnsiLine(rawLine);
252 if (BUN_PASS_COUNT.test(line)) {
253 this.pending[origin] = { failures: null };
254 return null;
255 }
256 const pending = this.pending[origin];
257 if (!pending) return null;
258 const fail = BUN_FAIL_COUNT.exec(line);
259 if (fail) {
260 pending.failures = Math.max(pending.failures ?? 0, Number.parseInt(fail[1], 10));
261 return null;
262 }
263 if (parseBunTerminalSummary(line) !== null) {
264 delete this.pending[origin];
265 return pending.failures;
266 }
267 return null;
268 }
269}
271export class BunTestOutputClassifier {
272 private readonly decoders: Record<ClassifierOrigin, StringDecoder> = {
273 stdout: new StringDecoder('utf8'),
274 stderr: new StringDecoder('utf8'),
275 };
276 private pending: Record<ClassifierOrigin, string> = { stdout: '', stderr: '' };
277 private failedTests = 0;
278 private reportedFailedTests = 0;
279 private readonly failureSummary = new BunFailureSummaryParser();
280 private unhandledBetweenTests = 0;
281 private terminalFileCounts: number[] = [];
282 private terminalTestCounts: number[] = [];
283 private skippedTests = 0;
284 private passedTests = 0;
286 write(chunk: Uint8Array | string, origin: ClassifierOrigin = 'stdout'): void {
287 this.pending[origin] += typeof chunk === 'string'
288 ? chunk
289 : this.decoders[origin].write(Buffer.from(chunk));
290 this.consumeCompleteLines(origin);
291 }
293 end(): BunTestOutputSummary {
294 for (const origin of ['stdout', 'stderr'] as const) {
295 this.pending[origin] += this.decoders[origin].end();
296 if (this.pending[origin].length > 0) this.classify(this.pending[origin], origin);
297 this.pending[origin] = '';
298 }
299 return this.summary();
300 }
302 summary(): BunTestOutputSummary {
303 return {
304 failedTests: Math.max(this.failedTests, this.reportedFailedTests),
305 unhandledBetweenTests: this.unhandledBetweenTests,
306 terminalFileCounts: [...this.terminalFileCounts],
307 terminalTestCounts: [...this.terminalTestCounts],
308 skippedTests: this.skippedTests,
309 passedTests: this.passedTests,
310 };
311 }
313 private consumeCompleteLines(origin: ClassifierOrigin): void {
314 let newline = this.pending[origin].indexOf('\n');
315 while (newline !== -1) {
316 this.classify(this.pending[origin].slice(0, newline), origin);
317 this.pending[origin] = this.pending[origin].slice(newline + 1);
318 newline = this.pending[origin].indexOf('\n');
319 }
320 }
322 private classify(line: string, origin: ClassifierOrigin): void {
323 const finding = classifyBunTestOutputLine(line);
324 if (finding === 'failed-test') this.failedTests += 1;
325 if (finding === 'unhandled-between-tests') this.unhandledBetweenTests += 1;
326 const stripped = stripAnsiLine(line);
327 const skip = BUN_SKIP_COUNT.exec(stripped);
328 if (skip !== null) this.skippedTests += Number.parseInt(skip[1], 10);
329 const pass = BUN_PASS_COUNT.exec(stripped);
330 if (pass !== null) this.passedTests += Number.parseInt(pass[1], 10);
331 const fail = this.failureSummary.consume(stripped, origin);
332 if (fail !== null) this.reportedFailedTests = Math.max(this.reportedFailedTests, fail);
333 const terminal = parseBunTerminalSummary(line);
334 if (terminal !== null) {
335 this.terminalFileCounts.push(terminal.files);
336 this.terminalTestCounts.push(terminal.tests);
337 }
338 }
339}
341export function strictTestExitCode(
342 childExitCode: number,
343 summary: BunTestOutputSummary,
344 expectedFiles?: number,
345): number {
346 if (childExitCode !== 0) return childExitCode;
347 if (summary.failedTests > 0 || summary.unhandledBetweenTests > 0) return 1;
348 if (expectedFiles !== undefined && !summary.terminalFileCounts.includes(expectedFiles)) return 1;
349 return 0;
350}
352export function normalizeRelativePath(filePath: string): string {
353 return filePath.replace(/\\/g, '/');
354}
356/**
357 * Bun treats positional test paths as substring filters. Resolve every
358 * canonical relative path before spawning so `test/foo.test.ts` cannot also
359 * select `browse/test/foo.test.ts`.
360 */
361export function exactTestFileSelectors(files: string[], rootDir = ROOT): string[] {
362 return files.map((file) => path.isAbsolute(file) ? path.normalize(file) : path.resolve(rootDir, file));
363}
365export function forwardAndClassify(
366 stream: NodeJS.ReadableStream,
367 destination: NodeJS.WriteStream,
368 classifier: BunTestOutputClassifier,
369 origin: ClassifierOrigin = 'stdout',
370): Promise<void> {
371 return new Promise((resolve, reject) => {
372 let ended = false;
373 const incomplete = () => reject(new Error(`incomplete ${origin} capture: stream closed before end`));
374 stream.on('data', (chunk: Buffer | string) => {
375 classifier.write(chunk, origin);
376 destination.write(chunk);
377 });
378 stream.once('end', () => { ended = true; resolve(); });
379 stream.on('error', reject);
380 stream.once('close', () => { if (!ended) incomplete(); });
381 // Bun can return an already-destroyed pipe whose close event is past.
382 if ('destroyed' in stream && stream.destroyed && !ended) {
383 if ('errored' in stream && stream.errored) reject(stream.errored);
384 else incomplete();
385 }
386 });
387}
389// --- Shared shard-child lifecycle ---
391export interface RunShardChildOptions {
392 command: string;
393 args: string[];
394 cwd: string;
395 env: NodeJS.ProcessEnv;
396 /** External wall-clock deadline; on expiry the child's process GROUP is SIGKILLed. */
397 timeoutMs: number;
398 deadlineMs?: number;
399 /**
400 * Hook the freshly-spawned child's stdout/stderr. Stream POLICY (classifier
401 * tees, log spooling, console forwarding, reporters) is entirely the
402 * caller's. Runs synchronously right after spawn; child close and every
403 * returned promise must settle within the same deadline. A wall-expired
404 * return reports incomplete capture instead of treating the prefix as final.
405 */
406 hookStreams: (child: ChildProcess) => Array<Promise<void>>;
407 /**
408 * Lane-owned companions of the child (the free lane's detached-browser
409 * tracker). Created right after spawn; `signal` runs after every forwarded
410 * group kill, and `settle` runs after the final group kill while parent
411 * signals are still forwarded.
412 */
413 attach?: (child: ChildProcess) => ShardChildCompanion;
414}
416export interface ShardChildCompanion {
417 signal(force: boolean): void;
418 settle(): Promise<void>;
419}
421export interface ShardChildResult {
422 exitCode: number | null;
423 /** True when the shared deadline expired; no further child work is allowed. */
424 timedOut: boolean;
425 /** The child's pid — the process-GROUP id on POSIX (detached spawn). */
426 groupPid: number | null;
427 incompleteCapture?: {
428 childClosed: boolean;
429 pendingStreams: number;
430 failedStreams: number;
431 deadlineMs: number;
432 };
433}
435/**
436 * The child lifecycle both sharded runners need, extracted from
437 * scripts/test-paid-shards.ts runPaidShard (scripts/test-free-shards.ts
438 * runFreeShard duplicates the same ~35 lines verbatim today and is designed
439 * to migrate here in a later change):
440 *
441 * - spawn detached on POSIX so the child owns its process group,
442 * - forward parent SIGINT/SIGTERM to the whole group (not just the child),
443 * - arm an EXTERNAL wall-clock timer that SIGKILLs the group — a spinning
444 * child main thread never fires its own in-process timer,
445 * - in EVERY exit path: disarm the timer, detach the signal forwarder, and
446 * signal group survivors with SIGKILL.
447 *
448 * Caller-side cleanup that must run even on a spawn failure (log streams,
449 * reporters, temp dirs) belongs in the caller's own try/finally around this
450 * call: a spawn 'error' event THROWS from here after the finally block runs,
451 * preserving the runners' existing could-not-run handling.
452 */
453const REAP_GRACE_MS = 250;
455export async function runShardChild(options: RunShardChildOptions): Promise<ShardChildResult> {
456 const deadlineMs = Math.min(options.deadlineMs ?? Infinity, Date.now() + options.timeoutMs);
457 if (!Number.isFinite(deadlineMs)) throw new Error('Shard deadline must be finite');
458 if (deadlineMs <= Date.now()) {
459 return { exitCode: null, timedOut: true, groupPid: null,
460 incompleteCapture: { childClosed: false, pendingStreams: 0, failedStreams: 0, deadlineMs } };
461 }
462 const child = spawn(options.command, options.args, {
463 cwd: options.cwd,
464 env: options.env,
465 stdio: ['ignore', 'pipe', 'pipe'],
466 detached: process.platform !== 'win32',
467 windowsHide: true,
468 });
469 const groupPid = child.pid ?? null;
470 const companion = options.attach?.(child);
471 // Group-kill on parent SIGINT/SIGTERM too, not just on timeout.
472 const forwarding = installChildSignalForwarding({
473 kill: (signal?: NodeJS.Signals | number) => {
474 killProcessGroup(child, (signal as NodeJS.Signals) ?? 'SIGTERM');
475 companion?.signal(signal === 'SIGKILL');
476 return true;
477 },
478 });
480 let timedOut = false;
481 let childClosed = false;
482 let pendingStreams = 0;
483 let failedStreams = 0;
484 let exitCode: number | null = null;
485 let failed = false;
486 let firstError: unknown;
487 const rememberError = (error: unknown) => {
488 if (failed) return;
489 failed = true;
490 firstError = error;
491 };
492 const kill = () => {
493 try { killProcessGroup(child, 'SIGKILL'); }
494 catch (error) { rememberError(error); }
495 };
496 let close!: () => void;
497 const closed = new Promise<void>(resolve => { close = resolve; });
498 let reap!: () => void;
499 const reaped = new Promise<void>(resolve => { reap = resolve; });
500 const onExit = (code: number | null) => { exitCode = code; reap(); };
501 const onClose = (code: number | null) => { exitCode = code; childClosed = true; close(); reap(); };
502 child.once('exit', onExit);
503 child.once('close', onClose);
504 child.on('error', rememberError);
505 let expire!: () => void;
506 const expired = new Promise<void>(resolve => { expire = resolve; });
507 const killTimer = setTimeout(() => {
508 timedOut = true;
509 kill();
510 expire();
511 }, Math.max(0, deadlineMs - Date.now()));
513 try {
514 let streams: Array<Promise<void>> = [];
515 try { streams = options.hookStreams(child); }
516 catch (error) {
517 rememberError(error);
518 kill();
519 child.stdout?.destroy();
520 child.stderr?.destroy();
521 }
522 pendingStreams = streams.length;
523 const drainage = Promise.all(streams.map(stream => Promise.resolve(stream).then(
524 () => { pendingStreams -= 1; },
525 (error: unknown) => { pendingStreams -= 1; failedStreams += 1; rememberError(error); },
526 )));
527 await Promise.race([Promise.all([closed, drainage]), expired]);
528 if (Date.now() >= deadlineMs) timedOut = true;
529 // A wall-killed child is normally reaped within milliseconds; wait that
530 // long (bounded) so callers never observe a live pid after a timeout.
531 let reapTimer: ReturnType<typeof setTimeout> | undefined;
532 await Promise.race([reaped, new Promise<void>(resolve => { reapTimer = setTimeout(resolve, REAP_GRACE_MS); })]);
533 clearTimeout(reapTimer);
534 } finally {
535 clearTimeout(killTimer);
536 kill();
537 if (companion) {
538 try { await companion.settle(); }
539 catch (error) { rememberError(error); }
540 }
541 forwarding.dispose();
542 child.off('exit', onExit);
543 child.off('close', onClose);
544 if (!childClosed || pendingStreams > 0) {
545 child.stdout?.destroy();
546 child.stderr?.destroy();
547 child.unref();
548 }
549 }
550 const result: ShardChildResult = { exitCode, timedOut, groupPid };
551 if (!childClosed || pendingStreams > 0 || failedStreams > 0) {
552 result.incompleteCapture = { childClosed, pendingStreams, failedStreams, deadlineMs };
553 }
554 if (failed) {
555 if (firstError instanceof Error && Object.isExtensible(firstError)) {
556 Reflect.defineProperty(firstError, 'shardResult', { value: result, configurable: true });
557 }
558 throw firstError;
559 }
560 return result;
561}
563// --- Per-shard sandbox, logs, duration seeds, verdicts, CLI flags ---
565/**
566 * Per-shard temp + Chromium-profile isolation. Two concurrent shards on one
567 * profile dir kill each other's browser, and shared tmp cross-contaminates;
568 * a group-SIGKILLed shard never runs its own cleanup, so the lane removes
569 * `stateDir` afterwards (the cleanup backstop). `realpath` resolves a
570 * symlinked tmpdir (macOS /var -> /private/var) for lanes that compare paths.
571 */
572export function createShardSandbox(
573 prefix: string,
574 baseEnv: NodeJS.ProcessEnv,
575 options: { realpath?: boolean } = {},
576): { stateDir: string; tmp: string; env: NodeJS.ProcessEnv } {
577 const created = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
578 const stateDir = options.realpath ? fs.realpathSync(created) : created;
579 const tmp = path.join(stateDir, 'tmp');
580 fs.mkdirSync(tmp);
581 const env: NodeJS.ProcessEnv = {
582 ...baseEnv, TMPDIR: tmp, TEMP: tmp, TMP: tmp,
583 CHROMIUM_PROFILE: path.join(stateDir, 'chromium-profile'),
584 };
585 for (const name of CAPY_LAUNCH_MARKERS) delete env[name];
586 return { stateDir, tmp, env };
587}
589/**
590 * The Capy installer's `GSTACK_LAUNCH` markers. A lane started from a shell
591 * that sourced the launch file would hand them to every product-under-test
592 * child a shard spawns, so fixtures would see an unattended Capy session they
593 * never set up (GSTA-99); each fixture sets its own session kind.
594 */
595export const CAPY_LAUNCH_MARKERS = ['GSTACK_SESSION_KIND', 'GSTACK_CAPY', 'GSTACK_EPHEMERAL', 'GSTACK_INSTALL_VERDICT'] as const;
597/**
598 * Asynchronous best-effort backstop removal: a SIGKILLed shard can leave a
599 * full git workspace plus a Chromium profile, and a synchronous recursive
600 * delete would stall every sibling shard's classification and timers.
601 */
602export async function removeShardSandbox(stateDir: string): Promise<void> {
603 try { await fs.promises.rm(stateDir, { recursive: true, force: true }); }
604 catch { /* a locked file must not turn a real verdict into an exception */ }
605}
607/** Run once per file, alone: readers `jobs` at a time, then exclusive files one by one. Stops on termination. */
608export async function forEachFileAlone(
609 readers: string[], exclusive: string[], jobs: number, run: (file: string, index: number) => Promise<void>,
610): Promise<void> {
611 let cursor = 0;
612 const worker = async (files: string[], offset: number) => {
613 while (cursor < files.length && !isTerminationRequested()) {
614 const index = cursor++;
615 await run(files[index], offset + index);
616 }
617 };
618 await Promise.all(Array.from({ length: Math.max(1, jobs) }, () => worker(readers, 0)));
619 cursor = 0;
620 await worker(exclusive, readers.length);
621}
623let shardLogSequence = 0;
625/** Timestamped log path; pid + sequence defeat same-millisecond collisions. */
626export function nextShardLogPath(directory: string, stem: string): string {
627 const stamp = new Date().toISOString().replace(/[:.]/g, '-');
628 shardLogSequence += 1;
629 return path.join(directory, `${stem}-${stamp}-${process.pid}-${shardLogSequence}.log`);
630}
632export interface ShardLog {
633 readonly path: string;
634 readonly stream: fs.WriteStream;
635 /** Set on the first write error; later chunks are dropped, never thrown. */
636 failed: boolean;
637 write(chunk: Buffer | string): void;
638}
640/** Full-stream capture: every child byte is spooled to disk, never held in RAM. */
641export function openShardLog(logPath: string, label: string, mode?: number): ShardLog {
642 const stream = fs.createWriteStream(logPath, mode === undefined ? undefined : { mode });
643 const log: ShardLog = {
644 path: logPath, stream, failed: false,
645 write(chunk) { if (!log.failed) stream.write(chunk); },
646 };
647 stream.on('error', (err) => {
648 if (log.failed) return;
649 log.failed = true;
650 console.error(`${label} could not write the full log at ${logPath}: ${err.message}`);
651 });
652 return log;
653}
655export type DurationSeedRead =
656 | { status: 'missing' }
657 | { status: 'corrupt'; error: Error }
658 | { status: 'ok'; durations: Record<string, number> };
660/** One reader for `{ durations: { file: ms } }` seeds; the lane decides which values count. */
661export function readDurationSeed(file: string, accepts: (ms: number) => boolean): DurationSeedRead {
662 let raw: string;
663 try { raw = fs.readFileSync(file, 'utf-8'); }
664 catch { return { status: 'missing' }; }
665 try {
666 const parsed = JSON.parse(raw) as { durations?: Record<string, unknown> };
667 return { status: 'ok', durations: Object.fromEntries(Object.entries(parsed.durations ?? {})
668 .filter((entry): entry is [string, number] =>
669 typeof entry[1] === 'number' && Number.isFinite(entry[1]) && accepts(entry[1]))) };
670 } catch (error) {
671 return { status: 'corrupt', error: error as Error };
672 }
673}
675/** Atomic temp+rename: a killed writer never leaves a truncated seed behind. */
676export function writeDurationSeed(file: string, durations: Record<string, number>): void {
677 const payload = {
678 version: 1,
679 recordedAt: new Date().toISOString(),
680 durations: Object.fromEntries(Object.entries(durations).sort(([a], [b]) => (a < b ? -1 : 1))),
681 };
682 const temporary = `${file}.tmp-${process.pid}`;
683 fs.writeFileSync(temporary, `${JSON.stringify(payload, null, 2)}\n`);
684 fs.renameSync(temporary, file);
685}
687/** What a strictly passed shard that executed zero tests becomes. */
688export type ZeroExecutionVerdict = 'passed' | 'passed-with-warning' | 'passed-empty';
690/** Classification rules each lane injects; the engine applies them, never decides them. */
691export interface LanePolicy {
692 /** Which recorded seed durations the lane trusts (free >= 0, paid > 0). */
693 acceptsSeedDuration(ms: number): boolean;
694 /** `promisedAll`: the run promised every test (EVALS_ALL) rather than a selection. */
695 zeroExecution(run: { promisedAll: boolean }): ZeroExecutionVerdict;
696}
698export function zeroExecutionVerdict(
699 executedTests: number | null,
700 policy: LanePolicy,
701 run: { promisedAll: boolean },
702): ZeroExecutionVerdict {
703 return executedTests === 0 ? policy.zeroExecution(run) : 'passed';
704}
706export type StrictShardStatus = 'passed' | 'failed' | 'timed-out';
708/**
709 * The verdict both lanes share: a wall timeout is its own status; otherwise
710 * a shard passes only with complete evidence (log, capture, cleanup) AND a
711 * strict exit of zero, which requires bun's summary to count every planned file.
712 */
713export function strictShardStatus(input: {
714 timedOut: boolean;
715 evidenceComplete: boolean;
716 exitCode: number | null;
717 summary: BunTestOutputSummary;
718 expectedFiles: number;
719}): StrictShardStatus {
720 if (input.timedOut) return 'timed-out';
721 return input.evidenceComplete && strictTestExitCode(input.exitCode ?? 1, input.summary, input.expectedFiles) === 0
722 ? 'passed' : 'failed';
723}
725/** Usage text for `--help`: the lane's own text, else its declared flags. */
726export function cliUsage(handlers: Record<string, unknown>, usage?: string, script = process.argv[1] ?? 'runner'): string {
727 return usage ?? [`Usage: bun run ${path.relative(process.cwd(), script) || script} [flags]`, '', 'Flags:',
728 ...Object.keys(handlers).sort().map(flag => ` ${flag}`), '', 'See docs/TESTING_INTERNALS.md for what each flag does.'].join('\n');
729}
731/**
732 * Shared flag loop. Each lane declares its flags; a handler that takes a
733 * value calls `next()` (the following argv entry, or undefined) and owns its
734 * own validation message. Any undeclared flag is `Unknown argument: <flag>`.
735 * `--help` / `-h` (unless a lane declares them) print the usage and exit 0
736 * before any work starts, so asking for help never runs a suite.
737 */
738export function parseCliFlags(
739 argv: string[],
740 handlers: Record<string, (next: () => string | undefined) => void>,
741 usage?: string,
742): void {
743 for (let index = 0; index < argv.length; index += 1) {
744 const arg = argv[index];
745 if ((arg === '--help' || arg === '-h') && !Object.hasOwn(handlers, arg)) {
746 process.stdout.write(`${cliUsage(handlers, usage)}\n`);
747 process.exit(0);
748 }
749 const handler = Object.hasOwn(handlers, arg) ? handlers[arg] : undefined;
750 if (!handler) throw new Error(`Unknown argument: ${arg}`);
751 handler(() => argv[++index]);
752 }
753}