paid-select.ts
521 lines · scripts/lib/paid-select.ts at 3f79d81
1/**
2 * Paid-lane selection and shard planning: tier classification, diff/PR case selection, shard keys, budgets and
3 * supervision bounds. Moved from scripts/test-paid-shards.ts; imports only leaf modules (no import cycle).
4 */
5import * as fs from 'node:fs';
6import * as path from 'node:path';
7import { spawnSync } from 'node:child_process';
8import { normalizeRelativePath } from './shard-engine';
9import { isPaidTestFile } from '../../test/helpers/paid-test-set';
10import { PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
11import { FILE_RETRY_BUDGETS } from '../../test/helpers/eval-budgets';
12import { OVERLAY_MIN_FILE_WALL_MS } from '../../test/helpers/overlay-case-policy';
13import { packageChangeOnlyVersion, prProfileFileCases, selectPrProfile, type PrProfileSelection } from '../test-pr-profile';
14import { detectBaseBranch, getChangedFiles, selectTests, E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES } from '../../test/helpers/touchfiles';
15import { caseTestNamePattern, fileCaseRegistration, shardCaseId, shardFile, shardTrial } from './paid-cases';
16import { DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_WITHIN_SHARD_CONCURRENCY, PAID_TIERS, ROOT, isOverlayTestFile,
17 type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, type ShardOutcome } from './paid-types';
19/** Compatibility helper for callers that only need the effective wall. */
20export function resolvePaidShardTimeoutMs(files: string[], explicitTimeoutMs?: number): number {
21 return resolvePaidShardBudget(files, explicitTimeoutMs).timeoutMs;
22}
24export function collectPaidTestFiles(rootDir = ROOT): string[] {
25 const testDir = path.join(rootDir, 'test');
26 if (!fs.existsSync(testDir)) return [];
27 return fs.readdirSync(testDir)
28 .map((name) => `test/${name}`)
29 .filter(isPaidTestFile)
30 .sort();
31}
33export interface TierClassification {
34 included: boolean;
35 reason: string;
36}
38/**
39 * Decide whether a paid test file has anything to run in `tier`.
40 *
41 * Per-TEST tier filtering already happens at runtime: test/helpers/e2e-helpers.ts
42 * intersects the selected tests with E2E_TIERS whenever EVALS_TIER is set, and
43 * this runner passes EVALS_TIER down to every shard. So this file-level pass is
44 * only an optimization — skipping a file merely saves one near-instant shard.
45 *
46 * Exclusion is the dangerous direction (a wrongly-skipped gate test is exactly
47 * the invisible-non-execution bug this runner exists to kill), so the only
48 * exclusion evidence accepted is an explicit whole-file tier guard: either the
49 * raw `EVALS_TIER === '<other>'` predicate or the consolidated helper form
50 * `describeE2ETier('<other>')` / `e2eTierEnabled('<other>')` from
51 * test/helpers/e2e-gate.ts (same semantics, read from env at module load).
52 * Inferring a file's tier from which E2E_TIERS names appear in its source
53 * is guesswork that silently drops real work: short keys like 'retro' match
54 * unrelated strings, and LLM-judge tests are keyed off LLM_JUDGE_TOUCHFILES and
55 * carry no E2E_TIERS name at all. Everything without an explicit other-tier
56 * guard runs and self-skips.
57 */
58export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassification {
59 const declares = (candidate: PaidTier) =>
60 new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source) ||
61 new RegExp(`\\b(?:describeE2ETier|e2eTierEnabled)\\(\\s*['"\`]${candidate}['"\`]`).test(source);
63 if (declares(tier)) return { included: true, reason: `declares tier '${tier}'` };
64 const others = PAID_TIERS.filter(candidate => candidate !== tier && declares(candidate));
65 if (others.length) return { included: false, reason: `declares tier ${others.map(other => `'${other}'`).join(' and ')} only` };
66 return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' };
67}
70/**
71 * A file is skipped for a tier lane only when it registers E2E ids, none of
72 * them has that tier, and no literal registration names an id outside its
73 * touchfile registration. A computed registration (a name built at runtime)
74 * can only run in this lane by producing an id of this tier (the child's
75 * EVALS_TIER filter drops every other), so it is skipped too when every id of
76 * the tier is registered to a test file (its touchfile list names one), which
77 * is not this file. An id of the tier that no test file registers, or no id
78 * at all, keeps today's scheduling (the child's runtime filter decides).
79 */
80export function tierSkipReason(
81 file: string, source: string, tier: PaidTier,
82 touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
83 tiers: Record<string, string> = E2E_TIERS,
84): string | null {
85 const { registered, computed, unregistered } = fileCaseRegistration(file, source, touchfiles, tiers);
86 if (registered.length === 0 || unregistered.length > 0 || registered.some(id => tiers[id] === tier)) return null;
87 if (!computed) return `skipped: no E2E_TIERS id has tier ${tier}`;
88 const unowned = Object.keys(tiers).some(id => tiers[id] === tier && !(touchfiles[id] ?? []).some(dep => /^test\/[^/]+\.test\.ts$/.test(dep)));
89 return unowned ? null : `skipped: no E2E_TIERS id has tier ${tier} (its computed names can only produce ${tier} ids other test files register)`;
90}
92/**
93 * The marathon lane selects positively: a file runs there only when it
94 * declares the marathon tier or registers a marathon-tier case. When its source
95 * names some registered ids, only those count: another case's touchfile entry
96 * is a dependency, not a registration (W2f:
97 * plan-decision-classification is a dependency of the marathon
98 * plan-ceo-split-overflow case and planned a hollow marathon shard). Files
99 * without marathon work never cost a marathon runner, and gate/periodic files
100 * never gain a third execution.
101 */
102export function marathonSkipReason(
103 file: string, source: string,
104 touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
105 tiers: Record<string, string> = E2E_TIERS,
106): string | null {
107 if (classifyPaidTestFile(source, 'marathon').reason === "declares tier 'marathon'") return null;
108 const { registered } = fileCaseRegistration(file, source, touchfiles, tiers);
109 // When the source names some of its registered ids, those are its own cases; the rest are cases that merely depend on it.
110 const named = registered.filter(id => knownTestNamesInSource(source, [id]).length > 0);
111 return (named.length ? named : registered).some(id => tiers[id] === 'marathon')
112 ? null : 'skipped: declares no marathon tier and registers no marathon case';
113}
115export interface TierSelection {
116 selected: string[];
117 excluded: Array<{ file: string; reason: string }>;
118}
120export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection {
121 const selected: string[] = [];
122 const excluded: Array<{ file: string; reason: string }> = [];
123 // Scheduled-lane exclusions (documented-red / manual-hardware files): a
124 // known-red weekly shard is triage waste locally AND in CI, so the list
125 // applies to every periodic and marathon run, with the reason surfaced per file.
126 const ciExcluded = (file: string): { reason: string; tracking: string } | undefined =>
127 tier !== 'gate' ? PERIODIC_CI_EXCLUDE[normalizeRelativePath(file)] : undefined;
128 for (const file of files) {
129 const exclusion = ciExcluded(file);
130 if (exclusion) {
131 excluded.push({ file, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` });
132 continue;
133 }
134 const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
135 const classification = classifyPaidTestFile(source, tier);
136 const skip = !classification.included ? null
137 : tier === 'marathon' ? marathonSkipReason(file, source) : tierSkipReason(file, source, tier);
138 if (classification.included && !skip) selected.push(file);
139 else excluded.push({ file, reason: skip ?? classification.reason });
140 }
141 return { selected, excluded };
142}
144// --- Parent-side diff selection (shard skipping) ---
146/**
147 * The test names the parent mapper recognizes: every E2E map key. LLM-judge
148 * keys are deliberately excluded — skill-llm-eval.test.ts is not a
149 * skill-e2e-* file, so it is always kept (child self-skip authoritative).
150 */
151export const PARENT_MAPPER_TEST_NAMES: string[] = [
152 ...new Set([...Object.keys(E2E_TOUCHFILES), ...Object.keys(E2E_TIERS)]),
153];
155/**
156 * Which of `names` appear in `source` as a quoted string ('x', "x", or `x`).
157 * Same class of detection test/e2e-tier-alignment.test.ts uses: exact
158 * quote-delimited match, raw source (comments count — a false hit can only
159 * KEEP a shard, and the registration union below covers constructed names).
160 */
161export function knownTestNamesInSource(source: string, names: Iterable<string>): string[] {
162 const hits: string[] = [];
163 for (const name of names) {
164 if (
165 source.includes(`'${name}'`)
166 || source.includes(`"${name}"`)
167 || source.includes(`\`${name}\``)
168 ) hits.push(name);
169 }
170 return hits;
171}
173export interface PaidDiffSelection {
174 /** null = run everything (EVALS_ALL, or no changes vs base). */
175 selectedNames: Set<string> | null;
176 reason: string;
177 totalTests: number;
178}
180/**
181 * Compute diff selection in the PARENT, mirroring the module-scope selection
182 * block in test/helpers/e2e-helpers.ts exactly: EVALS_ALL → run all;
183 * base = EVALS_BASE || detectBaseBranch || 'main'; empty changed-file union →
184 * run all. (e2e-helpers additionally gates on EVALS=1, which this runner sets
185 * for every child unconditionally, so the parent mirror omits it.)
186 *
187 * getChangedFiles THROWS on git errors (fail-closed) — the children would hit
188 * the same throw at module load, so the parent surfaces it before any shard
189 * spawns.
190 */
191export function computePaidDiffSelection(
192 env: NodeJS.ProcessEnv = process.env,
193 rootDir = ROOT,
194): PaidDiffSelection {
195 const totalTests = Object.keys(E2E_TOUCHFILES).length;
196 if (env.EVALS_ALL) {
197 return { selectedNames: null, reason: 'run-all (EVALS_ALL=1)', totalTests };
198 }
199 const baseBranch = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main';
200 const changedFiles = getChangedFiles(baseBranch, rootDir);
201 if (changedFiles.length === 0) {
202 return { selectedNames: null, reason: `run-all (no changes vs ${baseBranch})`, totalTests };
203 }
204 const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, {
205 baseRef: baseBranch, cwd: rootDir,
206 });
207 return { selectedNames: new Set(selection.selected), reason: selection.reason, totalTests };
208}
210/**
211 * Serialize the parent's diff selection for shard children (EVALS_SELECTION_JSON).
212 *
213 * Children's e2e-helpers module-load path adopts this instead of re-deriving
214 * the selection per shard — which, when touchfiles-data.ts is in the diff,
215 * spawned one bun subprocess PER CHILD to evaluate the old data file (the
216 * map-diff path in test/helpers/test-selection.ts, 20s timeout each; 46-68
217 * redundant children per full run). `selected: null` means run-all, mirroring
218 * PaidDiffSelection.selectedNames. The child-side parser lives in
219 * test/helpers/e2e-helpers.ts (parseEvalsSelectionJson); round-trip parity is
220 * pinned by test/paid-selection-propagation.test.ts.
221 */
222export function serializePaidDiffSelection(selection: PaidDiffSelection): string {
223 return JSON.stringify({
224 version: 1,
225 selected: selection.selectedNames === null ? null : [...selection.selectedNames].sort(),
226 reason: selection.reason,
227 });
228}
230/** Both selectors are computed once; execution consumes the exact persisted IDs. */
231export function computePaidCaseSelection(options: {
232 profile: PaidProfile;
233 env?: NodeJS.ProcessEnv;
234 rootDir?: string;
235 changedFiles?: string[];
236 /** Whether package.json differs from the base only in `version`; computed from git when omitted. */
237 packageVersionOnly?: boolean;
238}): { selection: PaidCaseSelection; reason: string; coverage?: PrProfileSelection } {
239 const env = options.env ?? process.env;
240 const rootDir = options.rootDir ?? ROOT;
241 const baseRef = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main';
242 const files = options.changedFiles ?? (env.EVALS_ALL ? [] : getChangedFiles(baseRef, rootDir));
243 const all = !!env.EVALS_ALL || files.length === 0;
244 const effectiveFiles = files.filter(file => options.profile !== 'pr' || file !== 'package.json' ||
245 !(options.packageVersionOnly ?? packageVersionOnlySinceBase(rootDir, baseRef)));
246 const sourceAliases = options.profile === 'pr' ? existingPromptSourceAliases(effectiveFiles, rootDir) : {};
247 const selectionFiles = [...new Set([...effectiveFiles, ...Object.values(sourceAliases)])];
248 const select = (table: Record<string, string[]>) => all ? null
249 : selectTests(selectionFiles, table, GLOBAL_TOUCHFILES, { baseRef, cwd: rootDir }).selected;
250 const selection = { e2e: select(E2E_TOUCHFILES), judges: select(LLM_JUDGE_TOUCHFILES) };
251 if (options.profile === 'full') return { selection, reason: all ? 'run-all' : 'diff' };
252 const coverage = selectPrProfile({ selectedE2E: selection.e2e, selectedJudges: selection.judges, changedFiles: effectiveFiles, sourceAliases,
253 screenPolicy: env.EVALS_SCREEN_POLICY === 'narrow' ? 'narrow' : 'broad' });
254 if (coverage.needsFullValidation) {
255 throw new Error(`PR profile requires full validation: ${coverage.missingCoverage.join(', ')}. Use --profile full and the relevant periodic cases.`);
256 }
257 return { selection: { e2e: coverage.e2e, judges: coverage.judges }, reason: coverage.reasons.join('; '), coverage };
258}
260export function existingPromptSourceAliases(files: readonly string[], rootDir = ROOT): Record<string, string> {
261 const aliases: Record<string, string> = {};
262 for (const file of files) {
263 if (!file.endsWith('.md')) continue;
264 const template = /^[^/]+\/sections\/preamble\.md$/.test(file) ? file.replace(/sections\/preamble\.md$/, 'SKILL.md.tmpl') : `${file}.tmpl`;
265 try { if (fs.statSync(path.join(rootDir, template)).isFile()) aliases[file] = template; }
266 catch { /* Unknown/generated-only content must keep its own dependency identity. */ }
267 }
268 return aliases;
269}
271function packageVersionOnlySinceBase(rootDir: string, baseRef: string): boolean {
272 try {
273 const options = { cwd: rootDir, encoding: 'utf8' as const, timeout: 10_000, maxBuffer: 1024 * 1024 };
274 const base = spawnSync('git', ['merge-base', baseRef, 'HEAD'], options);
275 const sha = base.stdout?.trim() ?? '';
276 if (base.status !== 0 || !/^[a-f0-9]{40,64}$/.test(sha)) return false;
277 const old = spawnSync('git', ['show', `${sha}:package.json`], options);
278 return old.status === 0 && packageChangeOnlyVersion(old.stdout, fs.readFileSync(path.join(rootDir, 'package.json'), 'utf8'));
279 } catch { return false; }
280}
282/** Only audited per-case files, plus the separately selected judge, enter the fast profile. */
283/** The selected PR-profile case ids a shard key owns (a case key owns at most its own case). */
284/** `exclude`: isolated case ids a file shard leaves to their trial shards. */
285export function prProfileShardIds(key: string, selection: PaidCaseSelection, exclude: readonly string[] = []): string[] {
286 const caseId = shardCaseId(key);
287 return prProfileFileCases(shardFile(key), selection.e2e)
288 .filter(id => (caseId === null || id === caseId) && (selection.e2e === null || selection.e2e.includes(id)) && !exclude.includes(id));
289}
291export function prProfileFileSelected(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): boolean {
292 if (file === 'test/skill-llm-eval.test.ts') return selection.judges === null || selection.judges.length > 0;
293 return prProfileShardIds(file, selection, exclude).length > 0;
294}
296export function expectedPrCaseCount(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): number {
297 if (file === 'test/skill-llm-eval.test.ts') return selection.judges?.length ?? Object.keys(LLM_JUDGE_TOUCHFILES).length;
298 return prProfileShardIds(file, selection, exclude).length;
299}
301export function prProfileTestNamePattern(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): string {
302 const ids = file === 'test/skill-llm-eval.test.ts'
303 ? selection.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES)
304 : prProfileShardIds(file, selection, exclude);
305 if (ids.length === 0) throw new Error(`No selected PR cases for ${file}`);
306 return caseTestNamePattern(ids);
307}
309export function paidSelectionEnv(profile: PaidProfile, selection: PaidCaseSelection, reason: string): NodeJS.ProcessEnv {
310 const encode = (selected: string[] | null) => JSON.stringify({ version: 1, selected, reason });
311 return { EVALS_PROFILE: profile, EVALS_SELECTION_JSON: encode(selection.e2e), EVALS_JUDGE_SELECTION_JSON: encode(selection.judges) };
312}
314export interface ShardSkipDecision {
315 file: string;
316 kept: boolean;
317 reason: string;
318}
320export interface DiffSkipOptions {
321 rootDir?: string;
322 /** Injectable for tests. Throwing reads fail OPEN (shard kept). */
323 readSource?: (file: string) => string;
324 /** Injectable name census (default: PARENT_MAPPER_TEST_NAMES). */
325 allNames?: string[];
326 /** Injectable registration map (default: E2E_TOUCHFILES). */
327 e2eTouchfiles?: Record<string, string[]>;
328 /** File shard -> isolated case ids its trial shards run instead. */
329 excludeCases?: Record<string, string[]>;
330}
332/**
333 * Decide whether a paid test file can be skipped under the current diff
334 * selection. A file's MAPPED names are the union of:
335 * - E2E map keys quoted in its source, and
336 * - E2E map keys whose dep list registers the file (the tier-alignment
337 * mapping) — this covers files whose testNames are constructed rather
338 * than literal.
339 *
340 * FAIL-OPEN by construction: run-all selection, non-skill-e2e paid files
341 * (llm-judge / codex-e2e / routing, keyed off other maps),
342 * unreadable sources, and files with zero mapped names all KEEP their shard —
343 * the child's self-skip stays authoritative. A parent bug may only run
344 * extra work, never drop it.
345 */
346export function diffSkipDecisionForFile(
347 file: string,
348 selectedNames: Set<string> | null,
349 options: DiffSkipOptions = {},
350): ShardSkipDecision {
351 if (selectedNames === null) return { file, kept: true, reason: 'run-all selection' };
352 const caseId = shardCaseId(file);
353 if (caseId !== null) {
354 return selectedNames.has(caseId) ? { file, kept: true, reason: `selected: ${caseId}` } : { file, kept: false, reason: `case ${caseId} not selected` };
355 }
356 const rel = normalizeRelativePath(file);
357 if (!/^test\/skill-e2e-.*\.test\.ts$/.test(rel)) {
358 return { file, kept: true, reason: 'non-skill-e2e paid file — child self-skip authoritative' };
359 }
360 let source: string;
361 try {
362 const read = options.readSource
363 ?? ((f: string) => fs.readFileSync(path.join(options.rootDir ?? ROOT, f), 'utf8'));
364 source = read(file);
365 } catch {
366 return { file, kept: true, reason: 'source unreadable — fail-open' };
367 }
368 const allNames = options.allNames ?? PARENT_MAPPER_TEST_NAMES;
369 const touchfiles = options.e2eTouchfiles ?? E2E_TOUCHFILES;
370 const quoted = knownTestNamesInSource(source, allNames);
371 const registered = Object.keys(touchfiles).filter((k) => touchfiles[k].includes(rel));
372 const isolated = options.excludeCases?.[rel] ?? [];
373 const mapped = [...new Set([...quoted, ...registered])].filter(name => !isolated.includes(name));
374 if (mapped.length === 0) {
375 return { file, kept: true, reason: 'no mappable test names — fail-open, child self-skip authoritative' };
376 }
377 const selectedHere = mapped.filter((n) => selectedNames.has(n));
378 if (selectedHere.length > 0) {
379 const shown = selectedHere.slice(0, 3).join(', ') + (selectedHere.length > 3 ? ', …' : '');
380 return { file, kept: true, reason: `selected: ${shown}` };
381 }
382 return { file, kept: false, reason: `none of its ${mapped.length} mapped test(s) selected` };
383}
385/**
386 * Partition planned shards into runnable vs skipped-by-diff. A shard is
387 * skipped only when EVERY file in it is skippable.
388 */
389export function partitionShardsByDiffSelection(
390 shards: string[][],
391 selectedNames: Set<string> | null,
392 options: DiffSkipOptions = {},
393): { runnable: string[][]; skipped: Array<{ files: string[]; reason: string }> } {
394 if (selectedNames === null) return { runnable: shards, skipped: [] };
395 const runnable: string[][] = [];
396 const skipped: Array<{ files: string[]; reason: string }> = [];
397 for (const shard of shards) {
398 const decisions = shard.map((file) => diffSkipDecisionForFile(file, selectedNames, options));
399 if (decisions.every((d) => !d.kept)) {
400 skipped.push({ files: shard, reason: [...new Set(decisions.map((d) => d.reason))].join('; ') });
401 } else {
402 runnable.push(shard);
403 }
404 }
405 return { runnable, skipped };
406}
408export function planPaidShards(
409 files: string[],
410 options: { maxFilesPerShard?: number; ownShard?: ReadonlySet<string> } = {},
411): string[][] {
412 const size = Math.max(1, options.maxFilesPerShard ?? DEFAULT_MAX_FILES_PER_SHARD);
413 const unique = [...new Set(files.map(normalizeRelativePath))].sort();
414 const shards: string[][] = [];
415 let pending: string[] = [];
416 for (const file of unique) {
417 if (isOverlayTestFile(file) || shardCaseId(file) !== null || FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file))
418 || options.ownShard?.has(file)) {
419 if (pending.length) shards.push(pending);
420 pending = [];
421 shards.push([file]);
422 } else {
423 pending.push(file);
424 if (pending.length === size) { shards.push(pending); pending = []; }
425 }
426 }
427 if (pending.length) shards.push(pending);
428 return shards;
429}
432/** Explicit caller limits win; registered supervision preserves existing attempts. */
433export function resolvePaidShardBudget(files: string[], overrideMs?: number): PaidShardBudget {
434 const finding = FILE_RETRY_BUDGETS.find(budget => files.map(shardFile).includes(budget.file));
435 if (finding && files.length !== 1) throw new Error('Registered retry budget requires its own shard');
436 if (overrideMs !== undefined && (!Number.isSafeInteger(overrideMs) || overrideMs <= 0 || overrideMs > 2_147_483_647)) {
437 throw new Error('Shard timeout must be a finite positive timer-safe integer');
438 }
439 const overlay = files.some(isOverlayTestFile);
440 if (overlay && files.length !== 1) throw new Error('Overlay budget requires its own shard');
441 if (overlay && overrideMs !== undefined && overrideMs < OVERLAY_MIN_FILE_WALL_MS) {
442 throw new Error(`Overlay shard requires at least ${OVERLAY_MIN_FILE_WALL_MS}ms; explicit wall ${overrideMs}ms cannot preserve its work and finalization budget`);
443 }
444 // A registered file's case shard supervises its one case.
445 const registeredMs = finding && shardCaseId(files[0]!) !== null
446 ? finding.caseMs + finding.shardReserveMs : finding?.shardMs;
447 return {
448 timeoutMs: overrideMs ?? (registeredMs ?? (overlay ? OVERLAY_MIN_FILE_WALL_MS : DEFAULT_SHARD_TIMEOUT_MS)),
449 source: overrideMs !== undefined ? 'explicit' : finding ? 'registered' : 'default',
450 policyId: finding?.id ?? null,
451 };
452}
454export function sameBudget(actual: PaidShardBudget | undefined, expected: PaidShardBudget): boolean {
455 return actual?.timeoutMs === expected.timeoutMs && actual.source === expected.source && actual.policyId === expected.policyId;
456}
458export function buildPaidShardArgs(
459 files: string[],
460 timeoutMs: number,
461 maxConcurrency: number = DEFAULT_WITHIN_SHARD_CONCURRENCY,
462 retries?: number,
463): string[] {
464 // Explicit --concurrent/--max-concurrency: the legacy path always set one;
465 // omitting it here made within-shard parallelism differ silently between
466 // the two runners (observed: 1.6x sumdur/wall sharded vs 8x legacy).
467 // Paid evals never retry (retriesForFiles); `--retry 0` is explicit so a
468 // bunfig default can never reintroduce one.
469 return ['test', ...files, '--retry', String(retries ?? 0), '--concurrent', `--max-concurrency=${maxConcurrency}`, `--timeout=${timeoutMs}`];
470}
472/**
473 * Stable per-shard eval-dir slug: test filename sans extension, sanitized.
474 * Stable across runs so each shard baselines against its own prior run.
475 */
476export function shardSlug(files: string[]): string {
477 return files
478 .map((file) => path.basename(shardFile(file)).replace(/\.test\.(?:[cm]?[jt]s|tsx|jsx)$/, '')
479 + (shardCaseId(file) === null ? '' : `--${shardCaseId(file)}`)
480 + (shardTrial(file) === null ? '' : `.t${shardTrial(file)}`))
481 .join('+')
482 .replace(/[^a-zA-Z0-9._+-]/g, '-');
483}
486/**
487 * True when a shard "passed" without verifying anything: every test bun ran
488 * was a skip. Legitimate for external-service files on hosts without the
489 * binary, but it must surface as a census warning, never read as coverage.
490 */
491export function isAllSkippedPass(outcome: Pick<ShardOutcome, 'status' | 'executedTests' | 'skippedTests'>): boolean {
492 return outcome.status === 'passed'
493 && outcome.executedTests !== null
494 && outcome.executedTests > 0
495 && outcome.skippedTests === outcome.executedTests;
496}
498/** Upper bound for one ordered FIFO group with the same admission limit.
499 * At each launch the least-loaded worker has at most total prior work / jobs,
500 * and at most floor(prior files / jobs) files of the largest prior wall.
501 * Both bounds hold when earlier files finish below their ceilings. Overlay
502 * groups must use their separate admission limit, as the runner does.
503 */
504export function paidShardWallUpperBoundMs(files: string[], jobs: number, overrideMs?: number): number {
505 if (!Number.isSafeInteger(jobs) || jobs < 1) throw new Error('Worker count must be a positive integer');
506 let priorWork = 0, priorLargest = 0, bound = 0;
507 files.forEach((file, index) => {
508 const wall = resolvePaidShardTimeoutMs([file], overrideMs);
509 const start = Math.min(priorWork / jobs, Math.floor(index / jobs) * priorLargest);
510 bound = Math.max(bound, start + wall);
511 priorWork += wall;
512 priorLargest = Math.max(priorLargest, wall);
513 });
514 return Math.ceil(bound);
515}
517export function validatedProfile(value: string | undefined, source: string): PaidProfile {
518 if (value === undefined || value === '') return 'full';
519 if (value !== 'pr' && value !== 'full') throw new Error(`${source} must be pr or full. Received: ${value}`);
520 return value;
521}