e2e-shard-reuse.ts
377 lines · scripts/e2e-shard-reuse.ts at 3f79d81
1/**
2 * Verified first-attempt reuse for PR-lane E2E shards, on the same receipts as
3 * the workflow-judge reuse (scripts/eval-input-cache.ts).
4 *
5 * A PR-profile shard (a file, or one case of a case-sharded file) is reused
6 * only when every consumed input is byte-identical to a fresh pass recorded in
7 * this PR within the receipt age:
8 * - files: the test file's literal import closure (helpers, fixtures loaded
9 * as modules, installed packages), every tracked file matched by the
10 * touchfile patterns of every case the file registers, the global
11 * touchfiles, the paid runner and this module, the workflow and its setup
12 * actions, bun.lock and the CI Dockerfile;
13 * - prompts: the test source that builds each selected case's prompt;
14 * - parameters: case ids, name pattern, expected count, retries, wall,
15 * within-shard concurrency, tier/profile, the root package without its
16 * release label, and every EVALS_/GSTACK_/CLAUDE_/ANTHROPIC_/... variable
17 * the child receives (secrets contribute presence only);
18 * - runtime: the immutable CI image manifest, Bun, Node, OS/arch and the
19 * Claude CLI version.
20 * Anything unknown fails closed: a computed case registration, a touchfile
21 * pattern matching no tracked file, retries other than zero (a retried pass
22 * cannot prove its first attempt), custom preload/endpoints, missing scope,
23 * or a lane other than the PR gate. Failures are never stored; the weekly
24 * census, marathon and release lanes always execute fresh.
25 */
26import * as fs from 'node:fs';
27import * as path from 'node:path';
28import { spawnSync } from 'node:child_process';
29import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure, storeEvalInputCache,
30 type EvalCacheValue, type EvalInputIdentity, type EvalPassingProof } from './eval-input-cache';
31import { matchGlob } from '../test/helpers/test-selection';
32import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data';
33import { EVAL_CACHE_MAX_AGE_MS as RECEIPT_MAX_AGE_MS } from './eval-input-cache';
34import { panelVerdict, TRIAL_ENV, type EvalCaseKind, type PanelShape, type PanelTrial } from '../test/helpers/eval-store';
36export interface E2EShardReuseRequest {
37 root: string;
38 /** Shard key: `<file>` or `<file>#<case id>`. */
39 key: string;
40 file: string;
41 /** Selected case ids this shard executes (the PR profile's exact expectation). */
42 caseIds: string[];
43 /** Every E2E id the file registers; all of their touchfiles are consumed inputs. */
44 registeredIds: string[];
45 registrationKnown: boolean;
46 casePattern: string;
47 expectedCases: number;
48 retries: number;
49 timeoutMs: number;
50 withinShardConcurrency: number;
51 tier: string;
52 profile: string;
53 /** The exact environment the child receives. */
54 env: NodeJS.ProcessEnv;
55 /** Isolated trial shard: its panel policy is part of the identity; the trial index is run-scoped. */
56 panel?: { kind: EvalCaseKind; panel: PanelShape; quarantined: boolean };
57}
59export interface E2EShardReuseHit { key: string; source: EvalPassingProof['source'] }
61const HARNESS_FILES = ['scripts/test-paid-shards.ts', 'scripts/e2e-shard-reuse.ts', 'scripts/eval-input-cache.ts',
62 'lib/eval-model.ts', 'bun.lock', '.github/docker/Dockerfile.ci', '.github/workflows/evals.yml',
63 '.github/actions/fix-bun-temp/action.yml', '.github/actions/restore-deps/action.yml',
64 '.github/actions/seed-claude-config/action.yml', '.github/actions/register-gstack-skills/action.yml'];
65const ENV_PREFIXES = ['EVALS_', 'GSTACK_', 'CLAUDE_', 'ANTHROPIC_', 'OPENAI_', 'GEMINI_', 'BUN_', 'NODE_', 'PLAYWRIGHT_'];
66/** Run-scoped values: provenance or transport, never behavior. Selection is bound as case ids. */
67const RUN_SCOPED_ENV = new Set(['EVALS_RUN_ID', 'GSTACK_EVAL_DIR', 'EVALS_CACHE_DIR', 'EVALS_CACHE_PR', 'EVALS_CACHE_REPOSITORY',
68 'EVALS_CACHE_RUNTIME_ID', 'EVALS_CACHE_PURPOSE', 'EVALS_SELECTION_JSON', 'EVALS_JUDGE_SELECTION_JSON', TRIAL_ENV.trial]);
69const SECRET_ENV = /KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/;
71/** The reuse-relevant environment the child sees; secrets contribute presence only. */
72export function e2eReuseEnvironment(env: NodeJS.ProcessEnv): Record<string, string> {
73 return Object.fromEntries(Object.keys(env).sort()
74 .filter(name => (ENV_PREFIXES.some(prefix => name.startsWith(prefix)) || ['PATH', 'HOME'].includes(name)) && !RUN_SCOPED_ENV.has(name))
75 .map(name => [name, SECRET_ENV.test(name) ? 'set' : env[name] ?? '']));
76}
78/** Why reuse cannot apply to this lane or environment, else null. */
79export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string | undefined): string | null {
80 const pr = Number(env.EVALS_CACHE_PR);
81 if (profileMode !== 'pr') return 'Only the fast PR profile reuses results';
82 if (!env.EVALS_CACHE_DIR || !env.EVALS_CACHE_REPOSITORY || !Number.isSafeInteger(pr) || pr <= 0) return 'No trusted same-PR cache scope';
83 if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity';
84 if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested';
85 if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh';
86 if ((env.EVALS_HOST ?? '').trim() && env.EVALS_HOST!.trim() !== 'claude') return 'Non-Claude hosts execute fresh';
87 if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source';
88 if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint';
89 return null;
90}
92function trackedFiles(root: string): string[] {
93 const listed = spawnSync('git', ['ls-files', '-z'], { cwd: root, encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 * 1024 });
94 if (listed.status !== 0) throw new Error('Cannot list tracked files');
95 return listed.stdout.split('\0').filter(Boolean);
96}
98/**
99 * Every repository file one shard consumes: the test's import closure plus the
100 * harness closure, and every tracked file the registered cases' touchfiles and
101 * the global touchfiles match. Throws when a pattern matches nothing (unknown).
102 */
103export function e2eShardInputFiles(request: Pick<E2EShardReuseRequest, 'root' | 'file' | 'registeredIds'>): string[] {
104 const tracked = trackedFiles(request.root);
105 const declared = new Set<string>();
106 for (const pattern of new Set([...request.registeredIds.flatMap(id => E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES])) {
107 const matches = tracked.filter(file => matchGlob(file, pattern));
108 if (!matches.length) throw new Error(`Touchfile pattern matches no tracked file: ${pattern}`);
109 for (const file of matches) declared.add(file);
110 }
111 const closure = sourceDependencyClosure(request.root, [request.file, ...HARNESS_FILES]);
112 return [...new Set([...closure, ...declared])].filter(file => file !== 'package.json').sort();
113}
115/** The consumed input identity of one PR shard, or why it is ineligible. */
116export function e2eShardIdentity(request: E2EShardReuseRequest): { status: 'eligible'; identity: EvalInputIdentity } | { status: 'ineligible'; reason: string } {
117 try {
118 if (!/^test\/skill-e2e-.+\.test\.ts$/.test(request.file) || /overlay-harness/.test(request.file)) return { status: 'ineligible', reason: 'Not an audited E2E file' };
119 if (!request.registrationKnown || !request.registeredIds.length) return { status: 'ineligible', reason: 'Case registration is not statically complete' };
120 if (!request.caseIds.length || request.caseIds.length !== request.expectedCases || request.caseIds.some(id => !request.registeredIds.includes(id))) {
121 return { status: 'ineligible', reason: 'Selected cases are not exactly known' };
122 }
123 if (request.retries !== 0) return { status: 'ineligible', reason: 'A retried pass cannot prove its first attempt' };
124 const files = e2eShardInputFiles(request);
125 const { version: _releaseLabel, ...rootPackage } = JSON.parse(fs.readFileSync(path.join(request.root, 'package.json'), 'utf8'));
126 const source = fs.readFileSync(path.join(request.root, request.file), 'utf8');
127 const env = request.env;
128 const result = buildEvalInputIdentity({
129 root: request.root,
130 scope: { repository: env.EVALS_CACHE_REPOSITORY!, pullRequest: Number(env.EVALS_CACHE_PR) },
131 coverage: { dependencies: 'complete', prompts: 'complete', environment: 'complete' }, unknownDependencies: [],
132 files,
133 prompts: Object.fromEntries(request.caseIds.map(id => [id, source])),
134 parameters: { rootPackage, key: request.panel ? request.key.replace(/~t\d+$/, '') : request.key,
135 ...(request.panel ? { panel: { kind: request.panel.kind, n: request.panel.panel.n, k: request.panel.panel.k, quarantined: request.panel.quarantined } } : {}), caseIds: [...request.caseIds].sort(), casePattern: request.casePattern,
136 expectedCases: request.expectedCases, retries: request.retries, timeoutMs: request.timeoutMs,
137 withinShardConcurrency: request.withinShardConcurrency, tier: request.tier, profile: request.profile,
138 environment: e2eReuseEnvironment(env) },
139 runtime: { image: env.EVALS_CACHE_RUNTIME_ID!, bun: Bun.version, node: process.versions.node,
140 platform: process.platform, arch: process.arch, claudeCli: claudeCliVersion(env) },
141 });
142 return result;
143 } catch (error) {
144 return { status: 'ineligible', reason: error instanceof Error ? error.message : 'Cannot identify consumed inputs' };
145 }
146}
148function claudeCliVersion(env: NodeJS.ProcessEnv): string {
149 const version = spawnSync('claude', ['--version'], { encoding: 'utf8', timeout: 5_000, env });
150 const line = version.status === 0 ? version.stdout.split('\n')[0]!.trim() : '';
151 if (!line) throw new Error('Claude CLI version is unknown');
152 return line;
153}
155const validResult = (identity: EvalInputIdentity, key: string) => (value: EvalCacheValue) =>
156 !!value && typeof value === 'object' && !Array.isArray(value)
157 && value.key === key && JSON.stringify(value.cases) === JSON.stringify(identity.caseIds);
159/**
160 * Prepare reuse for one shard: `lookup` returns a verified receipt for these
161 * exact inputs; `publish` stores a receipt after a fresh first-attempt pass
162 * whose inputs did not change during execution.
163 */
164export function prepareE2EShardReuse(request: E2EShardReuseRequest): {
165 /** The input identity key: recorded on the outcome so the report can store verdicts against it. */
166 inputKey: string;
167 lookup(): E2EShardReuseHit | null;
168 /** Trial shards only: this trial's record from a whole PASS panel receipt of the plan's receipts. */
169 lookupPanelTrial(trial: number): { hit: E2EShardReuseHit; trial: PanelTrial } | null;
170 /** True when the inputs are unchanged since `before` (the outcome may carry inputKey). */
171 unchanged(): boolean;
172 publish(): void;
173 /**
174 * A completed single-trial shard that did not pass: write its negative
175 * receipt at the execution boundary (CEO-25), beside the pass receipts the
176 * slice artifact carries, so a cancelled run's FAIL still blocks an older
177 * PASS for the same inputs when its report never runs.
178 */
179 publishFailure(): void;
180} | null {
181 if (e2eReuseLaneProblem(request.env, 'pr') !== null) return null;
182 const before = e2eShardIdentity(request);
183 if (before.status !== 'eligible') return null;
184 const common = { cacheDir: request.env.EVALS_CACHE_DIR!, purpose: 'gate' as const };
185 return {
186 inputKey: before.identity.key,
187 unchanged() {
188 const after = e2eShardIdentity(request);
189 return after.status === 'eligible' && after.identity.key === before.identity.key;
190 },
191 lookupPanelTrial(trial) {
192 if (!request.panel) return null;
193 const receipt = readPanelReceipt(common.cacheDir, before.identity.key);
194 if (!receipt || receipt.case !== request.caseIds[0] || receipt.kind !== request.panel.kind
195 || receipt.panel.n !== request.panel.panel.n || receipt.panel.k !== request.panel.panel.k) return null;
196 const record = receipt.trials.find(t => t.trial === trial);
197 return record ? { hit: { key: receipt.key, source: receipt.source }, trial: record } : null;
198 },
199 lookup() {
200 const found = lookupEvalInputCache({ ...common, identity: before.identity, validateResult: validResult(before.identity, request.key) });
201 return found.status === 'reused' ? { key: found.key, source: found.source } : null;
202 },
203 publish() {
204 const after = e2eShardIdentity(request);
205 const source = executionSource(request);
206 if (after.status !== 'eligible' || !source) return;
207 storeEvalInputCache({ ...common, before: before.identity, after: after.identity, proof: {
208 execution: 'new', finalized: true, completeAttemptHistory: true, exitCode: 0, timedOut: false,
209 cancelled: false, skipped: 0, failed: 0, passed: before.identity.caseIds.length,
210 cases: before.identity.caseIds.map(id => ({ id, outcome: 'passed' as const, attempt: 1 as const })),
211 source, result: { key: request.key, cases: before.identity.caseIds },
212 } });
213 },
214 publishFailure() {
215 const source = executionSource(request);
216 if (source) writeNegativeReceipt(common.cacheDir, { schema: 1, key: before.identity.key, source });
217 },
218 };
219}
221/** Provenance of a receipt written by this execution: `<run id>/<attempt>`, the checked-out revision and now. */
222function executionSource(request: E2EShardReuseRequest): { runId: string; revision: string; completedAt: number } | null {
223 const env = request.env;
224 const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}/${env.GITHUB_RUN_ATTEMPT ?? '1'}` : env.EVALS_RUN_ID;
225 const revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: request.root, encoding: 'utf8', timeout: 3_000 });
226 if (!runId || revision.status !== 0) return null;
227 return { runId, revision: revision.stdout.trim(), completedAt: Date.now() };
228}
230// ─── Panel receipts, negative receipts and the planner's receipt selection ──
231//
232// Reuse is decided by the planner, once per panel: it ships the plan a
233// receipt set in which every panel receipt is a whole PASS panel from one run
234// and no pass receipt has a newer FAIL for the same identity. Executors look
235// up only that set, so every trial of a panel sees the same receipts. The
236// report writes panel receipts (all n trials fresh, one identity) and
237// negative receipts (FAIL verdicts) after the verdict is known.
239export interface PanelReceipt {
240 schema: 1;
241 key: string;
242 case: string;
243 kind: EvalCaseKind;
244 panel: PanelShape;
245 trials: PanelTrial[];
246 source: { runId: string; revision: string; completedAt: number };
247}
249export interface NegativeReceipt { schema: 1; key: string; source: { runId: string; revision: string; completedAt: number } }
251const RECEIPT_KEY = /^[a-f0-9]{64}$/;
252const RECEIPT_FILE = /^[a-f0-9]{64}(?:\.panel|\.fail)?\.json$/;
253const validSource = (source: any) => !!source && typeof source.runId === 'string' && /^[\w./-]{1,160}$/.test(source.runId)
254 && typeof source.revision === 'string' && /^[a-f0-9]{40}$/.test(source.revision) && Number.isSafeInteger(source.completedAt) && source.completedAt > 0;
256function readJson(file: string, maxBytes = 64 * 1024): any {
257 try {
258 const stat = fs.lstatSync(file);
259 if (!stat.isFile() || stat.size > maxBytes) return null;
260 return JSON.parse(fs.readFileSync(file, 'utf8'));
261 } catch { return null; }
262}
264/** A whole, unexpired PASS panel receipt for `key`, re-verified with panelVerdict(); else null. */
265export function readPanelReceipt(cacheDir: string, key: string, now = Date.now()): PanelReceipt | null {
266 if (!RECEIPT_KEY.test(key)) return null;
267 const receipt = readJson(path.join(cacheDir, `${key}.panel.json`));
268 if (!receipt || receipt.schema !== 1 || receipt.key !== key || typeof receipt.case !== 'string' || !validSource(receipt.source)
269 || receipt.source.completedAt > now || now - receipt.source.completedAt >= RECEIPT_MAX_AGE_MS || !Array.isArray(receipt.trials)) return null;
270 try {
271 const verdict = panelVerdict({ case: receipt.case, kind: receipt.kind, panel: receipt.panel,
272 trials: receipt.trials.map((t: PanelTrial) => ({ ...t, attempt: 1 })) });
273 if (verdict.status !== 'PASS' || verdict.trials.length !== receipt.panel.n) return null;
274 } catch { return null; }
275 const negative = readJson(path.join(cacheDir, `${key}.fail.json`));
276 if (negative && validSource(negative.source) && negative.source.completedAt >= receipt.source.completedAt) return null;
277 return receipt as PanelReceipt;
278}
280export function writePanelReceipt(dir: string, receipt: PanelReceipt): void {
281 fs.mkdirSync(dir, { recursive: true });
282 fs.writeFileSync(path.join(dir, `${receipt.key}.panel.json`), `${JSON.stringify(receipt)}\n`, { mode: 0o600 });
283}
285export function writeNegativeReceipt(dir: string, receipt: NegativeReceipt): void {
286 fs.mkdirSync(dir, { recursive: true });
287 fs.writeFileSync(path.join(dir, `${receipt.key}.fail.json`), `${JSON.stringify(receipt)}\n`, { mode: 0o600 });
288}
290const receiptSource = (file: string): { runId: string; completedAt: number } => {
291 const parsed = readJson(file);
292 const source = parsed?.source ?? parsed?.proof?.source;
293 return { runId: typeof source?.runId === 'string' ? source.runId : '', completedAt: Number(source?.completedAt) || 0 };
294};
295const receiptTime = (file: string): number => receiptSource(file).completedAt;
297/**
298 * Receipt order (ENG-6): within one CI run, a later attempt is newer
299 * whatever its clock says; otherwise the later `completedAt` is newer.
300 * Positive when `a` is newer than `b`, zero on a tie.
301 */
302export function compareReceiptSources(a: { runId: string; completedAt: number }, b: { runId: string; completedAt: number }): number {
303 const [runA, attemptA] = a.runId.split('/');
304 const [runB, attemptB] = b.runId.split('/');
305 if (runA && runA === runB && Number(attemptA) !== Number(attemptB)) return (Number(attemptA) || 0) - (Number(attemptB) || 0);
306 return a.completedAt - b.completedAt;
307}
309/**
310 * Planner-side selection: copy `from` into `to`, dropping every pass or panel
311 * receipt that has a same-or-newer negative receipt for its identity, and
312 * every panel receipt that is not a whole PASS panel. Workflow-judge and
313 * other receipts pass through for their own validation at lookup.
314 */
315export function selectPlanReceipts(from: string, to: string, now = Date.now()): { shipped: number; blocked: string[] } {
316 fs.mkdirSync(to, { recursive: true });
317 const blocked: string[] = [];
318 let shipped = 0;
319 let names: string[] = [];
320 try { names = fs.readdirSync(from).filter(name => RECEIPT_FILE.test(name)); } catch { return { shipped, blocked }; }
321 for (const name of names) {
322 const file = path.join(from, name);
323 const [key, suffix] = [name.slice(0, 64), name.slice(64)];
324 const negative = RECEIPT_KEY.test(key) ? readJson(path.join(from, `${key}.fail.json`)) : null;
325 const newerFail = negative && validSource(negative.source) && negative.source.completedAt >= receiptTime(file);
326 if (suffix === '.panel.json' && (newerFail || !readPanelReceipt(from, key, now))) { blocked.push(name); continue; }
327 if (suffix === '.json' && newerFail) { blocked.push(name); continue; }
328 fs.copyFileSync(file, path.join(to, name));
329 shipped++;
330 }
331 return { shipped, blocked };
332}
334/**
335 * Merge receipt directories into one store, keeping the newest file per name
336 * by (run attempt, completedAt); a tie keeps the file already there. Negative
337 * receipts merge before passes, so a reader of a partial merge never sees a
338 * PASS without the FAIL that blocks it. The recovery checkpoint carries over.
339 */
340export function mergeReceiptDirs(out: string, dirs: string[]): number {
341 fs.mkdirSync(out, { recursive: true });
342 let merged = 0;
343 const negativesFirst = (a: string, b: string) => Number(b.endsWith('.fail.json')) - Number(a.endsWith('.fail.json')) || (a < b ? -1 : a > b ? 1 : 0);
344 for (const dir of dirs) {
345 let names: string[] = [];
346 try { names = fs.readdirSync(dir).filter(name => RECEIPT_FILE.test(name)).sort(negativesFirst); } catch { continue; }
347 for (const name of names) {
348 const source = path.join(dir, name);
349 const target = path.join(out, name);
350 if (!fs.lstatSync(source).isFile()) continue;
351 if (fs.existsSync(target) && compareReceiptSources(receiptSource(source), receiptSource(target)) <= 0) continue;
352 fs.copyFileSync(source, target);
353 merged++;
354 }
355 // The recovery checkpoint (scripts/recover-receipts.ts) travels with the store; the newest wins.
356 const checkpoint = path.join(dir, 'recovery.json');
357 const current = path.join(out, 'recovery.json');
358 if (fs.existsSync(checkpoint) && fs.lstatSync(checkpoint).isFile()
359 && (Number(readJson(checkpoint)?.updatedAt) || 0) > (fs.existsSync(current) ? Number(readJson(current)?.updatedAt) || 0 : -1)) {
360 fs.copyFileSync(checkpoint, current);
361 }
362 }
363 return merged;
364}
366if (import.meta.main) {
367 const [command, first, ...rest] = process.argv.slice(2);
368 if (command === 'select' && first && rest[0]) {
369 const result = selectPlanReceipts(first, rest[0]);
370 console.log(`[e2e-reuse] shipped ${result.shipped} receipt(s) to the plan; blocked ${result.blocked.length} (newer FAIL or partial panel)`);
371 } else if (command === 'merge' && first) {
372 console.log(`[e2e-reuse] merged ${mergeReceiptDirs(first, rest)} receipt(s) into ${first}`);
373 } else {
374 console.error('usage: bun run scripts/e2e-shard-reuse.ts select <from> <to> | merge <out> <dir...>');
375 process.exit(2);
376 }
377}