replay-pr-selection.ts
199 lines · scripts/replay-pr-selection.ts at 3f79d81
1/**
2 * CEO-23 preserved-detection replay for the PR paid lane (W1).
3 *
4 * Replays recorded PR runs' changed files through today's selector and
5 * reports, per push, the recorded mode and case count against the new mode
6 * and case count, the gate cases the new selector no longer runs, and every
7 * MISS: a case that actually failed in that run, still exists, and depends
8 * on the diff (today's touchfiles), yet is not selected now. Acceptance is
9 * zero misses. The replay uses today's dependency graph (ENG-18): cases that
10 * no longer exist are counted separately, never as misses.
11 *
12 * Inputs come from GitHub with `gh` (read-only): each run's `paid-plan`
13 * manifest, its trial-outcomes artifact, and the three-dot compare of the
14 * merge-base with the run's head SHA. Downloads are cached under --cache.
15 *
16 * bun run scripts/replay-pr-selection.ts --runs <runs.json> [--cache <dir>] [--repo garrytan/gstack] [--json]
17 *
18 * <runs.json> is `gh run list --workflow evals.yml --json databaseId,headSha,event,createdAt,conclusion -L 200` output.
19 */
20import * as fs from 'node:fs';
21import * as os from 'node:os';
22import * as path from 'node:path';
23import { spawnSync } from 'node:child_process';
24import { computePaidCaseSelection } from './test-paid-shards';
25import { packageChangeOnlyVersion } from './test-pr-profile';
26import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data';
27import { matchGlob } from '../test/helpers/test-selection';
28import { derivedDependencies } from './pr-dependencies';
29import { knownNonBehaviorFile, PR_PROFILE_MAPS } from './test-pr-profile';
31export interface ReplayInput {
32 runId: number;
33 oldMode: string;
34 oldCases: number;
35 changedFiles: string[];
36 /** The compare API lists at most 300 files; a longer diff is replayed on that prefix and flagged. */
37 truncated: boolean;
38 failedCases: string[];
39 packageVersionOnly: boolean;
40}
42export interface ReplayRow {
43 runId: number;
44 /** Changed files in the replayed diff (the wave/ordinary split reads it). */
45 changed: number;
46 /** Changed files missing from today's tree (deleted or renamed since); their consumers are the live files that still reference them. */
47 vanished: number;
48 oldMode: string;
49 newMode: string;
50 oldCases: number;
51 newCases: number;
52 dropped: string[];
53 misses: string[];
54 retiredFailures: string[];
55 truncated: boolean;
56 error?: string;
57}
59/**
60 * Whether a case depends on any changed file under today's dependency graph:
61 * its touchfiles, the globals, and the derived reference closure
62 * (scripts/pr-dependencies.ts, its global runner closure included). Like
63 * the selector, a touchfile match wins over the audited free-only exemptions,
64 * and an exempt file reaches no case through the derived closure.
65 */
66export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean {
67 const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES];
68 const derived = derivedDependencies(PR_PROFILE_MAPS, path.resolve(import.meta.dir, '..'));
69 // A path absent from today's tree reaches a case through the live files that still reference it.
70 const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file);
71 return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern))
72 || (!knownNonBehaviorFile(file) && via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id))));
73}
75export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow {
76 const tracked = derivedDependencies(PR_PROFILE_MAPS, path.resolve(import.meta.dir, '..')).tracked;
77 const base = { runId: input.runId, oldMode: input.oldMode, oldCases: input.oldCases, truncated: input.truncated,
78 changed: input.changedFiles.length, vanished: input.changedFiles.filter(file => !tracked.has(file)).length };
79 const retiredFailures = input.failedCases.filter(id => !Object.hasOwn(E2E_TIERS, id));
80 try {
81 const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: input.changedFiles, packageVersionOnly: input.packageVersionOnly });
82 const selected = new Set(result.selection.e2e ?? Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate'));
83 const misses = input.failedCases.filter(id => Object.hasOwn(E2E_TIERS, id) && E2E_TIERS[id] === 'gate'
84 && caseTouchesDiff(id, input.changedFiles) && !selected.has(id));
85 return { ...base, newMode: result.coverage?.mode ?? 'full', newCases: selected.size,
86 dropped: oldSelection.filter(id => Object.hasOwn(E2E_TIERS, id) && !selected.has(id)), misses, retiredFailures };
87 } catch (error) {
88 // The PR profile refuses prompts without coverage; CI fails the plan the same way, so nothing is silently dropped.
89 return { ...base, newMode: 'needs-full-validation', newCases: 0, dropped: [], misses: [], retiredFailures,
90 error: error instanceof Error ? error.message.slice(0, 200) : String(error) };
91 }
92}
94/** Pushes whose diff has at least this many files are wave branches (often legitimately the full gate). */
95export const WAVE_DIFF_FILES = 100;
97function segment(label: string, rows: readonly ReplayRow[]): string {
98 const pct = (n: number) => rows.length ? `${Math.round((100 * n) / rows.length)}%` : 'n/a';
99 const count = (mode: string) => rows.filter(row => row.newMode === mode).length;
100 const cases = rows.map(row => row.newCases).sort((a, b) => a - b);
101 const median = cases.length ? cases[Math.floor((cases.length - 1) / 2)] : 0;
102 return `- ${label}: ${rows.length} push(es); full-fallback ${rows.filter(row => row.oldMode === 'full-fallback').length} recorded -> ${count('full-fallback')} (${pct(count('full-fallback'))}) replayed; `
103 + `dependents ${count('dependents')}, pr ${count('pr')}; median selected gate cases ${median} (recorded median ${rows.map(row => row.oldCases).sort((a, b) => a - b)[Math.floor((rows.length - 1) / 2)] ?? 0})`;
104}
106export function summarize(rows: readonly ReplayRow[]): string[] {
107 const fallbackOld = rows.filter(row => row.oldMode === 'full-fallback').length;
108 const fallbackNew = rows.filter(row => row.newMode === 'full-fallback').length;
109 const misses = rows.flatMap(row => row.misses.map(id => `${row.runId}:${id}`));
110 const pct = (n: number) => rows.length ? `${Math.round((100 * n) / rows.length)}%` : 'n/a';
111 return [
112 `PR selection replay over ${rows.length} recorded push(es) (today's dependency graph)`,
113 `- full-fallback: ${fallbackOld} (${pct(fallbackOld)}) recorded -> ${fallbackNew} (${pct(fallbackNew)}) replayed`,
114 segment(`wave branches (diff >= ${WAVE_DIFF_FILES} files)`, rows.filter(row => row.changed >= WAVE_DIFF_FILES)),
115 segment(`ordinary pushes (diff < ${WAVE_DIFF_FILES} files)`, rows.filter(row => row.changed < WAVE_DIFF_FILES)),
116 `- pushes whose diff names files missing from today's tree: ${rows.filter(row => row.vanished > 0).length} (placed through the live files that still reference them)`,
117 `- needs full validation (plan refuses, as in CI): ${rows.filter(row => row.newMode === 'needs-full-validation').length}`,
118 `- diffs truncated at the compare API's 300-file limit: ${rows.filter(row => row.truncated).length}`,
119 `- failed cases that no longer exist: ${new Set(rows.flatMap(row => row.retiredFailures)).size}`,
120 `- MISSES (failed, depends on the diff, not selected now): ${misses.length}${misses.length ? ` -> ${misses.join(', ')}` : ''}`,
121 ];
122}
124function gh(args: string[]): string | null {
125 const result = spawnSync('gh', args, { encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
126 return result.status === 0 ? result.stdout : null;
127}
129function loadInput(repo: string, run: { databaseId: number; headSha: string }, cache: string): { input: ReplayInput; oldSelection: string[] } | null {
130 const dir = path.join(cache, String(run.databaseId));
131 const plan = path.join(dir, 'plan', 'manifest.json');
132 if (!fs.existsSync(plan)) gh(['run', 'download', String(run.databaseId), '-R', repo, '-n', 'paid-plan', '-D', path.join(dir, 'plan')]);
133 if (!fs.existsSync(plan)) return null;
134 const manifest = JSON.parse(fs.readFileSync(plan, 'utf8'));
135 if (manifest.profile !== 'pr') return null;
136 const outcomes = path.join(dir, 'outcomes');
137 if (!fs.existsSync(outcomes)) {
138 for (const name of ['trial-outcomes-pr-a1', 'trial-outcomes-pr']) {
139 if (gh(['run', 'download', String(run.databaseId), '-R', repo, '-n', name, '-D', outcomes]) !== null) break;
140 }
141 }
142 const failedCases = new Set<string>();
143 if (fs.existsSync(outcomes)) {
144 for (const file of fs.readdirSync(outcomes).filter(name => name.endsWith('.jsonl'))) {
145 for (const line of fs.readFileSync(path.join(outcomes, file), 'utf8').split('\n').filter(Boolean)) {
146 try { const record = JSON.parse(line); if (record.outcome === 'failed' && typeof record.case === 'string') failedCases.add(record.case); } catch { /* skip */ }
147 }
148 }
149 }
150 const compareFile = path.join(dir, 'compare.json');
151 if (!fs.existsSync(compareFile)) {
152 const compare = gh(['api', `repos/${repo}/compare/main...${run.headSha}`, '--jq', '{files: [.files[] | {filename, previous_filename}], merge_base: .merge_base_commit.sha}']);
153 if (compare === null) return null;
154 fs.writeFileSync(compareFile, compare);
155 }
156 const compare = JSON.parse(fs.readFileSync(compareFile, 'utf8')) as { files: Array<{ filename: string; previous_filename?: string }>; merge_base: string };
157 const changedFiles = [...new Set(compare.files.flatMap(f => [f.filename, ...(f.previous_filename ? [f.previous_filename] : [])]))];
158 let packageVersionOnly = false;
159 if (changedFiles.includes('package.json')) {
160 const read = (ref: string) => gh(['api', `repos/${repo}/contents/package.json?ref=${ref}`, '-H', 'Accept: application/vnd.github.raw']);
161 const [before, after] = [read(compare.merge_base), read(run.headSha)];
162 packageVersionOnly = before !== null && after !== null && packageChangeOnlyVersion(before, after);
163 }
164 return {
165 input: { runId: run.databaseId, oldMode: manifest.prCoverage?.mode ?? 'pr', oldCases: (manifest.selection?.e2e ?? []).length,
166 changedFiles, truncated: compare.files.length >= 300, failedCases: [...failedCases], packageVersionOnly },
167 oldSelection: manifest.selection?.e2e ?? [],
168 };
169}
171if (import.meta.main) {
172 const args = process.argv.slice(2);
173 const flag = (name: string) => { const i = args.indexOf(`--${name}`); return i >= 0 ? args[i + 1] : undefined; };
174 const runsFile = flag('runs');
175 if (args.includes('--help') || !runsFile) {
176 console.log('usage: bun run scripts/replay-pr-selection.ts --runs <runs.json> [--cache <dir>] [--repo garrytan/gstack] [--json]\n'
177 + 'runs.json: gh run list --workflow evals.yml --json databaseId,headSha,event,createdAt,conclusion -L 200');
178 process.exit(runsFile || args.includes('--help') ? 0 : 2);
179 }
180 const repo = flag('repo') ?? 'garrytan/gstack';
181 const cache = flag('cache') ?? path.join(os.tmpdir(), 'gstack-pr-replay');
182 const runs = (JSON.parse(fs.readFileSync(runsFile, 'utf8')) as Array<{ databaseId: number; headSha: string; event: string; conclusion: string }>)
183 .filter(run => run.event === 'pull_request' && !['action_required', 'skipped'].includes(run.conclusion));
184 const rows: ReplayRow[] = [];
185 for (const run of runs) {
186 const loaded = loadInput(repo, run, cache);
187 if (loaded) rows.push(replayRun(loaded.input, loaded.oldSelection));
188 }
189 if (args.includes('--json')) console.log(JSON.stringify(rows, null, 2));
190 else {
191 console.log('| run | recorded mode | cases | replayed mode | cases | dropped gate cases | misses |');
192 console.log('|---|---|---|---|---|---|---|');
193 for (const row of rows) {
194 console.log(`| ${row.runId}${row.truncated ? ' (diff >300 files)' : ''} | ${row.oldMode} | ${row.oldCases} | ${row.newMode} | ${row.newCases} | ${row.dropped.length} | ${row.misses.join(', ') || '0'} |`);
195 }
196 console.log('');
197 for (const line of summarize(rows)) console.log(line);
198 }
199}