gate-cost-replay.ts
617 lines · scripts/gate-cost-replay.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * gate-cost-replay — what the paid PR gate costs and what each selection
4 * policy would have cost on recent merges (GSTA-49 plan lane T0). Free and
5 * read-only: it downloads recorded report artifacts with `gh` into a cache,
6 * replays origin/main's first-parent squash merges through the selectors in
7 * this checkout, and models merge trains. Logic lives in lib/gate-cost.ts.
8 *
9 * Usage:
10 * bun run scripts/gate-cost-replay.ts --runs <id,id,...> [--since YYYY-MM-DD --until YYYY-MM-DD | --prs 3201,3202]
11 * [--ref origin/main] [--repo owner/name] [--cache .context/gate-cost-cache] [--window-days N]
12 * [--usd-per-second R]
13 * [--train-size 5] [--reps 400] [--seed 1] [--cases] [--history] [--json]
14 *
15 * --history (needs --since/--until) adds the red-head replay and the
16 * READY-arrival train replay (lib/gate-cost-history.ts): it lists the
17 * window's E2E Evals and Free Tests runs and PRs, downloads report-verdict
18 * and trial-outcomes artifacts of red runs, and fetches PR heads by SHA.
19 *
20 * --runs names complete full-gate runs (108 rule cases + 24 judges). Bare
21 * dates are UTC days; --until is inclusive. Cases without a recorded cost are
22 * estimated from their duration (default rate: recorded $ over recorded
23 * session-seconds; --usd-per-second pins it). Screening figures are labelled
24 * screen-provisional: the preamble detector is a heuristic until T2 lands.
25 */
26import * as fs from 'node:fs';
27import * as path from 'node:path';
28import { spawnSync } from 'node:child_process';
29import {
30 SCREEN_LABEL, TODAY_WEEKLY_USD, AFTER_GSTA48_WEEKLY_USD, caseCostTable, changedLinesByFile, fullGateUsd, isSkillMd,
31 parseEvalReport, parseTrialOutcomes, prSelectionCost, screenSelection, simulateTrains, summarizeGates, summarizeSelector, todaySelection,
32 type CaseCostEntry, type GateRecord, type GateSummary, type PrDiff, type PrSelectionCost, type ScreenPolicy,
33 type SelectionDeps, type SelectorSummary, type TrainModelResult,
34} from '../lib/gate-cost';
35import {
36 STAMP_FILES, classifyReds, headGreenAt, latencySummary, parseReportVerdict, prForRun, readyTimes, screenHit, simulateArrivalTrains,
37 stampFreeSignature, summarizeRedHeads, SCREEN_CANARY_CASES,
38 type ArrivalTrainResult, type CheckRun, type GatedRun, type HeadDiff, type LatencySummary, type ReadyRow, type RedHeadRow, type RedHeadSummary, type RedRun,
39} from '../lib/gate-cost-history';
40import { GH_CLIENT, GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
41import { computePaidCaseSelection } from './lib/paid-select';
42import { packageChangeOnlyVersion } from './test-pr-profile';
43import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
45const ROOT = path.resolve(import.meta.dir, '..');
46const DAY_MS = 86_400_000;
47const ARTIFACT = /^(paid-slice-\d+|trial-outcomes[A-Za-z0-9_-]*?)-a(\d+)$/;
48export const MODEL_POLICIES: ScreenPolicy[] = [
49 { kind: 'all' }, { kind: 'random', fraction: 0.8 }, { kind: 'random', fraction: 0.5 },
50 { kind: 'cheapest-half' }, { kind: 'expensive-half' }, { kind: 'none' },
51];
52export const MODEL_OUTSIDE_RED = [0.10, 0.15];
54export interface ReplayArgs {
55 runs: string[];
56 since?: string;
57 until?: string;
58 prs: number[];
59 ref: string;
60 repo: string;
61 cache: string;
62 windowDays?: number;
63 usdPerSecond?: number;
64 trainSize: number;
65 reps: number;
66 seed: number;
67 cases: boolean;
68 history: boolean;
69 json: boolean;
70}
72export function parseArgs(argv: string[]): ReplayArgs {
73 const args: ReplayArgs = { runs: [], prs: [], ref: 'origin/main', repo: '', cache: path.join(ROOT, '.context', 'gate-cost-cache'),
74 trainSize: 5, reps: 400, seed: 1, cases: false, history: false, json: false };
75 const list = (value: string) => value.split(/[\s,]+/).filter(Boolean);
76 const positive = (flag: string, value: string) => {
77 const n = Number(value);
78 if (!Number.isFinite(n) || n <= 0) throw new Error(`${flag} needs a positive number, got ${value}`);
79 return n;
80 };
81 for (let i = 0; i < argv.length; i += 1) {
82 const flag = argv[i]!;
83 if (flag === '--json') { args.json = true; continue; }
84 if (flag === '--cases') { args.cases = true; continue; }
85 if (flag === '--history') { args.history = true; continue; }
86 const value = argv[i + 1];
87 if (value === undefined || value.startsWith('--')) throw new Error(`${flag} needs a value`);
88 i += 1;
89 if (flag === '--runs') args.runs.push(...list(value));
90 else if (flag === '--prs') args.prs.push(...list(value).map(pr => Number(pr.replace(/^#/, ''))));
91 else if (flag === '--since') args.since = value;
92 else if (flag === '--until') args.until = value;
93 else if (flag === '--ref') args.ref = value;
94 else if (flag === '--repo') args.repo = value;
95 else if (flag === '--cache') args.cache = path.resolve(value);
96 else if (flag === '--window-days') args.windowDays = positive(flag, value);
97 else if (flag === '--usd-per-second') args.usdPerSecond = positive(flag, value);
98 else if (flag === '--train-size') args.trainSize = Math.round(positive(flag, value));
99 else if (flag === '--reps') args.reps = Math.round(positive(flag, value));
100 else if (flag === '--seed') args.seed = Math.round(Number(value));
101 else throw new Error(`unknown flag ${flag}`);
102 }
103 if (args.runs.length === 0) throw new Error('--runs <id,...> is required: the full-gate runs whose recorded costs price every case');
104 if (args.runs.some(run => !/^\d+$/.test(run))) throw new Error('--runs takes numeric GitHub Actions run ids');
105 if (args.prs.some(pr => !Number.isInteger(pr) || pr <= 0)) throw new Error('--prs takes PR numbers');
106 if (args.prs.length && (args.since || args.until)) throw new Error('use either --prs or --since/--until, not both');
107 if (!!args.since !== !!args.until) throw new Error('--since and --until go together');
108 if (args.history && !args.since) throw new Error('--history needs a window: --since and --until');
109 return args;
110}
112/** A bare date is a UTC day; `end` makes it inclusive (the next midnight). */
113export function parseWindowDate(value: string, end: boolean): Date {
114 const bare = /^\d{4}-\d{2}-\d{2}$/.test(value);
115 const date = new Date(bare ? `${value}T00:00:00Z` : value);
116 if (Number.isNaN(date.getTime())) throw new Error(`not a date: ${value}`);
117 return bare && end ? new Date(date.getTime() + DAY_MS) : date;
118}
120function git(args: string[], timeout = 60_000): string {
121 const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 });
122 if (result.status !== 0) throw new Error(`git ${args.slice(0, 3).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
123 return result.stdout;
124}
126// --- Artifacts ---
128/** Download a run's paid-slice and trial-outcomes artifacts once; a run with a .complete marker never calls gh again. */
129export function fetchRun(runId: string, cacheDir: string, repo: string): string {
130 const dir = path.join(cacheDir, runId);
131 if (fs.existsSync(path.join(dir, '.complete'))) return dir;
132 downloadRunArtifacts({ repo, run: { id: Number(runId) }, cacheDir, match: name => ARTIFACT.test(name), maxBytes: 512 * 1024 * 1024, fetcher: GH_HISTORY });
133 fs.writeFileSync(path.join(dir, '.complete'), '');
134 return dir;
135}
137/** One run directory -> GateRecord, reading only each artifact's highest attempt. */
138export function readGateRecord(runDir: string, runId = path.basename(runDir)): GateRecord {
139 const latest = new Map<string, { name: string; attempt: number }>();
140 for (const name of fs.existsSync(runDir) ? fs.readdirSync(runDir) : []) {
141 const match = name.match(ARTIFACT);
142 if (!match || (latest.get(match[1]!)?.attempt ?? 0) >= Number(match[2])) continue;
143 latest.set(match[1]!, { name, attempt: Number(match[2]) });
144 }
145 const gate: GateRecord = { runId, tests: [], trials: [] };
146 for (const { name } of [...latest.values()].sort((a, b) => a.name.localeCompare(b.name))) {
147 const artifact = path.join(runDir, name);
148 if (name.startsWith('trial-outcomes')) {
149 const file = path.join(artifact, 'trial-outcomes.jsonl');
150 if (fs.existsSync(file)) gate.trials.push(...parseTrialOutcomes(fs.readFileSync(file, 'utf8')));
151 continue;
152 }
153 const shards = path.join(artifact, 'shards');
154 for (const shard of fs.existsSync(shards) ? fs.readdirSync(shards).sort() : []) {
155 const shardDir = path.join(shards, shard);
156 if (!fs.statSync(shardDir).isDirectory()) continue;
157 for (const file of fs.readdirSync(shardDir).sort()) {
158 if (!file.endsWith('.json') || file.startsWith('_partial') || file.startsWith('slice-')) continue;
159 let report: unknown;
160 try { report = JSON.parse(fs.readFileSync(path.join(shardDir, file), 'utf8')); } catch { continue; }
161 gate.tests.push(...parseEvalReport(shard, report));
162 }
163 }
164 }
165 return gate;
166}
168// --- Merged PRs ---
170export interface MergedPr { pr: number; sha: string; committedAt: string }
172/** First-parent squash merges whose subject ends in (#NNN). */
173export function parseMergeLog(log: string): MergedPr[] {
174 return log.split('\n').filter(Boolean).flatMap(line => {
175 const [sha, committedAt, ...subject] = line.split('\t');
176 const match = subject.join('\t').match(/\(#(\d+)\)\s*$/);
177 return match && sha && committedAt ? [{ pr: Number(match[1]), sha, committedAt }] : [];
178 });
179}
181function mergedPrs(args: ReplayArgs): { prs: MergedPr[]; windowDays: number | null; since?: Date; until?: Date } {
182 const format = '--format=%H%x09%cI%x09%s';
183 const shallowHint = (since: string) => git(['rev-parse', '--is-shallow-repository']).trim() === 'true'
184 ? `; this checkout is shallow: run "git fetch --shallow-since=${since} origin main" and rerun` : '';
185 if (args.since && args.until) {
186 const since = parseWindowDate(args.since, false), until = parseWindowDate(args.until, true);
187 const oldest = git(['log', '--first-parent', '--format=%cI', args.ref]).trim().split('\n').pop() ?? '';
188 const hint = shallowHint(since.toISOString().slice(0, 10));
189 if (hint && new Date(oldest) > since) throw new Error(`history before ${oldest} is missing${hint}`);
190 const prs = parseMergeLog(git(['log', '--first-parent', `--since=${since.toISOString()}`, `--until=${until.toISOString()}`, format, args.ref])).reverse();
191 return { prs, windowDays: args.windowDays ?? (until.getTime() - since.getTime()) / DAY_MS, since, until };
192 }
193 if (args.prs.length === 0) return { prs: [], windowDays: null };
194 const all = new Map(parseMergeLog(git(['log', '--first-parent', format, args.ref])).map(pr => [pr.pr, pr]));
195 const missing = args.prs.filter(pr => !all.has(pr));
196 if (missing.length) throw new Error(`no squash merge of #${missing.join(', #')} on ${args.ref}'s first-parent history${shallowHint('2026-01-01')}`);
197 return { prs: args.prs.map(pr => all.get(pr)!), windowDays: args.windowDays ?? null };
198}
200/** The PR diff the selectors see between two revisions. */
201function revisionDiff(pr: number, from: string, to: string): PrDiff {
202 const files = git(['diff', '--name-only', '--no-renames', from, to]).split('\n').filter(Boolean);
203 let packageVersionOnly = false;
204 if (files.includes('package.json')) {
205 const show = (rev: string) => spawnSync('git', ['show', `${rev}:package.json`], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
206 const before = show(from), after = show(to);
207 packageVersionOnly = before.status === 0 && after.status === 0 && packageChangeOnlyVersion(before.stdout, after.stdout);
208 }
209 const skillMds = files.filter(isSkillMd);
210 const skillMdChangedLines = skillMds.length >= 10
211 ? changedLinesByFile(git(['diff', '-U0', '--no-renames', from, to, '--', ...skillMds], 120_000)) : {};
212 return { pr, sha: to.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
213}
215const prDiff = (merge: MergedPr): PrDiff => revisionDiff(merge.pr, `${merge.sha}^`, merge.sha);
217/** The selectors in this checkout: today's PR profile (with R4's shadow) and the plain touchfile selection. */
218export function checkoutSelectionDeps(rootDir = ROOT): SelectionDeps {
219 const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate');
220 const judgeCases = Object.keys(LLM_JUDGE_TOUCHFILES);
221 return {
222 gateCases, judgeCases,
223 today(files, packageVersionOnly) {
224 try {
225 const result = computePaidCaseSelection({ profile: 'pr', env: {}, rootDir, changedFiles: files, packageVersionOnly });
226 return { e2e: result.selection.e2e ?? gateCases, judges: result.selection.judges ?? judgeCases,
227 shadowE2e: result.coverage?.shadow?.e2e ?? null, mode: result.coverage?.mode ?? 'pr' };
228 } catch {
229 return { e2e: gateCases, judges: judgeCases, shadowE2e: null, mode: 'needs-full' };
230 }
231 },
232 touchfiles(files) {
233 const result = computePaidCaseSelection({ profile: 'full', env: {}, rootDir, changedFiles: files });
234 return { e2e: result.selection.e2e ?? gateCases, global: files.some(file => GLOBAL_TOUCHFILES.some(pattern => matchGlob(file, pattern))) };
235 },
236 };
237}
239// --- Report ---
241export interface ReplayReport {
242 repo: string;
243 ref: string;
244 head: string;
245 runs: Array<{ runId: string; tests: number; trials: number; recordedUsd: number }>;
246 gate: GateSummary;
247 fullScaleUsd: number;
248 usdPerSecond: number;
249 cases: CaseCostEntry[];
250 window: { since: string; until: string } | null;
251 windowDays: number | null;
252 prs: PrSelectionCost[];
253 selection: { today: SelectorSummary; shadow: SelectorSummary; screenProvisional: SelectorSummary } | null;
254 model: { screenScale: number; fullGateUsd: number; trainSize: number; reps: number; seed: number; rows: TrainModelResult[] } | null;
255 history?: HistoryReport;
256}
258export function buildReport(args: ReplayArgs, gates: GateRecord[], merges: ReturnType<typeof mergedPrs>, deps: SelectionDeps,
259 diffs: (merge: MergedPr) => PrDiff = prDiff): Omit<ReplayReport, 'repo' | 'ref' | 'head'> {
260 const table = caseCostTable(gates, { usdPerSecond: args.usdPerSecond });
261 const gate = summarizeGates(gates);
262 const fullScaleUsd = fullGateUsd(deps, table);
263 const prs = merges.prs.map(merge => prSelectionCost(diffs(merge), deps, table));
264 const selection = prs.length ? {
265 today: summarizeSelector(prs.map(row => row.today), fullScaleUsd),
266 shadow: summarizeSelector(prs.map(row => row.shadow), fullScaleUsd),
267 screenProvisional: summarizeSelector(prs.map(row => row.screenProvisional), fullScaleUsd),
268 } : null;
269 const screenScale = fullScaleUsd ? gate.recordedUsd.mean / fullScaleUsd : 0;
270 const model = prs.length && merges.windowDays ? {
271 screenScale, fullGateUsd: gate.recordedUsd.mean, trainSize: args.trainSize, reps: args.reps, seed: args.seed,
272 rows: MODEL_POLICIES.flatMap(policy => MODEL_OUTSIDE_RED.map(outsideRed => simulateTrains({
273 prs: prs.map(row => ({ paidRelevant: row.today.usd > 0, screenUsd: row.screenProvisional.usd * screenScale })),
274 policy, outsideRed, windowDays: merges.windowDays!, trainSize: args.trainSize, fullGateUsd: gate.recordedUsd.mean, reps: args.reps, seed: args.seed,
275 }))),
276 } : null;
277 return {
278 runs: gates.map(g => ({ runId: g.runId, tests: g.tests.length, trials: g.trials.length, recordedUsd: g.tests.reduce((sum, t) => sum + t.costUsd, 0) })),
279 gate, fullScaleUsd, usdPerSecond: table.usdPerSecond, cases: Object.values(table.cases).sort((a, b) => b.costUsd - a.costUsd),
280 window: merges.since && merges.until ? { since: merges.since.toISOString(), until: merges.until.toISOString() } : null,
281 windowDays: merges.windowDays, prs, selection, model,
282 };
283}
285const usd = (value: number) => value.toFixed(2);
286const pct = (value: number, total: number) => (total ? `${(100 * value / total).toFixed(1)}%` : '-');
287const mtok = (tokens: number) => (tokens >= 1e6 ? `${(tokens / 1e6).toFixed(2)}M` : `${Math.round(tokens / 1e3)}k`);
288const pad = (cells: Array<string | number>, widths: number[]) => cells.map((cell, i) => String(cell).padEnd(widths[i] ?? 0)).join(' ').trimEnd();
290export function formatReport(report: ReplayReport, options: { cases?: boolean } = {}): string {
291 const out: string[] = [];
292 const { gate } = report;
293 const gateUsd = gate.recordedUsd.mean;
294 out.push(`gate-cost replay: ${report.repo} at ${report.head.slice(0, 8)} (selectors from this checkout)`, '');
295 out.push(`Gates: ${report.runs.length} run(s), recorded $ per gate mean ${usd(gateUsd)} (min ${usd(gate.recordedUsd.min)}, max ${usd(gate.recordedUsd.max)})`);
296 for (const run of report.runs) out.push(` run ${run.runId}: ${run.trials} trials, ${run.tests} cost entries, $${usd(run.recordedUsd)}`);
297 out.push('', 'Recorded $ per gate by model:');
298 for (const [model, value] of Object.entries(gate.byModel).sort((a, b) => b[1] - a[1])) out.push(` ${pad([model, usd(value), pct(value, gateUsd)], [34, 8])}`);
299 out.push('', 'Token classes per gate (lib/pricing.ts prices; cache writes at 1.25x input; share of the recorded gate):');
300 const classes = Object.values(gate.tokenClasses);
301 const sum = (pick: (c: (typeof classes)[number]) => number) => classes.reduce((total, c) => total + pick(c), 0);
302 const thinking = sum(c => c.output.thinkingTokens);
303 for (const [label, key] of [['cache write', 'cacheWrite'], [`output (incl. ${mtok(thinking)} thinking)`, 'output'], ['cache read', 'cacheRead'], ['uncached input', 'uncachedInput']] as const) {
304 const value = sum(c => c[key].usd);
305 out.push(` ${pad([label, mtok(sum(c => c[key].tokens)), usd(value), pct(value, gateUsd)], [34, 8, 8])}`);
306 }
307 for (const [model, c] of Object.entries(gate.tokenClasses)) {
308 out.push(` ${model}: ${c.priced ? `priced $${usd(c.pricedUsd)} vs recorded $${usd(c.recordedUsd)}` : `no lib/pricing.ts row, recorded $${usd(c.recordedUsd)} unpriced`}`);
309 }
310 const estimated = report.cases.filter(c => c.estimated);
311 const estimatedUsd = estimated.reduce((total, c) => total + c.costUsd, 0);
312 out.push('', `Cases: ${report.cases.length}; ${estimated.length} without recorded cost, estimated at $${report.usdPerSecond.toFixed(4)} per session-second: ~${usd(estimatedUsd)} per gate`);
313 for (const c of estimated) out.push(` ${pad([c.caseId, `~${usd(c.costUsd)} (est.)`, `${Math.round(c.meanDurationMs / 1000)} s`, c.runner ?? '-'], [40, 14, 7])}`);
314 out.push(`Full-gate scale: $${usd(report.fullScaleUsd)} per gate (recorded case means plus estimates)`);
315 if (options.cases) {
316 out.push('', 'Per-case mean cost:');
317 for (const c of report.cases) out.push(` ${pad([c.caseId, (c.estimated ? '~' : '') + usd(c.costUsd), c.kind, c.runner ?? '-', c.estimated ? 'estimated' : `${c.recordedGates} gate(s)`], [50, 8, 6, 10])}`);
318 }
319 if (!report.prs.length) return out.join('\n');
320 const label = SCREEN_LABEL;
321 out.push('', `Per merged PR ($ on the $${usd(report.fullScaleUsd)} full-gate scale; cases are gate e2e cases, judges extra):`);
322 out.push(` ${pad(['PR', 'sha', 'files', 'today $ (cases)', 'R4 shadow $ (cases)', `${label} $ (cases)`], [6, 8, 5, 16, 20])}`);
323 for (const row of report.prs) {
324 const cell = (s: { usd: number; cases: number }) => `${usd(s.usd)} (${s.cases})`;
325 out.push(` ${pad([`#${row.pr}`, row.sha, row.files, cell(row.today), cell(row.shadow), cell(row.screenProvisional) + (row.screenProvisional.canary ? ' +canary' : '')], [6, 8, 5, 16, 20])}`);
326 }
327 const s = report.selection!;
328 out.push('', `Selection over ${report.prs.length} merged PRs (one cohort: every merged PR counts, zeros included)${report.window ? `, ${report.window.since} to ${report.window.until}` : ''}:`);
329 out.push(` ${pad(['selector', '$ / merged PR', 'median $', 'mean cases', 'PRs >= 90% of gate'], [20, 13, 9, 10])}`);
330 for (const [name, row] of [['today', s.today], ['R4 shadow', s.shadow], [label, s.screenProvisional]] as const) {
331 out.push(` ${pad([name, usd(row.meanUsd), usd(row.medianUsd), row.meanCases.toFixed(1), row.nearFull], [20, 13, 9, 10])}`);
332 }
333 out.push(` ${label}: its preamble detector is a line-commonality heuristic, not T2's hunk classifier; heuristic until T2 lands.`);
334 if (!report.model) {
335 out.push('', 'Train model: skipped (needs a window: --since/--until, or --window-days with --prs).');
336 return out.join('\n');
337 }
338 const m = report.model;
339 const perWeek = report.prs.length * 7 / report.windowDays!;
340 out.push('', `Train model (seeded Monte Carlo, ${m.reps} reps, seed ${m.seed}; trains of ${m.trainSize}; full gate $${usd(m.fullGateUsd)}, bisect $6, repair re-screens x1.3,`,
341 ` red rates screened 6% / unscreened 25%; ${label} screen costs scaled by ${m.screenScale.toFixed(4)} to the recorded gate;`,
342 ` window ${report.windowDays!.toFixed(1)} days, ${perWeek.toFixed(1)} merged PRs a week; multiples against $${TODAY_WEEKLY_USD} and $${AFTER_GSTA48_WEEKLY_USD} a week):`);
343 const widths = [15, 5, 7, 6, 7, 7, 22, 9, 9, 9, 8, 9, 10];
344 out.push(` ${pad(['policy', 'o', '$/week', '$/PR', 'x9950', 'x8400', `${label} $/w`, 'trains/w', 'gates/w', 'bisect/w', 'rebld/w', 'stalls/w', 'first-red'], widths)} attempt-red`);
345 for (const r of m.rows) {
346 out.push(` ${pad([r.policy, r.outsideRed.toFixed(2), Math.round(r.weeklyUsd), r.perMergedPrUsd.toFixed(1), r.multipleVsToday.toFixed(2), r.multipleVsAfterGsta48.toFixed(2),
347 Math.round(r.screenWeeklyUsd), r.initialTrainsWeekly.toFixed(2), r.gateAttemptsWeekly.toFixed(2), r.bisectsWeekly.toFixed(2), r.rebuildOnceWeekly.toFixed(2),
348 r.stallsWeekly.toFixed(2), pct(r.firstCandidateRedShare, 1)], widths)} ${pct(r.attemptRedShare, 1)}`);
349 }
350 return out.join('\n');
351}
353// --- History (--history): red-head replay and READY-arrival trains ---
355const REPORT_ARTIFACT = /^report-verdict-a\d+$/;
356const TRIALS_ARTIFACT = /^trial-outcomes[A-Za-z0-9_-]*?-a\d+$/;
357const TRAIN_POLICIES: ScreenPolicy[] = [{ kind: 'all' }, { kind: 'random', fraction: 0.5 }, { kind: 'none' }];
358const GH_RUN_PAGE = 100;
359const HISTORY_GIT_TIMEOUT_MS = 120_000;
360const FETCH_TIMEOUT_MS = 300_000;
362interface HistoryRun { id: number; sha: string; branch: string; createdAt: string; updatedAt: string; conclusion: string | null; prNumbers: number[] }
363interface PullRow { number: number; ref: string; head: string; createdAt: string; mergedAt: string | null; closedAt: string | null }
364type Merges = ReturnType<typeof mergedPrs>;
366export interface HistoryReport {
367 window: { since: string; until: string };
368 runs: { evals: number; gated: number; red: number; free: number; unmapped: number };
369 redRuns: RedRun[];
370 redHeads: RedHeadRow[];
371 redSummary: RedHeadSummary;
372 perPr: Array<{ pr: number; paidRelevant: boolean; prCausedRed: boolean; screenedMiss: boolean }>;
373 ready: ReadyRow[];
374 baseline: LatencySummary & { gatesAfterReady: number; sources: Record<string, number> };
375 trainParams: { trainSize: number; fillLimitHours: number; gateMinutes: number; bisectMinutes: number; redScreened: number; redUnscreened: number; reps: number; seed: number };
376 trains: ArrivalTrainResult[];
377}
379function tryGit(args: string[], timeout = HISTORY_GIT_TIMEOUT_MS): string | null {
380 const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 });
381 return result.status === 0 ? result.stdout : null;
382}
384function cachedJson<T>(file: string, load: () => T): T {
385 if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf8')) as T;
386 const value = load();
387 fs.writeFileSync(file, JSON.stringify(value));
388 return value;
389}
391/** Completed runs of one workflow, one UTC day per query: GitHub caps a filtered run listing at 1,000 results. */
392function listRunsByDay(repo: string, workflow: string, since: Date, until: Date, event?: string): HistoryRun[] {
393 const out: HistoryRun[] = [];
394 for (let day = new Date(since.toISOString().slice(0, 10)); day < until; day = new Date(day.getTime() + DAY_MS)) {
395 const date = day.toISOString().slice(0, 10);
396 for (let page = 1; page <= 10; page += 1) {
397 const query = [`status=completed`, `per_page=${GH_RUN_PAGE}`, `page=${page}`, `created=${date}..${date}`, ...(event ? [`event=${event}`] : [])];
398 const body = GH_CLIENT.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}`) as { workflow_runs?: any[] };
399 const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
400 out.push(...runs.map(run => ({ id: Number(run.id), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''), createdAt: String(run.created_at ?? ''),
401 updatedAt: String(run.updated_at ?? ''), conclusion: run.conclusion ?? null,
402 prNumbers: Array.isArray(run.pull_requests) ? run.pull_requests.map((pr: any) => Number(pr.number)).filter(Number.isFinite) : [] })));
403 if (runs.length < GH_RUN_PAGE) break;
404 }
405 }
406 return out.filter(run => run.createdAt >= since.toISOString() && run.createdAt < until.toISOString());
407}
409function listPulls(repo: string, since: Date): PullRow[] {
410 const out: PullRow[] = [];
411 for (let page = 1; page <= 30; page += 1) {
412 const body = GH_CLIENT.getJson(`repos/${repo}/pulls?state=all&sort=updated&direction=desc&per_page=100&page=${page}`) as any[];
413 if (!Array.isArray(body)) break;
414 out.push(...body.map(p => ({ number: Number(p.number), ref: String(p.head?.ref ?? ''), head: String(p.head?.sha ?? ''), createdAt: String(p.created_at ?? ''),
415 mergedAt: p.merged_at ?? null, closedAt: p.closed_at ?? null })));
416 if (body.length < 100 || String(body[body.length - 1]?.updated_at ?? '') < since.toISOString()) break;
417 }
418 return out;
419}
421/** Artifacts of one run matching a name pattern, downloaded once (a .history marker records that the run was listed). */
422function runArtifactDirs(runId: number, repo: string, cache: string, match: RegExp): string[] {
423 const dir = path.join(cache, String(runId));
424 const marker = path.join(dir, `.history-${match.source.replace(/[^a-z]/gi, '')}`);
425 if (!fs.existsSync(marker)) {
426 downloadRunArtifacts({ repo, run: { id: runId }, cacheDir: cache, match: name => match.test(name), maxBytes: 16 * 1024 * 1024, fetcher: GH_HISTORY });
427 fs.mkdirSync(dir, { recursive: true });
428 fs.writeFileSync(marker, '');
429 }
430 return fs.readdirSync(dir).filter(name => match.test(name)).sort((a, b) => Number(a.split('-a').pop()) - Number(b.split('-a').pop())).map(name => path.join(dir, name));
431}
433/** The run's report verdict over its attempts: the last attempt's verdict, every attempt's failures; null when no report exists. */
434function runVerdict(runId: number, repo: string, cache: string): { green: boolean; failures: GatedRun['failures'] } | null {
435 const reports = runArtifactDirs(runId, repo, cache, REPORT_ARTIFACT).map(dir => path.join(dir, 'report.txt')).filter(file => fs.existsSync(file))
436 .map(file => parseReportVerdict(fs.readFileSync(file, 'utf8')));
437 if (!reports.length) return null;
438 return { green: reports[reports.length - 1]!.verdict === 'GREEN', failures: reports.flatMap(r => r.failures) };
439}
441/** The tested merge commit of a run, from its trial outcomes. */
442function testedMergeSha(runId: number, repo: string, cache: string): string | null {
443 for (const dir of runArtifactDirs(runId, repo, cache, TRIALS_ARTIFACT)) {
444 const file = path.join(dir, 'trial-outcomes.jsonl');
445 const first = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').find(Boolean) : undefined;
446 try { const sha = first ? JSON.parse(first).sha : null; if (typeof sha === 'string' && /^[0-9a-f]{40}$/.test(sha)) return sha; } catch { /* torn line */ }
447 }
448 return null;
449}
451/** Fetch commits this clone lacks by SHA (GitHub serves PR heads and merge refs that no branch holds anymore). */
452function ensureCommits(shas: string[]): void {
453 const missing = [...new Set(shas)].filter(sha => tryGit(['cat-file', '-e', `${sha}^{commit}`], 30_000) === null);
454 for (let i = 0; i < missing.length; i += 20) spawnSync('git', ['fetch', '-q', 'origin', ...missing.slice(i, i + 20)], { cwd: ROOT, timeout: FETCH_TIMEOUT_MS });
455}
457function headDiff(sha: string, ref: string): HeadDiff | null {
458 const mergeBase = tryGit(['merge-base', ref, sha])?.trim();
459 if (!mergeBase) return null;
460 const files = tryGit(['diff', '--name-only', '--no-renames', mergeBase, sha]);
461 const diff = tryGit(['diff', '-U0', '--no-color', '--no-renames', '--no-ext-diff', mergeBase, sha, '--', '.', ...STAMP_FILES.map(file => `:(exclude)${file}`)]);
462 return files === null || diff === null ? null : { ownSig: stampFreeSignature(diff), files: files.split('\n').filter(Boolean) };
463}
465/** The base a red run tested: its merge commit's first parent, else main's first-parent tip when the run was created. */
466function testedBase(run: RedRun, args: ReplayArgs, repo: string): { base: string; source: RedHeadRow['baseSource'] } | null {
467 const merge = testedMergeSha(run.runId, repo, args.cache);
468 if (merge) {
469 ensureCommits([merge]);
470 const parents = (tryGit(['rev-list', '--parents', '-n1', merge]) ?? '').trim().split(' ').slice(1);
471 if (parents.length === 2 && parents[1] === run.head) return { base: parents[0]!, source: 'tested-merge' };
472 }
473 const tip = tryGit(['rev-list', '-1', '--first-parent', `--before=${run.createdAt}`, args.ref])?.trim();
474 return tip ? { base: tip, source: 'main-at-run' } : null;
475}
477function collectHistory(args: ReplayArgs, repo: string, merges: Merges, body: Omit<ReplayReport, 'repo' | 'ref' | 'head'>, deps: SelectionDeps, gates: GateRecord[]): HistoryReport {
478 const since = merges.since!, until = merges.until!;
479 const key = `${since.toISOString()}_${until.toISOString()}`.replace(/[^0-9TZ_]/g, '');
480 const cached = <T>(name: string, load: () => T) => cachedJson(path.join(args.cache, `history-${key}-${name}.json`), load);
481 const evals = cached('evals', () => listRunsByDay(repo, 'evals.yml', since, until));
482 const free = cached('free', () => listRunsByDay(repo, 'free-tests.yml', since, until, 'pull_request'));
483 const pulls = cached('pulls', () => listPulls(repo, since));
484 const prOf = new Map([...evals, ...free].map(run => [run.id, prForRun(run, pulls)]));
485 const gated: GatedRun[] = evals.flatMap(run => {
486 const pr = prOf.get(run.id) ?? null;
487 if (run.conclusion === 'success') return [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, green: true, failures: [] }];
488 const verdict = run.conclusion === 'failure' ? runVerdict(run.id, repo, args.cache) : null;
489 return verdict ? [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, ...verdict }] : [];
490 });
491 const mergedSet = new Set(merges.prs.map(m => m.pr));
492 const pullOf = new Map(pulls.map(p => [p.number, p]));
493 const checkRuns = [...evals.map(run => ({ run, workflow: 'evals' as const })), ...free.map(run => ({ run, workflow: 'free' as const }))]
494 .filter(({ run }) => mergedSet.has(prOf.get(run.id) ?? -1));
495 const heads = [...new Set([...gated.filter(r => r.pr !== null).map(r => r.head), ...checkRuns.map(({ run }) => run.sha),
496 ...merges.prs.map(m => pullOf.get(m.pr)?.head ?? '')].filter(Boolean))];
497 ensureCommits(heads);
498 const diffs = cached('heads', () => Object.fromEntries(heads.map(sha => [sha, headDiff(sha, args.ref)])));
499 const headMap = new Map(Object.entries(diffs).filter((e): e is [string, HeadDiff] => e[1] !== null));
500 const caseFiles = new Map(gates.flatMap(g => g.trials.map(t => [t.caseId, t.file] as const)));
501 const caseFile = (id: string) => caseFiles.get(id) ?? null;
502 const inputsOf = (id: string) => /^test\/.+\.test\.ts$/.test(id)
503 ? [id, ...[...caseFiles].filter(([, file]) => file === id).flatMap(([c]) => E2E_TOUCHFILES[c] ?? [])]
504 : [...(E2E_TOUCHFILES[id] ?? []), ...(caseFile(id) ? [caseFile(id)!] : [])];
505 const redRuns = classifyReds(gated, {
506 heads: headMap,
507 delta: (from, to) => tryGit(['diff', '--name-only', '--no-renames', from, to])?.split('\n').filter(Boolean) ?? null,
508 touches: (id, files) => files.filter(file => [...inputsOf(id), ...GLOBAL_TOUCHFILES].some(pattern => matchGlob(file, pattern))),
509 });
510 const redHeads: RedHeadRow[] = redRuns.filter(run => run.cause === 'pr').flatMap(run => {
511 const tested = testedBase(run, args, repo);
512 const mergeBase = tested && tryGit(['merge-base', tested.base, run.head])?.trim();
513 if (!tested || !mergeBase) return [];
514 const diff = revisionDiff(run.pr, mergeBase, run.head);
515 const selection = screenSelection(diff, deps, todaySelection(diff, deps));
516 return run.cases.filter(c => c.cause === 'pr').map(c => ({ caseId: c.caseId, kind: c.kind, pr: run.pr, runId: run.runId, head: run.head.slice(0, 8),
517 base: tested.base.slice(0, 8), baseSource: tested.source, files: diff.files.length, hit: screenHit(c, selection, caseFile), canaryReasons: selection.canaryReasons }));
518 });
519 const selected = new Map(body.prs.map(row => [row.pr, row]));
520 const merged = merges.prs.map(m => ({ pr: m.pr, paidRelevant: (selected.get(m.pr)?.today.usd ?? 0) > 0 }));
521 const redSummary = summarizeRedHeads(redHeads, redRuns, merged);
522 const ready = readyTimes(merges.prs.flatMap(m => {
523 const pull = pullOf.get(m.pr);
524 if (!pull?.mergedAt) return [];
525 const prHeads = [...new Set([...checkRuns.filter(({ run }) => prOf.get(run.id) === m.pr).map(({ run }) => run.sha), pull.head])];
526 return [{ pr: m.pr, mergedAt: pull.mergedAt, finalHead: pull.head, heads: prHeads.map(sha => ({ sha, ownSig: headMap.get(sha)?.ownSig ?? null })),
527 gatedRuns: gated.filter(r => r.pr === m.pr).map(r => r.createdAt) }];
528 }), headGreenAt(checkRuns.map(({ run, workflow }): CheckRun => ({ head: run.sha, workflow, createdAt: run.createdAt, completedAt: run.updatedAt, conclusion: run.conclusion }))));
529 const withReady = ready.filter(row => row.ready !== null);
530 const sources: Record<string, number> = {};
531 for (const row of ready) sources[row.source] = (sources[row.source] ?? 0) + 1;
532 const scale = body.model?.screenScale ?? 1;
533 const arrivals = withReady.map(row => ({ pr: row.pr, readyMs: Date.parse(row.ready!), paidRelevant: (selected.get(row.pr)?.today.usd ?? 0) > 0,
534 screenUsd: (selected.get(row.pr)?.screenProvisional.usd ?? 0) * scale }));
535 const trainParams = { trainSize: args.trainSize, fillLimitHours: 3, gateMinutes: 45, bisectMinutes: 45, redScreened: 0.06, redUnscreened: 0.25, reps: args.reps, seed: args.seed };
536 return {
537 window: { since: since.toISOString(), until: until.toISOString() },
538 runs: { evals: evals.length, gated: gated.length, red: gated.filter(r => !r.green).length, free: free.length, unmapped: evals.filter(run => prOf.get(run.id) === null).length },
539 redRuns, redHeads, redSummary,
540 perPr: merged.map(m => ({ ...m, prCausedRed: redSummary.prsWithPrRed.includes(m.pr), screenedMiss: redSummary.missedPrs.includes(m.pr) })),
541 ready, baseline: { ...latencySummary(withReady.map(row => row.readyToMergeHours!)), gatesAfterReady: withReady.reduce((sum, row) => sum + row.gatesAfterReady, 0), sources },
542 trainParams,
543 trains: TRAIN_POLICIES.flatMap(policy => MODEL_OUTSIDE_RED.map(outsideRed => simulateArrivalTrains({ ...trainParams, arrivals, policy, outsideRed, windowDays: merges.windowDays! }))),
544 };
545}
547export function formatHistory(h: HistoryReport): string {
548 const s = h.redSummary;
549 const label = SCREEN_LABEL;
550 const share = (n: number, d: number) => `${n} of ${d} (${pct(n, d)})`;
551 const causes = (cause: string) => h.redRuns.filter(run => run.cause === cause).length;
552 const out = [`History replays, E2E Evals and Free Tests runs created ${h.window.since} to ${h.window.until}:`,
553 ` E2E Evals runs ${h.runs.evals} (${h.runs.unmapped} not mapped to a PR), gated ${h.runs.gated}, red ${h.runs.red}: PR-caused ${causes('pr')}, outside ${causes('outside')}, unknown ${causes('unknown')}; Free Tests PR runs ${h.runs.free}`,
554 ' Classification (GSTA-47): per failed case, first match wins: transcript diagnosis; red in another PR within 72 h -> outside;',
555 ' no later gated run -> unknown; next gated run has the same own diff -> outside; the push to it changed the case\'s inputs',
556 ' (touchfiles, test file, harness globals) among either head\'s own files -> PR-caused; else outside. A run with any PR-caused case is PR-caused.',
557 '', `Red-head replay (${label} on the failing head's diff from its tested base; canary set fixed before the replay: ${Object.values(SCREEN_CANARY_CASES).join(', ')}):`,
558 ` ${pad(['PR', 'run', 'head', 'base', 'files', 'case', 'kind', `${label} hit`], [6, 12, 9, 22, 5, 46, 6])}`];
559 for (const row of h.redHeads) {
560 out.push(` ${pad([`#${row.pr}`, row.runId, row.head, `${row.base} ${row.baseSource}`, row.files, row.caseId, row.kind, row.hit ?? 'MISSED'], [6, 12, 9, 22, 5, 46, 6])}`);
561 }
562 out.push(` ${label} hits by kind: ${Object.entries(s.byHit).map(([k, n]) => `${k} ${n}`).join(', ') || 'none'}`,
563 ` ${label} misses: ${share(s.missedPrs.length, s.prsWithPrRed.length)} merged PRs with a PR-caused red (${s.missedCases} of ${s.prCausedCases} case results)${s.missedPrs.length ? `: #${s.missedPrs.join(', #')}` : ''}`,
564 ` ${label} misses with an empty canary set: ${s.missedPrsWithoutCanary.length}${s.missedPrsWithoutCanary.length ? ` (#${s.missedPrsWithoutCanary.join(', #')})` : ''}`,
565 ` P-i (${label} misses over paid-relevant merged PRs): ${share(s.missedPrs.length, s.paidRelevantPrs)}`,
566 ` P-k (merged PRs with any PR-caused red run): ${share(s.prsWithPrRed.length, s.mergedPrs)}${s.prsWithPrRed.length ? `: #${s.prsWithPrRed.join(', #')}` : ''}; ${s.prCausedRuns} PR-caused red runs`,
567 ` PR-caused red runs on PRs that did not merge in the window (not counted): ${s.otherPrCausedRuns}`,
568 '', `Per merged PR (paid-relevant: today's selector picks a case on the squash; red: any PR-caused red run; miss: a ${label} miss):`);
569 for (let i = 0; i < h.perPr.length; i += 6) {
570 out.push(` ${h.perPr.slice(i, i + 6).map(p => `#${p.pr} ${p.paidRelevant ? 'paid' : '----'} ${p.prCausedRed ? 'red ' : '--- '}${p.screenedMiss ? 'miss' : '----'}`).join(' ')}`);
571 }
572 const b = h.baseline;
573 const hrs = (v: number) => v.toFixed(1);
574 const p = h.trainParams;
575 out.push('', 'READY-to-merge. READY = when the merged own diff (modulo main merges and stamps) first had green Free Tests and E2E Evals on a',
576 ' head, before the merge (fallback: the final head\'s green time; READY reports are not machine-readable).',
577 ` READY signal: ${Object.entries(b.sources).map(([k, n]) => `${k} ${n}`).join(', ')}`,
578 ` today (one PR per gate): ${b.prs} PRs, READY-to-merge p50 ${hrs(b.p50Hours)} h, p90 ${hrs(b.p90Hours)} h, mean ${hrs(b.meanHours)} h; ${b.gatesAfterReady} gated E2E Evals runs after READY`,
579 ` trains (${p.reps} reps, seed ${p.seed}; size ${p.trainSize}, fill limit ${p.fillLimitHours} h, screened first, unscreened alone when ${p.trainSize}+ wait; ${p.gateMinutes} min a gate,`,
580 ` ${p.bisectMinutes} min a bisect, one train gating at a time; defects ${p.redScreened * 100}% screened / ${p.redUnscreened * 100}% unscreened; a dropped PR rejoins at once):`,
581 ` ${pad(['policy', 'o', 'trains/w', 'occupancy', 'full', 'sizes 1..k per window', 'gates/w', 'drop+gate/w', 'rebld/w', 'stalls/w', 'p50 h', 'p90 h'], [10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} mean h`);
582 for (const r of h.trains) {
583 out.push(` ${pad([r.policy, r.outsideRed.toFixed(2), r.trainsWeekly.toFixed(2), r.meanOccupancy.toFixed(2), pct(r.fullTrainShare, 1), r.occupancy.map(n => n.toFixed(1)).join('/'),
584 r.gateAttemptsWeekly.toFixed(2), r.attributedRebuildsWeekly.toFixed(2), r.rebuildOnceWeekly.toFixed(2), r.stallsWeekly.toFixed(2), hrs(r.readyToMerge.p50Hours), hrs(r.readyToMerge.p90Hours)],
585 [10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} ${hrs(r.readyToMerge.meanHours)}`);
586 }
587 out.push(` "screened" follows the ${label} policy column; PRs with no paid case ride as screened.`);
588 return out.join('\n');
589}
591export function main(argv: string[]): number {
592 let args: ReplayArgs;
593 try { args = parseArgs(argv); } catch (error) {
594 console.error(`gate-cost-replay: ${(error as Error).message}`);
595 return 2;
596 }
597 try {
598 const repo = args.repo || repoSlug(ROOT);
599 fs.mkdirSync(args.cache, { recursive: true });
600 const gates = args.runs.map(run => readGateRecord(fetchRun(run, args.cache, repo), run));
601 const empty = gates.filter(g => g.trials.length === 0);
602 if (empty.length) throw new Error(`run(s) ${empty.map(g => g.runId).join(', ')} have no trial outcomes (expired artifacts, or not a paid gate run)`);
603 const merges = mergedPrs(args);
604 const deps = checkoutSelectionDeps();
605 const body = buildReport(args, gates, merges, deps);
606 const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...body,
607 ...(args.history ? { history: collectHistory(args, repo, merges, body, deps, gates) } : {}) };
608 const text = formatReport(report, { cases: args.cases }) + (report.history ? `\n\n${formatHistory(report.history)}` : '');
609 console.log(args.json ? JSON.stringify(report, null, 2) : text);
610 return 0;
611 } catch (error) {
612 console.error(`gate-cost-replay: ${(error as Error).message}`);
613 return 1;
614 }
615}
617if (import.meta.main) process.exit(main(process.argv.slice(2)));