ship-measure-sweep.ts
352 lines · scripts/ship-measure-sweep.ts at 3f79d81
1/**
2 * ship-measure sweep — the weekly off-ship qualification sweep on main
3 * (approved by Garry 2026-10-06; docs/TESTING_INTERNALS.md#ship-measure-sweep).
4 *
5 * With ~130 gate verdicts, an all-green gate is unlikely even when every case
6 * is healthy, so flaky cases are measured on main before any ship hits them.
7 * The sweep reads eval:pass-rates history (evals-periodic.yml on main plus the
8 * branch census runs that pool into it), ranks the gate cases with a failed
9 * trial by how much each costs the gate's all-green probability Π(1 − p_i)
10 * (p_i: its red-verdict rate shrunk toward the pooled rate, as
11 * `eval:pass-rates --reds` computes it; ties broken by the lower per-trial
12 * pass rate), and measures the top K at their kind's N on the
13 * checked-out head with the same runner and measurement bar /ship uses
14 * (scripts/ship-measure.ts, scripts/lib/measure-bar.ts).
15 *
16 * Spend: ship_measure_sweep_budget_usd (default $150) is a weekly cap enforced
17 * through the same admission budget as /ship's loop, shared across the
18 * sweep's cases and every sweep report of the last 7 days found under --prior
19 * (CI downloads the earlier runs' reports there) and the local sweep root.
20 * When the cap is exhausted the sweep stops and lists what it skipped.
21 *
22 * Every trial is diagnostic: it never becomes a verdict and never enters
23 * EVAL_POLICY pooling or series history. The sweep never pushes, commits or
24 * opens a pull request; an agent fixes from its report.
25 */
26import * as fs from 'node:fs';
27import * as path from 'node:path';
28import { spawnSync } from 'node:child_process';
29import { CASE_QUARANTINE } from '../test/helpers/periodic-exclude-data';
30import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
31import { parseCliFlags } from './lib/shard-engine';
32import { RED_RATE_SHRINK, verdictsOf } from './lib/eval-history';
33import { trialPassed } from './lib/measure-bar';
34import { repoSlug } from './lib/ci-history';
35import { loadPassRateHistory, type TrialRecord as HistoryRecord } from './eval-flake-rank';
36import {
37 caseSelectionPreflight, caseSlug, commandRunner, gstackRunner, kindPlan, measureCase, measurementIdentity, readMeasureConfig,
38 type MeasureConfig, type MeasureKind, type Measurement, type SpendPool, type TrialRunner,
39} from './ship-measure';
41const ROOT = path.resolve(import.meta.dir, '..');
42const WEEK_MS = 7 * 24 * 3600_000;
43export const SWEEP_SCHEMA = 'gstack-ship-measure-sweep/1';
45export interface SweepCandidate {
46 case: string;
47 kind: MeasureKind;
48 /** Red-verdict rate over gate history, shrunk toward the pooled rate (K = RED_RATE_SHRINK, as eval:pass-rates --reds). */
49 redRate: number;
50 reds: number;
51 verdicts: number;
52 /** Per-trial pass rate over the history window, or null without scored trials. */
53 passRate: number | null;
54 scoredTrials: number;
55 /** All-green probability gained if this case never went red: P / (1 − p_i) − P. */
56 contribution: number;
57 /** Mean known cost per trial from history (else the gate-wide mean), or null. */
58 costPerTrialUsd: number | null;
59}
61export interface SweepPlan {
62 candidates: SweepCandidate[];
63 picked: SweepCandidate[];
64 /** All-green probability over the latest census's gate verdicts, from history. */
65 allGreen: number | null;
66 censusCases: string[];
67 pooledRate: number;
68 /** Per-case shrunk red rates for every case in the latest census. */
69 rates: Record<string, number>;
70}
72const liveCase = (id: string) => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id);
73export const sweepKind = (id: string): MeasureKind => (E2E_KINDS[id] as MeasureKind | undefined) ?? (Object.hasOwn(E2E_TIERS, id) ? 'rule' : 'judge');
75/**
76 * Rank the gate cases history flags: every live, non-quarantined case of the
77 * latest gate census with a failed trial in the window, by its contribution
78 * to the all-green probability, then by the lower per-trial pass rate. A red
79 * verdict here is a lane-failing verdict with at least one failed trial (an
80 * INCOMPLETE record-count verdict is a reporting fault, not a flake). Picks
81 * the top k.
82 */
83export function planSweep(records: readonly HistoryRecord[], k: number, opts: { quarantine?: Record<string, unknown> } = {}): SweepPlan {
84 const quarantine = opts.quarantine ?? CASE_QUARANTINE;
85 const gate = records.filter(r => r.tier === 'gate' && r.policy_version >= 1);
86 const verdicts = verdictsOf(gate).filter(v => v.status !== 'SKIPPED');
87 const red = (v: (typeof verdicts)[number]) => v.failsLane && v.trials.some(t => t.outcome === 'failed');
88 const pooledRate = verdicts.length ? verdicts.filter(red).length / verdicts.length : 0;
89 const byCase = new Map<string, { verdicts: number; reds: number }>();
90 for (const v of verdicts) {
91 const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 };
92 e.verdicts++; if (red(v)) e.reds++;
93 byCase.set(v.case, e);
94 }
95 const rates: Record<string, number> = Object.fromEntries([...byCase].map(([id, e]) => [id, (e.reds + RED_RATE_SHRINK * pooledRate) / (e.verdicts + RED_RATE_SHRINK)]));
96 const latestRun = verdicts.map(v => v.run).filter(run => run !== 'local').sort().at(-1) ?? 'local';
97 const censusCases = [...new Set(verdicts.filter(v => v.run === latestRun).map(v => v.case))].sort();
98 const allGreen = censusCases.length ? censusCases.reduce((p, id) => p * (1 - rates[id]!), 1) : null;
99 const scored = gate.filter(r => r.execution === 'executed' && (r.outcome === 'passed' || r.outcome === 'failed'));
100 const knownCost = (r: HistoryRecord) => r.execution === 'executed' && r.cost_known !== false && r.cost_usd > 0;
101 const mean = (xs: number[]) => xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null;
102 const gateCost = mean(records.filter(r => r.tier === 'gate' && knownCost(r)).map(r => r.cost_usd));
103 const candidates: SweepCandidate[] = [];
104 for (const id of censusCases) {
105 if (!liveCase(id) || Object.hasOwn(quarantine, id)) continue;
106 const trials = scored.filter(r => r.case === id);
107 const failedTrials = trials.filter(r => r.outcome === 'failed').length;
108 const e = byCase.get(id)!;
109 if (failedTrials === 0) continue;
110 candidates.push({
111 case: id, kind: sweepKind(id), redRate: rates[id]!, reds: e.reds, verdicts: e.verdicts,
112 passRate: trials.length ? (trials.length - failedTrials) / trials.length : null, scoredTrials: trials.length,
113 contribution: allGreen === null ? 0 : allGreen / (1 - rates[id]!) - allGreen,
114 costPerTrialUsd: mean(records.filter(r => r.case === id && knownCost(r)).map(r => r.cost_usd)) ?? gateCost,
115 });
116 }
117 candidates.sort((a, b) => b.contribution - a.contribution || b.redRate - a.redRate || (a.passRate ?? 1) - (b.passRate ?? 1) || (a.case < b.case ? -1 : 1));
118 return { candidates, picked: candidates.slice(0, k), allGreen, censusCases, pooledRate, rates };
119}
121/**
122 * A measured case's predicted red-verdict rate: its measured per-trial failure
123 * fraction shrunk toward the pooled rate with the same K as history, and for a
124 * behavior case the 3-trial panel's red risk 1 − P(at least 2 of 3 pass).
125 */
126export function measuredRedRate(kind: MeasureKind, failures: number, trials: number, pooledRate: number): number {
127 const q = (failures + RED_RATE_SHRINK * pooledRate) / (trials + RED_RATE_SHRINK);
128 return kind === 'behavior' ? 1 - ((1 - q) ** 3 + 3 * (1 - q) ** 2 * q) : q;
129}
131/** Π(1 − p_i) over the census cases, with measured cases' rates replacing history's. */
132export function predictedAllGreen(plan: SweepPlan, measured: Record<string, number>): number | null {
133 if (plan.allGreen === null) return null;
134 return plan.censusCases.reduce((p, id) => p * (1 - (measured[id] ?? plan.rates[id] ?? plan.pooledRate)), 1);
135}
137export interface SweepRow {
138 rank: number;
139 case: string;
140 kind: MeasureKind;
141 historyPassRate: number | null;
142 historyTrials: number;
143 reds: number;
144 verdicts: number;
145 redRate: number;
146 status: 'planned' | 'measured' | 'budget_exhausted' | 'skipped' | 'error';
147 decision: string;
148 measuredPasses: number | null;
149 measuredTrials: number | null;
150 estimatedUsd: number | null;
151 actualUsd: number;
152 costUnknownTrials: number;
153 captures: string | null;
154 note: string;
155}
157export interface SweepReport {
158 schema: typeof SWEEP_SCHEMA;
159 label: 'diagnostic';
160 verdict: null;
161 sweepId: string;
162 startedAt: string;
163 finishedAt: string | null;
164 ref: string;
165 sha: string;
166 dryRun: boolean;
167 k: number;
168 capUsd: number;
169 priorWeekUsd: number;
170 /** Charged against the weekly cap by this sweep: known costs plus the estimate for trials whose cost is unknown. */
171 chargedUsd: number;
172 historyScope: string;
173 allGreenBefore: number | null;
174 allGreenAfter: number | null;
175 censusVerdicts: number;
176 rows: SweepRow[];
177}
179/** Spend charged by sweep reports under dirs started within the last 7 days (deduplicated by sweep id). */
180export function weeklySpent(dirs: readonly string[], now: number, excludeId?: string): number {
181 const seen = new Map<string, number>();
182 for (const dir of dirs) {
183 if (!fs.existsSync(dir)) continue;
184 for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) {
185 if (path.basename(name) !== 'sweep-report.json') continue;
186 let report: SweepReport;
187 try { report = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')) as SweepReport; } catch { continue; }
188 if (report.schema !== SWEEP_SCHEMA || report.sweepId === excludeId || report.dryRun) continue;
189 const started = Date.parse(report.startedAt);
190 if (!Number.isFinite(started) || now - started > WEEK_MS || started > now) continue;
191 seen.set(report.sweepId, Math.max(seen.get(report.sweepId) ?? 0, report.chargedUsd));
192 }
193 }
194 return [...seen.values()].reduce((a, b) => a + b, 0);
195}
197const pct = (x: number | null) => x === null ? 'n/a' : `${Math.round(x * 1000) / 10}%`;
198const prob = (x: number | null) => x === null || x >= 0.001 || x === 0 ? pct(x) : `${(x * 100).toPrecision(2)}%`;
199const money = (x: number | null) => x === null ? 'unknown' : `$${x.toFixed(2)}`;
200const cell = (text: string) => text.replace(/\|/g, '/').replace(/\n/g, ' ');
202export function formatSweepReport(r: SweepReport): string {
203 const rows = r.rows.map(row => `| ${row.rank} | ${cell(row.case)} | ${row.kind} | ${pct(row.historyPassRate)} of ${row.historyTrials} trials; ${row.reds}/${row.verdicts} red verdicts (rate ${pct(row.redRate)}) | `
204 + `${row.measuredTrials === null ? '—' : `${row.measuredPasses}/${row.measuredTrials}`} | ${cell(row.decision)} | ${money(row.estimatedUsd)} / ${money(row.actualUsd)}${row.costUnknownTrials ? ` (+${row.costUnknownTrials} unknown)` : ''} | ${row.captures ? `\`${cell(row.captures)}\`` : '—'} |`);
205 const noted = r.rows.filter(row => row.status !== 'measured');
206 return [
207 `## ship-measure sweep${r.dryRun ? ' (dry run: nothing ran, $0)' : ''}: diagnostic measurements, not verdicts`,
208 '',
209 `Ref ${r.ref} at ${r.sha.slice(0, 12)}; K=${r.k}; weekly cap $${r.capUsd} (charged earlier this week $${r.priorWeekUsd.toFixed(2)}, this sweep $${r.chargedUsd.toFixed(2)}). History: ${r.historyScope}.`,
210 '',
211 `Predicted gate all-green probability (approximation, assumes independent verdicts, over the latest census's ${r.censusVerdicts} gate verdicts): before ${prob(r.allGreenBefore)} → after ${prob(r.allGreenAfter)}${r.dryRun ? ' (no measurements yet)' : ' (measured cases use this sweep\'s rates)'}.`,
212 '',
213 '| # | Case | Kind | History rate | Measured | Decision | Cost (est / actual) | Captures |',
214 '|---|---|---|---|---|---|---|---|',
215 ...(rows.length ? rows : ['| — | no case in the latest gate census has a failed trial or red verdict | | | | | | |']),
216 ...(noted.length ? ['', r.dryRun ? 'Plan:' : 'Skipped or stopped:', ...noted.map(row => `- ${row.case}: ${row.note}`)] : []),
217 '',
218 'Every trial is diagnostic: it never becomes a verdict and never enters pass-rate history. The sweep never pushes or opens a pull request. '
219 + 'Fix a BELOW case at its cause, classify a needs-classify case (`ship-measure classify --out <captures root> --case <id> --round baseline ...`), and measure again on the fix.',
220 ].join('\n');
221}
223export interface SweepOptions {
224 records: readonly HistoryRecord[];
225 historyScope: string;
226 k: number;
227 capUsd: number;
228 config: MeasureConfig;
229 outDir: string;
230 priorDirs: string[];
231 dryRun: boolean;
232 parallel: number;
233 ref: string;
234 sha: string;
235 now?: number;
236 /** One runner for every case (default: gstack's single-case runner after a selection preflight). */
237 runner?: TrialRunner;
238 runnerId?: string;
239 preflight?: (caseId: string) => { ok: boolean; detail: string };
240 identity?: (kind: MeasureKind, trials: number) => Record<string, string>;
241 evalDir?: string;
242 log?: (line: string) => void;
243}
245/** Plan, measure in rank order under the weekly cap, and write sweep-report.json/.md after every case. */
246export async function runSweep(o: SweepOptions): Promise<SweepReport> {
247 const log = o.log ?? ((line: string) => console.log(line));
248 const now = o.now ?? Date.now();
249 const sweepId = new Date(now).toISOString().replace(/[:.]/g, '-');
250 const dir = path.join(o.outDir, `sweep-${sweepId}`);
251 fs.mkdirSync(dir, { recursive: true });
252 const plan = planSweep(o.records, o.k);
253 const priorWeekUsd = weeklySpent([...o.priorDirs, o.outDir], now, sweepId);
254 const pool: SpendPool = { label: `weekly sweep cap`, capUsd: o.capUsd, spentUsd: priorWeekUsd };
255 const report: SweepReport = {
256 schema: SWEEP_SCHEMA, label: 'diagnostic', verdict: null, sweepId, startedAt: new Date(now).toISOString(), finishedAt: null, ref: o.ref, sha: o.sha,
257 dryRun: o.dryRun, k: o.k, capUsd: o.capUsd, priorWeekUsd, chargedUsd: 0, historyScope: o.historyScope,
258 allGreenBefore: plan.allGreen, allGreenAfter: plan.allGreen, censusVerdicts: plan.censusCases.length, rows: [],
259 };
260 const save = () => {
261 report.chargedUsd = Math.max(0, pool.spentUsd - priorWeekUsd);
262 fs.writeFileSync(path.join(dir, 'sweep-report.json'), `${JSON.stringify(report, null, 2)}\n`);
263 fs.writeFileSync(path.join(dir, 'sweep-report.md'), `${formatSweepReport(report)}\n`);
264 };
265 const measuredRates: Record<string, number> = {};
266 for (const [index, c] of plan.picked.entries()) {
267 const n = kindPlan(c.kind, o.config).trials;
268 const row: SweepRow = {
269 rank: index + 1, case: c.case, kind: c.kind, historyPassRate: c.passRate, historyTrials: c.scoredTrials, reds: c.reds, verdicts: c.verdicts, redRate: c.redRate,
270 status: 'planned', decision: 'planned', measuredPasses: null, measuredTrials: null,
271 estimatedUsd: c.costPerTrialUsd === null ? null : c.costPerTrialUsd * n, actualUsd: 0, costUnknownTrials: 0, captures: null, note: '',
272 };
273 report.rows.push(row);
274 if (o.dryRun) { row.note = `would measure ${n} trials at ${c.costPerTrialUsd === null ? 'an unknown cost (one calibration trial first)' : `${money(c.costPerTrialUsd)}/trial`}`; save(); continue; }
275 if (pool.spentUsd >= pool.capUsd) {
276 Object.assign(row, { status: 'skipped', decision: 'skipped', note: `weekly cap $${o.capUsd} exhausted ($${pool.spentUsd.toFixed(2)} charged in the last 7 days)` });
277 save();
278 continue;
279 }
280 const selected = (o.preflight ?? caseSelectionPreflight)(c.case);
281 if (!selected.ok) { Object.assign(row, { status: 'skipped', decision: 'skipped', note: `selection preflight failed (no paid call made): ${selected.detail}` }); save(); continue; }
282 let m: Measurement;
283 try {
284 m = await measureCase({
285 caseId: c.case, kind: c.kind, round: 'baseline', config: o.config, outDir: dir, parallel: o.parallel, costPerTrialUsd: c.costPerTrialUsd, approved: true,
286 runner: o.runner ?? gstackRunner(), pool, log,
287 identity: () => (o.identity ?? ((kind, trials) => measurementIdentity(kind, trials, o.runnerId ?? 'gstack')))(c.kind, n),
288 ...(o.evalDir ? { evalDir: o.evalDir } : {}),
289 });
290 } catch (error) {
291 Object.assign(row, { status: 'error', decision: 'error', note: (error as Error).message });
292 save();
293 continue;
294 }
295 const captures = path.relative(o.outDir, path.join(dir, caseSlug(c.case), 'baseline'));
296 const decided = m.status === 'measured' && m.decision !== 'void' && m.decision !== 'incomplete';
297 Object.assign(row, {
298 status: m.status === 'measured' ? 'measured' : 'budget_exhausted', decision: m.status === 'measured' ? m.decision : m.status.replace('_', ' '),
299 ...(decided ? { measuredPasses: m.passes, measuredTrials: m.counted } : { measuredPasses: m.trials.filter(t => trialPassed(c.kind, t)).length, measuredTrials: m.trials.length }),
300 actualUsd: m.actualUsd, costUnknownTrials: m.costUnknownTrials, captures,
301 note: m.status === 'measured' ? m.decisionReason : m.reason ?? '',
302 });
303 if (decided) {
304 measuredRates[c.case] = measuredRedRate(c.kind, m.counted - m.passes, m.counted, plan.pooledRate);
305 report.allGreenAfter = predictedAllGreen(plan, measuredRates);
306 }
307 save();
308 }
309 report.finishedAt = new Date().toISOString();
310 save();
311 log(formatSweepReport(report));
312 log(`[ship-measure] sweep report: ${path.join(dir, 'sweep-report.md')}`);
313 return report;
314}
316function gitText(args: string[]): string {
317 const r = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
318 return r.status === 0 ? r.stdout.trim() : 'unknown';
319}
321/** `ship-measure sweep` CLI. Exits 0 when the sweep completed (whatever it measured), 1 when history was unavailable, 4 on usage errors. */
322export async function sweepMain(argv: string[], env: NodeJS.ProcessEnv = process.env): Promise<number> {
323 const flags: Record<string, string> = {};
324 const historyDirs: string[] = [];
325 const priorDirs: string[] = [];
326 const value = (name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); flags[name] = v; };
327 const list = (into: string[], name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); into.push(path.resolve(v)); };
328 parseCliFlags(argv, {
329 ...Object.fromEntries(['--k', '--cap-usd', '--command', '--jobs', '--out', '--runs', '--repo'].map(name => [name, value(name)])),
330 '--history-dir': list(historyDirs, '--history-dir'), '--prior': list(priorDirs, '--prior'), '--dry-run': () => { flags['--dry-run'] = '1'; },
331 }, 'Usage: ship-measure sweep [--k K] [--cap-usd USD] [--dry-run] [--history-dir DIR]... [--prior DIR]... [--runs N] [--command \'CMD {case}\'] [--jobs N] [--out DIR]');
332 const config = readMeasureConfig();
333 const int = (raw: string | undefined, flag: string, fallback: number) => {
334 if (raw === undefined) return fallback;
335 if (!/^[1-9][0-9]*$/.test(raw)) throw new Error(`${flag} needs a positive integer. Received: ${raw}`);
336 return Number(raw);
337 };
338 const capUsd = flags['--cap-usd'] === undefined ? config.sweepBudgetUsd : Number(flags['--cap-usd']);
339 if (!Number.isFinite(capUsd) || capUsd <= 0) throw new Error(`--cap-usd needs a positive amount in USD. Received: ${flags['--cap-usd']}`);
340 const repo = flags['--repo'] ?? repoSlug(ROOT);
341 const history = loadPassRateHistory({ repo, workflow: 'evals-periodic.yml', branch: 'main', runsLimit: int(flags['--runs'], '--runs', 10), dirs: historyDirs, sinceDays: 60 });
342 if (history.historyError) { console.error(`[ship-measure] sweep: pass-rate history unavailable (${history.historyError}); nothing measured`); return 1; }
343 const outDir = path.resolve(flags['--out'] ?? path.join(process.cwd(), '.context', 'ship-measure'));
344 const report = await runSweep({
345 records: history.records, historyScope: history.scope, k: int(flags['--k'], '--k', config.sweepCases), capUsd, config, outDir, priorDirs,
346 dryRun: flags['--dry-run'] === '1', parallel: int(flags['--jobs'], '--jobs', Number(env.EVALS_JOBS) || 4),
347 ref: env.GITHUB_REF_NAME ?? gitText(['rev-parse', '--abbrev-ref', 'HEAD']), sha: env.GITHUB_SHA ?? gitText(['rev-parse', 'HEAD']),
348 ...(flags['--command'] ? { runner: commandRunner(flags['--command']), runnerId: `command:${flags['--command']}`, preflight: () => ({ ok: true, detail: 'documented command' }) } : {}),
349 });
350 if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, `${formatSweepReport(report)}\n`);
351 return 0;
352}