train-backstop.ts
219 lines · scripts/train-backstop.ts at 3f79d81
1/**
2 * The main backstop (evals-main-backstop.yml; plan GSTA-49 lane T1b): every
3 * push to main is covered by a full gate, and a train's merges are covered by
4 * the gate the train already ran, so a train costs no second gate.
5 *
6 * 1. Inert unless main's .gstack/ship-policy.json declares a `train` block.
7 * 2. Debounce: wait until no train is merging (refs/notes/gstack-trains
8 * holds no train in state `merging`), up to the settle window; a newer
9 * push to main supersedes this run.
10 * 3. Compute main's tree modulo stamps (scripts/train-gate.ts stampFreeTree)
11 * and look for a green-train record that lists it among its prefix trees
12 * T1..Tk: a successful train dispatch of the gate workflow whose record
13 * artifact is GREEN with a complete census and whose `full_gate_check`
14 * check run on the train head succeeded.
15 * 4. Otherwise dispatch the full gate on main (eval-bg's backstop nonce),
16 * wait for it, and on anything but success write the freeze record
17 * gstack-ship-queue reads (red cases, their declared inputs, the run, an
18 * issue) to refs/notes/gstack-trains, unless a freeze is already active.
19 */
20import { randomBytes } from 'node:crypto';
21import * as fs from 'node:fs';
22import * as os from 'node:os';
23import * as path from 'node:path';
24import { loadPolicyFromHead, type TrainPolicy } from '../lib/ship-policy';
25import { NOTES_REF, fetchNotes, nowIso, persist, repoIdentity, type FreezeRecord } from '../lib/ship-queue-train-record';
26import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
27import { DISPATCH_INPUTS, makeNonce, nonceScope } from './eval-bg';
28import { ghFailureReason, runGh, type GhRunner } from './recover-receipts';
29import { gitIn, stampFreeTree, type Git, type TrainRecord } from './train-gate';
31const SHA = /^[0-9a-f]{40}$/;
32const GH_MS = 60_000;
34export type BackstopAction = 'inert' | 'superseded' | 'covered' | 'deferred' | 'gated' | 'frozen' | 'failed';
35export interface BackstopOptions {
36 repo: string; branch: string; sha: string; repoRoot: string;
37 gh?: GhRunner; git?: Git; now?: () => number; sleep?: (ms: number) => void;
38 settleMs?: number; pollMs?: number; gateDeadlineMs?: number; random?: string;
39}
40interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string }
42function json<T>(r: { status: number | null; stdout: string }): T | null {
43 if (r.status !== 0) return null;
44 try { return JSON.parse(r.stdout) as T; } catch { return null; }
45}
47/** Notes records of one kind for this repository (the local notes ref, after a fetch). */
48function noteRecords<T extends { kind: string; repo_key?: string }>(git: Git, kind: string, repoKey: string): T[] {
49 const listed = git(['notes', `--ref=${NOTES_REF}`, 'list']);
50 if (!listed.ok || !listed.out.trim()) return [];
51 return listed.out.split('\n').map(line => line.split(' ')[0]!).filter(Boolean).flatMap(blob => {
52 try { const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as T; return r?.kind === kind && r.repo_key === repoKey ? [r] : []; } catch { return []; }
53 });
54}
56/**
57 * The newest green-train record with a prefix covering main, with its run; null when none. A plain prefix T_i
58 * covers main's tree modulo stamps (its digest); a stamped prefix S_i covers only main's exact tree.
59 */
60export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, main: { digest: string | null; tree: string }, lines: string[]): { run: Run; record: TrainRecord; index: number; exact: boolean } | null {
61 const runs = json<{ workflow_runs?: Run[] }>(gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/${train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'status=success', '-f', 'per_page=50'], GH_MS));
62 if (!runs) { lines.push('cannot list the train gate runs; the backstop gates main'); return null; }
63 const trains = (runs.workflow_runs ?? []).map(run => ({ run, nonce: /\[eval-bg ([A-Za-z0-9._-]+)\]/.exec(run.display_title ?? '')?.[1] ?? '' }))
64 .filter(t => t.run.conclusion === 'success' && nonceScope(t.nonce)?.purpose === 'train').sort((a, b) => b.run.id - a.run.id).slice(0, 20);
65 const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'train-backstop-'));
66 try {
67 for (const { run, nonce } of trains) {
68 const dir = path.join(scratch, String(run.id));
69 if (gh(['run', 'download', String(run.id), '-R', repo, '-n', train.record_artifact, '-D', dir], GH_MS).status !== 0) continue;
70 const file = fs.existsSync(dir) ? fs.readdirSync(dir).find(name => name.endsWith('.json')) : undefined;
71 let record: TrainRecord | null = null;
72 try { record = file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) as TrainRecord : null; } catch { record = null; }
73 if (!record || record.kind !== 'train-gate' || record.verdict !== 'GREEN' || record.census?.complete !== true || record.nonce !== nonce
74 || record.head_sha !== run.head_sha || !Array.isArray(record.prefixes)) continue;
75 const hit = record.prefixes.find(p => p.exact ? p.tree === main.tree : main.digest !== null && p.stamp_free_tree === main.digest);
76 if (!hit) continue;
77 const checks = json<{ check_runs?: Array<{ name?: string; conclusion?: string | null }> }>(
78 gh(['api', `repos/${repo}/commits/${run.head_sha}/check-runs?check_name=${encodeURIComponent(train.full_gate_check)}`], GH_MS));
79 if (!checks?.check_runs?.some(c => c.name === train.full_gate_check && c.conclusion === 'success')) {
80 lines.push(`train ${nonce} (run ${run.id}) lists this tree but its ${train.full_gate_check} check run is not a success`);
81 continue;
82 }
83 return { run, record, index: hit.index, exact: !!hit.exact };
84 }
85 return null;
86 } finally {
87 fs.rmSync(scratch, { recursive: true, force: true });
88 }
89}
91/** Red cases of a finished gate run, from its trial outcomes (first attempt), with their declared inputs. */
92function redCases(gh: GhRunner, repo: string, run: Run): Array<{ id: string; inputs: string[] }> {
93 const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'train-backstop-red-'));
94 try {
95 if (gh(['run', 'download', String(run.id), '-R', repo, '-n', 'trial-outcomes-pr-a1', '-D', dir], GH_MS).status !== 0) return [];
96 const text = fs.existsSync(path.join(dir, 'trial-outcomes.jsonl')) ? fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8') : '';
97 const ids = new Set<string>();
98 for (const line of text.split('\n').filter(Boolean)) {
99 try { const r = JSON.parse(line) as { case?: string; outcome?: string; attempt?: number }; if (r.outcome === 'failed' && (r.attempt ?? 1) === 1 && r.case) ids.add(r.case); } catch { /* skip */ }
100 }
101 return [...ids].sort().map(id => ({ id, inputs: E2E_TOUCHFILES[id] ?? LLM_JUDGE_TOUCHFILES[id] ?? [] }));
102 } finally {
103 fs.rmSync(dir, { recursive: true, force: true });
104 }
105}
107/** Dispatch the full gate on main, find its run by nonce, and follow it to its conclusion. */
108function gateMain(o: Required<Pick<BackstopOptions, 'repo' | 'branch' | 'sha'>> & { gh: GhRunner; train: TrainPolicy; now: () => number; sleep: (ms: number) => void; pollMs: number; deadline: number; nonce: string },
109 lines: string[]): Run | null {
110 const inputs = { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: o.sha, [DISPATCH_INPUTS.nonce]: o.nonce };
111 const sent = o.gh(['workflow', 'run', o.train.gate_workflow, '--repo', o.repo, '--ref', o.branch, ...Object.entries(inputs).flatMap(([k, v]) => ['-f', `${k}=${v}`])], GH_MS);
112 if (sent.status !== 0) { lines.push(`the gate dispatch failed (${ghFailureReason(sent.stderr)})`); return null; }
113 lines.push(`BACKSTOP: dispatched ${o.train.gate_workflow} on ${o.branch}@${o.sha.slice(0, 12)} (nonce ${o.nonce})`);
114 let run: Run | null = null;
115 while (o.now() < o.deadline) {
116 if (!run) {
117 const runs = json<{ workflow_runs?: Run[] }>(o.gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/${o.train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=30'], GH_MS));
118 run = runs?.workflow_runs?.find(r => (r.display_title ?? '').includes(`eval-bg ${o.nonce}]`)) ?? null;
119 if (run) lines.push(`BACKSTOP: gate run ${run.id} ${run.html_url ?? ''}`.trim());
120 } else {
121 run = json<Run>(o.gh(['api', `repos/${o.repo}/actions/runs/${run.id}`], GH_MS)) ?? run;
122 if (run.status === 'completed') return run;
123 }
124 o.sleep(o.pollMs);
125 }
126 lines.push(`the gate ${run ? `run ${run.id}` : 'dispatch'} did not finish before the backstop deadline`);
127 return run;
128}
130export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines: string[] } {
131 const gh = o.gh ?? runGh;
132 const git = o.git ?? gitIn(o.repoRoot);
133 const now = o.now ?? Date.now;
134 const sleep = o.sleep ?? ((ms: number) => Bun.sleepSync(ms));
135 const pollMs = o.pollMs ?? 60_000;
136 const lines: string[] = [];
137 const done = (action: BackstopAction) => ({ action, lines });
138 const loaded = loadPolicyFromHead(o.repoRoot);
139 const train = loaded.policy.train;
140 if (!train) { lines.push(`BACKSTOP: inert (${loaded.source.label} declares no train block; per-PR gates cover main)`); return done('inert'); }
141 if (!SHA.test(o.sha)) { lines.push(`BACKSTOP: ${o.sha} is not a commit SHA`); return done('failed'); }
142 const repoKey = repoIdentity(o.repoRoot).key;
143 const settleUntil = now() + (o.settleMs ?? 60 * 60_000);
144 for (;;) {
145 const fetched = fetchNotes(o.repoRoot);
146 const merging = noteRecords<{ kind: string; repo_key: string; nonce: string; state: string }>(git, 'train', repoKey).filter(t => t.state === 'merging');
147 const tip = git(['ls-remote', 'origin', `refs/heads/${o.branch}`]).out.split(/\s/)[0];
148 if (tip && tip !== o.sha) { lines.push(`BACKSTOP: superseded (${o.branch} moved to ${tip.slice(0, 12)}; that push's run gates it)`); return done('superseded'); }
149 if (!merging.length) break;
150 if (now() >= settleUntil) { lines.push(`BACKSTOP: train(s) ${merging.map(t => t.nonce).join(', ')} still merging after the settle window; gating anyway`); break; }
151 lines.push(`BACKSTOP: waiting for train(s) ${merging.map(t => t.nonce).join(', ')} to finish merging${fetched.ok ? '' : ` (${fetched.detail})`}`);
152 sleep(pollMs);
153 }
154 const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
155 if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); }
156 const tree = git(['rev-parse', `${o.sha}^{tree}`]).out;
157 const covered = coveringRecord(gh, o.repo, train, { digest: typeof digest === 'string' ? digest : null, tree }, lines);
158 if (covered) {
159 lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is ${covered.exact ? `S${covered.index}` : `T${covered.index}`} of green train ${covered.record.nonce} (run ${covered.run.id}, ${covered.exact ? `exact tree ${tree.slice(0, 12)}` : `tree modulo stamps ${String(digest).slice(0, 12)}`}); no gate`);
160 return done('covered');
161 }
162 const owned = noteRecords<FreezeRecord>(git, 'freeze', repoKey).find(f => !f.cleared_at && f.main.sha === o.sha && f.backstop);
163 if (owned?.backstop) {
164 lines.push(`BACKSTOP: deferred: train ${owned.train} froze the queue on this commit and dispatched backstop run ${owned.backstop.run ?? 'unknown'} (nonce ${owned.backstop.nonce}); no second gate`);
165 return done('deferred');
166 }
167 lines.push(`BACKSTOP: no green-train record lists main's tree modulo stamps${typeof digest === 'string' ? ` ${digest.slice(0, 12)}` : ''}; gating main`);
168 const nonce = makeNonce('gate', o.sha, o.sha, o.random ?? randomBytes(3).toString('hex'), { purpose: 'backstop' });
169 const run = gateMain({ repo: o.repo, branch: o.branch, sha: o.sha, gh, train, now, sleep, pollMs, deadline: now() + (o.gateDeadlineMs ?? 270 * 60_000), nonce }, lines);
170 if (run?.status === 'completed' && run.conclusion === 'success') { lines.push(`BACKSTOP: gated: run ${run.id} is green on main ${o.sha.slice(0, 12)}`); return done('gated'); }
171 return freeze(o, { gh, git, repoKey, nonce, run }, lines);
172}
174/** Write the freeze record (local state, then refs/notes/gstack-trains) unless a freeze is already active. */
175function freeze(o: BackstopOptions, c: { gh: GhRunner; git: Git; repoKey: string; nonce: string; run: Run | null }, lines: string[]): { action: BackstopAction; lines: string[] } {
176 const active = noteRecords<FreezeRecord>(c.git, 'freeze', c.repoKey).find(f => !f.cleared_at);
177 if (active) { lines.push(`BACKSTOP: red, and the queue is already frozen (${active.reason}); the existing freeze stands`); return { action: 'frozen', lines }; }
178 const red = c.run?.status === 'completed' ? redCases(c.gh, o.repo, c.run) : [];
179 const runText = c.run ? `run ${c.run.id} (${c.run.status}/${c.run.conclusion ?? '-'})` : 'no run';
180 const cases = red.length ? red.map(r => `${r.id} (inputs ${r.inputs.slice(0, 4).join(', ') || 'none declared'})`).join('; ') : 'no red case recorded';
181 const body = [`The main backstop gate is not green on ${o.sha}: ${runText}.`, '', `Red cases: ${cases}.`, '', 'gstack-ship-queue refuses ordinary trains until a repair train merges (docs/ship-policy.md#run-a-merge-train). Land the revert or fix-forward.'].join('\n');
182 const issue = c.gh(['issue', 'create', '-R', o.repo, '--title', `Main backstop red on ${o.sha.slice(0, 12)}`, '--body', body], GH_MS);
183 const issueUrl = issue.status === 0 ? issue.stdout.trim().split('\n').at(-1) ?? null : null;
184 const at = nowIso();
185 const record: FreezeRecord & { red_cases: typeof red; run_url: string | null; issue: string | null } = {
186 schema: 1, kind: 'freeze', repo_key: c.repoKey, train: 'backstop', merged: [], unmerged: [],
187 main: { sha: o.sha, tree: c.git(['rev-parse', `${o.sha}^{tree}`]).out },
188 reason: `backstop red on main ${o.sha.slice(0, 12)}: ${cases}; ${runText}; fix: land the revert or fix-forward${issueUrl ? ` (${issueUrl})` : ''}`,
189 backstop: { nonce: c.nonce, run: c.run?.id ?? null, state: c.run ? `${c.run.status}/${c.run.conclusion ?? '-'}` : 'unconfirmed' },
190 repair: null, created_at: at, cleared_at: null, cleared_by: null, updated_at: at,
191 red_cases: red, run_url: c.run?.html_url ?? null, issue: issueUrl,
192 };
193 const pushed = persist(o.repoRoot, [record]);
194 lines.push(`BACKSTOP: frozen: ${record.reason}`, `BACKSTOP: freeze record ${pushed.detail}`);
195 return { action: pushed.pushed ? 'frozen' : 'failed', lines };
196}
198const USAGE = `usage: bun run scripts/train-backstop.ts run --repo <owner/name> --branch <main> --sha <sha>
199 [--settle-minutes 60] [--poll-seconds 60] [--gate-deadline-minutes 270] [--summary <file>] [--outputs <file>]
200Exit 0 when inert, superseded, covered, deferred to a freeze's own backstop, or gated green; 1 when it froze the queue or could not finish.
201--outputs appends action=<action> (the workflow reads it as a step output and publishes the main-backstop check run from it).`;
203if (import.meta.main) {
204 const [command, ...rest] = process.argv.slice(2);
205 const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
206 const minutes = (name: string, fallback: number) => Number(flag(name) ?? fallback) * 60_000;
207 const [repo, branch, sha] = [flag('repo'), flag('branch'), flag('sha')];
208 if (command !== 'run' || !repo || !branch || !sha) { console.error(USAGE); process.exit(2); }
209 const { action, lines } = runBackstop({ repo: repo!, branch: branch!, sha: sha!, repoRoot: process.cwd(), settleMs: minutes('settle-minutes', 60),
210 pollMs: Number(flag('poll-seconds') ?? 60) * 1000, gateDeadlineMs: minutes('gate-deadline-minutes', 270) });
211 const summary = flag('summary');
212 for (const line of lines) {
213 console.log(line);
214 if (summary) fs.appendFileSync(summary, `- ${line}\n`);
215 }
216 const outputs = flag('outputs');
217 if (outputs) fs.appendFileSync(outputs, `action=${action}\n`);
218 process.exit(action === 'frozen' || action === 'failed' ? 1 : 0);
219}