train-gate.ts
404 lines · scripts/train-gate.ts at 3f79d81
1/**
2 * The merge-train side of evals.yml (plan GSTA-49 lane T1b;
3 * docs/ship-policy.md#run-a-merge-train). gstack-ship-queue (lib/ship-queue-train*.ts)
4 * dispatches the runs and reads what they publish; this file is the workflow's half.
5 *
6 * route (route job, dispatches) names the run's mode and validates
7 * its inputs: a screen needs a PR, its pinned merge SHA and a
8 * policy; a bisect needs case ids from the full gate's own plan
9 * and prefix SHAs that are ancestors of the train head on
10 * train/<nonce>. Anything else is refused before any spend.
11 * bisect-plan (plan-slices) one executor per prefix, each expecting its SHA.
12 * bisect-prefix (eval-slices) one prefix: every red case once, one trial,
13 * written as the per-prefix record `train bisect` reads.
14 * train-record (slices-report, train runs) the green-train record: the census
15 * from the run's own plan manifest, one panel per planned case,
16 * and the prefix trees T1..Tk modulo stamps. GREEN only for a
17 * complete census: every planned case executed and passed, no
18 * unbound result, partial or cancelled shard, or slice killed
19 * before its upload step.
20 * screen-summary (plan-slices, PR runs under a train policy) the screening
21 * selection and its estimated cost; no paid case runs.
22 */
23import { spawnSync } from 'node:child_process';
24import { createHash } from 'node:crypto';
25import * as fs from 'node:fs';
26import * as path from 'node:path';
27import { normalizeChangelog } from '../lib/changelog-check';
28import { normalizeLockfile } from '../lib/evidence-bundle';
29import { stampMentions, rewriteStamps, versionAtRef } from '../lib/restamp';
30import { loadPolicyFromHead, matchesAny, type ShipPolicy } from '../lib/ship-policy';
31import type { BisectPrefixRecord, TrainGateRecord } from '../lib/ship-queue-train-verdict';
32import type { CasePanel } from '../lib/ship-queue-train-record';
33import { E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
34import { panelVerdict } from '../test/helpers/eval-store';
35import { UNRECORDED_SHARD_MS, nonceScope, plannedSerialMs } from './eval-bg';
36import { caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile } from './lib/paid-cases';
37import { buildRunManifest, loadPaidTestDurations, parseRunManifest, recordedShardMs, type PaidRunManifest } from './lib/paid-plan';
38import { loadSliceArtifacts } from './lib/paid-report';
39import type { FlakeRecord } from './lib/paid-rerun';
40import { paidSelectionEnv } from './lib/paid-select';
41import { ROOT } from './lib/paid-types';
43const SHA = /^[0-9a-f]{40}$/;
44const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/;
45export const JUDGE_FILE = 'test/skill-llm-eval.test.ts';
46/** T1 caps a train at 6 PRs, so a bisect has at most 7 prefixes (T0..T6). */
47export const MAX_PREFIXES = 7;
48export const MAX_BISECT_CASES = 40;
49/** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */
50export const FULL_GATE_USD = 93.14;
51/** T1's prefix commits: `train T<i>: merge #<pr> at <sha12>` (lib/ship-queue-train.ts buildChain). */
52const PREFIX_SUBJECT = /^train (?:T(\d+): merge #(\d+) at [0-9a-f]{12}|S(\d+): merge #(\d+) at [0-9a-f]{12} v[0-9.]+)$/;
54export type Git = (args: string[]) => { ok: boolean; out: string };
55export const gitIn = (cwd: string): Git => (args) => {
56 const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
57 return r.status === 0 ? { ok: true, out: (r.stdout ?? '').replace(/\n$/, '') } : { ok: false, out: (r.stderr || String(r.error ?? '')).trim() };
58};
60export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch';
61export interface RouteInput {
62 nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean;
63 cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string;
64}
65export interface Route { mode: RunMode; cases: string[]; prefixes: string[] }
67const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean);
69/**
70 * The run's mode from a dispatch's inputs, or why it is refused. Screens,
71 * trains and bisects exist only under a train policy; a bisect's case ids must
72 * be in the full gate's plan and each prefix an ancestor of the train head.
73 */
74export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, head: string): boolean; gateCases(): Map<string, string> }): Route | { error: string } {
75 const scope = i.nonce ? nonceScope(i.nonce) : null;
76 const trainOnly = (mode: string) => (i.trainPolicy ? null : { error: `a ${mode} dispatch needs a train block in the default branch's .gstack/ship-policy.json` });
77 if (i.screenPr || i.screenMergeSha || i.screenPolicy) {
78 if (!/^[1-9]\d{0,8}$/.test(i.screenPr)) return { error: `screen_pr ${JSON.stringify(i.screenPr)} is not a PR number` };
79 if (!SHA.test(i.screenMergeSha)) return { error: `screen_merge_sha ${JSON.stringify(i.screenMergeSha)} is not a full commit SHA` };
80 if (i.screenPolicy !== 'narrow' && i.screenPolicy !== 'broad') return { error: `screen_policy ${JSON.stringify(i.screenPolicy)} is not narrow or broad` };
81 if (i.cases || i.prefixShas) return { error: 'a screen takes no cases or prefix_shas' };
82 if (i.evalsAll) return { error: 'a screen runs the PR selection: dispatch it with evals_all=false' };
83 if (i.expectedSha && i.expectedSha !== i.screenMergeSha) return { error: `expected_sha ${i.expectedSha.slice(0, 12)} is not the screen's merge SHA ${i.screenMergeSha.slice(0, 12)}` };
84 if (scope && (scope.purpose !== 'screen' || scope.policy !== i.screenPolicy)) return { error: `nonce ${i.nonce} is not a ${i.screenPolicy} screen nonce` };
85 return trainOnly('screen') ?? { mode: 'screen', cases: [], prefixes: [] };
86 }
87 if (i.cases || i.prefixShas || scope?.purpose === 'bisect') {
88 const cases = list(i.cases);
89 const prefixes = list(i.prefixShas);
90 if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` };
91 if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/<nonce>, not ${i.refName}` };
92 if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` };
93 if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` };
94 if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` };
95 const bad = prefixes.find(sha => !SHA.test(sha));
96 if (bad) return { error: `prefix ${JSON.stringify(bad)} is not a full commit SHA` };
97 const outside = prefixes.find(sha => !deps.isAncestor(sha, i.head));
98 if (outside) return { error: `prefix ${outside.slice(0, 12)} is not an ancestor of the train head ${i.head.slice(0, 12)} on ${i.refName}` };
99 const plan = deps.gateCases();
100 const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id));
101 if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` };
102 return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes };
103 }
104 if (scope?.purpose === 'train') {
105 if (!i.refName.startsWith('train/')) return { error: `a train gate runs on train/<nonce>, not ${i.refName}` };
106 if (!i.evalsAll) return { error: 'a train gate is the full gate: dispatch it with evals_all=true' };
107 if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)}` };
108 return trainOnly('train') ?? { mode: 'train', cases: [], prefixes: [] };
109 }
110 return { mode: 'dispatch', cases: [], prefixes: [] };
111}
113/** Every case a plan manifest plans, with its kind: the census a complete gate must execute. */
114export function plannedCases(manifest: PaidRunManifest, rootDir = ROOT): Map<string, string> {
115 const cases = new Map<string, string>();
116 const e2e = manifest.selection?.e2e ?? null;
117 for (const entry of manifest.entries.filter(item => item.status === 'planned')) {
118 const file = shardFile(entry.file);
119 if (entry.trial) { cases.set(shardCaseId(entry.file)!, entry.trial.kind); continue; }
120 if (file === JUDGE_FILE) { for (const id of manifest.selection?.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES)) cases.set(id, 'judge'); continue; }
121 const only = shardCaseId(entry.file);
122 const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
123 for (const id of fileCaseRegistration(file, source).registered) {
124 if (E2E_TIERS[id] === manifest.tier && (only === null || id === only) && (e2e === null || e2e.includes(id)) && !(entry.excludeCases ?? []).includes(id)) cases.set(id, 'rule');
125 }
126 }
127 return cases;
128}
130/** The full gate's planned cases on this checkout (what a bisect may name). */
131export function fullGateCases(rootDir = ROOT): Map<string, string> {
132 return plannedCases(buildRunManifest({ tier: 'gate', profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir }), rootDir);
133}
135/**
136 * A release output or stamp path without its stamps: every version mentioned
137 * in a stamp form (the built-in forms and the policy's `stamp_patterns`
138 * templates) collapses to one placeholder. Normalising only the commit's own
139 * version left a tentative `*_KEY_VERSION = '<next minor>'` as written on the
140 * unstamped prefix and normalised once the slot stamp moved it to the released
141 * version, so the two digests differed and train-backstop found no covering
142 * record for main after that train (#3512), and a base-version mention on a
143 * line the base already carried was normalised on the prefix and kept on the
144 * stamped main. Every stamp-form mention collapsing makes the digest blind to
145 * which version a stamp names; the receipt (`stampOnlyLine`) is what judges a
146 * stamp commit's content.
147 */
148export function stampFreeText(text: string, templates: readonly string[]): string {
149 let out = text;
150 for (const v of [...stampMentions(text, templates)].sort((a, b) => b.length - a.length)) out = rewriteStamps(out, v, '\0VERSION', templates).text;
151 return out;
152}
154/**
155 * A commit's tree modulo stamps: every path with its mode and blob, except the
156 * files a restamp rewrites, which contribute their content without the stamp
157 * (the version file nothing, package.json without its version, lockfiles
158 * without their root version, the CHANGELOG with its headings normalized,
159 * release outputs and stamp paths with every stamp-form version mention
160 * replaced). Two commits that differ only by a restamp have the same digest.
161 */
162export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy: ShipPolicy): string | { error: string } {
163 const listed = git(['ls-tree', '-r', '-z', '--full-tree', commit]);
164 if (!listed.ok) return { error: `cannot list ${commit.slice(0, 12)} (${listed.out.split('\n')[0]})` };
165 const at = versionAtRef(repoRoot, commit);
166 const versionRel = 'rel' in at ? at.rel : 'VERSION';
167 const lines: string[] = [];
168 for (const item of listed.out.split('\0').filter(Boolean)) {
169 const [meta, rel] = [item.slice(0, item.indexOf('\t')), item.slice(item.indexOf('\t') + 1)];
170 const [mode, , blob] = meta.split(' ') as [string, string, string];
171 const base = path.basename(rel);
172 const stamp = rel === versionRel ? 'version' : base === 'package.json' ? 'manifest' : ['bun.lock', 'package-lock.json', 'npm-shrinkwrap.json'].includes(base) ? 'lockfile'
173 : rel === policy.changelog ? 'changelog' : matchesAny([...policy.stamp_paths, ...policy.release_outputs, ...policy.stamp_files], rel) ? 'release' : null;
174 if (!stamp) { lines.push(`${rel}\0${mode}\0${blob}`); continue; }
175 if (stamp === 'version') { lines.push(`${rel}\0${mode}\0(version)`); continue; }
176 const shown = git(['cat-file', 'blob', blob]);
177 if (!shown.ok) return { error: `cannot read ${rel} at ${commit.slice(0, 12)}` };
178 let text = shown.out;
179 if (stamp === 'manifest') { try { const parsed = JSON.parse(text); delete parsed.version; text = JSON.stringify(parsed); } catch { /* not JSON: kept verbatim */ } }
180 else if (stamp === 'lockfile') text = normalizeLockfile(base, text);
181 else if (stamp === 'changelog') text = normalizeChangelog(text);
182 else text = stampFreeText(text, policy.stamp_patterns);
183 lines.push(`${rel}\0${mode}\0${createHash('sha256').update(text).digest('hex')}`);
184 }
185 return createHash('sha256').update(lines.sort().join('\n')).digest('hex');
186}
188/** `exact`: a stamped prefix S_i (changelog_fragments trains), which covers only its exact tree. */
189export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string; exact?: true }
191/** T1..Tk (or stamped S1..Sk, never mixed) from the train head, walking first parents over the prefix commits. */
192export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } {
193 const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]);
194 if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` };
195 const found: TrainPrefixTree[] = [];
196 for (const line of log.out.split('\n')) {
197 const [commit, parents, subject] = line.split('\0') as [string, string, string];
198 const m = PREFIX_SUBJECT.exec(subject ?? '');
199 if (!m || parents.split(' ').length !== 2) break;
200 const index = Number(m[1] ?? m[3]);
201 const exact = m[3] !== undefined;
202 if (found.length && exact !== !!found[0]!.exact) return { error: `${commit.slice(0, 12)} mixes stamped and plain prefixes` };
203 if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` };
204 const digest = stampFreeTree(git, commit, repoRoot, policy);
205 if (typeof digest !== 'string') return digest;
206 found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest, ...(exact ? { exact: true as const } : {}) });
207 }
208 if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` };
209 return found.reverse();
210}
212interface OutcomeLine { case?: string; kind?: string; attempt?: number; outcome?: string; failure_class?: string; cost_usd?: number; cost_known?: boolean; execution?: string }
213interface CollectorPanel { case: string; status: string; redClass: string | null }
215function readJson<T>(file: string): T | null {
216 try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; }
217}
219/** One panel per planned case from the run's trial outcomes (trial panels from the collector's panelVerdict). */
220function casePanels(planned: Map<string, string>, outcomes: OutcomeLine[], collector: CollectorPanel[], flaky: FlakeRecord[] = []): CasePanel[] {
221 return [...planned].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, kind]) => {
222 const mine = outcomes.filter(o => o.case === id && o.execution !== 'reused');
223 const cost = mine.length && mine.every(o => o.cost_known !== false && typeof o.cost_usd === 'number')
224 ? Math.round(mine.reduce((sum, o) => sum + o.cost_usd!, 0) * 10_000) / 10_000 : null;
225 // Rule 16: the report resolved this case FLAKY (failed once, passed its one rerun): a PASS that names both runs.
226 const flake = flaky.find(f => f.case === id);
227 if (flake) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost, flaky: { first: flake.first.where, rerun: flake.rerun.where } };
228 const panel = collector.find(p => p.case === id);
229 if (panel) return { case: id, kind, status: panel.status, red_class: panel.redClass, cost_usd: cost };
230 const failed = mine.filter(o => o.outcome === 'failed');
231 if (failed.length) return { case: id, kind, status: 'FAIL', red_class: failed.every(o => o.failure_class === 'infra') ? 'INFRA' : 'VERDICT', cost_usd: cost };
232 if (mine.some(o => o.outcome === 'passed')) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost };
233 return { case: id, kind, status: mine.length ? 'SKIPPED' : 'MISSING', red_class: 'INCOMPLETE', cost_usd: cost };
234 });
235}
237export type TrainRecord = TrainGateRecord & { run: { id: number; attempt: number; url: string }; prefixes: TrainPrefixTree[] };
239/** The green-train record a train's own run publishes (T1's TrainGateRecord plus the run and the prefix trees). */
240export function buildTrainRecord(o: { reportDir: string; nonce: string; head: string; tree: string; reconcileExit: number; rootDir?: string;
241 prefixes: TrainPrefixTree[]; run: TrainRecord['run'] }): TrainRecord {
242 const manifest = parseRunManifest(fs.readFileSync(path.join(o.reportDir, 'manifest.json'), 'utf8'));
243 const planned = plannedCases(manifest, o.rootDir);
244 let lines: OutcomeLine[] = [];
245 try { lines = fs.readFileSync(path.join(o.reportDir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as OutcomeLine); } catch { lines = []; }
246 const primary = Math.min(...lines.map(l => l.attempt ?? 1), Infinity);
247 const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[]; flaky?: FlakeRecord[] }>(path.join(o.reportDir, 'collector-outcomes.json'));
248 const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? [], collector?.flaky ?? []);
249 const artifacts = loadSliceArtifacts(o.reportDir);
250 const first = Math.min(...artifacts.map(a => a.result.attempt ?? 1), Infinity);
251 const results = artifacts.filter(a => (a.result.attempt ?? 1) === first).map(a => a.result);
252 const reportedSlices = new Set(results.map(r => r.sliceIndex));
253 const reportedFiles = new Set(results.flatMap(r => r.outcomes.flatMap(outcome => outcome.files)));
254 const count = (judge: boolean, from: Iterable<[string, string]>) => [...from].filter(([, kind]) => (kind === 'judge') === judge).length;
255 const executed = panels.filter(p => p.status === 'PASS' || p.status === 'FAIL').map(p => [p.case, p.kind] as [string, string]);
256 const census = {
257 complete: false,
258 planned: { rule: count(false, planned), judge: count(true, planned) },
259 executed: { rule: count(false, executed), judge: count(true, executed) },
260 unbound: collector?.census?.unattributed?.length ?? 0,
261 partial_shards: results.filter(r => r.checkpoint).length,
262 cancelled_shards: manifest.entries.filter(e => e.status === 'planned' && reportedSlices.has(e.slice) && !reportedFiles.has(e.file)).length,
263 killed_before_upload: Array.from({ length: manifest.sliceCount }, (_, i) => i + 1).filter(index => !reportedSlices.has(index)).length,
264 };
265 census.complete = o.reconcileExit === 0 && !!collector && planned.size > 0 && census.executed.rule === census.planned.rule
266 && census.executed.judge === census.planned.judge && !census.unbound && !census.partial_shards && !census.cancelled_shards
267 && !census.killed_before_upload && panels.every(p => p.status === 'PASS');
268 const verdict = census.complete && collector?.verdict?.verdict === 'GREEN' ? 'GREEN' : 'RED';
269 return { schema: 1, kind: 'train-gate', nonce: o.nonce, head_sha: o.head, tree: o.tree, verdict, census, panels, run: o.run, prefixes: o.prefixes };
270}
272export interface BisectPlan { schema: 1; nonce: string; cases: Array<{ id: string; kind: string }>; prefixes: string[] }
274/** One executor per prefix; each case runs one trial (and at most one rule-16 rerun inside its 30 minutes), so a slice's ceiling grows with the case count. */
275export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: number[]; refs: string[] } {
276 const minutes = Math.min(350, 20 + 30 * plan.cases.length);
277 return { slices: plan.prefixes.map((_, i) => i + 1), timeouts: [0, ...plan.prefixes.map(() => minutes)], refs: ['', ...plan.prefixes] };
278}
280export type CaseRunner = (id: string, kind: string, evalDir: string) => Promise<{ status: string; redClass: string | null }>;
282/**
283 * Run every red case once on the checked-out prefix; a case that leaves no
284 * verdict is INCOMPLETE (T1 relaunches it). Rule 16 holds here as in the
285 * gate: a case verdict that fails (red class VERDICT) reruns once on the same
286 * prefix; passing makes it a flaky PASS naming both runs, failing again keeps
287 * it red. INFRA and INCOMPLETE are never rerun.
288 */
289export async function runBisectPrefix(plan: BisectPlan, index: number, o: { rootDir: string; evalDir: string; checkedOut: string; runCase?: CaseRunner }): Promise<BisectPrefixRecord> {
290 const sha = plan.prefixes[index - 1];
291 if (!sha) throw new Error(`bisect slice ${index} names no prefix (${plan.prefixes.length} planned)`);
292 if (o.checkedOut !== sha) throw new Error(`bisect slice ${index} checked out ${o.checkedOut.slice(0, 12)}, not its prefix ${sha.slice(0, 12)}`);
293 const runCase = o.runCase ?? ((id, kind, evalDir) => runOneTrial(id, kind, evalDir, o.rootDir));
294 const panels: CasePanel[] = [];
295 for (const c of plan.cases) {
296 const dir = path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_'));
297 const run = async (evalDir: string): Promise<{ status: string; redClass: string | null }> => {
298 try { return await runCase(c.id, c.kind, evalDir); }
299 catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); return { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; }
300 };
301 const verdict = await run(dir);
302 const rerun = verdict.status === 'FAIL' && verdict.redClass === 'VERDICT' ? await run(`${dir}.rerun`) : null;
303 const flaky = rerun?.status === 'PASS';
304 const final = flaky ? rerun! : verdict;
305 panels.push({ case: c.id, kind: c.kind, status: final.status, red_class: final.redClass, cost_usd: null,
306 ...(flaky ? { flaky: { first: `prefix ${sha.slice(0, 12)} slice ${index}`, rerun: `prefix ${sha.slice(0, 12)} slice ${index} rerun` } } : {}) });
307 console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}`
308 + `${rerun ? `; rule-16 rerun ${rerun.status}${rerun.redClass ? ` (${rerun.redClass})` : ''}${flaky ? ': FLAKY, counted as PASS' : ': failed twice, red'}` : ''}`);
309 }
310 return { schema: 1, kind: 'train-bisect-prefix', nonce: plan.nonce, prefix_sha: sha, checked_out_sha: o.checkedOut, panels };
311}
313/** One trial of one case exactly as CI runs it: E2E cases through the panel runner, a judge through its file with only it selected. */
314async function runOneTrial(id: string, kind: string, evalDir: string, rootDir: string): Promise<{ status: string; redClass: string | null }> {
315 const { runPaidShards } = await import('./test-paid-shards');
316 if (kind !== 'judge') {
317 const { runCaseDiagnosis } = await import('./lib/paid-report');
318 const v = await runCaseDiagnosis(id, { trials: 1, jobs: 1, rootDir, evalDirBase: evalDir, runShards: runPaidShards });
319 return { status: v.status, redClass: v.redClass };
320 }
321 const env = { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [id] }, 'train bisect') };
322 const [outcome] = (await runPaidShards([[JUDGE_FILE]], { jobs: 1, rootDir, env, evalDirBase: evalDir })).outcomes;
323 const trials = outcome && outcome.runnerError === undefined && ['passed', 'failed', 'timed-out'].includes(outcome.status)
324 ? [{ trial: 1, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const,
325 ...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) }] : [];
326 const v = panelVerdict({ case: id, kind: 'judge', panel: { n: 1, k: 1 }, trials, quarantined: caseTrialPlan(id).quarantined });
327 return { status: v.status, redClass: v.redClass };
328}
330/** The free summary of a PR push under a train policy: how to ask for the screen, then its selection and estimated cost. */
331export function screenSummary(manifest: PaidRunManifest, pr: string, rootDir = ROOT): string[] {
332 const planned = plannedCases(manifest, rootDir);
333 const recorded = loadPaidTestDurations(rootDir, manifest.tier);
334 const serial = manifest.entries.filter(e => e.status === 'planned').reduce((sum, e) => sum + (recordedShardMs(recorded, e.file) ?? UNRECORDED_SHARD_MS), 0);
335 const full = plannedSerialMs('gate', rootDir);
336 const usd = full > 0 ? Math.min(FULL_GATE_USD, FULL_GATE_USD * serial / full) : null;
337 const judges = [...planned.values()].filter(kind => kind === 'judge').length;
338 return [
339 `To run the paid screen of this head: \`gstack-ship-queue ready --pr ${pr} --screen\` (add \`--broad\` for today's full PR selection). PR pushes start no paid slices under this repository's train policy; the train's full gate covers every merge.`,
340 `- Screening selection: ${planned.size - judges} case(s) + ${judges} judge(s), PR coverage mode \`${manifest.prCoverage?.mode ?? 'broad'}\``,
341 `- Estimated screen cost: ${usd === null ? 'unknown' : `~$${usd.toFixed(2)}`} (recorded shard durations scaled to the $${FULL_GATE_USD} full gate; shards without a recorded duration count ${UNRECORDED_SHARD_MS / 60_000} min). No paid case ran on this push.`,
342 ];
343}
345const USAGE = `usage:
346 bun run scripts/train-gate.ts route --github-output <file> (inputs from the route job's env)
347 bun run scripts/train-gate.ts bisect-plan --cases <ids> --prefixes <shas> --nonce <nonce> --out <bisect.json> [--github-output <file>]
348 bun run scripts/train-gate.ts bisect-prefix --plan <bisect.json> --index <n> --out <dir>
349 bun run scripts/train-gate.ts train-record <report dir> --nonce <nonce> --head <sha> --reconcile-exit <n> --run-id <id> --run-attempt <n> --repo <owner/name> --out <file>
350 bun run scripts/train-gate.ts screen-summary <manifest.json> --pr <n>`;
352if (import.meta.main) {
353 const [command, ...rest] = process.argv.slice(2);
354 const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
355 const fail = (message: string): never => { console.error(`::error::${message}`); process.exit(1); };
356 const usage = (): never => { console.error(USAGE); process.exit(2); };
357 const output = (lines: string[]) => { const file = flag('github-output'); if (file) fs.appendFileSync(file, `${lines.join('\n')}\n`); };
358 const git = gitIn(process.cwd());
359 if (command === 'route') {
360 const env = process.env;
361 const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '',
362 evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '',
363 screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' },
364 { isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) });
365 if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`);
366 const ok = route as Route;
367 output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]);
368 console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`);
369 } else if (command === 'bisect-plan') {
370 const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')];
371 if (!cases.length || !prefixes.length || !nonce || !out) usage();
372 const plan = fullGateCases(process.cwd());
373 const bisect: BisectPlan = { schema: 1, nonce: nonce!, cases: cases.map(id => ({ id, kind: plan.get(id) ?? fail(`case ${id} is not in the full gate's plan`) })), prefixes };
374 fs.mkdirSync(path.dirname(out!), { recursive: true });
375 fs.writeFileSync(out!, `${JSON.stringify(bisect, null, 2)}\n`);
376 const m = bisectMatrix(bisect);
377 output([`slices=${JSON.stringify(m.slices)}`, `slice_timeouts=${JSON.stringify(m.timeouts)}`, `prefix_refs=${JSON.stringify(m.refs)}`]);
378 console.log(`- Bisect: ${cases.length} case(s) x ${prefixes.length} prefix(es), one trial each`);
379 } else if (command === 'bisect-prefix') {
380 const [planPath, index, out] = [flag('plan'), Number(flag('index')), flag('out')];
381 if (!planPath || !Number.isSafeInteger(index) || index < 1 || !out) usage();
382 const plan = readJson<BisectPlan>(planPath!) ?? fail(`unreadable bisect plan ${planPath}`);
383 const record = await runBisectPrefix(plan, index, { rootDir: process.cwd(), evalDir: path.join(out!, 'eval'), checkedOut: git(['rev-parse', 'HEAD']).out });
384 fs.mkdirSync(out!, { recursive: true });
385 fs.writeFileSync(path.join(out!, 'bisect-prefix.json'), `${JSON.stringify(record, null, 2)}\n`);
386 } else if (command === 'train-record') {
387 const [reportDir, nonce, head, repo, out] = [rest[0], flag('nonce'), flag('head'), flag('repo'), flag('out')];
388 if (!reportDir || !nonce || !head || !SHA.test(head) || !repo || !out) usage();
389 const prefixes = trainPrefixes(git, head!, process.cwd(), loadPolicyFromHead(process.cwd()).policy);
390 if ('error' in prefixes) fail(`no train record: ${prefixes.error}`);
391 const runId = Number(flag('run-id'));
392 const record = buildTrainRecord({ reportDir: reportDir!, nonce: nonce!, head: head!, tree: git(['rev-parse', `${head}^{tree}`]).out,
393 reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? Number(flag('reconcile-exit')) : -1, prefixes: prefixes as TrainPrefixTree[],
394 run: { id: runId, attempt: Number(flag('run-attempt')), url: `https://github.com/${repo}/actions/runs/${runId}` } });
395 fs.mkdirSync(path.dirname(out!), { recursive: true });
396 fs.writeFileSync(out!, `${JSON.stringify(record, null, 2)}\n`);
397 const c = record.census;
398 console.log(`- Train record: ${record.verdict} (rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}, unbound ${c.unbound}, partial ${c.partial_shards}, cancelled ${c.cancelled_shards}, killed before upload ${c.killed_before_upload}); prefixes T1..T${record.prefixes.length}`);
399 } else if (command === 'screen-summary') {
400 const [manifestPath, pr] = [rest[0], flag('pr')];
401 if (!manifestPath || !pr) usage();
402 for (const line of screenSummary(parseRunManifest(fs.readFileSync(manifestPath!, 'utf8')), pr!)) console.log(line);
403 } else usage();
404}