eval-bg.ts
448 lines · scripts/eval-bg.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * eval-bg — the one entry point for background paid evals (`bun run eval:bg:<lane>`).
4 *
5 * Two backends, one completion contract (W5d, CEO-17, DX-1..3/7, ENG-12/19):
6 *
7 * dispatch The pushed, clean ref is dispatched to CI (evals.yml and/or
8 * evals-periodic.yml on garrytan/gstack) with an `expected_sha` the
9 * workflow verifies and a `nonce` its run-name carries; this process
10 * resolves the run by that nonce and follows it to its conclusion.
11 * local The lane's sharded runner on this machine, capped at
12 * ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min, at most 4 h.
13 *
14 * Both run under bin/gstack-detach with the `gstack-evals` lock, a run-scoped
15 * log under ~/.gstack-dev/eval-runs/ whose first line names the backend and the
16 * tested revision, and the `### gstack-detach EXIT=<code> ###` sentinel. A
17 * followed CI run maps its conclusion: success 0, failure 1, cancelled 130,
18 * anything else 2. `status <run-id|log>` reconnects later.
19 *
20 * Dispatch needs every one of: gh installed and authenticated, origin is
21 * garrytan/gstack with push rights, a branch whose remote head (read with
22 * `gh api`, never git fetch) equals HEAD, and a clean tree (no staged,
23 * unstaged or untracked files, the rule test/helpers/test-selection.ts uses).
24 * Otherwise it runs locally and prints why; `--dispatch` refuses instead.
25 * Dispatched runs are validation runs: fresh, never writing PR receipts.
26 */
27import * as fs from 'node:fs';
28import * as os from 'node:os';
29import * as path from 'node:path';
30import { spawnSync } from 'node:child_process';
31import { detectBaseBranch } from '../test/helpers/touchfiles';
32import { DEFAULT_JOBS, ROOT, type PaidTier } from './lib/paid-types';
33import { buildRunManifest, loadPaidTestDurations, recordedShardMs } from './lib/paid-plan';
35export type EvalBgLane = 'pr' | 'gate' | 'periodic' | 'release';
36export type Backend = 'dispatch' | 'local';
37export const EVAL_BG_LANES: readonly EvalBgLane[] = ['pr', 'gate', 'periodic', 'release'];
38export const CANONICAL_REPO = 'garrytan/gstack';
39export const EVAL_LOCK = 'gstack-evals';
40export const LOCAL_CAP_SECONDS = 4 * 60 * 60;
41export const LOCAL_SETUP_SECONDS = 20 * 60;
42export const LOCAL_CAP_FACTOR = 1.5;
43/** Packing weight of a shard with no recorded wall: the PR lane's ~10-minute case target. */
44export const UNRECORDED_SHARD_MS = 600_000;
45/** How long the dispatch follower may hold the lock: the longest CI lane plus queueing. */
46export const DISPATCH_FOLLOW_SECONDS = 4 * 60 * 60;
48export interface WorkflowDispatch { workflow: string; inputs: Record<string, string> }
50/**
51 * Every workflow_dispatch input name eval-bg and gstack-ship-queue send, in
52 * one place. The train inputs (cases, prefix_shas) and the screen inputs
53 * (screen_pr, screen_merge_sha, screen_policy) are the interface the
54 * workflow side (plan GSTA-49, lane T1b) adds to evals.yml.
55 */
56export const DISPATCH_INPUTS = {
57 evalsAll: 'evals_all', expectedSha: 'expected_sha', nonce: 'nonce', baseRef: 'base_ref', baseSha: 'base_sha', prReceipts: 'pr_receipts',
58 cases: 'cases', prefixShas: 'prefix_shas', screenPr: 'screen_pr', screenMergeSha: 'screen_merge_sha', screenPolicy: 'screen_policy',
59} as const;
61/** What a dispatch is for, beside its lane: the screen policy keeps a broad screen from adopting a narrow run. */
62export type DispatchPurpose = 'screen' | 'train' | 'bisect' | 'backstop';
63export type ScreenPolicy = 'narrow' | 'broad';
64export interface NonceScope { purpose: DispatchPurpose; policy?: ScreenPolicy }
66interface LaneSpec {
67 /** The package script the local backend runs. */
68 script: string;
69 /** Worker count the local command defaults to (EVALS_JOBS overrides). */
70 jobs: number;
71 /** Tiers the local command plans, in order. */
72 tiers: Array<{ tier: PaidTier; profile: 'pr' | 'full' }>;
73}
75export const LANE_SPECS: Record<EvalBgLane, LaneSpec> = {
76 pr: { script: 'test:pr', jobs: 2, tiers: [{ tier: 'gate', profile: 'pr' }] },
77 gate: { script: 'test:gate:sharded', jobs: DEFAULT_JOBS, tiers: [{ tier: 'gate', profile: 'full' }] },
78 periodic: { script: 'test:periodic:sharded', jobs: DEFAULT_JOBS, tiers: [{ tier: 'periodic', profile: 'full' }] },
79 release: { script: 'test:release', jobs: DEFAULT_JOBS, tiers: [{ tier: 'gate', profile: 'full' }, { tier: 'periodic', profile: 'full' }] },
80};
82/** The workflows a lane dispatches and their inputs (names agreed with evals.yml's workflow_dispatch). */
83export function dispatchPlan(lane: EvalBgLane, ids: { sha: string; baseRef: string; baseSha: string; nonce: string; prNumber?: number | null }): WorkflowDispatch[] {
84 const I = DISPATCH_INPUTS;
85 const evals = (all: boolean): WorkflowDispatch => ({ workflow: 'evals.yml', inputs: {
86 [I.evalsAll]: String(all), [I.expectedSha]: ids.sha, [I.nonce]: ids.nonce,
87 ...(all ? {} : { [I.baseRef]: ids.baseRef, [I.baseSha]: ids.baseSha }),
88 // CEO-29/ENG-8: a PR lane run on a ref with an open same-repo PR reads that PR's receipts read-only.
89 ...(!all && ids.prNumber ? { [I.prReceipts]: String(ids.prNumber) } : {}),
90 } });
91 const periodic: WorkflowDispatch = { workflow: 'evals-periodic.yml', inputs: { [I.expectedSha]: ids.sha, [I.nonce]: ids.nonce } };
92 if (lane === 'pr') return [evals(false)];
93 if (lane === 'gate') return [evals(true)];
94 if (lane === 'periodic') return [periodic];
95 return [evals(true), periodic];
96}
98/**
99 * Local cap: ceil(1.5 x planned serial seconds / jobs) + 20 min setup, at most
100 * 4 h. Serial seconds sum the recorded wall of every planned shard
101 * (scripts/paid-test-durations.json), unrecorded shards at UNRECORDED_SHARD_MS.
102 */
103export function localCapSeconds(serialMs: number, jobs: number): { seconds: number; capped: boolean } {
104 if (!Number.isSafeInteger(jobs) || jobs < 1) throw new Error('EVALS_JOBS must be a positive integer');
105 const computed = Math.ceil(LOCAL_CAP_FACTOR * serialMs / jobs / 1000) + LOCAL_SETUP_SECONDS;
106 return { seconds: Math.min(computed, LOCAL_CAP_SECONDS), capped: computed > LOCAL_CAP_SECONDS };
107}
109/** Planned serial work of a lane on this checkout, from the same planner CI uses. */
110export function plannedSerialMs(lane: EvalBgLane, rootDir = ROOT): number {
111 return LANE_SPECS[lane].tiers.reduce((total, { tier, profile }) => {
112 const plan = (all: boolean) => buildRunManifest({ tier, profile: all ? 'full' : profile, sliceCount: 1, evalsAll: all,
113 env: all ? { EVALS_ALL: '1' } : {}, rootDir });
114 let manifest;
115 try { manifest = plan(profile === 'full'); }
116 catch { manifest = plan(true); } // a PR diff that needs full validation runs the full gate locally
117 const recorded = loadPaidTestDurations(rootDir, tier);
118 return total + manifest.entries.filter(entry => entry.status === 'planned')
119 .reduce((sum, entry) => sum + (recordedShardMs(recorded, entry.file) ?? UNRECORDED_SHARD_MS), 0);
120 }, 0);
121}
123export interface CheckoutFacts {
124 sha: string;
125 branch: string | null;
126 /** `git status --porcelain --untracked-files=all` lines. */
127 dirty: string[];
128 /** owner/name of origin, null when unknown. */
129 repo: string | null;
130 ghReady: boolean;
131 canPush: boolean;
132 /** Remote head of the branch on garrytan/gstack (gh api), null when absent. */
133 remoteSha: string | null;
134 /** Commits HEAD is ahead of remoteSha, null when unknown. */
135 ahead: number | null;
136}
138export type BackendChoice =
139 | { backend: Backend; reason: string }
140 | { backend: 'refuse'; reason: string };
142/** DX-1 checkout contract, one row per reason dispatch is not possible. */
143export function dispatchBlockers(facts: CheckoutFacts): string[] {
144 const sha = facts.sha.slice(0, 12);
145 const blockers: string[] = [];
146 if (!facts.ghReady) blockers.push('gh is not installed or not authenticated (run `gh auth status`)');
147 if (facts.repo !== CANONICAL_REPO) blockers.push(`origin is ${facts.repo ?? 'unknown'}, not ${CANONICAL_REPO} (a fork cannot dispatch its workflows)`);
148 else if (facts.ghReady && !facts.canPush) blockers.push(`this gh account cannot dispatch workflows on ${CANONICAL_REPO}`);
149 if (!facts.branch) blockers.push(`HEAD ${sha} is detached; dispatch needs a pushed branch`);
150 else if (facts.ghReady && facts.repo === CANONICAL_REPO && facts.remoteSha === null) blockers.push(`branch ${facts.branch} is not on ${CANONICAL_REPO}; push it`);
151 else if (facts.remoteSha !== null && facts.remoteSha !== facts.sha) {
152 blockers.push(facts.ahead !== null
153 ? `HEAD ${sha} is ${facts.ahead} commits ahead of origin/${facts.branch}`
154 : `HEAD ${sha} differs from origin/${facts.branch} (${facts.remoteSha.slice(0, 12)})`);
155 }
156 if (facts.dirty.length) blockers.push(`${facts.dirty.length} files are modified or untracked`);
157 return blockers;
158}
160/** Choose the backend: an explicit mode wins; otherwise dispatch when nothing blocks it. */
161export function chooseBackend(facts: CheckoutFacts, mode: Backend | 'auto'): BackendChoice {
162 if (mode === 'local') return { backend: 'local', reason: 'local requested (--local or GSTACK_EVAL_BG_MODE=local)' };
163 const blockers = dispatchBlockers(facts);
164 if (!blockers.length) return { backend: 'dispatch', reason: `HEAD ${facts.sha.slice(0, 12)} is clean and pushed to ${CANONICAL_REPO}/${facts.branch}` };
165 if (mode === 'dispatch') return { backend: 'refuse', reason: `${blockers.join(' and ')}; commit and push, or rerun with --local` };
166 return { backend: 'local', reason: `cannot dispatch: ${blockers.join('; ')}` };
167}
169/** CI conclusion -> the detach exit code (DX-2). */
170export function conclusionExitCode(conclusion: string | null | undefined): number {
171 if (conclusion === 'success') return 0;
172 if (conclusion === 'failure') return 1;
173 if (conclusion === 'cancelled') return 130;
174 return 2;
175}
177/** Status words `status` prints, from a detach EXIT code or a CI run. */
178export function exitStatusWord(code: string): 'passed' | 'failed' | 'cancelled' {
179 if (code === '0') return 'passed';
180 return code === '130' ? 'cancelled' : 'failed';
181}
183/**
184 * Nonce: lane, tested revision and base, plus a random suffix; the dedupe key
185 * is everything but the suffix. A scoped dispatch (a screen, a train, a
186 * bisect, a backstop) adds its purpose and screen policy to the lane field
187 * (`pr.screen.broad-…`), so runs of another purpose or policy never match
188 * its key and the plain lane keys are unchanged.
189 */
190export function makeNonce(lane: EvalBgLane, sha: string, baseSha: string, random = Math.random().toString(36).slice(2, 8), scope?: NonceScope): string {
191 return `${dedupeKey(lane, sha, baseSha, scope)}${random}`;
192}
193export function dedupeKey(lane: EvalBgLane, sha: string, baseSha: string, scope?: NonceScope): string {
194 const field = scope ? [lane, scope.purpose, ...(scope.policy ? [scope.policy] : [])].join('.') : lane;
195 return `${field}-${sha.slice(0, 12)}-${(baseSha || 'none').slice(0, 12)}-`;
196}
198/** The scope a nonce was minted with (null for a plain lane nonce). */
199export function nonceScope(nonce: string): NonceScope | null {
200 const m = /^(?:pr|gate|periodic|release)\.(screen|train|bisect|backstop)(?:\.(narrow|broad))?-[0-9a-f]{1,12}-/.exec(nonce);
201 return m ? { purpose: m[1] as DispatchPurpose, ...(m[2] ? { policy: m[2] as ScreenPolicy } : {}) } : null;
202}
204export interface RunSummary { databaseId: number; displayTitle: string; status: string; conclusion: string | null; url: string; event: string }
206/**
207 * DX-3 / ENG-12: an existing dispatch of the same lane, revision and base that
208 * is queued, running or completed successfully is followed instead of
209 * dispatching a duplicate. Only dispatches match (by the nonce in run-name); a
210 * failed or cancelled run is never reused.
211 */
212export function findDuplicateRun(runs: readonly RunSummary[], key: string): RunSummary | null {
213 return runs.find(run => run.event === 'workflow_dispatch' && run.displayTitle.includes(`eval-bg ${key}`)
214 && (run.status !== 'completed' || run.conclusion === 'success')) ?? null;
215}
217// ─── Process plumbing (git / gh / detach) ────────────────────────────────────
219export type Exec = (command: string, args: string[]) => { status: number | null; stdout: string };
221const RUN_FIELDS = 'databaseId,displayTitle,status,conclusion,url,event';
223/** Recent workflow_dispatch runs of one workflow; null when gh could not list them (unknown, never "none"). */
224export function listDispatchRuns(workflow: string, run: Exec, repo = CANONICAL_REPO, limit = 30): RunSummary[] | null {
225 const out = run('gh', ['run', 'list', '--repo', repo, '--workflow', workflow, '--event', 'workflow_dispatch', '--limit', String(limit), '--json', RUN_FIELDS]);
226 if (out.status !== 0) return null;
227 try { const parsed = JSON.parse(out.stdout); return Array.isArray(parsed) ? parsed as RunSummary[] : null; } catch { return null; }
228}
230/** The dispatch run whose run-name carries exactly this nonce. */
231export function runForNonce(runs: readonly RunSummary[], nonce: string): RunSummary | null {
232 return runs.find(item => item.event === 'workflow_dispatch' && item.displayTitle.includes(`eval-bg ${nonce}]`)) ?? null;
233}
235/** `gh workflow run` with the plan's inputs on `ref`; false when gh refused it. */
236export function dispatchWorkflow(dispatch: WorkflowDispatch, ref: string, run: Exec, repo = CANONICAL_REPO): boolean {
237 const args = ['workflow', 'run', dispatch.workflow, '--repo', repo, '--ref', ref,
238 ...Object.entries(dispatch.inputs).flatMap(([name, value]) => ['-f', `${name}=${value}`])];
239 return run('gh', args).status === 0;
240}
242/** Poll for the run a dispatch started, by its nonce, for up to ten poll intervals; null when none appeared. */
243export async function awaitNonceRun(workflow: string, nonce: string, run: Exec, repo = CANONICAL_REPO): Promise<RunSummary | null> {
244 const deadline = Date.now() + 10 * pollMs();
245 while (Date.now() < deadline) {
246 await sleep(Math.min(pollMs(), 10_000));
247 const found = runForNonce(listDispatchRuns(workflow, run, repo) ?? [], nonce);
248 if (found) return found;
249 }
250 return null;
251}
253const exec: Exec = (command, args) => {
254 const result = spawnSync(command, args, { cwd: ROOT, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 });
255 return { status: result.error ? null : result.status, stdout: result.stdout ?? '' };
256};
258function originRepo(run: Exec): string | null {
259 const url = run('git', ['config', '--get', 'remote.origin.url']).stdout.trim();
260 return /github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?\/?$/.exec(url)?.[1] ?? null;
261}
263/** Gather the DX-1 facts. Remote state comes from `gh api` only (ENG-19). */
264export function gatherCheckoutFacts(run: Exec = exec): CheckoutFacts {
265 const sha = run('git', ['rev-parse', 'HEAD']).stdout.trim();
266 const branch = run('git', ['branch', '--show-current']).stdout.trim() || null;
267 const dirty = run('git', ['status', '--porcelain', '--untracked-files=all']).stdout.split('\n').filter(Boolean);
268 const repo = originRepo(run);
269 const ghReady = run('gh', ['auth', 'status']).status === 0;
270 const canPush = ghReady && repo === CANONICAL_REPO
271 && run('gh', ['api', `repos/${CANONICAL_REPO}`, '--jq', '.permissions.push']).stdout.trim() === 'true';
272 const remote = ghReady && repo === CANONICAL_REPO && branch
273 ? run('gh', ['api', `repos/${CANONICAL_REPO}/branches/${encodeURIComponent(branch)}`, '--jq', '.commit.sha']) : null;
274 const remoteSha = remote && remote.status === 0 && /^[0-9a-f]{40}$/.test(remote.stdout.trim()) ? remote.stdout.trim() : null;
275 const count = remoteSha && remoteSha !== sha ? run('git', ['rev-list', '--count', `${remoteSha}..HEAD`]) : null;
276 return { sha, branch, dirty, repo, ghReady, canPush, remoteSha,
277 ahead: count && count.status === 0 ? Number(count.stdout.trim()) : null };
278}
280/** Run-scoped log path, the same layout gstack-detach uses for its own default. */
281export function runScopedLog(label: string, branch: string | null, now = new Date()): string {
282 const stamp = now.toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15);
283 return path.join(os.homedir(), '.gstack-dev', 'eval-runs',
284 `${label}-${path.basename(ROOT)}-${(branch ?? 'nobranch').replace(/\//g, '-')}-${stamp}-${process.pid}.log`);
285}
287/** Start `command` under gstack-detach with the eval lock; the header becomes the log's first line. */
288export function launchDetached(opts: { log: string; label: string; timeoutSeconds: number; header: string; command: string[]; env?: NodeJS.ProcessEnv }): string {
289 fs.mkdirSync(path.dirname(opts.log), { recursive: true });
290 fs.writeFileSync(opts.log, `${opts.header}\n`, { flag: 'a' });
291 const result = spawnSync(path.join(ROOT, 'bin', 'gstack-detach'), ['--log', opts.log, '--lock', EVAL_LOCK,
292 '--timeout', String(opts.timeoutSeconds), '--label', opts.label, '--', ...opts.command],
293 { cwd: ROOT, encoding: 'utf8', timeout: 30_000, env: opts.env ?? process.env });
294 if (result.status !== 0) throw new Error(`gstack-detach failed to start: ${result.stderr || result.error?.message}`);
295 return result.stdout.trim();
296}
298/** The run id a local lane exports: the run-scoped log's name, reduced to [A-Za-z0-9_.-] as the native-capture cases require. */
299export function localRunId(log: string): string {
300 return path.basename(log, '.log').replace(/[^\w.-]/g, '-');
301}
303/**
304 * The local backend: CI sets EVALS_RUN_ID per slice, and the native-capture
305 * cases refuse to start without it, so a local lane exports one too (an
306 * exported value is kept). The id is logged under the header.
307 */
308export function launchLocal(opts: { log: string; label: string; timeoutSeconds: number; header: string; command: string[]; env?: NodeJS.ProcessEnv }): string {
309 const env = opts.env ?? process.env;
310 const runId = env.EVALS_RUN_ID || localRunId(opts.log);
311 return launchDetached({ ...opts, header: `${opts.header}\n[eval-bg] EVALS_RUN_ID=${runId}`, env: { ...env, EVALS_RUN_ID: runId } });
312}
314const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
315const pollMs = () => Number(process.env.GSTACK_EVAL_BG_POLL_MS) || 30_000;
317/** Dispatch one workflow (or adopt a duplicate) and follow it; returns the mapped exit code. */
318async function followWorkflow(dispatch: WorkflowDispatch, ref: string, key: string, run: Exec): Promise<number> {
319 let target = findDuplicateRun(listDispatchRuns(dispatch.workflow, run) ?? [], key);
320 if (target) console.log(`[eval-bg] ${dispatch.workflow}: following existing run ${target.url} (same lane, revision and base; no duplicate dispatch)`);
321 else {
322 if (!dispatchWorkflow(dispatch, ref, run)) { console.log(`[eval-bg] ${dispatch.workflow}: gh workflow run failed`); return 2; }
323 target = await awaitNonceRun(dispatch.workflow, dispatch.inputs[DISPATCH_INPUTS.nonce]!, run);
324 if (!target) { console.log(`[eval-bg] ${dispatch.workflow}: dispatched, but no run named ${dispatch.inputs.nonce} appeared`); return 2; }
325 }
326 console.log(`[eval-bg] ${dispatch.workflow}: run ${target.databaseId} ${target.url}`);
327 let last = '';
328 while (true) {
329 const view = run('gh', ['run', 'view', String(target.databaseId), '--repo', CANONICAL_REPO, '--json', 'status,conclusion']);
330 let state: { status?: string; conclusion?: string | null } = {};
331 try { state = JSON.parse(view.stdout); } catch { /* transient gh failure: poll again */ }
332 if (state.status && state.status !== last) console.log(`[eval-bg] ${dispatch.workflow}: ${last = state.status}`);
333 if (state.status === 'completed') {
334 const code = conclusionExitCode(state.conclusion);
335 console.log(`[eval-bg] ${dispatch.workflow}: conclusion ${state.conclusion ?? 'unknown'} -> exit ${code}`);
336 return code;
337 }
338 await sleep(pollMs());
339 }
340}
342/** The detached follower: every planned workflow, then the first non-zero code. */
343export async function follow(plan: WorkflowDispatch[], ref: string, key: string, run: Exec = exec): Promise<number> {
344 const codes = await Promise.all(plan.map(dispatch => followWorkflow(dispatch, ref, key, run)));
345 return codes.find(code => code !== 0) ?? 0;
346}
348/** `status <run-id|log>`: running / passed / failed / cancelled / incomplete. */
349export function statusOf(target: string, run: Exec = exec, alive: (pgid: number) => boolean = pgidAlive): string {
350 if (/^\d+$/.test(target)) {
351 const view = run('gh', ['run', 'view', target, '--repo', CANONICAL_REPO, '--json', 'status,conclusion']);
352 const state = JSON.parse(view.stdout || '{}') as { status?: string; conclusion?: string };
353 return state.status === 'completed' ? exitStatusWord(String(conclusionExitCode(state.conclusion))) : 'running';
354 }
355 const log = fs.readFileSync(target, 'utf8');
356 const exit = /### gstack-detach EXIT=(\S+) ###/.exec(log)?.[1];
357 if (exit) return exitStatusWord(exit);
358 const pgid = Number(/### gstack-detach START label=\S+ pgid=(\d+) ###/.exec(log)?.[1]);
359 return pgid && alive(pgid) ? 'running' : 'incomplete';
360}
362function pgidAlive(pgid: number): boolean {
363 try { process.kill(-pgid, 0); return true; } catch { return false; }
364}
366const HELP = `Usage:
367 bun run scripts/eval-bg.ts <pr|gate|periodic|release> [--local | --dispatch] [--timeout SECS] [--base REF]
368 bun run scripts/eval-bg.ts status <run-id|log>
370Runs a paid eval lane in the background and returns immediately. Poll the printed
371log; it ends with "### gstack-detach EXIT=<code> ###" (0 passed, 1 failed,
372130 cancelled, 2 other).
374 pr diff-selected PR gate (CI: evals.yml, evals_all=false)
375 gate full gate tier (CI: evals.yml, evals_all=true)
376 periodic full periodic tier (CI: evals-periodic.yml, periodic lane only)
377 release gate + periodic (CI: both workflows)
379Backend: a clean HEAD pushed to ${CANONICAL_REPO} dispatches the CI workflow and
380follows it; anything else runs locally under bin/gstack-detach, capped at
381ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min (at most 4 h), with
382EVALS_RUN_ID set to the log's name unless one is exported.
383 --local / --dispatch force a backend (also GSTACK_EVAL_BG_MODE=local|dispatch);
384 --dispatch refuses instead of falling back
385 --timeout SECS override the computed local cap
386 --base REF base branch for the pr lane (default: detected base)
387Needs: gh with repo access for dispatch; ANTHROPIC_API_KEY for local runs.`;
389async function main(argv: string[]): Promise<number> {
390 if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) { console.log(HELP); return argv.length ? 0 : 2; }
391 const [command, ...rest] = argv;
392 if (command === 'status') {
393 if (!rest[0]) throw new Error('status needs a run id or a log path');
394 console.log(statusOf(rest[0]));
395 return 0;
396 }
397 if (command === '__follow') {
398 const [planJson, ref, key] = rest;
399 return follow(JSON.parse(planJson!), ref!, key!);
400 }
401 if (!EVAL_BG_LANES.includes(command as EvalBgLane)) throw new Error(`Unknown lane ${command}; expected ${EVAL_BG_LANES.join(', ')} or status (see --help)`);
402 const lane = command as EvalBgLane;
403 let mode: Backend | 'auto' = process.env.GSTACK_EVAL_BG_MODE === 'local' || process.env.GSTACK_EVAL_BG_MODE === 'dispatch' ? process.env.GSTACK_EVAL_BG_MODE : 'auto';
404 let timeoutOverride: number | null = null;
405 let baseRef = detectBaseBranch(ROOT) || 'main';
406 for (let index = 0; index < rest.length; index++) {
407 const flag = rest[index];
408 if (flag === '--local' || flag === '--dispatch') mode = flag.slice(2) as Backend;
409 else if (flag === '--timeout') {
410 timeoutOverride = Number(rest[++index]);
411 if (!Number.isSafeInteger(timeoutOverride) || timeoutOverride <= 0) throw new Error('--timeout needs positive seconds');
412 } else if (flag === '--base') baseRef = rest[++index] ?? baseRef;
413 else throw new Error(`Unknown argument: ${flag} (see --help)`);
414 }
415 const facts = gatherCheckoutFacts();
416 const choice = chooseBackend(facts, mode);
417 if (choice.backend === 'refuse') { console.error(`[eval-bg] refusing to dispatch: ${choice.reason}`); return 1; }
418 const label = `evals-${lane}`;
419 const log = runScopedLog(label, facts.branch);
420 const tested = `${facts.sha}${facts.dirty.length ? ` + ${facts.dirty.length} uncommitted file(s)` : ''}`;
421 if (choice.backend === 'dispatch') {
422 const baseSha = exec('git', ['merge-base', `origin/${baseRef}`, 'HEAD']).stdout.trim();
423 const pr = lane === 'pr' ? exec('gh', ['api', `repos/${CANONICAL_REPO}/pulls?head=${CANONICAL_REPO.split('/')[0]}:${facts.branch}&state=open`, '--jq', '.[0].number']).stdout.trim() : '';
424 const nonce = makeNonce(lane, facts.sha, baseSha);
425 const plan = dispatchPlan(lane, { sha: facts.sha, baseRef, baseSha, nonce, prNumber: Number(pr) || null });
426 const header = `[eval-bg] backend=dispatch lane=${lane} tested=${facts.sha} ref=${facts.branch} workflows=${plan.map(item => item.workflow).join(',')} nonce=${nonce} (${choice.reason})`;
427 console.log(header);
428 console.log(launchDetached({ log, label, timeoutSeconds: DISPATCH_FOLLOW_SECONDS, header,
429 command: [process.execPath, 'run', path.join(ROOT, 'scripts', 'eval-bg.ts'), '__follow', JSON.stringify(plan), facts.branch!, dedupeKey(lane, facts.sha, baseSha)] }));
430 return 0;
431 }
432 const jobs = Number(process.env.EVALS_JOBS) || LANE_SPECS[lane].jobs;
433 const cap = timeoutOverride !== null ? { seconds: timeoutOverride, capped: false } : localCapSeconds(plannedSerialMs(lane), jobs);
434 const header = `[eval-bg] backend=local lane=${lane} tested=${tested} cap=${cap.seconds}s jobs=${jobs}${timeoutOverride !== null ? ' (--timeout)' : ''} (${choice.reason})`;
435 console.log(header);
436 if (cap.capped) console.log(`[eval-bg] WARNING: planned work exceeds the ${LOCAL_CAP_SECONDS}s local cap; the tail may not run. Dispatch to CI or raise EVALS_JOBS.`);
437 console.log(launchLocal({ log, label, timeoutSeconds: cap.seconds, header, command: ['bun', 'run', LANE_SPECS[lane].script] }));
438 return 0;
439}
441if (import.meta.main) {
442 try {
443 process.exitCode = await main(process.argv.slice(2));
444 } catch (error) {
445 console.error(`[eval-bg] ${error instanceof Error ? error.message : String(error)}`);
446 process.exitCode = 1;
447 }
448}