eval-readiness.ts
191 lines · scripts/eval-readiness.ts at 3f79d81
1/**
2 * Readiness-gated dispatch for the PR paid lane (R1, GSTA-48).
3 *
4 * The `readiness` job in evals.yml runs this before any paid work: a PR
5 * head's paid slices start only after (1) the PR has been quiet for the
6 * window (no newer push since this run was created; `vars.EVALS_QUIET_MINUTES`,
7 * default 10; the `evals-no-debounce` label sets it to 0) and (2) Free Tests
8 * (free-tests.yml) has concluded success on the same head SHA, as GitHub's
9 * pull_request run or as a workflow_dispatch run on the branch. Selection,
10 * trials and verdict rules are unchanged: only the start time moves.
11 *
12 * A newer push cancels the run through the workflow's cancel-in-progress
13 * group, so the quiet window is a plain sleep with no API calls. After it,
14 * Free Tests is polled every `--poll-seconds`. Free Tests concluding anything
15 * but success on this head fails the run without paid work: that head cannot
16 * merge, and re-running this workflow after a green Free Tests rerun starts
17 * the gate (a re-run keeps the run's original creation time, so it does not
18 * wait out the window again). A head that is no longer the PR's head reports
19 * superseded and plans nothing.
20 *
21 * Dispatched screens (plan GSTA-49 T1b): `--merge-sha` names the
22 * refs/pull/<n>/merge commit `gstack-ship-queue ready --screen` pinned. The
23 * screen is accepted only for an open same-repository PR whose current head
24 * is that merge commit's second parent; its first parent is the tested base.
25 * The same Free Tests rule then applies to the PR head, with no quiet window
26 * (a screen is an explicit request).
27 */
28import * as fs from 'node:fs';
29import { runGh, type GhRunner } from './recover-receipts';
31export interface FreeTestsRun { id: number; status: string; conclusion: string | null; head_sha: string; event: string }
32export type FreeTestsState =
33 | { state: 'missing' }
34 | { state: 'running'; runId: number }
35 | { state: 'success'; runId: number }
36 | { state: 'failed'; runId: number; conclusion: string };
38export type ReadinessDecision =
39 | { kind: 'wait'; reason: string }
40 | { kind: 'ready'; reason: string }
41 | { kind: 'superseded'; by: string }
42 | { kind: 'blocked'; reason: string };
44/**
45 * The Free Tests run events that count as this head's CI, in order of
46 * precedence: GitHub's own pull_request run (it builds the merge ref, the tree
47 * a merge would test), then a workflow_dispatch run on the branch (the
48 * dispatch route in docs/ship-policy.md for a head whose only conflict with
49 * the base is the CHANGELOG, which GitHub builds no merge ref for and so never
50 * runs pull_request CI on; #3508). A dispatched run builds the head alone, so
51 * it decides only for a head with no pull_request run: it never overrides a red
52 * pull_request run (rerun that run), and a lane's extra dispatch never holds a
53 * head its pull_request CI cleared. A push run is a branch build, not this
54 * head's gate.
55 */
56export const FREE_TESTS_EVENTS: readonly string[] = ['pull_request', 'workflow_dispatch'];
58/** The newest Free Tests run on this exact head under the first event present decides; a rerun updates that run in place. */
59export function freeTestsState(runs: FreeTestsRun[], headSha: string): FreeTestsState {
60 const atHead = runs.filter(run => run.head_sha === headSha);
61 const event = FREE_TESTS_EVENTS.find(e => atHead.some(run => run.event === e));
62 const newest = atHead.filter(run => run.event === event).sort((a, b) => b.id - a.id)[0];
63 if (!newest) return { state: 'missing' };
64 if (newest.status !== 'completed') return { state: 'running', runId: newest.id };
65 if (newest.conclusion === 'success') return { state: 'success', runId: newest.id };
66 return { state: 'failed', runId: newest.id, conclusion: newest.conclusion ?? 'unknown' };
67}
69export function readinessDecision(input: { headSha: string; prHead: string | null; free: FreeTestsState; quietLeftMs: number }): ReadinessDecision {
70 if (input.prHead && input.prHead !== input.headSha) return { kind: 'superseded', by: input.prHead };
71 const free = input.free;
72 if (free.state === 'failed') {
73 return { kind: 'blocked', reason: `Free Tests run ${free.runId} concluded ${free.conclusion} on ${input.headSha.slice(0, 12)}; the paid gate does not start for a head that cannot merge. Fix and push, or re-run Free Tests and then re-run this workflow` };
74 }
75 if (input.quietLeftMs > 0) return { kind: 'wait', reason: `quiet window: ${Math.ceil(input.quietLeftMs / 1000)} s left` };
76 if (free.state === 'missing') return { kind: 'wait', reason: 'Free Tests has not started on this head yet' };
77 if (free.state === 'running') return { kind: 'wait', reason: `Free Tests run ${free.runId} is still running` };
78 return { kind: 'ready', reason: `Free Tests run ${free.runId} passed on ${input.headSha.slice(0, 12)} and the PR has been quiet for the window` };
79}
81export interface WaitOptions {
82 repo: string;
83 pr: number;
84 headSha: string;
85 runId: number;
86 quietMs: number;
87 deadlineMs: number;
88 pollMs: number;
89 gh?: GhRunner;
90 now?: () => number;
91 sleep?: (ms: number) => void;
92}
94function ghJson<T>(gh: GhRunner, args: string[]): { ok: true; value: T } | { ok: false; error: string } {
95 const out = gh(args, 30_000);
96 if (out.status !== 0) return { ok: false, error: out.stderr.trim().split('\n')[0]?.slice(0, 200) || `gh exited ${out.status}` };
97 try { return { ok: true, value: JSON.parse(out.stdout) as T }; } catch { return { ok: false, error: 'gh returned unreadable JSON' }; }
98}
100export interface ScreenTarget { head: string; base: string; branch: string }
102/** A screen's merge commit, checked against the PR: same repository, open, and merging the PR's current head into its base. */
103export function screenTarget(gh: GhRunner, repo: string, pr: number, mergeSha: string): ScreenTarget | { error: string } {
104 if (!/^[0-9a-f]{40}$/.test(mergeSha)) return { error: `screen_merge_sha ${JSON.stringify(mergeSha)} is not a full commit SHA` };
105 const pull = ghJson<{ state?: string; head?: { sha?: string; ref?: string; repo?: { full_name?: string } | null }; base?: { repo?: { full_name?: string } } }>(gh, ['api', `repos/${repo}/pulls/${pr}`]);
106 if (!pull.ok) return { error: `cannot read PR #${pr} (${pull.error})` };
107 const p = pull.value;
108 if (p.base?.repo?.full_name !== repo || p.head?.repo?.full_name !== repo) return { error: `PR #${pr} is not a same-repository PR of ${repo}; fork work is screened on a trusted branch` };
109 if (p.state !== 'open') return { error: `PR #${pr} is ${p.state ?? 'unreadable'}, not open` };
110 const commit = ghJson<{ parents?: Array<{ sha?: string }> }>(gh, ['api', `repos/${repo}/commits/${mergeSha}`]);
111 if (!commit.ok) return { error: `cannot read the merge commit ${mergeSha.slice(0, 12)} (${commit.error})` };
112 const parents = (commit.value.parents ?? []).map(parent => parent.sha ?? '');
113 if (parents.length !== 2 || !parents.every(sha => /^[0-9a-f]{40}$/.test(sha))) return { error: `${mergeSha.slice(0, 12)} is not a two-parent merge commit` };
114 return { head: parents[1]!, base: parents[0]!, branch: p.head?.ref ?? '' };
115}
117/** Waits until the head is ready, superseded or blocked; never longer than the deadline. */
118export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision; lines: string[] } {
119 const gh = o.gh ?? runGh;
120 const now = o.now ?? Date.now;
121 const sleep = o.sleep ?? ((ms: number) => Bun.sleepSync(ms));
122 const start = now();
123 const lines: string[] = [];
124 const run = ghJson<{ created_at?: string }>(gh, ['api', `repos/${o.repo}/actions/runs/${o.runId}`]);
125 const created = run.ok ? Date.parse(run.value.created_at ?? '') : NaN;
126 const quietUntil = (Number.isFinite(created) ? created : start) + o.quietMs;
127 lines.push(`Readiness (R1): paid slices start after Free Tests passes on ${o.headSha.slice(0, 12)} and the PR is quiet for ${Math.round(o.quietMs / 60_000)} min (from ${new Date(quietUntil - o.quietMs).toISOString()})`);
128 if (quietUntil > now()) sleep(quietUntil - now());
129 let lastError = '';
130 for (;;) {
131 // Every event at the head; freeTestsState keeps the pull_request and workflow_dispatch runs.
132 const listed = ghJson<{ workflow_runs?: FreeTestsRun[] }>(gh, ['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/free-tests.yml/runs`,
133 '-f', `head_sha=${o.headSha}`, '-f', 'per_page=20']);
134 const pr = ghJson<{ head?: { sha?: string } }>(gh, ['api', `repos/${o.repo}/pulls/${o.pr}`]);
135 lastError = !listed.ok ? listed.error : !pr.ok ? pr.error : '';
136 if (!lastError && listed.ok && pr.ok) {
137 const decision = readinessDecision({ headSha: o.headSha, prHead: pr.value.head?.sha ?? null,
138 free: freeTestsState(listed.value.workflow_runs ?? [], o.headSha), quietLeftMs: quietUntil - now() });
139 if (decision.kind !== 'wait') {
140 lines.push(decision.kind === 'superseded' ? `superseded by ${decision.by}; this run plans no slices` : decision.reason);
141 lines.push(`Readiness wait: ${Math.round((now() - start) / 1000)} s`);
142 return { decision, lines };
143 }
144 lastError = decision.reason;
145 }
146 if (now() + o.pollMs > start + o.deadlineMs) {
147 const reason = `not ready after ${Math.round(o.deadlineMs / 60_000)} min (${lastError}); re-run this workflow once Free Tests has passed on this head`;
148 lines.push(reason);
149 return { decision: { kind: 'blocked', reason }, lines };
150 }
151 sleep(o.pollMs);
152 }
153}
155const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo <owner/name> --pr <n> (--head-sha <sha> | --merge-sha <sha>) --run-id <id>
156 [--quiet-minutes 10] [--deadline-minutes 80] [--poll-seconds 60] [--github-output <file>] [--summary <file>]
157--merge-sha (a dispatched screen) waits on the PR head that merge commit merges and writes head_sha, base_sha and
158head_branch outputs. Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`;
160if (import.meta.main) {
161 const [command, ...rest] = process.argv.slice(2);
162 const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
163 const num = (name: string, fallback: number) => { const value = flag(name); return value === undefined || value === '' ? fallback : Number(value); };
164 const [repo, mergeSha] = [flag('repo'), flag('merge-sha')];
165 const [pr, runId, quiet, deadline, poll] = [Number(flag('pr')), Number(flag('run-id')), num('quiet-minutes', 10), num('deadline-minutes', 80), num('poll-seconds', 60)];
166 if (command !== 'wait' || !repo || !(flag('head-sha') ?? mergeSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId)
167 || ![quiet, deadline, poll].every(value => Number.isFinite(value) && value >= 0)) {
168 console.error(USAGE);
169 process.exit(2);
170 }
171 const output = flag('github-output');
172 const summary = flag('summary');
173 const target = mergeSha ? screenTarget(runGh, repo, pr, mergeSha) : null;
174 if (target && 'error' in target) {
175 console.error(`::error::screen refused: ${target.error}`);
176 if (summary) fs.appendFileSync(summary, `- Screen refused: ${target.error}\n`);
177 process.exit(1);
178 }
179 const headSha = target ? target.head : flag('head-sha')!;
180 if (!/^[0-9a-f]{40}$/.test(headSha)) { console.error(USAGE); process.exit(2); }
181 if (target && output) fs.appendFileSync(output, `head_sha=${target.head}\nbase_sha=${target.base}\nhead_branch=${target.branch}\n`);
182 const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 });
183 if (target) lines.unshift(`Screen of PR #${pr}: merge ${mergeSha} merges head ${target.head.slice(0, 12)} into base ${target.base.slice(0, 12)}`);
184 for (const line of lines) {
185 console.log(line);
186 if (summary) fs.appendFileSync(summary, `- ${line}\n`);
187 }
188 if (output && decision.kind === 'superseded') fs.appendFileSync(output, 'superseded=true\n');
189 if (decision.kind === 'blocked') console.error(`::error::${decision.reason}`);
190 process.exit(decision.kind === 'blocked' ? 1 : 0);
191}