ci-history.ts
335 lines · scripts/lib/ci-history.ts at 3f79d81
1/**
2 * The one reader of GitHub Actions history: run lists, run jobs, artifact
3 * downloads and the free lane's CI flake ledgers. `scripts/eval-flake-rank.ts`
4 * (pass-rates) and `scripts/test-health-report.ts` (test:health) both read CI
5 * history through this module; every call goes through an injectable client so
6 * the free tests never touch the network.
7 *
8 * Downloads are data only: artifacts are size-capped, name-checked and parsed,
9 * never executed. A failed call throws; callers that report metrics turn the
10 * error into a named "unavailable: <reason>" with `unavailableReason`, never 0.
11 */
12import * as fs from 'node:fs';
13import * as path from 'node:path';
14import { spawnSync } from 'node:child_process';
15import type { FlakeLedgerEntry } from '../test-free-shards';
17// --- Clients ---
19export interface WeeklyRun { id: number; attempt: number; sha: string; branch: string; createdAt: string; event?: string }
20export interface RunArtifact { id: number; name: string; size: number }
22/** The GitHub calls pass-rates makes; injectable so the free tests never touch the network. */
23export interface HistoryFetcher {
24 listRuns(repo: string, workflow: string, branch: string, limit: number): WeeklyRun[];
25 listArtifacts(repo: string, runId: number): RunArtifact[];
26 downloadZip(repo: string, artifactId: number, destination: string): void;
27}
29/** One GitHub REST page as parsed JSON, plus artifact zip downloads. */
30export interface GhClient {
31 getJson(apiPath: string): unknown;
32 downloadZip(repo: string, artifactId: number, destination: string): void;
33}
35/** Per-file cap for downloaded artifacts: CI history is parsed as data only, never executed. */
36export const TRIAL_OUTCOMES_MAX_BYTES = 8 * 1024 * 1024;
38function gh(args: string[]): Buffer {
39 const result = spawnSync('gh', args, { timeout: 300_000, maxBuffer: 256 * 1024 * 1024 });
40 if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
41 return result.stdout;
42}
44function jsonLines<T>(buffer: Buffer): T[] {
45 return buffer.toString('utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as T);
46}
48export const GH_CLIENT: GhClient = {
49 getJson: (apiPath) => JSON.parse(gh(['api', apiPath]).toString('utf8')),
50 downloadZip: (repo, artifactId, destination) => fs.writeFileSync(destination, gh(['api', `repos/${repo}/actions/artifacts/${artifactId}/zip`])),
51};
53export const GH_HISTORY: HistoryFetcher = {
54 listRuns: (repo, workflow, branch, limit): WeeklyRun[] => jsonLines(gh(['api',
55 `repos/${repo}/actions/workflows/${workflow}/runs?${branch ? `branch=${encodeURIComponent(branch)}&` : ''}status=completed&per_page=${limit}`,
56 '--jq', '.workflow_runs[] | {id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at, event}'])),
57 listArtifacts: (repo, runId): RunArtifact[] => jsonLines(gh(['api', `repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`,
58 '--paginate', '--jq', '.artifacts[] | select(.expired | not) | {id, name, size: .size_in_bytes}'])),
59 downloadZip: GH_CLIENT.downloadZip,
60};
62/**
63 * A named reason plus the next step for a failed GitHub read (DX-6): rate
64 * limits, server errors, auth and a missing CLI each say what to do next.
65 */
66export function unavailableReason(error: unknown): string {
67 const message = (error instanceof Error ? error.message : String(error)).replace(/\s+/g, ' ').trim();
68 if (/rate limit|secondary rate|abuse detection/i.test(message)) {
69 return `GitHub API rate limit (${message.slice(0, 160)}); next: \`gh api rate_limit --jq .resources.core.reset\` gives the reset time, rerun after it`;
70 }
71 if (/HTTP 5\d\d|\b50[0234]\b|bad gateway|service unavailable|gateway time-?out/i.test(message)) {
72 return `GitHub API server error (${message.slice(0, 160)}); next: rerun later, the history is unchanged`;
73 }
74 if (/ENOENT|not found: gh|command not found/i.test(message)) {
75 return 'gh CLI not installed; next: install it (https://cli.github.com) and run `gh auth login`';
76 }
77 if (/HTTP 401|HTTP 403|auth|not logged|credentials/i.test(message)) {
78 return `gh cannot read this repository (${message.slice(0, 160)}); next: \`gh auth status\` and \`gh auth login\` with repo read access`;
79 }
80 if (/HTTP 404|not found/i.test(message)) {
81 return `not found (${message.slice(0, 160)}); next: check --repo and that the workflow or artifact still exists (artifacts expire)`;
82 }
83 return `${message.slice(0, 200) || 'unknown error'}; next: fix the input named above, then rerun`;
84}
86// --- Runs, jobs and artifacts ---
88export interface CiRun {
89 id: number; attempt: number; sha: string; branch: string; event: string;
90 status: string; conclusion: string | null; createdAt: string; updatedAt: string; startedAt: string;
91 prNumbers: number[];
92}
94export interface CiJob { id: number; name: string; conclusion: string | null; startedAt: string | null; completedAt: string | null }
96const PAGE = 100;
98/** The runs API's `created` filter, by UTC day (both ends inclusive): `>=since`, or `since..until` when the window has an end. */
99export function createdFilter(since: string, until?: string): string {
100 return `created=${encodeURIComponent(until ? `${since.slice(0, 10)}..${until.slice(0, 10)}` : `>=${since.slice(0, 10)}`)}`;
101}
103/**
104 * Every completed run of `workflow` created at or after `since` (and before
105 * `until` when given), newest first, paging until a page is short (the
106 * `created` filter bounds the range server-side). A run whose created_at does
107 * not parse is skipped. A failure on any page throws: a partial history is
108 * never reported as a complete one.
109 */
110export function listWorkflowRuns(client: GhClient, repo: string, workflow: string,
111 opts: { since: string; until?: string; branch?: string; event?: string; maxPages?: number }): CiRun[] {
112 const out: CiRun[] = [];
113 const query = [`status=completed`, `per_page=${PAGE}`, createdFilter(opts.since, opts.until),
114 ...(opts.branch ? [`branch=${encodeURIComponent(opts.branch)}`] : []), ...(opts.event ? [`event=${encodeURIComponent(opts.event)}`] : [])];
115 for (let page = 1; page <= (opts.maxPages ?? 20); page++) {
116 const body = client.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}&page=${page}`) as { workflow_runs?: any[] };
117 const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
118 for (const run of runs) {
119 const created = Date.parse(String(run.created_at));
120 if (!(created >= Date.parse(opts.since)) || (opts.until && created >= Date.parse(opts.until))) continue;
121 if ((opts.branch && run.head_branch !== opts.branch) || (opts.event && run.event !== opts.event)) continue;
122 out.push({ id: Number(run.id), attempt: Number(run.run_attempt ?? 1), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''),
123 event: String(run.event ?? ''), status: String(run.status ?? ''), conclusion: run.conclusion ?? null,
124 createdAt: String(run.created_at ?? ''), updatedAt: String(run.updated_at ?? ''), startedAt: String(run.run_started_at ?? run.created_at ?? ''),
125 prNumbers: Array.isArray(run.pull_requests) ? run.pull_requests.map((pr: any) => Number(pr.number)).filter(Number.isFinite) : [] });
126 }
127 if (runs.length < PAGE) break;
128 }
129 return out.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
130}
132/** Every job of one run attempt (all pages). */
133export function listRunJobs(client: GhClient, repo: string, runId: number, attempt = 1): CiJob[] {
134 const out: CiJob[] = [];
135 for (let page = 1; page <= 10; page++) {
136 const body = client.getJson(`repos/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=${PAGE}&page=${page}`) as { jobs?: any[] };
137 const jobs = Array.isArray(body?.jobs) ? body.jobs : [];
138 out.push(...jobs.map(job => ({ id: Number(job.id), name: String(job.name ?? ''), conclusion: job.conclusion ?? null,
139 startedAt: job.started_at ?? null, completedAt: job.completed_at ?? null })));
140 if (jobs.length < PAGE) break;
141 }
142 return out;
143}
145/** Unexpired artifacts of one run (all pages). */
146export function listRunArtifacts(client: GhClient, repo: string, runId: number): RunArtifact[] {
147 const out: RunArtifact[] = [];
148 for (let page = 1; page <= 10; page++) {
149 const body = client.getJson(`repos/${repo}/actions/runs/${runId}/artifacts?per_page=${PAGE}&page=${page}`) as { artifacts?: any[] };
150 const artifacts = Array.isArray(body?.artifacts) ? body.artifacts : [];
151 out.push(...artifacts.filter(artifact => !artifact.expired)
152 .map(artifact => ({ id: Number(artifact.id), name: String(artifact.name ?? ''), size: Number(artifact.size_in_bytes ?? 0) })));
153 if (artifacts.length < PAGE) break;
154 }
155 return out;
156}
158/** A HistoryFetcher over a GhClient (artifact listings memoized per run), so downloads share one cache and one parser. */
159export function historyFetcher(client: GhClient): HistoryFetcher {
160 const artifacts = new Map<string, RunArtifact[]>();
161 return {
162 listRuns: (repo, workflow, branch, limit) => listWorkflowRuns(client, repo, workflow, { since: '1970-01-01', branch, maxPages: 1 })
163 .slice(0, limit).map(run => ({ id: run.id, attempt: run.attempt, sha: run.sha, branch: run.branch, createdAt: run.createdAt, event: run.event })),
164 listArtifacts: (repo, runId) => {
165 const key = `${repo}#${runId}`;
166 if (!artifacts.has(key)) artifacts.set(key, listRunArtifacts(client, repo, runId));
167 return artifacts.get(key)!;
168 },
169 downloadZip: (repo, artifactId, destination) => client.downloadZip(repo, artifactId, destination),
170 };
171}
173/**
174 * EVAL_POLICY v2 weekly history (W4b): scheduled runs on main plus main
175 * dispatches, never a branch dispatch, so back-to-back validation runs on a
176 * branch neither feed pass rates nor count as weeks for quarantine expiry.
177 */
178export function isWeeklyHistoryRun(run: Pick<WeeklyRun, 'branch' | 'event'>): boolean {
179 return run.branch === 'main' && (run.event === 'schedule' || run.event === 'workflow_dispatch');
180}
182/**
183 * Branch census runs whose trials may pool into a case's main series (D1
184 * option b, approved 2026-10-05): any completed non-main run of the census
185 * workflow. A pooled trial counts only toward a series main has also run
186 * (same case-owned bytes, HARNESS_VERSION, model and CLI); weeks for
187 * quarantine expiry still come from isWeeklyHistoryRun alone.
188 */
189export function isPooledTrialRun(run: Pick<WeeklyRun, 'branch'>): boolean {
190 return run.branch !== '' && run.branch !== 'main';
191}
193/** The last `limit` completed runs of `workflow` on each branch, newest first, deduplicated. */
194export function listWeeklyRuns(opts: { repo: string; workflow: string; branches: string[]; limit: number; fetcher?: HistoryFetcher }): WeeklyRun[] {
195 const fetcher = opts.fetcher ?? GH_HISTORY;
196 const runs = new Map<number, WeeklyRun>();
197 for (const branch of opts.branches) for (const run of fetcher.listRuns(opts.repo, opts.workflow, branch, opts.limit)) runs.set(run.id, run);
198 return [...runs.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
199}
201/**
202 * Download the artifacts of one run whose names match into a per-run cache
203 * directory (reused on later calls) and return the extracted directories.
204 * Oversized or oddly named artifacts are skipped: downloads are data only.
205 */
206export function downloadRunArtifacts(opts: { repo: string; run: Pick<WeeklyRun, 'id'>; match: (name: string) => boolean; cacheDir: string;
207 fetcher?: HistoryFetcher; maxBytes?: number; artifacts?: RunArtifact[] }): string[] {
208 const fetcher = opts.fetcher ?? GH_HISTORY;
209 const dirs: string[] = [];
210 for (const artifact of opts.artifacts ?? fetcher.listArtifacts(opts.repo, opts.run.id)) {
211 if (!opts.match(artifact.name) || !/^[A-Za-z0-9._-]+$/.test(artifact.name)) continue;
212 if (artifact.size > (opts.maxBytes ?? TRIAL_OUTCOMES_MAX_BYTES)) continue;
213 const dir = path.join(opts.cacheDir, `${opts.run.id}`, artifact.name);
214 if (!fs.existsSync(path.join(dir, '.complete'))) {
215 fs.rmSync(dir, { recursive: true, force: true });
216 fs.mkdirSync(dir, { recursive: true });
217 const zip = path.join(dir, 'artifact.zip');
218 fetcher.downloadZip(opts.repo, artifact.id, zip);
219 const unzip = spawnSync('unzip', ['-o', '-q', zip, '-d', dir], { timeout: 120_000 });
220 if (unzip.status !== 0) throw new Error(`unzip failed for ${artifact.name}: ${String(unzip.stderr || unzip.error || '')}`);
221 fs.rmSync(zip, { force: true });
222 fs.writeFileSync(path.join(dir, '.complete'), '');
223 }
224 dirs.push(dir);
225 }
226 return dirs;
227}
229/** Every regular file under `dir` whose basename matches, recursively. */
230export function filesUnder(dir: string, match: (base: string) => boolean): string[] {
231 if (!fs.existsSync(dir)) return [];
232 return (fs.readdirSync(dir, { recursive: true }) as string[])
233 .filter(name => match(path.basename(name)))
234 .map(name => path.join(dir, name))
235 .filter(full => fs.statSync(full).isFile());
236}
238// --- Free-lane flake ledgers (W7a) ---
240/** Per-line parse: one torn or hand-edited JSONL line drops that line, never the file. */
241export function parseFlakeLedger(text: string): FlakeLedgerEntry[] {
242 const out: FlakeLedgerEntry[] = [];
243 for (const line of text.split('\n')) {
244 if (!line.trim()) continue;
245 try {
246 const entry = JSON.parse(line);
247 if (entry && entry.kind === 'flaky-pass' && typeof entry.file === 'string') out.push(entry);
248 } catch { /* torn line: skip */ }
249 }
250 return out;
251}
253/** Every flaky-pass entry in the `*.jsonl` files under a directory of downloaded `flake-ledger-*` artifacts. */
254export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] {
255 return filesUnder(dir, base => base.endsWith('.jsonl')).flatMap(file => parseFlakeLedger(fs.readFileSync(file, 'utf8')));
256}
258/** The pre-registered flaky-file rule (CEO-15): > 5% of main free-tests runs over a window of >= 20 runs. */
259export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const;
261export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean }
262export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean }
264/**
265 * Rank files by the share of runs in which they needed the retry pass. A file
266 * fails only when the window holds at least `minRuns` runs and its rate is
267 * strictly above `maxRate`; a thinner window is report-only.
268 */
269export function rankFlakyFiles(runEntries: Map<number, FlakeLedgerEntry[]>, runCount: number, since: string,
270 policy: { maxRate: number; minRuns: number } = FLAKE_POLICY): FlakeWindow {
271 const perFile = new Map<string, { runs: Set<number>; entries: number }>();
272 for (const [runId, entries] of runEntries) for (const entry of entries) {
273 const slot = perFile.get(entry.file) ?? { runs: new Set<number>(), entries: 0 };
274 slot.runs.add(runId);
275 slot.entries += 1;
276 perFile.set(entry.file, slot);
277 }
278 const enforceable = runCount >= policy.minRuns;
279 const files = [...perFile].map(([file, slot]) => {
280 const rate = runCount ? slot.runs.size / runCount : 0;
281 return { file, flakyRuns: slot.runs.size, entries: slot.entries, rate, enforced: enforceable, failing: enforceable && rate > policy.maxRate };
282 }).sort((a, b) => b.rate - a.rate || a.file.localeCompare(b.file));
283 return { runs: runCount, flakyRuns: runEntries.size, since, files, enforceable };
284}
286/**
287 * Read the flake ledgers of main's completed free-tests runs: the runs created
288 * in the last `sinceDays`, widened to the newest `minRuns` runs (within
289 * `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run
290 * with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped
291 * when the ledger file does not exist).
292 */
293export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string;
294 minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow {
295 const minRuns = opts.minRuns ?? FLAKE_POLICY.minRuns;
296 const iso = (days: number) => new Date(opts.now - days * 86_400_000).toISOString();
297 const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml',
298 { since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' })
299 .filter(run => run.conclusion === 'success' || run.conclusion === 'failure');
300 const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays));
301 const window = recent.length >= minRuns ? recent : all.slice(0, minRuns);
302 const fetcher = historyFetcher(opts.client);
303 const runEntries = new Map<number, FlakeLedgerEntry[]>();
304 for (const run of window) {
305 const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') });
306 const entries = dirs.flatMap(readFlakeLedgerDir);
307 if (entries.length) runEntries.set(run.id, entries);
308 }
309 return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays));
310}
312/** Markdown table of one run's merged flake ledgers, for a job summary. */
313export function formatFlakeSummary(entries: FlakeLedgerEntry[]): string {
314 if (entries.length === 0) return '### Free-suite flaky passes\n\nNone: every file passed on its first run.\n';
315 const byFile = new Map<string, FlakeLedgerEntry[]>();
316 for (const entry of entries) byFile.set(entry.file, [...(byFile.get(entry.file) ?? []), entry]);
317 const rows = [...byFile].sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0]))
318 .map(([file, list]) => `| \`${file}\` | ${list.length} | ${[...new Set(list.map(entry => entry.shard).filter(shard => shard !== undefined))].join(', ') || '-'} |`);
319 return ['### Free-suite flaky passes', '',
320 `${entries.length} flaky pass(es) in ${byFile.size} file(s): each failed, then passed on the serial retry. They are green, not fixed.`, '',
321 '| File | Flaky passes | Shard(s) |', '|---|---:|---|', ...rows, '',
322 'Weekly enforcement: `bun run test:health --enforce` fails when a file flakes in more than 5% of main runs over at least 20 runs.', ''].join('\n');
323}
325// --- Repository ---
327export function gitOutput(args: string[], cwd?: string): string | null {
328 const result = spawnSync('git', args, { encoding: 'utf8', timeout: 5_000, cwd });
329 return result.status === 0 ? result.stdout.trim() : null;
330}
332export function repoSlug(cwd?: string): string {
333 const url = gitOutput(['remote', 'get-url', 'origin'], cwd) ?? '';
334 return url.match(/[:/]([^/:]+\/[^/]+?)(?:\.git)?$/)?.[1] ?? 'garrytan/gstack';
335}