test-health-report.ts
460 lines · scripts/test-health-report.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * test:health — the audit's success metrics as one command, read from CI
4 * history (scripts/lib/ci-history.ts, the same reader pass-rates uses) plus
5 * the local tree. Every metric prints its counting rule; a metric whose input
6 * cannot be read prints "unavailable: <reason>; next: <step>", never 0.
7 *
8 * Enforced checks (`--enforce`): a free test file that flakes in more than 5%
9 * of main free-tests runs over a window of at least 20 runs (W7a), and more
10 * than 5 free test files missing from scripts/free-test-durations.json
11 * (CEO-10). Without `--enforce` the command always exits 0; an unavailable
12 * check never fails it. Needs `gh` with read access to the repository.
13 *
14 * Usage:
15 * bun run test:health [--since-days 7] [--json] [--enforce] [--summary <file>]
16 * [--repo owner/name] [--complexity-base <ref>]
17 * bun run test:health flake-summary <dir> # job-summary table of downloaded flake-ledger-* artifacts
18 */
19import * as fs from 'node:fs';
20import * as path from 'node:path';
21import { spawnSync } from 'node:child_process';
22import { GH_CLIENT, downloadRunArtifacts, formatFlakeSummary, historyFetcher, listRunJobs, listWorkflowRuns, readCiFlakeLedgers,
23 readFlakeLedgerDir, repoSlug, unavailableReason, type CiRun, type FlakeWindow, type GhClient } from './lib/ci-history';
24import { analyzePassRates, readTrialOutcomeDir, type TrialRecord } from './eval-flake-rank';
25import { collectFreeTestFiles, loadFreeTestDurations, unseededFreeFiles } from './test-free-shards';
26import { EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
27import { resolveStateRoot } from '../lib/state-root';
29const ROOT = path.resolve(import.meta.dir, '..');
30export const UNSEEDED_LIMIT = 5;
31export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows'] as const;
33export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string }
34export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string }
35export type IssueAction = 'upsert' | 'keep-open' | 'close';
36export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
38export interface HealthInputs {
39 client: GhClient;
40 repo: string;
41 now: number;
42 sinceDays: number;
43 cacheDir: string;
44 root?: string;
45 complexityBase?: string;
46}
48// --- Pure arithmetic ---
50export function percentile(values: number[], p: number): number | null {
51 if (values.length === 0) return null;
52 const sorted = [...values].sort((a, b) => a - b);
53 return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))]!;
54}
56function choose(n: number, k: number): number {
57 let out = 1;
58 for (let i = 1; i <= k; i++) out = (out * (n - k + i)) / i;
59 return out;
60}
62/** P(fewer than k of n independent trials pass) when one trial fails with probability `failRate`. */
63export function panelRedProbability(failRate: number, n: number, k: number): number {
64 let red = 0;
65 for (let passes = 0; passes < k; passes++) red += choose(n, passes) * (1 - failRate) ** passes * failRate ** (n - passes);
66 return red;
67}
69export interface CensusStats {
70 runs: number;
71 trials: number;
72 failed: number;
73 byKind: Record<string, { trials: number; failed: number }>;
74 byClass: Record<string, number>;
75 /** P(lane red) with each case's pooled trial failure rate, under the recorded panels. */
76 redInForce: number;
77 /** The same evidence with every case judged on one trial. */
78 redSingleTrial: number;
79}
81/**
82 * CEO-06/22 census arithmetic over one lane's trial records. Every observed
83 * failure is treated as nondeterminism, so both red probabilities are upper
84 * bounds for known-good inputs; they are reported separately from the raw
85 * trial failure rate so a verdict-policy change cannot read as a reliability gain.
86 */
87export function censusStats(records: TrialRecord[], runIds: string[]): CensusStats {
88 const scored = records.filter(record => record.outcome !== 'skipped');
89 const byKind: CensusStats['byKind'] = {};
90 const byClass: Record<string, number> = {};
91 const perCase = new Map<string, { trials: number; failed: number; n: number; k: number }>();
92 for (const record of scored) {
93 const failed = record.outcome === 'failed';
94 const kind = (byKind[record.kind] ??= { trials: 0, failed: 0 });
95 kind.trials += 1;
96 if (failed) { kind.failed += 1; byClass[record.failure_class ?? 'unknown'] = (byClass[record.failure_class ?? 'unknown'] ?? 0) + 1; }
97 const slot = perCase.get(record.case) ?? { trials: 0, failed: 0, n: 1, k: 1 };
98 slot.trials += 1;
99 if (failed) slot.failed += 1;
100 if (record.panel.n > slot.n) { slot.n = record.panel.n; slot.k = record.panel.k; }
101 perCase.set(record.case, slot);
102 }
103 let greenInForce = 1, greenSingle = 1;
104 for (const slot of perCase.values()) {
105 const p = slot.failed / slot.trials;
106 greenInForce *= 1 - panelRedProbability(p, slot.n, slot.k);
107 greenSingle *= 1 - p;
108 }
109 return { runs: runIds.length, trials: scored.length, failed: scored.filter(record => record.outcome === 'failed').length,
110 byKind, byClass, redInForce: 1 - greenInForce, redSingleTrial: 1 - greenSingle };
111}
113const pct = (value: number) => `${(Math.round(value * 1000) / 10).toFixed(1)}%`;
114const minutes = (ms: number | null) => ms === null ? 'n/a' : `${(ms / 60_000).toFixed(1)} min`;
115const span = (from: string | null, to: string | null) => from && to ? Math.max(0, Date.parse(to) - Date.parse(from)) : null;
117/** DX-9: close only when every enforced check has evidence and passes; missing evidence keeps the issue open. */
118export function issueActionFor(checks: Check[]): IssueAction {
119 if (checks.some(check => check.status === 'fail')) return 'upsert';
120 if (checks.some(check => check.status === 'unavailable')) return 'keep-open';
121 return 'close';
122}
124// --- Checks ---
126export function flakeCheck(window: FlakeWindow | Error): Check {
127 const base = { id: 'flaky-files', title: 'Free-suite flaky files (W7a)',
128 fix: 'Fix the race at source (docs/TESTING_INTERNALS.md#flake-ledger); never raise the timeout or add a retry. Reproduce with `bun test <file>` under load.' };
129 if (window instanceof Error) return { ...base, status: 'unavailable', detail: `unavailable: ${unavailableReason(window)}` };
130 const failing = window.files.filter(file => file.failing);
131 if (!window.enforceable) {
132 return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)` };
133 }
134 if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs` };
135 return { ...base, status: 'fail',
136 detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') };
137}
139export function unseededCheck(root: string): Check {
140 const base = { id: 'unseeded-free-files', title: 'Free test files missing from the duration seed (CEO-10)',
141 fix: 'Refresh the seed with UBICLOUD_API_KEY set: `bun run test:ubicloud --record-durations` times every free file on a CI-mirrored VM and copies scripts/free-test-durations.json back into this checkout; commit it (docs/TESTING_INTERNALS.md#free-suite-duration-seed). Without UBICLOUD_API_KEY, leave the refresh to a maintainer: a 4-core laptop or sandbox recording skips or slows browser and display tests, so it is not accepted.' };
142 const durations = loadFreeTestDurations(root);
143 if (!durations) return { ...base, status: 'unavailable', detail: 'unavailable: scripts/free-test-durations.json is missing or unreadable; next: refresh the seed' };
144 const unseeded = unseededFreeFiles(collectFreeTestFiles(root), durations);
145 const shown = unseeded.slice(0, 10).join(', ') + (unseeded.length > 10 ? `, +${unseeded.length - 10} more` : '');
146 if (unseeded.length > UNSEEDED_LIMIT) return { ...base, status: 'fail', detail: `stale seed: ${unseeded.length} unseeded file(s) > ${UNSEEDED_LIMIT}: ${shown}` };
147 return { ...base, status: 'pass', detail: `${unseeded.length} unseeded file(s) (limit ${UNSEEDED_LIMIT})${unseeded.length ? `: ${shown}` : ''}` };
148}
150// --- Metric collection ---
152function attempt<T>(fn: () => T): T | Error {
153 try { return fn(); } catch (error) { return error instanceof Error ? error : new Error(String(error)); }
154}
156function metric(id: string, title: string, rule: string, compute: () => { value: string; detail?: string[] }): Metric {
157 const result = attempt(compute);
158 if (result instanceof Error) return { id, title, rule, unavailable: unavailableReason(result) };
159 return { id, title, rule, ...result };
160}
162function records(dirs: string[]): TrialRecord[] {
163 return dirs.flatMap(dir => readTrialOutcomeDir(dir).records);
164}
166function readJson(file: string): any {
167 return JSON.parse(fs.readFileSync(file, 'utf8'));
168}
170/** Net LOC under COMPLEXITY_PATHS and the package-script count at one ref. */
171export function complexityAt(root: string, ref: string): { loc: number; scripts: number } {
172 const grep = spawnSync('git', ['grep', '-c', '-I', '', ref, '--', ...COMPLEXITY_PATHS], { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 60_000 });
173 if (grep.status !== 0 && grep.status !== 1) throw new Error(`git grep ${ref} failed: ${grep.stderr.trim()}`);
174 const loc = grep.stdout.split('\n').filter(Boolean).reduce((sum, line) => sum + Number(line.slice(line.lastIndexOf(':') + 1) || 0), 0);
175 const pkg = spawnSync('git', ['show', `${ref}:package.json`], { cwd: root, encoding: 'utf8', timeout: 10_000 });
176 if (pkg.status !== 0) throw new Error(`git show ${ref}:package.json failed: ${pkg.stderr.trim()}`);
177 return { loc, scripts: Object.keys(JSON.parse(pkg.stdout).scripts ?? {}).length };
178}
180export function collectHealth(inputs: HealthInputs): HealthReport {
181 const { client, repo, now, sinceDays, cacheDir } = inputs;
182 const root = inputs.root ?? ROOT;
183 const since = new Date(now - sinceDays * 86_400_000).toISOString();
184 const fetcher = historyFetcher(client);
185 const artifactDirs = (run: CiRun, match: (name: string) => boolean) =>
186 downloadRunArtifacts({ repo, run, cacheDir, fetcher, match, maxBytes: 64 * 1024 * 1024 });
187 const prRuns = attempt(() => listWorkflowRuns(client, repo, 'evals.yml', { since, event: 'pull_request' }));
188 const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
189 const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled')
190 .map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) })));
191 const prManifests = attempt(() => need(prRuns).flatMap(run => {
192 const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate));
193 return file ? [{ run, manifest: readJson(file) }] : [];
194 }));
195 const metrics: Metric[] = [];
196 let costPerMergedPr: string | undefined;
198 metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => {
199 const modes = new Map<string, number>();
200 for (const { manifest } of need(prManifests)) {
201 const mode = String(manifest.prCoverage?.mode ?? 'none');
202 modes.set(mode, (modes.get(mode) ?? 0) + 1);
203 }
204 const total = [...modes.values()].reduce((a, b) => a + b, 0);
205 if (total === 0) throw new Error('no paid-plan manifests in the window');
206 return { value: `${pct((modes.get('full-fallback') ?? 0) / total)} (${modes.get('full-fallback') ?? 0}/${total} runs)`,
207 detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) };
208 }));
210 metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => {
211 const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges));
212 if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window');
213 const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`;
214 const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort();
215 return { value: `${mean(planned)} per run (${planned.length} runs)`,
216 detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) };
217 }));
219 metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => {
220 const entries = need(prRecords);
221 if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window');
222 const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0);
223 const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length;
224 const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')]));
225 const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`;
226 const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort();
227 return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`,
228 detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) };
229 }));
231 metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => {
232 const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error);
233 if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window');
234 const records = prRecords instanceof Error ? [] : prRecords;
235 const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length;
236 const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0);
237 const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0);
238 const misses: string[] = [];
239 const flaky: string[] = [];
240 for (const { run, manifest } of shadowed) {
241 const trials = records.find(entry => entry.run.id === run.id)?.trials ?? [];
242 for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) {
243 if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue;
244 const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha
245 && (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0);
246 if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`);
247 }
248 }
249 const week = 7 / sinceDays;
250 const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost';
251 return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`,
252 detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] };
253 }));
255 metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => {
256 const all = need(prRecords).flatMap(entry => entry.trials);
257 const reused = all.filter(record => record.execution === 'reused').length;
258 return { value: `${reused} of ${all.length} trial records` };
259 }));
261 metrics.push(metric('pr-spend', 'Recorded PR-lane spend', 'sum of executed trial cost_usd in PR runs; per merged PR = / PRs merged in the window; per validated revision = / distinct head SHAs with a success or failure verdict', () => {
262 const entries = need(prRecords);
263 const spend = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed')
264 .reduce((sum, record) => sum + record.cost_usd, 0);
265 const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size;
266 const merged = listMergedPulls(client, repo, since);
267 if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`;
268 return { value: `$${spend.toFixed(2)} recorded`, detail: [
269 `per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`,
270 `per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] };
271 }));
273 metrics.push(metric('cancelled-slice-minutes', 'Cancelled slice-minutes per week', 'wall minutes of eval-slices jobs in cancelled evals.yml PR runs, scaled to 7 days', () => {
274 const cancelled = need(prRuns).filter(run => run.conclusion === 'cancelled');
275 const total = cancelled.flatMap(run => listRunJobs(client, repo, run.id, run.attempt))
276 .filter(job => job.name.startsWith('eval-slices')).reduce((sum, job) => sum + (span(job.startedAt, job.completedAt) ?? 0), 0);
277 return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` };
278 }));
280 metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait, which pr-summary runs skip since GSTA-149, so their verdicts land about 10 min sooner)', () => {
281 const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!);
282 if (walls.length === 0) throw new Error('no PR run reached a verdict in the window');
283 return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` };
284 }));
286 metrics.push(...censusMetrics(inputs, artifactDirs));
288 const flakes = attempt(() => readCiFlakeLedgers({ client, repo, sinceDays, now, cacheDir }));
289 metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs', () => {
290 const window = need(flakes);
291 return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s))`,
292 detail: window.files.slice(0, 15).map(file => `${file.file}: ${file.flakyRuns}/${window.runs} runs (${pct(file.rate)})${file.failing ? ' FAIL > 5%' : ''}`) };
293 }));
295 metrics.push(metric('free-wall', 'Free-suite and Windows wall', 'p50 of run_started_at to updated_at for successful main runs of free-tests.yml and successful pull_request runs of windows-free-tests.yml (its workflow_dispatch duration recordings excluded)', () => {
296 const walls = (workflow: string, filter: { branch?: string; event?: string }) =>
297 listWorkflowRuns(client, repo, workflow, { since, ...filter }).filter(run => run.conclusion === 'success').map(run => span(run.startedAt, run.updatedAt)!);
298 const free = walls('free-tests.yml', { branch: 'main' });
299 const windows = walls('windows-free-tests.yml', { event: 'pull_request' });
300 const windowsValue = windows.length ? `${minutes(percentile(windows, 0.5))} (${windows.length} runs)` : 'no successful PR runs in window';
301 return { value: `free-tests ${minutes(percentile(free, 0.5))} (${free.length} runs); windows-free-tests (successful PR runs) ${windowsValue}` };
302 }));
304 metrics.push(metric('free-serial', 'Free-suite serial time', 'sum of scripts/free-test-durations.json (the committed seed)', () => {
305 const durations = loadFreeTestDurations(root);
306 if (!durations) throw new Error('scripts/free-test-durations.json is missing or unreadable');
307 return { value: `${Math.round(Object.values(durations).reduce((a, b) => a + b, 0) / 1000)} s over ${Object.keys(durations).length} files` };
308 }));
310 metrics.push(metric('net-complexity', 'Net complexity (CEO-32, reported, no target)', `lines under ${COMPLEXITY_PATHS.join(', ')} and package.json script count, HEAD vs --complexity-base`, () => {
311 const head = complexityAt(root, 'HEAD');
312 if (!inputs.complexityBase) return { value: `HEAD: ${head.loc} lines, ${head.scripts} package scripts (pass --complexity-base <ref> for the delta)` };
313 const base = complexityAt(root, inputs.complexityBase);
314 const sign = (n: number) => (n >= 0 ? `+${n}` : `${n}`);
315 return { value: `${sign(head.loc - base.loc)} lines (${base.loc} -> ${head.loc}), ${sign(head.scripts - base.scripts)} package scripts (${base.scripts} -> ${head.scripts}) vs ${inputs.complexityBase}` };
316 }));
318 const checks = [flakeCheck(flakes), unseededCheck(root)];
319 const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`;
320 return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) };
321}
323function listMergedPulls(client: GhClient, repo: string, since: string): number {
324 let merged = 0;
325 for (let page = 1; page <= 10; page++) {
326 const pulls = client.getJson(`repos/${repo}/pulls?state=closed&base=main&sort=updated&direction=desc&per_page=100&page=${page}`) as any[];
327 if (!Array.isArray(pulls)) throw new Error('pulls listing returned no array');
328 merged += pulls.filter(pull => pull.merged_at && String(pull.merged_at) >= since).length;
329 if (pulls.length < 100 || pulls.every(pull => String(pull.updated_at) < since)) break;
330 }
331 return merged;
332}
334/**
335 * Census metrics from main's evals-periodic history (scheduled runs plus main
336 * dispatches, the W4b weekly history): CEO-06/22 per lane over the trailing 4
337 * scheduled censuses, the INFRA re-dispatch count, slice wall, and DX-12
338 * quarantine evidence (qualifying vs required trials) over the last 10 runs.
339 */
340function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: (name: string) => boolean) => string[]): Metric[] {
341 const { client, repo, now } = inputs;
342 const history = attempt(() => listWorkflowRuns(client, repo, 'evals-periodic.yml',
343 { since: new Date(now - Math.max(inputs.sinceDays, 84) * 86_400_000).toISOString(), branch: 'main' })
344 .filter(run => run.event === 'schedule' || run.event === 'workflow_dispatch'));
345 const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
346 const censuses = attempt(() => {
347 const out: { run: CiRun; trials: TrialRecord[] }[] = [];
348 for (const run of need(history)) {
349 if (out.length === 4) break;
350 const trials = records(artifactDirs(run, name => name.startsWith('trial-outcomes')));
351 if (trials.length) out.push({ run, trials });
352 }
353 return out;
354 });
355 const out: Metric[] = [];
356 for (const lane of ['periodic', 'gate'] as const) {
357 out.push(metric(`census-${lane}`, `${lane === 'gate' ? 'Gate' : 'Periodic'} census reliability (CEO-06/22)`,
358 'trailing 4 main censuses with trial outcomes (scheduled runs and main dispatches): expected failed trials per run, raw trial failure rate per kind and class, and P(lane red) from pooled per-case rates under the recorded panels vs single-trial aggregation (every failure treated as nondeterminism: upper bounds for known-good inputs)', () => {
359 const runs = need(censuses);
360 const laneRuns = runs.filter(entry => entry.trials.some(record => (record.lane ?? '').startsWith(`${lane}/`)));
361 if (laneRuns.length === 0) throw new Error(`no main census with ${lane} trial outcomes in the last 84 days`);
362 const stats = censusStats(laneRuns.flatMap(entry => entry.trials).filter(record => (record.lane ?? '').startsWith(`${lane}/`)), laneRuns.map(entry => `${entry.run.id}`));
363 return { value: `${(stats.failed / stats.runs).toFixed(2)} expected failed trials/run (${stats.failed}/${stats.trials} over ${stats.runs} run(s)); P(red) ${pct(stats.redInForce)} in force, ${pct(stats.redSingleTrial)} single-trial`,
364 detail: [...Object.entries(stats.byKind).map(([kind, slot]) => `kind ${kind}: ${slot.failed}/${slot.trials} failed (${pct(slot.failed / Math.max(1, slot.trials))})`),
365 ...Object.entries(stats.byClass).map(([cls, count]) => `class ${cls}: ${count}`)] };
366 }));
367 }
368 out.push(metric('census-redispatch', 'INFRA re-dispatches (ENG-10)', 'trailing 4 main census runs whose redispatch job succeeded; longest paid slice job wall in the same runs', () => {
369 const runs = need(history).slice(0, 4);
370 const jobs = runs.map(run => listRunJobs(client, repo, run.id, run.attempt));
371 const redispatched = jobs.filter(list => list.some(job => job.name === 'redispatch' && job.conclusion === 'success')).length;
372 const walls = jobs.flat().filter(job => /^(eval-slices|eval-codex-slices|gate-census)/.test(job.name)).map(job => span(job.startedAt, job.completedAt) ?? 0);
373 return { value: `${redispatched} of ${runs.length} census(es) re-dispatched; longest paid slice job ${minutes(walls.length ? Math.max(...walls) : null)}` };
374 }));
375 out.push(metric('quarantine-evidence', 'Quarantine evidence (DX-12)', `blocking cases with >= ${EVAL_POLICY.quarantine.entry.minTrials} qualifying v${EVAL_POLICY.version} trials on their current series, over the last 10 main census runs`, () => {
376 const runs = need(history).slice(0, 10);
377 const report = analyzePassRates(runs.flatMap(run => records(artifactDirs(run, name => name.startsWith('trial-outcomes')))), { weeklyRuns: runs.map(run => run.createdAt) });
378 const blocking = report.cases.filter(entry => entry.tier === 'gate' || entry.tier === 'periodic');
379 const sufficient = blocking.filter(entry => (entry.current?.trials ?? 0) >= EVAL_POLICY.quarantine.entry.minTrials);
380 const labels = new Map<string, number>();
381 for (const entry of sufficient) labels.set(entry.label, (labels.get(entry.label) ?? 0) + 1);
382 return { value: `${sufficient.length} of ${blocking.length} blocking case(s) have enough evidence; ${blocking.length - sufficient.length} insufficient evidence (not the same as healthy)`,
383 detail: [...[...labels].map(([label, count]) => `${label}: ${count}`),
384 ...(report.newerPolicyTrials ? [`ignored ${report.newerPolicyTrials} newer-policy trial(s)`] : []),
385 ...(report.olderPolicyTrials ? [`${report.olderPolicyTrials} older-policy trial(s) display only`] : [])] };
386 }));
387 return out;
388}
390// --- Rendering ---
392export function formatHealth(report: HealthReport): string {
393 const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])];
394 for (const entry of report.metrics) {
395 lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`);
396 lines.push(` rule: ${entry.rule}`);
397 for (const detail of entry.detail ?? []) lines.push(` ${detail}`);
398 }
399 lines.push('Enforced checks:');
400 for (const check of report.checks) {
401 lines.push(`- [${check.status.toUpperCase()}] ${check.title}: ${check.detail}`);
402 if (check.status !== 'pass') lines.push(` fix: ${check.fix}`);
403 }
404 const unavailable = report.checks.filter(check => check.status === 'unavailable').map(check => check.id);
405 if (unavailable.length) lines.push(`Missing inputs (an unavailable check never fails the run, and keeps a tracking issue open): ${unavailable.join(', ')}`);
406 return lines.join('\n');
407}
409export function formatHealthMarkdown(report: HealthReport): string {
410 return ['## test:health', '', '```', formatHealth(report), '```', ''].join('\n');
411}
413export const HELP = `Usage: bun run test:health [options]
414 bun run test:health flake-summary <dir>
416Prints the test/eval/CI success metrics from GitHub Actions history (needs gh
417with read access to the repository; \`gh auth status\` checks it). A metric whose
418input cannot be read prints "unavailable: <reason>; next: <step>", never 0.
420Options:
421 --since-days <n> window in days (default 7)
422 --enforce exit 1 when an enforced check fails (flaky file > 5% of
423 >= 20 main free-tests runs; > 5 unseeded free files);
424 without it the command always exits 0
425 --json machine-readable report (includes issueAction)
426 --summary <file> append a markdown report (job summary / issue body)
427 --repo <owner/name> repository (default: origin)
428 --complexity-base <ref> report the net-complexity delta against <ref>
429 --help this text
431flake-summary <dir> markdown table of the flake-ledger-* artifacts under <dir>`;
433if (import.meta.main) {
434 const argv = process.argv.slice(2);
435 if (argv.includes('--help') || argv.includes('-h')) { console.log(HELP); process.exit(0); }
436 if (argv[0] === 'flake-summary') {
437 if (!argv[1]) { console.error('usage: bun run test:health flake-summary <dir>'); process.exit(2); }
438 console.log(formatFlakeSummary(readFlakeLedgerDir(argv[1])));
439 process.exit(0);
440 }
441 const valued = new Set(['--since-days', '--summary', '--repo', '--complexity-base']);
442 const known = new Set([...valued, '--enforce', '--json']);
443 for (let index = 0; index < argv.length; index++) {
444 if (!known.has(argv[index]!)) { console.error(`test:health: unknown argument ${argv[index]}\n\n${HELP}`); process.exit(2); }
445 if (valued.has(argv[index]!)) {
446 if (argv[index + 1] === undefined || argv[index + 1]!.startsWith('--')) { console.error(`test:health: ${argv[index]} needs a value`); process.exit(2); }
447 index++;
448 }
449 }
450 const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
451 const sinceDays = Number(flag('--since-days') ?? 7);
452 if (!Number.isInteger(sinceDays) || sinceDays < 1) { console.error('test:health: --since-days must be a positive integer'); process.exit(2); }
453 const repo = flag('--repo') ?? repoSlug(ROOT);
454 const cacheDir = path.join(path.resolve(resolveStateRoot()), 'test-health-cache', repo.replace('/', '-'));
455 const report = collectHealth({ client: GH_CLIENT, repo, now: Date.now(), sinceDays, cacheDir, complexityBase: flag('--complexity-base') });
456 console.log(argv.includes('--json') ? JSON.stringify(report, null, 2) : formatHealth(report));
457 const summary = flag('--summary');
458 if (summary) fs.appendFileSync(summary, formatHealthMarkdown(report));
459 if (argv.includes('--enforce') && report.checks.some(check => check.status === 'fail')) process.exit(1);
460}