eval-flake-rank.ts
696 lines · scripts/eval-flake-rank.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * eval-pass-rates (alias: eval-flake-rank) — per-case trial pass rates.
4 *
5 * Reads trial records (one JSONL line per trial: case, kind, trial, outcome,
6 * exit_reason, duration, cost, model, CLI version, series identity, run id,
7 * sha, policy_version) from the last N completed `evals-periodic.yml` runs of
8 * the weekly history: scheduled runs on `main` plus `main` dispatches
9 * (`--branch <name>` reads one branch's runs for inspection). Trials from
10 * branch census runs in the same window pool into a series main has also
11 * run, never into a new one; weeks for quarantine expiry count main only,
12 * downloading only each run's small `trial-outcomes` artifact through `gh`,
13 * plus any local eval dirs, and
14 * prints per-case per-trial pass rates with 95% Wilson intervals.
15 *
16 * A series is one case under one input identity (EVAL_POLICY v2: the bytes
17 * the case owns plus HARNESS_VERSION, stamped by scripts/eval-trial-series.ts
18 * caseSeriesIdentitiesV2), grouped by model and CLI version, per
19 * policy_version. A new identity starts a new series; earlier
20 * series stay visible. Only trials of the current series under the reader's
21 * own EVAL_POLICY.version feed the labels and alarms: an older policy's
22 * trials are display-only, and a newer policy's trials (written by a later
23 * checkout) are ignored with a printed count, so a reader rolled back past a
24 * policy bump never pools records it cannot interpret. Legacy eval-store records (`--backfill`, `--dir`) are
25 * imported as pre-policy trials (first attempt only; a missing attempt means
26 * 1) and are display-only.
27 *
28 * Labels: INCONCLUSIVE (below the entry rule's minimum trials), BROKEN (latest run 0/n
29 * after a prior interval at or above the entry rate), FLAKY (failures and an
30 * interval straddling the entry rate), FAILING (interval below the entry
31 * rate), PASSING (otherwise).
32 *
33 * The weekly gate (`--gate`) exits non-zero with ACTION REQUIRED when a
34 * non-quarantined case meets the quarantine entry rule, a rule case behaves
35 * like a behavior case, a blocking case's current-identity rate is
36 * significantly below its previous identity (one-sided Fisher exact,
37 * Holm-controlled across cases), or a CASE_QUARANTINE entry has met its exit
38 * rule, expired, or pushed its tier over the cap. History that cannot be
39 * fetched fails the gate closed.
40 *
41 * Usage:
42 * bun run eval:pass-rates # last 10 weekly runs (main: scheduled + dispatched)
43 * bun run eval:pass-rates --case <id> --runs 20
44 * bun run eval:pass-rates --dir <path> # local eval dirs / downloaded artifacts (repeatable)
45 * bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy
46 * bun run eval:pass-rates --json | --gate
47 * bun run eval:pass-rates --headroom | --reds | --run <id> # report views (scripts/lib/eval-history.ts)
48 * bun run eval:pass-rates --help # every flag, no network
49 */
51import * as fs from 'node:fs';
52import * as path from 'node:path';
53import { isPartialEval, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, parseTrialOutcomes, sanitizeTrialError,
54 TRIAL_OUTCOME_SCHEMA, type EvalCaseKind, type EvalResult, type TrialOutcomeRecord } from '../test/helpers/eval-store';
55import { flakeLedgerPath, type FlakeLedgerEntry } from './test-free-shards';
56import { E2E_KINDS, E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
57import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
58import { CASE_TEST_NAMES } from './test-paid-shards';
59import { resolveStateRoot } from '../lib/state-root';
60import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
61 redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history';
62import { downloadRunArtifacts, GH_HISTORY, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES, type WeeklyRun } from './lib/ci-history';
64interface TestSeries {
65 name: string;
66 runs: number;
67 passes: number;
68 fails: number;
69 manualAccepted: number;
70 retriedPasses: number;
71 totalAttempts: number;
72 totalCostUsd: number;
73 totalDurationMs: number;
74 lastSeen: string;
75}
77export function aggregate(evalFiles: string[]): Map<string, TestSeries> {
78 const series = new Map<string, TestSeries>();
79 for (const file of evalFiles) {
80 let run: EvalResult;
81 try {
82 run = JSON.parse(fs.readFileSync(file, 'utf-8'));
83 } catch { continue; }
84 if (isPartialEval(run, file)) continue; // in-progress accumulators are not runs
85 if (!Array.isArray(run.tests)) continue;
86 // Group this run's entries by name so N attempts = 1 run of that test.
87 const byName = new Map<string, typeof run.tests>();
88 for (const t of run.tests) {
89 const list = byName.get(t.name) ?? [];
90 list.push(t);
91 byName.set(t.name, list);
92 }
93 for (const [name, entries] of byName) {
94 const s = series.get(name) ?? {
95 name, runs: 0, passes: 0, fails: 0, manualAccepted: 0, retriedPasses: 0,
96 totalAttempts: 0, totalCostUsd: 0, totalDurationMs: 0, lastSeen: '',
97 };
98 const final = entries[entries.length - 1];
99 s.totalAttempts += entries.length;
100 const outcome = evalEntryOutcome(final);
101 if (outcome === 'manual-review') s.manualAccepted += 1;
102 else {
103 s.runs += 1;
104 if (outcome === 'passed') s.passes += 1; else s.fails += 1;
105 if (outcome === 'passed' && entries.length > 1) s.retriedPasses += 1;
106 }
107 for (const e of entries) {
108 s.totalCostUsd += e.cost_usd || 0;
109 s.totalDurationMs += e.duration_ms || 0;
110 }
111 if (run.timestamp > s.lastSeen) s.lastSeen = run.timestamp;
112 series.set(name, s);
113 }
114 }
115 return series;
116}
118export function collectEvalFiles(dir: string, sinceDays = 60): string[] {
119 if (!fs.existsSync(dir)) return [];
120 const cutoff = Date.now() - sinceDays * 86_400_000;
121 const out: string[] = [];
122 for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) {
123 if (!isFinalizedEvalResultFile(name)) continue;
124 const full = path.join(dir, name);
125 try {
126 // Recency bound (review finding): E2E results embed full transcripts
127 // (MBs each) and the scan is otherwise unbounded over all-time history.
128 if (fs.statSync(full).mtimeMs < cutoff) continue;
129 } catch { continue; }
130 out.push(full);
131 }
132 return out;
133}
135/** The local free-suite flake ledger (CI ledgers are read by test:health through scripts/lib/ci-history.ts). */
136function readFreeLedger(): FlakeLedgerEntry[] {
137 try {
138 return parseFlakeLedger(fs.readFileSync(flakeLedgerPath(), 'utf-8'));
139 } catch { return []; }
140}
142// --- Trial records ---
144/**
145 * A trial record as pass-rates reads it: eval-store's trial-outcomes schema
146 * plus what the report job stamps (scripts/eval-trial-series.ts): the series
147 * identity, the full consumed-input fingerprint and HARNESS_VERSION, both
148 * provenance only. policy_version 0 marks a pre-policy (backfilled) record.
149 */
150export type TrialRecord = TrialOutcomeRecord & { series_identity?: string; series_fingerprint?: string; harness_version?: number };
153/** Every `trial-outcomes*.jsonl` file under a directory, size-capped, schema-validated by eval-store. */
154export function readTrialOutcomeDir(dir: string): { records: TrialRecord[]; errors: string[] } {
155 const records: TrialRecord[] = [];
156 const errors: string[] = [];
157 if (!fs.existsSync(dir)) return { records, errors };
158 for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) {
159 if (!/^trial-outcomes[^/\\]*\.jsonl$/.test(path.basename(name))) continue;
160 const full = path.join(dir, name);
161 const parsed = parseTrialOutcomes(fs.readFileSync(full, 'utf8'), { maxBytes: TRIAL_OUTCOMES_MAX_BYTES });
162 records.push(...parsed.records.map(record => ({
163 ...record, series_identity: typeof (record as TrialRecord).series_identity === 'string'
164 ? (record as TrialRecord).series_identity!.slice(0, 64) : undefined })));
165 errors.push(...parsed.errors.map(error => `${full}: ${error}`));
166 }
167 return { records, errors };
168}
170// --- Registry attribution and series identity ---
172export interface Registry {
173 kinds: Record<string, EvalCaseKind>;
174 tiers: Record<string, string>;
175 touchfiles: Record<string, string[]>;
176 judgeTouchfiles: Record<string, string[]>;
177 globals: readonly string[];
178 testNames: Record<string, string>;
179}
181export const LIVE_REGISTRY: Registry = {
182 kinds: E2E_KINDS, tiers: E2E_TIERS, touchfiles: E2E_TOUCHFILES, judgeTouchfiles: LLM_JUDGE_TOUCHFILES,
183 globals: GLOBAL_TOUCHFILES, testNames: CASE_TEST_NAMES,
184};
186/** A case's tier: its E2E_TIERS value, or 'judge' for an LLM-judge entry. */
187export function caseTier(id: string, registry: Registry = LIVE_REGISTRY): string {
188 return registry.tiers[id] ?? (id in registry.judgeTouchfiles ? 'judge' : 'unknown');
189}
191/**
192 * Attribute a legacy eval-store record to a registry id: the case-shard slug
193 * suffix (`<file>--<id>`), the recorded name or its exact slug (`/qa b6-static`
194 * is `qa-b6-static`), a CASE_TEST_NAMES label, or the only id its shard file
195 * registers. Anything else is unattributed (null).
196 */
197export function attributeLegacyRecord(name: string, shard: string | undefined, registry: Registry = LIVE_REGISTRY): string | null {
198 const known = (id: string) => id in registry.kinds;
199 const [slugFile, slugCase] = (shard ?? '').split('--');
200 if (slugCase && known(slugCase)) return slugCase;
201 if (known(name)) return name;
202 const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
203 if (known(slug)) return slug;
204 const labeled = Object.entries(registry.testNames).find(([, label]) => label === name)?.[0];
205 if (labeled && known(labeled)) return labeled;
206 if (slugFile) {
207 const file = `test/${slugFile}.test.ts`;
208 const owners = Object.keys(registry.touchfiles).filter(id => registry.touchfiles[id]!.includes(file));
209 if (owners.length === 1 && known(owners[0]!)) return owners[0]!;
210 }
211 return null;
212}
214/**
215 * Import legacy eval-store result files as pre-policy trials (policy_version
216 * 0, source 'backfill'): first attempt only (a missing attempt means 1),
217 * attributed by registry id, never guessed. A manual-review acceptance carries
218 * no automated verdict: it is counted and shown, never scored. Without a CI
219 * run, each local result file is its own run.
220 */
221export function backfillEvalFiles(files: string[], run?: { run_id: string; sha?: string; timestamp?: string },
222 registry: Registry = LIVE_REGISTRY): { records: TrialRecord[]; unattributed: string[]; manualReviews: string[] } {
223 const records: TrialRecord[] = [];
224 const unattributed = new Set<string>();
225 const manualReviews: string[] = [];
226 for (const file of files) {
227 let result: EvalResult & { shard?: string; claude_cli_version?: string };
228 try { result = JSON.parse(fs.readFileSync(file, 'utf8')); } catch { continue; }
229 if (isPartialEval(result, file) || !Array.isArray(result.tests)) continue;
230 const seen = new Set<string>();
231 for (const entry of result.tests) {
232 if ((entry.attempt ?? 1) !== 1 || seen.has(entry.name)) continue;
233 seen.add(entry.name);
234 const id = attributeLegacyRecord(entry.name, result.shard, registry);
235 if (!id) { unattributed.add(entry.name); continue; }
236 const outcome = evalEntryOutcome(entry);
237 if (outcome === 'manual-review') { manualReviews.push(id); continue; }
238 records.push({
239 schema: TRIAL_OUTCOME_SCHEMA, case: id,
240 file: result.shard ? `test/${result.shard.split('--')[0]}.test.ts` : 'unknown',
241 tier: caseTier(id, registry), kind: registry.kinds[id]!, trial: 1, panel: { n: 1, k: 1 }, attempt: 1,
242 outcome, ...(outcome === 'failed' ? { failure_class: failureClassOf(entry) } : {}),
243 exit_reason: entry.exit_reason, error: sanitizeTrialError(entry.error),
244 duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0), ...(entry.cost_known === false ? { cost_known: false } : {}),
245 model: entry.model, cli_version: result.claude_cli_version, policy_version: 0, quarantined: false,
246 execution: entry.execution === 'reused' ? 'reused' : 'executed', source: 'backfill',
247 run_id: run?.run_id ?? `local:${file}`, sha: run?.sha ?? result.git_sha, recorded_at: run?.timestamp ?? result.timestamp,
248 });
249 }
250 }
251 return { records, unattributed: [...unattributed].sort(), manualReviews };
252}
254// --- Statistics ---
256/** 95% Wilson score interval for k successes in n trials. */
257export function wilsonInterval(k: number, n: number, z = 1.96): { lo: number; hi: number } {
258 if (n <= 0) return { lo: 0, hi: 1 };
259 const p = k / n, z2 = z * z, denom = 1 + z2 / n;
260 const center = (p + z2 / (2 * n)) / denom;
261 const half = (z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n))) / denom;
262 return { lo: Math.max(0, center - half), hi: k === n ? 1 : Math.min(1, center + half) };
263}
265function logChoose(n: number, k: number): number {
266 let sum = 0;
267 for (let i = 1; i <= k; i++) sum += Math.log(n - k + i) - Math.log(i);
268 return sum;
269}
271/**
272 * One-sided Fisher exact p-value that the CURRENT pass rate is below the
273 * PREVIOUS one: P(X <= curPass) under the hypergeometric null with the
274 * observed margins.
275 */
276export function fisherOneSidedLower(curPass: number, curN: number, prevPass: number, prevN: number): number {
277 const passes = curPass + prevPass, total = curN + prevN;
278 const denom = logChoose(total, passes);
279 let p = 0;
280 for (let x = Math.max(0, passes - prevN); x <= curPass; x++) p += Math.exp(logChoose(curN, x) + logChoose(prevN, passes - x) - denom);
281 return Math.min(1, p);
282}
284/** Holm step-down: the indices whose p-values are rejected at family-wise alpha. */
285export function holmRejections(pValues: number[], alpha: number): Set<number> {
286 const order = pValues.map((p, index) => ({ p, index })).sort((a, b) => a.p - b.p);
287 const rejected = new Set<number>();
288 for (let rank = 0; rank < order.length; rank++) {
289 if (order[rank]!.p > alpha / (order.length - rank)) break;
290 rejected.add(order[rank]!.index);
291 }
292 return rejected;
293}
295// --- Analysis ---
297export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING';
298export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired'
299 | 'quarantine-cap' | 'quarantine-invalid' | 'headroom';
301/** The EVAL_POLICY fields pass-rates reads (structural, so tests can vary them). */
302export interface PassRatePolicy {
303 version: number;
304 quarantine: { entry: { rate: number; minTrials: number }; exit: { rate: number; minTrials: number }; capFraction: number; expiryWeeklyRuns: number };
305 drift: { fisherAlpha: number; fisherMinPerSide: number };
306}
308export type QuarantineEntry = (typeof CASE_QUARANTINE)[string];
310/** Tiers whose cases block a lane; quarantine applies only to them. */
311export const BLOCKING_TIERS: readonly string[] = ['gate', 'periodic'];
312const QUARANTINE_FAILURE_CLASSES: readonly string[] = ['detector', 'harness', 'model-latency'];
314export interface SeriesStats {
315 key: string;
316 identity: string;
317 model: string;
318 cli: string;
319 policyVersion: number;
320 passes: number;
321 /** Scored trials: passed + failed (skipped trials carry no verdict). */
322 trials: number;
323 infra: number;
324 interval: { lo: number; hi: number };
325 firstSeen: string;
326 lastSeen: string;
327 runs: string[];
328}
330export interface CasePassRate {
331 case: string;
332 kind: EvalCaseKind;
333 tier: string;
334 quarantined: boolean;
335 label: PassRateLabel;
336 /** Manual-review acceptances: visible, never scored. */
337 manualReviews: number;
338 current: SeriesStats | null;
339 previous: SeriesStats | null;
340 prePolicy: SeriesStats | null;
341 series: SeriesStats[];
342 latestRun: { runId: string; passes: number; trials: number } | null;
343}
345export interface Alarm { kind: AlarmKind; case: string; message: string }
347export interface PassRateReport {
348 policyVersion: number;
349 cases: CasePassRate[];
350 alarms: Alarm[];
351 /** Trials under this reader's EVAL_POLICY.version: the only ones scored. */
352 postPolicyTrials: number;
353 /** Backfilled pre-policy trials (policy_version 0): display only. */
354 prePolicyTrials: number;
355 /** Trials under an earlier policy version: display only, never scored. */
356 olderPolicyTrials: number;
357 /** Trials under a later policy version than this reader: ignored entirely. */
358 newerPolicyTrials: number;
359 unattributed: string[];
360 errors: string[];
361}
363export interface AnalyzeOptions {
364 registry?: Registry;
365 quarantine?: Record<string, QuarantineEntry>;
366 policy?: PassRatePolicy;
367 /** Completed weekly-run timestamps in the window, for quarantine expiry. */
368 weeklyRuns?: string[];
369 /**
370 * Run ids of branch census runs (isPooledTrialRun). Their trials count only
371 * toward a series that a non-pooled record also has, so a branch can extend
372 * main's history but never start, or become, a case's current series.
373 */
374 pooledRunIds?: ReadonlySet<string>;
375 now?: number;
376 unattributed?: string[];
377 errors?: string[];
378 manualReviews?: string[];
379}
381const at = (record: TrialRecord) => record.recorded_at ?? '';
382const runOf = (record: TrialRecord) => `${record.run_id ?? record.sha ?? 'local'}#${record.attempt}`;
384function seriesStats(key: string, records: TrialRecord[]): SeriesStats {
385 const scored = records.filter(record => record.outcome !== 'skipped');
386 const passes = scored.filter(record => record.outcome === 'passed').length;
387 const times = records.map(at).sort();
388 const first = records[0]!;
389 return {
390 key, identity: first.series_identity ?? 'unknown', model: first.model ?? 'unknown', cli: first.cli_version ?? 'unknown',
391 policyVersion: first.policy_version, passes, trials: scored.length,
392 infra: scored.filter(record => record.outcome === 'failed' && record.failure_class === 'infra').length,
393 interval: wilsonInterval(passes, scored.length), firstSeen: times[0] ?? '', lastSeen: times[times.length - 1] ?? '',
394 runs: [...new Set(records.map(runOf))],
395 };
396}
398/** Weekly runs completed after an entry's enteredAt; offline, whole weeks elapsed. */
399export function quarantineRunsSince(enteredAt: string, weeklyRuns: string[] | undefined, now: number): number {
400 const entered = Date.parse(enteredAt);
401 if (!Number.isFinite(entered)) return Number.POSITIVE_INFINITY;
402 if (weeklyRuns && weeklyRuns.length) return weeklyRuns.filter(time => Date.parse(time) > entered).length;
403 return Math.floor((now - entered) / (7 * 86_400_000));
404}
406/**
407 * Static CASE_QUARANTINE problems, shared by the free policy test and the
408 * weekly gate: an id that is not a blocking-tier E2E case, a missing field,
409 * a failure class outside detector / harness / model-latency (a product
410 * defect is fixed or named, never quarantined), a malformed or future date,
411 * and a tier over its cap.
412 */
413export function quarantinePolicyProblems(quarantine: Record<string, QuarantineEntry>,
414 registry: Registry = LIVE_REGISTRY, policy: PassRatePolicy = EVAL_POLICY, now = Date.now()): Alarm[] {
415 const problems: Alarm[] = [];
416 const invalid = (id: string, message: string) => problems.push({ kind: 'quarantine-invalid', case: id, message: `${id}: ${message}` });
417 const perTier = new Map<string, number>();
418 for (const [id, entry] of Object.entries(quarantine)) {
419 const tier = registry.tiers[id];
420 if (!tier || !(id in registry.kinds)) { invalid(id, 'CASE_QUARANTINE names no registered E2E case'); continue; }
421 if (!BLOCKING_TIERS.includes(tier)) invalid(id, `tier ${tier} is not blocking; only ${BLOCKING_TIERS.join(' and ')} cases are quarantined`);
422 for (const field of ['reason', 'failureClass', 'tracking', 'owner', 'enteredAt', 'exit'] as const) {
423 if (typeof entry[field] !== 'string' || !entry[field].trim()) invalid(id, `missing ${field}`);
424 }
425 if (typeof entry.reason === 'string' && entry.reason.trim().length < 40) invalid(id, 'reason must be a written diagnosis (at least 40 characters)');
426 if (!QUARANTINE_FAILURE_CLASSES.includes(entry.failureClass)) {
427 invalid(id, `failureClass ${JSON.stringify(entry.failureClass)} is not ${QUARANTINE_FAILURE_CLASSES.join(', ')}; a product defect is fixed or named as a red, never quarantined`);
428 }
429 const entered = Date.parse(entry.enteredAt);
430 if (!/^\d{4}-\d{2}-\d{2}$/.test(entry.enteredAt ?? '') || !Number.isFinite(entered)) invalid(id, 'enteredAt must be YYYY-MM-DD');
431 else if (entered > now) invalid(id, 'enteredAt is in the future');
432 perTier.set(tier, (perTier.get(tier) ?? 0) + 1);
433 }
434 for (const [tier, count] of perTier) {
435 const size = Object.values(registry.tiers).filter(value => value === tier).length;
436 const cap = Math.floor(size * policy.quarantine.capFraction);
437 if (count > cap) problems.push({ kind: 'quarantine-cap', case: tier,
438 message: `${count} quarantined ${tier} cases exceed the ${pct(policy.quarantine.capFraction)} cap (${cap} of ${size})` });
439 }
440 return problems;
441}
443export function analyzePassRates(records: TrialRecord[], options: AnalyzeOptions = {}): PassRateReport {
444 const registry = options.registry ?? LIVE_REGISTRY;
445 const quarantine = options.quarantine ?? CASE_QUARANTINE;
446 const policy = options.policy ?? EVAL_POLICY;
447 const now = options.now ?? Date.now();
448 const newerPolicyTrials = records.filter(record => record.policy_version > policy.version).length;
449 const byCase = new Map<string, TrialRecord[]>();
450 for (const record of records) {
451 if (record.policy_version > policy.version) continue;
452 const list = byCase.get(record.case) ?? [];
453 list.push(record);
454 byCase.set(record.case, list);
455 }
456 for (const id of options.manualReviews ?? []) if (!byCase.has(id)) byCase.set(id, []);
457 const cases: CasePassRate[] = [];
458 for (const [id, list] of [...byCase].sort(([a], [b]) => a.localeCompare(b))) {
459 list.sort((a, b) => at(a).localeCompare(at(b)) || runOf(a).localeCompare(runOf(b)) || a.trial - b.trial);
460 const groups = new Map<string, TrialRecord[]>();
461 const keyOf = (record: TrialRecord) => record.policy_version === 0 ? 'pre-policy'
462 : [record.series_identity ?? 'unknown', record.model ?? 'unknown', record.cli_version ?? 'unknown', `v${record.policy_version}`].join('|');
463 const pooled = (record: TrialRecord) => !!options.pooledRunIds?.has(record.run_id ?? '');
464 const mainKeys = new Set(list.filter(record => !pooled(record)).map(keyOf));
465 for (const record of list) {
466 const key = keyOf(record);
467 if (pooled(record) && !mainKeys.has(key)) continue;
468 const group = groups.get(key) ?? [];
469 group.push(record);
470 groups.set(key, group);
471 }
472 const series = [...groups].map(([key, group]) => seriesStats(key, group))
473 .sort((a, b) => a.lastSeen.localeCompare(b.lastSeen));
474 const post = series.filter(entry => entry.policyVersion === policy.version);
475 const current = post[post.length - 1] ?? null;
476 const previous = post[post.length - 2] ?? null;
477 const scored = current ? groups.get(current.key)!.filter(record => record.outcome !== 'skipped') : [];
478 const latestRun = scored.length ? runOf(scored[scored.length - 1]!) : null;
479 const latest = scored.filter(record => runOf(record) === latestRun);
480 const prior = scored.filter(record => runOf(record) !== latestRun);
481 const priorPasses = prior.filter(record => record.outcome === 'passed').length;
482 const entryRate = policy.quarantine.entry.rate;
483 let label: PassRateLabel;
484 if (latest.length > 0 && latest.every(record => record.outcome === 'failed')
485 && prior.length > 0 && wilsonInterval(priorPasses, prior.length).lo >= entryRate) label = 'BROKEN';
486 else if (!current || current.trials < policy.quarantine.entry.minTrials) label = 'INCONCLUSIVE';
487 else if (current.interval.hi < entryRate) label = 'FAILING';
488 else if (current.passes < current.trials && current.interval.lo < entryRate) label = 'FLAKY';
489 else label = 'PASSING';
490 cases.push({
491 case: id, kind: registry.kinds[id] ?? list[0]!.kind, tier: caseTier(id, registry),
492 quarantined: id in quarantine, label, current, previous,
493 manualReviews: (options.manualReviews ?? []).filter(name => name === id).length,
494 prePolicy: series.find(entry => entry.policyVersion === 0) ?? null, series,
495 latestRun: latestRun ? { runId: latestRun, passes: latest.filter(record => record.outcome === 'passed').length, trials: latest.length } : null,
496 });
497 }
499 const alarms: Alarm[] = [];
500 const rate = (stats: SeriesStats) => stats.passes / stats.trials;
501 for (const entry of cases) {
502 const current = entry.current;
503 if (!current) continue;
504 const below = current.trials >= policy.quarantine.entry.minTrials && rate(current) < policy.quarantine.entry.rate;
505 if (below && !entry.quarantined && BLOCKING_TIERS.includes(entry.tier)) alarms.push({ kind: 'drift', case: entry.case,
506 message: `${entry.case} passes ${current.passes}/${current.trials} (below ${pct(policy.quarantine.entry.rate)} over >= ${policy.quarantine.entry.minTrials} trials): fix it, or propose a CASE_QUARANTINE entry with a written diagnosis (product defects are never quarantined)` });
507 if (below && entry.kind === 'rule') alarms.push({ kind: 'rule-as-behavior', case: entry.case,
508 message: `${entry.case}: rule case behaving like behavior (${current.passes}/${current.trials}): fix or reclassify` });
509 if (entry.quarantined && current.trials >= policy.quarantine.exit.minTrials && rate(current) >= policy.quarantine.exit.rate) {
510 alarms.push({ kind: 'quarantine-exit', case: entry.case,
511 message: `${entry.case} passes ${current.passes}/${current.trials} (>= ${pct(policy.quarantine.exit.rate)}): remove its CASE_QUARANTINE entry` });
512 }
513 }
514 const tested = cases.filter(entry => BLOCKING_TIERS.includes(entry.tier) && entry.current && entry.previous
515 && entry.current.trials >= policy.drift.fisherMinPerSide && entry.previous.trials >= policy.drift.fisherMinPerSide);
516 const pValues = tested.map(entry => fisherOneSidedLower(entry.current!.passes, entry.current!.trials, entry.previous!.passes, entry.previous!.trials));
517 for (const index of holmRejections(pValues, policy.drift.fisherAlpha)) {
518 const entry = tested[index]!;
519 alarms.push({ kind: 'regression', case: entry.case,
520 message: `${entry.case}: current identity ${entry.current!.passes}/${entry.current!.trials} is significantly below the previous ${entry.previous!.passes}/${entry.previous!.trials} (one-sided Fisher p=${pValues[index]!.toFixed(4)}, Holm over ${tested.length} cases)` });
521 }
522 for (const [id, entry] of Object.entries(quarantine)) {
523 const runs = quarantineRunsSince(entry.enteredAt, options.weeklyRuns, now);
524 if (runs >= policy.quarantine.expiryWeeklyRuns) alarms.push({ kind: 'quarantine-expired', case: id,
525 message: `${id}: entered ${runs} weekly runs ago (limit ${policy.quarantine.expiryWeeklyRuns}): fix it, name it as a red, or re-diagnose with fresh evidence` });
526 }
527 alarms.push(...quarantinePolicyProblems(quarantine, registry, policy, now));
529 const count = (match: (version: number) => boolean) => records.filter(record => match(record.policy_version)).length;
530 return { policyVersion: policy.version, cases, alarms, postPolicyTrials: count(version => version === policy.version),
531 prePolicyTrials: count(version => version === 0), olderPolicyTrials: count(version => version > 0 && version < policy.version),
532 newerPolicyTrials, unattributed: options.unattributed ?? [], errors: options.errors ?? [] };
533}
535function pct(value: number): string { return `${Math.round(value * 1000) / 10}%`; }
537function formatStats(stats: SeriesStats | null): string {
538 if (!stats) return '-';
539 return `${stats.passes}/${stats.trials} [${pct(stats.interval.lo)}–${pct(stats.interval.hi)}]${stats.infra ? ` (${stats.infra} infra)` : ''}`;
540}
542export function formatPassRates(report: PassRateReport, options: { caseFilter?: string } = {}): string {
543 const lines: string[] = [];
544 lines.push(`pass-rates: policy v${report.policyVersion}, ${report.postPolicyTrials} post-policy trial(s), ${report.prePolicyTrials} pre-policy (display only)`
545 + (report.olderPolicyTrials ? `, ${report.olderPolicyTrials} under an older policy (display only)` : ''));
546 if (report.newerPolicyTrials) {
547 lines.push(` ignored ${report.newerPolicyTrials} trial(s) recorded under a policy newer than v${report.policyVersion}: this checkout predates them. `
548 + 'Fix: run pass-rates from a checkout at or after the commit that bumped EVAL_POLICY.version (docs/TESTING_INTERNALS.md#pass-rate-policy-versions).');
549 }
550 if (report.postPolicyTrials === 0) lines.push(' no post-policy trials yet: every series starts INCONCLUSIVE');
551 const cases = report.cases.filter(entry => !options.caseFilter || entry.case === options.caseFilter);
552 lines.push(' label kind tier current series pre-policy manual case');
553 for (const entry of cases) {
554 const group = entry.current ? ` ${entry.current.model} / ${entry.current.cli}` : '';
555 const reset = entry.previous ? ' (baseline reset)' : '';
556 lines.push(` ${entry.label.padEnd(12)} ${entry.kind.padEnd(8)} ${entry.tier.padEnd(8)} ${formatStats(entry.current).padEnd(29)} `
557 + `${formatStats(entry.prePolicy).padEnd(18)} ${String(entry.manualReviews).padStart(6)} ${entry.case}${entry.quarantined ? ' [quarantined]' : ''}${group}${reset}`);
558 }
559 if (report.unattributed.length) lines.push(` unattributed records (${report.unattributed.length}, never guessed): ${report.unattributed.slice(0, 20).join(', ')}`);
560 if (report.errors.length) lines.push(` rejected ${report.errors.length} invalid trial line(s): ${report.errors.slice(0, 5).join('; ')}`);
561 if (report.alarms.length) {
562 lines.push(`ACTION REQUIRED (${report.alarms.length}):`);
563 for (const alarm of report.alarms) lines.push(` [${alarm.kind}] ${alarm.message}`);
564 }
565 return lines.join('\n');
566}
568export interface PassRateHistory {
569 records: TrialRecord[];
570 unattributed: string[];
571 errors: string[];
572 manualReviews: string[];
573 weeklyRuns?: string[];
574 fetched: WeeklyRun[];
575 pooledRunIds?: Set<string>;
576 historyError: string | null;
577 scope: string;
578 cacheDir: string;
579}
581/**
582 * The trial history every eval:pass-rates view reads (and the ship-measure
583 * sweep ranks from): local dirs, or the trial-outcomes artifacts of the last
584 * N weekly runs on the branch plus the branch census runs that pool into
585 * main's series. A fetch failure is returned as historyError, never thrown.
586 */
587export function loadPassRateHistory(o: { repo: string; workflow: string; branch: string; runsLimit: number; dirs: string[]; sinceDays: number; backfill?: boolean; runId?: number }): PassRateHistory {
588 const records: TrialRecord[] = [];
589 const unattributed = new Set<string>();
590 const errors: string[] = [];
591 const manualReviews: string[] = [];
592 let historyError: string | null = null;
593 let weeklyRuns: string[] | undefined;
594 let fetched: WeeklyRun[] = [];
595 let pooledRunIds: Set<string> | undefined;
596 const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', o.repo.replace('/', '-'));
597 const importDir = (dir: string, run: { run_id: string; sha?: string; timestamp?: string } | undefined, legacyDays: number) => {
598 const trials = readTrialOutcomeDir(dir);
599 records.push(...trials.records);
600 errors.push(...trials.errors);
601 const legacy = backfillEvalFiles(collectEvalFiles(dir, legacyDays), run);
602 records.push(...legacy.records);
603 manualReviews.push(...legacy.manualReviews);
604 legacy.unattributed.forEach(name => unattributed.add(name));
605 };
606 if (o.dirs.length) {
607 for (const dir of o.dirs) importDir(dir, undefined, o.sinceDays);
608 } else {
609 try {
610 const weekly = listWeeklyRuns({ repo: o.repo, workflow: o.workflow, branches: [o.branch], limit: o.runsLimit }).filter(run => o.branch !== 'main' || isWeeklyHistoryRun(run));
611 weeklyRuns = weekly.map(run => run.createdAt);
612 // Branch census trials pool into main's matching series over the same window (isPooledTrialRun).
613 const oldest = weekly[weekly.length - 1]?.createdAt;
614 const pooledRuns = o.branch === 'main' && oldest
615 ? listWeeklyRuns({ repo: o.repo, workflow: o.workflow, branches: [''], limit: 100 }).filter(run => isPooledTrialRun(run) && run.createdAt >= oldest)
616 : [];
617 pooledRunIds = new Set(pooledRuns.map(run => `${run.id}`));
618 const runs = [...weekly, ...pooledRuns];
619 if (o.runId !== undefined && !runs.some(run => run.id === o.runId)) runs.push(GH_JOBS.getRun(o.repo, o.runId));
620 fetched = runs;
621 const match = o.backfill
622 ? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name)
623 : (name: string) => name.startsWith('trial-outcomes');
624 for (const run of runs) {
625 const dirsForRun = downloadRunArtifacts({ repo: o.repo, run, match, cacheDir, maxBytes: o.backfill ? 64 * 1024 * 1024 : undefined });
626 for (const dir of dirsForRun) importDir(dir, { run_id: `${run.id}`, sha: run.sha, timestamp: run.createdAt }, 3650);
627 }
628 } catch (error) {
629 historyError = error instanceof Error ? error.message : String(error);
630 }
631 }
632 const scope = o.dirs.length ? `local dirs ${o.dirs.join(', ')}` : `${o.repo} ${o.workflow} on ${o.branch}${pooledRunIds?.size ? ` + ${pooledRunIds.size} pooled branch census run(s)` : ''}, `
633 + `last ${o.runsLimit} completed run(s): ${fetched.map(run => run.id).join(', ') || 'none'}`;
634 return { records, unattributed: [...unattributed].sort(), errors, manualReviews, weeklyRuns, fetched, pooledRunIds, historyError, scope, cacheDir };
635}
637// --- GitHub history (shared reader: scripts/lib/ci-history.ts) ---
639export { downloadRunArtifacts, GH_HISTORY, listWeeklyRuns, TRIAL_OUTCOMES_MAX_BYTES,
640 type HistoryFetcher, type RunArtifact, type WeeklyRun } from './lib/ci-history';
642if (import.meta.main) {
643 const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)
644 || (/^test\/[\w./-]+\.test\.ts$/.test(id) && fs.existsSync(id)));
645 if ('help' in parsed) { console.log(PASS_RATES_USAGE); process.exit(0); }
646 if ('error' in parsed) { console.error(`eval:pass-rates: ${parsed.error}\n${PASS_RATES_USAGE}`); process.exit(2); }
647 const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed;
648 const repo = parsed.repo ?? repoSlug();
649 const workflow = parsed.workflow ?? 'evals-periodic.yml';
650 const branch = parsed.branch ?? 'main';
652 const history = loadPassRateHistory({ repo, workflow, branch, runsLimit, dirs, sinceDays, backfill, runId: parsed.runId });
653 const { records, errors, manualReviews, weeklyRuns, fetched, pooledRunIds, historyError, scope, cacheDir } = history;
654 const unattributed = new Set(history.unattributed);
655 const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records;
656 const caseHeadroom = headroom(scoped);
657 if (view !== 'rates') {
658 const critical: CriticalPath[] = [];
659 if (view !== 'reds' && !dirs.length) {
660 for (const run of fetched.filter(run => view === 'headroom' || run.id === parsed.runId)) {
661 try { critical.push(criticalPath(String(run.id), GH_JOBS.listJobs(repo, run.id))); } catch { critical.push({ run: String(run.id), jobs: 0, slowest: null, wallMs: null }); }
662 }
663 }
664 const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped))
665 : historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY,
666 download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) })
667 .concat(formatCriticalPath(critical), historyError ? [] : [`guide: ${CENSUS_RED_GUIDE}`]);
668 if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2));
669 else {
670 console.log(`scope: ${scope}`);
671 if (historyError) console.log(`history unavailable (${historyError})`);
672 for (const line of lines) console.log(line);
673 }
674 process.exit(historyError ? 1 : 0);
675 }
676 const full = analyzePassRates(records, { weeklyRuns, pooledRunIds, unattributed: [...unattributed].sort(), errors, manualReviews });
677 const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter);
678 const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms };
679 const ledger = readFreeLedger();
680 if (asJson) {
681 console.log(JSON.stringify({ repo, workflow, branch, scope, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
682 } else {
683 console.log(`scope: ${scope}`);
684 if (historyError) console.log(`pass-rates: history unavailable (${historyError}); every label below is INCONCLUSIVE`);
685 console.log(formatPassRates(report, { caseFilter }));
686 if (ledger.length > 0) {
687 const byFile = new Map<string, number>();
688 for (const e of ledger) byFile.set(e.file, (byFile.get(e.file) ?? 0) + 1);
689 console.log(`free-suite flaky-passes (${flakeLedgerPath()}):`);
690 for (const [file, n] of [...byFile.entries()].sort((a, b) => b[1] - a[1])) {
691 console.log(` ${String(n).padStart(3)}x ${file}`);
692 }
693 }
694 }
695 if (gate && (historyError || report.alarms.length)) process.exit(1);
696}