free-ci-health.ts
164 lines · scripts/lib/free-ci-health.ts at 3f79d81
1/**
2 * Free-lane health signals that keep the duration seed honest: the
3 * unseeded-file warning (--ci-plan), the shard overrun warning (--ci-verify
4 * job summary), and the
5 * seed growth ratchet (60 s ceiling with a shrink-only allowlist).
6 * Every message names the problem, the offending entry, the fix command and
7 * the docs anchor.
8 */
9import { spawnSync } from 'node:child_process';
10import * as fs from 'node:fs';
11import type { FreeCiPlan, FreeCiResult } from '../test-free-shards';
12import type { CurationResult } from './windows-curation';
14export const UNSEEDED_WARN_LIMIT = 5;
15export const SHARD_OVERRUN_FACTOR = 1.5;
16/** Overruns smaller than this are runner noise (startup, cache restore), not a stale seed. */
17export const SHARD_OVERRUN_FLOOR_MS = 30_000;
18export const SEED_CEILING_MS = 60_000;
19export const SEED_ALLOWLIST_FILE = 'scripts/free-test-seed-allowlist.json';
20export const SEED_REFRESH_COMMAND = 'bun run test:ubicloud --record-durations';
21export const SEED_DOCS_ANCHOR = 'docs/TESTING_INTERNALS.md#free-suite-duration-seed';
23const seconds = (ms: number) => `${Math.round(ms / 1000)}s`;
25export function unseededWarning(unseeded: string[]): string | null {
26 if (unseeded.length <= UNSEEDED_WARN_LIMIT) return null;
27 const shown = unseeded.slice(0, 10).join(', ') + (unseeded.length > 10 ? `, +${unseeded.length - 10} more` : '');
28 return `WARNING: ${unseeded.length} free test files have no recorded duration (limit ${UNSEEDED_WARN_LIMIT}); `
29 + `they are packed at the 99th-percentile estimate, so shard balance degrades until the seed is refreshed: ${shown}. `
30 + `Fix: run \`${SEED_REFRESH_COMMAND}\` and commit scripts/free-test-durations.json. See ${SEED_DOCS_ANCHOR}.`;
31}
33export interface ShardTiming { shard: number; predictedMs: number; elapsedMs: number }
35export function shardOverruns(timings: ShardTiming[]): ShardTiming[] {
36 return timings.filter(({ predictedMs, elapsedMs }) =>
37 elapsedMs > predictedMs * SHARD_OVERRUN_FACTOR && elapsedMs - predictedMs >= SHARD_OVERRUN_FLOOR_MS);
38}
40export function overrunWarning(overruns: ShardTiming[]): string | null {
41 if (overruns.length === 0) return null;
42 const listed = overruns.map(({ shard, predictedMs, elapsedMs }) =>
43 `shard ${shard} took ${seconds(elapsedMs)} vs ${seconds(predictedMs)} predicted`).join('; ');
44 return `WARNING: ${overruns.length} shard(s) exceeded ${SHARD_OVERRUN_FACTOR}x their predicted duration (${listed}). `
45 + `A file in them is slower than its seed entry or unseeded. Fix: run \`${SEED_REFRESH_COMMAND}\` `
46 + `and commit scripts/free-test-durations.json; split any file over ${seconds(SEED_CEILING_MS)}. See ${SEED_DOCS_ANCHOR}.`;
47}
49/** Job-summary signal derived from the strict results: shards that outran their seed. */
50export function ciHealthSummary(plan: FreeCiPlan, results: FreeCiResult[]): string[] {
51 const ordered = [...results].sort((a, b) => a.outcome.shard - b.outcome.shard);
52 const overruns = shardOverruns(ordered.flatMap(({ outcome }) => {
53 const predictedMs = plan.shards[outcome.shard - 1]?.predictedMs;
54 return predictedMs === undefined ? [] : [{ shard: outcome.shard, predictedMs, elapsedMs: outcome.elapsedMs }];
55 }));
56 const warning = overrunWarning(overruns);
57 return warning ? [warning] : [];
58}
60/** Printed by every Windows run and its job summary so silent shrinkage of the curated set is visible. */
61export function windowsCurationLine({ safe, excluded }: CurationResult): string {
62 const byReason = new Map<string, number>();
63 for (const { reason } of excluded) byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
64 const top = [...byReason].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 4)
65 .map(([reason, count]) => `${count} ${reason}`).join('; ');
66 return `curated ${safe.length} Windows-safe tests (${excluded.length} excluded${top ? `: ${top}` : ''})`;
67}
69/** Append markdown to the GitHub job summary when running in Actions; no-op locally. */
70export function appendStepSummary(markdown: string, env: NodeJS.ProcessEnv = process.env): void {
71 if (!env.GITHUB_STEP_SUMMARY) return;
72 fs.appendFileSync(env.GITHUB_STEP_SUMMARY, `${markdown}\n\n`);
73}
75// ─── Seed growth ratchet ────────────────────────────────────────────────────
77export interface SeedAllowlistEntry {
78 file: string;
79 reason: string;
80 /** Required on an entry the merge-base allowlist lacks: the annotated exception names its follow-up. */
81 todo?: string;
82}
84export function parseSeedAllowlist(raw: string, source = SEED_ALLOWLIST_FILE): SeedAllowlistEntry[] {
85 const parsed = JSON.parse(raw) as { entries?: unknown };
86 if (!Array.isArray(parsed.entries)) throw new Error(`${source}: expected {"entries": [...]}`);
87 return parsed.entries.map((entry, index) => {
88 const { file, reason, todo } = (entry ?? {}) as Record<string, unknown>;
89 if (typeof file !== 'string' || !file || typeof reason !== 'string' || !reason.trim()) {
90 throw new Error(`${source}: entries[${index}] needs a non-empty "file" and "reason"`);
91 }
92 if (todo !== undefined && (typeof todo !== 'string' || !todo.trim())) {
93 throw new Error(`${source}: entries[${index}] "todo" must be a non-empty string when present`);
94 }
95 return todo === undefined ? { file, reason } : { file, reason, todo };
96 });
97}
99export interface SeedRatchetInput {
100 durations: Record<string, number>;
101 allowlist: SeedAllowlistEntry[];
102 /** The merge-base allowlist; null when the merge-base has no allowlist file (the first allowlist). */
103 baseAllowlist: SeedAllowlistEntry[] | null;
104}
106export interface SeedRatchetResult {
107 violations: string[];
108 /** Annotated exceptions added on this branch; printed so reviewers see them. */
109 exceptions: string[];
110}
112export function seedRatchet({ durations, allowlist, baseAllowlist }: SeedRatchetInput): SeedRatchetResult {
113 const violations: string[] = [];
114 const exceptions: string[] = [];
115 const allowed = new Map(allowlist.map(entry => [entry.file, entry]));
116 const fix = `See ${SEED_DOCS_ANCHOR}.`;
118 for (const [file, ms] of Object.entries(durations).sort(([a], [b]) => a.localeCompare(b))) {
119 if (ms > SEED_CEILING_MS && !allowed.has(file)) {
120 violations.push(`${file} is seeded at ${seconds(ms)}, over the ${seconds(SEED_CEILING_MS)} ceiling. `
121 + `Fix: split it (by describe block or fixture) so the packer can balance it, then rerun \`${SEED_REFRESH_COMMAND}\`. ${fix}`);
122 }
123 }
124 for (const entry of allowlist) {
125 const ms = durations[entry.file];
126 if (ms === undefined || ms <= SEED_CEILING_MS) {
127 violations.push(`${SEED_ALLOWLIST_FILE} lists ${entry.file}, which is ${ms === undefined ? 'not in the seed' : `seeded at ${seconds(ms)}`}. `
128 + `Fix: delete its entry; the allowlist holds only files still over ${seconds(SEED_CEILING_MS)}. ${fix}`);
129 }
130 }
131 if (baseAllowlist === null) return { violations, exceptions };
132 const baseFiles = new Set(baseAllowlist.map(entry => entry.file));
133 for (const entry of allowlist) {
134 if (baseFiles.has(entry.file)) continue;
135 if (entry.todo) {
136 exceptions.push(`annotated exception: ${entry.file} (reason: ${entry.reason}; todo: ${entry.todo})`);
137 } else {
138 violations.push(`${SEED_ALLOWLIST_FILE} adds ${entry.file}, but the allowlist is shrink-only. `
139 + `Fix: split the file under ${seconds(SEED_CEILING_MS)}, or add a "todo" naming the follow-up that splits it (an annotated exception). ${fix}`);
140 }
141 }
142 return { violations, exceptions };
143}
145export type BaseAllowlistRead =
146 | { status: 'ok'; entries: SeedAllowlistEntry[] | null }
147 | { status: 'unavailable'; reason: string };
149/**
150 * The merge-base copy of the allowlist (`git show <base>:<file>`). A base that
151 * predates the allowlist file yields null (the first allowlist); a base git
152 * cannot resolve (shallow checkout, missing ref) is unavailable, never "empty".
153 */
154export function readBaseSeedAllowlist(root: string, baseRef: string): BaseAllowlistRead {
155 const commit = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${baseRef}^{commit}`], { cwd: root, encoding: 'utf8', timeout: 10_000 });
156 if (commit.status !== 0) return { status: 'unavailable', reason: `cannot resolve merge-base ${baseRef}` };
157 const sha = commit.stdout.trim();
158 const listed = spawnSync('git', ['ls-tree', '--name-only', sha, '--', SEED_ALLOWLIST_FILE], { cwd: root, encoding: 'utf8', timeout: 10_000 });
159 if (listed.status !== 0) return { status: 'unavailable', reason: `cannot list ${SEED_ALLOWLIST_FILE} at ${sha}` };
160 if (!listed.stdout.trim()) return { status: 'ok', entries: null };
161 const shown = spawnSync('git', ['show', `${sha}:${SEED_ALLOWLIST_FILE}`], { cwd: root, encoding: 'utf8', timeout: 10_000 });
162 if (shown.status !== 0) return { status: 'unavailable', reason: `cannot read ${SEED_ALLOWLIST_FILE} at ${sha}` };
163 return { status: 'ok', entries: parseSeedAllowlist(shown.stdout, `${sha}:${SEED_ALLOWLIST_FILE}`) };
164}