typecheck-test.ts
173 lines · scripts/typecheck-test.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Test-code type-debt ratchet: `bun run typecheck:test [--write-baseline]`.
4 *
5 * Product code must typecheck clean (`bun run typecheck`). Test code carries
6 * historical diagnostics, so it is held to a committed baseline instead:
7 * each diagnostic identity (file + TS code + message, line-insensitive) maps
8 * to how many times it occurs. The check fails on a new identity, on a higher
9 * count, and on a stale baseline (fixed diagnostics must be locked in with
10 * --write-baseline in the same change, so the allowance only ever shrinks).
11 * Fixing one error and adding an identical-message error in the same file is
12 * the one substitution this cannot see.
13 */
14import { spawnSync } from 'node:child_process';
15import * as fs from 'node:fs';
16import * as path from 'node:path';
18export const BASELINE_FILE = 'scripts/typecheck-test-baseline.json';
19const ROOT = path.resolve(import.meta.dir, '..');
21export type DiagnosticCounts = Record<string, number>;
23/**
24 * tsc prints a synthesized object type's members in checker order, which
25 * shifts when the program's file set changes (`{ content?: undefined; isError?: undefined; … }`
26 * vs `{ isError?: undefined; content?: undefined; … }` for one unchanged
27 * diagnostic). Sort the members of every `{ …; }` group so that order is not
28 * part of the identity.
29 */
30export function sortTypeMembers(text: string): string {
31 const group = (start: number): [string, number] => {
32 const members: string[] = [];
33 let current = '';
34 let i = start;
35 for (; i < text.length && text[i] !== '}'; i++) {
36 if (text[i] === '{') {
37 const [inner, end] = group(i + 1);
38 current += inner;
39 i = end;
40 } else if (text[i] === ';') {
41 members.push(current.trim());
42 current = '';
43 } else current += text[i];
44 }
45 if (i >= text.length) return [`{${members.map(m => ` ${m};`).join('')}${current}`, i];
46 if (members.length === 0) return [`{${current}}`, i];
47 return [`{ ${members.sort().map(m => `${m}; `).join('')}${current.trim() ? `${current.trim()} ` : ''}}`, i];
48 };
49 let out = '';
50 for (let i = 0; i < text.length; i++) {
51 if (text[i] !== '{') { out += text[i]; continue; }
52 const [inner, end] = group(i + 1);
53 out += inner;
54 i = end;
55 }
56 return out;
57}
59/** Parse `tsc --pretty false` output into identity → count. Continuation lines belong to the preceding diagnostic. */
60export function parseDiagnostics(output: string, root = ROOT): DiagnosticCounts {
61 const counts: DiagnosticCounts = {};
62 // Messages can embed absolute import paths; strip the checkout root so the
63 // identity is the same in every clone and CI workspace.
64 const roots = [root, root.replaceAll('\\', '/')].filter(Boolean);
65 const portable = (text: string) => roots.reduce((value, prefix) => value.split(prefix + '/').join('').split(prefix).join('.'), text);
66 let current: string | null = null;
67 const flush = () => {
68 if (current !== null) {
69 const identity = sortTypeMembers(current);
70 counts[identity] = (counts[identity] ?? 0) + 1;
71 }
72 current = null;
73 };
74 for (const line of output.split(/\r?\n/)) {
75 const match = /^(.+?)\(\d+,\d+\): error (TS\d+): (.*)$/.exec(line);
76 if (match) {
77 flush();
78 current = `${portable(match[1]!).replaceAll('\\', '/')}\t${match[2]}\t${portable(match[3]!.trim())}`;
79 } else if (current !== null && /^\s+\S/.test(line)) {
80 current += ` ${portable(line.trim())}`;
81 } else {
82 flush();
83 }
84 }
85 flush();
86 return Object.fromEntries(Object.entries(counts).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
87}
89export interface RatchetResult {
90 added: Array<{ identity: string; baseline: number; current: number }>;
91 fixed: Array<{ identity: string; baseline: number; current: number }>;
92}
94export function compareDiagnostics(baseline: DiagnosticCounts, current: DiagnosticCounts): RatchetResult {
95 const added: RatchetResult['added'] = [], fixed: RatchetResult['fixed'] = [];
96 for (const identity of new Set([...Object.keys(baseline), ...Object.keys(current)])) {
97 const before = baseline[identity] ?? 0, now = current[identity] ?? 0;
98 if (now > before) added.push({ identity, baseline: before, current: now });
99 else if (now < before) fixed.push({ identity, baseline: before, current: now });
100 }
101 return { added, fixed };
102}
104export function readBaseline(file: string): DiagnosticCounts {
105 let parsed: unknown;
106 try {
107 parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
108 } catch (error) {
109 throw new Error(`test typecheck baseline ${file} is missing or unreadable (${(error as Error).message}). ` +
110 'Regenerate it from a clean tree with: bun run typecheck:test --write-baseline');
111 }
112 const diagnostics = (parsed as { version?: unknown; diagnostics?: unknown })?.diagnostics;
113 if ((parsed as { version?: unknown })?.version !== 1 || !diagnostics || typeof diagnostics !== 'object' ||
114 !Object.values(diagnostics).every(n => Number.isSafeInteger(n) && (n as number) > 0)) {
115 throw new Error(`test typecheck baseline ${file} is malformed. Regenerate it with: bun run typecheck:test --write-baseline`);
116 }
117 const counts: DiagnosticCounts = {};
118 for (const [identity, n] of Object.entries(diagnostics as DiagnosticCounts)) {
119 const key = sortTypeMembers(identity);
120 counts[key] = (counts[key] ?? 0) + n;
121 }
122 return counts;
123}
125function describe(identity: string): string {
126 const [file, code, message] = identity.split('\t');
127 return `${file} ${code}: ${message}`;
128}
130function main(): number {
131 const write = process.argv.includes('--write-baseline');
132 const tsc = spawnSync(process.execPath, ['x', 'tsc', '-p', 'tsconfig.test.json', '--pretty', 'false'], {
133 cwd: ROOT, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024,
134 });
135 if (tsc.error || tsc.signal) {
136 console.error(`test typecheck could not run tsc: ${tsc.error?.message ?? tsc.signal}`);
137 return 2;
138 }
139 const current = parseDiagnostics(`${tsc.stdout}\n${tsc.stderr}`);
140 const total = Object.values(current).reduce((sum, n) => sum + n, 0);
141 if (tsc.status !== 0 && total === 0) {
142 console.error(`tsc exited ${tsc.status} without parseable diagnostics:\n${tsc.stdout}${tsc.stderr}`);
143 return 2;
144 }
145 const baselinePath = path.join(ROOT, BASELINE_FILE);
146 if (write) {
147 fs.writeFileSync(baselinePath, JSON.stringify({ version: 1, diagnostics: current }, null, 2) + '\n');
148 console.log(`test typecheck ratchet: wrote ${BASELINE_FILE} (${total} diagnostics, ${Object.keys(current).length} identities)`);
149 return 0;
150 }
151 let baseline: DiagnosticCounts;
152 try {
153 baseline = readBaseline(baselinePath);
154 } catch (error) {
155 console.error((error as Error).message);
156 return 1;
157 }
158 const { added, fixed } = compareDiagnostics(baseline, current);
159 if (added.length) {
160 console.error(`test typecheck ratchet: ${added.length} new or more frequent diagnostic(s). Fix them; do not add them to the baseline.`);
161 for (const d of added) console.error(` + ${describe(d.identity)} (${d.baseline} → ${d.current})`);
162 console.error('Reproduce with: bunx tsc -p tsconfig.test.json --pretty false');
163 }
164 if (fixed.length) {
165 console.error(`test typecheck ratchet: ${fixed.length} diagnostic(s) fixed. Lock the smaller allowance in with: bun run typecheck:test --write-baseline`);
166 for (const d of fixed) console.error(` - ${describe(d.identity)} (${d.baseline} → ${d.current})`);
167 }
168 if (added.length || fixed.length) return 1;
169 console.log(`test typecheck ratchet: ${total} known diagnostics, none new.`);
170 return 0;
171}
173if (import.meta.main) process.exit(main());