gate-diff-calibrate.ts
147 lines · scripts/gate-diff-calibrate.ts at 3f79d81
1/**
2 * gate-diff calibration — free, unshipped-to-skills tool inventory for the
3 * gate-edit detector (W1 of docs/designs/HONEST_WORK_GATE_INTEGRITY.md).
4 *
5 * bun scripts/gate-diff-calibrate.ts [--ref origin/main] [--count 50] [--repo <path>] [--markdown]
6 *
7 * Runs the pure core (lib/gate-diff) over the first-parent delta of the last
8 * N commits on <ref>'s first-parent line (one per landed PR: a merge commit
9 * or a squash merge), exactly what `gstack-gate-diff --commit <sha>` scans. Prints one row per
10 * commit: listed, eligible, read, unread, tags by id, unmatched, unsupported.
11 *
12 * What this measures: the TOOL's inventory (how much it lists, how often each
13 * tag fires, how often the read cap bites). It does NOT measure agent findings,
14 * kept/restored/open dispositions or reviewer value; those are collected after
15 * ship from the persisted review records. The hand read of tagged hunks is a
16 * human step this script only supports (--markdown gives the table to annotate).
17 * No artifact is written: calibration never touches the state root.
18 */
19import * as path from 'node:path';
20import { EMPTY_PROJECT_CONFIG, parseGateIntegrityBlock, scanCandidate, specDirectoryHoldsSpecs, type ScanResult } from '../lib/gate-diff/index';
21import { commitInfo, diffCommit, firstParent, readClaudeMd, repoRoot, runGit, treePaths } from '../lib/gate-diff/git';
22import { COVERAGE_TAGS } from '../lib/gate-diff/tags';
24interface Args {
25 ref: string;
26 count: number;
27 repo: string;
28 markdown: boolean;
29}
31function parseArgs(argv: string[]): Args {
32 const a: Args = { ref: 'origin/main', count: 50, repo: process.cwd(), markdown: false };
33 for (let i = 0; i < argv.length; i++) {
34 const x = argv[i];
35 if (x === '--ref') a.ref = argv[++i] ?? a.ref;
36 else if (x === '--count') a.count = Number(argv[++i] ?? a.count);
37 else if (x === '--repo') a.repo = argv[++i] ?? a.repo;
38 else if (x === '--markdown') a.markdown = true;
39 else if (x === '--help' || x === '-h') {
40 console.log('usage: bun scripts/gate-diff-calibrate.ts [--ref origin/main] [--count 50] [--repo <path>] [--markdown]');
41 process.exit(0);
42 } else throw new Error(`unknown argument ${x}`);
43 }
44 if (!Number.isFinite(a.count) || a.count <= 0) throw new Error('--count must be a positive integer');
45 return a;
46}
48/** The last N commits on <ref>'s first-parent line: merge commits and squash merges alike, one per landed PR. */
49function firstParentCommits(ref: string, count: number, cwd: string): string[] {
50 const r = runGit(['log', '--first-parent', '--format=%H', '-n', String(count), ref, '--'], cwd);
51 if (r.status !== 0) throw new Error(`git log ${ref} failed: ${r.stderr.trim()}`);
52 return r.stdout.toString('utf-8').split('\n').filter(Boolean);
53}
55interface Row {
56 sha: string;
57 subject: string;
58 result?: ScanResult;
59 error?: string;
60}
62function scanOne(sha: string, root: string, project: ReturnType<typeof parseGateIntegrityBlock>): Row {
63 const info = commitInfo(sha, root);
64 const subject = (info?.message.split('\n')[0] ?? '').slice(0, 72);
65 if (!firstParent(sha, root)) return { sha, subject, error: 'parent missing (shallow history: git fetch --deepen)' };
66 try {
67 const files = diffCommit(sha, root);
68 const result = scanCandidate({
69 files, commits: info ? [info] : [], project, specIsTestPath: specDirectoryHoldsSpecs(treePaths(root, sha)),
70 candidate: sha.slice(0, 12), artifact: '(calibration: not written)',
71 });
72 return { sha, subject, result };
73 } catch (e: any) {
74 return { sha, subject, error: String(e?.message ?? e) };
75 }
76}
78function tagCell(tagged: Record<string, number>): string {
79 const keys = Object.keys(tagged).sort((a, b) => Number.parseInt(a.slice(3), 10) - Number.parseInt(b.slice(3), 10) || a.localeCompare(b));
80 return keys.length ? keys.map((k) => `${k}:${tagged[k]}`).join(' ') : '-';
81}
83function median(xs: number[]): number {
84 if (xs.length === 0) return 0;
85 const s = [...xs].sort((a, b) => a - b);
86 const mid = Math.floor(s.length / 2);
87 return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
88}
90function percentile(xs: number[], p: number): number {
91 if (xs.length === 0) return 0;
92 const s = [...xs].sort((a, b) => a - b);
93 return s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))];
94}
96function main(): number {
97 const args = parseArgs(process.argv.slice(2));
98 const root = repoRoot(path.resolve(args.repo));
99 if (!root) throw new Error(`${args.repo} is not inside a git repository`);
100 const shas = firstParentCommits(args.ref, args.count, root);
101 const project = parseGateIntegrityBlock(readClaudeMd(root));
102 const rows = shas.map((sha) => scanOne(sha, root, project));
104 const ok = rows.filter((r) => r.result).map((r) => r.result as ScanResult);
105 const totals: Record<string, number> = {};
106 const commitsWithTag: Record<string, number> = {};
107 for (const r of ok) {
108 for (const [k, v] of Object.entries(r.summary.tagged)) {
109 totals[k] = (totals[k] ?? 0) + v;
110 commitsWithTag[k] = (commitsWithTag[k] ?? 0) + 1;
111 }
112 }
113 const listed = ok.map((r) => r.summary.listed);
114 const eligible = ok.map((r) => r.summary.eligible);
115 const partial = ok.filter((r) => r.summary.unread > 0).length;
117 const sep = args.markdown ? ' | ' : '\t';
118 const line = (cells: string[]) => (args.markdown ? `| ${cells.join(' | ')} |` : cells.join(sep));
119 const out: string[] = [];
120 out.push(line(['commit', 'subject', 'listed', 'eligible', 'read', 'unread', 'unmatched', 'unsupported', 'tags']));
121 if (args.markdown) out.push('|---|---|---:|---:|---:|---:|---:|---:|---|');
122 for (const r of rows) {
123 if (!r.result) {
124 out.push(line([r.sha.slice(0, 12), r.subject.replace(/\|/g, '\\|'), '-', '-', '-', '-', '-', '-', `ERROR: ${r.error}`]));
125 continue;
126 }
127 const s = r.result.summary;
128 out.push(line([
129 r.sha.slice(0, 12), r.subject.replace(/\|/g, '\\|'), String(s.listed), String(s.eligible), String(s.inspected), String(s.unread),
130 String(s.unmatched), String(r.result.unsupportedPaths.length), tagCell(s.tagged),
131 ]));
132 }
133 out.push('');
134 out.push(`${args.markdown ? '**Scanned:**' : 'Scanned:'} ${ok.length} of ${rows.length} first-parent commits on ${args.ref}; ${rows.length - ok.length} errored.`);
135 out.push(`${args.markdown ? '**Listed per commit:**' : 'Listed per commit:'} median ${median(listed)}, p75 ${percentile(listed, 75)}, max ${Math.max(0, ...listed)}; eligible median ${median(eligible)}, p75 ${percentile(eligible, 75)}, max ${Math.max(0, ...eligible)}; partial (read cap hit) in ${partial} commit(s).`);
136 const tagLines = [...COVERAGE_TAGS, 'RH-3?', 'RH-15?'].filter((t) => totals[t]).map((t) => `${t}: ${totals[t]} hunk(s) in ${commitsWithTag[t]} commit(s)`);
137 out.push(`${args.markdown ? '**Tags:**' : 'Tags:'} ${tagLines.length ? tagLines.join('; ') : 'none fired'}.`);
138 console.log(out.join('\n'));
139 return 0;
140}
142try {
143 process.exitCode = main();
144} catch (e: any) {
145 console.error(`gate-diff-calibrate: ${e?.message ?? e}`);
146 process.exitCode = 1;
147}