review-dispatch-compare.ts
181 lines · scripts/review-dispatch-compare.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * review-dispatch-compare — how many reviewers a /ship run and an /autoplan run dispatch under the
4 * legacy policy and under docs/review-policy.md#review-depth, on fixed fixture diffs, with an input
5 * plus output token estimate per run. The proof-of-saving table in the GSTA-131 PR comes from here.
6 *
7 * bun scripts/review-dispatch-compare.ts [--fixture <file>]... [--tokens <measured.json>] [--json]
8 * bun scripts/review-dispatch-compare.ts --capture <sha> --name <name> --out <file>
9 *
10 * --capture writes a fixture from one commit (its diff against its first parent) through
11 * gstack-review-plan --json. --tokens takes the live per-dispatch output-token medians by role and
12 * effort ({ "specialist": { "full": n, "light": n }, ... }); without it the script uses the shipped
13 * defaults below and says so. Counts are exact for the modelled scenarios; tokens are estimates.
14 */
15import { spawnSync } from 'node:child_process';
16import * as fs from 'node:fs';
17import * as os from 'node:os';
18import * as path from 'node:path';
19import { runScriptSync } from '../lib/script-invocation';
20import { planReview, specialistsFor, type DiffClass, type History, type NumbersResult, type PathLines } from '../lib/review-depth';
22const ROOT = path.resolve(import.meta.dir, '..');
23const BYTES_PER_TOKEN = 4;
24const FIXED_AUDITS = ['coverage-audit', 'plan-completion', 'docs-audit'];
25const RANGE_FRACTION = 0.15;
26const FIX_CHECK_FRACTION = 0.1;
27/** Output plus reasoning tokens per dispatch when no live measurement is given (labelled DEFAULT in the output). */
28const DEFAULT_OUT: Record<string, { full: number; light: number }> = {
29 specialist: { full: 6000, light: 3000 }, 'red-team': { full: 8000, light: 4000 }, 'adversarial-native': { full: 8000, light: 4000 },
30 'outside-voice': { full: 6000, light: 6000 }, 'fix-check': { full: 4000, light: 2000 }, audit: { full: 5000, light: 2500 }, 'autoplan-voice': { full: 9000, light: 4500 },
31};
33/** per_path and registry_outputs (captured since GSTA-147) feed the runtime-line count; an older fixture counts every code-class line as runtime. */
34export interface Fixture { name: string; commit: string; paths: string[]; deleted: string[]; stamp_only?: string[]; scope: Record<string, string>; gate_tagged: number | null; diff_lines: number; diff_bytes: number; numbers: NumbersResult; per_path?: PathLines[]; registry_outputs?: string[] }
35/** clean: pass 1 finds nothing; nits: pass 1 fixes below-floor findings; blocker: pass 1 fixes a blocker; adversarial-fix: Step 11 finds a should that is fixed; fix-regression: a pass-1 fix introduces a blocker the next step finds. */
36type Scenario = 'clean' | 'nits' | 'blocker' | 'adversarial-fix' | 'fix-regression';
37const SCENARIOS: Scenario[] = ['clean', 'nits', 'blocker', 'adversarial-fix', 'fix-regression'];
38interface Call { role: string; name?: string; effort: 'full' | 'medium' | 'light' | 'host'; diffBytes: number }
40const fileBytes = (rel: string) => { try { return fs.statSync(path.join(ROOT, rel)).size; } catch { return 0; } };
41function promptBytes(role: string, name: string): number {
42 if (role === 'specialist') return fileBytes(`review/specialists/${name}.md`) + fileBytes('review/checklist.md') / 4;
43 if (role === 'red-team') return fileBytes('review/specialists/red-team.md');
44 if (role === 'adversarial-native' || role === 'outside-voice') return fileBytes('ship/sections/adversarial.md') / 2;
45 if (role === 'fix-check') return fileBytes('review/checklist.md') / 2;
46 return 20_000;
47}
49function legacyShip(f: Fixture, sc: Scenario): Call[] {
50 const calls: Call[] = [];
51 const passes = sc === 'clean' ? 1 : sc === 'fix-regression' ? 3 : 2;
52 const step11Runs = sc === 'adversarial-fix' ? 2 : 1;
53 const legacySpecialists = specialistsFor('code', { ...base(f), history: { fullPasses: 0, last: 'none' } });
54 for (let p = 1; p <= passes; p++) {
55 const diffBytes = p === 1 ? f.diff_bytes : f.diff_bytes * RANGE_FRACTION;
56 for (const name of legacySpecialists) calls.push({ role: 'specialist', name, effort: 'full', diffBytes });
57 const critical = (sc === 'blocker' && p === 1) || (sc === 'fix-regression' && p === 2);
58 if (f.diff_lines > 200 || critical) calls.push({ role: 'red-team', effort: 'full', diffBytes });
59 }
60 for (let s = 0; s < step11Runs; s++) calls.push({ role: 'adversarial-native', effort: 'full', diffBytes: f.diff_bytes }, { role: 'outside-voice', effort: 'host', diffBytes: f.diff_bytes });
61 for (const a of FIXED_AUDITS) calls.push({ role: 'audit', effort: 'full', diffBytes: f.diff_bytes / 2 });
62 return calls;
63}
65function base(f: Fixture) {
66 return { paths: f.paths, deleted: f.deleted, stampOnly: f.stamp_only ?? [], scope: f.scope, gateTagged: f.gate_tagged, diffLines: f.diff_lines, numbers: f.numbers, perPath: f.per_path, registry: f.registry_outputs ? { sha: null, outputs: f.registry_outputs } : undefined };
67}
69/** The new policy, step by step through lib/review-depth's own transition table. */
70function newShip(f: Fixture, sc: Scenario): { calls: Call[]; ending: string } {
71 const calls: Call[] = [];
72 const outcomes: Record<Scenario, Array<Partial<History>>> = {
73 clean: [{}],
74 nits: [{ fixedBelow: true }, {}],
75 blocker: [{ fixedBlocking: true }, {}],
76 'adversarial-fix': [{ fixedBelow: true }, {}],
77 'fix-regression': [{ fixedBelow: true }, { blockingOpen: true }],
78 };
79 let h: History = { fullPasses: 0, last: 'none' };
80 let ending = 'done';
81 let adversarialRan = false;
82 for (let i = 0; i < 6; i++) {
83 const plan = planReview({ ...base(f), history: h, criticalFound: sc === 'blocker' && i === 0, adversarialRan });
84 if (plan.next.step === 'done' || plan.next.step === 'stop') { ending = plan.next.step; break; }
85 const diffBytes = plan.next.step === 'fix-check' ? f.diff_bytes * FIX_CHECK_FRACTION : h.fullPasses === 0 ? f.diff_bytes : f.diff_bytes * RANGE_FRACTION;
86 for (const d of plan.dispatches) calls.push({ role: d.role, name: d.name, effort: d.effort, diffBytes: d.role === 'outside-voice' ? f.diff_bytes : diffBytes });
87 adversarialRan ||= plan.dispatches.some(d => d.role === 'adversarial-native');
88 const o = outcomes[sc][Math.min(i, outcomes[sc].length - 1)] ?? {};
89 const last = plan.next.step === 'scoped' ? 'full' : plan.next.step;
90 h = { fullPasses: h.fullPasses + (last === 'full' ? 1 : 0), last, ...o };
91 }
92 for (const a of FIXED_AUDITS) calls.push({ role: 'audit', effort: 'light', diffBytes: f.diff_bytes / 2 });
93 return { calls, ending };
94}
96function tokens(calls: Call[], out: typeof DEFAULT_OUT): number {
97 return Math.round(calls.reduce((n, c) => {
98 const o = out[c.role] ?? out.audit!;
99 return n + (promptBytes(c.role, c.name ?? '') + c.diffBytes) / BYTES_PER_TOKEN + (c.effort === 'light' ? o.light : o.full);
100 }, 0));
101}
103/** /autoplan: phases x voices x rounds. Legacy reopens every touched phase on any accepted change, up to 3 rounds. */
104function autoplanRuns(docsOnly: boolean): Array<{ scenario: string; legacy: number; next: number }> {
105 const phases = ['ceo', 'eng'];
106 const legacyRounds = { 'no change': 1, 'medium-only change': 2, 'high change': 2, 'high twice': 3 } as Record<string, number>;
107 const newRounds = { 'no change': 1, 'medium-only change': 1, 'high change': 2, 'high twice': 2 } as Record<string, number>;
108 return Object.keys(legacyRounds).map(s => ({
109 scenario: s,
110 legacy: phases.length * 2 * legacyRounds[s]!,
111 next: (docsOnly ? 3 : phases.length * 2) * newRounds[s]!,
112 }));
113}
115function capture(sha: string, name: string, out: string): number {
116 const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-compare-'));
117 const captureBase = `review-compare-base-${process.pid}-${sha.slice(0, 8)}`;
118 const git = (args: string[], cwd = ROOT) => spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
119 try {
120 if (git(['worktree', 'add', '-q', '--detach', dir, sha]).status !== 0) throw new Error(`cannot check out ${sha}`);
121 git(['branch', '-f', captureBase, `${sha}^`], dir);
122 const r = runScriptSync(path.join(ROOT, 'bin', 'gstack-review-plan'), ['--base', captureBase, '--json'], { cwd: dir, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 });
123 if (r.status !== 0) throw new Error(`gstack-review-plan failed: ${r.stderr}`);
124 const plan = JSON.parse(r.stdout);
125 const diffBytes = Buffer.byteLength(git(['diff', `${sha}^`, sha], dir).stdout ?? '');
126 const fixture: Fixture = { name, commit: sha, paths: plan.paths, deleted: plan.deleted, stamp_only: plan.stamp_only, scope: plan.scope, gate_tagged: plan.gate_tagged, diff_lines: plan.diff_lines, diff_bytes: diffBytes, numbers: plan.numbers, per_path: plan.per_path, registry_outputs: plan.registry.outputs };
127 fs.mkdirSync(path.dirname(out), { recursive: true });
128 fs.writeFileSync(out, JSON.stringify(fixture, null, 2) + '\n');
129 process.stdout.write(`CAPTURED: ${name} ${sha} class=${plan.class} depth=${plan.depth} runtime=${plan.runtime_lines} paths=${plan.paths.length} lines=${plan.diff_lines} -> ${out}\n`);
130 return 0;
131 } finally {
132 git(['worktree', 'remove', '--force', dir]);
133 git(['branch', '-D', captureBase]);
134 }
135}
137export function compare(fixtures: Fixture[], out: typeof DEFAULT_OUT) {
138 const rows: Array<Record<string, string | number>> = [];
139 for (const f of fixtures) {
140 const first = planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } });
141 const cls: DiffClass = first.class;
142 for (const sc of SCENARIOS) {
143 const legacy = legacyShip(f, sc);
144 const next = newShip(f, sc);
145 const lt = tokens(legacy, out); const nt = tokens(next.calls, out);
146 rows.push({ fixture: f.name, class: cls, depth: first.depth, runtime_lines: first.runtimeLines, scenario: sc, legacy_dispatches: legacy.length, new_dispatches: next.calls.length, legacy_tokens: lt, new_tokens: nt, ratio: Math.round((nt / lt) * 100) / 100, ending: next.ending });
147 }
148 }
149 return rows;
150}
152function main(argv: string[]): number {
153 const flag = (n: string) => { const i = argv.indexOf(n); return i >= 0 ? argv[i + 1] : undefined; };
154 const cap = flag('--capture');
155 if (cap) return capture(cap, flag('--name') ?? cap, flag('--out') ?? path.join(ROOT, 'test/fixtures/review-depth', `${flag('--name') ?? cap}.json`));
156 const files = argv.flatMap((a, i) => (a === '--fixture' ? [argv[i + 1]!] : []));
157 const fixtureFiles = files.length ? files : fs.readdirSync(path.join(ROOT, 'test/fixtures/review-depth')).filter(f => f.endsWith('.json')).map(f => path.join(ROOT, 'test/fixtures/review-depth', f));
158 const fixtures = fixtureFiles.map(f => JSON.parse(fs.readFileSync(f, 'utf8')) as Fixture);
159 const tokFile = flag('--tokens');
160 const out = tokFile ? { ...DEFAULT_OUT, ...JSON.parse(fs.readFileSync(tokFile, 'utf8')) } : DEFAULT_OUT;
161 const rows = compare(fixtures, out);
162 const ap = fixtures.map(f => ({ fixture: f.name, runs: autoplanRuns(planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } }).class === 'docs') }));
163 if (argv.includes('--json')) { process.stdout.write(JSON.stringify({ tokens_source: tokFile ?? 'DEFAULT', ship: rows, autoplan: ap }) + '\n'); return 0; }
164 const lines = [`TOKENS: ${tokFile ? `measured (${tokFile})` : 'DEFAULT estimates (pass --tokens for measured output)'}`, '', '| fixture | class | depth (runtime lines) | scenario | /ship dispatches legacy → new | est. tokens legacy → new | ratio | ending |', '|---|---|---|---|---|---|---|---|'];
165 for (const r of rows) lines.push(`| ${r.fixture} | ${r.class} | ${r.depth} (${r.runtime_lines}) | ${r.scenario} | ${r.legacy_dispatches} → ${r.new_dispatches} | ${r.legacy_tokens} → ${r.new_tokens} | ${r.ratio} | ${r.ending} |`);
166 lines.push('', '| fixture | /autoplan scenario | voice dispatches legacy → new |', '|---|---|---|');
167 for (const a of ap) for (const r of a.runs) lines.push(`| ${a.fixture} | ${r.scenario} | ${r.legacy} → ${r.next} |`);
168 const median = (v: number[]) => { const m = Math.floor(v.length / 2); return v.length % 2 ? v[m]! : Math.round(((v[m - 1]! + v[m]!) / 2) * 100) / 100; };
169 const ratios = rows.map(r => Number(r.ratio)).sort((a, b) => a - b);
170 lines.push('', `SHIP_TOKEN_RATIO: median=${median(ratios)} min=${ratios[0]} max=${ratios[ratios.length - 1]} (n=${ratios.length})`);
171 const byClass = new Map<string, number[]>();
172 for (const r of rows) byClass.set(String(r.class), [...(byClass.get(String(r.class)) ?? []), Number(r.ratio)].sort((x, y) => x - y));
173 lines.push(`SHIP_TOKEN_RATIO_BY_CLASS: ${[...byClass].map(([c, v]) => `${c} median=${median(v)} (n=${v.length})`).join('; ')}`);
174 const byDepth = new Map<string, number[]>();
175 for (const r of rows) byDepth.set(String(r.depth), [...(byDepth.get(String(r.depth)) ?? []), Number(r.ratio)].sort((x, y) => x - y));
176 lines.push(`SHIP_TOKEN_RATIO_BY_DEPTH: ${[...byDepth].map(([d, v]) => `${d} median=${median(v)} (n=${v.length})`).join('; ')}`);
177 process.stdout.write(lines.join('\n') + '\n');
178 return 0;
179}
181if (import.meta.main) process.exit(main(process.argv.slice(2)));