eval-trial-series.ts
240 lines · scripts/eval-trial-series.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Stamp `series_identity`, `series_fingerprint` and `harness_version` on a
4 * report's trial-outcomes JSONL (the pass-rates history key). A separate step
5 * after `test-paid-shards.ts --report`, so the paid runner's closure never
6 * imports the history tool.
7 *
8 * EVAL_POLICY v2 identity (`caseSeriesIdentitiesV2`, CEO-27/ENG-3): only the
9 * bytes a case owns (its paid test file, its fixtures, and the prompt files of
10 * the skills it names: templates, sections and the generated SKILL.md the
11 * model reads) plus HARNESS_VERSION. Shared helpers no longer reset every
12 * case's history; the execution harness every case consumes is pinned by
13 * scripts/harness-version.json and changes only by a deliberate bump
14 * (scripts/bump-harness-version.ts). Every record also carries the full
15 * consumed-input fingerprint (all touchfiles, globals included) as provenance.
16 *
17 * Usage:
18 * bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl>
19 * bun run scripts/eval-trial-series.ts --backtest [--weeks 30] [--records <dir>]
20 */
21import * as fs from 'node:fs';
22import * as path from 'node:path';
23import { spawnSync } from 'node:child_process';
24import { createHash } from 'node:crypto';
25import { LIVE_REGISTRY, readTrialOutcomeDir, type Registry } from './eval-flake-rank';
26import { formatTrialOutcomes, parseTrialOutcomes } from '../test/helpers/eval-store';
27import harnessManifest from './harness-version.json';
29const ROOT = path.resolve(import.meta.dir, '..');
31/** Deliberate shared-harness version: part of every v2 series identity. Bumped only by scripts/bump-harness-version.ts. */
32export const HARNESS_VERSION: number = harnessManifest.version;
33/** Harness patterns beyond GLOBAL_TOUCHFILES whose bytes HARNESS_VERSION pins. */
34export const HARNESS_PATTERNS: readonly string[] = harnessManifest.patterns;
36export interface TreeEntry { file: string; blob: string }
38/** test-selection's matchGlob semantics, compiled once per pattern (the backtest matches thousands of files per revision). */
39const globCache = new Map<string, RegExp>();
40export function globRegex(pattern: string): RegExp {
41 let regex = globCache.get(pattern);
42 if (!regex) {
43 regex = new RegExp(`^${pattern.replace(/\./g, '\\.').replace(/\*\*/g, '{{GLOBSTAR}}').replace(/\*/g, '[^/]*').replace(/\{\{GLOBSTAR\}\}/g, '.*')}$`);
44 globCache.set(pattern, regex);
45 }
46 return regex;
47}
48const matches = (file: string, patterns: readonly string[]) => patterns.some(pattern => globRegex(pattern).test(file));
50/** Tracked files with their blob ids, at the index (`ref` omitted) or at a commit. */
51export function treeEntries(root: string, ref?: string): TreeEntry[] {
52 const args = ref ? ['ls-tree', '-r', ref] : ['ls-files', '-s'];
53 const listed = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
54 if (listed.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${listed.stderr}`);
55 return listed.stdout.split('\n').filter(Boolean).map(line => {
56 const [meta, file] = line.split('\t');
57 const parts = meta!.split(' ');
58 return { file: file!, blob: (ref ? parts[2] : parts[1])! };
59 });
60}
62/** Top-level directories that hold a skill template. */
63export function skillDirs(entries: TreeEntry[]): Set<string> {
64 return new Set(entries.filter(entry => /^[^/]+\/SKILL\.md\.tmpl$/.test(entry.file)).map(entry => entry.file.split('/')[0]!));
65}
67/**
68 * A file a case owns: a paid test file (not a unit test under test/helpers),
69 * a fixture, or a skill prompt file (markdown, template or JSON manifest in a
70 * skill directory, generated SKILL.md included: it is what the model reads).
71 */
72export function isCaseOwnedFile(file: string, skills: Set<string>): boolean {
73 if (file.endsWith('.test.ts')) return !file.startsWith('test/helpers/');
74 if (/(^|\/)fixtures\//.test(file)) return true;
75 if (file === 'SKILL.md' || file === 'SKILL.md.tmpl') return true;
76 return skills.has(file.split('/')[0]!) && /\.(md|tmpl|json)$/.test(file);
77}
79/** Files HARNESS_VERSION pins: GLOBAL_TOUCHFILES plus HARNESS_PATTERNS, unit tests excluded. */
80export function harnessFiles(entries: TreeEntry[], registry: Registry = LIVE_REGISTRY, patterns: readonly string[] = HARNESS_PATTERNS): TreeEntry[] {
81 const all = [...registry.globals, ...patterns];
82 return entries.filter(entry => !entry.file.endsWith('.test.ts') && matches(entry.file, all));
83}
85const digest = (text: string) => createHash('sha256').update(text).digest('hex').slice(0, 16);
86const lines = (entries: TreeEntry[]) => entries.map(entry => `${entry.file} ${entry.blob}`).sort().join('\n');
88/** v2 series identity and full provenance fingerprint per case. */
89export function caseSeriesIdentitiesV2(ids: string[], entries: TreeEntry[], registry: Registry = LIVE_REGISTRY,
90 harnessVersion = HARNESS_VERSION): Record<string, { identity: string; fingerprint: string }> {
91 const skills = skillDirs(entries);
92 return Object.fromEntries(ids.map(id => {
93 const patterns = registry.touchfiles[id] ?? registry.judgeTouchfiles[id] ?? [];
94 const consumed = entries.filter(entry => matches(entry.file, patterns));
95 const owned = consumed.filter(entry => isCaseOwnedFile(entry.file, skills));
96 const full = entries.filter(entry => matches(entry.file, [...patterns, ...registry.globals]));
97 return [id, { identity: digest(`${id}\nharness ${harnessVersion}\n${lines(owned)}`), fingerprint: digest(`${id}\n${lines(full)}`) }];
98 }));
99}
101/** Rewrite the file with every record stamped; an invalid line fails the whole stamp. */
102export function stampTrialSeries(file: string, root = ROOT): number {
103 const { records, errors } = parseTrialOutcomes(fs.readFileSync(file, 'utf8'));
104 if (errors.length) throw new Error(`${file}: ${errors.join('; ')}`);
105 const identities = caseSeriesIdentitiesV2([...new Set(records.map(record => record.case))], treeEntries(root));
106 const stamped = records.map(record => ({ ...record, series_identity: identities[record.case]!.identity,
107 series_fingerprint: identities[record.case]!.fingerprint, harness_version: HARNESS_VERSION }));
108 fs.writeFileSync(file, formatTrialOutcomes(stamped));
109 return stamped.length;
110}
112// --- Backtest (ENG-3/DX-12): would the identity let cases reach the quarantine minimum? ---
114export interface BacktestRow { variant: string; latestQualifying: number; present: number; meanShareLast8: number; medianChanges: number }
116/**
117 * Over weekly main revisions (Monday 06:00 UTC, the census cron), how many
118 * blocking cases would hold one identity long enough to reach `minTrials`
119 * within the last `window` weekly runs (rule and judge cases record one
120 * trial per run, behavior cases a panel of 3). `v1` is today's identity
121 * (own touchfiles minus GLOBAL_TOUCHFILES); `v2` is caseSeriesIdentitiesV2;
122 * `v2+harness` also resets every series whenever the pinned harness bytes
123 * change (the cost of bumping HARNESS_VERSION on every harness edit).
124 */
125export function weeklyBacktest(opts: { root: string; weeks: number; now: number; registry?: Registry; minTrials?: number; window?: number }):
126 { revisions: string[]; harnessChanges: number; rows: BacktestRow[] } {
127 const registry = opts.registry ?? LIVE_REGISTRY;
128 const minTrials = opts.minTrials ?? 10, window = opts.window ?? 10;
129 const ids = Object.keys(registry.tiers).filter(id => registry.tiers[id] === 'gate' || registry.tiers[id] === 'periodic');
130 const monday = new Date(opts.now);
131 monday.setUTCHours(6, 0, 0, 0);
132 monday.setUTCDate(monday.getUTCDate() - ((monday.getUTCDay() + 6) % 7));
133 const revisions: string[] = [];
134 for (let week = opts.weeks - 1; week >= 0; week--) {
135 const at = new Date(monday.getTime() - week * 7 * 86_400_000).toISOString();
136 const sha = spawnSync('git', ['rev-list', '-1', '--first-parent', `--before=${at}`, 'HEAD'], { cwd: opts.root, encoding: 'utf8', timeout: 10_000 }).stdout.trim();
137 if (sha) revisions.push(sha);
138 }
139 const series: Record<string, Record<string, string[]>> = { v1: {}, v2: {} };
140 const harness: string[] = [];
141 for (const sha of revisions) {
142 const entries = treeEntries(opts.root, sha);
143 const present = new Set(ids.filter(id => entries.some(entry => entry.file.endsWith('.test.ts')
144 && matches(entry.file, registry.touchfiles[id] ?? []))));
145 const v2 = caseSeriesIdentitiesV2(ids, entries, registry, 0);
146 const nonGlobal = entries.filter(entry => !matches(entry.file, registry.globals));
147 for (const id of ids) {
148 const own = nonGlobal.filter(entry => matches(entry.file, registry.touchfiles[id] ?? []));
149 (series.v1![id] ??= []).push(present.has(id) ? digest(`${id}\n${lines(own)}`) : '');
150 (series.v2![id] ??= []).push(present.has(id) ? v2[id]!.identity : '');
151 }
152 harness.push(digest(lines(harnessFiles(entries, registry))));
153 }
154 const perRun = (id: string) => (registry.kinds[id] === 'behavior' ? 3 : 1);
155 const row = (variant: string, values: Record<string, string[]>, resetOnHarness: boolean): BacktestRow => {
156 const shares: number[] = [];
157 let latestQualifying = 0, present = 0;
158 for (let week = 0; week < revisions.length; week++) {
159 let qualifying = 0, live = 0;
160 for (const id of ids) {
161 const list = values[id]!;
162 if (!list[week]) continue;
163 live++;
164 let runs = 1;
165 while (runs < window && week - runs >= 0 && list[week - runs] === list[week] && (!resetOnHarness || harness[week - runs] === harness[week])) runs++;
166 if (runs * perRun(id) >= minTrials) qualifying++;
167 }
168 shares.push(live ? qualifying / live : 0);
169 if (week === revisions.length - 1) { latestQualifying = qualifying; present = live; }
170 }
171 const changes = ids.map(id => values[id]!.filter(Boolean)).map(list => list.slice(1).filter((value, index) => value !== list[index]).length).sort((a, b) => a - b);
172 const last8 = shares.slice(-8);
173 return { variant, latestQualifying, present, meanShareLast8: last8.reduce((a, b) => a + b, 0) / Math.max(1, last8.length), medianChanges: changes[Math.floor(changes.length / 2)] ?? 0 };
174 };
175 return { revisions, harnessChanges: harness.slice(1).filter((value, index) => value !== harness[index]).length,
176 rows: [row('v1', series.v1!, false), row('v2', series.v2!, false), row('v2+harness', series.v2!, true)] };
177}
179/**
180 * Recorded-trial backtest: cases reaching `minTrials` on one series when the
181 * recorded trials (any lane) are re-keyed by the v2 identity of the revision
182 * that produced them, vs the series_identity they were stamped with.
183 */
184export function recordBacktest(opts: { root: string; dir: string; registry?: Registry; minTrials?: number }):
185 { trials: number; revisions: number; missing: string[]; recorded: number; v2: number; blocking: number } {
186 const registry = opts.registry ?? LIVE_REGISTRY;
187 const minTrials = opts.minTrials ?? 10;
188 const seen = new Set<string>();
189 const records = readTrialOutcomeDir(opts.dir).records.filter(record => {
190 const key = `${record.run_id}|${record.attempt}|${record.case}|${record.trial}`;
191 if (record.outcome === 'skipped' || seen.has(key) || !record.sha) return false;
192 seen.add(key);
193 return true;
194 });
195 const cases = [...new Set(records.map(record => record.case))];
196 const identity = new Map<string, Record<string, { identity: string }>>();
197 const missing: string[] = [];
198 for (const sha of new Set(records.map(record => record.sha!))) {
199 try { identity.set(sha, caseSeriesIdentitiesV2(cases, treeEntries(opts.root, sha), registry, 0)); } catch { missing.push(sha); }
200 }
201 const qualifying = (key: (record: (typeof records)[number]) => string | undefined) => {
202 const counts = new Map<string, number>();
203 for (const record of records) {
204 const series = key(record);
205 if (series) counts.set(`${record.case}|${series}|${record.model}|${record.cli_version}`, (counts.get(`${record.case}|${series}|${record.model}|${record.cli_version}`) ?? 0) + 1);
206 }
207 const best = new Map<string, number>();
208 for (const [series, count] of counts) best.set(series.split('|')[0]!, Math.max(best.get(series.split('|')[0]!) ?? 0, count));
209 return [...best].filter(([id, count]) => (registry.tiers[id] === 'gate' || registry.tiers[id] === 'periodic') && count >= minTrials).length;
210 };
211 return { trials: records.length, revisions: identity.size, missing,
212 recorded: qualifying(record => record.series_identity), v2: qualifying(record => identity.get(record.sha!)?.[record.case]?.identity),
213 blocking: cases.filter(id => registry.tiers[id] === 'gate' || registry.tiers[id] === 'periodic').length };
214}
216if (import.meta.main) {
217 const argv = process.argv.slice(2);
218 if (argv[0] === '--backtest') {
219 const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
220 const weekly = weeklyBacktest({ root: ROOT, weeks: Number(flag('--weeks') ?? 30), now: Date.now() });
221 console.log(`weekly main revisions: ${weekly.revisions.length}; pinned-harness changes between consecutive weeks: ${weekly.harnessChanges}`);
222 console.log('variant qualifying now mean share (last 8 weeks) median identity changes/case');
223 for (const row of weekly.rows) {
224 console.log(`${row.variant.padEnd(11)} ${`${row.latestQualifying}/${row.present}`.padEnd(15)} ${`${(row.meanShareLast8 * 100).toFixed(1)}%`.padEnd(26)} ${row.medianChanges}`);
225 }
226 const dir = flag('--records');
227 if (dir) {
228 const recorded = recordBacktest({ root: ROOT, dir });
229 console.log(`recorded trials: ${recorded.trials} over ${recorded.revisions} revision(s)${recorded.missing.length ? ` (${recorded.missing.length} revision(s) not in this clone: fetch them)` : ''}`);
230 console.log(`blocking cases with >= 10 trials on one series: recorded identity ${recorded.recorded}/${recorded.blocking}, v2 identity ${recorded.v2}/${recorded.blocking}`);
231 }
232 process.exit(0);
233 }
234 const file = argv[0];
235 if (!file) {
236 console.error('usage: bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl> | --backtest [--weeks N] [--records <dir>]');
237 process.exit(2);
238 }
239 console.log(`[eval-trial-series] stamped ${stampTrialSeries(file)} record(s) in ${file}`);
240}