bump-harness-version.ts
113 lines · scripts/bump-harness-version.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Keep HARNESS_VERSION honest (CEO-27/ENG-3/DX-6). scripts/harness-version.json
4 * pins the blob of every shared-harness file (GLOBAL_TOUCHFILES plus its
5 * `patterns`). test/harness-version.test.ts fails when a harness file changes
6 * without a decision, and this script records the decision:
7 *
8 * --bump "<reason>" the change alters how cases behave: version + 1,
9 * which starts a new pass-rate series for every case
10 * --non-behavioral "<reason>" the change cannot alter a verdict (refactor,
11 * comment, logging): the new blobs are recorded as
12 * compatible and no series resets
13 * --check print drift and the fix; exit 1 on drift
14 *
15 * Entries are one line per file, so two PRs touching different harness files
16 * merge cleanly. When both bump, or both touch one file, git conflicts on the
17 * manifest: take the higher version, then rerun this script.
18 */
19import * as fs from 'node:fs';
20import * as path from 'node:path';
21import { spawnSync } from 'node:child_process';
22import { harnessFiles, treeEntries, type TreeEntry } from './eval-trial-series';
24const ROOT = path.resolve(import.meta.dir, '..');
25export const MANIFEST = 'scripts/harness-version.json';
26export const DOC_ANCHOR = 'docs/TESTING_INTERNALS.md#harness-version';
28export interface ManifestEntry { blob: string; since: number; nonBehavioral?: string }
29export interface HarnessManifest { version: number; patterns: string[]; files: Record<string, ManifestEntry> }
30export interface Drift { changed: string[]; added: string[]; removed: string[] }
32export function readManifest(root = ROOT): HarnessManifest {
33 return JSON.parse(fs.readFileSync(path.join(root, MANIFEST), 'utf8'));
34}
36/** The tracked harness files with blobs of their working-tree bytes (an unstaged edit counts; a deleted file drops out). */
37export function workingHarnessFiles(root: string, patterns: readonly string[]): TreeEntry[] {
38 const tracked = harnessFiles(treeEntries(root), undefined, patterns).filter(entry => fs.existsSync(path.join(root, entry.file)));
39 if (tracked.length === 0) return [];
40 const hashed = spawnSync('git', ['hash-object', '--', ...tracked.map(entry => entry.file)], { cwd: root, encoding: 'utf8', timeout: 30_000 });
41 if (hashed.status !== 0) throw new Error(`git hash-object failed: ${hashed.stderr}`);
42 const blobs = hashed.stdout.trim().split('\n');
43 return tracked.map((entry, index) => ({ file: entry.file, blob: blobs[index]! }));
44}
46/** Harness files whose blob differs from, is missing from, or no longer matches an entry in the manifest. */
47export function harnessDrift(manifest: HarnessManifest, files: TreeEntry[]): Drift {
48 const current = new Map(files.map(entry => [entry.file, entry.blob]));
49 return {
50 changed: files.filter(entry => manifest.files[entry.file] && manifest.files[entry.file]!.blob !== entry.blob).map(entry => entry.file).sort(),
51 added: files.filter(entry => !manifest.files[entry.file]).map(entry => entry.file).sort(),
52 removed: Object.keys(manifest.files).filter(file => !current.has(file)).sort(),
53 };
54}
56export const hasDrift = (drift: Drift) => drift.changed.length + drift.added.length + drift.removed.length > 0;
58/** Problem, offending files, both fix commands and the docs anchor (DX-6). */
59export function formatDrift(drift: Drift, version: number): string {
60 const list = (label: string, files: string[]) => files.length ? [` ${label}: ${files.join(', ')}`] : [];
61 return [
62 `Shared-harness files changed without a HARNESS_VERSION decision (${MANIFEST}, version ${version}):`,
63 ...list('changed', drift.changed), ...list('added', drift.added), ...list('removed', drift.removed),
64 'Fix (pick one):',
65 ' bun run scripts/bump-harness-version.ts --non-behavioral "<why no verdict can change>" # refactor/comment/logging: no series reset',
66 ' bun run scripts/bump-harness-version.ts --bump "<what behavior changed>" # starts a new pass-rate series for every case',
67 `Merge conflict in ${MANIFEST}: take the higher version, then rerun the script. See ${DOC_ANCHOR}.`,
68 ].join('\n');
69}
71/** Record a decision over the current harness files; returns the new manifest. */
72export function decide(manifest: HarnessManifest, files: TreeEntry[], decision: 'bump' | 'non-behavioral', reason: string): HarnessManifest {
73 if (reason.trim().length < 10) throw new Error('a reason of at least 10 characters is required');
74 const drift = harnessDrift(manifest, files);
75 if (decision === 'bump') {
76 const version = manifest.version + 1;
77 return { ...manifest, version, files: Object.fromEntries(files.map(entry => [entry.file, { blob: entry.blob, since: version }])) };
78 }
79 const touched = new Set([...drift.changed, ...drift.added]);
80 return { ...manifest, files: Object.fromEntries(files.map(entry => [entry.file, touched.has(entry.file)
81 ? { blob: entry.blob, since: manifest.version, nonBehavioral: reason.trim() }
82 : manifest.files[entry.file]!])) };
83}
85/** Stable, one-entry-per-line JSON so concurrent edits to different files merge cleanly. */
86export function serializeManifest(manifest: HarnessManifest): string {
87 const files = Object.keys(manifest.files).sort().map(file => ` ${JSON.stringify(file)}: ${JSON.stringify(manifest.files[file])}`);
88 return `{\n "version": ${manifest.version},\n "patterns": [\n${manifest.patterns.map(pattern => ` ${JSON.stringify(pattern)}`).join(',\n')}\n ],\n`
89 + ` "files": {\n${files.join(',\n')}\n }\n}\n`;
90}
92if (import.meta.main) {
93 const argv = process.argv.slice(2);
94 const manifest = readManifest();
95 const files = workingHarnessFiles(ROOT, manifest.patterns);
96 const drift = harnessDrift(manifest, files);
97 const mode = argv[0];
98 if (mode === '--check' || mode === undefined) {
99 if (hasDrift(drift)) { console.error(formatDrift(drift, manifest.version)); process.exit(1); }
100 console.log(`harness version ${manifest.version}: ${files.length} pinned file(s), no drift`);
101 process.exit(0);
102 }
103 if (mode !== '--bump' && mode !== '--non-behavioral') {
104 console.error('usage: bun run scripts/bump-harness-version.ts --check | --bump "<reason>" | --non-behavioral "<reason>"');
105 process.exit(2);
106 }
107 if (mode === '--non-behavioral' && !hasDrift(drift)) { console.log('no harness drift: nothing to record'); process.exit(0); }
108 const next = decide(manifest, files, mode === '--bump' ? 'bump' : 'non-behavioral', argv[1] ?? '');
109 fs.writeFileSync(path.join(ROOT, MANIFEST), serializeManifest(next));
110 console.log(mode === '--bump'
111 ? `HARNESS_VERSION ${manifest.version} -> ${next.version}: every case starts a new pass-rate series (reason: ${argv[1]})`
112 : `recorded ${drift.changed.length + drift.added.length} file(s) as non-behavioral at HARNESS_VERSION ${next.version}${drift.removed.length ? `, dropped ${drift.removed.length} removed file(s)` : ''}`);
113}