eval-compare.ts
108 lines · scripts/eval-compare.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Compare two eval runs from the project eval dir (~/.gstack/projects/<slug>/evals;
4 * legacy fallback ~/.gstack-dev/evals)
5 *
6 * Usage:
7 * bun run eval:compare # compare two most recent of same tier
8 * bun run eval:compare <file> # compare file against its predecessor
9 * bun run eval:compare <file-a> <file-b> # compare two specific files
10 */
12import * as fs from 'fs';
13import * as path from 'path';
14import {
15 findPreviousRun,
16 compareEvalResults,
17 formatComparison,
18 getProjectEvalDir,
19 isPartialEval,
20 listEvalJsonFiles,
21} from '../test/helpers/eval-store';
22import type { EvalResult } from '../test/helpers/eval-store';
24const EVAL_DIR = getProjectEvalDir();
26function loadResult(filepath: string): EvalResult {
27 // Resolve relative to EVAL_DIR if not absolute
28 const resolved = path.isAbsolute(filepath) ? filepath : path.join(EVAL_DIR, filepath);
29 if (!fs.existsSync(resolved)) {
30 console.error(`File not found: ${resolved}`);
31 process.exit(1);
32 }
33 return JSON.parse(fs.readFileSync(resolved, 'utf-8'));
34}
36const args = process.argv.slice(2);
38let beforeFile: string;
39let afterFile: string;
41if (args.length === 2) {
42 // Two explicit files
43 beforeFile = args[0];
44 afterFile = args[1];
45} else if (args.length === 1) {
46 // One file — find its predecessor
47 afterFile = args[0];
48 const resolved = path.isAbsolute(afterFile) ? afterFile : path.join(EVAL_DIR, afterFile);
49 const afterResult = loadResult(resolved);
50 const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, resolved);
51 if (!prev) {
52 console.log('No previous run found to compare against.');
53 process.exit(0);
54 }
55 beforeFile = prev;
56} else {
57 // No args — find two most recent of the same tier. Scans the flat dir plus
58 // one level of shards/<slug>/; in-progress accumulators are never the
59 // "after" run (comparing against a half-finished run says nothing).
60 const files = listEvalJsonFiles(EVAL_DIR)
61 .sort((a, b) => path.basename(b).localeCompare(path.basename(a)));
63 if (files.length === 0) {
64 console.log('No eval runs yet. Run: bun run eval:bg:pr');
65 process.exit(0);
66 }
67 if (files.length < 2) {
68 console.log('Need at least 2 eval runs to compare. Run evals again.');
69 process.exit(0);
70 }
72 // Most recent finalized file
73 const latest = files.find(f => {
74 try {
75 return !isPartialEval(JSON.parse(fs.readFileSync(f, 'utf-8')), f);
76 } catch {
77 return false;
78 }
79 });
80 if (!latest) {
81 console.log('No completed eval runs yet. Run: bun run eval:bg:pr');
82 process.exit(0);
83 }
84 afterFile = latest;
85 const afterResult = loadResult(afterFile);
86 const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, afterFile);
87 if (!prev) {
88 console.log('No previous run of the same tier found to compare against.');
89 process.exit(0);
90 }
91 beforeFile = prev;
92}
94const beforeResult = loadResult(beforeFile);
95const afterResult = loadResult(afterFile);
97// Warn if different tiers
98if (beforeResult.tier !== afterResult.tier) {
99 console.warn(`Warning: comparing different tiers (${beforeResult.tier} vs ${afterResult.tier})`);
100}
102// Warn on schema mismatch
103if (beforeResult.schema_version !== afterResult.schema_version) {
104 console.warn(`Warning: schema version mismatch (${beforeResult.schema_version} vs ${afterResult.schema_version})`);
105}
107const comparison = compareEvalResults(beforeResult, afterResult, beforeFile, afterFile);
108console.log(formatComparison(comparison));