eval-list.ts
131 lines · scripts/eval-list.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * List eval runs from the project eval dir (~/.gstack/projects/<slug>/evals;
4 * legacy fallback ~/.gstack-dev/evals)
5 *
6 * Usage: bun run eval:list [--branch <name>] [--tier e2e|llm-judge] [--limit N]
7 */
9import * as fs from 'fs';
10import { evalEntryOutcome, getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
12const EVAL_DIR = getProjectEvalDir();
14// Parse args
15const args = process.argv.slice(2);
16let filterBranch: string | null = null;
17let filterTier: string | null = null;
18let limit = 20;
20function parseLimit(raw: string | undefined): number {
21 if (!raw || !/^[1-9]\d*$/.test(raw)) {
22 console.error('eval:list: --limit requires a positive integer');
23 process.exit(1);
24 }
25 const parsed = Number(raw);
26 if (!Number.isSafeInteger(parsed)) {
27 console.error('eval:list: --limit requires a positive integer');
28 process.exit(1);
29 }
30 return parsed;
31}
33for (let i = 0; i < args.length; i++) {
34 if (args[i] === '--branch' && args[i + 1]) { filterBranch = args[++i]; }
35 else if (args[i] === '--tier' && args[i + 1]) { filterTier = args[++i]; }
36 else if (args[i] === '--limit') { limit = parseLimit(args[++i]); }
37}
39// Read eval files (flat dir plus one level of shards/<slug>/)
40const files = listEvalJsonFiles(EVAL_DIR);
42if (files.length === 0) {
43 console.log('No eval runs yet. Run: bun run eval:bg:pr');
44 process.exit(0);
45}
47// Parse top-level fields from each file
48interface RunSummary {
49 file: string;
50 timestamp: string;
51 branch: string;
52 tier: string;
53 version: string;
54 passed: number;
55 manual: Array<{ name: string; approvedBy: string; approvalUrl: string }>;
56 total: number;
57 cost: number;
58 duration: number;
59 turns: number;
60}
62const runs: RunSummary[] = [];
63for (const file of files) {
64 try {
65 const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
66 if (filterBranch && data.branch !== filterBranch) continue;
67 if (filterTier && data.tier !== filterTier) continue;
68 const totalTurns = (data.tests || []).reduce((s: number, t: any) => s + (t.turns_used || 0), 0);
69 const tests = Array.isArray(data.tests) ? data.tests : null;
70 const final = tests ? [...new Map<string, any>(tests.map((t: any) => [t.name, t] as const)).values()] : [];
71 const manual = final.filter((t: any) => evalEntryOutcome(t) === 'manual-review').map((t: any) => ({
72 name: t.name, approvedBy: t.manual_review.approval.approved_by, approvalUrl: t.manual_review.approval.approval_url,
73 }));
74 runs.push({
75 file,
76 timestamp: data.timestamp || '',
77 branch: data.branch || 'unknown',
78 tier: data.tier || 'unknown',
79 version: data.version || '?',
80 passed: tests ? tests.filter((t: any) => evalEntryOutcome(t) === 'passed').length : data.passed || 0,
81 manual,
82 total: data.total_tests || 0,
83 cost: data.total_cost_usd || 0,
84 duration: data.total_duration_ms || 0,
85 turns: totalTurns,
86 });
87 } catch { continue; }
88}
90// Sort by timestamp descending
91runs.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
93// Apply limit
94const displayed = runs.slice(0, limit);
96// Print table
97console.log('');
98console.log(`Eval History (${runs.length} total runs)`);
99console.log('═'.repeat(105));
100console.log(
101 ' ' +
102 'Date'.padEnd(17) +
103 'Branch'.padEnd(25) +
104 'Tier'.padEnd(12) +
105 'Pass'.padEnd(8) +
106 'Cost'.padEnd(8) +
107 'Turns'.padEnd(7) +
108 'Duration'.padEnd(10) +
109 'Version'
110);
111console.log('─'.repeat(105));
113for (const run of displayed) {
114 const date = run.timestamp.replace('T', ' ').slice(0, 16);
115 const branch = run.branch.length > 23 ? run.branch.slice(0, 20) + '...' : run.branch.padEnd(25);
116 const pass = `${run.passed}/${run.total}`.padEnd(8);
117 const cost = `$${run.cost.toFixed(2)}`.padEnd(8);
118 const turns = run.turns > 0 ? `${run.turns}t`.padEnd(7) : ''.padEnd(7);
119 const dur = run.duration > 0 ? `${Math.round(run.duration / 1000)}s`.padEnd(10) : ''.padEnd(10);
120 const manual = run.manual.map(entry => `MANUAL/unscored ${entry.name}: approved by ${entry.approvedBy} (${entry.approvalUrl})`).join('; ');
121 console.log(` ${date.padEnd(17)}${branch}${run.tier.padEnd(12)}${pass}${cost}${turns}${dur}v${run.version}${manual ? ` ${manual}` : ''}`);
122}
124console.log('─'.repeat(105));
126const totalCost = runs.reduce((s, r) => s + r.cost, 0);
127const totalDur = runs.reduce((s, r) => s + r.duration, 0);
128const totalTurns = runs.reduce((s, r) => s + r.turns, 0);
129console.log(` ${runs.length} runs | $${totalCost.toFixed(2)} total | ${totalTurns} turns | ${Math.round(totalDur / 1000)}s | Showing: ${displayed.length}`);
130console.log(` Dir: ${EVAL_DIR}`);
131console.log('');