eval-summary.ts
196 lines · scripts/eval-summary.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Aggregate summary of eval runs from the project eval dir
4 * (~/.gstack/projects/<slug>/evals; legacy fallback ~/.gstack-dev/evals)
5 *
6 * Usage: bun run eval:summary
7 */
9import * as fs from 'fs';
10import type { EvalResult } from '../test/helpers/eval-store';
11import { evalEntryOutcome, getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
13const EVAL_DIR = getProjectEvalDir();
15// Flat dir plus one level of shards/<slug>/
16const files = listEvalJsonFiles(EVAL_DIR);
18if (files.length === 0) {
19 console.log('No eval runs yet. Run: bun run eval:bg:pr');
20 process.exit(0);
21}
23// Load all results
24const results: EvalResult[] = [];
25for (const file of files) {
26 try {
27 results.push(JSON.parse(fs.readFileSync(file, 'utf-8')));
28 } catch { continue; }
29}
31// Aggregate stats
32const e2eRuns = results.filter(r => r.tier === 'e2e');
33const judgeRuns = results.filter(r => r.tier === 'llm-judge');
34const totalCost = results.reduce((s, r) => s + (r.total_cost_usd || 0), 0);
35const avgE2ECost = e2eRuns.length > 0 ? e2eRuns.reduce((s, r) => s + r.total_cost_usd, 0) / e2eRuns.length : 0;
36const avgJudgeCost = judgeRuns.length > 0 ? judgeRuns.reduce((s, r) => s + r.total_cost_usd, 0) / judgeRuns.length : 0;
38// Duration + turns from E2E runs
39const avgE2EDuration = e2eRuns.length > 0
40 ? e2eRuns.reduce((s, r) => s + (r.total_duration_ms || 0), 0) / e2eRuns.length
41 : 0;
42const e2eTurns: number[] = [];
43for (const r of e2eRuns) {
44 const runTurns = r.tests.reduce((s, t) => s + (t.turns_used || 0), 0);
45 if (runTurns > 0) e2eTurns.push(runTurns);
46}
47const avgE2ETurns = e2eTurns.length > 0
48 ? e2eTurns.reduce((a, b) => a + b, 0) / e2eTurns.length
49 : 0;
51// Per-test efficiency stats (avg turns + duration across runs)
52const testEfficiency = new Map<string, { turns: number[]; durations: number[]; costs: number[] }>();
53for (const r of e2eRuns) {
54 for (const t of r.tests) {
55 if (!testEfficiency.has(t.name)) {
56 testEfficiency.set(t.name, { turns: [], durations: [], costs: [] });
57 }
58 const stats = testEfficiency.get(t.name)!;
59 if (t.turns_used !== undefined) stats.turns.push(t.turns_used);
60 if (t.duration_ms > 0) stats.durations.push(t.duration_ms);
61 if (t.cost_usd > 0) stats.costs.push(t.cost_usd);
62 }
63}
65// Detection rates from outcome evals
66const detectionRates: number[] = [];
67for (const r of e2eRuns) {
68 for (const t of r.tests) {
69 if (t.detection_rate !== undefined) {
70 detectionRates.push(t.detection_rate);
71 }
72 }
73}
74const avgDetection = detectionRates.length > 0
75 ? detectionRates.reduce((a, b) => a + b, 0) / detectionRates.length
76 : null;
78// Flaky tests (passed in some runs, failed in others)
79const testResults = new Map<string, boolean[]>();
80const manualAccepted: Array<{ name: string; approvedBy: string; approvalUrl: string }> = [];
81for (const r of results) {
82 const final = new Map(r.tests.map(t => [t.name, t]));
83 const manuallyAcceptedNames = new Set([...final.values()].filter(t => evalEntryOutcome(t) === 'manual-review').map(t => t.name));
84 for (const t of final.values()) {
85 if (evalEntryOutcome(t) === 'manual-review') manualAccepted.push({ name: t.name,
86 approvedBy: t.manual_review!.approval.approved_by, approvalUrl: t.manual_review!.approval.approval_url });
87 }
88 for (const t of r.tests) {
89 if (manuallyAcceptedNames.has(t.name)) continue;
90 const key = `${r.tier}:${t.name}`;
91 const outcome = evalEntryOutcome(t);
92 if (outcome === 'manual-review') continue;
93 if (!testResults.has(key)) testResults.set(key, []);
94 testResults.get(key)!.push(outcome === 'passed');
95 }
96}
97const flakyTests: string[] = [];
98for (const [name, outcomes] of testResults) {
99 if (outcomes.length >= 2) {
100 const hasPass = outcomes.some(o => o);
101 const hasFail = outcomes.some(o => !o);
102 if (hasPass && hasFail) flakyTests.push(name);
103 }
104}
106// Branch stats
107const branchStats = new Map<string, { runs: number; avgDetection: number; detections: number[] }>();
108for (const r of e2eRuns) {
109 if (!branchStats.has(r.branch)) {
110 branchStats.set(r.branch, { runs: 0, avgDetection: 0, detections: [] });
111 }
112 const stats = branchStats.get(r.branch)!;
113 stats.runs++;
114 for (const t of r.tests) {
115 if (t.detection_rate !== undefined) {
116 stats.detections.push(t.detection_rate);
117 }
118 }
119}
120for (const stats of branchStats.values()) {
121 stats.avgDetection = stats.detections.length > 0
122 ? stats.detections.reduce((a, b) => a + b, 0) / stats.detections.length
123 : 0;
124}
126// Print summary
127console.log('');
128console.log('Eval Summary');
129console.log('═'.repeat(70));
130console.log(` Total runs: ${results.length} (${e2eRuns.length} e2e, ${judgeRuns.length} llm-judge)`);
131console.log(` Total spend: $${totalCost.toFixed(2)}`);
132if (manualAccepted.length) {
133 console.log(` Manual accepted: ${manualAccepted.length} unscored provider refusal(s)`);
134 for (const entry of manualAccepted) console.log(` ${entry.name}: approved by ${entry.approvedBy} (${entry.approvalUrl})`);
135}
136console.log(` Avg cost/e2e: $${avgE2ECost.toFixed(2)}`);
137console.log(` Avg cost/judge: $${avgJudgeCost.toFixed(2)}`);
138if (avgE2EDuration > 0) {
139 console.log(` Avg duration/e2e: ${Math.round(avgE2EDuration / 1000)}s`);
140}
141if (avgE2ETurns > 0) {
142 console.log(` Avg turns/e2e: ${Math.round(avgE2ETurns)}`);
143}
144if (avgDetection !== null) {
145 console.log(` Avg detection: ${avgDetection.toFixed(1)} bugs`);
146}
147console.log('─'.repeat(70));
149// Per-test efficiency averages (only if we have enough data)
150if (testEfficiency.size > 0 && e2eRuns.length >= 2) {
151 console.log(' Per-test efficiency (averages across runs):');
152 const sorted = [...testEfficiency.entries()]
153 .filter(([, s]) => s.turns.length >= 2)
154 .sort((a, b) => {
155 const avgA = a[1].costs.reduce((s, c) => s + c, 0) / a[1].costs.length;
156 const avgB = b[1].costs.reduce((s, c) => s + c, 0) / b[1].costs.length;
157 return avgB - avgA;
158 });
159 for (const [name, stats] of sorted) {
160 const avgT = Math.round(stats.turns.reduce((a, b) => a + b, 0) / stats.turns.length);
161 const avgD = Math.round(stats.durations.reduce((a, b) => a + b, 0) / stats.durations.length / 1000);
162 const avgC = (stats.costs.reduce((a, b) => a + b, 0) / stats.costs.length).toFixed(2);
163 const label = name.length > 30 ? name.slice(0, 27) + '...' : name.padEnd(30);
164 console.log(` ${label} $${avgC} ${avgT}t ${avgD}s (${stats.turns.length} runs)`);
165 }
166 console.log('─'.repeat(70));
167}
169if (flakyTests.length > 0) {
170 console.log(` Flaky tests (${flakyTests.length}):`);
171 for (const name of flakyTests) {
172 console.log(` - ${name}`);
173 }
174 console.log('─'.repeat(70));
175}
177if (branchStats.size > 0) {
178 console.log(' Branches:');
179 const sorted = [...branchStats.entries()].sort((a, b) => b[1].avgDetection - a[1].avgDetection);
180 for (const [branch, stats] of sorted) {
181 const det = stats.detections.length > 0 ? ` avg det: ${stats.avgDetection.toFixed(1)}` : '';
182 console.log(` ${branch.padEnd(30)} ${stats.runs} runs${det}`);
183 }
184 console.log('─'.repeat(70));
185}
187// Date range
188const timestamps = results.map(r => r.timestamp).filter(Boolean).sort();
189if (timestamps.length > 0) {
190 const first = timestamps[0].replace('T', ' ').slice(0, 16);
191 const last = timestamps[timestamps.length - 1].replace('T', ' ').slice(0, 16);
192 console.log(` Date range: ${first} → ${last}`);
193}
195console.log(` Dir: ${EVAL_DIR}`);
196console.log('');