eval-select.ts
119 lines · scripts/eval-select.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Show which E2E and LLM-judge tests would run based on the current git diff.
4 *
5 * The default profile is `pr` (the PR lane's selection, matching `test:pr`
6 * and `eval:bg:pr`); `--profile full` (or EVALS_PROFILE=full) shows the plain
7 * touchfile selection over every E2E and LLM-judge entry.
8 *
9 * Usage:
10 * bun run eval:select # PR-lane selection, human-readable
11 * bun run eval:select --json # machine-readable JSON
12 * bun run eval:select --base main # override base branch
13 * bun run eval:select --profile full # full touchfile selection
14 */
16import * as path from 'path';
17import { computePaidCaseSelection } from './test-paid-shards';
18import {
19 selectTests,
20 detectBaseBranch,
21 getChangedFiles,
22 E2E_TOUCHFILES,
23 LLM_JUDGE_TOUCHFILES,
24 GLOBAL_TOUCHFILES,
25} from '../test/helpers/touchfiles';
27const ROOT = path.resolve(import.meta.dir, '..');
28const args = process.argv.slice(2);
29const jsonMode = args.includes('--json');
30const baseIdx = args.indexOf('--base');
31const baseOverride = baseIdx >= 0 ? args[baseIdx + 1] : undefined;
32const profileIdx = args.indexOf('--profile');
33const profile = profileIdx >= 0 ? args[profileIdx + 1] : process.env.EVALS_PROFILE ?? 'pr';
34if (profile !== 'pr' && profile !== 'full') throw new Error('--profile must be pr or full');
36// Detect base branch
37const baseBranch = baseOverride || detectBaseBranch(ROOT) || 'main';
38const changedFiles = getChangedFiles(baseBranch, ROOT);
40if (profile === 'pr') {
41 const result = computePaidCaseSelection({ profile, rootDir: ROOT, changedFiles,
42 env: { ...process.env, EVALS_BASE: baseBranch } });
43 const output = { base: baseBranch, changed_files: changedFiles, profile,
44 e2e: { selected: result.selection.e2e }, llm_judge: { selected: result.selection.judges },
45 coverage: result.coverage, reason: result.reason };
46 if (jsonMode) console.log(JSON.stringify(output, null, 2));
47 else {
48 console.log(`Base: ${baseBranch}\nProfile: pr (${result.coverage?.mode})\nChanged files: ${changedFiles.length}`);
49 console.log(`Behavioral probes: ${result.selection.e2e?.join(', ') || 'none'}`);
50 console.log(`Quality judges: ${result.selection.judges?.join(', ') || 'none'}`);
51 console.log(`Deferred to broad coverage: ${result.coverage?.deferred.map(item => item.id).join(', ') || 'none'}`);
52 console.log(result.reason);
53 }
54 process.exit(0);
55}
57if (changedFiles.length === 0) {
58 if (jsonMode) {
59 console.log(JSON.stringify({ base: baseBranch, changed_files: 0, e2e: 'all', llm_judge: 'all', reason: 'no diff — would run all tests' }));
60 } else {
61 console.log(`Base: ${baseBranch}`);
62 console.log('No changed files detected — all tests would run.');
63 }
64 process.exit(0);
65}
67// baseRef/cwd scope the map-diff path (used when touchfiles-data.ts changed)
68// to the same base this script diffed against — including a --base override.
69const selectOpts = { baseRef: baseBranch, cwd: ROOT };
70const e2eSelection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectOpts);
71const llmSelection = selectTests(changedFiles, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES, selectOpts);
73if (jsonMode) {
74 console.log(JSON.stringify({
75 base: baseBranch,
76 changed_files: changedFiles,
77 e2e: {
78 selected: e2eSelection.selected,
79 skipped: e2eSelection.skipped,
80 reason: e2eSelection.reason,
81 removed_tests: e2eSelection.removedTests ?? [],
82 count: `${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length}`,
83 },
84 llm_judge: {
85 selected: llmSelection.selected,
86 skipped: llmSelection.skipped,
87 reason: llmSelection.reason,
88 count: `${llmSelection.selected.length}/${Object.keys(LLM_JUDGE_TOUCHFILES).length}`,
89 },
90 }, null, 2));
91} else {
92 console.log(`Base: ${baseBranch}`);
93 console.log(`Changed files: ${changedFiles.length}`);
94 console.log();
96 console.log(`E2E: selected ${e2eSelection.selected.length} of ${Object.keys(E2E_TOUCHFILES).length}, reason: ${e2eSelection.reason}`);
97 if (e2eSelection.removedTests && e2eSelection.removedTests.length > 0) {
98 console.log(` Removed from maps (reported, not selected): ${e2eSelection.removedTests.join(', ')}`);
99 }
100 if (e2eSelection.selected.length > 0 && e2eSelection.selected.length < Object.keys(E2E_TOUCHFILES).length) {
101 console.log(` Selected: ${e2eSelection.selected.join(', ')}`);
102 console.log(` Skipped: ${e2eSelection.skipped.join(', ')}`);
103 } else if (e2eSelection.selected.length === 0) {
104 console.log(' No E2E tests affected.');
105 } else {
106 console.log(' All E2E tests selected.');
107 }
108 console.log();
110 console.log(`LLM-judge: selected ${llmSelection.selected.length} of ${Object.keys(LLM_JUDGE_TOUCHFILES).length}, reason: ${llmSelection.reason}`);
111 if (llmSelection.selected.length > 0 && llmSelection.selected.length < Object.keys(LLM_JUDGE_TOUCHFILES).length) {
112 console.log(` Selected: ${llmSelection.selected.join(', ')}`);
113 console.log(` Skipped: ${llmSelection.skipped.join(', ')}`);
114 } else if (llmSelection.selected.length === 0) {
115 console.log(' No LLM-judge tests affected.');
116 } else {
117 console.log(' All LLM-judge tests selected.');
118 }
119}