paid-cases.ts
317 lines · scripts/lib/paid-cases.ts at 3f79d81
1/**
2 * Case and trial shard keys for the paid lane: which files shard per case, how a case shard and its trials are named, and expansion of files into case/trial shards. Moved from scripts/test-paid-shards.ts.
3 */
4import * as fs from 'node:fs';
5import * as path from 'node:path';
6import { normalizeRelativePath } from './shard-engine';
7import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
8import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store';
9import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
10import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles';
11import { type PaidTier, ROOT } from './paid-types';
12import { type EvalHost, hostSkipReason, resolveEvalHost } from '../../test/helpers/e2e-hosts';
14/**
15 * The E2E ids a paid file registers: the touchfile registrations that list the
16 * file. `known` is true only when those ids are complete: no computed
17 * registration (testName, *IfSelected, describeIfSelected with a non-literal
18 * argument) and every literal registration argument is among them
19 * (`unregistered` lists the literal ids that are not). Quoted strings
20 * elsewhere (comments, skill paths) never count.
21 */
22export function fileCaseRegistration(
23 file: string, source: string,
24 touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
25 tiers: Record<string, string> = E2E_TIERS,
26): { registered: string[]; known: boolean; computed: boolean; unregistered: string[] } {
27 const rel = normalizeRelativePath(file);
28 const registered = Object.keys(touchfiles).filter(key => touchfiles[key]!.includes(rel));
29 const computed = /testName\s*:\s*(?!string\b)(?:`[^`]*\$\{|[A-Za-z_$])/.test(source)
30 || /\btest(?:Concurrent)?IfSelected\s*\(\s*(?:`[^`]*\$\{|[A-Za-z_$])/.test(source)
31 || /\bdescribeIfSelected\s*\([^,]*,(?!\s*\[)/.test(source)
32 || [...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)].some(m => m[1]!.split(',')
33 .map(item => item.trim()).some(item => item && !/^(['"`])[^'"`$]*\1$/.test(item)));
34 const literal = [
35 ...[...source.matchAll(/testName\s*:\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!),
36 ...[...source.matchAll(/\btest(?:Concurrent)?IfSelected\s*\(\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!),
37 ...[...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)]
38 .flatMap(m => [...m[1]!.matchAll(/(['"`])([^'"`]+)\1/g)].map(n => n[2]!)),
39 ].filter(id => id in tiers);
40 const unregistered = [...new Set(literal.filter(id => !registered.includes(id)))];
41 return { registered, known: registered.length > 0 && !computed && unregistered.length === 0, computed, unregistered };
42}
45/**
46 * Files whose cases run in separate processes, one shard per registered E2E
47 * case (`<file>#<case id>`): the file's lane wall exceeds one runner's budget
48 * while every case is short. Separate processes also give each case its own
49 * SDK semaphore, so shared-libs(-paths) capture waves never queue inside a
50 * sibling case's wall (the reason paths runs test.serial in one process).
51 * Every case must be a registered, literal E2E id whose Bun test name is the
52 * id or its CASE_TEST_NAMES label (test/paid-shards.test.ts scans the sources).
53 */
54export const CASE_SHARDED_FILES: readonly string[] = [
55 // W5a: one carved skill per case shard (was 15 one-line wrapper files under one case id).
56 'test/carve-section-loading.test.ts',
57 'test/skill-e2e-deploy.test.ts',
58 'test/skill-e2e-design.test.ts',
59 'test/skill-e2e-plan.test.ts',
60 'test/skill-e2e-review-army.test.ts',
61 'test/skill-e2e-shared-libs-paths.test.ts',
62 'test/skill-e2e-shared-libs.test.ts',
63 'test/skill-e2e-ship-docsync.test.ts',
64 'test/skill-e2e-qa-callers.test.ts',
65 // W5c: its gate cases (qa-quick, qa-only-no-fix, qa-bootstrap) no longer share one ~7-minute runner.
66 'test/skill-e2e-qa-workflow.test.ts',
67];
69/** Bun test names that differ from their E2E id. */
70export const CASE_TEST_NAMES: Record<string, string> = {
71 'plan-review-report': '/plan-eng-review writes GSTACK REVIEW REPORT to plan file',
72 'auq-format-gate': "/plan-ceo-review's first AskUserQuestion is a compliant decision brief (7/7 + substance)",
73 'autoplan-dual-voice': 'both Claude + Codex voices produce output in Phase 1 (within timeout)',
74 'cso-full-audit': '/cso persists supported tenant-boundary findings with redacted evidence',
75 'cso-diff-mode': '/cso --diff records its base and investigates changed security paths',
76 'cso-infra-scope': '/cso --infra finds an attacker-to-credential execution path',
77 'plan-ceo-review-plan-mode': 'first terminal outcome is asked (Step 0 fires before any plan write)',
78 'plan-eng-review-artifact': 'an interactive review writes one QA test plan about the reviewed change',
79 'plan-eng-review-artifact-full': 'a fresh interactive review reaches Test review and writes one QA test plan',
80};
82export const CASE_KEY_SEPARATOR = '#';
83const TRIAL_SUFFIX = /~t([1-9][0-9]*)$/;
85/** The test file behind a shard key (`<file>`, `<file>#<case id>` or `<file>#<case id>~t<N>`). */
86export function shardFile(key: string): string {
87 return normalizeRelativePath(key).split(CASE_KEY_SEPARATOR)[0]!;
88}
90/** The E2E case id of a case or trial shard key, else null. */
91export function shardCaseId(key: string): string | null {
92 const [, id] = normalizeRelativePath(key).split(CASE_KEY_SEPARATOR);
93 return id === undefined ? null : id.replace(TRIAL_SUFFIX, '');
94}
96/** The 1-based trial index of an isolated trial shard key, else null. */
97export function shardTrial(key: string): number | null {
98 const [, id] = normalizeRelativePath(key).split(CASE_KEY_SEPARATOR);
99 const match = id === undefined ? null : TRIAL_SUFFIX.exec(id);
100 return match ? Number(match[1]) : null;
101}
103/** Shard key of one trial of an isolated case. */
104export function trialShardKey(file: string, id: string, trial: number): string {
105 return `${normalizeRelativePath(file)}${CASE_KEY_SEPARATOR}${id}~t${trial}`;
106}
108/** Trial policy of one case, fixed from the registries before the run. */
109export interface CaseTrialPlan { kind: EvalCaseKind; panel: PanelShape; quarantined: boolean }
111/**
112 * `behavior` cases run EVAL_POLICY.panel; a quarantined case runs a full panel
113 * whose k keeps its kind's meaning (k = n for rule); everything else runs one
114 * trial. Only behavior and quarantined cases are isolated into trial shards.
115 */
116export function caseTrialPlan(id: string, kinds: Record<string, EvalCaseKind> = E2E_KINDS,
117 quarantine: Record<string, unknown> = CASE_QUARANTINE): CaseTrialPlan {
118 const kind = kinds[id] ?? 'rule';
119 const quarantined = Object.hasOwn(quarantine, id);
120 if (kind === 'behavior') return { kind, panel: { ...EVAL_POLICY.panel }, quarantined };
121 if (quarantined) return { kind, panel: { n: EVAL_POLICY.panel.n, k: EVAL_POLICY.panel.n }, quarantined };
122 return { kind, panel: { n: 1, k: 1 }, quarantined };
123}
125export function isIsolatedCase(plan: CaseTrialPlan): boolean {
126 return plan.kind === 'behavior' || plan.quarantined;
127}
129export function sameTrialPlan(a: CaseTrialPlan | undefined, b: CaseTrialPlan | undefined): boolean {
130 return !!a && !!b && a.kind === b.kind && a.quarantined === b.quarantined && a.panel?.n === b.panel?.n && a.panel?.k === b.panel?.k;
131}
133/** Bun name pattern that runs every case of a file except `ids` (their trial shards run them). */
134export function excludedCasesNamePattern(ids: string[]): string {
135 const escaped = ids.map(id => (CASE_TEST_NAMES[id] ?? id).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
136 return `^(?!.*(?:^|\\s)(?:${escaped.join('|')})$)`;
137}
139/** Exact Bun name pattern for a set of case ids (labels where the test name differs). */
140export function caseTestNamePattern(ids: string[]): string {
141 const escaped = ids.map(id => (CASE_TEST_NAMES[id] ?? id).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
142 return `(?:^|\\s)(?:${escaped.join('|')})$`;
143}
145/**
146 * Replace each case-sharded file with one key per registered case of `tier`.
147 * Throws when such a file's registration is not statically complete: an
148 * unregistered case would otherwise silently never run.
149 */
150export function expandCaseShards(files: string[], tier: PaidTier, rootDir = ROOT,
151 touchfiles: Record<string, string[]> = E2E_TOUCHFILES, tiers: Record<string, string> = E2E_TIERS): string[] {
152 return files.flatMap(file => {
153 const rel = normalizeRelativePath(file);
154 if (!CASE_SHARDED_FILES.includes(rel)) return [file];
155 const { registered, known } = fileCaseRegistration(rel, fs.readFileSync(path.join(rootDir, rel), 'utf8'), touchfiles, tiers);
156 if (!known) throw new Error(`Case-sharded ${rel} needs a complete literal case registration`);
157 return registered.filter(id => tiers[id] === tier).sort().map(id => `${rel}${CASE_KEY_SEPARATOR}${id}`);
158 });
159}
161export interface TrialExpansion {
162 keys: string[];
163 /** Trial policy per trial shard key. */
164 trials: Record<string, CaseTrialPlan>;
165 /** File shard key -> isolated case ids its name pattern excludes. */
166 excludeCases: Record<string, string[]>;
167}
169/**
170 * Isolate every behavior or quarantined case of `tier` into its panel of trial
171 * shards (`<file>#<id>~t1..tn`), each selected by EVALS_SELECTION_JSON=[id] and
172 * its exact test name. The file shard keeps the remaining ids of the tier and
173 * excludes the isolated ones by name; with none remaining it is dropped. A case
174 * may be isolated only when its file's registration is statically known.
175 */
176export function expandTrialShards(keys: string[], tier: PaidTier, rootDir = ROOT, opts: {
177 kinds?: Record<string, EvalCaseKind>; quarantine?: Record<string, unknown>;
178 touchfiles?: Record<string, string[]>; tiers?: Record<string, string>;
179} = {}): TrialExpansion {
180 const touchfiles = opts.touchfiles ?? E2E_TOUCHFILES;
181 const tiers = opts.tiers ?? E2E_TIERS;
182 const planOf = (id: string) => caseTrialPlan(id, opts.kinds, opts.quarantine);
183 const out: TrialExpansion = { keys: [], trials: {}, excludeCases: {} };
184 const addPanel = (file: string, id: string) => {
185 const plan = planOf(id);
186 for (let trial = 1; trial <= plan.panel.n; trial++) {
187 const key = trialShardKey(file, id, trial);
188 out.keys.push(key);
189 out.trials[key] = plan;
190 }
191 };
192 for (const key of keys) {
193 const file = shardFile(key);
194 const caseId = shardCaseId(key);
195 if (caseId !== null) {
196 if (isIsolatedCase(planOf(caseId))) addPanel(file, caseId);
197 else out.keys.push(key);
198 continue;
199 }
200 const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(rootDir, file), 'utf8'), touchfiles, tiers);
201 const inTier = registered.filter(id => tiers[id] === tier);
202 const isolated = inTier.filter(id => isIsolatedCase(planOf(id)));
203 if (isolated.length === 0) { out.keys.push(key); continue; }
204 if (!known) {
205 throw new Error(`${file}: behavior or quarantined case(s) ${isolated.join(', ')} need a statically known case registration`);
206 }
207 for (const id of isolated) addPanel(file, id);
208 if (inTier.length > isolated.length) {
209 out.keys.push(key);
210 out.excludeCases[normalizeRelativePath(key)] = [...isolated].sort();
211 }
212 }
213 return out;
214}
216/**
217 * Split expanded shard keys into runnable keys and CI-unrunnable cases
218 * (CASE_CI_EXCLUDE), each with its surfaced reason; never an empty shard.
219 */
220export function partitionCaseExclusions(keys: string[]): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } {
221 const excluded: Array<{ file: string; reason: string }> = [];
222 const runnable = keys.filter(key => {
223 const exclusion = CASE_CI_EXCLUDE[normalizeRelativePath(key)];
224 if (exclusion) excluded.push({ file: key, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` });
225 return !exclusion;
226 });
227 return { runnable, excluded };
228}
230/**
231 * Host exclusions (EVALS_HOST, test/helpers/e2e-hosts.ts): on a non-Claude
232 * host a case key whose case is Claude-only, or a file key none of whose
233 * `tier` cases runs on the host, is excluded with its reason. A file that
234 * keeps one host case stays runnable; its Claude-only cases skip at module
235 * load and the report deselects them. The Claude host excludes nothing.
236 */
237export function partitionHostExclusions(keys: string[], tier: PaidTier, host: EvalHost, rootDir = ROOT): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } {
238 if (host === 'claude') return { runnable: keys, excluded: [] };
239 const excluded: Array<{ file: string; reason: string }> = [];
240 const runnable = keys.filter(key => {
241 const caseId = shardCaseId(key);
242 const ids = caseId !== null ? [caseId]
243 : fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(rootDir, shardFile(key)), 'utf8')).registered.filter(id => E2E_TIERS[id] === tier);
244 if (ids.some(id => hostSkipReason(id, host) === null)) return true;
245 const reason = ids.length === 1 ? hostSkipReason(ids[0]!, host)!
246 : ids.length === 0 ? `Claude-only on the ${host} host: no Codex host metadata (no ${tier} case registered)`
247 : `Claude-only on the ${host} host: no case runs on it (${ids.join(', ')})`;
248 excluded.push({ file: key, reason: `skipped: ${reason}` });
249 return false;
250 });
251 return { runnable, excluded };
252}
254/**
255 * Codex access in the CI eval image, scoped per paid shard.
256 *
257 * The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR)
258 * and the eval workflows log it in under a non-default home
259 * (GSTACK_CI_CODEX_HOME). On the Codex host (EVALS_HOST=codex) Codex is the
260 * agent, so every shard gets the CLI on PATH and the home as CODEX_HOME.
261 * Otherwise a gate shard never sees either. A non-gate shard
262 * whose every file exercises Codex gets the CLI on PATH and the home as
263 * CODEX_HOME. A file that mixes one Codex case with other cases keeps only
264 * the two variables, and that case opts in on its own session; hermetic
265 * child environments drop GSTACK_* names, so its sibling sessions see no
266 * Codex. Every other shard sees neither, so a skill's outside-voice probe
267 * (`command -v codex`) reports not_installed exactly as it did before Codex
268 * entered the image. Outside CI neither variable is set and nothing changes.
269 */
270export const CODEX_CI_ENV = { binDir: 'GSTACK_CI_CODEX_BIN_DIR', home: 'GSTACK_CI_CODEX_HOME' } as const;
272/** Paid files whose every case exercises the real Codex CLI. */
273export const CODEX_CI_FILES: readonly string[] = [
274 'test/codex-e2e.test.ts',
275 'test/codex-e2e-sol-scope.test.ts',
276 'test/codex-e2e-shared-libs.test.ts',
277 'test/codex-e2e-recommendation-substance.test.ts',
278 'test/skill-e2e-outside-voice.test.ts',
279 'test/skill-e2e-outside-plan-disabled.test.ts',
280 'test/skill-e2e-safety-codex-boundary.test.ts',
281 'test/codex-e2e-multiblock-live.test.ts',
282];
284/**
285 * Codex cases inside files whose other cases must not see Codex. Each listed
286 * case opts in by putting $GSTACK_CI_CODEX_BIN_DIR on its own session PATH
287 * (inline, so the file's other touchfile keys do not grow a helper).
288 */
289export const CODEX_CI_CASES: Readonly<Record<string, readonly string[]>> = {
290 'test/skill-e2e-workflow.test.ts': ['codex-review'],
291};
293export type CodexShardAccess = 'none' | 'case-opt-in' | 'path';
295/** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */
296export function codexShardAccess(files: readonly string[], tier: string | undefined, host: EvalHost = 'claude'): CodexShardAccess {
297 if (files.length === 0) return 'none';
298 if (host === 'codex') return 'path';
299 if (tier === 'gate') return 'none';
300 if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path';
301 return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none';
302}
304/** Scope one shard's child environment in place; `files` are the shard keys' test files. */
305export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[]): void {
306 const binDir = env[CODEX_CI_ENV.binDir];
307 if (!binDir) return;
308 const home = env[CODEX_CI_ENV.home];
309 const access = codexShardAccess(files, env.EVALS_TIER, resolveEvalHost(env));
310 delete env.CODEX_HOME;
311 if (access === 'case-opt-in') return;
312 delete env[CODEX_CI_ENV.binDir];
313 delete env[CODEX_CI_ENV.home];
314 if (access !== 'path') return;
315 env.PATH = env.PATH ? `${binDir}${path.delimiter}${env.PATH}` : binDir;
316 if (home) env.CODEX_HOME = home;
317}