ship-measure.ts
842 lines · scripts/ship-measure.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * ship-measure — /ship's measure-then-fix runner for a red eval case or a red
4 * free-suite shard, and the weekly off-ship qualification sweep. Every trial
5 * it runs is DIAGNOSTIC: it never changes a recorded verdict, its artifacts
6 * are labeled `diagnostic` with `verdict: null`, and they live under
7 * `.context/ship-measure/`, never in the project eval dir that pass-rate
8 * history and CI uploads read. The lane verdict still comes from the one full
9 * gate run /ship makes after the case measures at or above the bar
10 * (docs/TESTING_INTERNALS.md#ship-measure).
11 *
12 * Trials per batch (config keys in bin/gstack-config): rule
13 * ship_measure_rule_trials (10); behavior ship_measure_behavior_panels panels
14 * of 3 (4, so 12 trials); judge ship_measure_judge_outputs outputs (10), each
15 * scored by its median 3-sample panel. The decision is the measurement bar in
16 * scripts/lib/measure-bar.ts: MEETS at 9/10, MEETS-qualified at 8/10 with
17 * every red in a qualifying class, EXTEND at 7/10 (N more trials on identical
18 * inputs, decided once on the pooled 2N), else BELOW (a fix round). A batch
19 * with more than 30% provider-evidence failures is void and is redispatched
20 * once.
21 *
22 * Spend: ship_measure_budget_usd is an estimated ADMISSION budget per red case
23 * across the baseline, extension, redispatch and every repair round. Before
24 * each batch the runner reserves the estimated cost of every concurrent trial
25 * and admits only what fits beside what was already spent; actual costs are
26 * reconciled after the batch. With no per-trial estimate it asks once
27 * ("estimate unknown"), then runs one calibration trial alone.
28 * ship_measure_ask_per_trial_usd asks first above that per-trial estimate.
29 * ship_measure_max_rounds caps repair rounds.
30 *
31 * Usage:
32 * bun run scripts/ship-measure.ts table
33 * bun run scripts/ship-measure.ts measure --case ID --round baseline|round-N
34 * [--kind rule|behavior|judge] [--command 'CMD {case}'] [--cost-per-trial USD]
35 * [--approved] [--fix TEXT] [--jobs N] [--out DIR]
36 * bun run scripts/ship-measure.ts classify --case ID --round R --trials 2,5 --class C --evidence TEXT [--out DIR]
37 * bun run scripts/ship-measure.ts decide --case ID --round R [--out DIR]
38 * bun run scripts/ship-measure.ts extend --case ID --round R [--command ..] [--cost-per-trial USD] [--jobs N] [--out DIR]
39 * bun run scripts/ship-measure.ts skip --case ID --reason TEXT [--out DIR]
40 * bun run scripts/ship-measure.ts report [--out DIR]
41 * bun run scripts/ship-measure.ts free (--files a,b,... | --shard I) [--reruns N]
42 * [--concurrency C] [--backend local|ubicloud] [--wall-cap SECS] [--out DIR]
43 * bun run scripts/ship-measure.ts sweep [--k K] [--cap-usd USD] [--dry-run] [--history-dir DIR]...
44 * [--prior DIR] [--command 'CMD {case}'] [--jobs N] [--out DIR] (scripts/ship-measure-sweep.ts)
45 *
46 * measure, extend, classify and decide exit 0 at MEETS or MEETS-qualified, 1
47 * at BELOW (a fix round), 2 when it needs the user's approval (nothing ran), 3
48 * on a named-red stop (budget exhausted, round limit, a void batch twice), 4 on
49 * a usage, setup or refused-classification error, 5 at needs-classify, 6 at
50 * EXTEND. free exits 0 when every completed rerun passed, else 1.
51 */
52import * as fs from 'node:fs';
53import * as os from 'node:os';
54import * as path from 'node:path';
55import { spawn, spawnSync } from 'node:child_process';
56import { EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
57import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
58import { getClaudeCliVersion, getProjectEvalDir, isFinalizedEvalResultFile } from '../test/helpers/eval-store';
59import { parseCliFlags } from './lib/shard-engine';
60import { collectPaidTestFiles, paidSelectionEnv } from './lib/paid-select';
61import { DEFAULT_JOBS } from './lib/paid-types';
62import { privateFreeHome } from './lib/free-home-guard';
63import {
64 FAILURE_CLASSES, barThresholds, classificationProblem, decide, identityMismatch,
65 type BarDecision, type BarNext, type BarThresholds, type Classification, type FailureClass, type SetSummary, type TrialSet,
66} from './lib/measure-bar';
67import harnessManifest from './harness-version.json';
68import {
69 TREE_MUTATING, assignFilesToShards, collectFreeTestFiles, fullSuiteJobs, loadFreeTestDurations, packShardsByDuration,
70 runFreeShard, wallTimeoutForShard, type FreeShardOutcome,
71} from './test-free-shards';
73const ROOT = path.resolve(import.meta.dir, '..');
75export type MeasureKind = 'rule' | 'behavior' | 'judge';
76export type RerunBackend = 'local' | 'ubicloud';
77export const PANEL = EVAL_POLICY.panel;
78export const JUDGE_SAMPLES = EVAL_POLICY.judge.samples;
80export interface MeasureConfig {
81 ruleTrials: number; behaviorPanels: number; judgeOutputs: number;
82 askPerTrialUsd: number; budgetUsd: number; maxRounds: number; rerunBackend: RerunBackend;
83 sweepCases: number; sweepBudgetUsd: number;
84}
86export const MEASURE_DEFAULTS: MeasureConfig = {
87 ruleTrials: 10, behaviorPanels: 4, judgeOutputs: 10, askPerTrialUsd: 2, budgetUsd: 25, maxRounds: 3, rerunBackend: 'local',
88 sweepCases: 5, sweepBudgetUsd: 150,
89};
91const USD_FIELDS = new Set<keyof MeasureConfig>(['askPerTrialUsd', 'budgetUsd', 'sweepBudgetUsd']);
93function gstackConfigGet(key: string): string | undefined {
94 const r = spawnSync(path.join(ROOT, 'bin', 'gstack-config'), ['get', key], { encoding: 'utf8', timeout: 10_000 });
95 return r.status === 0 ? r.stdout.trim() : undefined;
96}
98/** The measure controls from gstack config; an empty value takes the default, an invalid one throws. */
99export function readMeasureConfig(configGet: (key: string) => string | undefined = gstackConfigGet): MeasureConfig {
100 const raw: Array<[keyof MeasureConfig, string, string | undefined]> = [
101 ['ruleTrials', 'ship_measure_rule_trials', configGet('ship_measure_rule_trials')],
102 ['behaviorPanels', 'ship_measure_behavior_panels', configGet('ship_measure_behavior_panels')],
103 ['judgeOutputs', 'ship_measure_judge_outputs', configGet('ship_measure_judge_outputs')],
104 ['askPerTrialUsd', 'ship_measure_ask_per_trial_usd', configGet('ship_measure_ask_per_trial_usd')],
105 ['budgetUsd', 'ship_measure_budget_usd', configGet('ship_measure_budget_usd')],
106 ['maxRounds', 'ship_measure_max_rounds', configGet('ship_measure_max_rounds')],
107 ['rerunBackend', 'ship_rerun_backend', configGet('ship_rerun_backend')],
108 ['sweepCases', 'ship_measure_sweep_cases', configGet('ship_measure_sweep_cases')],
109 ['sweepBudgetUsd', 'ship_measure_sweep_budget_usd', configGet('ship_measure_sweep_budget_usd')],
110 ];
111 const config = { ...MEASURE_DEFAULTS };
112 for (const [field, key, rawValue] of raw) {
113 const value = rawValue?.trim();
114 if (!value) continue;
115 if (field === 'rerunBackend') {
116 if (value !== 'local' && value !== 'ubicloud') throw new Error(`${key} '${value}' is not local or ubicloud. Fix: gstack-config set ${key} local`);
117 config.rerunBackend = value;
118 continue;
119 }
120 const usd = USD_FIELDS.has(field);
121 if (!(usd ? Number.isFinite(Number(value)) && Number(value) > 0 : /^[1-9][0-9]*$/.test(value))) {
122 throw new Error(`${key} '${value}' is not a ${usd ? 'positive amount in USD' : 'positive integer'}. Fix: gstack-config set ${key} ${MEASURE_DEFAULTS[field]}`);
123 }
124 (config as Record<string, unknown>)[field] = Number(value);
125 }
126 return config;
127}
129export interface KindPlan {
130 kind: MeasureKind;
131 /** Behavior trials launch in panels of 3; rule and judge trials one at a time. */
132 trialsPerUnit: number;
133 /** Trials per batch (outputs for judge). */
134 trials: number;
135 bar: BarThresholds;
136 pooledBar: BarThresholds;
137}
139export function kindPlan(kind: MeasureKind, config: MeasureConfig): KindPlan {
140 const trials = kind === 'behavior' ? config.behaviorPanels * PANEL.n : kind === 'judge' ? config.judgeOutputs : config.ruleTrials;
141 return { kind, trialsPerUnit: kind === 'behavior' ? PANEL.n : 1, trials, bar: barThresholds(trials), pooledBar: barThresholds(2 * trials) };
142}
144/** The per-kind table /ship prints before it runs anything. */
145export function formatKindTable(config: MeasureConfig): string {
146 const budget = `$${config.budgetUsd} total; asks above $${config.askPerTrialUsd}/trial or with no estimate`;
147 const row = (kind: MeasureKind, label: string) => {
148 const { trials: n, bar, pooledBar } = kindPlan(kind, config);
149 const unit = kind === 'judge' ? ' outputs' : '';
150 return `| ${kind} | ${label} | ${bar.strict} of ${n}${unit} | ${bar.qualified} of ${n}, every red qualifying | ${bar.floor} of ${n} (or ${bar.qualified} that does not qualify): ${n} more on identical inputs, then ${pooledBar.strict} of ${2 * n}, or ${pooledBar.qualified} of ${2 * n} qualified | ${budget} |`;
151 };
152 return [
153 '| Kind | Trials per batch | MEETS | MEETS-qualified | EXTEND, then the pooled decision | Budget per red case |',
154 '|---|---|---|---|---|---|',
155 row('rule', String(config.ruleTrials)),
156 row('behavior', `${config.behaviorPanels * PANEL.n} (${config.behaviorPanels} panels of ${PANEL.n})`),
157 row('judge', `${config.judgeOutputs} outputs, each its median ${JUDGE_SAMPLES}-sample panel`),
158 '',
159 'Qualifying reds: provider (affirmative provider or transport evidence only), judge noise at the threshold, a model miss citing its evidence.',
160 'Never qualifying (a fix round): a timeout, a hang, a regression, a contract violation, any known fixable cause. Below the EXTEND floor: BELOW, a fix round.',
161 'A batch with more than 30% of trials failing on provider evidence is void and is redispatched once; both batches are reported. There is never a third batch.',
162 `Repair rounds: at most ${config.maxRounds}. Free-suite rerun backend: ${config.rerunBackend}. Every trial is diagnostic and never changes a recorded verdict.`,
163 ].join('\n');
164}
166// ─── Trials ────────────────────────────────────────────────────────────────
168export interface TrialResult {
169 passed: boolean;
170 /** A contract violation fails the measurement at any count. */
171 contract?: boolean;
172 /** Billed cost; absent means unknown. */
173 costUsd?: number;
174 /** Judge outputs: each sample's pass. */
175 samples?: boolean[];
176 failureCause?: string;
177 failureDetail?: string;
178 /** The eval store's one-line failure_cause_evidence. */
179 failureEvidence?: string;
180}
182export interface TrialRequest { caseId: string; kind: MeasureKind; round: string; trial: number; dir: string }
183export type TrialRunner = (request: TrialRequest) => Promise<TrialResult>;
185export interface TrialRecord extends TrialResult { trial: number; set: TrialSet; unit: number; batch: number; dir: string }
187// ─── One measurement: baseline or a repair round ───────────────────────────
189export type MeasureStatus = 'measured' | 'needs_approval' | 'budget_exhausted' | 'round_limit';
191export interface Measurement {
192 schema: 'gstack-ship-measure/2';
193 label: 'diagnostic';
194 verdict: null;
195 case: string;
196 kind: MeasureKind;
197 round: string;
198 fix?: string;
199 status: MeasureStatus;
200 reason?: string;
201 plan: KindPlan;
202 /** Inputs every pooled batch must share: tree, runner, CLI, model, image, policy and harness versions. */
203 identity: Record<string, string>;
204 trials: TrialRecord[];
205 decision: BarDecision;
206 next: BarNext;
207 decisionReason: string;
208 passes: number;
209 counted: number;
210 sets: SetSummary[];
211 /** The per-trial estimate later batches reserve; null until known. */
212 costPerTrialUsd: number | null;
213 /** Reserved before admission; null when a batch ran without an estimate. */
214 estimatedUsd: number | null;
215 actualUsd: number;
216 costUnknownTrials: number;
217 parallel: number;
218}
220interface Ledger { case: string; kind: MeasureKind; approved: boolean; spentUsd: number; rounds: string[] }
222/** A spend cap shared across cases (the sweep's weekly cap), checked beside each case's own budget. */
223export interface SpendPool { label: string; capUsd: number; spentUsd: number }
225export interface MeasureOptions {
226 caseId: string;
227 kind: MeasureKind;
228 round: string;
229 config: MeasureConfig;
230 runner: TrialRunner;
231 outDir: string;
232 parallel: number;
233 costPerTrialUsd: number | null;
234 approved: boolean;
235 fix?: string;
236 /** Pass-rate history directory diagnostic artifacts must stay out of (default: the project eval dir). */
237 evalDir?: string;
238 /** The batch identity (default: measurementIdentity of the working tree). */
239 identity?: () => Record<string, string>;
240 pool?: SpendPool;
241 log?: (line: string) => void;
242}
244export const caseSlug = (id: string) => id.replace(/[^A-Za-z0-9._-]+/g, '_');
245const readJson = <T>(file: string): T | null => { try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; } };
246const writeJson = (file: string, value: unknown) => { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); };
247const inside = (child: string, parent: string) => { const rel = path.relative(path.resolve(parent), path.resolve(child)); return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); };
249/** Throws when diagnostic artifacts would land where verdict history is read. */
250export function assertDiagnosticDir(outDir: string, evalDir: string): void {
251 if (inside(outDir, evalDir)) throw new Error(`ship-measure: ${outDir} is inside the eval history dir ${evalDir}; diagnostic trials are never recorded as verdicts. Use the default .context/ship-measure.`);
252}
254function roundNumber(round: string): number {
255 if (round === 'baseline') return 0;
256 const match = /^round-([1-9][0-9]*)$/.exec(round);
257 if (!match) throw new Error(`--round must be baseline or round-N. Received: ${round}`);
258 return Number(match[1]);
259}
261/** The git tree of the working tree as it would be committed now (tracked and unignored files), without touching the real index. */
262export function frozenTree(cwd: string): string {
263 const git = (args: string[], env: NodeJS.ProcessEnv = process.env) => spawnSync('git', args, { cwd, env, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
264 const top = git(['rev-parse', '--show-toplevel']);
265 if (top.status !== 0) return 'no-git';
266 const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ship-measure-index-'));
267 try {
268 const index = path.resolve(cwd, git(['rev-parse', '--git-path', 'index']).stdout.trim());
269 if (fs.existsSync(index)) fs.copyFileSync(index, path.join(tmp, 'index'));
270 const env = { ...process.env, GIT_INDEX_FILE: path.join(tmp, 'index') };
271 const add = git(['-C', top.stdout.trim(), 'add', '-A'], env);
272 const tree = add.status === 0 ? git(['write-tree'], env) : add;
273 if (tree.status !== 0) throw new Error(`ship-measure: could not snapshot the working tree: ${tree.stderr.trim()}`);
274 return tree.stdout.trim();
275 } finally { fs.rmSync(tmp, { recursive: true, force: true }); }
276}
278/** Everything a pooled batch must share with the batch it extends. */
279export function measurementIdentity(kind: MeasureKind, trials: number, runner: string, cwd = process.cwd(), env: NodeJS.ProcessEnv = process.env): Record<string, string> {
280 return {
281 tree: frozenTree(cwd), kind, trials: String(trials), runner, policy: String(EVAL_POLICY.version), harness: String(harnessManifest.version),
282 claude_cli: getClaudeCliVersion(), model: env.EVALS_MODEL ?? 'default', image: env.GSTACK_CI_IMAGE ?? 'local', bun: Bun.version,
283 };
284}
286const roundDirOf = (outDir: string, caseId: string, round: string) => path.join(outDir, caseSlug(caseId), round);
287const classificationsPath = (roundDir: string) => path.join(roundDir, 'classifications.jsonl');
289export function readClassifications(roundDir: string): Classification[] {
290 if (!fs.existsSync(classificationsPath(roundDir))) return [];
291 return fs.readFileSync(classificationsPath(roundDir), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as Classification);
292}
294/** Apply the bar to the measurement's trials and its classification records. */
295function withDecision(m: Measurement, roundDir: string): Measurement {
296 const outcome = decide(m.kind, m.plan.trials, m.trials, readClassifications(roundDir));
297 return { ...m, decision: outcome.decision, next: outcome.next, decisionReason: outcome.reason, passes: outcome.passes, counted: outcome.thresholds.trials, sets: outcome.sets };
298}
300function loadMeasurement(outDir: string, caseId: string, round: string): { m: Measurement; roundDir: string } {
301 const roundDir = roundDirOf(outDir, caseId, round);
302 const m = readJson<Measurement>(path.join(roundDir, 'measurement.json'));
303 if (!m) throw new Error(`ship-measure: no measurement at ${roundDir}; run measure --case ${caseId} --round ${round} first`);
304 return { m, roundDir };
305}
307function saveMeasurement(m: Measurement, roundDir: string, log: (line: string) => void): Measurement {
308 const decided = withDecision(m, roundDir);
309 writeJson(path.join(roundDir, 'measurement.json'), decided);
310 log(formatMeasurementLine(decided));
311 return decided;
312}
314export async function measureCase(o: MeasureOptions): Promise<Measurement> {
315 const log = o.log ?? ((line: string) => console.log(line));
316 const plan = kindPlan(o.kind, o.config);
317 assertDiagnosticDir(o.outDir, o.evalDir ?? getProjectEvalDir());
318 const caseDir = path.join(o.outDir, caseSlug(o.caseId));
319 const ledgerPath = path.join(caseDir, 'ledger.json');
320 const ledger: Ledger = readJson<Ledger>(ledgerPath) ?? { case: o.caseId, kind: o.kind, approved: false, spentUsd: 0, rounds: [] };
321 const base: Measurement = {
322 schema: 'gstack-ship-measure/2', label: 'diagnostic', verdict: null, case: o.caseId, kind: o.kind, round: o.round,
323 ...(o.fix ? { fix: o.fix } : {}), status: 'measured', plan, identity: {}, trials: [], decision: 'incomplete', next: 'stop', decisionReason: '',
324 passes: 0, counted: 0, sets: [], costPerTrialUsd: o.costPerTrialUsd, estimatedUsd: 0, actualUsd: 0, costUnknownTrials: 0, parallel: Math.max(1, o.parallel),
325 };
326 if (roundNumber(o.round) > o.config.maxRounds) {
327 return { ...base, status: 'round_limit', reason: `repair-round limit ${o.config.maxRounds} reached; stop with the named-red report` };
328 }
329 const estimate = o.costPerTrialUsd;
330 const ask = estimate === null ? 'estimate unknown' : estimate > o.config.askPerTrialUsd
331 ? `estimated $${estimate.toFixed(2)}/trial is above $${o.config.askPerTrialUsd}/trial` : null;
332 if (ask && !o.approved && !ledger.approved) {
333 return { ...base, status: 'needs_approval', reason: `${ask}; ${plan.trials} trials, budget $${o.config.budgetUsd} per red case. Ask once, then rerun with --approved.` };
334 }
335 if (ask) ledger.approved = true;
336 const roundDir = path.join(caseDir, o.round);
337 if (fs.existsSync(roundDir)) throw new Error(`ship-measure: ${roundDir} already exists; measurements are never overwritten. Use the next round.`);
338 fs.mkdirSync(roundDir, { recursive: true });
339 let m: Measurement = { ...base, identity: (o.identity ?? (() => measurementIdentity(o.kind, plan.trials, 'gstack')))() };
340 m = withDecision(await runSet(o, m, 'initial', roundDir, ledger, log), roundDir);
341 if (m.status === 'measured' && m.next === 'redispatch') {
342 log(`[ship-measure] ${o.caseId} ${o.round}: ${m.decisionReason}`);
343 m = await runSet(o, m, 'redispatch', roundDir, ledger, log);
344 }
345 ledger.rounds.push(o.round);
346 writeJson(ledgerPath, ledger);
347 return saveMeasurement(m, roundDir, log);
348}
350export interface ExtendOptions extends Omit<MeasureOptions, 'kind' | 'fix' | 'approved' | 'costPerTrialUsd'> { costPerTrialUsd?: number | null }
352/** EXTEND: N more trials on identical inputs (the identity must match field for field), decided once on the pooled 2N. */
353export async function extendCase(o: ExtendOptions): Promise<Measurement> {
354 const log = o.log ?? ((line: string) => console.log(line));
355 assertDiagnosticDir(o.outDir, o.evalDir ?? getProjectEvalDir());
356 let { m, roundDir } = loadMeasurement(o.outDir, o.caseId, o.round);
357 m = withDecision(m, roundDir);
358 if (m.trials.some(t => t.set === 'extend')) throw new Error(`ship-measure: ${o.caseId} ${o.round} was already extended; there is never a third batch`);
359 if (m.decision !== 'EXTEND') throw new Error(`ship-measure: ${o.caseId} ${o.round} is ${m.decision}, not EXTEND (${m.decisionReason})`);
360 const now = (o.identity ?? (() => measurementIdentity(m.kind, m.plan.trials, m.identity.runner ?? 'gstack')))();
361 const changed = identityMismatch(m.identity, now);
362 if (changed) throw new Error(`ship-measure: refusing to pool ${o.caseId} ${o.round}: ${changed}. EXTEND needs identical inputs; measure the changed tree as a new round.`);
363 const ledgerPath = path.join(o.outDir, caseSlug(o.caseId), 'ledger.json');
364 const ledger = readJson<Ledger>(ledgerPath) ?? { case: o.caseId, kind: m.kind, approved: true, spentUsd: m.actualUsd, rounds: [o.round] };
365 const opts: MeasureOptions = { ...o, kind: m.kind, approved: true, costPerTrialUsd: o.costPerTrialUsd ?? m.costPerTrialUsd };
366 m = withDecision(await runSet(opts, { ...m, status: 'measured' }, 'extend', roundDir, ledger, log), roundDir);
367 if (m.status === 'measured' && m.next === 'redispatch') {
368 log(`[ship-measure] ${o.caseId} ${o.round}: ${m.decisionReason}`);
369 m = await runSet(opts, m, 'extend-redispatch', roundDir, ledger, log);
370 }
371 writeJson(ledgerPath, ledger);
372 return saveMeasurement(m, roundDir, log);
373}
375/** Append one immutable classification record per trial (a later record supersedes, never edits), then re-decide. */
376export function classifyTrials(outDir: string, caseId: string, round: string, trials: number[], cls: string, evidence: string, log = (line: string) => console.log(line)): Measurement {
377 const { m, roundDir } = loadMeasurement(outDir, caseId, round);
378 for (const trial of trials) {
379 const problem = classificationProblem(m.trials.find(t => t.trial === trial), m.kind, cls, evidence);
380 if (problem) throw new Error(`ship-measure: classify t${trial} refused: ${problem}`);
381 }
382 const at = new Date().toISOString();
383 fs.appendFileSync(classificationsPath(roundDir), trials.map(trial => `${JSON.stringify({ trial, class: cls as FailureClass, evidence: evidence.trim(), by: 'agent', at } satisfies Classification)}\n`).join(''));
384 return saveMeasurement(m, roundDir, log);
385}
387export function decideRound(outDir: string, caseId: string, round: string, log = (line: string) => console.log(line)): Measurement {
388 const { m, roundDir } = loadMeasurement(outDir, caseId, round);
389 return saveMeasurement(m, roundDir, log);
390}
392/** Run one set of N trials (initial, redispatch, extend or extend-redispatch) under the admission budget. */
393async function runSet(o: MeasureOptions, m: Measurement, set: TrialSet, roundDir: string, ledger: Ledger, log: (line: string) => void): Promise<Measurement> {
394 const plan = m.plan;
395 const first = m.trials.length;
396 const trials: TrialRecord[] = [];
397 const initialEstimate = m.costPerTrialUsd;
398 let estimate = initialEstimate;
399 let estimatedUsd = m.estimatedUsd;
400 let actualUsd = m.actualUsd;
401 let costUnknownTrials = m.costUnknownTrials;
402 let knownCosts = m.trials.filter(t => typeof t.costUsd === 'number').length;
403 let batch = 0;
404 const parallel = Math.max(1, o.parallel);
405 const done = (status: MeasureStatus, reason?: string): Measurement => ({
406 ...m, status, ...(reason ? { reason } : {}), trials: [...m.trials, ...trials], estimatedUsd, actualUsd, costUnknownTrials, parallel,
407 costPerTrialUsd: estimate,
408 });
409 while (trials.length < plan.trials) {
410 const remaining = plan.trials - trials.length;
411 const headroom = Math.min(o.config.budgetUsd - ledger.spentUsd, o.pool ? o.pool.capUsd - o.pool.spentUsd : Infinity);
412 let size = Math.min(parallel, remaining);
413 if (estimate === null) size = knownCosts === 0 && trials.length === 0 ? 1 : size;
414 else size = Math.min(size, Math.floor((headroom + 1e-9) / estimate));
415 if (size < 1 || headroom <= 0) {
416 const cap = o.pool && o.pool.capUsd - o.pool.spentUsd <= o.config.budgetUsd - ledger.spentUsd
417 ? `${o.pool.label} $${o.pool.capUsd}: spent $${o.pool.spentUsd.toFixed(2)}`
418 : `budget $${o.config.budgetUsd} per red case: spent $${ledger.spentUsd.toFixed(2)}`;
419 return done('budget_exhausted', `${cap}, next trial reserves $${(estimate ?? 0).toFixed(2)}; ${trials.length} of ${plan.trials} ${set} trials ran`);
420 }
421 batch += 1;
422 estimatedUsd = estimate === null || estimatedUsd === null ? null : estimatedUsd + size * estimate;
423 log(`[ship-measure] ${o.caseId} ${o.round} ${set}: batch ${batch} admits ${size} trial(s)${estimate === null ? ' (calibration, no estimate)' : `, reserves $${(size * estimate).toFixed(2)}`}; spent $${ledger.spentUsd.toFixed(2)} of $${o.config.budgetUsd}`);
424 const start = first + trials.length;
425 const results = await Promise.all(Array.from({ length: size }, async (_, i) => {
426 const trial = start + i + 1;
427 const dir = path.join(roundDir, `t${String(trial).padStart(2, '0')}`);
428 fs.mkdirSync(dir, { recursive: false });
429 let result: TrialResult;
430 try { result = await o.runner({ caseId: o.caseId, kind: m.kind, round: o.round, trial, dir }); }
431 catch (error) { result = { passed: false, failureCause: 'unknown', failureDetail: `runner error: ${(error as Error).message}` }; }
432 return { ...result, trial, set, unit: Math.ceil((trial - first) / plan.trialsPerUnit), batch, dir } satisfies TrialRecord;
433 }));
434 for (const record of results) {
435 writeJson(path.join(record.dir, 'trial.json'), { label: 'diagnostic', verdict: null, ...record });
436 trials.push(record);
437 const charge = typeof record.costUsd === 'number' && Number.isFinite(record.costUsd) ? record.costUsd : estimate;
438 if (typeof record.costUsd === 'number' && Number.isFinite(record.costUsd)) { actualUsd += record.costUsd; knownCosts += 1; }
439 else costUnknownTrials += 1;
440 if (charge !== null) { ledger.spentUsd += charge; if (o.pool) o.pool.spentUsd += charge; }
441 }
442 if (initialEstimate === null && knownCosts > 0) estimate = actualUsd / knownCosts;
443 }
444 return done('measured');
445}
447const DECISION_WORDS: Record<BarDecision, string> = {
448 MEETS: 'MEETS', 'MEETS-qualified': 'MEETS-qualified', EXTEND: 'EXTEND', BELOW: 'BELOW (fix round)',
449 'needs-classify': 'needs-classify', void: 'void', incomplete: 'incomplete',
450};
452export function formatMeasurementLine(m: Measurement): string {
453 const when = m.fix ? ` after fix at ${m.fix}` : m.round === 'baseline' ? ' (baseline)' : '';
454 const verdict = m.status === 'measured' ? DECISION_WORDS[m.decision] : m.status.replace('_', ' ');
455 const sets = m.sets.length > 1 ? `; batches ${m.sets.map(s => `${s.set} ${s.passes}/${s.trials}${s.void ? ' void' : ''}`).join(', ')}` : '';
456 return `[ship-measure] DIAGNOSTIC ${m.case} ${m.round}: observed ${m.passes}/${m.counted} ${m.kind === 'judge' ? 'outputs' : 'trials'}${when}; ${verdict}${m.status === 'measured' ? ` — ${m.decisionReason}` : m.reason ? ` — ${m.reason}` : ''}${sets}; est ${usd(m.estimatedUsd)}, actual ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (+${m.costUnknownTrials} trial(s) cost unknown)` : ''}`;
457}
459const usd = (value: number | null) => value === null ? 'unknown' : `$${value.toFixed(2)}`;
461/** measure/extend/classify/decide exit code for a measurement (see the header). */
462export function measurementExit(m: Measurement): number {
463 if (m.status === 'needs_approval') return 2;
464 if (m.status !== 'measured') return 3;
465 return ({ MEETS: 0, 'MEETS-qualified': 0, BELOW: 1, 'needs-classify': 5, EXTEND: 6, void: 3, incomplete: 3 } as const)[m.decision];
466}
468// ─── Unmeasured skip and the PR-body report ────────────────────────────────
470export function recordUnmeasured(outDir: string, caseId: string, reason: string): string {
471 if (!reason.trim()) throw new Error('skip needs --reason: why this red is infrastructure and is not measured');
472 const file = path.join(outDir, caseSlug(caseId), 'unmeasured.json');
473 writeJson(file, { schema: 'gstack-ship-measure/2', label: 'diagnostic', verdict: null, case: caseId, status: 'unmeasured', reason: reason.trim() });
474 return file;
475}
477/** The per-case table for the PR body: every measurement, estimated and actual spend, never a pass for an unmeasured case. */
478export function formatReport(outDir: string): string {
479 const rows = ['| Case | Kind | Measurement | Observed | Bar | Decision | Est. spend | Actual spend |', '|---|---|---|---|---|---|---|---|'];
480 const cases = fs.existsSync(outDir) ? fs.readdirSync(outDir).filter(name => fs.statSync(path.join(outDir, name)).isDirectory()).sort() : [];
481 const qualified: string[] = [];
482 for (const name of cases) {
483 const dir = path.join(outDir, name);
484 const skip = readJson<{ case: string; reason: string }>(path.join(dir, 'unmeasured.json'));
485 if (skip) { rows.push(`| ${skip.case} | — | — | — | — | unmeasured (${skip.reason.replace(/\|/g, '/')}); not a pass | — | — |`); continue; }
486 const ledger = readJson<Ledger>(path.join(dir, 'ledger.json'));
487 for (const round of ledger?.rounds ?? []) {
488 const m = readJson<Measurement>(path.join(dir, round, 'measurement.json'));
489 if (!m) continue;
490 if (m.schema !== 'gstack-ship-measure/2') {
491 rows.push(`| ${m.case} | ${m.kind ?? '—'} | ${round} | — | — | older measurement format (${String(m.schema).replace(/\|/g, '/')}); not read, re-measure on this release | — | — |`);
492 continue;
493 }
494 const bar = m.counted > m.plan.trials ? m.plan.pooledBar : m.plan.bar;
495 const what = m.fix ? `${m.round}: after fix at ${m.fix.replace(/\|/g, '/')}` : m.round;
496 const batches = m.sets.length > 1 ? ` (${m.sets.map(s => `${s.set} ${s.passes}/${s.trials}${s.void ? ' void' : ''}`).join(', ')})` : '';
497 const decision = m.status === 'measured' ? DECISION_WORDS[m.decision] : m.status.replace('_', ' ');
498 rows.push(`| ${m.case} | ${m.kind} | ${what} | observed ${m.passes}/${m.counted}${batches} | ${bar.strict}/${bar.trials} strict, ${bar.qualified}/${bar.trials} qualified | ${decision} | ${usd(m.estimatedUsd)} | ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (${m.costUnknownTrials} unknown)` : ''} |`);
499 if (m.status === 'measured' && m.decision === 'MEETS-qualified') qualified.push(`- ${m.case} ${m.round}: ${m.decisionReason.replace(/\|/g, '/')}`);
500 }
501 }
502 return [...rows, ...(qualified.length ? ['', 'Qualified MEETS (every red classified into a qualifying class):', ...qualified] : []),
503 '', 'Diagnostic measurements: they never change a recorded verdict; the lane verdict is the full gate run.'].join('\n');
504}
506// ─── Trial runners ─────────────────────────────────────────────────────────
508const TRIAL_TIMEOUT_MS = 60 * 60_000;
510/** Cost, failure fields and judge records a trial's eval dir holds (any finalized eval-store result under it). */
511export function readTrialRecords(evalDir: string): Array<Record<string, any>> {
512 if (!fs.existsSync(evalDir)) return [];
513 return (fs.readdirSync(evalDir, { recursive: true }) as string[]).filter(isFinalizedEvalResultFile)
514 .flatMap(name => readJson<{ tests?: unknown[] }>(path.join(evalDir, name))?.tests ?? [])
515 .filter((t): t is Record<string, any> => !!t && typeof t === 'object');
516}
518/**
519 * No-cost proof that gstack's runner selects the case before any paid trial:
520 * `--list` must name the case's own test file. A selection bug otherwise
521 * shows up only after money is spent (two such bugs were found in #3033).
522 */
523export function caseSelectionPreflight(caseId: string, rootDir = ROOT): { ok: boolean; detail: string } {
524 const isJudge = !Object.hasOwn(E2E_TIERS, caseId);
525 if (isJudge) {
526 if (!Object.hasOwn(LLM_JUDGE_TOUCHFILES, caseId)) return { ok: false, detail: `${caseId} is neither an E2E case nor a standalone judge` };
527 const file = judgeFile(caseId, rootDir);
528 return fs.existsSync(path.join(rootDir, file)) ? { ok: true, detail: `judge ${caseId} runs from ${file}` } : { ok: false, detail: `judge file ${file} is missing` };
529 }
530 const r = spawnSync(process.execPath, ['run', path.join(rootDir, 'scripts/test-paid-shards.ts'), '--tier', E2E_TIERS[caseId]!, '--case', caseId, '--trials', '1', '--list'],
531 { cwd: rootDir, encoding: 'utf8', timeout: 60_000 });
532 const out = `${r.stdout ?? ''}${r.stderr ?? ''}`;
533 if (r.status !== 0) return { ok: false, detail: `--list exited ${r.status}: ${out.trim().split('\n').slice(-1)[0] ?? ''}` };
534 const file = new RegExp(`--case ${caseId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}: \\d+ trial\\(s\\) of (\\S+)`).exec(out)?.[1];
535 const esc = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
536 // A file-mode case lists `<file> (trial 1)`; a name-mode case lists its trial shard `<file>#<id>~t1`.
537 const listed = file && new RegExp(`^\\s+${esc(file)}(?: \\(trial 1\\)|#${esc(caseId)}~t1)$`, 'm').test(out);
538 if (!file || !listed || !fs.existsSync(path.join(rootDir, file))) return { ok: false, detail: `--list did not plan a trial of ${caseId}'s test file` };
539 return { ok: true, detail: `--case ${caseId} selects ${file}` };
540}
542/**
543 * The environment every diagnostic trial starts from. Trials always run fresh
544 * and never publish: with the judge input cache enabled, a trial could reuse a
545 * stored pass, and its own passes would flow into the gate's reuse.
546 */
547export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string, trialId: string): NodeJS.ProcessEnv {
548 const { EVALS_CACHE_DIR: _cacheDir, EVALS_CACHE_RUNTIME_ID: _cacheRuntime, ...inherited } = base;
549 // Cases that retain native evidence (functional QA, docs faults) require EVALS_RUN_ID, which only CI sets;
550 // each trial gets its own so parallel trials never share an evidence directory.
551 return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1', EVALS_RUN_ID: `${base.EVALS_RUN_ID || 'local'}-measure-${trialId}` };
552}
554/** True when some junit.xml under evalDir holds an executed, passing testcase and no failed or errored one. */
555export function junitExecuted(evalDir: string): boolean {
556 if (!fs.existsSync(evalDir)) return false;
557 let executed = 0;
558 for (const name of fs.readdirSync(evalDir, { recursive: true }) as string[]) {
559 if (path.basename(name) !== 'junit.xml') continue;
560 const xml = fs.readFileSync(path.join(evalDir, name), 'utf8');
561 for (const m of xml.matchAll(/<testcase\b[^>]*?(?:\/>|>([\s\S]*?)<\/testcase>)/g)) {
562 const body = m[1] ?? '';
563 if (/<(?:failure|error)\b/.test(body)) return false;
564 if (!/<skipped\b/.test(body)) executed++;
565 }
566 }
567 return executed > 0;
568}
570function costOf(records: Array<Record<string, any>>): number | undefined {
571 if (!records.length || records.some(r => r.cost_known === false || typeof r.cost_usd !== 'number')) return undefined;
572 return records.reduce((sum, r) => sum + r.cost_usd, 0);
573}
575/** Runs argv as one trial in its own process group, logging to <dir>/output.log; returns exit code and text. */
576async function runTrialProcess(argv: string[], cwd: string, env: NodeJS.ProcessEnv, dir: string, timeoutMs = TRIAL_TIMEOUT_MS): Promise<{ code: number | null; output: string }> {
577 const logFile = fs.createWriteStream(path.join(dir, 'output.log'));
578 const chunks: string[] = [];
579 const child = spawn(argv[0]!, argv.slice(1), { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], detached: process.platform !== 'win32' });
580 for (const stream of [child.stdout, child.stderr]) stream?.on('data', (chunk: Buffer) => { chunks.push(chunk.toString('utf8')); logFile.write(chunk); });
581 const timer = setTimeout(() => { try { process.kill(-(child.pid ?? 0), 'SIGKILL'); } catch { child.kill('SIGKILL'); } }, timeoutMs);
582 const code = await new Promise<number | null>(resolve => { child.on('error', () => resolve(null)); child.on('close', resolve); });
583 clearTimeout(timer);
584 await new Promise<void>(resolve => logFile.end(resolve));
585 return { code, output: chunks.join('') };
586}
588function parseTrialOutput(output: string): Partial<TrialResult> {
589 const field = (name: string) => new RegExp(`^\\s*${name}\\s*[:=]\\s*(.+?)\\s*$`, 'm').exec(output)?.[1];
590 const cost = field('cost_usd');
591 const samples = field('samples');
592 return {
593 ...(field('failure_cause') ? { failureCause: field('failure_cause') } : {}),
594 ...(field('failure_detail') ? { failureDetail: field('failure_detail') } : {}),
595 ...(field('failure_evidence') ? { failureEvidence: field('failure_evidence') } : {}),
596 ...(cost !== undefined && Number.isFinite(Number(cost)) ? { costUsd: Number(cost) } : {}),
597 ...(samples ? { samples: samples.split(/[,\s]+/).filter(Boolean).map(s => s === '1' || s === 'pass' || s === 'true') } : {}),
598 ...(/^\s*contract_violation\b/m.test(output) ? { contract: true } : {}),
599 };
600}
602/**
603 * The project's documented single-case command, split on whitespace (no
604 * shell), with {case} replaced by the case id. Exit 0 is a pass; the trial's
605 * own lines `cost_usd:`, `failure_cause:`, `failure_detail:`, `samples:` and
606 * `contract_violation` are read, and so are eval-store records it writes to
607 * GSTACK_EVAL_DIR.
608 */
609export function commandRunner(template: string, cwd = process.cwd()): TrialRunner {
610 const words = template.trim().split(/\s+/).filter(Boolean);
611 if (!words.length) throw new Error('--command is empty');
612 return async ({ caseId, trial, dir }) => {
613 const env = { ...process.env, GSTACK_EVAL_DIR: path.join(dir, 'eval'), GSTACK_SHIP_MEASURE_DIR: dir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', GSTACK_SHIP_MEASURE_TRIAL: String(trial) };
614 const { code, output } = await runTrialProcess(words.map(w => w.replaceAll('{case}', caseId)), cwd, env, dir);
615 const parsed = parseTrialOutput(output);
616 const records = readTrialRecords(path.join(dir, 'eval'));
617 const failed = records.find(r => r.passed !== true);
618 return {
619 passed: code === 0, ...parsed,
620 ...(parsed.costUsd === undefined && costOf(records) !== undefined ? { costUsd: costOf(records) } : {}),
621 ...(code !== 0 && !parsed.failureCause && failed?.failure_cause ? { failureCause: String(failed.failure_cause) } : {}),
622 ...(code !== 0 && !parsed.failureDetail && failed?.failure_detail ? { failureDetail: JSON.stringify(failed.failure_detail).slice(0, 300) } : {}),
623 ...(code !== 0 && !parsed.failureEvidence && failed?.failure_cause_evidence ? { failureEvidence: String(failed.failure_cause_evidence).slice(0, 300) } : {}),
624 };
625 };
626}
628/** The one paid file that names a standalone judge id, else a thrown reason. */
629export function judgeFile(id: string, rootDir = ROOT): string {
630 const owners = collectPaidTestFiles(rootDir).filter(file => {
631 const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
632 return source.includes(`'${id}'`) || source.includes(`"${id}"`);
633 });
634 if (owners.length !== 1) throw new Error(`judge ${id}: ${owners.length ? `named by ${owners.join(', ')}` : 'no paid file names it'}; it needs exactly one`);
635 return owners[0]!;
636}
638/**
639 * gstack's default runner. An E2E case runs one trial through
640 * `scripts/test-paid-shards.ts --case <id> --trials 1` (the CI panel runner,
641 * whose exit code is that trial's verdict); a standalone judge runs its file
642 * with the judge selected alone and passes only on its own passing record.
643 */
644export function gstackRunner(rootDir = ROOT): TrialRunner {
645 return async ({ caseId, round, trial, dir }) => {
646 const evalDir = path.join(dir, 'eval');
647 const isJudge = !Object.hasOwn(E2E_TIERS, caseId);
648 if (isJudge && !Object.hasOwn(LLM_JUDGE_TOUCHFILES, caseId)) throw new Error(`${caseId} is neither an E2E case nor a standalone judge`);
649 const argv = isJudge
650 ? [process.execPath, 'test', path.join(rootDir, judgeFile(caseId, rootDir))]
651 : [process.execPath, 'run', path.join(rootDir, 'scripts/test-paid-shards.ts'), '--tier', E2E_TIERS[caseId]!, '--case', caseId, '--trials', '1'];
652 const env = {
653 ...diagnosticBaseEnv(process.env, evalDir, `${caseSlug(caseId)}-${round}-t${trial}-${process.pid}`),
654 ...(isJudge ? { EVALS: '1', EVALS_TIER: 'gate', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [caseId] }, 'ship-measure judge') } : {}),
655 };
656 const { code, output } = await runTrialProcess(argv, rootDir, env, dir);
657 const records = readTrialRecords(evalDir);
658 const own = isJudge ? records.filter(r => r.name === caseId || r.case_id === caseId) : records;
659 const failed = own.find(r => r.passed !== true);
660 // An E2E trial passes only with proof the case ran: a JUnit testcase that
661 // executed and passed (a --case that selected nothing also exits 0).
662 const passed = code === 0 && (isJudge ? own.length > 0 && !failed : junitExecuted(evalDir));
663 const cost = costOf(own);
664 return {
665 passed, ...(cost !== undefined ? { costUsd: cost } : {}),
666 ...(own.some(r => r.failure_class === 'contract') ? { contract: true } : {}),
667 ...(passed ? {} : {
668 failureCause: failed?.failure_cause ? String(failed.failure_cause) : own.length ? 'assertion' : 'unknown',
669 ...(failed?.failure_cause_evidence ? { failureEvidence: String(failed.failure_cause_evidence).slice(0, 300) } : {}),
670 failureDetail: failed?.failure_detail ? JSON.stringify(failed.failure_detail).slice(0, 300)
671 : own.length ? String(failed?.error ?? '').slice(0, 300) : `no trial record; exit ${code}; ${output.trim().split('\n').slice(-1)[0] ?? ''}`.slice(0, 300),
672 }),
673 };
674 };
675}
677// ─── Free-suite shard reruns (CEO-1) ───────────────────────────────────────
679export interface FreeRerunOutcome { rerun: number; mode: 'baseline' | 'parallel'; status: string; completed: boolean; failingFiles: string[]; elapsedMs: number; log: string }
681export interface FreeMeasurement {
682 schema: 'gstack-ship-measure/1'; label: 'diagnostic'; verdict: null;
683 files: string[]; reruns: number; completed: number; passed: number; parallel: number; concurrency: number;
684 backend: 'local'; wallCapMs: number; capHit: boolean; outcomes: FreeRerunOutcome[];
685}
687export interface FreeRerunOptions {
688 files: string[];
689 reruns: number;
690 /** Within-shard concurrency of the original run (the free lane runs each shard at 1). */
691 concurrency: number;
692 wallCapMs: number;
693 outDir: string;
694 cores?: number;
695 runShard?: typeof runFreeShard;
696 log?: (line: string) => void;
697}
699/** Local parallelism for free reruns: max(1, floor(cores / shard concurrency)). */
700export const freeParallelism = (cores: number, concurrency: number) => Math.max(1, Math.floor(cores / Math.max(1, concurrency)));
702/**
703 * Rerun one shard's exact file list N times with the flaky retry off: rerun 1
704 * alone as the baseline, the rest in parallel up to the cap, each with its own
705 * HOME, state root and flake ledger. No rerun starts after the wall cap, and
706 * one the cap cuts short is not counted as completed.
707 */
708export async function measureFreeShard(o: FreeRerunOptions): Promise<FreeMeasurement> {
709 const log = o.log ?? ((line: string) => console.log(line));
710 const runShard = o.runShard ?? runFreeShard;
711 const parallel = freeParallelism(o.cores ?? os.availableParallelism(), o.concurrency);
712 fs.mkdirSync(path.join(o.outDir, 'free'), { recursive: true });
713 const dir = fs.mkdtempSync(path.join(o.outDir, 'free', 'rerun-'));
714 const deadline = Date.now() + o.wallCapMs;
715 const shardWall = wallTimeoutForShard(o.files.length);
716 const outcomes: FreeRerunOutcome[] = [];
717 const runOne = async (rerun: number, mode: FreeRerunOutcome['mode']) => {
718 const runDir = path.join(dir, `r${String(rerun).padStart(2, '0')}`);
719 const state = path.join(runDir, 'state');
720 fs.mkdirSync(state, { recursive: true });
721 const remaining = deadline - Date.now();
722 const wallTimeoutMs = Math.max(1, Math.min(shardWall, remaining));
723 const env = { ...process.env, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, GSTACK_FREE_RETRY_FLAKY: '0', GSTACK_FLAKE_LEDGER: path.join(runDir, 'flake-ledger.jsonl') };
724 const logPath = path.join(runDir, 'shard.log');
725 const outcome: FreeShardOutcome = await runShard(o.files, rerun, o.reruns, { env, wallTimeoutMs, quiet: true, log: () => {}, logFilePath: logPath, homeGuard: privateFreeHome });
726 const cut = outcome.status === 'timed-out' && wallTimeoutMs < shardWall;
727 outcomes.push({ rerun, mode, status: cut ? 'cut by wall cap' : outcome.status, completed: !cut, failingFiles: outcome.failingFiles, elapsedMs: outcome.elapsedMs, log: logPath });
728 };
729 if (o.reruns > 0) await runOne(1, 'baseline');
730 let next = 2;
731 const worker = async () => {
732 while (next <= o.reruns && Date.now() < deadline) await runOne(next++, 'parallel');
733 };
734 await Promise.all(Array.from({ length: Math.min(parallel, Math.max(0, o.reruns - 1)) }, worker));
735 outcomes.sort((a, b) => a.rerun - b.rerun);
736 const done = outcomes.filter(r => r.completed);
737 const measurement: FreeMeasurement = {
738 schema: 'gstack-ship-measure/1', label: 'diagnostic', verdict: null, files: o.files, reruns: o.reruns,
739 completed: done.length, passed: done.filter(r => r.status === 'passed').length, parallel, concurrency: o.concurrency,
740 backend: 'local', wallCapMs: o.wallCapMs, capHit: done.length < o.reruns, outcomes,
741 };
742 writeJson(path.join(dir, 'free-measurement.json'), measurement);
743 log(`[ship-measure] DIAGNOSTIC free shard (${o.files.length} files): observed ${measurement.passed}/${measurement.completed} passed; ${measurement.completed} of ${o.reruns} completed${measurement.capHit ? ` (wall cap ${Math.round(o.wallCapMs / 1000)}s)` : ''}; rerun 1 alone as baseline, the rest at parallelism ${parallel}; flaky retry off. Captures: ${dir}`);
744 return measurement;
745}
747/** The exact file list of local free-suite shard I, as `bun run test` packs it. */
748export function localFreeShardFiles(index: number, rootDir = ROOT): string[] {
749 const files = collectFreeTestFiles(rootDir);
750 const exclusive = files.filter(f => f in TREE_MUTATING);
751 const readers = files.filter(f => !(f in TREE_MUTATING));
752 const jobs = fullSuiteJobs();
753 const durations = loadFreeTestDurations(rootDir);
754 const shards = durations ? packShardsByDuration(readers, jobs, durations).shards : assignFilesToShards(readers, jobs);
755 if (exclusive.length) shards.push(exclusive);
756 const shard = shards[index - 1];
757 if (!shard) throw new Error(`--shard ${index}: the local free suite has ${shards.length} shards`);
758 return shard;
759}
761// ─── CLI ───────────────────────────────────────────────────────────────────
763function positive(raw: string | undefined, flag: string, integer = true): number {
764 const value = Number(raw);
765 if (raw === undefined || !(integer ? /^[1-9][0-9]*$/.test(raw) : Number.isFinite(value) && value >= 0)) throw new Error(`${flag} needs a ${integer ? 'positive integer' : 'non-negative number'}. Received: ${raw}`);
766 return value;
767}
769export async function main(argv: string[], env: NodeJS.ProcessEnv = process.env): Promise<number> {
770 const [command, ...rest] = argv;
771 if (command === 'sweep') return (await import('./ship-measure-sweep')).sweepMain(rest, env);
772 const flags: Record<string, string> = {};
773 const value = (name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); flags[name] = v; };
774 const handlers = Object.fromEntries(['--case', '--round', '--kind', '--command', '--cost-per-trial', '--fix', '--jobs', '--out', '--reason',
775 '--files', '--shard', '--reruns', '--concurrency', '--backend', '--wall-cap', '--trials', '--class', '--evidence'].map(name => [name, value(name)]));
776 parseCliFlags(rest, { ...handlers, '--approved': () => { flags['--approved'] = '1'; } }, 'Usage: see the header of scripts/ship-measure.ts');
777 const config = readMeasureConfig();
778 const outDir = path.resolve(flags['--out'] ?? path.join(process.cwd(), '.context', 'ship-measure'));
779 if (command === 'table') { console.log(formatKindTable(config)); return 0; }
780 if (command === 'report') { console.log(formatReport(outDir)); return 0; }
781 if (command === 'skip') { console.log(`[ship-measure] ${flags['--case']} labeled unmeasured: ${recordUnmeasured(outDir, flags['--case'] ?? '', flags['--reason'] ?? '')}`); return 0; }
782 if (command === 'free') return runFreeCli(flags, config, outDir, env);
783 if (command !== 'measure' && command !== 'classify' && command !== 'decide' && command !== 'extend') {
784 throw new Error(`Unknown command: ${command ?? '(none)'}. Use table, measure, classify, decide, extend, skip, report, free or sweep.`);
785 }
786 const caseId = flags['--case'];
787 if (!caseId) throw new Error(`${command} needs --case`);
788 const round = flags['--round'] ?? 'baseline';
789 if (command === 'classify') {
790 const trials = (flags['--trials'] ?? '').split(',').map(t => t.trim().replace(/^t/, '')).filter(Boolean).map(t => positive(t, '--trials'));
791 if (!trials.length || !flags['--class']) throw new Error(`classify needs --trials 2,5 and --class (${FAILURE_CLASSES.join(', ')})`);
792 return measurementExit(classifyTrials(outDir, caseId, round, trials, flags['--class'], flags['--evidence'] ?? ''));
793 }
794 if (command === 'decide') return measurementExit(decideRound(outDir, caseId, round));
795 const isGstack = path.resolve(process.cwd()) === ROOT;
796 if (!flags['--command'] && !isGstack) throw new Error('No single-case eval command: pass --command \'<documented command> {case}\' (ask the user once and record the answer).');
797 const runner = flags['--command'] ? commandRunner(flags['--command']) : gstackRunner();
798 const runnerId = flags['--command'] ? `command:${flags['--command']}` : 'gstack';
799 const parallel = flags['--jobs'] ? positive(flags['--jobs'], '--jobs') : Number(env.EVALS_JOBS) || DEFAULT_JOBS;
800 const costPerTrialUsd = flags['--cost-per-trial'] !== undefined ? positive(flags['--cost-per-trial'], '--cost-per-trial', false) : null;
801 if (command === 'extend') {
802 return measurementExit(await extendCase({ caseId, round, config, outDir, runner, parallel, ...(costPerTrialUsd !== null ? { costPerTrialUsd } : {}) }));
803 }
804 const kind = (flags['--kind'] ?? E2E_KINDS[caseId] ?? 'rule') as MeasureKind;
805 if (!['rule', 'behavior', 'judge'].includes(kind)) throw new Error(`--kind must be rule, behavior or judge. Received: ${kind}`);
806 console.log(formatKindTable(config));
807 if (!flags['--command']) {
808 const selected = caseSelectionPreflight(caseId);
809 if (!selected.ok) throw new Error(`--case ${caseId} does not select its test (no paid call made): ${selected.detail}`);
810 console.log(`[ship-measure] preflight: ${selected.detail}`);
811 }
812 const plan = kindPlan(kind, config);
813 const m = await measureCase({
814 caseId, kind, round, config, outDir, fix: flags['--fix'], runner, parallel, costPerTrialUsd, approved: flags['--approved'] === '1',
815 identity: () => measurementIdentity(kind, plan.trials, runnerId),
816 });
817 if (m.status === 'needs_approval') console.log(`[ship-measure] NEEDS APPROVAL ${caseId}: ${m.reason}`);
818 else if (m.status !== 'measured' || m.next === 'stop') console.log(`[ship-measure] NAMED RED ${caseId}: ${m.reason ?? m.decisionReason}`);
819 return measurementExit(m);
820}
822async function runFreeCli(flags: Record<string, string>, config: MeasureConfig, outDir: string, env: NodeJS.ProcessEnv): Promise<number> {
823 const files = flags['--files'] ? flags['--files'].split(',').map(f => f.trim()).filter(Boolean)
824 : flags['--shard'] ? localFreeShardFiles(positive(flags['--shard'], '--shard')) : [];
825 if (!files.length) throw new Error('free needs --files a,b,... (the failing shard\'s exact file list) or --shard I');
826 const reruns = flags['--reruns'] ? positive(flags['--reruns'], '--reruns') : config.ruleTrials;
827 const backend = (flags['--backend'] ?? config.rerunBackend) as RerunBackend;
828 if (backend === 'ubicloud') {
829 if (!env.UBICLOUD_API_KEY) throw new Error('ship_rerun_backend is ubicloud but UBICLOUD_API_KEY is not set; set it or use --backend local');
830 const r = spawnSync('bash', [path.join(ROOT, 'scripts/ubicloud/test-free.sh'), '--diagnostic', '--files', files.join(','), '--reruns', String(reruns)],
831 { stdio: 'inherit', env, timeout: 30 * 60_000 });
832 return r.status ?? 1;
833 }
834 if (backend !== 'local') throw new Error(`--backend must be local or ubicloud. Received: ${backend}`);
835 const m = await measureFreeShard({ files, reruns, outDir, concurrency: flags['--concurrency'] ? positive(flags['--concurrency'], '--concurrency') : 1,
836 wallCapMs: (flags['--wall-cap'] ? positive(flags['--wall-cap'], '--wall-cap') : 600) * 1000 });
837 return m.completed > 0 && m.passed === m.completed ? 0 : 1;
838}
840if (import.meta.main) {
841 main(process.argv.slice(2)).then(code => process.exit(code), (error: Error) => { console.error(`[ship-measure] ${error.message}`); process.exit(4); });
842}