measure-bar.ts
205 lines · scripts/lib/measure-bar.ts at 3f79d81
1/**
2 * The ship-measure measurement bar (approved by Garry 2026-10-06): one pure
3 * decision over a case's diagnostic trials, shared by /ship's measure loop
4 * (scripts/ship-measure.ts) and the weekly off-ship sweep
5 * (scripts/ship-measure-sweep.ts). Nothing here runs a trial or reads a file.
6 *
7 * For N trials per batch (rule and judge N = 10; behavior N = 12, four panels
8 * of 3; a judge trial is one output scored by its median 3-sample panel), the
9 * counts round up from 9/10, 8/10 and 7/10 of N:
10 *
11 * MEETS passes >= 9/10 of N.
12 * MEETS-qualified passes >= 8/10 of N and every failed trial is in a
13 * qualifying class: provider (affirmative provider or
14 * transport evidence only), judge-noise, or model-miss
15 * citing its evidence (path:line).
16 * EXTEND passes >= 7/10 of N without meeting either bar, and no red
17 * has a known fixable cause: run N more trials on identical
18 * inputs, then decide once on the pooled 2N (>= 9/10 strict,
19 * >= 8/10 qualified). There is never a third batch.
20 * BELOW everything else, and any red in a fix-round class
21 * (timeout, hang, regression, fixable) or a contract
22 * violation: a fix round.
23 *
24 * Before the counts: a batch where more than 30% of trials fail with
25 * affirmative provider evidence is void (an outage, not a measurement) and may
26 * be redispatched once; both batches are reported. A batch below strict whose
27 * failed trials are not all classified returns needs-classify when the
28 * classification decides the outcome (passes at or above the 7/10 floor).
29 */
31export type BarKind = 'rule' | 'behavior' | 'judge';
32export type TrialSet = 'initial' | 'redispatch' | 'extend' | 'extend-redispatch';
33export const QUALIFYING_CLASSES = ['provider', 'judge-noise', 'model-miss'] as const;
34export const FIX_ROUND_CLASSES = ['timeout', 'hang', 'regression', 'fixable'] as const;
35export const FAILURE_CLASSES = [...QUALIFYING_CLASSES, ...FIX_ROUND_CLASSES, 'unclassified'] as const;
36export type FailureClass = (typeof FAILURE_CLASSES)[number];
38/** A batch is void when more than this share of its trials fail on affirmative provider evidence. */
39export const VOID_SHARE = { num: 3, den: 10 } as const;
41export interface BarTrial {
42 trial: number;
43 set: TrialSet;
44 passed: boolean;
45 contract?: boolean;
46 /** Judge outputs: each sample's pass; the output passes at 2 of 3. */
47 samples?: boolean[];
48 failureCause?: string;
49 failureDetail?: string;
50 failureEvidence?: string;
51}
53export interface Classification { trial: number; class: FailureClass; evidence: string; by: 'machine' | 'agent'; at: string }
55export type BarDecision = 'MEETS' | 'MEETS-qualified' | 'EXTEND' | 'BELOW' | 'needs-classify' | 'void' | 'incomplete';
56export type BarNext = 'none' | 'redispatch' | 'extend' | 'classify' | 'fix' | 'stop';
58export interface BarThresholds { trials: number; strict: number; qualified: number; floor: number }
59export interface SetSummary { set: TrialSet; trials: number; passes: number; providerFailures: number; void: boolean }
61export interface BarOutcome {
62 decision: BarDecision;
63 next: BarNext;
64 passes: number;
65 thresholds: BarThresholds;
66 pooled: boolean;
67 sets: SetSummary[];
68 /** Failed counted trials that still need a class. */
69 needsClassify: number[];
70 reason: string;
71}
73const atLeast = (n: number, num: number) => Math.ceil((n * num) / 10);
75/** Strict 9/10, qualified 8/10 and extend floor 7/10 of n, rounded up (10: 9/8/7; 12: 11/10/9; 20: 18/16/14; 24: 22/20/17). */
76export function barThresholds(n: number): BarThresholds {
77 return { trials: n, strict: atLeast(n, 9), qualified: atLeast(n, 8), floor: atLeast(n, 7) };
78}
80/** A trial's pass under the bar: a judge output at 2 of its 3 samples; a contract violation never passes. */
81export function trialPassed(kind: BarKind, t: BarTrial): boolean {
82 if (t.contract) return false;
83 if (kind === 'judge' && t.samples) return t.samples.length === 3 && t.samples.filter(Boolean).length >= 2;
84 return t.passed;
85}
87const PROVIDER_TEXT = /\b(?:429|5\d\d)\b|rate[ _-]?limit|overloaded|api_error|ECONNRESET|connection reset|socket hang up|ETIMEDOUT|EAI_AGAIN|service unavailable|bad gateway/i;
88const TIMEOUT_CAUSES = new Set(['session_timeout', 'observer_timeout', 'provider_stall']);
90/**
91 * The affirmative provider or transport evidence for a failed trial, or null.
92 * `api_error` is provider evidence by itself; `pre_turn_infra` only when its
93 * evidence or detail names a 429/5xx, rate limit, overload or connection reset
94 * (a crash or module-load failure is a harness defect, never an outage).
95 */
96export function providerEvidence(t: BarTrial): string | null {
97 if (t.passed && !t.contract) return null;
98 const text = [t.failureEvidence, t.failureDetail].filter(Boolean).join(' ');
99 if (t.failureCause === 'api_error') return text || 'failure_cause api_error';
100 if (t.failureCause === 'pre_turn_infra' && PROVIDER_TEXT.test(text)) return text;
101 return null;
102}
104/** The class a failed trial gets without review: provider on affirmative evidence; timeout or hang from its cause. */
105export function machineClass(t: BarTrial): FailureClass | null {
106 if (providerEvidence(t)) return 'provider';
107 if (t.failureCause === 'provider_stall') return 'hang';
108 if (t.failureCause && TIMEOUT_CAUSES.has(t.failureCause)) return 'timeout';
109 return null;
110}
112const CITATION = /[\w./-]+:\d+/;
114/** Why an agent classification is refused, or null when it is accepted. */
115export function classificationProblem(t: BarTrial | undefined, kind: BarKind, cls: string, evidence: string): string | null {
116 if (!t) return 'no such trial in this measurement';
117 if (trialPassed(kind, t)) return `trial ${t.trial} passed; only failed trials are classified`;
118 if (!(FAILURE_CLASSES as readonly string[]).includes(cls)) return `class must be one of ${FAILURE_CLASSES.join(', ')}. Received: ${cls}`;
119 if (!evidence.trim()) return 'every classification needs --evidence (what you read: a capture path:line, a failure line, a transcript line)';
120 if (cls === 'provider' && !providerEvidence(t)) {
121 return `provider needs affirmative provider or transport evidence (failure_cause api_error, or pre_turn_infra naming a 429/5xx, rate limit or connection reset); trial ${t.trial}'s failure_cause is ${t.failureCause ?? 'unrecorded'}`;
122 }
123 if ((QUALIFYING_CLASSES as readonly string[]).includes(cls) && t.failureCause && TIMEOUT_CAUSES.has(t.failureCause)) {
124 return `trial ${t.trial} failed on ${t.failureCause}: a timeout or hang never qualifies; it goes to a fix round`;
125 }
126 return null;
127}
129/** Each failed trial's effective class: its latest agent record, else its machine class. */
130export function effectiveClasses(trials: readonly BarTrial[], records: readonly Classification[]): Map<number, Classification> {
131 const out = new Map<number, Classification>();
132 for (const t of trials) {
133 const cls = machineClass(t);
134 if (cls) out.set(t.trial, { trial: t.trial, class: cls, evidence: providerEvidence(t) ?? t.failureCause ?? '', by: 'machine', at: '' });
135 }
136 for (const r of records) if (r.by === 'agent') out.set(r.trial, r);
137 return out;
138}
140const qualifies = (c: Classification) => (QUALIFYING_CLASSES as readonly string[]).includes(c.class) && (c.class !== 'model-miss' || CITATION.test(c.evidence));
141const fixRound = (c: Classification) => (FIX_ROUND_CLASSES as readonly string[]).includes(c.class);
143function summarize(kind: BarKind, set: TrialSet, trials: BarTrial[]): SetSummary {
144 const providerFailures = trials.filter(t => !trialPassed(kind, t) && providerEvidence(t)).length;
145 return { set, trials: trials.length, passes: trials.filter(t => trialPassed(kind, t)).length, providerFailures,
146 void: trials.length > 0 && providerFailures * VOID_SHARE.den > trials.length * VOID_SHARE.num };
147}
149/** The counted batch of a pair (first try, its one redispatch), or a void/incomplete stop. */
150function countedBatch(n: number, first: SetSummary, retry: SetSummary, firstTrials: BarTrial[], retryTrials: BarTrial[], label: string):
151 { trials: BarTrial[] } | { stop: Pick<BarOutcome, 'decision' | 'next' | 'reason'> } {
152 const use = first.void ? retry : first;
153 if (use.trials === 0) return { stop: { decision: 'void', next: 'redispatch', reason: `${label} batch is void: ${first.providerFailures} of ${first.trials} trials failed on provider evidence (> 30%); redispatch it once` } };
154 if (use.void) return { stop: { decision: 'void', next: 'stop', reason: `${label} batch and its one redispatch are both void (provider evidence in ${first.providerFailures}/${first.trials} and ${retry.providerFailures}/${retry.trials}); stop with a named infrastructure red` } };
155 if (use.trials < n) return { stop: { decision: 'incomplete', next: 'stop', reason: `${label} batch ran ${use.trials} of ${n} trials` } };
156 return { trials: first.void ? retryTrials : firstTrials };
157}
159/**
160 * The bar's decision over a measurement's trials and its classification
161 * records (append-only; the latest agent record for a trial wins).
162 */
163export function decide(kind: BarKind, n: number, trials: readonly BarTrial[], records: readonly Classification[] = []): BarOutcome {
164 const of = (set: TrialSet) => trials.filter(t => t.set === set);
165 const sets = (['initial', 'redispatch', 'extend', 'extend-redispatch'] as const).map(set => summarize(kind, set, of(set)));
166 const [initial, redispatch, extend, extendRedispatch] = sets as [SetSummary, SetSummary, SetSummary, SetSummary];
167 const reported = sets.filter(s => s.trials > 0);
168 const base = countedBatch(n, initial, redispatch, of('initial'), of('redispatch'), 'the first');
169 const empty = { passes: 0, pooled: false, sets: reported, needsClassify: [] as number[] };
170 if ('stop' in base) return { ...empty, thresholds: barThresholds(n), ...base.stop };
171 const pooled = extend.trials > 0;
172 const ext = pooled ? countedBatch(n, extend, extendRedispatch, of('extend'), of('extend-redispatch'), 'the extension') : { trials: [] };
173 if ('stop' in ext) return { ...empty, pooled, thresholds: barThresholds(2 * n), ...ext.stop };
174 const counted = [...base.trials, ...ext.trials];
175 const thresholds = barThresholds(counted.length);
176 const passes = counted.filter(t => trialPassed(kind, t)).length;
177 const failed = counted.filter(t => !trialPassed(kind, t));
178 const classes = effectiveClasses(failed, records);
179 const result = (decision: BarDecision, next: BarNext, reason: string, needsClassify: number[] = []): BarOutcome =>
180 ({ decision, next, passes, thresholds, pooled, sets: reported, needsClassify, reason });
181 const count = `${passes}/${counted.length}${pooled ? ' pooled' : ''}`;
182 const contract = failed.find(t => t.contract);
183 if (contract) return result('BELOW', 'fix', `${count}; trial ${contract.trial} violated a contract: a fix round, whatever the count`);
184 if (passes >= thresholds.strict) return result('MEETS', 'none', `${count} meets the strict bar (${thresholds.strict}/${counted.length})`);
185 if (passes < (pooled ? thresholds.qualified : thresholds.floor)) {
186 return result('BELOW', 'fix', `${count} is under the ${pooled ? `qualified pooled bar (${thresholds.qualified}/${counted.length})` : `extend floor (${thresholds.floor}/${counted.length})`}: a fix round`);
187 }
188 const unclassified = failed.filter(t => !classes.has(t.trial)).map(t => t.trial);
189 if (unclassified.length) return result('needs-classify', 'classify', `${count} is below strict; classify trial(s) ${unclassified.map(t => `t${t}`).join(', ')}, then decide`, unclassified);
190 const fix = failed.find(t => fixRound(classes.get(t.trial)!));
191 if (fix) return result('BELOW', 'fix', `${count}; trial ${fix.trial} is ${classes.get(fix.trial)!.class}, which never qualifies: a fix round`);
192 if (passes >= thresholds.qualified && failed.every(t => qualifies(classes.get(t.trial)!))) {
193 return result('MEETS-qualified', 'none', `${count} meets the qualified bar (${thresholds.qualified}/${counted.length}); every red qualifies (${failed.map(t => `t${t.trial} ${classes.get(t.trial)!.class}`).join(', ')})`);
194 }
195 if (pooled) return result('BELOW', 'fix', `${count} does not meet the pooled bar (${thresholds.strict} strict, or ${thresholds.qualified} with every red qualifying); there is no third batch: a fix round`);
196 return result('EXTEND', 'extend', `${count} is at or above the extend floor (${thresholds.floor}/${counted.length}) without meeting a bar: run ${n} more trials on identical inputs, then decide once on the pooled ${2 * n}`);
197}
199/** The first identity field that differs between two batches, or null when they may pool. */
200export function identityMismatch(a: Record<string, string>, b: Record<string, string>): string | null {
201 for (const key of [...new Set([...Object.keys(a), ...Object.keys(b)])].sort()) {
202 if (a[key] !== b[key]) return `${key} changed (${a[key] ?? 'absent'} -> ${b[key] ?? 'absent'})`;
203 }
204 return null;
205}