cso-eval.ts
984 lines · scripts/cso-eval.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/** Trusted corpus preparation, producer receipt collection, and objective result accounting. */
3import { createHash } from 'node:crypto';
4import { execFileSync } from 'node:child_process';
5import { existsSync, lstatSync, mkdirSync, readdirSync, realpathSync } from 'node:fs';
6import { dirname, join, resolve } from 'node:path';
7import { readBoundedStable } from '../lib/cso/bounded-file';
8import { canonical } from '../lib/cso/contracts';
9import { validateRuntimeCatalog, type RuntimeCatalog } from '../lib/cso/runtime-catalog';
10import { EMPTY_GLOBAL_GIT_CONFIG, executable } from '../lib/cso/process';
11import { atomicWriteSync } from '../lib/fs-atomic';
12import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorpusManifest, type CorpusManifest, type EvalCorpus, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize';
13import {
14 PRODUCER_PLATFORMS,
15 producerArtifactInventoryHash,
16 producerExecution,
17 producerInputHash,
18 producerInstallationIdentityHash,
19 producerProviderIdentityHash,
20 producerReceiptHash,
21 sha256,
22 type ProducerCell,
23 type ProducerArtifactInventory,
24 type ProducerHost,
25 type ProducerInput,
26 type ProducerInstallationIdentity,
27 type ProducerPlatform,
28 type ProducerProviderIdentity,
29 type ProducerReceipt,
30 type ProducerReceiptIndex,
31 type ProducerSourceEntry,
32} from './cso-eval-protocol';
34export type EvalVersion = 'v2' | 'v3';
35export type EvalMode = 'daily' | 'comprehensive';
36export type Outcome = 'passed' | 'failed' | 'blocked' | 'not_attempted';
37/**
38 * full: matched v2/v3 × 3 repetitions on one producer installation.
39 * release: v3 only, 1 repetition, each cell pinned to a native platform.
40 * baseline: v2 comprehensive only, 1 repetition; scored into a reusable EvalBaseline.
41 */
42export type EvalProfile = 'full' | 'release' | 'baseline';
43export type EvalPlatform = ProducerPlatform;
44type EvalCorpusManifest = CorpusManifest<string>;
45export interface EvalCell extends ProducerCell {
46 id: string; caseId: string; stack: EvalStack; variant: EvalVariant; version: EvalVersion; mode: EvalMode;
47 repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; platform?: EvalPlatform;
48}
49interface EvalMatrixBase {
50 schemaVersion: 1; corpusVersion: string; corpusHash: string; model: string; host: ProducerHost;
51 budgets: { daily: number; comprehensive: number }; cells: EvalCell[];
52}
53export interface FullEvalMatrix extends EvalMatrixBase { profile?: undefined; skillHashes: { v2: string; v3: string }; repetitions: 3 }
54export interface ReleaseEvalMatrix extends EvalMatrixBase {
55 profile: 'release'; skillHashes: { v3: string }; repetitions: 1;
56 /** Daily cells run on one platform; each comprehensive case is fixed to one platform. */
57 platforms: { daily: EvalPlatform; comprehensive: Record<string, EvalPlatform> };
58 /** Pre-registered external v2 baseline, or null when the regression gate stays unmeasured. */
59 baselineHash: string | null;
60}
61export interface BaselineEvalMatrix extends EvalMatrixBase {
62 profile: 'baseline'; skillHashes: { v2: string }; repetitions: 1;
63 platforms: { comprehensive: Record<string, EvalPlatform> };
64}
65export type EvalMatrix = FullEvalMatrix | ReleaseEvalMatrix | BaselineEvalMatrix;
66interface EvalMatrixCommonOptions { model: string; host: ProducerHost; budgets?: { daily: number; comprehensive: number } }
67export interface FullEvalMatrixOptions extends EvalMatrixCommonOptions { profile?: 'full'; skillHashes: { v2: string; v3: string } }
68export interface ReleaseEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'release'; skillHashes: { v3: string }; platforms: ReleaseEvalMatrix['platforms']; baseline?: EvalBaseline }
69export interface BaselineEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'baseline'; skillHashes: { v2: string }; platforms: BaselineEvalMatrix['platforms'] }
70export type EvalMatrixOptions = FullEvalMatrixOptions | ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions;
71/**
72 * A scored v2 comprehensive run, reused by release matrices while every bound
73 * input is unchanged. Release matrices pin its hash before results exist.
74 */
75export interface EvalBaseline {
76 schemaVersion: 1; version: 'v2';
77 corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; budgetSeconds: number; skillHash: string;
78 platforms: Record<string, EvalPlatform>;
79 matrixHash: string; producerBatchHash: string; modelUsed: string;
80 provider: { family: ProducerProviderIdentity['family']; policyRevision: string; version: string };
81 /** One entry per corpus case, in manifest order: whether the vulnerable cell found the seeded issue. */
82 cases: Array<{ caseId: string; found: boolean }>;
83 baselineHash: string;
84}
85export interface PortableSkillFile {
86 path: string;
87 contents: string;
88}
89export interface PortableSkillFileIdentity {
90 path: string;
91 sha256: string;
92 bytes: number;
93}
94export interface PortableSkillPayloadIdentity {
95 schemaVersion: 1;
96 version: EvalVersion;
97 files: PortableSkillFileIdentity[];
98}
99export interface EvalFinding {
100 id: string; evidence: 'supported' | 'hypothesis' | 'legacy_review'; claimedTested: boolean;
101 /**
102 * Written only by the independent evaluator after reviewing the reported trace. `correct`: the seeded issue.
103 * `incidental`: a different issue a unanimous judge panel confirmed is real in this source; a precision true
104 * positive, never recall. `instrumentation`: it targets the evaluator's declared fixture instrumentation
105 * (canary overlay files, egress probe), so precision leaves it out of both terms.
106 */
107 judgment: 'correct' | 'incidental' | 'instrumentation' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string;
108 /** `incidental` only: the evaluator's opaque key for the confirmed issue, so restatements count once per cell. */
109 incidentalKey?: string;
110 /**
111 * Written only by the independent evaluator. A producer's tested claim is
112 * valid only when this finding, rather than merely another finding in the
113 * same corpus cell, owns the trusted repair and recheck observations.
114 */
115 trustedVerification?: {
116 repair: Outcome;
117 repairEvidenceHash?: string;
118 recheck: Outcome;
119 recheckEvidenceHash?: string;
120 };
121 /**
122 * Written only by the independent evaluator, for a tested claim on an `incidental` finding: the result of
123 * replaying the helper's signed repair bundle for that finding against the trusted source. `passed` (with the
124 * replay's evidence hash) certifies the claim; `failed` or `missing` (no bound bundle) leaves it falsely tested.
125 * It never counts toward recall, repair or held-out rates, which need the seeded issue.
126 */
127 trustedReplay?: { result: 'passed' | 'failed' | 'missing'; replayEvidenceHash?: string };
128}
129export interface EvalResult {
130 cellId: string; sourceHash: string; skillHash: string; model: string; host: string; budgetSeconds: number;
131 /**
132 * `reportFinished`: the helper report's status is `finished`; its findings and runtime outcomes count whatever its
133 * completeness, and the report gates require it. `reportComplete`: it is also `complete` (no gap, every domain
134 * assessed), measured as its own ungated rate because the corpus fixtures carry gaps by design.
135 */
136 reportPresent: boolean; reportFinished: boolean; reportComplete: boolean; findings: EvalFinding[];
137 setup: Outcome; reproduction: Outcome; repair: Outcome; recheck: Outcome;
138 /** Hash of trusted private observations; never a producer's claimed verification status. */
139 oracleEvidenceHash?: string;
140 oracleVersion?: string;
141 heldOutAssertionsPassed: boolean;
142 currentSourceHash?: string;
143 recheckEvidenceHash?: string;
144 freshRecheck: boolean;
145 latencyMs: number;
146 firstUsefulResultMs: number | null;
147 /** Required by the release scorer; omitted only by isolated accounting tests. */
148 producerReceiptHash?: string;
149 usage?: { source: 'host'; tokens?: number; costUSD?: number };
150 prerequisite?: string;
151}
152export interface Rate { numerator: number; denominator: number; value: number | null }
153export interface ReleaseQualification {
154 containment: Record<string, 'passed' | 'failed' | 'not_run'>;
155}
156export interface PreparedEvalSchedule {
157 schemaVersion: 1;
158 matrixHash: string;
159 scheduledCells: number;
160 preparedCells: number;
161 jobs: Array<{ cellId: string; relativePath: string; inputHash: string }>;
162}
163export interface ProducerGroupSummary {
164 version: EvalVersion;
165 mode: EvalMode;
166 scheduled: number;
167 submitted: number;
168 missing: number;
169 succeeded: number;
170 failed: number;
171 latency: { samples: number; denominator: number; p95Ms: number | null };
172 firstUsefulResult: { samples: 0; denominator: number; p95Ms: null };
173 modelTokens: { measured: number; denominator: number; total: number | null };
174 estimatedCost: { measured: number; denominator: number; totalUSD: number | null; source: 'pricing-table-estimate' };
175}
176export interface ProducerBatch {
177 schemaVersion: 1;
178 matrixHash: string;
179 scheduleHash: string;
180 receipts: ProducerReceiptIndex[];
181 summary: {
182 scheduled: number;
183 prepared: number;
184 submitted: number;
185 missing: number;
186 matchedPairsExpected: number;
187 matchedPairsSubmitted: number;
188 modelMismatches: Array<{ pair: string; v2: string; v3: string }>;
189 groups: ProducerGroupSummary[];
190 note: string;
191 };
192 /** Present only when platforms embedded catalogs that differ in qualification time alone; see assertPlatformIdentities. */
193 embeddedCatalogs?: RuntimeCatalog[];
194 batchHash: string;
195}
196export const REQUIRED_CONTAINMENT = [
197 'hostile-startup-configuration', 'environment-canary', 'remote-docker-context', 'ipv4-egress', 'ipv6-egress', 'dns-egress',
198 'archive-traversal-and-poisoning', 'symlink-traversal', 'split-output-secrets', 'concurrent-admission', 'expired-snapshot-replay',
199 'watchdog-survival', 'failed-report-writes', 'held-out-oracle-visibility',
200] as const;
201const digest = (value: string) => createHash('sha256').update(value).digest('hex');
202const HEX = /^[a-f0-9]{64}$/;
203const rate = (numerator: number, denominator: number): Rate => ({ numerator, denominator, value: denominator ? numerator / denominator : null });
204const validNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value >= 0;
205const PORTABLE_PAYLOAD_HEADER = '<!-- GSTACK_CSO_EVAL_PAYLOAD ';
206const PORTABLE_PAYLOAD_HEADER_END = ' -->\n';
207const PORTABLE_FILE_END = '\n<<<END_GSTACK_CSO_EVAL_FILE>>>\n';
208const PORTABLE_FILE_LIMIT = 2 * 1024 * 1024;
209const PORTABLE_PAYLOAD_LIMIT = 4 * 1024 * 1024;
211function portablePayloadPreamble(version: EvalVersion): string {
212 return `# Complete portable CSO evaluation instructions (${version})\n\nThis package is the sole CSO instruction input for this evaluation cell. Apply the embedded SKILL.md and every embedded manifest-listed section. Resolve any SKILL.md reference to cso/sections/<file> or sections/<file> from the matching embedded file below. Do not read an installed, user-home, repository, or other-version CSO skill or section. The file bodies are exact generated bytes; package markers are transport metadata.\n\n`;
213}
215function portableFileOpen(identity: PortableSkillFileIdentity): string {
216 return `<<<GSTACK_CSO_EVAL_FILE ${JSON.stringify(identity)}>>>\n`;
217}
219function parsePortableManifest(contents: string): { files: string[] } {
220 let manifest: any;
221 try { manifest = JSON.parse(contents); } catch { throw new Error('INVALID_CSO_EVAL_PAYLOAD_MANIFEST'); }
222 if (!manifest || manifest.skill !== 'cso' || manifest.version !== 1 || !Array.isArray(manifest.sections) || manifest.sections.length === 0) throw new Error('INVALID_CSO_EVAL_PAYLOAD_MANIFEST');
223 const files: string[] = [];
224 const ids = new Set<string>();
225 for (const section of manifest.sections) {
226 if (!section || typeof section.id !== 'string' || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(section.id) || ids.has(section.id) || typeof section.file !== 'string' || !/^[a-z0-9]+(?:-[a-z0-9]+)*\.md$/.test(section.file) || files.includes(section.file) || typeof section.title !== 'string' || !section.title.trim() || typeof section.trigger !== 'string' || !section.trigger.trim()) throw new Error('INVALID_CSO_EVAL_PAYLOAD_MANIFEST');
227 ids.add(section.id); files.push(section.file);
228 }
229 return { files };
230}
232function validatePortableSkillFiles(version: EvalVersion, files: PortableSkillFile[]): PortableSkillFile[] {
233 if (!['v2', 'v3'].includes(version) || !Array.isArray(files)) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
234 const byPath = new Map<string, string>();
235 for (const file of files) {
236 if (!file || typeof file.path !== 'string' || typeof file.contents !== 'string' || byPath.has(file.path) || Buffer.byteLength(file.contents) > PORTABLE_FILE_LIMIT || file.contents.includes(PORTABLE_FILE_END.trim())) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
237 byPath.set(file.path, file.contents);
238 }
239 const root = byPath.get('SKILL.md'), manifest = byPath.get('sections/manifest.json');
240 if (root === undefined || manifest === undefined) throw new Error('INCOMPLETE_CSO_EVAL_PAYLOAD');
241 const frontmatter = root.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)?.[1];
242 const versionMatch = frontmatter?.match(/^version:\s*([0-9]+)(?:\.[0-9]+){0,2}\s*$/m);
243 if (!versionMatch || Number(versionMatch[1]) !== Number(version.slice(1))) throw new Error('CSO_EVAL_PAYLOAD_VERSION_MISMATCH');
244 const sectionFiles = parsePortableManifest(manifest).files;
245 const expectedPaths = ['SKILL.md', 'sections/manifest.json', ...sectionFiles.map(file => `sections/${file}`)];
246 if (byPath.size !== expectedPaths.length || expectedPaths.some(path => !byPath.has(path))) throw new Error('INCOMPLETE_CSO_EVAL_PAYLOAD');
247 return expectedPaths.map(path => ({ path, contents: byPath.get(path)! }));
248}
250/** Render the only portable instruction format accepted by matrix preparation. */
251export function createPortableSkillPayload(version: EvalVersion, inputFiles: PortableSkillFile[]): string {
252 const files = validatePortableSkillFiles(version, inputFiles);
253 const identities = files.map(file => ({ path: file.path, sha256: sha256(file.contents), bytes: Buffer.byteLength(file.contents) }));
254 const identity: PortableSkillPayloadIdentity = { schemaVersion: 1, version, files: identities };
255 let payload = `${PORTABLE_PAYLOAD_HEADER}${JSON.stringify(identity)}${PORTABLE_PAYLOAD_HEADER_END}${portablePayloadPreamble(version)}`;
256 for (let index = 0; index < files.length; index++) payload += `${portableFileOpen(identities[index])}${files[index].contents}${PORTABLE_FILE_END}`;
257 if (Buffer.byteLength(payload) > PORTABLE_PAYLOAD_LIMIT) throw new Error('CSO_EVAL_PAYLOAD_TOO_LARGE');
258 return payload;
259}
261/** Validate canonical serialization and return the identities bound by its hash. */
262export function validatePortableSkillPayload(payload: string, expectedVersion?: EvalVersion): PortableSkillPayloadIdentity {
263 if (typeof payload !== 'string' || Buffer.byteLength(payload) > PORTABLE_PAYLOAD_LIMIT || !payload.startsWith(PORTABLE_PAYLOAD_HEADER)) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
264 const headerEnd = payload.indexOf(PORTABLE_PAYLOAD_HEADER_END, PORTABLE_PAYLOAD_HEADER.length);
265 if (headerEnd < 0) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
266 let identity: PortableSkillPayloadIdentity;
267 try { identity = JSON.parse(payload.slice(PORTABLE_PAYLOAD_HEADER.length, headerEnd)); } catch { throw new Error('INVALID_CSO_EVAL_PAYLOAD'); }
268 if (identity?.schemaVersion !== 1 || !['v2', 'v3'].includes(identity.version) || (expectedVersion && identity.version !== expectedVersion) || !Array.isArray(identity.files)) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
269 let cursor = headerEnd + PORTABLE_PAYLOAD_HEADER_END.length;
270 const preamble = portablePayloadPreamble(identity.version);
271 if (payload.slice(cursor, cursor + preamble.length) !== preamble) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
272 cursor += preamble.length;
273 const files: PortableSkillFile[] = [];
274 for (const file of identity.files) {
275 if (!file || typeof file.path !== 'string' || !HEX.test(file.sha256) || !Number.isSafeInteger(file.bytes) || file.bytes < 0 || file.bytes > PORTABLE_FILE_LIMIT) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
276 const open = portableFileOpen(file);
277 if (payload.slice(cursor, cursor + open.length) !== open) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
278 cursor += open.length;
279 const end = payload.indexOf(PORTABLE_FILE_END, cursor);
280 if (end < 0) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
281 const contents = payload.slice(cursor, end);
282 if (Buffer.byteLength(contents) !== file.bytes || sha256(contents) !== file.sha256) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
283 files.push({ path: file.path, contents }); cursor = end + PORTABLE_FILE_END.length;
284 }
285 if (cursor !== payload.length || createPortableSkillPayload(identity.version, files) !== payload) throw new Error('INVALID_CSO_EVAL_PAYLOAD');
286 return identity;
287}
289/** Read one generated CSO skill tree and package every manifest-listed section. */
290export function loadPortableSkillPayload(version: EvalVersion, skillDirectory: string): string {
291 const root = resolve(skillDirectory), stat = lstatSync(root);
292 if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(root) !== root) throw new Error('UNSAFE_CSO_EVAL_SKILL_DIRECTORY');
293 const read = (relativePath: string) => {
294 const bytes = readBoundedStable(join(root, ...relativePath.split('/')), PORTABLE_FILE_LIMIT, `CSO ${version} ${relativePath}`);
295 const contents = bytes.toString('utf8');
296 if (!Buffer.from(contents).equals(bytes)) throw new Error('INVALID_CSO_EVAL_PAYLOAD_ENCODING');
297 return contents;
298 };
299 const rootSkill = read('SKILL.md'), manifest = read('sections/manifest.json');
300 const sectionFiles = parsePortableManifest(manifest).files;
301 const sectionsRoot = join(root, 'sections'), sectionStat = lstatSync(sectionsRoot);
302 if (!sectionStat.isDirectory() || sectionStat.isSymbolicLink() || realpathSync(sectionsRoot) !== sectionsRoot) throw new Error('UNSAFE_CSO_EVAL_SKILL_DIRECTORY');
303 const listed = new Set(sectionFiles);
304 const inventory = () => readdirSync(sectionsRoot, { withFileTypes: true }).filter(entry => entry.name.endsWith('.md')).map(entry => {
305 if (!entry.isFile() || entry.isSymbolicLink() || !listed.has(entry.name)) throw new Error('UNLISTED_CSO_EVAL_SECTION');
306 return entry.name;
307 }).sort();
308 const firstInventory = inventory();
309 const files = [
310 { path: 'SKILL.md', contents: rootSkill },
311 { path: 'sections/manifest.json', contents: manifest },
312 ...sectionFiles.map(file => ({ path: `sections/${file}`, contents: read(`sections/${file}`) })),
313 ];
314 if (JSON.stringify(inventory()) !== JSON.stringify(firstInventory) || files.some(file => read(file.path) !== file.contents)) throw new Error('CSO_EVAL_SKILL_CHANGED_DURING_PACKAGING');
315 return createPortableSkillPayload(version, files);
316}
318function matrixHash(matrix: EvalMatrix): string { return digest(JSON.stringify(matrix)); }
320function safeWriteNew(path: string, value: unknown): void {
321 const target = resolve(path), parent = dirname(target), stat = lstatSync(parent);
322 if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(parent) !== parent) throw new Error('UNSAFE_EVAL_DESTINATION');
323 atomicWriteSync(target, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, noReplace: true });
324}
326function readJsonBounded(path: string, max: number, label: string): any {
327 try { return JSON.parse(readBoundedStable(resolve(path), max, label).toString('utf8')); }
328 catch (error) { if (error instanceof SyntaxError) throw new Error(`INVALID_${label.toUpperCase().replaceAll(' ', '_')}`); throw error; }
329}
331function initializeFixtureRepository(path: string): void {
332 const nullPath = process.platform === 'win32' ? 'NUL' : '/dev/null';
333 const env = {
334 PATH: process.env.PATH ?? '', LANG: 'C', LC_ALL: 'C',
335 GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: EMPTY_GLOBAL_GIT_CONFIG, GIT_ATTR_NOSYSTEM: '1',
336 GIT_AUTHOR_NAME: 'CSO Eval', GIT_AUTHOR_EMAIL: 'cso-eval@invalid',
337 GIT_COMMITTER_NAME: 'CSO Eval', GIT_COMMITTER_EMAIL: 'cso-eval@invalid',
338 };
339 const git = executable('git');
340 // Prepared sources are copied immediately. A detached auto-maintenance child
341 // can remove .git/objects/maintenance.lock while that copy is in progress.
342 const safe = ['-c', 'maintenance.auto=false', '-c', 'gc.auto=0',
343 '-c', `core.hooksPath=${nullPath}`, '-c', `core.attributesFile=${nullPath}`, '-c', `core.excludesFile=${nullPath}`];
344 execFileSync(git, [...safe, 'init', '--quiet', '--initial-branch=main'], { cwd: path, env, stdio: 'ignore' });
345 execFileSync(git, [...safe, 'add', '--all'], { cwd: path, env, stdio: 'ignore' });
346 execFileSync(git, [...safe, 'commit', '--quiet', '-m', 'immutable evaluation fixture'], { cwd: path, env, stdio: 'ignore' });
347}
349function sourceEntries(corpus: EvalCorpus, caseId: string, variant: EvalVariant): ProducerSourceEntry[] {
350 return Object.entries(corpus.sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) }));
351}
353/**
354 * Prepare independent one-cell jobs. The returned schedule is trusted
355 * evaluator metadata; transfer one job at a time and never give the schedule
356 * or sibling jobs to the producing agent.
357 */
358export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial<Record<EvalVersion, string>>, destination: string, selectedCellIds?: string[], corpus: EvalCorpus = publicCorpus()): PreparedEvalSchedule {
359 validateMatrix(matrix, corpus.manifest);
360 const skillHashes: Partial<Record<EvalVersion, string>> = matrix.skillHashes;
361 const versions = (['v2', 'v3'] as const).filter(version => skillHashes[version] !== undefined);
362 if (!payloads || Object.keys(payloads).sort().join(',') !== versions.join(',')) throw new Error('EVAL_SKILL_HASH_MISMATCH');
363 for (const version of versions) {
364 validatePortableSkillPayload(payloads[version]!, version);
365 if (sha256(payloads[version]!) !== skillHashes[version]) throw new Error('EVAL_SKILL_HASH_MISMATCH');
366 }
367 const root = resolve(destination);
368 if (existsSync(root)) throw new Error('EVAL_DESTINATION_EXISTS');
369 const parent = dirname(root);
370 if (realpathSync(parent) !== parent || !lstatSync(parent).isDirectory()) throw new Error('UNSAFE_EVAL_DESTINATION');
371 const selected = selectedCellIds === undefined ? matrix.cells : selectedCellIds.map(id => {
372 const cell = matrix.cells.find(item => item.id === id);
373 if (!cell) throw new Error('UNKNOWN_EVAL_CELL');
374 return cell;
375 });
376 if (new Set(selected.map(cell => cell.id)).size !== selected.length) throw new Error('DUPLICATE_EVAL_CELL');
377 mkdirSync(root, { mode: 0o700 });
378 const jobsRoot = join(root, 'jobs'); mkdirSync(jobsRoot, { mode: 0o700 });
379 const jobs: PreparedEvalSchedule['jobs'] = [];
380 for (const cell of selected) {
381 const jobRoot = join(jobsRoot, cell.id); mkdirSync(jobRoot, { mode: 0o700 });
382 const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus);
383 if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH');
384 initializeFixtureRepository(source.path);
385 const input: ProducerInput = { schemaVersion: 2, cellRef: cell.id, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant), execution: producerExecution(cell) };
386 const inputHash = producerInputHash(input);
387 safeWriteNew(join(jobRoot, 'producer-input.json'), input);
388 jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash });
389 }
390 const schedule: PreparedEvalSchedule = { schemaVersion: 1, matrixHash: matrixHash(matrix), scheduledCells: matrix.cells.length, preparedCells: jobs.length, jobs };
391 safeWriteNew(join(root, 'schedule.json'), schedule);
392 return schedule;
393}
395function validateSchedule(matrix: EvalMatrix, schedule: PreparedEvalSchedule): void {
396 if (schedule?.schemaVersion !== 1 || schedule.matrixHash !== matrixHash(matrix) || schedule.scheduledCells !== matrix.cells.length || !Array.isArray(schedule.jobs) || schedule.preparedCells !== schedule.jobs.length) throw new Error('INVALID_EVAL_SCHEDULE');
397 const known = new Set(matrix.cells.map(cell => cell.id));
398 const seen = new Set<string>();
399 for (const job of schedule.jobs) {
400 if (!job || !known.has(job.cellId) || seen.has(job.cellId) || job.relativePath !== `jobs/${job.cellId}` || !HEX.test(job.inputHash)) throw new Error('INVALID_EVAL_SCHEDULE');
401 seen.add(job.cellId);
402 }
403}
405function validateReceipt(receipt: ProducerReceipt, cell: EvalCell, inputHash: string): void {
406 if (receipt?.schemaVersion !== 2 || receipt.cellRef !== cell.id || 'cell' in receipt || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
407 const { receiptHash, ...withoutHash } = receipt;
408 if (producerReceiptHash(withoutHash) !== receiptHash) throw new Error('INVALID_PRODUCER_RECEIPT');
409 const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
410 if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started) throw new Error('INVALID_PRODUCER_RECEIPT');
411 if (!receipt.usage || !['inputTokens', 'outputTokens', 'cachedTokens', 'estimatedCostUSD'].every(field => receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage]))) throw new Error('INVALID_PRODUCER_RECEIPT');
412 const hasError = !!receipt.error;
413 if ((receipt.status === 'failed') !== hasError || (hasError && (!receipt.error!.code || typeof receipt.error!.reason !== 'string'))) throw new Error('INVALID_PRODUCER_RECEIPT');
414 validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_RECEIPT');
415 validateProviderIdentity(receipt.providerIdentity, cell.host === 'codex' ? 'gpt' : cell.host, 'INVALID_PRODUCER_RECEIPT');
416 validateArtifactInventory(receipt.artifacts, 'INVALID_PRODUCER_RECEIPT');
417 const timeline = receipt.toolTimeline;
418 if (timeline !== undefined && (!Array.isArray(timeline) || timeline.length > 4096 || timeline.some(entry => !entry || !Number.isSafeInteger(entry.atMs) || entry.atMs < 0 || (entry.endMs !== undefined && (!Number.isSafeInteger(entry.endMs) || entry.endMs < entry.atMs)) || typeof entry.tool !== 'string' || !/^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(entry.tool) || !['main', 'subagent'].includes(entry.agent) || Object.keys(entry).some(key => !['atMs', 'endMs', 'tool', 'agent'].includes(key))))) throw new Error('INVALID_PRODUCER_RECEIPT');
419}
421function validateInstallationIdentity(identity: ProducerInstallationIdentity, error: string): void {
422 const artifact = (value: any) => value && HEX.test(value.sha256) && Number.isSafeInteger(value.bytes) && value.bytes > 0;
423 if (!identity || identity.schemaVersion !== 1 || !artifact(identity.producer) || !artifact(identity.launcher) || !artifact(identity.core) || !artifact(identity.watchdog) ||
424 !identity.embeddedCatalogs || !identity.embeddedCatalogs.runtimeRevision || !identity.embeddedCatalogs.runtimeBuildRevision || !HEX.test(identity.embeddedCatalogs.runtimeSha256) ||
425 !identity.embeddedCatalogs.scannerRevision || !HEX.test(identity.embeddedCatalogs.scannerSha256) || !HEX.test(identity.identityHash)) throw new Error(error);
426 const { identityHash, ...withoutHash } = identity;
427 if (producerInstallationIdentityHash(withoutHash) !== identityHash) throw new Error(error);
428}
430function validateProviderIdentity(identity: ProducerProviderIdentity, family: 'claude' | 'gpt' | 'gemini', error: string): void {
431 if (!identity || identity.schemaVersion !== 1 || identity.family !== family || !identity.policyRevision || !identity.version ||
432 !identity.executable || !HEX.test(identity.executable.sha256) || !Number.isSafeInteger(identity.executable.bytes) || identity.executable.bytes <= 0 ||
433 !Array.isArray(identity.argsPrefix) || identity.argsPrefix.some(value => typeof value !== 'string' || value.includes('\0')) || !HEX.test(identity.identityHash)) throw new Error(error);
434 const { identityHash, ...withoutHash } = identity;
435 if (producerProviderIdentityHash(withoutHash) !== identityHash) throw new Error(error);
436}
438/** Paths are unique; their order is whatever the producer bound into identityHash (1.91.75 producers sorted by locale). */
439function validateArtifactInventory(inventory: ProducerArtifactInventory, error: string): void {
440 if (!inventory || inventory.schemaVersion !== 1 || inventory.root !== 'security/cso' || !Array.isArray(inventory.entries) || inventory.entries.length > 4096 ||
441 !Number.isSafeInteger(inventory.totalBytes) || inventory.totalBytes < 0 || inventory.totalBytes > 128 * 1024 * 1024 || !HEX.test(inventory.identityHash)) throw new Error(error);
442 const paths = new Set<string>();
443 let total = 0;
444 for (const entry of inventory.entries) {
445 if (!entry || typeof entry.path !== 'string' || entry.path.length > 1024 || !/^[A-Za-z0-9._/-]+$/.test(entry.path) || entry.path.split('/').some(part => !part || part === '.' || part === '..') ||
446 paths.has(entry.path) || !HEX.test(entry.sha256) || !Number.isSafeInteger(entry.bytes) || entry.bytes < 0 || entry.bytes > 32 * 1024 * 1024) throw new Error(error);
447 paths.add(entry.path); total += entry.bytes;
448 }
449 if (total !== inventory.totalBytes) throw new Error(error);
450 const { identityHash, ...withoutHash } = inventory;
451 if (producerArtifactInventoryHash(withoutHash) !== identityHash) throw new Error(error);
452}
454function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): void {
455 if (!receipt || receipt.schemaVersion !== 2 || receipt.cellRef !== cell.id || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
456 const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
457 if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started || !['inputTokens', 'outputTokens', 'cachedTokens', 'estimatedCostUSD'].every(field => receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage])) || ((receipt.status === 'failed') !== !!receipt.error) || (receipt.error && !receipt.error.code)) throw new Error('INVALID_PRODUCER_BATCH');
458 validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_BATCH');
459 validateProviderIdentity(receipt.providerIdentity, cell.host === 'codex' ? 'gpt' : cell.host, 'INVALID_PRODUCER_BATCH');
460 validateArtifactInventory(receipt.artifacts, 'INVALID_PRODUCER_BATCH');
461}
463const matchedPairKey = (cell: EvalCell): string => [cell.caseId, cell.variant, cell.mode, cell.repetition, cell.model, cell.host, cell.budgetSeconds, cell.sourceHash].join('|');
464const PROFILE_GROUPS: Record<EvalProfile, ReadonlyArray<readonly [EvalVersion, EvalMode]>> = {
465 full: [['v2', 'daily'], ['v2', 'comprehensive'], ['v3', 'daily'], ['v3', 'comprehensive']],
466 release: [['v3', 'daily'], ['v3', 'comprehensive']],
467 baseline: [['v2', 'comprehensive']],
468};
469const profileOf = (matrix: EvalMatrix): EvalProfile => matrix.profile ?? 'full';
470const expectedMatchedPairs = (matrix: EvalMatrix): number => profileOf(matrix) === 'full' ? matrix.cells.length / 2 : 0;
472type IdentityBearing = Pick<ProducerReceiptIndex, 'cell' | 'installationIdentity' | 'providerIdentity'>;
473/**
474 * Each platform has exactly one producer installation and provider binary.
475 * Platforms differ only in native binaries: the embedded catalogs and the
476 * reviewed provider policy must be identical across them. Evaluation units that
477 * generated their catalog once per platform embed copies differing only in each
478 * runtime's `qualifiedAt` and the evidence digest over it; `bodies`, the catalogs
479 * as embedded, prove that, each hashing to its platform's runtimeSha256.
480 */
481function assertPlatformIdentities(receipts: IdentityBearing[], bodies: RuntimeCatalog[] = []): void {
482 const byPlatform = new Map<string, IdentityBearing[]>();
483 for (const receipt of receipts) {
484 const key = receipt.cell.platform ?? 'unpinned', list = byPlatform.get(key) ?? [];
485 list.push(receipt); byPlatform.set(key, list);
486 }
487 for (const list of byPlatform.values()) {
488 if (new Set(list.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
489 if (new Set(list.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
490 }
491 const representatives = [...byPlatform.values()].map(list => list[0]);
492 if (representatives.length < 2) return;
493 if (new Set(representatives.map(receipt => receipt.installationIdentity.identityHash)).size !== representatives.length) throw new Error('UNMATCHED_PRODUCER_PLATFORMS');
494 const catalogs = (receipt: IdentityBearing) => JSON.stringify(receipt.installationIdentity.embeddedCatalogs);
495 const policy = ({ providerIdentity: { family, policyRevision, version, argsPrefix } }: IdentityBearing) => JSON.stringify({ family, policyRevision, version, argsPrefix });
496 if (new Set(representatives.map(catalogs)).size !== 1 && !catalogsDifferOnlyInQualificationTime(representatives, bodies)) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
497 if (new Set(representatives.map(policy)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
498}
500function catalogsDifferOnlyInQualificationTime(representatives: IdentityBearing[], bodies: RuntimeCatalog[]): boolean {
501 if (!Array.isArray(bodies) || bodies.length !== representatives.length) return false;
502 const timeless = (body: RuntimeCatalog) => canonical({ ...body, evaluation: { ...body.evaluation, evidenceDigest: null }, runtimes: body.runtimes.map(({ qualifiedAt: _qualifiedAt, ...runtime }) => runtime) });
503 const shared = ({ runtimeSha256: _runtimeSha256, ...rest }: ProducerInstallationIdentity['embeddedCatalogs']) => JSON.stringify(rest);
504 const matched = representatives.map(receipt => bodies.find(body => sha256(JSON.stringify(body)) === receipt.installationIdentity.embeddedCatalogs.runtimeSha256));
505 if (matched.some(body => !body) || new Set(matched).size !== bodies.length) return false;
506 try { for (const body of bodies) validateRuntimeCatalog(body); } catch { return false; }
507 return new Set(representatives.map(receipt => shared(receipt.installationIdentity.embeddedCatalogs))).size === 1 && new Set(bodies.map(timeless)).size === 1 &&
508 bodies.every(body => body.revision === representatives[0].installationIdentity.embeddedCatalogs.runtimeRevision && body.evaluation);
509}
511/** `embeddedCatalogs`: the runtime catalog each platform's producer unit embedded, required only when those differ (see assertPlatformIdentities). */
512export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[], corpus: EvalCorpusManifest = loadCorpusManifest(), embeddedCatalogs: RuntimeCatalog[] = []): ProducerBatch {
513 validateMatrix(matrix, corpus); validateSchedule(matrix, schedule);
514 if (!Array.isArray(receipts)) throw new Error('INVALID_PRODUCER_RECEIPTS');
515 const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
516 const jobs = new Map(schedule.jobs.map(job => [job.cellId, job]));
517 const seen = new Set<string>();
518 for (const receipt of receipts) {
519 const cell = cells.get(receipt?.cellRef), job = jobs.get(receipt?.cellRef);
520 if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
521 validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id);
522 }
523 // Receipts name only the opaque cellRef; the trusted matrix supplies the cell.
524 const bound = receipts.map(receipt => ({ ...receipt, cell: cells.get(receipt.cellRef)! }));
525 assertPlatformIdentities(bound, embeddedCatalogs);
526 const platformCatalogs = new Set(bound.map(receipt => receipt.installationIdentity.embeddedCatalogs.runtimeSha256)).size > 1 ? embeddedCatalogs : undefined;
527 const pairs = new Map<string, Partial<Record<EvalVersion, (typeof bound)[number]>>>();
528 if (profileOf(matrix) === 'full') for (const receipt of bound) {
529 const key = matchedPairKey(receipt.cell as EvalCell);
530 const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair);
531 }
532 else if (new Set(bound.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
533 const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: (typeof bound)[number]; v3: (typeof bound)[number] }]>;
534 const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed }));
535 if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`);
536 const groups: ProducerGroupSummary[] = [];
537 for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) {
538 const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
539 const submitted = bound.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
540 const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]);
541 const costs = submitted.flatMap(receipt => receipt.usage.estimatedCostUSD === null ? [] : [receipt.usage.estimatedCostUSD]);
542 groups.push({ version, mode, scheduled: expected.length, submitted: submitted.length, missing: expected.length - submitted.length, succeeded: submitted.filter(receipt => receipt.status === 'succeeded').length, failed: submitted.filter(receipt => receipt.status === 'failed').length,
543 latency: { samples: submitted.length, denominator: expected.length, p95Ms: quantile(submitted.map(receipt => receipt.durationMs), 0.95) },
544 firstUsefulResult: { samples: 0, denominator: expected.length, p95Ms: null },
545 modelTokens: { measured: tokens.length, denominator: expected.length, total: tokens.length ? tokens.reduce((sum, value) => sum + value, 0) : null },
546 estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' },
547 });
548 }
549 const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, toolTimeline: _toolTimeline, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
550 const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: {
551 scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length,
552 matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
553 note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.',
554 }, ...(platformCatalogs ? { embeddedCatalogs: platformCatalogs } : {}) };
555 return { ...base, batchHash: digest(JSON.stringify(base)) };
556}
558function normalizedPlatforms(corpus: EvalCorpusManifest, value: unknown): Record<string, EvalPlatform> {
559 const assignment = value as Record<string, EvalPlatform>;
560 if (!assignment || typeof assignment !== 'object' || Array.isArray(assignment) || Object.keys(assignment).length !== corpus.cases.length ||
561 corpus.cases.some(fixture => !PRODUCER_PLATFORMS.includes(assignment[fixture.id]))) throw new Error('INVALID_EVAL_PLATFORMS');
562 return Object.fromEntries(corpus.cases.map(fixture => [fixture.id, assignment[fixture.id]]));
563}
565function buildEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest, baselineHash: string | null): EvalMatrix {
566 const profile = options.profile ?? 'full';
567 const skillHashes: Partial<Record<EvalVersion, string>> = options.skillHashes ?? {};
568 const versions = profile === 'full' ? ['v2', 'v3'] : profile === 'release' ? ['v3'] : ['v2'];
569 if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !['full', 'release', 'baseline'].includes(profile) ||
570 Object.keys(skillHashes).sort().join(',') !== versions.join(',') || versions.some(version => !HEX.test(skillHashes[version as EvalVersion] ?? '')) ||
571 (profile === 'full' && skillHashes.v2 === skillHashes.v3)) throw new Error('INVALID_MATCHED_EVAL_INPUT');
572 const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 };
573 if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET');
574 const corpusHash = digest(JSON.stringify(corpus));
575 const cells: EvalCell[] = [];
576 if (options.profile === undefined || options.profile === 'full') {
577 for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) {
578 const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] };
579 cells.push({ id: digest(JSON.stringify(cell)), ...cell });
580 }
581 return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash, model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells };
582 }
583 const pinned = options as ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions;
584 const platformKeys = pinned.profile === 'release' ? 'comprehensive,daily' : 'comprehensive';
585 if (!pinned.platforms || Object.keys(pinned.platforms).sort().join(',') !== platformKeys) throw new Error('INVALID_EVAL_PLATFORMS');
586 const comprehensive = normalizedPlatforms(corpus, pinned.platforms.comprehensive);
587 const daily = pinned.profile === 'release' ? pinned.platforms.daily : undefined;
588 if (pinned.profile === 'release' && !PRODUCER_PLATFORMS.includes(daily!)) throw new Error('INVALID_EVAL_PLATFORMS');
589 const version: EvalVersion = pinned.profile === 'release' ? 'v3' : 'v2';
590 const modes = pinned.profile === 'release' ? ['daily', 'comprehensive'] as const : ['comprehensive'] as const;
591 for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of modes) {
592 const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition: 1 as const, model: pinned.model, host: pinned.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: skillHashes[version]!, platform: mode === 'daily' ? daily! : comprehensive[fixture.id] };
593 cells.push({ id: digest(JSON.stringify(cell)), ...cell });
594 }
595 const common = { schemaVersion: 1 as const, corpusVersion: corpus.version, corpusHash, model: pinned.model, host: pinned.host, budgets };
596 if (pinned.profile === 'release') {
597 if (baselineHash !== null && !HEX.test(baselineHash)) throw new Error('UNBOUND_EVAL_BASELINE');
598 return { ...common, profile: 'release', skillHashes: { v3: pinned.skillHashes.v3 }, repetitions: 1, platforms: { daily: daily!, comprehensive }, baselineHash, cells };
599 }
600 return { ...common, profile: 'baseline', skillHashes: { v2: pinned.skillHashes.v2 }, repetitions: 1, platforms: { comprehensive }, cells };
601}
603const BASELINE_FIELDS = 'baselineHash,budgetSeconds,cases,corpusHash,corpusVersion,host,matrixHash,model,modelUsed,platforms,producerBatchHash,provider,schemaVersion,skillHash,version';
604/** Check that a baseline was measured under the same corpus, model, host, budget and platform assignment. */
605function assertBaselineBinding(baseline: EvalBaseline, target: { corpus: EvalCorpusManifest; model: string; host: ProducerHost; budgetSeconds: number; platforms: Record<string, EvalPlatform> }): void {
606 if (!baseline || typeof baseline !== 'object' || Object.keys(baseline).sort().join(',') !== BASELINE_FIELDS) throw new Error('UNBOUND_EVAL_BASELINE');
607 const { baselineHash, ...record } = baseline;
608 const provider = baseline.provider as unknown as Record<string, unknown>;
609 if (baseline.schemaVersion !== 1 || baseline.version !== 'v2' || !HEX.test(baselineHash) || digest(JSON.stringify(record)) !== baselineHash ||
610 baseline.corpusVersion !== target.corpus.version || baseline.corpusHash !== digest(JSON.stringify(target.corpus)) ||
611 baseline.model !== target.model || baseline.host !== target.host || baseline.budgetSeconds !== target.budgetSeconds ||
612 JSON.stringify(baseline.platforms) !== JSON.stringify(target.platforms) || !HEX.test(baseline.skillHash) || !HEX.test(baseline.matrixHash) ||
613 !HEX.test(baseline.producerBatchHash) || typeof baseline.modelUsed !== 'string' || !baseline.modelUsed ||
614 !provider || typeof provider !== 'object' || Object.keys(provider).sort().join(',') !== 'family,policyRevision,version' ||
615 !['claude', 'gpt', 'gemini'].includes(String(provider.family)) || typeof provider.policyRevision !== 'string' || !provider.policyRevision ||
616 typeof provider.version !== 'string' || !provider.version || !Array.isArray(baseline.cases) || baseline.cases.length !== target.corpus.cases.length ||
617 baseline.cases.some((entry, index) => !entry || Object.keys(entry).sort().join(',') !== 'caseId,found' || entry.caseId !== target.corpus.cases[index].id || typeof entry.found !== 'boolean'))
618 throw new Error('UNBOUND_EVAL_BASELINE');
619}
621export function createEvalMatrix(options: FullEvalMatrixOptions, corpus?: EvalCorpusManifest): FullEvalMatrix;
622export function createEvalMatrix(options: ReleaseEvalMatrixOptions, corpus?: EvalCorpusManifest): ReleaseEvalMatrix;
623export function createEvalMatrix(options: BaselineEvalMatrixOptions, corpus?: EvalCorpusManifest): BaselineEvalMatrix;
624export function createEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest = loadCorpusManifest()): EvalMatrix {
625 let baselineHash: string | null = null;
626 if (options.profile === 'release' && options.baseline !== undefined) {
627 assertBaselineBinding(options.baseline, { corpus, model: options.model, host: options.host, budgetSeconds: (options.budgets ?? { comprehensive: 1800 }).comprehensive, platforms: normalizedPlatforms(corpus, options.platforms?.comprehensive) });
628 baselineHash = options.baseline.baselineHash;
629 }
630 return buildEvalMatrix(options, corpus, baselineHash);
631}
633export function validateMatrix(matrix: EvalMatrix, corpus: EvalCorpusManifest = loadCorpusManifest()): void {
634 if (!matrix || typeof matrix !== 'object') throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
635 const common = { model: matrix.model, host: matrix.host, budgets: matrix.budgets };
636 let expected: EvalMatrix;
637 try {
638 expected = matrix.profile === 'release' ? buildEvalMatrix({ ...common, profile: 'release', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, matrix.baselineHash)
639 : matrix.profile === 'baseline' ? buildEvalMatrix({ ...common, profile: 'baseline', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, null)
640 : buildEvalMatrix({ ...common, skillHashes: matrix.skillHashes }, corpus, null);
641 } catch { throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); }
642 if (JSON.stringify(matrix) !== JSON.stringify(expected)) throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
643}
644function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusManifest): void {
645 if (result.sourceHash !== cell.sourceHash || result.skillHash !== cell.skillHash || result.model !== cell.model || result.host !== cell.host || result.budgetSeconds !== cell.budgetSeconds) throw new Error('UNMATCHED_EVAL_RESULT');
646 for (const field of ['setup', 'reproduction', 'repair', 'recheck'] as const) if (!['passed', 'failed', 'blocked', 'not_attempted'].includes(result[field])) throw new Error('INVALID_EVAL_OUTCOME');
647 for (const field of ['reportPresent', 'reportFinished', 'reportComplete', 'heldOutAssertionsPassed', 'freshRecheck'] as const) if (typeof result[field] !== 'boolean') throw new Error('INVALID_EVAL_RESULT');
648 if (!Array.isArray(result.findings) || !validNumber(result.latencyMs) || (result.firstUsefulResultMs !== null && (!validNumber(result.firstUsefulResultMs) || result.firstUsefulResultMs > result.latencyMs))) throw new Error('INVALID_EVAL_RESULT');
649 if ((result.reportComplete && !result.reportFinished) || (result.reportFinished && !result.reportPresent)) throw new Error('MISSING_EVAL_REPORT');
650 if (!result.reportPresent && result.findings.length) throw new Error('FINDINGS_WITHOUT_REPORT');
651 if (result.setup === 'blocked' && !result.prerequisite?.trim()) throw new Error('MISSING_SETUP_PREREQUISITE');
652 if (result.oracleEvidenceHash !== undefined && (!HEX.test(result.oracleEvidenceHash) || result.oracleVersion !== corpus.version)) throw new Error('INVALID_ORACLE_PROVENANCE');
653 if (result.currentSourceHash !== undefined && !HEX.test(result.currentSourceHash)) throw new Error('INVALID_RECHECK_SOURCE');
654 if (result.recheckEvidenceHash !== undefined && !HEX.test(result.recheckEvidenceHash)) throw new Error('INVALID_RECHECK_EVIDENCE');
655 if (result.usage && (result.usage.source !== 'host' || (result.usage.tokens !== undefined && !validNumber(result.usage.tokens)) || (result.usage.costUSD !== undefined && !validNumber(result.usage.costUSD)))) throw new Error('INVALID_EVAL_USAGE');
656 if (result.producerReceiptHash !== undefined && !HEX.test(result.producerReceiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT_PROVENANCE');
657 const ids = new Set<string>(), repairEvidence = new Set<string>(), recheckEvidence = new Set<string>(), replayEvidence = new Set<string>();
658 for (const finding of result.findings) {
659 if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incidental', 'instrumentation', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT');
660 ids.add(finding.id);
661 if (finding.judgment === 'correct' && (cell.variant !== 'vulnerable' || finding.matchedCaseId !== cell.caseId)) throw new Error('INVALID_ORACLE_MATCH');
662 if ((finding.judgment === 'incidental' || finding.judgment === 'instrumentation') && (finding.evidence === 'hypothesis' || finding.matchedCaseId !== undefined)) throw new Error('INVALID_FINDING_JUDGMENT');
663 if ((finding.judgment === 'incidental') !== (finding.incidentalKey !== undefined) || (finding.incidentalKey !== undefined && !HEX.test(finding.incidentalKey))) throw new Error('INVALID_FINDING_JUDGMENT');
664 if (finding.trustedVerification !== undefined) {
665 const verification = finding.trustedVerification as Record<string, unknown>, allowed = new Set(['repair', 'repairEvidenceHash', 'recheck', 'recheckEvidenceHash']);
666 if (!verification || typeof verification !== 'object' || Array.isArray(verification) || Object.keys(verification).some(key => !allowed.has(key)) ||
667 !['passed', 'failed', 'blocked', 'not_attempted'].includes(String(verification.repair)) ||
668 !['passed', 'failed', 'blocked', 'not_attempted'].includes(String(verification.recheck)) || !finding.claimedTested || cell.version !== 'v3' || cell.mode !== 'comprehensive')
669 throw new Error('INVALID_TRUSTED_FINDING_VERIFICATION');
670 const repairHash = verification.repairEvidenceHash, recheckHash = verification.recheckEvidenceHash;
671 if (!(repairHash === undefined || (typeof repairHash === 'string' && HEX.test(repairHash))) ||
672 !(recheckHash === undefined || (typeof recheckHash === 'string' && HEX.test(recheckHash))) ||
673 (verification.repair === 'passed') !== (repairHash !== undefined) ||
674 (verification.recheck === 'passed') !== (recheckHash !== undefined) ||
675 (verification.recheck === 'passed' && verification.repair !== 'passed'))
676 throw new Error('INVALID_TRUSTED_FINDING_VERIFICATION');
677 if (typeof repairHash === 'string') {
678 if (repairEvidence.has(repairHash)) throw new Error('DUPLICATE_TRUSTED_REPAIR_BINDING');
679 repairEvidence.add(repairHash);
680 }
681 if (typeof recheckHash === 'string') {
682 if (recheckEvidence.has(recheckHash)) throw new Error('DUPLICATE_TRUSTED_RECHECK_BINDING');
683 recheckEvidence.add(recheckHash);
684 }
685 }
686 if (finding.trustedReplay !== undefined) {
687 const replay = finding.trustedReplay as Record<string, unknown>, hash = replay?.replayEvidenceHash;
688 if (!replay || typeof replay !== 'object' || Array.isArray(replay) || Object.keys(replay).some(key => key !== 'result' && key !== 'replayEvidenceHash') ||
689 !['passed', 'failed', 'missing'].includes(String(replay.result)) || finding.judgment !== 'incidental' || !finding.claimedTested ||
690 (replay.result === 'passed') !== (typeof hash === 'string' && HEX.test(hash)) || (replay.result !== 'passed' && hash !== undefined))
691 throw new Error('INVALID_TRUSTED_REPLAY');
692 if (typeof hash === 'string') {
693 if (replayEvidence.has(hash)) throw new Error('DUPLICATE_TRUSTED_REPLAY_BINDING');
694 replayEvidence.add(hash);
695 }
696 }
697 }
698 if (cell.mode === 'daily' && [result.setup, result.reproduction, result.repair, result.recheck].some(value => value !== 'not_attempted')) throw new Error('DAILY_EVAL_EXECUTED_APPLICATION');
699}
700function quantile(values: number[], fraction: number): number | null {
701 if (!values.length) return null;
702 return [...values].sort((left, right) => left - right)[Math.ceil(values.length * fraction) - 1];
703}
704function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: EvalCorpusManifest): Rate {
705 return rate(cells.filter(cell => {
706 const result = results.get(cell.id);
707 if (!result?.reportFinished || result[field] !== 'passed') return false;
708 if (field === 'setup') return true;
709 if (!result.oracleEvidenceHash || result.oracleVersion !== corpus.version || result.setup !== 'passed') return false;
710 if (field === 'reproduction') return true;
711 if (result.reproduction !== 'passed' || result.repair !== 'passed' || !result.heldOutAssertionsPassed) return false;
712 if (cell.version === 'v3' && !result.findings.some(finding => trustedFindingVerification(cell, result, finding, 'repair', corpus))) return false;
713 if (field === 'repair') return true;
714 // Correct alternative patches need not match the reference fix byte-for-byte.
715 // Closure needs a fresh source and a separate trusted current-source observation.
716 return result.freshRecheck && !!result.currentSourceHash && result.currentSourceHash !== cell.sourceHash &&
717 !!result.recheckEvidenceHash && result.recheckEvidenceHash !== result.oracleEvidenceHash &&
718 (cell.version !== 'v3' || result.findings.some(finding => trustedFindingVerification(cell, result, finding, 'recheck', corpus)));
719 }).length, cells.length);
720}
722function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: EvalCorpusManifest): boolean {
723 const verification = finding.trustedVerification;
724 if (!result.reportFinished || cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' ||
725 finding.judgment !== 'correct' || finding.matchedCaseId !== cell.caseId || !finding.claimedTested || !verification ||
726 result.setup !== 'passed' || result.reproduction !== 'passed' || result.repair !== 'passed' || !result.heldOutAssertionsPassed ||
727 verification.repair !== 'passed' || verification.repairEvidenceHash !== result.oracleEvidenceHash || result.oracleVersion !== corpus.version)
728 return false;
729 if (stage === 'repair') return true;
730 return result.recheck === 'passed' && verification.recheck === 'passed' && verification.recheckEvidenceHash === result.recheckEvidenceHash &&
731 result.freshRecheck && !!result.currentSourceHash && result.currentSourceHash !== cell.sourceHash &&
732 !!result.recheckEvidenceHash && result.recheckEvidenceHash !== result.oracleEvidenceHash;
733}
735function validatedResults(matrix: EvalMatrix, observations: EvalResult[], corpus: EvalCorpusManifest): Map<string, EvalResult> {
736 if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS');
737 const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
738 const results = new Map<string, EvalResult>();
739 for (const result of observations) {
740 const cell = cells.get(result.cellId);
741 if (!cell || results.has(result.cellId)) throw new Error('UNKNOWN_OR_DUPLICATE_EVAL_RESULT');
742 validateResult(result, cell, corpus); results.set(result.cellId, result);
743 }
744 return results;
745}
746const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review');
747const foundSeededIssue = (cell: EvalCell, result: EvalResult | undefined) => result?.reportFinished === true &&
748 result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct');
749/**
750 * A tested claim is false unless trusted evidence certifies it: the seeded repair's held-out verification, or, for an
751 * `incidental` finding, a passing replay of the helper's signed bundle. Claims on declared instrumentation count in
752 * neither direction (Garry, 2026-10-18, option 1), the same way precision leaves them out.
753 */
754const falselyTested = (cell: EvalCell, result: EvalResult, finding: EvalFinding, corpus: EvalCorpusManifest) =>
755 finding.claimedTested && finding.judgment !== 'instrumentation' && !trustedFindingVerification(cell, result, finding, 'repair', corpus) &&
756 !(finding.judgment === 'incidental' && finding.trustedReplay?.result === 'passed');
757const isHighCritical = (corpus: EvalCorpusManifest, caseId: string) => ['critical', 'high'].includes(corpus.cases.find(fixture => fixture.id === caseId)!.severity);
759/** High/critical recall of a pre-registered external baseline, after proving it binds to this release matrix. */
760function externalBaselineRecall(matrix: ReleaseEvalMatrix, baseline: EvalBaseline, corpus: EvalCorpusManifest): Rate {
761 if (matrix.baselineHash === null || baseline?.baselineHash !== matrix.baselineHash) throw new Error('UNBOUND_EVAL_BASELINE');
762 assertBaselineBinding(baseline, { corpus, model: matrix.model, host: matrix.host, budgetSeconds: matrix.budgets.comprehensive, platforms: matrix.platforms.comprehensive });
763 const highCritical = baseline.cases.filter(entry => isHighCritical(corpus, entry.caseId));
764 return rate(highCritical.filter(entry => entry.found).length, highCritical.length);
765}
767export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) {
768 validateMatrix(matrix, corpus);
769 const profile = profileOf(matrix);
770 if (profile === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline');
771 if (profile !== 'release' && baseline !== undefined) throw new Error('UNBOUND_EVAL_BASELINE');
772 const results = validatedResults(matrix, observations, corpus);
773 const found = (cell: EvalCell) => foundSeededIssue(cell, results.get(cell.id));
774 const groups: any[] = [];
775 for (const [version, mode] of PROFILE_GROUPS[profile]) {
776 const selected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
777 const completed = selected.map(cell => results.get(cell.id)).filter((result): result is EvalResult => !!result);
778 const expected = selected.filter(cell => cell.variant === 'vulnerable');
779 const highCritical = expected.filter(cell => isHighCritical(corpus, cell.caseId));
780 let correct = 0, reported = 0, unadjudicated = 0;
781 for (const cell of selected) {
782 const result = results.get(cell.id);
783 const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis' && finding.judgment !== 'instrumentation') : [];
784 reported += findings.length;
785 unadjudicated += findings.filter(finding => finding.judgment === 'unadjudicated').length;
786 correct += Number(findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct')); // Duplicate reports do not increase true positives.
787 correct += new Set(findings.filter(finding => eligibleFinding(cell, finding) && finding.judgment === 'incidental').map(finding => finding.incidentalKey)).size;
788 }
789 const firstUseful = completed.flatMap(result => result.firstUsefulResultMs === null ? [] : [result.firstUsefulResultMs]);
790 const costs = completed.flatMap(result => result.usage?.costUSD === undefined ? [] : [result.usage.costUSD]);
791 const tokens = completed.flatMap(result => result.usage?.tokens === undefined ? [] : [result.usage.tokens]);
792 const falseTested = selected.reduce((count, cell) => {
793 const result = results.get(cell.id); if (!result) return count;
794 return count + result.findings.filter(finding => falselyTested(cell, result, finding, corpus)).length;
795 }, 0);
796 groups.push({ version, mode, cells: selected.length, submitted: completed.length, missing: selected.length - completed.length,
797 reports: rate(completed.filter(result => result.reportPresent && result.reportComplete).length, selected.length), finishedReports: rate(completed.filter(result => result.reportFinished).length, selected.length), precision: rate(correct, reported), recall: rate(expected.filter(found).length, expected.length), highCriticalRecall: rate(highCritical.filter(found).length, highCritical.length),
798 unadjudicated, falseTested, setup: mode === 'comprehensive' ? measuredRate(selected, results, 'setup', corpus) : null,
799 reproduction: mode === 'comprehensive' ? measuredRate(expected, results, 'reproduction', corpus) : null,
800 repair: mode === 'comprehensive' ? measuredRate(expected, results, 'repair', corpus) : null,
801 recheck: mode === 'comprehensive' ? measuredRate(expected, results, 'recheck', corpus) : null,
802 setupBlocked: completed.filter(result => result.setup === 'blocked').length,
803 firstUsefulResult: { samples: firstUseful.length, denominator: selected.length, medianMs: quantile(firstUseful, 0.5), p95Ms: quantile(firstUseful, 0.95) },
804 latency: { samples: completed.length, p95Ms: quantile(completed.map(result => result.latencyMs), 0.95) },
805 cost: { measured: costs.length, denominator: selected.length, totalUSD: costs.length ? costs.reduce((sum, cost) => sum + cost, 0) : null },
806 modelTokens: { measured: tokens.length, denominator: selected.length, total: tokens.length ? tokens.reduce((sum, count) => sum + count, 0) : null },
807 });
808 }
809 const daily = groups.find(group => group.version === 'v3' && group.mode === 'daily');
810 const comprehensive = groups.find(group => group.version === 'v3' && group.mode === 'comprehensive');
811 const inBatchBaseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive');
812 const external = matrix.profile === 'release' && baseline !== undefined ? externalBaselineRecall(matrix, baseline, corpus) : null;
813 const perStack = Object.fromEntries(STACKS.map(stack => {
814 const eligible = matrix.cells.filter(cell => cell.version === 'v3' && cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.stack === stack);
815 // A repaired oracle case demonstrates the full find-to-repair workflow only
816 // when the producer also reported the matching supported finding.
817 const successful = new Set(eligible.filter(cell => {
818 const result = results.get(cell.id);
819 return result?.findings.some(finding => trustedFindingVerification(cell, result, finding, 'repair', corpus));
820 }).map(cell => cell.caseId));
821 return [stack, { correctHeldOutRepairs: successful.size, denominator: new Set(eligible.map(cell => cell.caseId)).size }];
822 }));
823 const assessedAll = observations.length === matrix.cells.length && observations.every(result => result.reportFinished) && groups.every(group => !group.unadjudicated);
824 const gate = (condition: boolean | null, hasData: boolean): 'pass' | 'fail' | 'unmeasured' => !hasData || condition === null ? 'unmeasured' : condition ? 'pass' : 'fail';
825 const containmentValues = REQUIRED_CONTAINMENT.map(name => qualification.containment?.[name]);
826 if (containmentValues.some(value => value !== undefined && !['passed', 'failed', 'not_run'].includes(value))) throw new Error('INVALID_CONTAINMENT_RESULT');
827 const gates = {
828 matchedCompleteMatrix: gate(assessedAll, observations.length > 0),
829 mandatoryReports: gate(groups.every(group => group.finishedReports.numerator === group.finishedReports.denominator), observations.length === matrix.cells.length),
830 dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated),
831 comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated),
832 noHighCriticalRecallRegression: profile === 'full'
833 ? gate(comprehensive.highCriticalRecall.value >= inBatchBaseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && inBatchBaseline.submitted === inBatchBaseline.cells && !comprehensive.unadjudicated && !inBatchBaseline.unadjudicated)
834 : gate(external === null ? null : comprehensive.highCriticalRecall.value >= external.value!, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated && external !== null),
835 allCoreColdStarts: gate(comprehensive.setup.numerator === comprehensive.setup.denominator, comprehensive.submitted === comprehensive.cells),
836 zeroFalselyTestedRepairs: gate(comprehensive.falseTested === 0 && daily.falseTested === 0, comprehensive.submitted === comprehensive.cells && daily.submitted === daily.cells),
837 heldOutRepairEachStack: gate(Object.values(perStack).every(value => value.correctHeldOutRepairs >= 1), comprehensive.submitted === comprehensive.cells),
838 containmentAndCanaries: containmentValues.includes('failed') ? 'fail' as const : containmentValues.every(value => value === 'passed') ? 'pass' as const : 'unmeasured' as const,
839 };
840 return { schemaVersion: 1, corpusVersion: corpus.version, model: matrix.model, host: matrix.host,
841 ...(profile === 'release' ? { profile, baseline: external === null ? null : { baselineHash: baseline!.baselineHash, highCriticalRecall: external } } : {}),
842 status: Object.values(gates).every(value => value === 'pass') ? 'qualified' : observations.length ? 'partial' : 'unmeasured', groups, perStack, gates,
843 notes: ['Missing and setup-blocked supported scenarios remain in recall and workflow denominators.', 'Costs and tokens come only from host-reported usage; wall-clock budgets are not model-spend caps.', 'Oracle judgments and qualification receipts must come from the trusted evaluator, never producing agents.'] };
844}
846/** Bind a complete collected batch and every trusted judgment to the matrix's producer receipts. */
847/** Validate the batch against its judgments; a failed producer receipt's report counts nothing, whatever the judge read. */
848function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[]): EvalResult[] {
849 if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS');
850 if (!batch || batch.schemaVersion !== 1 || batch.matrixHash !== matrixHash(matrix) || !HEX.test(batch.scheduleHash) || !HEX.test(batch.batchHash)) throw new Error('INVALID_PRODUCER_BATCH');
851 const { batchHash, ...withoutHash } = batch;
852 const pairs = expectedMatchedPairs(matrix);
853 if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== pairs || batch.summary.matchedPairsSubmitted !== pairs || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH');
854 const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
855 const receipts = new Map<string, ProducerReceiptIndex>();
856 for (const receipt of batch.receipts) {
857 const cell = cells.get(receipt?.cellRef);
858 if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH');
859 validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt);
860 }
861 assertPlatformIdentities(batch.receipts, batch.embeddedCatalogs);
862 if (profileOf(matrix) === 'full') {
863 const actualModels = new Map<string, Partial<Record<EvalVersion, string>>>();
864 for (const receipt of batch.receipts) {
865 const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {};
866 pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair);
867 }
868 if (actualModels.size !== pairs || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
869 } else if (new Set(batch.receipts.map(receipt => receipt.modelUsed)).size !== 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
870 for (const result of observations) {
871 const receipt = receipts.get(result.cellId);
872 if (!receipt || result.producerReceiptHash !== receipt.receiptHash) throw new Error('UNBOUND_EVAL_RESULT');
873 }
874 return observations.map(result => receipts.get(result.cellId)!.status === 'failed' ? { ...result, reportFinished: false, reportComplete: false } : result);
875}
877/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */
878export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) {
879 validateMatrix(matrix, corpus);
880 if (profileOf(matrix) === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline');
881 const counted = validateCollectedBatch(matrix, batch, observations);
882 if (baseline !== undefined) {
883 const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0];
884 if (baseline?.modelUsed !== modelUsed || JSON.stringify(baseline?.provider) !== JSON.stringify({ family, policyRevision, version })) throw new Error('UNBOUND_EVAL_BASELINE');
885 }
886 const score = scoreEval(matrix, counted, qualification, corpus, baseline);
887 return { ...score, producerBatchHash: batch.batchHash };
888}
890/**
891 * Turn a complete, fully adjudicated v2 baseline run into a reusable record.
892 * Missing reports, and reports behind a failed producer receipt, count as misses; an incomplete or unadjudicated run is refused.
893 */
894export function createEvalBaseline(matrix: BaselineEvalMatrix, batch: ProducerBatch, observations: EvalResult[], corpus: EvalCorpusManifest = loadCorpusManifest()): EvalBaseline {
895 if (matrix?.profile !== 'baseline') throw new Error('INVALID_EVAL_BASELINE_MATRIX');
896 validateMatrix(matrix, corpus);
897 const results = validatedResults(matrix, validateCollectedBatch(matrix, batch, observations), corpus);
898 if (results.size !== matrix.cells.length) throw new Error('INCOMPLETE_EVAL_BASELINE');
899 if ([...results.values()].some(result => result.reportFinished && result.findings.some(finding => finding.evidence !== 'hypothesis' && finding.judgment === 'unadjudicated'))) throw new Error('UNADJUDICATED_EVAL_BASELINE');
900 const vulnerable = new Map(matrix.cells.filter(cell => cell.variant === 'vulnerable').map(cell => [cell.caseId, cell]));
901 const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0];
902 const record: Omit<EvalBaseline, 'baselineHash'> = {
903 schemaVersion: 1, version: 'v2', corpusVersion: corpus.version, corpusHash: matrix.corpusHash, model: matrix.model, host: matrix.host,
904 budgetSeconds: matrix.budgets.comprehensive, skillHash: matrix.skillHashes.v2, platforms: matrix.platforms.comprehensive,
905 matrixHash: matrixHash(matrix), producerBatchHash: batch.batchHash, modelUsed, provider: { family, policyRevision, version },
906 cases: corpus.cases.map(fixture => ({ caseId: fixture.id, found: foundSeededIssue(vulnerable.get(fixture.id)!, results.get(vulnerable.get(fixture.id)!.id)) })),
907 };
908 return { ...record, baselineHash: digest(JSON.stringify(record)) };
909}
911function splitFlags(args: string[], allowed: string[]): { positional: string[]; flags: Record<string, string> } {
912 const positional: string[] = [], flags: Record<string, string> = {};
913 for (let index = 0; index < args.length; index++) {
914 if (!args[index].startsWith('--')) { positional.push(args[index]); continue; }
915 if (!allowed.includes(args[index]) || !args[index + 1] || flags[args[index]]) throw new Error(`Unsupported or repeated flag: ${args[index]}`);
916 flags[args[index]] = args[++index];
917 }
918 return { positional, flags };
919}
921function cli(args: string[]): void {
922 const [command, ...rest] = args;
923 if (command === 'payload') {
924 if (rest.length !== 6 || rest[0] !== '--version' || !['v2', 'v3'].includes(rest[1]) || rest[2] !== '--skill-dir' || rest[4] !== '--output') throw new Error('Usage: cso-eval payload --version <v2|v3> --skill-dir <directory> --output <new-file>');
925 const payload = loadPortableSkillPayload(rest[1] as EvalVersion, rest[3]);
926 const target = resolve(rest[5]), parent = dirname(target), stat = lstatSync(parent);
927 if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(parent) !== parent) throw new Error('UNSAFE_EVAL_DESTINATION');
928 atomicWriteSync(target, payload, { mode: 0o600, noReplace: true });
929 console.log(JSON.stringify({ version: rest[1], files: validatePortableSkillPayload(payload).files.length, sha256: sha256(payload), output: target, paidCalls: 0 })); return;
930 }
931 if (command === 'materialize') {
932 if (rest.length !== 3 || !['vulnerable', 'fixed'].includes(rest[1])) throw new Error('Usage: cso-eval materialize <case-id> <vulnerable|fixed> <new-directory>');
933 console.log(JSON.stringify(materializeCase(rest[0], rest[1] as EvalVariant, rest[2]), null, 2)); return;
934 }
935 if (command === 'matrix') {
936 const values: Record<string, string> = {};
937 for (let index = 0; index < rest.length; index += 2) {
938 if (!['--model', '--host', '--v2-payload', '--v3-payload', '--output'].includes(rest[index]) || !rest[index + 1] || values[rest[index]]) throw new Error('Usage: cso-eval matrix --model <id> --host <claude|codex|gemini> --v2-payload <file> --v3-payload <file> --output <new-file>');
939 values[rest[index]] = rest[index + 1];
940 }
941 if (Object.keys(values).length !== 5) throw new Error('All matrix flags are required; exact portable skill payloads and model identity must be pinned.');
942 const payloads = { v2: readBoundedStable(resolve(values['--v2-payload']), PORTABLE_PAYLOAD_LIMIT, 'v2 portable skill payload').toString('utf8'), v3: readBoundedStable(resolve(values['--v3-payload']), PORTABLE_PAYLOAD_LIMIT, 'v3 portable skill payload').toString('utf8') };
943 validatePortableSkillPayload(payloads.v2, 'v2'); validatePortableSkillPayload(payloads.v3, 'v3');
944 const matrix = createEvalMatrix({ model: values['--model'], host: values['--host'] as ProducerHost, skillHashes: { v2: digest(payloads.v2), v3: digest(payloads.v3) } });
945 safeWriteNew(values['--output'], matrix); console.log(JSON.stringify({ cells: matrix.cells.length, output: values['--output'], paidCalls: 0 })); return;
946 }
947 if (command === 'prepare') {
948 if (rest.length !== 7 || rest[1] !== '--v2-payload' || rest[3] !== '--v3-payload' || rest[5] !== '--output') throw new Error('Usage: cso-eval prepare <matrix.json> --v2-payload <file> --v3-payload <file> --output <new-directory>');
949 const matrix = readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix;
950 const schedule = prepareEvalJobs(matrix, { v2: readBoundedStable(resolve(rest[2]), PORTABLE_PAYLOAD_LIMIT, 'v2 portable skill payload').toString('utf8'), v3: readBoundedStable(resolve(rest[4]), PORTABLE_PAYLOAD_LIMIT, 'v3 portable skill payload').toString('utf8') }, rest[6]);
951 console.log(JSON.stringify({ scheduled: schedule.scheduledCells, prepared: schedule.preparedCells, output: resolve(rest[6]), paidCalls: 0 })); return;
952 }
953 const { positional, flags } = splitFlags(rest, ['--corpus-manifest', '--baseline']);
954 const corpus = flags['--corpus-manifest'] ? validateCorpusManifest(readJsonBounded(flags['--corpus-manifest'], 16 * 1024 * 1024, 'corpus manifest')) : loadCorpusManifest();
955 if (command === 'collect') {
956 if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval collect <matrix.json> <schedule.json> <receipts-directory> <new-batch.json> [--corpus-manifest <manifest.json>]');
957 const [matrixPath, schedulePath, receiptDirectory, output] = positional;
958 const matrix = readJsonBounded(matrixPath, 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix;
959 const schedule = readJsonBounded(schedulePath, 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule;
960 const receiptRoot = resolve(receiptDirectory), stat = lstatSync(receiptRoot);
961 if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(receiptRoot) !== receiptRoot) throw new Error('UNSAFE_RECEIPT_DIRECTORY');
962 const receipts: ProducerReceipt[] = [];
963 for (const entry of readdirSync(receiptRoot, { withFileTypes: true })) {
964 if (!entry.isFile() || entry.isSymbolicLink() || !/^[a-f0-9]{64}\.json$/.test(entry.name)) throw new Error('UNSAFE_RECEIPT_DIRECTORY');
965 receipts.push(readJsonBounded(join(receiptRoot, entry.name), 40 * 1024 * 1024, 'producer receipt'));
966 }
967 const batch = collectProducerReceipts(matrix, schedule, receipts, corpus);
968 safeWriteNew(resolve(output), batch);
969 console.log(JSON.stringify({ output: resolve(output), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return;
970 }
971 if (command === 'score') {
972 if (positional.length < 3 || positional.length > 4) throw new Error('Usage: cso-eval score <matrix.json> <producer-batch.json> <trusted-results.json> [qualification.json] [--corpus-manifest <manifest.json>] [--baseline <baseline.json>]');
973 const baseline = flags['--baseline'] ? readJsonBounded(flags['--baseline'], 4 * 1024 * 1024, 'evaluation baseline') as EvalBaseline : undefined;
974 console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), positional[3] ? readJsonBounded(positional[3], 4 * 1024 * 1024, 'qualification') : undefined, corpus, baseline), null, 2)); return;
975 }
976 if (command === 'baseline') {
977 if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval baseline <baseline-matrix.json> <producer-batch.json> <trusted-results.json> <new-baseline.json> [--corpus-manifest <manifest.json>]');
978 const baseline = createEvalBaseline(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), corpus);
979 safeWriteNew(resolve(positional[3]), baseline);
980 console.log(JSON.stringify({ output: resolve(positional[3]), baselineHash: baseline.baselineHash, paidCalls: 0 })); return;
981 }
982 throw new Error('Usage: cso-eval <payload|matrix|prepare|materialize|collect|score|baseline>');
983}
984if (import.meta.main) { try { cli(process.argv.slice(2)); } catch (error) { console.error(error instanceof Error ? error.message : 'CSO evaluation failed'); process.exitCode = 1; } }