cso-eval-protocol.ts
189 lines · scripts/cso-eval-protocol.ts at 3f79d81
1import { createHash } from 'node:crypto';
3export type ProducerHost = 'claude' | 'codex' | 'gemini';
4export type ProducerVersion = 'v2' | 'v3';
5export type ProducerMode = 'daily' | 'comprehensive';
6export type ProducerStack = 'node' | 'bun' | 'python' | 'rails';
7export type ProducerVariant = 'vulnerable' | 'fixed';
8export type ProducerPlatform = 'linux/amd64' | 'linux/arm64';
9export const PRODUCER_PLATFORMS: readonly ProducerPlatform[] = ['linux/amd64', 'linux/arm64'];
11export interface ProducerCell {
12 id: string;
13 caseId: string;
14 stack: ProducerStack;
15 variant: ProducerVariant;
16 version: ProducerVersion;
17 mode: ProducerMode;
18 repetition: 1 | 2 | 3;
19 model: string;
20 host: ProducerHost;
21 budgetSeconds: number;
22 sourceHash: string;
23 skillHash: string;
24 /** Release and baseline cells pin the native producer platform; full-profile cells omit it. */
25 platform?: ProducerPlatform;
26}
28export interface ProducerSourceEntry {
29 path: string;
30 sha256: string;
31 bytes: number;
32}
34/**
35 * Only what a producer needs to run one cell. It never names the corpus case,
36 * variant, version, or repetition: an agent that can read its own control file
37 * or receipt still cannot tell which answer it is being graded against.
38 */
39export interface ProducerExecution {
40 mode: ProducerMode;
41 model: string;
42 host: ProducerHost;
43 budgetSeconds: number;
44 sourceHash: string;
45 skillHash: string;
46 platform?: ProducerPlatform;
47}
49/**
50 * This control file is consumed before a producing agent starts. It must never
51 * be placed inside the application repository or retained on the producer.
52 */
53export interface ProducerInput {
54 schemaVersion: 2;
55 /** The opaque matrix cell id; only the trusted evaluator maps it back to a cell. */
56 cellRef: string;
57 /** Exact canonical portable payload: root SKILL.md plus its manifest-listed sections. */
58 skill: string;
59 source: ProducerSourceEntry[];
60 execution: ProducerExecution;
61}
63export function producerExecution(cell: ProducerCell): ProducerExecution {
64 return {
65 mode: cell.mode, model: cell.model, host: cell.host, budgetSeconds: cell.budgetSeconds,
66 sourceHash: cell.sourceHash, skillHash: cell.skillHash, ...(cell.platform === undefined ? {} : { platform: cell.platform }),
67 };
68}
70export interface ProducerArtifactIdentity {
71 sha256: string;
72 bytes: number;
73}
75export interface ProducerInstallationIdentity {
76 schemaVersion: 1;
77 producer: ProducerArtifactIdentity;
78 launcher: ProducerArtifactIdentity;
79 core: ProducerArtifactIdentity;
80 watchdog: ProducerArtifactIdentity;
81 /** The adjacent manifest is exactly `<coreSha256>\n`; bind both its bytes and declaration. */
82 generation: {
83 coreSha256: string;
84 manifest: ProducerArtifactIdentity;
85 };
86 embeddedCatalogs: {
87 runtimeRevision: string;
88 runtimeBuildRevision: string;
89 runtimeSha256: string;
90 scannerRevision: string;
91 scannerSha256: string;
92 };
93 identityHash: string;
94}
96export interface ProducerProviderIdentity {
97 schemaVersion: 1;
98 family: 'claude' | 'gpt' | 'gemini';
99 policyRevision: string;
100 executable: ProducerArtifactIdentity;
101 argsPrefix: string[];
102 version: string;
103 identityHash: string;
104}
106export interface ProducerArtifactInventory {
107 schemaVersion: 1;
108 root: 'security/cso';
109 /** Sorted safe paths relative to root within the retained helper home. */
110 entries: ProducerSourceEntry[];
111 totalBytes: number;
112 identityHash: string;
113}
115export interface ProducerReceipt {
116 schemaVersion: 2;
117 /** The opaque cell id from the consumed input; bound together with inputHash. */
118 cellRef: string;
119 inputHash: string;
120 installationIdentity: ProducerInstallationIdentity;
121 providerIdentity: ProducerProviderIdentity;
122 artifacts: ProducerArtifactInventory;
123 startedAt: string;
124 finishedAt: string;
125 status: 'succeeded' | 'failed';
126 requestedModel: string;
127 modelUsed: string;
128 /** Provider-reported when it differs; otherwise the exact CLI model pin. */
129 modelIdentitySource: 'provider_reported' | 'requested_pin';
130 durationMs: number;
131 firstUsefulResultMs: null;
132 toolCalls: number;
133 /** Content-free tool timeline (name, offset, thread) for diagnosing overruns; absent when the adapter has none. */
134 toolTimeline?: Array<{ atMs: number; endMs?: number; tool: string; agent: 'main' | 'subagent' }>;
135 output: string;
136 outputHash: string;
137 usage: {
138 /** CLI/provider-reported tokens. Null means the adapter did not report usage. */
139 inputTokens: number | null;
140 outputTokens: number | null;
141 cachedTokens: number | null;
142 /** Pricing-table estimate, not a host-billed amount. */
143 estimatedCostUSD: number | null;
144 };
145 error?: { code: string; reason: string };
146 receiptHash: string;
147}
149/**
150 * Compact trusted index; raw output remains in the separately retained receipt.
151 * Collection adds the matrix cell that the receipt's cellRef names.
152 */
153export type ProducerReceiptIndex = Omit<ProducerReceipt, 'output' | 'error' | 'toolTimeline'> & {
154 cell: ProducerCell;
155 error?: { code: string };
156};
158export const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex');
160/** The native platform this producer process runs on, or null when no CSO runtime platform matches it. */
161export function producerHostPlatform(platform: string = process.platform, arch: string = process.arch): ProducerPlatform | null {
162 if (platform !== 'linux') return null;
163 return arch === 'x64' ? 'linux/amd64' : arch === 'arm64' ? 'linux/arm64' : null;
164}
166export function producerInputHash(input: ProducerInput): string {
167 return sha256(JSON.stringify(input));
168}
170export function producerReceiptHash(receipt: Omit<ProducerReceipt, 'receiptHash'>): string {
171 return sha256(JSON.stringify(receipt));
172}
174export function producerInstallationIdentityHash(identity: Omit<ProducerInstallationIdentity, 'identityHash'>): string {
175 if (!identity.generation || !identity.generation.manifest || identity.generation.coreSha256 !== identity.core.sha256 ||
176 identity.generation.manifest.bytes !== 65 ||
177 identity.generation.manifest.sha256 !== sha256(`${identity.generation.coreSha256}\n`)) {
178 throw new Error('INVALID_PRODUCER_GENERATION_IDENTITY');
179 }
180 return sha256(JSON.stringify(identity));
181}
183export function producerProviderIdentityHash(identity: Omit<ProducerProviderIdentity, 'identityHash'>): string {
184 return sha256(JSON.stringify(identity));
185}
187export function producerArtifactInventoryHash(inventory: Omit<ProducerArtifactInventory, 'identityHash'>): string {
188 return sha256(JSON.stringify(inventory));
189}