eval-input-cache.ts
272 lines · scripts/eval-input-cache.ts at 3f79d81
1/**
2 * Conservative reuse of verified paid results. No provider calls or import-time I/O.
3 *
4 * The caller must audit the complete consumed input/dependency closure; touchfiles
5 * alone are selection hints, not that proof. Hash the actual expanded prompts and
6 * generated documents, fixtures, runner/rubric code, dependencies, resolved models,
7 * runtime versions, parameters and relevant environment. Unknown inputs disable reuse.
8 *
9 * Receipts contain only hashes and public provenance, never prompts or credentials.
10 * They are NOT signed evidence: the caller must restore through trusted same-repo,
11 * same-PR cache transport (including its GitHub PR ref), never arbitrary artifacts
12 * or cross-PR/base fallback keys. Scope checks below supplement that trust boundary.
13 *
14 * complete inputs -> key -> fresh successful execution -> same inputs -> receipt
15 * receipt + identical key + trusted scope + age -> explicitly reported reused result
16 */
17import * as fs from 'node:fs';
18import * as path from 'node:path';
19import { isBuiltin } from 'node:module';
20import { createHash } from 'node:crypto';
21import { atomicWriteSync } from '../lib/fs-atomic';
23/**
24 * Follow literal module imports from `entries` (repo-relative) without
25 * executing them, including installed package bytes and the package.json that
26 * governs each resolved module; root bunfig/tsconfig/jsconfig are included
27 * when present. The root package.json is left to the caller, which hashes its
28 * semantic fields without the release version label.
29 */
30export function sourceDependencyClosure(root: string, entries: string[]): string[] {
31 const seen = new Set<string>();
32 const scan = new Bun.Transpiler({ loader: 'tsx' });
33 const visit = (file: string) => {
34 file = path.resolve(file);
35 const relative = path.relative(root, file).split(path.sep).join('/');
36 if (relative.startsWith('../') || path.isAbsolute(relative)) throw new Error('Dependency outside checkout');
37 if (relative === 'package.json') return;
38 if (seen.has(relative)) return;
39 seen.add(relative);
40 const source = fs.readFileSync(file, 'utf8');
41 if (!/\.[cm]?[jt]sx?$/.test(file)) return;
42 // Entrypoint scripts carry hashbangs, which scanImports does not accept.
43 // Strip only for parsing; buildEvalInputIdentity still hashes the full file.
44 for (const entry of scan.scanImports(source.replace(/^#![^\n]*(?:\n|$)/, '\n'))) {
45 if (isBuiltin(entry.path) || entry.path.startsWith('bun:')) continue;
46 const resolved = Bun.resolveSync(entry.path, path.dirname(file));
47 visit(resolved);
48 // Package export maps/defaults affect resolution independently of code.
49 let directory = path.dirname(resolved);
50 while (directory !== root && directory.startsWith(root + path.sep)) {
51 const manifest = path.join(directory, 'package.json');
52 if (fs.existsSync(manifest)) { visit(manifest); break; }
53 directory = path.dirname(directory);
54 }
55 }
56 };
57 for (const file of entries) visit(path.join(root, file));
58 for (const file of ['bunfig.toml', 'tsconfig.json', 'jsconfig.json'])
59 if (fs.existsSync(path.join(root, file))) visit(path.join(root, file));
60 return [...seen].sort();
61}
63export const EVAL_CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
64export const EVAL_CACHE_RESULT_MAX_BYTES = 16 * 1024;
65const SCHEMA = 1;
66export type EvalCacheValue = null | boolean | number | string | EvalCacheValue[] | { [key: string]: EvalCacheValue };
67type Json = EvalCacheValue;
68export interface EvalCacheScope { repository: string; pullRequest: number }
69export interface EvalInputIdentity {
70 key: string;
71 scope: EvalCacheScope;
72 caseIds: string[];
73}
74export interface EvalInputManifest {
75 root: string;
76 scope: EvalCacheScope;
77 /** These claims require an audited adapter, not automatic touchfile inference. */
78 coverage: { dependencies: 'complete'; prompts: 'complete'; environment: 'complete' };
79 unknownDependencies: string[];
80 /** Complete declared and imported source/fixture/rubric/generated-file closure. */
81 files: string[];
82 /** Complete actual prompt bytes keyed by expected case identity. */
83 prompts: Record<string, string>;
84 /** Rubric, model/request parameters, selection, retry policy and work budgets. */
85 parameters: Record<string, Json>;
86 /** Resolved models, provider endpoint, OS/arch, Bun/Node/CLI and relevant env. */
87 runtime: Record<string, Json>;
88}
89export type EvalInputIdentityResult =
90 | { status: 'eligible'; identity: EvalInputIdentity }
91 | { status: 'ineligible'; reason: string };
92export interface EvalCachePolicy {
93 purpose: 'gate' | 'periodic' | 'release' | 'marathon';
94 fresh?: boolean;
95 now?: number;
96 maxAgeMs?: number;
97}
98export interface EvalPassingProof {
99 execution: 'new';
100 finalized: true;
101 completeAttemptHistory: true;
102 exitCode: 0;
103 timedOut: false;
104 cancelled: false;
105 skipped: 0;
106 failed: 0;
107 passed: number;
108 /** Include EVERY attempt. Retry passes cannot hide their previous failures. */
109 cases: Array<{ id: string; outcome: 'passed'; attempt: 1 }>;
110 source: { runId: string; revision: string; completedAt: number };
111 /** Small public assertion input (e.g. JudgeScore); never prompts or API state. */
112 result: EvalCacheValue;
113}
114interface Receipt {
115 schema: 1;
116 identity: EvalInputIdentity;
117 proof: EvalPassingProof;
118 resultSha256: string;
119}
120export type EvalCacheLookup =
121 | { status: 'reused'; key: string; caseIds: string[]; source: EvalPassingProof['source']; result: EvalCacheValue }
122 | { status: 'miss'; reason: string };
124const hash = (bytes: string | Buffer) => createHash('sha256').update(bytes).digest('hex');
125const object = (value: unknown): value is Record<string, unknown> =>
126 value !== null && typeof value === 'object' && !Array.isArray(value)
127 && [Object.prototype, null].includes(Object.getPrototypeOf(value));
128const positive = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) > 0;
129const sorted = (values: string[]) => [...values].sort();
130const validIds = (value: unknown): value is string[] => Array.isArray(value) && value.length > 0
131 && value.every(id => typeof id === 'string' && id.trim().length > 0)
132 && new Set(value).size === value.length;
133const validScope = (value: unknown): value is EvalCacheScope => object(value)
134 && typeof value.repository === 'string' && /^[\w.-]+\/[\w.-]+$/.test(value.repository)
135 && positive(value.pullRequest);
137/** Reject non-JSON inputs instead of silently dropping undefined/functions/NaN. */
138function canonical(value: unknown): string {
139 if (value === null || typeof value === 'string' || typeof value === 'boolean') return JSON.stringify(value);
140 if (typeof value === 'number' && Number.isFinite(value)) return JSON.stringify(value);
141 if (Array.isArray(value)) return `[${Array.from(value, canonical).join(',')}]`;
142 if (object(value)) return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;
143 throw new Error('Input contains an unknown or non-JSON value');
144}
146export function buildEvalInputIdentity(input: EvalInputManifest): EvalInputIdentityResult {
147 try {
148 if (!validScope(input.scope)) throw new Error('A repository and positive PR number are required');
149 if (!object(input.coverage) || (['dependencies', 'prompts', 'environment'] as const).some(key => input.coverage[key] !== 'complete')
150 || !Array.isArray(input.unknownDependencies) || input.unknownDependencies.length !== 0) {
151 throw new Error('Consumed input coverage is incomplete or unknown');
152 }
153 if (!object(input.prompts) || !validIds(Object.keys(input.prompts))
154 || Object.values(input.prompts).some(prompt => typeof prompt !== 'string' || !prompt.trim())) {
155 throw new Error('Every expected case needs its complete nonempty prompt');
156 }
157 if (!object(input.runtime) || !Object.keys(input.runtime).length
158 || !object(input.parameters) || !Object.keys(input.parameters).length) {
159 throw new Error('Explicit runtime and request parameters are required');
160 }
161 if (!validIds(input.files)) throw new Error('A unique nonempty file closure is required');
162 const root = fs.realpathSync(input.root);
163 const files = sorted(input.files).map(relative => {
164 if (relative.includes('\\') || path.isAbsolute(relative) || relative !== path.posix.normalize(relative)
165 || relative === '..' || relative.startsWith('../')) throw new Error('Input paths must be canonical repository-relative paths');
166 const target = fs.realpathSync(path.join(root, relative));
167 const inside = path.relative(root, target);
168 if (inside === '..' || inside.startsWith(`..${path.sep}`) || path.isAbsolute(inside)) throw new Error('Input escapes repository');
169 const stat = fs.statSync(target);
170 if (!stat.isFile()) throw new Error('Input is not a regular file');
171 return { path: relative, target: inside.split(path.sep).join('/'), sha256: hash(fs.readFileSync(target)), mode: stat.mode & 0o777 };
172 });
173 const caseIds = sorted(Object.keys(input.prompts));
174 const key = hash(canonical({ schema: SCHEMA, scope: input.scope, files,
175 prompts: caseIds.map(id => ({ id, sha256: hash(input.prompts[id]!) })),
176 parameters: input.parameters, runtime: input.runtime }));
177 return { status: 'eligible', identity: { key,
178 scope: { repository: input.scope.repository, pullRequest: input.scope.pullRequest }, caseIds } };
179 } catch (error) {
180 return { status: 'ineligible', reason: error instanceof Error ? error.message : 'Cannot identify consumed inputs' };
181 }
182}
184function validIdentity(value: unknown): value is EvalInputIdentity {
185 return object(value) && typeof value.key === 'string' && /^[a-f0-9]{64}$/.test(value.key)
186 && validScope(value.scope) && validIds(value.caseIds)
187 && canonical(value.caseIds) === canonical(sorted(value.caseIds));
188}
189function bypass(policy: EvalCachePolicy): string | null {
190 if (policy.purpose !== 'gate') return 'Periodic, marathon and release validation must execute fresh';
191 if (policy.fresh) return 'Fresh validation requested';
192 if (!positive(policy.now ?? Date.now()) || !positive(policy.maxAgeMs ?? EVAL_CACHE_MAX_AGE_MS)) return 'Invalid cache age policy';
193 return null;
194}
195function validProof(proof: unknown, identity: EvalInputIdentity, now: number): proof is EvalPassingProof {
196 if (!object(proof) || proof.execution !== 'new' || proof.finalized !== true || proof.completeAttemptHistory !== true
197 || proof.exitCode !== 0 || proof.timedOut !== false || proof.cancelled !== false || proof.skipped !== 0 || proof.failed !== 0
198 || proof.passed !== identity.caseIds.length || !Array.isArray(proof.cases) || proof.cases.length !== identity.caseIds.length) return false;
199 const ids: string[] = [];
200 for (const entry of proof.cases) {
201 if (!object(entry) || typeof entry.id !== 'string' || entry.outcome !== 'passed' || entry.attempt !== 1) return false;
202 ids.push(entry.id);
203 }
204 if (canonical(sorted(ids)) !== canonical(identity.caseIds)) return false;
205 if (Buffer.byteLength(canonical(proof.result)) > EVAL_CACHE_RESULT_MAX_BYTES) return false;
206 return object(proof.source) && typeof proof.source.runId === 'string' && /^[\w./-]{1,160}$/.test(proof.source.runId)
207 && typeof proof.source.revision === 'string' && /^[a-f0-9]{40}$/.test(proof.source.revision)
208 && positive(proof.source.completedAt) && proof.source.completedAt <= now;
209}
211export function lookupEvalInputCache(options: EvalCachePolicy & {
212 cacheDir: string; identity: EvalInputIdentity;
213 /** Validate result shape AND current assertions before granting cache credit. */
214 validateResult: (value: EvalCacheValue) => boolean;
215}): EvalCacheLookup {
216 const disabled = bypass(options);
217 if (disabled) return { status: 'miss', reason: disabled };
218 try {
219 if (!validIdentity(options.identity)) throw new Error('Invalid current input identity');
220 const filename = path.join(options.cacheDir, `${options.identity.key}.json`);
221 if (!fs.lstatSync(filename).isFile()) throw new Error('Receipt is not a regular file');
222 // Read the same inode we inspect; do not follow a link swapped in after lstat.
223 const fd = fs.openSync(filename, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
224 let receipt: unknown;
225 try {
226 const stat = fs.fstatSync(fd);
227 if (!stat.isFile() || stat.size > EVAL_CACHE_RESULT_MAX_BYTES + 32 * 1024) throw new Error('Invalid receipt file');
228 receipt = JSON.parse(fs.readFileSync(fd, 'utf8'));
229 } finally { fs.closeSync(fd); }
230 const now = options.now ?? Date.now();
231 if (!object(receipt) || receipt.schema !== SCHEMA || !validIdentity(receipt.identity)
232 || canonical(receipt.identity) !== canonical(options.identity)
233 || !validProof(receipt.proof, options.identity, now)) throw new Error('Invalid or incomplete passing receipt');
234 if (receipt.resultSha256 !== hash(canonical(receipt.proof.result)) || options.validateResult(receipt.proof.result) !== true) {
235 throw new Error('Cached result failed current validation');
236 }
237 if (now - receipt.proof.source.completedAt >= (options.maxAgeMs ?? EVAL_CACHE_MAX_AGE_MS)) throw new Error('Passing receipt expired');
238 return { status: 'reused', key: options.identity.key, caseIds: [...options.identity.caseIds],
239 source: { ...receipt.proof.source }, result: receipt.proof.result };
240 } catch {
241 return { status: 'miss', reason: 'No valid unexpired passing receipt for these inputs' };
242 }
243}
245export function storeEvalInputCache(options: EvalCachePolicy & {
246 cacheDir: string; before: EvalInputIdentity; after: EvalInputIdentity; proof: EvalPassingProof;
247}): { status: 'stored'; key: string } | { status: 'not-stored'; reason: string } {
248 const disabled = bypass(options);
249 if (disabled) return { status: 'not-stored', reason: disabled };
250 try {
251 if (!validIdentity(options.before) || !validIdentity(options.after)
252 || canonical(options.before) !== canonical(options.after)) throw new Error('Inputs changed during execution');
253 const now = options.now ?? Date.now();
254 if (!validProof(options.proof, options.before, now)) throw new Error('Execution did not produce complete fresh passing evidence');
255 if (now - options.proof.source.completedAt >= (options.maxAgeMs ?? EVAL_CACHE_MAX_AGE_MS)) throw new Error('Execution evidence expired');
256 fs.mkdirSync(options.cacheDir, { recursive: true, mode: 0o700 });
257 if (!fs.lstatSync(options.cacheDir).isDirectory()) throw new Error('Cache directory is not a regular directory');
258 // Whitelist proof fields: callers may pass richer collector objects containing
259 // transcripts, prompts or operational paths which must not enter cache storage.
260 const p = options.proof;
261 const proof: EvalPassingProof = { execution: p.execution, finalized: p.finalized,
262 completeAttemptHistory: p.completeAttemptHistory, exitCode: p.exitCode,
263 timedOut: p.timedOut, cancelled: p.cancelled, skipped: p.skipped, failed: p.failed,
264 passed: p.passed, cases: p.cases.map(c => ({ id: c.id, outcome: c.outcome, attempt: c.attempt })),
265 source: { runId: p.source.runId, revision: p.source.revision, completedAt: p.source.completedAt }, result: p.result };
266 const receipt: Receipt = { schema: SCHEMA, identity: options.before, proof, resultSha256: hash(canonical(proof.result)) };
267 atomicWriteSync(path.join(options.cacheDir, `${options.before.key}.json`), JSON.stringify(receipt) + '\n', { mode: 0o600 });
268 return { status: 'stored', key: options.before.key };
269 } catch (error) {
270 return { status: 'not-stored', reason: error instanceof Error ? error.message : 'Could not save passing evidence' };
271 }
272}