preflight-agent-sdk.ts
161 lines · scripts/preflight-agent-sdk.ts at 3f79d81
1/**
2 * Preflight for the overlay efficacy harness.
3 *
4 * Confirms, before any paid eval runs:
5 * 1. `@anthropic-ai/claude-agent-sdk` loads and `query()` is the expected shape.
6 * 2. `claude-opus-4-7` is a live API model ID (not a Claude Code alias).
7 * 3. The SDK event stream contains the types we assume (system init, assistant,
8 * result) with the fields we destructure.
9 * 4. `scripts/resolvers/model-overlay.ts` resolves `{{INHERIT:claude}}` against
10 * `opus-4-7.md` with no unresolved inheritance directives.
11 * 5. A local `claude` binary exists at `which claude` so binary pinning is possible.
12 *
13 * Run: bun run scripts/preflight-agent-sdk.ts [--live]
14 *
15 * Checks 1-2 and 4-5 are free. Check 3 (the live SDK query, ~15 tokens of API
16 * spend) runs only with --live. Exit 0 on success, 1 on any failed check,
17 * 2 on a bad argument.
18 */
20import '../lib/conductor-env-shim';
21import { query, type SDKMessage } from '@anthropic-ai/claude-agent-sdk';
22import { readOverlay } from './resolvers/model-overlay';
23import { resolveClaudeBinary } from '../lib/claude-bin';
25const USAGE = `Usage: bun run scripts/preflight-agent-sdk.ts [--live]
27Free checks: overlay resolution and local claude binary pinning.
28--live also runs one end-to-end SDK query against claude-opus-4-7
29 (~15 tokens of API spend; needs ANTHROPIC_API_KEY).`;
31function parsePreflightArgs(argv: string[]): { help: boolean; live: boolean } {
32 const parsed = { help: false, live: false };
33 for (const arg of argv) {
34 if (arg === '--help' || arg === '-h') parsed.help = true;
35 else if (arg === '--live') parsed.live = true;
36 else throw new Error(`unknown argument "${arg}"`);
37 }
38 return parsed;
39}
41async function main() {
42 let args: { help: boolean; live: boolean };
43 try {
44 args = parsePreflightArgs(process.argv.slice(2));
45 } catch (err) {
46 console.error(`preflight-agent-sdk: ${(err as Error).message}\n\n${USAGE}`);
47 process.exit(2);
48 }
49 if (args.help) {
50 console.log(USAGE);
51 return;
52 }
54 const failures: string[] = [];
55 const pass = (msg: string) => console.log(` ok ${msg}`);
56 const fail = (msg: string) => {
57 console.log(` FAIL ${msg}`);
58 failures.push(msg);
59 };
61 // 1. Overlay resolver
62 console.log('1. Overlay resolver');
63 const resolved = readOverlay('opus-4-7');
64 if (!resolved) {
65 fail("readOverlay('opus-4-7') returned empty");
66 } else {
67 pass(`resolved overlay length: ${resolved.length} chars`);
68 if (resolved.includes('{{INHERIT:')) {
69 fail('resolved overlay still contains {{INHERIT:...}} directive');
70 } else {
71 pass('no unresolved INHERIT directives');
72 }
73 }
75 // 2. Local claude binary exists
76 console.log('\n2. Binary pinning');
77 let claudePath: string | null = resolveClaudeBinary();
78 if (claudePath) {
79 pass(`local claude binary: ${claudePath}`);
80 } else {
81 fail('`Bun.which("claude")` failed — cannot pin binary (set GSTACK_CLAUDE_BIN to override)');
82 }
84 // 3. SDK query end-to-end
85 console.log('\n3. SDK query end-to-end');
86 if (!args.live) {
87 console.log(' skip live query not requested — rerun with --live to spend ~15 tokens on it');
88 } else if (!process.env.ANTHROPIC_API_KEY) {
89 console.log(' skip ANTHROPIC_API_KEY not set — cannot test live query');
90 } else {
91 try {
92 const events: SDKMessage[] = [];
93 const q = query({
94 prompt: 'say pong',
95 options: {
96 model: 'claude-opus-4-7',
97 systemPrompt: '',
98 tools: [],
99 permissionMode: 'bypassPermissions',
100 allowDangerouslySkipPermissions: true,
101 settingSources: [],
102 maxTurns: 1,
103 pathToClaudeCodeExecutable: claudePath ?? undefined,
104 env: { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY },
105 },
106 });
107 for await (const ev of q) events.push(ev);
108 pass(`received ${events.length} events`);
110 const init = events.find(
111 (e) => e.type === 'system' && (e as { subtype?: string }).subtype === 'init',
112 ) as { claude_code_version?: string; model?: string } | undefined;
113 if (!init) {
114 fail('no system/init event received');
115 } else {
116 pass(`system init: claude_code_version=${init.claude_code_version}, model=${init.model}`);
117 }
119 const assistantEvents = events.filter((e) => e.type === 'assistant');
120 if (assistantEvents.length === 0) {
121 fail('no assistant events received — model ID may be rejected');
122 } else {
123 pass(`received ${assistantEvents.length} assistant event(s)`);
124 const first = assistantEvents[0] as { message?: { content?: unknown[] } };
125 const content = first.message?.content;
126 if (!Array.isArray(content)) {
127 fail('first assistant event has no content[] array');
128 } else {
129 pass(`first assistant content[] has ${content.length} block(s)`);
130 }
131 }
133 const result = events.find((e) => e.type === 'result') as
134 | { subtype?: string; total_cost_usd?: number; num_turns?: number }
135 | undefined;
136 if (!result) {
137 fail('no result event received');
138 } else {
139 pass(
140 `result: subtype=${result.subtype}, cost=$${result.total_cost_usd?.toFixed(4)}, turns=${result.num_turns}`,
141 );
142 }
143 } catch (err) {
144 fail(`SDK query threw: ${err instanceof Error ? err.message : String(err)}`);
145 }
146 }
148 console.log();
149 if (failures.length > 0) {
150 console.log(`PREFLIGHT FAILED: ${failures.length} check(s) failed`);
151 process.exit(1);
152 }
153 console.log('PREFLIGHT OK');
154}
156if (import.meta.main) {
157 main().catch((err) => {
158 console.error(err);
159 process.exit(1);
160 });
161}