terminal-agent-control.ts
429 lines · browse/src/terminal-agent-control.ts at 3f79d81
1/**
2 * terminal-agent process-control primitives shared by cli.ts spawn site,
3 * server.ts shutdown teardown, and the v1.44 watchdog/respawn loop.
4 *
5 * Why this exists: pre-v1.44 used `pkill -f terminal-agent\.ts`, which
6 * matches any process whose argv contains the string and would kill
7 * sibling gstack sessions on the same host. The agent now writes a
8 * structured `terminal-agent-pid` record (`{pid, gen, startedAt}`) and
9 * every kill site routes through `killAgentByRecord` here — identity-based,
10 * no regex.
11 *
12 * The `gen` field is a per-boot generation counter. Loopback /internal/*
13 * calls from the parent server include `X-Browse-Gen` so a slow agent that
14 * the watchdog respawned around can't accidentally service a stale grant
15 * from the old generation.
16 */
17import * as fs from 'fs';
18import * as path from 'path';
19import * as crypto from 'crypto';
20import { spawnSync } from 'child_process';
21import { safeUnlink, isProcessAlive } from './error-handling';
22import { restrictFilePermissions, mkdirSecure } from './file-permissions';
23import { atomicWriteSync } from '../../lib/fs-atomic';
24import { readPidCmdline, readPidStartTime } from './xvfb';
25import { emptyBunConfig, findWindowsBunExe, type BunExeLookup } from './bun-child-config';
27const FILETIME_UNIX_EPOCH_MS = 11_644_473_600_000n;
28let win32Kernel: any;
30function windowsProcessQuery(pid: number): { startTime: string; commandLine: string } {
31 const script = `(Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}' | Select-Object CreationDate,CommandLine | ConvertTo-Json -Compress)`;
32 try {
33 const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
34 if (result.status !== 0 || !result.stdout) return { startTime: '', commandLine: '' };
35 const processInfo = JSON.parse(result.stdout);
36 return { startTime: processInfo?.CreationDate || '', commandLine: processInfo?.CommandLine || '' };
37 } catch { return { startTime: '', commandLine: '' }; }
38}
40export const win32CreationTimeSource = {
41 read(pid: number): string | undefined {
42 if (!process.versions.bun) return;
43 try {
44 const { dlopen, FFIType, ptr } = require('bun:ffi');
45 win32Kernel ??= dlopen('kernel32.dll', {
46 OpenProcess: { args: [FFIType.u32, FFIType.i32, FFIType.u32], returns: FFIType.u64 },
47 GetProcessTimes: { args: [FFIType.u64, FFIType.ptr, FFIType.ptr, FFIType.ptr, FFIType.ptr], returns: FFIType.i32 },
48 CloseHandle: { args: [FFIType.u64], returns: FFIType.i32 },
49 }).symbols;
50 const handle = win32Kernel.OpenProcess(0x1000, 0, pid);
51 if (!handle) return;
52 try {
53 const times = new BigUint64Array(4);
54 if (!win32Kernel.GetProcessTimes(handle, ptr(times, 0), ptr(times, 8), ptr(times, 16), ptr(times, 24)) || times[0] === 0n) return;
55 return `/Date(${times[0] / 10_000n - FILETIME_UNIX_EPOCH_MS})/`;
56 } finally {
57 win32Kernel.CloseHandle(handle);
58 }
59 } catch { return; }
60 },
61};
63function agentProcessInfo(pid: number): { startTime: string; commandLine: string } {
64 if (!Number.isSafeInteger(pid) || pid <= 0) return { startTime: '', commandLine: '' };
65 if (process.platform !== 'win32') return { startTime: readPidStartTime(pid), commandLine: readPidCmdline(pid) };
66 const query = windowsProcessQuery(pid);
67 return { startTime: win32CreationTimeSource.read(pid) ?? query.startTime, commandLine: query.commandLine };
68}
70export function readAgentStartTime(pid: number): string {
71 if (!Number.isSafeInteger(pid) || pid <= 0) return '';
72 if (process.platform !== 'win32') return readPidStartTime(pid);
73 return win32CreationTimeSource.read(pid) ?? windowsProcessQuery(pid).startTime;
74}
76const pendingAgentExits = new Set<any>();
78function reclaimPublicationLock(stateDir: string, lockPath: string): boolean {
79 try {
80 const inode = fs.lstatSync(lockPath, { bigint: true });
81 if (!inode.isFile() || inode.size === 0n || inode.size > 4096n) return false;
82 const contents = fs.readFileSync(lockPath, 'utf8');
83 const lock = JSON.parse(contents);
84 const record = readAgentRecord(stateDir);
85 if (lock?.kind !== 'agent-publication-v1' || !record
86 || !Number.isSafeInteger(record.pid) || record.pid <= 0
87 || typeof record.gen !== 'string' || !record.gen
88 || typeof record.startTime !== 'string' || !record.startTime
89 || record.ownerPid !== process.pid
90 || typeof record.ownerStartTime !== 'string' || !record.ownerStartTime) return false;
91 const fields = ['pid', 'gen', 'startTime', 'ownerPid', 'ownerStartTime'] as const;
92 if (fields.some(field => lock[field] !== record[field])
93 || readAgentStartTime(process.pid) !== record.ownerStartTime) return false;
94 let present = true;
95 try { process.kill(record.pid, 0); }
96 catch (err: any) {
97 if (err?.code !== 'ESRCH') return false;
98 present = false;
99 }
100 if (present) {
101 if (readAgentStartTime(record.pid) !== record.startTime) return false;
102 if (process.platform === 'linux') {
103 const state = fs.readFileSync(`/proc/${record.pid}/stat`, 'utf8').match(/^\d+ \(.*\) ([A-Z])/u)?.[1];
104 if (state !== 'Z') return false;
105 } else if (process.platform === 'darwin') {
106 const result = spawnSync('ps', ['-p', String(record.pid), '-o', 'stat='], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
107 if (result.status !== 0 || result.stdout?.trim()?.[0] !== 'Z') return false;
108 } else return false;
109 }
110 const currentRecord = readAgentRecord(stateDir);
111 if (!currentRecord || fields.some(field => currentRecord[field] !== record[field])) return false;
112 if (present && readAgentStartTime(record.pid) !== record.startTime) return false;
113 if (fs.readFileSync(lockPath, 'utf8') !== contents) return false;
114 const current = fs.lstatSync(lockPath, { bigint: true });
115 if (!current.isFile() || current.dev !== inode.dev || current.ino !== inode.ino) return false;
116 fs.unlinkSync(lockPath);
117 return true;
118 } catch { return false; }
119}
121export function acquireAgentStateLock(stateDir: string, waitMs = 5000, publicationGen?: string): () => void {
122 mkdirSecure(stateDir);
123 const lockPath = path.join(stateDir, 'terminal-agent-pid.lock');
124 let publication: string | undefined;
125 if (publicationGen !== undefined) {
126 const record = readAgentRecord(stateDir);
127 if (!record || record.pid !== process.pid || record.gen !== publicationGen
128 || !record.ownerPid || !isOurAgent(record, record.ownerPid)) {
129 throw new Error('terminal-agent publication lock identity is unavailable');
130 }
131 publication = JSON.stringify({ kind: 'agent-publication-v1', pid: record.pid, gen: record.gen,
132 startTime: record.startTime, ownerPid: record.ownerPid, ownerStartTime: record.ownerStartTime });
133 }
134 const deadline = Date.now() + waitMs;
135 let reclaimed = false;
136 let fd: number;
137 while (true) {
138 try {
139 if (publication !== undefined) {
140 atomicWriteSync(lockPath, publication, { mode: 0o600, noReplace: true });
141 fd = fs.openSync(lockPath, 'r');
142 } else {
143 fd = fs.openSync(lockPath, 'wx', 0o600);
144 }
145 break;
146 } catch (err: any) {
147 if (err?.code === 'EEXIST' && !reclaimed && reclaimPublicationLock(stateDir, lockPath)) {
148 reclaimed = true;
149 continue;
150 }
151 if (err?.code !== 'EEXIST' || Date.now() >= deadline) {
152 throw new Error(`terminal-agent state lock unavailable at ${lockPath}: ${err?.code || err}; inspect the owning process before manual recovery`);
153 }
154 Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
155 }
156 }
157 const owned = fs.fstatSync(fd, { bigint: true });
158 return () => {
159 try {
160 const current = fs.statSync(lockPath, { bigint: true });
161 if (current.dev === owned.dev && current.ino === owned.ino) fs.unlinkSync(lockPath);
162 } catch (err: any) {
163 if (err?.code !== 'ENOENT') throw err;
164 } finally { fs.closeSync(fd); }
165 };
166}
168/**
169 * Locate the terminal-agent script on disk. In dev (cli.ts running via
170 * `bun run`), it lives next to this file in browse/src. In a compiled
171 * binary, Bun's --compile bakes the source into the executable and
172 * exposes it relative to process.execPath. Either path must work or
173 * the agent can't be spawned at all.
174 */
175/**
176 * Flags for every `bun run` child the daemon spawns. The child runs in the
177 * user's project, and Bun would otherwise load that project's .env (which can
178 * set GSTACK_CHROMIUM_NO_SANDBOX) and bunfig.toml (whose preload runs code).
179 * Built per spawn so the empty config is re-checked each time.
180 */
181export function bunChildFlags(): string[] {
182 return ['--no-env-file', `--config=${emptyBunConfig()}`];
183}
185/**
186 * The bun executable to spawn the agent with. On Windows a bare 'bun' under
187 * the Node polyfill resolves (via cross-spawn) to npm's bun.cmd shim, run as
188 * `cmd.exe /c bun ...`. That breaks two ways: windowsHide only hides cmd.exe,
189 * and the detached cmd.exe has no console, so bun.exe allocates a fresh
190 * visible one (a window that steals focus on every spawn); and proc.pid is
191 * cmd.exe's, so the agent's startup check (record.pid === process.pid) never
192 * matches and it exits after 2s, which the 60s watchdog then respawns.
193 * Spawning bun.exe directly fixes both. Falls back to 'bun' when no .exe is
194 * found, which is the old behavior.
195 */
196export function resolveBunExecutable(opts: BunExeLookup & { platform?: string } = {}): string {
197 if ((opts.platform ?? process.platform) !== 'win32') return 'bun';
198 return findWindowsBunExe(opts) ?? 'bun';
199}
201export function resolveTerminalAgentScript(searchHints: { metaDir?: string; execPath?: string } = {}): string | null {
202 const meta = searchHints.metaDir || __dirname;
203 const exec = searchHints.execPath || process.execPath;
204 const candidates = [
205 path.resolve(meta, 'terminal-agent.ts'),
206 path.resolve(path.dirname(exec), '..', 'src', 'terminal-agent.ts'),
207 ];
208 for (const c of candidates) {
209 if (fs.existsSync(c)) return c;
210 }
211 return null;
212}
214/**
215 * Spawn an owned terminal-agent. A prior record is retained until its exact
216 * process exits, and the new generation is recorded before it may bind.
217 *
218 * Used by both the CLI cold-start path (cli.ts) and the v1.44 watchdog in
219 * server.ts. Centralizing here removes a copy-paste between them and means
220 * spawn-env additions (BROWSE_OWNER_PID being the first) land in one place.
221 */
222export function spawnTerminalAgent(opts: {
223 stateFile: string;
224 serverPort: number;
225 /** PID of the browse server that owns this agent. */
226 ownerPid: number;
227 cwd?: string;
228 /** Optional extra env vars to add to the agent's process env. */
229 extraEnv?: Record<string, string>;
230 /** Override script lookup for tests. */
231 scriptPath?: string;
232}): number | null {
233 if (!Number.isSafeInteger(opts.ownerPid) || opts.ownerPid <= 0) throw new Error('terminal-agent requires a daemon owner PID');
234 const stateDir = path.dirname(opts.stateFile);
235 const script = opts.scriptPath || resolveTerminalAgentScript();
236 if (!script || !fs.existsSync(script)) return null;
237 const release = acquireAgentStateLock(stateDir);
238 try {
239 const prior = readAgentRecord(stateDir);
240 if (prior) {
241 if (prior.pid === 0) {
242 console.warn('[browse] terminal-agent startup or failed exit remains unconfirmed; retaining its reservation');
243 return null;
244 }
245 if (isAgentRecordLive(prior) && !stopAgentByRecord(prior)) {
246 console.warn(`[browse] terminal-agent PID ${prior.pid} is still running or its identity cannot be confirmed; refusing a second agent`);
247 return null;
248 }
249 clearAgentRecord(stateDir, prior);
250 safeUnlink(path.join(stateDir, 'terminal-port'));
251 safeUnlink(path.join(stateDir, 'terminal-internal-token'));
252 }
253 const ownerStartTime = readAgentStartTime(opts.ownerPid);
254 if (!ownerStartTime) throw new Error('terminal-agent owner identity is unavailable');
255 const gen = crypto.randomBytes(16).toString('base64url');
256 const reservation: AgentRecord = { pid: 0, gen, startedAt: Date.now(), ownerPid: opts.ownerPid, ownerStartTime };
257 writeAgentRecord(stateDir, reservation);
258 let proc: any;
259 try {
260 proc = (Bun as any).spawn([resolveBunExecutable(), 'run', ...bunChildFlags(), script, `--agent-gen=${gen}`], {
261 cwd: opts.cwd || process.cwd(),
262 env: {
263 ...process.env,
264 ...(opts.extraEnv || {}),
265 BROWSE_STATE_FILE: opts.stateFile,
266 BROWSE_SERVER_PORT: String(opts.serverPort),
267 BROWSE_OWNER_PID: String(opts.ownerPid),
268 BROWSE_OWNER_START_TIME: ownerStartTime,
269 BROWSE_AGENT_GEN: gen,
270 },
271 stdio: ['ignore', 'ignore', 'ignore'],
272 windowsHide: true,
273 // A non-detached Windows child is killed with its parent's job, so an
274 // agent started by `browse connect` died when the CLI exited (#2637).
275 // The owner watchdog (BROWSE_OWNER_PID) still ends it with the daemon.
276 detached: process.platform === 'win32',
277 });
278 } catch (err) {
279 clearAgentRecord(stateDir, reservation);
280 throw err;
281 }
282 const retainUntilExit = () => {
283 pendingAgentExits.add(proc);
284 proc.exited?.then(() => {
285 try {
286 const releasePending = acquireAgentStateLock(stateDir);
287 try { clearAgentRecord(stateDir, reservation); } finally { releasePending(); }
288 } catch (err) { console.warn('[browse] terminal-agent pending exit cleanup failed:', err); }
289 pendingAgentExits.delete(proc);
290 }, (err: unknown) => console.warn('[browse] terminal-agent exit remains unconfirmed:', err));
291 };
292 const pid = proc.pid;
293 const startTime = pid ? readAgentStartTime(pid) : '';
294 if (!pid || !startTime) {
295 try { proc.kill('SIGTERM'); } catch {}
296 retainUntilExit();
297 throw new Error('terminal-agent process identity is unavailable');
298 }
299 const record: AgentRecord = { pid, gen, startedAt: Date.now(), startTime, ownerPid: opts.ownerPid, ownerStartTime };
300 try {
301 writeAgentRecord(stateDir, record);
302 } catch (err) {
303 if (!stopAgentByRecord(record)) {
304 retainUntilExit();
305 throw new Error(`terminal-agent record update failed and child ${pid} exit is unconfirmed: ${err}`);
306 }
307 clearAgentRecord(stateDir, reservation);
308 throw err;
309 }
310 proc.unref?.();
311 return pid;
312 } finally { release(); }
313}
315export interface AgentRecord {
316 pid: number;
317 /** Random per-boot identifier. Loopback /internal/* sees X-Browse-Gen: <gen>. */
318 gen: string;
319 /** ms since epoch. Reserved for future PID-reuse guards. */
320 startedAt: number;
321 startTime?: string;
322 ownerPid?: number;
323 ownerStartTime?: string;
324}
326export function agentRecordPath(stateDir: string): string {
327 return path.join(stateDir, 'terminal-agent-pid');
328}
330/** Read the current record. Returns null on missing/malformed file. */
331export function readAgentRecord(stateDir: string): AgentRecord | null {
332 try {
333 const raw = fs.readFileSync(agentRecordPath(stateDir), 'utf-8');
334 const j = JSON.parse(raw);
335 if (typeof j?.pid === 'number' && typeof j?.gen === 'string' && typeof j?.startedAt === 'number') {
336 return j as AgentRecord;
337 }
338 return null;
339 } catch {
340 return null;
341 }
342}
344/** Atomic write (throws on failure — boot must not proceed on a bad record). */
345export function writeAgentRecord(stateDir: string, record: AgentRecord): void {
346 try { mkdirSecure(stateDir); } catch {}
347 const target = agentRecordPath(stateDir);
348 atomicWriteSync(target, JSON.stringify(record), { mode: 0o600 });
349 // Windows ACL hardening (POSIX chmod is redundant with mode above).
350 restrictFilePermissions(target);
351}
353export function clearAgentRecord(stateDir: string, expected?: AgentRecord): void {
354 if (expected) {
355 const current = readAgentRecord(stateDir);
356 if (!current || current.pid !== expected.pid || current.gen !== expected.gen) return;
357 }
358 safeUnlink(agentRecordPath(stateDir));
359}
361export function isAgentRecordLive(record: AgentRecord): boolean {
362 return Number.isSafeInteger(record.pid) && record.pid > 0 && isProcessAlive(record.pid);
363}
365function agentStatus(record: AgentRecord, ownerPid?: number): 'owned' | 'gone' | 'unknown' {
366 if (!isAgentRecordLive(record)) return 'gone';
367 if (!record.startTime || !record.ownerPid || !record.ownerStartTime) return 'unknown';
368 if (ownerPid !== undefined && (record.ownerPid !== ownerPid || record.ownerStartTime !== readAgentStartTime(ownerPid))) return 'unknown';
369 const actual = agentProcessInfo(record.pid);
370 if (!actual.startTime) return isAgentRecordLive(record) ? 'unknown' : 'gone';
371 if (actual.startTime !== record.startTime && process.platform !== 'win32') return 'gone';
372 if (actual.startTime !== record.startTime && !actual.commandLine) return 'unknown';
373 try {
374 let state: string | undefined;
375 if (process.platform === 'linux') {
376 state = fs.readFileSync(`/proc/${record.pid}/stat`, 'utf8').match(/^\d+ \(.*\) ([A-Z])/u)?.[1];
377 } else if (process.platform === 'darwin') {
378 const result = spawnSync('ps', ['-p', String(record.pid), '-o', 'stat='], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
379 if (result.status === 0) state = result.stdout?.trim()?.[0];
380 }
381 if (state === 'Z') return 'gone';
382 } catch {}
383 if (!isAgentRecordLive(record)) return 'gone';
384 return actual.commandLine.split(/\s+/).some(arg => arg.replace(/^['"]|['"]$/g, '') === `--agent-gen=${record.gen}`)
385 ? 'owned' : actual.startTime === record.startTime ? 'unknown' : 'gone';
386}
388export function isOurAgent(record: AgentRecord, ownerPid?: number): boolean {
389 return agentStatus(record, ownerPid) === 'owned';
390}
392export function isAgentRecordGone(record: AgentRecord): boolean {
393 return agentStatus(record) === 'gone';
394}
396/**
397 * Kill the agent identified by `record`. Signal defaults to SIGTERM (give
398 * the agent a chance to run its own SIGTERM cleanup). Returns true if a
399 * signal reached an exact-generation live process, false otherwise.
400 */
401export function killAgentByRecord(
402 record: AgentRecord,
403 signal: NodeJS.Signals = 'SIGTERM',
404): boolean {
405 if (!isOurAgent(record)) return false;
406 try { process.kill(record.pid, signal); return true; } catch { return false; }
407}
409export function stopAgentByRecord(record: AgentRecord, graceMs = 1000): boolean {
410 const initial = agentStatus(record);
411 if (initial === 'gone') return true;
412 if (initial !== 'owned') return false;
413 const waitForExit = (ms: number) => {
414 const deadline = Date.now() + ms;
415 while (Date.now() < deadline) {
416 const status = agentStatus(record);
417 if (status === 'gone') return true;
418 if (status === 'unknown') return false;
419 Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
420 }
421 return agentStatus(record) === 'gone';
422 };
423 if (!killAgentByRecord(record, 'SIGTERM')) return agentStatus(record) === 'gone';
424 if (waitForExit(graceMs)) return true;
425 const afterGrace = agentStatus(record);
426 if (afterGrace !== 'owned') return afterGrace === 'gone';
427 if (!killAgentByRecord(record, 'SIGKILL')) return agentStatus(record) === 'gone';
428 return waitForExit(graceMs);
429}