xprotect-heal.ts
439 lines · browse/src/xprotect-heal.ts at 3f79d81
1/**
2 * XProtect launch-kill self-heal (P0 #2554).
3 *
4 * macOS XProtect definition updates can start killing the exact Chromium
5 * revision the committed bun.lock pins (observed: revision 1208 under
6 * playwright 1.58.2 — xprotectd SIGKILLs chrome-headless-shell at spawn, so
7 * the failure surfaces as a Playwright launch timeout or a "Browser closed"
8 * error carrying `signal=SIGKILL`, never anything naming XProtect).
9 *
10 * The heal, in order, at most ONCE per process (F4):
11 * 1. Classify the launch failure against the XProtect kill signature
12 * (positive AND negative fixtures under test, F9).
13 * 2. Clear com.apple.quarantine on the Playwright cache bundles ONLY —
14 * a GSTACK_CHROMIUM_PATH bundle belongs to the wrapper/embedder and is
15 * never touched (same scope contract as probePoisonedChromiumBundle).
16 * 3. Force-reinstall Chromium FROM THE GSTACK INSTALL ROOT (ENG-OV3: the
17 * root whose node_modules pins the same playwright-core our compiled
18 * binary embeds — a cwd-resolved `bunx playwright install` would fetch
19 * the LATEST playwright's revision, which the embedded playwright-core
20 * won't find, and the one-shot guard would then block the retry).
21 * The install is BOUNDED (~120s, process-GROUP kill on timeout; E1).
22 * 4. Verify the revision dir the embedded playwright-core EXPECTS exists
23 * post-heal (registry-derived expectation, not merely install exit 0).
24 *
25 * Every action emits one structured stderr line (F11). When the heal cannot
26 * complete (offline, timeout, no install root, one-shot spent), the caller
27 * surfaces the ORIGINAL launch error plus manual
28 * `bunx playwright install chromium` guidance — the CLI never hangs on it.
29 */
31import * as fs from 'fs';
32import * as os from 'os';
33import * as path from 'path';
34import { spawn } from 'child_process';
35import { chromium } from 'playwright';
37/** F11: one structured stderr line per self-heal action. */
38function logHeal(action: string, fields: Record<string, unknown> = {}): void {
39 console.error(`[browse:xprotect-heal] ${JSON.stringify({ action, ...fields })}`);
40}
42// ─── Classifier (F9: positives AND negatives) ────────────────────────────
44/**
45 * Failure shapes that are definitively NOT an XProtect kill. Checked before
46 * the positives so an ambiguous message never triggers a pointless reinstall:
47 * - missing executable (browser was never installed / cache wiped)
48 * - spawn-level permission errors (EACCES / EPERM / ENOENT)
49 * - Linux sandbox denials (wrong OS anyway, but the text is distinctive)
50 */
51const NEGATIVE_SIGNATURES: RegExp[] = [
52 /executable doesn't exist/i,
53 /spawn\s+\S+\s+(EACCES|EPERM|ENOENT)/i,
54 /\b(EACCES|EPERM)\b/,
55 /no usable sandbox/i,
56 /failed to move to new namespace/i,
57 /suid sandbox helper/i,
58];
60/**
61 * Failure shapes an OS-level kill produces (sourced from the #2554 report
62 * plus Playwright's launch-error format): the browser process SPAWNED, then
63 * died to SIGKILL, or never became ready (launch timeout with a `<launched>`
64 * marker — the report's visible symptom, since xprotectd kills the child
65 * without Playwright ever learning why).
66 */
67const POSITIVE_SIGNATURES: RegExp[] = [
68 /<process did exit:[^>]*signal=SIGKILL/i,
69 /signal[:=]\s*['"]?SIGKILL/i,
70];
72/**
73 * True when a launch failure message matches the macOS XProtect kill
74 * signature. Platform-gated: XProtect exists only on darwin.
75 */
76export function isXProtectKillSignature(
77 message: string,
78 platform: NodeJS.Platform = process.platform,
79): boolean {
80 if (platform !== 'darwin') return false;
81 if (!message) return false;
82 for (const neg of NEGATIVE_SIGNATURES) {
83 if (neg.test(message)) return false;
84 }
85 for (const pos of POSITIVE_SIGNATURES) {
86 if (pos.test(message)) return true;
87 }
88 // XProtect kill at spawn also surfaces as a launch timeout where the
89 // process DID launch (<launched> marker present) but never became ready —
90 // this is the exact symptom the #2554 report describes.
91 return /timeout \d+\s*ms exceeded/i.test(message) && /<launched>/i.test(message);
92}
94// ─── Playwright cache path helpers (pure) ────────────────────────────────
96const REVISION_DIR_RE = /^chromium(?:_headless_shell)?-\d+$/;
98/**
99 * Walk up from a Chromium executable to its Playwright cache revision dir
100 * (e.g. …/ms-playwright/chromium-1234 or …/chromium_headless_shell-1234).
101 * Returns null when the executable is not in the standard cache layout.
102 */
103export function findPlaywrightRevisionDir(executablePath: string): string | null {
104 let dir = path.dirname(executablePath);
105 for (let i = 0; i < 8; i++) {
106 if (REVISION_DIR_RE.test(path.basename(dir))) return dir;
107 const parent = path.dirname(dir);
108 if (parent === dir) return null;
109 dir = parent;
110 }
111 return null;
112}
114/**
115 * The Chromium revision the EMBEDDED playwright-core expects, derived from
116 * the registry-computed executable path (chromium.executablePath() embeds
117 * the revision from playwright-core's browsers.json — it is not read from
118 * disk, so it stays correct even when nothing is installed yet).
119 */
120export function expectedChromiumRevision(executablePath: string): string | null {
121 const revDir = findPlaywrightRevisionDir(executablePath);
122 if (!revDir) return null;
123 const m = path.basename(revDir).match(/-(\d+)$/);
124 return m ? m[1] : null;
125}
127/**
128 * Find the gstack install root whose node_modules pins the SAME
129 * playwright-core revision our binary embeds (ENG-OV3). Candidates:
130 * the dev checkout (source runs) and the global ./setup install. A candidate
131 * qualifies only when its playwright-core/browsers.json chromium revision
132 * matches — running the reinstall anywhere else heals to the WRONG revision.
133 */
134export function findGstackInstallRoot(
135 expectedRevision: string,
136 candidates?: string[],
137): string | null {
138 const roots = candidates ?? [
139 // Dev checkout: browse/src/ → repo root. In the compiled binary
140 // __dirname points into the bunfs bundle and won't exist on disk,
141 // so this candidate simply fails the existsSync below.
142 path.resolve(__dirname, '..', '..'),
143 // Global install root (the ./setup target). os.homedir() rather than
144 // process.env.HOME: with HOME unset the env form produced the RELATIVE
145 // path '.claude/skills/gstack' under the daemon's cwd — often an
146 // untrusted repo being QA'd, whose planted node_modules would then be
147 // where the heal runs the playwright install (repo-controlled code
148 // execution). The absolute-or-skip guard below backstops the class.
149 path.join(os.homedir(), '.claude', 'skills', 'gstack'),
150 ];
151 for (const root of roots) {
152 if (!path.isAbsolute(root)) continue;
153 try {
154 const browsersJson = path.join(root, 'node_modules', 'playwright-core', 'browsers.json');
155 if (!fs.existsSync(browsersJson)) continue;
156 const parsed = JSON.parse(fs.readFileSync(browsersJson, 'utf-8'));
157 const rev = parsed?.browsers?.find((b: { name?: string }) => b?.name === 'chromium')?.revision;
158 if (String(rev) === String(expectedRevision)) return root;
159 } catch {
160 continue; // unreadable/malformed candidate — try the next one
161 }
162 }
163 return null;
164}
166// ─── Quarantine clear ────────────────────────────────────────────────────
168function defaultRunXattr(target: string): number | null {
169 const res = Bun.spawnSync(['xattr', '-dr', 'com.apple.quarantine', target], {
170 windowsHide: true,
171 stdout: 'pipe',
172 stderr: 'pipe',
173 timeout: 10_000,
174 });
175 return res.exitCode;
176}
178/**
179 * Clear com.apple.quarantine on every chromium* revision dir in the
180 * Playwright cache (the headless shell is what XProtect actually killed in
181 * #2554; the headed bundle rides along so a later headed launch doesn't
182 * re-trip). Scope contract mirrors probePoisonedChromiumBundle: NEVER act
183 * on a GSTACK_CHROMIUM_PATH bundle — that belongs to the wrapper/embedder.
184 * Best-effort: xattr failures are logged, never thrown (the forced
185 * reinstall below is the real heal).
186 */
187export function clearQuarantineOnPlaywrightCache(
188 executablePath: string,
189 runXattr: (target: string) => number | null = defaultRunXattr,
190): boolean {
191 const customPath = process.env.GSTACK_CHROMIUM_PATH;
192 if (customPath && path.resolve(executablePath) === path.resolve(customPath)) {
193 logHeal('quarantine-clear-skipped', { reason: 'custom-chromium-path' });
194 return false;
195 }
196 const revDir = findPlaywrightRevisionDir(executablePath);
197 if (!revDir) {
198 logHeal('quarantine-clear-skipped', { reason: 'not-in-playwright-cache', executablePath });
199 return false;
200 }
201 const cacheRoot = path.dirname(revDir);
202 let cleared = 0;
203 let entries: string[];
204 try {
205 entries = fs.readdirSync(cacheRoot);
206 } catch (err) {
207 logHeal('quarantine-clear-skipped', {
208 reason: 'cache-unreadable',
209 error: err instanceof Error ? err.message : String(err),
210 });
211 return false;
212 }
213 for (const entry of entries) {
214 if (!REVISION_DIR_RE.test(entry)) continue;
215 const target = path.join(cacheRoot, entry);
216 try {
217 const exitCode = runXattr(target);
218 // Non-zero usually means "no such xattr" — nothing to clear, fine.
219 logHeal('quarantine-clear', { target, exitCode });
220 cleared++;
221 } catch (err) {
222 logHeal('quarantine-clear', {
223 target,
224 error: err instanceof Error ? err.message : String(err),
225 });
226 }
227 }
228 return cleared > 0;
229}
231// ─── Bounded forced reinstall (E1) ───────────────────────────────────────
233export const XPROTECT_REINSTALL_TIMEOUT_MS = 120_000;
235export interface ReinstallResult {
236 ok: boolean;
237 reason?: string;
238 exitCode?: number | null;
239}
241/**
242 * Run `bunx playwright install --force chromium` from the gstack install
243 * root, bounded at ~120s. The child gets its own process group (detached)
244 * so a timeout kills the WHOLE tree (bunx → playwright CLI → download
245 * workers), never leaving a zombie download saturating the network.
246 */
247export function runBoundedChromiumReinstall(
248 installRoot: string,
249 timeoutMs: number = XPROTECT_REINSTALL_TIMEOUT_MS,
250): Promise<ReinstallResult> {
251 return new Promise((resolve) => {
252 let settled = false;
253 let child: ReturnType<typeof spawn>;
254 try {
255 child = spawn('bunx', ['playwright', 'install', '--force', 'chromium'], {
256 cwd: installRoot,
257 detached: true, // own process group → group-kill on timeout
258 stdio: ['ignore', 'ignore', 'pipe'],
259 windowsHide: true,
260 });
261 } catch (err) {
262 resolve({ ok: false, reason: `spawn-error: ${err instanceof Error ? err.message : String(err)}` });
263 return;
264 }
265 let stderrTail = '';
266 child.stderr?.on('data', (d: Buffer) => {
267 stderrTail = (stderrTail + String(d)).slice(-2000);
268 });
269 const timer = setTimeout(() => {
270 if (settled) return;
271 settled = true;
272 try {
273 if (child.pid) process.kill(-child.pid, 'SIGKILL'); // whole group
274 } catch (err: unknown) {
275 if ((err as NodeJS.ErrnoException)?.code !== 'ESRCH') {
276 try { child.kill('SIGKILL'); } catch { /* already gone */ }
277 }
278 }
279 resolve({ ok: false, reason: 'timeout' });
280 }, timeoutMs);
281 child.on('error', (err) => {
282 if (settled) return;
283 settled = true;
284 clearTimeout(timer);
285 resolve({ ok: false, reason: `spawn-error: ${err.message}` });
286 });
287 child.on('exit', (code) => {
288 if (settled) return;
289 settled = true;
290 clearTimeout(timer);
291 if (code === 0) {
292 resolve({ ok: true, exitCode: code });
293 } else {
294 resolve({
295 ok: false,
296 reason: `install-exit-${code}${stderrTail ? `: ${stderrTail.slice(-300)}` : ''}`,
297 exitCode: code,
298 });
299 }
300 });
301 });
302}
304// ─── One-shot orchestration (F4) ─────────────────────────────────────────
306let healAttempted = false;
308/** Test seam only — production never resets the one-shot guard. */
309export function resetXProtectHealForTests(): void {
310 healAttempted = false;
311}
313export interface XProtectHealDeps {
314 platform?: NodeJS.Platform;
315 executablePath?: () => string;
316 clearQuarantine?: (execPath: string) => boolean;
317 installRoot?: (expectedRevision: string) => string | null;
318 runReinstall?: (installRoot: string) => Promise<ReinstallResult>;
319 verifyInstalled?: (execPath: string) => boolean;
320}
322/**
323 * Attempt the XProtect self-heal for a classified launch failure.
324 *
325 * Returns true when the heal completed AND the revision dir the embedded
326 * playwright-core expects exists on disk — the caller should retry the
327 * launch exactly once. Returns false when the error doesn't match the
328 * signature, the launch used a custom executable, the one-shot guard
329 * already fired, or any heal step failed (the caller then surfaces the
330 * original error + manual guidance).
331 */
332export async function maybeHealXProtectKill(
333 err: unknown,
334 opts: { usesCustomExecutable?: boolean } = {},
335 deps: XProtectHealDeps = {},
336): Promise<boolean> {
337 const message = err instanceof Error ? err.message : String(err);
338 if (!isXProtectKillSignature(message, deps.platform ?? process.platform)) return false;
339 if (opts.usesCustomExecutable) {
340 // A GSTACK_CHROMIUM_PATH bundle belongs to the wrapper/embedder — never
341 // quarantine-clear or reinstall over it (probePoisonedChromiumBundle's
342 // scope contract).
343 logHeal('skip', { reason: 'custom-executable' });
344 return false;
345 }
346 if (healAttempted) {
347 logHeal('skip', { reason: 'already-attempted-this-process' });
348 return false;
349 }
350 healAttempted = true; // F4: at most one heal per process, even on failure
351 logHeal('classified', { signature: 'xprotect-kill' });
353 const execPath = (deps.executablePath ?? (() => chromium.executablePath()))();
354 (deps.clearQuarantine ?? clearQuarantineOnPlaywrightCache)(execPath);
356 const revision = expectedChromiumRevision(execPath);
357 if (!revision) {
358 logHeal('reinstall-skipped', { reason: 'no-revision-in-path', execPath });
359 return false;
360 }
361 const root = (deps.installRoot ?? findGstackInstallRoot)(revision);
362 if (!root) {
363 // No install root pins our revision — a cwd-resolved install would heal
364 // to the WRONG revision (ENG-OV3), so surface guidance instead.
365 logHeal('reinstall-skipped', { reason: 'no-install-root', revision });
366 return false;
367 }
369 logHeal('reinstall-start', { installRoot: root, revision, timeoutMs: XPROTECT_REINSTALL_TIMEOUT_MS });
370 const result = await (deps.runReinstall ?? runBoundedChromiumReinstall)(root);
371 if (!result.ok) {
372 logHeal('reinstall-failed', { reason: result.reason });
373 return false;
374 }
376 // F9/ENG-OV3: assert the revision dir the embedded playwright-core
377 // EXPECTS exists post-heal — install exit 0 alone can mean "installed the
378 // wrong revision" when resolution went sideways.
379 const verify = deps.verifyInstalled ?? ((p: string) => fs.existsSync(p));
380 if (!verify(execPath)) {
381 logHeal('verify-failed', { expected: execPath });
382 return false;
383 }
384 logHeal('reinstall-ok', { installRoot: root, revision });
385 return true;
386}
388/**
389 * Original launch error + manual remediation, for classified failures the
390 * heal could not fix (offline, timeout, one-shot spent, no install root).
391 */
392export function buildXProtectGuidance(originalMessage: string): string {
393 return (
394 `${originalMessage}\n` +
395 '[browse] This launch failure matches the macOS XProtect kill signature (#2554): ' +
396 "the OS killed Playwright's Chromium at spawn. Automatic self-heal did not complete. " +
397 'Fix manually: run `bunx playwright install chromium` from your gstack install ' +
398 '(the directory whose node_modules pins playwright — ~/.claude/skills/gstack for ' +
399 'global installs), then retry.'
400 );
401}
403/**
404 * Wrap a Playwright launch call with the XProtect self-heal: on a classified
405 * failure, heal once and retry the launch once. On a classified failure the
406 * heal could not fix, throw the ORIGINAL error text augmented with manual
407 * guidance. Unclassified failures pass through untouched.
408 */
409export async function launchWithXProtectHeal<T>(
410 doLaunch: () => Promise<T>,
411 opts: { usesCustomExecutable?: boolean } = {},
412 deps: XProtectHealDeps = {},
413): Promise<T> {
414 try {
415 return await doLaunch();
416 } catch (err) {
417 const healed = await maybeHealXProtectKill(err, opts, deps);
418 if (healed) {
419 logHeal('retry-launch', {});
420 try {
421 return await doLaunch();
422 } catch (retryErr) {
423 // The heal ran but the retry died too. Without this wrap the second
424 // error propagated raw and the manual-remediation guidance was lost
425 // exactly when the automatic path had just proven insufficient.
426 const retryMessage = retryErr instanceof Error ? retryErr.message : String(retryErr);
427 if (isXProtectKillSignature(retryMessage, deps.platform ?? process.platform)) {
428 throw new Error(buildXProtectGuidance(retryMessage), { cause: retryErr });
429 }
430 throw retryErr;
431 }
432 }
433 const message = err instanceof Error ? err.message : String(err);
434 if (isXProtectKillSignature(message, deps.platform ?? process.platform)) {
435 throw new Error(buildXProtectGuidance(message), { cause: err });
436 }
437 throw err;
438 }
439}