tunnel-bootstrap.ts
472 lines · ios-qa/daemon/src/tunnel-bootstrap.ts at 3f79d81
1// Bootstrap the CoreDevice tunnel to a connected iPhone or iPad running the
2// iOS app under test. Orchestrates the full hand-rolled flow we verified
3// end-to-end:
4//
5// 1. find a paired, connected device via devicectl list devices
6// 2. launch the app on it (no-op if already running)
7// 3. wait briefly for the in-app StateServer to start
8// 4. copy the boot token from the app's sandbox via devicectl copy from
9// If an earlier daemon already consumed it, relaunch the app once to mint
10// a fresh boot token, then verify the relaunched StateServer again.
11// 5. POST /auth/rotate to swap boot token → fresh in-memory token
12// 6. return a DeviceTunnel pointing at the device's IPv6 with the rotated
13// bearer that subsequent proxied requests carry
14//
15// Step 5 is critical: rotation deletes the on-disk token file, so anything
16// that copied it sees a dead credential. The Mac daemon holds the only live
17// token, which it scopes per-tailnet-session via /auth/mint.
18//
19// recoverTunnel() handles a dropped route afterwards without repeating this
20// flow: the app still holds the rotated bearer, and a second bootstrap could
21// only get a new token by relaunching the app.
23import { randomBytes } from 'crypto';
24import { spawnSync } from 'child_process';
25import type { DeviceTunnel } from './proxy';
26import {
27 listDevices,
28 resolveTunnelIPv6,
29 getDeviceTunnelIPv6,
30 getDeviceTunnelIPv6FromDevicectl,
31 isAppRunning,
32 launchApp,
33 launchEnvArgs,
34 copyFileFromAppContainer,
35 type DeviceEntry,
36 type SpawnImpl,
37 type ResolveImpl,
38} from './devicectl';
40export interface BootstrapOptions {
41 /** Target iPhone/iPad UDID. If null, picks the best connected paired device. */
42 udid?: string;
43 /** Bundle ID of the iOS app hosting the StateServer. */
44 bundleId: string;
45 /** StateServer port. Defaults to 9999. */
46 port?: number;
47 /** Token-path inside the app sandbox (relative to data container). */
48 bootTokenPath?: string;
49 /** Max time to wait for the StateServer to start after launch (ms). */
50 startupTimeoutMs?: number;
51 /** Test injection. */
52 spawnImpl?: SpawnImpl;
53 resolveImpl?: ResolveImpl;
54 fetchImpl?: typeof fetch;
55}
57export type BootstrapResult =
58 | { ok: true; tunnel: DeviceTunnel }
59 | { ok: false; error: BootstrapErrorReason; detail?: string };
61export type BootstrapErrorReason =
62 | 'no_devices'
63 | 'no_paired_device'
64 | 'device_not_found'
65 | 'multiple_devices'
66 | 'launch_failed'
67 | 'device_locked'
68 | 'state_server_unreachable'
69 | 'wrong_app'
70 | 'boot_token_unavailable'
71 | 'rotate_failed'
72 | 'resolve_failed';
74function isSupportedIOSDevice(device: DeviceEntry): boolean {
75 const platform = device.platform.trim().toLowerCase();
76 const deviceType = device.deviceType.trim().toLowerCase();
77 const model = device.model.trim().toLowerCase();
79 // productType is present even on older CoreDevice versions. Prefer the
80 // explicit platform/type fields when available, but retain productType as
81 // a compatibility fallback. An explicit non-iOS platform always loses.
82 if (platform && platform !== 'ios' && platform !== 'ipados') return false;
83 return (
84 deviceType === 'iphone'
85 || deviceType === 'ipad'
86 || model.startsWith('iphone')
87 || model.startsWith('ipad')
88 );
89}
91function isAvailableDevice(device: Pick<DeviceEntry, 'state' | 'transport'>): boolean {
92 const state = device.state.trim().toLowerCase();
93 const transport = device.transport.trim().toLowerCase();
94 // Xcode 26.6 / iOS 27 beta can report a USB-reachable iPhone as
95 // tunnelState=disconnected until the next devicectl command establishes
96 // the CoreDevice tunnel. The wired transport is the authoritative signal
97 // in that transitional state. Stale devices have no wired transport.
98 return state === 'connected'
99 || state.startsWith('available')
100 || (state === 'disconnected' && transport === 'wired');
101}
103function defaultDeviceRank(device: DeviceEntry): number {
104 if (!device.paired || !isSupportedIOSDevice(device) || !isAvailableDevice(device)) return -1;
106 const state = device.state.trim().toLowerCase();
107 const transport = device.transport.trim().toLowerCase();
108 // Prefer the USB-connected phone the user is actively working with. Then
109 // prefer an established CoreDevice tunnel over a merely available device.
110 return (transport === 'wired' ? 100 : 0)
111 + (state === 'connected' ? 10 : 0)
112 + (state.startsWith('available') ? 1 : 0);
113}
115type DeviceSelection =
116 | { ok: true; device: DeviceEntry }
117 | { ok: false; error: BootstrapErrorReason; detail?: string };
119/**
120 * Choose the device a QA session targets: the explicit UDID when one is set,
121 * otherwise the best-ranked paired iPhone or iPad. Ranks that tie (an iPhone
122 * and an iPad both on USB, say) are ambiguous, so the caller gets every
123 * candidate with its UDID instead of a silent pick. `prefer` breaks a tie in
124 * favor of the device a live session already uses.
125 */
126export function selectDevice(devices: DeviceEntry[], udid?: string, prefer?: string): DeviceSelection {
127 if (devices.length === 0) return { ok: false, error: 'no_devices' };
128 if (udid) {
129 const explicit = devices.find((d) => d.identifier === udid);
130 return explicit ? { ok: true, device: explicit } : { ok: false, error: 'device_not_found', detail: udid };
131 }
133 const bestRank = Math.max(...devices.map(defaultDeviceRank));
134 const best = bestRank < 0 ? [] : devices.filter((d) => defaultDeviceRank(d) === bestRank);
135 if (best.length === 1) return { ok: true, device: best[0]! };
136 if (best.length > 1) {
137 const preferred = best.find((d) => d.identifier === prefer);
138 if (preferred) return { ok: true, device: preferred };
139 const listing = best.map((d) => ` ${d.name} (${d.deviceType || d.model}): ${d.identifier}`).join('\n');
140 return {
141 ok: false,
142 error: 'multiple_devices',
143 detail: `${best.length} iPhones/iPads are connected and none is selected:\n${listing}\n`
144 + `Pick one, then restart the daemon:\n export GSTACK_IOS_TARGET_UDID=${best[0]!.identifier}`,
145 };
146 }
148 const pairedIOS = devices.find((d) => d.paired && isSupportedIOSDevice(d));
149 if (pairedIOS) {
150 return {
151 ok: false,
152 error: 'device_not_found',
153 detail: `paired device ${pairedIOS.name} (${pairedIOS.identifier}) is ${pairedIOS.state}; connect it over USB and unlock it`,
154 };
155 }
156 const firstIOS = devices.find(isSupportedIOSDevice);
157 if (!firstIOS) {
158 return {
159 ok: false,
160 error: 'device_not_found',
161 detail: 'no iPhone or iPad is connected; non-iOS devices are not eligible for iOS QA',
162 };
163 }
164 return {
165 ok: false,
166 error: 'no_paired_device',
167 detail: `device ${firstIOS.name} (${firstIOS.identifier}) is ${firstIOS.state}; run \`xcrun devicectl manage pair --device ${firstIOS.identifier}\` and tap Trust on the device`,
168 };
169}
171const defaultSpawn: SpawnImpl = (cmd, args) => spawnSync(cmd, args, {
172 stdio: 'pipe',
173 timeout: 60_000,
174});
176function relaunchApp(
177 udid: string,
178 bundleId: string,
179 spawn: SpawnImpl = defaultSpawn,
180): { ok: true } | { ok: false; error: 'device_locked' | 'launch_failed'; detail?: string } {
181 const env = launchEnvArgs();
182 if (!env.ok) return { ok: false, error: 'launch_failed', detail: env.error };
183 const r = spawn('xcrun', [
184 'devicectl', 'device', 'process', 'launch',
185 '--device', udid,
186 '--terminate-existing',
187 ...env.args,
188 bundleId,
189 ]);
190 if (r.status === 0) return { ok: true };
192 const detail = `${r.stderr?.toString() ?? ''}${r.stdout?.toString() ?? ''}`.trim();
193 if (detail.includes('was not, or could not be, unlocked')) {
194 return { ok: false, error: 'device_locked', detail };
195 }
196 return { ok: false, error: 'launch_failed', detail };
197}
199/**
200 * Bootstrap a real CoreDevice tunnel to an iOS app's StateServer. Used by
201 * the daemon's default tunnelProvider when GSTACK_IOS_TARGET_UDID is set
202 * (or when the user wants real-device control instead of a stub).
203 */
204export async function bootstrapTunnel(opts: BootstrapOptions): Promise<BootstrapResult> {
205 const port = opts.port ?? 9999;
206 const tokenPath = opts.bootTokenPath ?? 'tmp/gstack-ios-qa.token';
207 const startupTimeoutMs = opts.startupTimeoutMs ?? 5_000;
208 const spawn = opts.spawnImpl;
209 const resolve = opts.resolveImpl;
210 const fetchFn = opts.fetchImpl ?? fetch;
212 // Step 1: pick a device
213 const selection = selectDevice(listDevices(spawn), opts.udid);
214 if (!selection.ok) return selection;
215 const target = selection.device;
216 if (!isSupportedIOSDevice(target)) {
217 return {
218 ok: false,
219 error: 'device_not_found',
220 detail: `device ${target.name} (${target.identifier}) is ${target.platform || target.model}, not an iPhone or iPad`,
221 };
222 }
223 if (!target.paired) {
224 return {
225 ok: false,
226 error: 'no_paired_device',
227 detail: `device ${target.name} (${target.identifier}) is ${target.state}; run \`xcrun devicectl manage pair --device ${target.identifier}\` and tap Trust on the device`,
228 };
229 }
230 if (!isAvailableDevice(target)) {
231 return {
232 ok: false,
233 error: 'device_not_found',
234 detail: `device ${target.name} (${target.identifier}) is ${target.state}; connect it over USB and unlock it`,
235 };
236 }
238 // Step 2: launch app (idempotent — devicectl returns success if already running)
239 if (!isAppRunning(target.identifier, opts.bundleId, spawn)) {
240 const launched = launchApp(target.identifier, opts.bundleId, spawn);
241 if (!launched.ok) {
242 return { ok: false, error: launched.error === 'device_locked' ? 'device_locked' : 'launch_failed', detail: launched.error };
243 }
244 }
246 // Step 3: resolve tunnel IPv6. Try devicectl `info details` first (most
247 // reliable on macOS 26.x), fall through to mDNS via dns.lookup, then
248 // dns.resolve6 as a last-ditch fallback. See devicectl.ts:resolveTunnelIPv6
249 // for the rationale.
250 // When tests inject `resolve`, use it for both the mDNS-lookup path AND the
251 // legacy resolve6 path — otherwise the legacy path would make a real DNS
252 // call. In production, only `resolve` is set (to the dns.lookup-based
253 // default) and the legacy path uses the real dns.resolve6.
254 const ipv6 = await resolveTunnelIPv6({
255 udid: target.identifier,
256 deviceName: target.name,
257 spawn,
258 resolve,
259 legacyResolve: resolve,
260 });
261 if (!ipv6) {
262 return { ok: false, error: 'resolve_failed', detail: target.name };
263 }
265 // Step 4: wait for StateServer to become reachable, then scrape boot token.
266 // Probe /healthz with retries (the listener can take a moment to bind).
267 let bootTokenWriteError: string | undefined;
268 const waitForStateServer = async (): Promise<BootstrapResult | null> => {
269 const deadline = Date.now() + startupTimeoutMs;
270 while (Date.now() < deadline) {
271 try {
272 const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
273 signal: AbortSignal.timeout(2_000),
274 });
275 if (r.ok) {
276 const health = await r.json().catch(() => null) as { bundle_id?: string; boot_token_error?: string } | null;
277 // Older bridges did not identify their bundle. Preserve compatibility,
278 // but reject an explicit mismatch from current bridges: another debug
279 // app already owns the fixed StateServer port on this device.
280 if (health?.bundle_id && health.bundle_id !== opts.bundleId) {
281 return {
282 ok: false,
283 error: 'wrong_app',
284 detail: `expected ${opts.bundleId} but StateServer port ${port} belongs to ${health.bundle_id}; terminate the other debug app`,
285 };
286 }
287 bootTokenWriteError = health?.boot_token_error;
288 return null;
289 }
290 } catch { /* retry */ }
291 await new Promise((res) => setTimeout(res, 250));
292 }
293 return {
294 ok: false,
295 error: 'state_server_unreachable',
296 detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms`,
297 };
298 };
300 const healthFailure = await waitForStateServer();
301 if (healthFailure) return healthFailure;
303 const readBootToken = () => copyFileFromAppContainer({
304 udid: target.identifier,
305 bundleId: opts.bundleId,
306 sourceRelativePath: tokenPath,
307 spawn,
308 });
310 let bootToken = readBootToken();
311 if (!bootToken && bootTokenWriteError) {
312 // The app reported that it could not write the file. A relaunch would
313 // only fail the same way and wipe the app's state, so stop here.
314 return {
315 ok: false,
316 error: 'boot_token_unavailable',
317 detail: `${opts.bundleId} could not write ${tokenPath}: ${bootTokenWriteError}; fix the app's tmp/ directory, then relaunch the app`,
318 };
319 }
320 if (!bootToken) {
321 // A healthy running app can lack a boot token when an earlier daemon
322 // already rotated it. A new daemon has no way to recover that in-memory
323 // bearer, so restart exactly once to make StateServer mint a fresh one.
324 // The explicit bundle check above prevents disrupting an unrelated app
325 // that happens to own the fixed StateServer port.
326 const relaunched = relaunchApp(target.identifier, opts.bundleId, spawn);
327 if (!relaunched.ok) {
328 return { ok: false, error: relaunched.error, detail: relaunched.detail };
329 }
331 // The token is written before StateServer opens its listener. Waiting for
332 // it first prevents a stale response from the terminating process from
333 // being mistaken for readiness of the replacement process.
334 const tokenDeadline = Date.now() + startupTimeoutMs;
335 while (!bootToken && Date.now() < tokenDeadline) {
336 bootToken = readBootToken();
337 if (!bootToken) await new Promise((res) => setTimeout(res, 250));
338 }
339 if (!bootToken) {
340 return {
341 ok: false,
342 error: 'boot_token_unavailable',
343 detail: `couldn't read ${tokenPath} from ${opts.bundleId} after relaunch`,
344 };
345 }
347 const relaunchedHealthFailure = await waitForStateServer();
348 if (relaunchedHealthFailure) return relaunchedHealthFailure;
349 }
351 // Step 5: rotate the boot token to a fresh in-memory-only one.
352 const rotatedToken = randomBytes(32).toString('base64url');
353 try {
354 const r = await fetchFn(`http://[${ipv6}]:${port}/auth/rotate`, {
355 method: 'POST',
356 headers: {
357 'Authorization': `Bearer ${bootToken}`,
358 'Content-Type': 'application/json',
359 },
360 body: JSON.stringify({ new_token: rotatedToken }),
361 signal: AbortSignal.timeout(5_000),
362 });
363 if (!r.ok) {
364 return { ok: false, error: 'rotate_failed', detail: `HTTP ${r.status}` };
365 }
366 } catch (err) {
367 return { ok: false, error: 'rotate_failed', detail: (err as Error).message };
368 }
370 return {
371 ok: true,
372 tunnel: {
373 udid: target.identifier,
374 ipv6Addr: ipv6,
375 port,
376 bootTokenRotated: rotatedToken,
377 },
378 };
379}
381export type TunnelRecovery =
382 /** The app still holds the bearer; keep the session at this (maybe new) address. */
383 | { action: 'reuse'; tunnel: DeviceTunnel }
384 /** Full bootstrap needed. The old bearer was not sent to anything unproven. */
385 | { action: 'bootstrap'; reason: string }
386 /** Route still down with the app running: surface the error, keep the session. */
387 | { action: 'unavailable'; reason: string };
389export interface RecoveryOptions {
390 /** Same explicit UDID the bootstrap used, if any. */
391 udid?: string;
392 bundleId: string;
393 /**
394 * Send the bearer only to the address devicectl reports for the UDID. A
395 * session persisted by an earlier daemon has no address proven in this
396 * process, so it never falls back to mDNS or a remembered address.
397 */
398 pinnedAddressOnly?: boolean;
399 probeTimeoutMs?: number;
400 spawnImpl?: SpawnImpl;
401 resolveImpl?: ResolveImpl;
402 fetchImpl?: typeof fetch;
403}
405/**
406 * Decide how to recover a cached tunnel after a route failure (503
407 * device_disconnected / 504 upstream_timeout) without relaunching the app.
408 * CoreDevice routes blip while the app keeps running with the rotated bearer
409 * in memory, and the one-shot boot-token file is already gone, so a full
410 * bootstrap would have to relaunch the app and wipe its QA state (#1975).
411 *
412 * The bearer is only ever sent to the address `devicectl` reports for the
413 * pinned UDID, or to the address this session already used. Only a 401 (the
414 * app was replaced) or a confirmed app-absent state asks for a bootstrap.
415 */
416export async function recoverTunnel(failed: DeviceTunnel, opts: RecoveryOptions): Promise<TunnelRecovery> {
417 const spawn = opts.spawnImpl;
418 const fetchFn = opts.fetchImpl ?? fetch;
419 const timeoutMs = opts.probeTimeoutMs ?? 3_000;
421 const selection = selectDevice(listDevices(spawn), opts.udid, failed.udid);
422 if (!selection.ok) {
423 return { action: 'unavailable', reason: `${selection.error}${selection.detail ? `: ${selection.detail}` : ''}` };
424 }
425 const device = selection.device;
426 if (device.identifier !== failed.udid) {
427 return { action: 'bootstrap', reason: `target device changed from ${failed.udid} to ${device.identifier}` };
428 }
430 let address = getDeviceTunnelIPv6FromDevicectl(device.identifier, spawn);
431 if (!address && opts.pinnedAddressOnly) {
432 return { action: 'bootstrap', reason: `devicectl reported no tunnel address for ${device.identifier}` };
433 }
434 if (!address) {
435 const byName = await getDeviceTunnelIPv6(device.name, opts.resolveImpl);
436 if (byName && byName !== failed.ipv6Addr) {
437 return { action: 'bootstrap', reason: `tunnel address changed to ${byName} and devicectl could not tie it to ${failed.udid}` };
438 }
439 address = failed.ipv6Addr;
440 }
442 const base = `http://[${address}]:${failed.port}`;
443 const appGone = (reason: string): TunnelRecovery => (isAppRunning(device.identifier, opts.bundleId, spawn)
444 ? { action: 'unavailable', reason: `${reason}; ${opts.bundleId} is still running, so the session is kept` }
445 : { action: 'bootstrap', reason: `${opts.bundleId} is not running on ${device.identifier}` });
447 let health: Response;
448 try {
449 health = await fetchFn(`${base}/healthz`, { signal: AbortSignal.timeout(timeoutMs) });
450 } catch {
451 return appGone(`no /healthz response from ${base}`);
452 }
453 const owner = (await health.json().catch(() => null) as { bundle_id?: string } | null)?.bundle_id;
454 if (owner && owner !== opts.bundleId) {
455 return { action: 'bootstrap', reason: `StateServer at ${base} now belongs to ${owner}` };
456 }
458 let probe: Response;
459 try {
460 probe = await fetchFn(`${base}/state/snapshot`, {
461 headers: { 'Authorization': `Bearer ${failed.bootTokenRotated}` },
462 signal: AbortSignal.timeout(timeoutMs),
463 });
464 } catch {
465 return appGone(`authenticated probe to ${base} failed`);
466 }
467 await probe.arrayBuffer().catch(() => undefined);
468 if (probe.status === 401) {
469 return { action: 'bootstrap', reason: `${opts.bundleId} rejected the session bearer (the app was relaunched)` };
470 }
471 return { action: 'reuse', tunnel: { ...failed, ipv6Addr: address } };
472}