sandbox-doctor.sh
126 lines · scripts/sandbox-doctor.sh at 3f79d81
1#!/bin/sh
2# sandbox-doctor — make a syscall-supervised cloud sandbox (Vercel sandbox /
3# Conductor cloud workspace) able to run `bun run test` green.
4#
5# Root causes this script treats (all measured on a live Vercel sandbox,
6# Amazon Linux 2023, PID 1 = sandbox-init with a seccomp filter):
7#
8# 1. /dev/fd is missing on fresh boots — every bash process substitution
9# `<(...)` fails with "/dev/fd/63: No such file or directory".
10# 2. /dev/shm is 64M — concurrent Chromium instances crash.
11# 3. The seccomp supervisor spuriously fails access(2)-family syscalls for
12# BUSY processes: `git init` dies with "Cannot access work tree:
13# Permission denied", bun's existsSync returns false for files written
14# microseconds earlier (statx succeeds while access fails on the same
15# path). Per-process pressure matters: 1 serial mega-shard and 6-way
16# sharding both fail hard; 2 shards is the sweet spot. Under blanket
17# denial the whole /tmp subtree is denied while $HOME stays clean, so
18# tests run with TMPDIR under HOME.
19# 4. Every process runs with FULL capabilities (CapEff=1ffffffffff) despite
20# uid 1000 — CAP_DAC_OVERRIDE makes chmod-denial tests unfailable.
21# Tests must run under `setpriv --ambient-caps=-all --bounding-set=-all`.
22# 5. No X server — headed-browser tests (browse handoff) need Xvfb.
23# 6. No git identity — fixtures that rely on ambient user.name/email fail.
24# 7. Conductor's /conductor/bin/git shim captures $? AFTER its `if`
25# construct (POSIX resets it to 0 on a false condition with no else), so
26# every push/pull/fetch/clone/ls-remote FAILURE exits 0. Tests that
27# inject remote failures (pre-receive hooks) see phantom successes.
28# Report upstream via Conductor Help -> Send Feedback; patched locally.
29#
30# Idempotent. Run once per sandbox boot (or source ~/.bashrc, which this
31# script also seeds). Then:
32#
33# DISPLAY=:99 TMPDIR=$HOME/tmp GSTACK_FREE_JOBS=2 GSTACK_FREE_RETRY_FLAKY=1 \
34# setpriv --ambient-caps=-all --bounding-set=-all bun run test
35set -eu
36# When bash runs this script, keep heredoc bodies on temp files rather than
37# the 512-65536B pipe window (see test/heredoc-pipe-deadlock.test.ts).
38BASH_COMPAT=50
40say() { printf 'sandbox-doctor: %s\n' "$1"; }
42# 1. /dev/fd
43if [ ! -e /dev/fd ]; then
44 sudo ln -sfn /proc/self/fd /dev/fd
45 say 'restored /dev/fd -> /proc/self/fd'
46fi
48# 2. /dev/shm size (missing /dev/shm => empty df output => skip, don't abort:
49# an empty operand would be a test(1) error, fatal under set -eu)
50shm_kb="$(df -k /dev/shm 2>/dev/null | awk 'NR==2 {print $2}')"
51if [ "${shm_kb:-0}" -gt 0 ] && [ "$shm_kb" -lt 1048576 ]; then
52 sudo mount -o remount,size=4G /dev/shm
53 say 'remounted /dev/shm at 4G'
54fi
56# 3. TMPDIR under HOME (persisted via bashrc below; created here)
57mkdir -p "$HOME/tmp"
59# 5. Xvfb for headed-browser tests. dnf-gated so a non-dnf distro degrades to
60# a warning instead of aborting the remaining fixes under set -eu; the
61# running-check looks for the :99 socket specifically (an Xvfb on another
62# display would otherwise satisfy pgrep while DISPLAY=:99 points at nothing).
63if ! command -v Xvfb >/dev/null 2>&1; then
64 if command -v dnf >/dev/null 2>&1; then
65 sudo dnf install -y xorg-x11-server-Xvfb >/dev/null
66 else
67 say 'WARNING Xvfb missing and no dnf — install Xvfb manually for headed-browser tests'
68 fi
69fi
70if command -v Xvfb >/dev/null 2>&1 && [ ! -e /tmp/.X11-unix/X99 ]; then
71 Xvfb :99 -screen 0 1280x800x24 >/dev/null 2>&1 &
72 say 'started Xvfb on :99'
73fi
75# 6. git identity (only if absent — never clobber a configured one)
76git config --global user.name >/dev/null 2>&1 || {
77 git config --global user.name "$(whoami)"
78 git config --global user.email "$(whoami)@localhost"
79 say 'seeded global git identity'
80}
82# 7. Conductor git-shim exit-code bug
83if [ -f /conductor/bin/git ] && grep -q '^status=\$?' /conductor/bin/git 2>/dev/null; then
84 sudo python3 - <<'EOF'
85import os
86p = '/conductor/bin/git'
87src = open(p).read()
88old = 'exit 0\nfi\nstatus=$?'
89new = 'exit 0\nelse\n\tstatus=$?\nfi'
90if old in src:
91 # Atomic: a concurrently spawned git must never exec a truncated shim.
92 open(p + '.orig', 'w').write(src)
93 tmp = p + '.tmp'
94 open(tmp, 'w').write(src.replace(old, new))
95 os.chmod(tmp, 0o755)
96 os.replace(tmp, p)
97 print('sandbox-doctor: patched /conductor/bin/git exit-code laundering (backup: git.orig)')
98elif new not in src:
99 print('sandbox-doctor: WARNING git shim drifted from the patch pattern; laundering NOT fixed, patch by hand')
100EOF
101fi
103# Persist the env recipe for interactive shells.
104if ! grep -q 'GSTACK sandbox test env' "$HOME/.bashrc" 2>/dev/null; then
105 cat >> "$HOME/.bashrc" <<'EOF'
107# GSTACK sandbox test env (written by scripts/sandbox-doctor.sh)
108# RETRY_FLAKY=1 overrides the runner's default-OFF contract on purpose:
109# the seccomp supervisor injects spurious one-off failures under load.
110# Delete this block to restore the default.
111export TMPDIR="$HOME/tmp"
112export GSTACK_FREE_JOBS=2
113export GSTACK_FREE_RETRY_FLAKY=1
114export DISPLAY=:99
115[ -e /dev/fd ] || sudo -n ln -sfn /proc/self/fd /dev/fd 2>/dev/null || true
116[ -e /tmp/.X11-unix/X99 ] || (Xvfb :99 -screen 0 1280x800x24 >/dev/null 2>&1 &)
117# The 4G /dev/shm remount does not survive sandbox restarts; a 64M shm makes
118# concurrent Chromium (multi-tab browse tests) fail under suite load.
119shm_kb="$(df -k /dev/shm 2>/dev/null | awk 'NR==2 {print $2}')"
120[ "${shm_kb:-0}" -gt 0 ] && [ "$shm_kb" -lt 1048576 ] && sudo -n mount -o remount,size=4G /dev/shm 2>/dev/null || true
121EOF
122 say 'seeded ~/.bashrc test env'
123fi
125say 'done. run tests with:'
126say ' setpriv --ambient-caps=-all --bounding-set=-all bun run test'