pregate-remote.sh
319 lines · scripts/ubicloud/pregate-remote.sh at 3f79d81
1#!/usr/bin/env bash
2# pregate-remote.sh — gstack's declared remote_gate (.gstack/pregate.json).
3#
4# gstack-pregate runs this under 8 cores for --stage tests --remote:
5#
6# gstack-pregate --stage tests --remote --spend-cap 2 --allow-repo-commands
7#
8# with PREGATE_FILES_FILE (the selected test files, one per line),
9# PREGATE_TREE (the working-tree fingerprint being tested) and
10# PREGATE_SPEND_CAP_USD. It prints the contract line pregate reads
11# (docs/pregate.md#remote-gate):
12#
13# PREGATE_REMOTE: tree=<tree tested> tests=<n> failed=<n> spent=<usd>
14#
15# With UBICLOUD_API_KEY set it runs the files on one Ubicloud VM with the
16# environment of bun run test:ubicloud (scripts/ubicloud/test-free.sh:
17# setup-free-suite.sh, xvfb-run, GSTACK_EXPECT_BINARIES=1, the free retry
18# policy), through test-free-shards.ts --files-from. The wall timeout is the
19# whole minutes the cap buys at the size's list price (UBI_SIZE, default
20# standard-16), at most 45; under 20 (the shortest useful run) it refuses
21# before the VM is created. On the VM it recomputes bin/gstack-wtree
22# after unpack (the tree reported) and checks that the planned set's digest
23# equals the list's before any test runs. Setup rebuilds committed build
24# output exactly as the CI free lane does (lib/diagram-render/dist is not
25# byte-reproducible there); the script prints what setup rewrote, and the
26# tests must leave the post-setup tree unchanged. spent= is billed minutes (create to confirmed
27# teardown) times the size's price. Without a key it runs the same files
28# locally in parallel shards and reports spent=0.
29#
30# Reattach: the tree and the planned set's digest go into ubi-runner's run key
31# (--key-input), so a rerun after the local runner died (a machine stop, a
32# SIGKILL) reattaches to the VM still running this gate instead of paying for a
33# second one. Its minutes since created_at count against the cap: the wall
34# budget shrinks by them (PREGATE_REMOTE_CAP_SPENT when none is left), and
35# spent= bills from the original create.
36#
37# Exit codes: 0 pass (or a paid-only selection, tests=0); 1 test failures (or
38# a rejected file list); 2 missing input; 3 tree or plan mismatch (or the tree
39# moved during the run); 4 infrastructure failure (create, setup, SSH, stop
40# deadline, another gate on this machine owning the VM to reattach to); 5 the
41# spend cap or pregate's deadline refuses the run. Every refusal line starts with a
42# PREGATE_REMOTE_* reason code (docs/pregate.md#remote-gate). Nothing is
43# uploaded or spent before the local tree check.
44set -euo pipefail
45# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
46BASH_COMPAT=50
48# The whole script is one braced block, so bash parses it before running and an
49# edit to this file during a run cannot change what the run executes.
50{
51here=$(cd "$(dirname "$0")" && pwd)
52root=$(cd "$here/../.." && pwd)
53rerun='gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands'
55say() { printf 'pregate-remote: %s\n' "$@"; }
56stop() { local code=$1 reason=$2; shift 2; say "$reason: $1"; shift; [ $# -eq 0 ] || say "$@"; say "see docs/pregate.md#remote-gate"; exit "$code"; }
58[ -n "${PREGATE_FILES_FILE:-}" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_FILES_FILE is not set; run: $rerun"
59[ -n "${PREGATE_TREE:-}" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_TREE is not set; run: $rerun"
60[ -f "$PREGATE_FILES_FILE" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_FILES_FILE=$PREGATE_FILES_FILE does not exist; run: $rerun"
62started=$(date +%s)
63wtree() { (cd "$root" && bin/gstack-wtree); }
64tree=$(wtree) || stop 3 PREGATE_REMOTE_TREE_MISMATCH "bin/gstack-wtree could not fingerprint $root" "nothing uploaded, nothing spent" "rerun: $rerun"
65if [ "$tree" != "$PREGATE_TREE" ]; then
66 stop 3 PREGATE_REMOTE_TREE_MISMATCH "the working tree is ${tree:0:12}, not the planned tree ${PREGATE_TREE:0:12}" "nothing uploaded, nothing spent" "rerun: $rerun"
67fi
69logs="$root/.context/pregate-remote/$(date -u +%Y%m%dT%H%M%SZ)-$$"
70mkdir -p "$logs"
71log="$logs/run.log"
73# pregate's selection can name paid test files (skill-e2e and other paid
74# evals); the paid PR gate covers those, and test-free-shards --files-from
75# rejects them, so only the free files go on.
76files="$logs/free-files.txt"
77paid=$(cd "$root" && bun --no-env-file --config=/dev/null -e 'import { isPaidTestFile } from "./test/helpers/paid-test-set.ts"; import * as fs from "node:fs";
78const lines = fs.readFileSync(process.argv[1], "utf8").split(/\r?\n/).filter(Boolean);
79fs.writeFileSync(process.argv[2], lines.filter(f => !isPaidTestFile(f)).map(f => f + "\n").join(""));
80console.log(lines.filter(f => isPaidTestFile(f)).join("\n"));' "$PREGATE_FILES_FILE" "$files") || stop 2 PREGATE_REMOTE_USAGE "could not split $PREGATE_FILES_FILE into free and paid test files"
81if [ -n "$paid" ]; then
82 say "$(grep -c . <<<"$paid") paid test file(s) are covered by the paid PR gate, not the free tier: $(head -5 <<<"$paid" | paste -sd, - | sed 's/,/, /g')$( [ "$(grep -c . <<<"$paid")" -le 5 ] || echo " ...")"
83fi
84count=$(grep -c . "$files" || true)
85if [ "$count" -eq 0 ]; then
86 say "PREGATE_REMOTE_PAID_ONLY: every selected file is a paid test file; no free test ran, so this receipt cannot clear the obligation (tests=0)" "see docs/pregate.md#remote-gate"
87 echo "PREGATE_REMOTE: tree=$tree tests=0 failed=0 spent=0"
88 exit 0
89fi
91result_line() { grep '^\[test:free\] FILES-FROM RESULT: ' "$1" | tail -1 || true; }
92field() { sed -n "s/.* $1=\([0-9][0-9]*\).*/\1/p" <<<"$2"; }
94if [ -z "${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}" ]; then
95 say "no UBICLOUD_API_KEY: running $count files locally in parallel shards"
96 set +e
97 (cd "$root" && bun --no-env-file --config=/dev/null run scripts/test-free-shards.ts --files-from "$files") 2>&1 | tee "$log"
98 rc=${PIPESTATUS[0]}
99 set -e
100 after=$(wtree) || after=unavailable
101 if [ "$after" != "$tree" ]; then
102 (cd "$root" && git diff --name-status "$tree" "$after" 2>&1 | head -20 | sed 's/^/ moved: /') || true
103 stop 3 PREGATE_REMOTE_TREE_MOVED "the working tree moved during the run (${tree:0:12} before, ${after:0:12} after); no result reported" "log: $log" "rerun on a tree nothing else edits: $rerun"
104 fi
105 result=$(result_line "$log")
106 [ -n "$result" ] || stop "$(( rc == 0 ? 1 : rc ))" PREGATE_REMOTE_NO_RESULT "test-free-shards exited $rc without a FILES-FROM RESULT line" "log: $log"
107 say "log: $log"
108 echo "backend=local ($(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu) cores; not the CI environment)"
109 echo "PREGATE_REMOTE: tree=$tree tests=$(field tests "$result") failed=$(field failed "$result") spent=0"
110 exit "$rc"
111fi
113[ -n "${UBI_OWNER:-}" ] || stop 2 PREGATE_REMOTE_OWNER_REQUIRED "UBI_OWNER is not set: every VM must carry its owner" "nothing created, nothing spent" "rerun with UBI_OWNER=<your thread code>: $rerun"
114size=${UBI_SIZE:-standard-16}
115case $size in
116 standard-2) ppm=0.00104 ;; standard-4) ppm=0.00208 ;; standard-8) ppm=0.00416 ;;
117 standard-16) ppm=0.00831 ;; standard-30) ppm=0.01559 ;; standard-60) ppm=0.03117 ;;
118 *) stop 5 PREGATE_REMOTE_NO_PRICE "no price for UBI_SIZE=$size (priced: standard-2, -4, -8, -16, -30, -60; Ubicloud list price per minute)" "nothing created, nothing spent" ;;
119esac
120cap=${PREGATE_SPEND_CAP_USD:-}
121[[ "$cap" =~ ^[0-9]+(\.[0-9]+)?$ ]] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_SPEND_CAP_USD is not a dollar amount ('$cap'); run: $rerun"
122max_min=45 min_min=20
123wall_min=$(awk -v c="$cap" -v p="$ppm" -v m="$max_min" 'BEGIN { w = int(c / p); print (w < m ? w : m) }')
124usd() { awk -v m="$1" -v p="$ppm" 'BEGIN { s = m * p; c = int(s * 100); if (c < s * 100) c++; printf "%.2f", c / 100 }'; }
125if [ "$wall_min" -lt "$min_min" ]; then
126 stop 5 PREGATE_REMOTE_CAP_TOO_LOW "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more"
127fi
128# The enclosing gstack-pregate kills this gate at its --timeout (default the
129# policy's timeout_s, else 600 s; pregate passes it as PREGATE_TIMEOUT_S)
130# with a 5 s grace, so the VM must be told to stop early enough for teardown to
131# be confirmed first: 120 s reserved for teardown plus 15 s margin. The deadline
132# comes from PREGATE_TIMEOUT_S, else from the gstack-pregate process this gate
133# runs under (its --timeout, else 600); run by hand there is none.
134outer_s=${PREGATE_TIMEOUT_S:-} outer_from=PREGATE_TIMEOUT_S
135if [ -z "$outer_s" ]; then
136 outer_from="" pid=$PPID
137 for _ in 1 2 3 4 5 6 7 8; do
138 { [ -n "$pid" ] && [ "$pid" -gt 1 ]; } || break
139 cmdline=$(ps -o args= -p "$pid" 2>/dev/null) || break
140 read -r arg0 arg1 _ <<<"$cmdline" || true
141 case "${arg0##*/} ${arg1##*/}" in
142 "gstack-pregate "*|*" gstack-pregate")
143 outer_s=$(sed -n 's/.*--timeout[ =]\([0-9][0-9]*\).*/\1/p' <<<"$cmdline")
144 outer_s=${outer_s:-600} outer_from="gstack-pregate --timeout"
145 break ;;
146 esac
147 pid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ')
148 done
149fi
150stop_after=$(( (wall_min - 3) * 60 )) stop_why="the cap's $wall_min min wall minus 3 min for teardown"
151if [ -n "$outer_s" ]; then
152 outer_stop=$(( outer_s - ($(date +%s) - started) - 135 ))
153 if [ "$outer_stop" -lt 300 ]; then
154 stop 5 PREGATE_REMOTE_TIMEOUT_TOO_SHORT "$outer_from ${outer_s}s leaves ${outer_stop}s for the VM after 120 s reserved for teardown and 15 s margin (at least 300 s needed)" "nothing created, nothing spent" "rerun with --timeout 2700: $rerun"
155 fi
156 if [ "$outer_stop" -lt "$stop_after" ]; then stop_after=$outer_stop stop_why="$outer_from ${outer_s}s minus 120 s teardown reserve and 15 s margin"; fi
157fi
158digest=$(tr -d '\r' <"$files" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
159prelude="$logs/vm-setup.sh"
160{
161 printf '%s\n' 'set -euo pipefail' 'BASH_COMPAT=50' \
162 'command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git >/dev/null; }' \
163 'vm_tree=$(bin/gstack-wtree) || vm_tree=unavailable' \
164 "if [ \"\$vm_tree\" != $tree ]; then echo \"PREGATE_VM_TREE_MISMATCH vm=\$vm_tree\"; exit 3; fi" \
165 'echo "PREGATE_VM_TREE $vm_tree"' \
166 "cat > ~/.pregate-files.txt <<'PREGATE_FILES'"
167 tr -d '\r' <"$files" | grep .
168 printf '%s\n' 'PREGATE_FILES' "cat > ~/.pregate-run.sh <<'PREGATE_RUN'" \
169 'planned=$(bun --no-env-file --config=/dev/null run scripts/test-free-shards.ts --files-from ~/.pregate-files.txt --dry-run | sed -n "s/^\[test:free\] files-from .*: [0-9]* files, sha256=\([0-9a-f]*\)$/\1/p")' \
170 "if [ \"\$planned\" != $digest ]; then echo \"PREGATE_VM_PLAN_MISMATCH planned=\${planned:-none} expected=$digest\"; exit 3; fi" \
171 'xvfb-run -a bun run test:free --files-from ~/.pregate-files.txt' \
172 'rc=$?' \
173 'after=$(bin/gstack-wtree || echo unavailable)' \
174 'built=$(cat ~/.pregate-built-tree)' \
175 'echo "PREGATE_VM_TREE_AFTER $after"' \
176 '[ "$after" = "$built" ] || git diff --name-status "$built" "$after" 2>&1 | head -20 | sed "s/^/PREGATE_VM_MOVED /"' \
177 'exit $rc' \
178 'PREGATE_RUN' \
179 'bash scripts/ubicloud/setup-free-suite.sh' \
180 'built=$(bin/gstack-wtree || echo unavailable)' \
181 'echo "$built" > ~/.pregate-built-tree' \
182 'echo "PREGATE_VM_TREE_BUILT $built"' \
183 '[ "$built" = "$vm_tree" ] || git diff --name-status "$vm_tree" "$built" 2>&1 | head -20 | sed "s/^/PREGATE_VM_REBUILT /"'
184} >"$prelude"
186args=(run -s "$size" --src "$root" --setup "$prelude"
187 --env GSTACK_EXPECT_BINARIES=1 --env GSTACK_FREE_RETRY_FLAKY=1
188 --env GSTACK_FLAKE_LEDGER=/tmp/gstack-free-test-flake-ledger.jsonl
189 --pull "/tmp/gstack-free-test-*:$logs"
190 --pull "work/$(basename "$root")/.context/free-test-logs:$logs")
191[ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS")
192args+=(--key-input "pregate-remote tree=$tree plan=$digest")
194# A rerun of this gate for the same tree and planned set has the same
195# ubi-runner run key, so it reattaches to the VM a killed run left behind
196# (`ubi-runner: reattached to <vm> created_at=<epoch>`). The minutes that VM
197# has billed since created_at come off the cap's wall budget first.
198prior_vm=""
199attach=$(bash "$here/ubi-runner.sh" "${args[@]}" --probe -- "bash ~/.pregate-run.sh" 2>/dev/null \
200 | sed -n 's/^UBI_ATTACH_INFO: vm=\([^ ]*\) created_at=\([0-9]*\) key=[0-9a-f]* busy=\([01]\)$/\1 \2 \3/p' | tail -1) || attach=""
201if [ -n "$attach" ]; then
202 read -r prior_vm prior_created prior_busy <<<"$attach"
203 [ "$prior_busy" = 0 ] || stop 4 PREGATE_REMOTE_ATTACH_BUSY "another gate on this machine still runs VM $prior_vm for this tree and planned set" "wait for it; this run created nothing and spent nothing"
204 prior_min=$(( ($(date +%s) - prior_created + 59) / 60 ))
205 left_s=$(( (wall_min - 3 - prior_min) * 60 ))
206 if [ "$left_s" -le 0 ]; then
207 stop 5 PREGATE_REMOTE_CAP_SPENT "VM $prior_vm, created ${prior_min} min ago for this tree and planned set, has already used the cap's $wall_min min wall (\$$(usd "$prior_min") of \$$cap)" \
208 "nothing reattached; destroy it: bash scripts/ubicloud/ubi-runner.sh down $prior_vm" "or rerun with a higher --spend-cap to let it finish: $rerun"
209 fi
210 if [ "$left_s" -lt "$stop_after" ]; then stop_after=$left_s stop_why="the cap's $wall_min min wall minus 3 min for teardown and $prior_min min already billed since $prior_vm was created"; fi
211 say "reattaching to VM $prior_vm, created ${prior_min} min ago by an earlier run of this gate for the same tree and planned set"
212fi
213say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
215# The VM run gets its own session (python3, which ubi-runner already needs, for
216# setsid on Linux and macOS): pregate's --timeout kill signals this
217# script's process group and then SIGKILL it, which must never cut ubi-runner's
218# teardown short. Output goes to files (a dead reader would SIGPIPE ubi-runner),
219# stamped for the phase timings; this script streams it. A watchdog sends
220# ubi-runner TERM at the cap-derived wall timeout, and so does a TERM or INT
221# to this script; ubi-runner then destroys the VM.
222vm_session='
223exec </dev/null > >(exec >/dev/null 2>&1; trap "" INT TERM HUP; while IFS= read -r line; do printf "%s %s\n" "$(date +%s)" "$line" >>"$2"; printf "%s\n" "$line" >>"$3"; done) 2>&1
224bash "$4" "${@:5}" -- "bash ~/.pregate-run.sh" &
225runner=$!
226echo "$runner" >"$2.pid"
227( sleep "$1"; touch "$2.timedout"; kill -TERM "$runner" ) </dev/null >/dev/null 2>&1 &
228watchdog=$!
229wait "$runner"
230rc=$?
231kill "$watchdog" 2>/dev/null
232[ ! -f "$2.timedout" ] || rc=124
233exit "$rc"
234'
235stream="$logs/stream.log"
236at() { grep -m1 -E "^[0-9]+ $1" "$log" | cut -d' ' -f1 || true; }
237( while sleep 60 </dev/null >/dev/null 2>&1; do say "still running ($(( $(date +%s) - started )) s)"; done ) &
238heartbeat=$!
239vmrun="" tailer=""
240trap 'kill "$heartbeat" $tailer 2>/dev/null || true' EXIT
241trap '[ -z "$vmrun" ] || kill -TERM "$(cat "$log.pid" 2>/dev/null)" 2>/dev/null; stop 130 PREGATE_REMOTE_INTERRUPTED "interrupted: ubi-runner is tearing the VM down on its own (log: $log)"' INT TERM
243billed=0 attempt=0
244while :; do
245 attempt=$(( attempt + 1 ))
246 : >"$log"
247 : >"$stream"
248 rm -f "$log.pid" "$log.timedout" "$log.left"
249 t0=$(date +%s)
250 vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$(( (stop_after + 900 + 3599) / 3600 ))"
251 [ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
252 UBI_LEFT_FILE="$log.left" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
253 bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" &
254 vmrun=$!
255 tail -n +1 -f "$stream" &
256 tailer=$!
257 set +e
258 wait "$vmrun"
259 rc=$?
260 set -e
261 vmrun=""
262 sleep 1
263 kill "$tailer" 2>/dev/null || true
264 tailer=""
265 t_end=$(date +%s)
266 t_create=$(at 'ubi-runner: creating ')
267 # A reattached VM is billed from its original create.
268 reattached=$(sed -n 's/^[0-9]* ubi-runner: reattached to [^ ]* created_at=\([0-9][0-9]*\) .*/\1/p' "$log" | tail -1)
269 [ -z "$reattached" ] || t_create=$reattached
270 if [ -n "$t_create" ] && ! grep -q 'ubi-runner: create failed' "$log"; then minutes=$(( (t_end - t_create + 59) / 60 )); billed=$(( billed + (minutes > 0 ? minutes : 1) )); fi
271 if [ "$rc" -ne 0 ] && [ "$attempt" -eq 1 ] && grep -q 'vCPU quota exhausted' "$log"; then
272 say "vCPU quota refused the VM; one retry in ${PREGATE_REMOTE_QUOTA_WAIT_S:-60} s"
273 cp "$log" "$logs/attempt-1.log"
274 sleep "${PREGATE_REMOTE_QUOTA_WAIT_S:-60}"
275 continue
276 fi
277 break
278done
279spent=$(usd "$billed")
281t_ready=$(at 'ubi-runner: ready: ')
282t_setup=$(at 'ubi-runner: running setup')
283t_run=$(at 'ubi-runner: running on ')
284t_exit=$(at 'ubi-runner: command exited')
285span() { if [ -n "$1" ] && [ -n "$2" ]; then echo "$(( $2 - $1 ))s"; else echo "-"; fi; }
286say "phases: create $(span "$t0" "$t_ready"), setup $(span "$t_setup" "$t_run"), run $(span "$t_run" "$t_exit"), teardown $(span "$t_exit" "$t_end"); total $(( t_end - started ))s (target 600s)"
287say "log: $log"
289left=""
290[ ! -f "$log.left" ] || left=$(sed -n 's/^UBICLOUD: left=\([^ ]*\) .*/\1/p' "$log.left" | tail -1)
291if [ "$left" != 0 ]; then
292 stop 4 PREGATE_REMOTE_INFRA "VM cleanup not confirmed (UBICLOUD: left=${left:-missing}); no result reported" \
293 "VM: $vm_name; destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" "log: $log" "spent=$spent ($billed billed min on $size)"
294fi
296vm_tree=$(sed -n 's/^[0-9]* PREGATE_VM_TREE \([0-9a-f]*\)$/\1/p' "$log" | tail -1)
297mismatch=$(grep -m1 -oE 'PREGATE_VM_(TREE|PLAN)_MISMATCH .*' "$log" || true)
298if [ -n "$mismatch" ]; then
299 stop 3 PREGATE_REMOTE_VM_TREE "the VM refused before running tests: $mismatch (expected tree ${tree:0:12})" "no tests ran; spent=$spent ($billed billed min on $size)" "rerun: $rerun"
300fi
301result=$(result_line <(cut -d' ' -f2- "$log"))
302if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
303 after=$(sed -n 's/^[0-9]* PREGATE_VM_TREE_AFTER \(.*\)$/\1/p' "$log" | tail -1)
304 built=$(sed -n 's/^[0-9]* PREGATE_VM_TREE_BUILT \(.*\)$/\1/p' "$log" | tail -1)
305 sed -n 's/^[0-9]* PREGATE_VM_REBUILT / setup rebuilt (committed build output, as the CI free lane rebuilds it): /p' "$log"
306 if [ "$after" != "$built" ]; then
307 sed -n 's/^[0-9]* PREGATE_VM_MOVED / moved: /p' "$log"
308 stop 3 PREGATE_REMOTE_TREE_MOVED "the VM's tree moved during the run (${built:0:12} after setup, ${after:0:12} after tests); no result reported" "spent=$spent ($billed billed min on $size)" "rerun: $rerun"
309 fi
310 echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
311 exit $(( rc == 0 ? 0 : 1 ))
312fi
313vm=$(sed -n -e 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' -e 's/^[0-9]* ubi-runner: reattached to \([^ ]*\) .*/\1/p' "$log" | tail -1)
314down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
315cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
316[ "$rc" -ne 124 ] || cause="the VM was told to stop after ${stop_after}s ($stop_why) and torn down"
317stop 4 PREGATE_REMOTE_INFRA "infrastructure failure (ubi-runner exit $rc): ${cause:-no ubi-runner message}" \
318 "VM: ${vm:-none created}${down:+; $down}" "log: $log" "spent=$spent ($billed billed min on $size)"
319}