ubi-runner.sh
1600 lines · scripts/ubicloud/ubi-runner.sh at 3f79d81
1#!/usr/bin/env bash
2# ubi-runner — run a command on an ephemeral Ubicloud VM.
3#
4# UBICLOUD_API_KEY=... scripts/ubicloud/ubi-runner.sh run --setup S.sh -- 'make test'
5#
6# UBICLOUD_API_TOKEN is accepted as an alias for UBICLOUD_API_KEY. SSH
7# connections are multiplexed per VM, and `pack` / `unpack` upload one prebuilt
8# checkout tarball to many VMs. This is the repo copy the user-scope Capy
9# `ubicloud` skill and gbrain's scripts/ubicloud/ubi-runner.sh are kept in sync
10# with (plan E2); `bun run test:ubicloud` drives it through test-free.sh.
11#
12# Scratch record: every VM this machine creates is appended to a VM-id file
13# (UBI_VM_RECORD; default ~/.capy/work/ubi-runner/vms.tsv when ~/.capy/work
14# exists, else $STATE_ROOT/vms.tsv) as `name location owner created_at pid`,
15# and removed once `down` confirms it gone. A resumed Capy thread reads that
16# file to find VMs an earlier turn left behind (docs/TESTING_INTERNALS.md,
17# "Resuming after a wake").
18#
19# Creates a VM (standard-16 by default), streams the checkout to it (tracked +
20# untracked-unignored files + .git, so uncommitted edits are included), runs an
21# optional setup script and the command, copies requested artifacts back, and
22# destroys the VM on every exit path. Exits with the command's status.
23# Needs only bash, curl, python3, ssh, ssh-keygen, and tar locally, so it works
24# from dev boxes, containers, and cloud sandboxes.
25#
26# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>-t<hours>. <owner> is UBI_OWNER
27# (lowercased, letters and digits only, at most 12 characters, starting with a
28# letter). `up`, `run` and `verify` refuse without it (UBI_OWNER_REQUIRED), so
29# every VM names whoever can be asked about it; read-only commands fall back to a
30# per-machine id kept in $STATE_ROOT/owner-id. Ubicloud names allow 63 characters
31# of [a-z0-9-], so a full name stays under 45.
32#
33# Teardown: `up` records each VM's state (location, create status) before it
34# sends the create request, and `down` destroys and polls until the VM is
35# confirmed gone or provably never existed, waiting out a create that may still
36# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, QUIT, and HUP
37# until a remote phase is live) runs that `down`; further signals are ignored
38# until it finishes. A create refused
39# for vCPU quota prints the project's usage per owner.
40#
41# The owner-age sweep is off by default (the TTL reap below always runs). With UBI_GC_HOURS set to a positive
42# number, every `up` first destroys this owner's VMs older than that; `gc HOURS`
43# runs the same sweep on demand. Neither ever touches another owner's VMs.
44#
45# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
46# 1-999, default 3), the creator's own declaration of how long the VM may live.
47# `reap` destroys every VM in the project whose TTL expired more than ten
48# minutes ago, whichever owner made it, and this owner's own ubirun-<owner>-*
49# names with no TTL once they are UBI_LEGACY_HOURS old (default 4; another
50# owner's or an untagged TTL-less VM is never reaped); every `up` runs it first (UBI_REAP=0 skips it), so a runner
51# killed by a sleeping machine leaves nothing behind for longer than its TTL.
52# Names outside the ubirun- prefix are never reaped.
53#
54# Leftover line: `run` and `verify` end with
55# UBICLOUD: left=<0|1|unknown> owner=<owner> vm=<name> others=<n>
56# after teardown: left counts this invocation's own VM, checked against the
57# project's VM list, and others counts this owner's other VMs (information only).
58# A left other than 0 turns a passing command into exit 1; `run --keep` reports
59# its live VM as left=1 kept=1 and keeps its exit status. With UBI_LEFT_FILE set,
60# the line is also appended to that file, out of reach of the command's output.
61#
62# Quota preflight: when the project's vCPU ceiling is known (UBI_QUOTA_VCPUS, or
63# the maximum a refusal printed, cached in $STATE_ROOT/quota-max), `up` adds the
64# size's vCPUs to the project's running VMs and refuses before any create when
65# that exceeds it; UBI_QUOTA_WAIT=<seconds> waits for room instead.
66#
67# Watchdog: `run`, `verify` and `ssh NAME COMMAND` stream the remote output
68# through a watchdog that logs a heartbeat every UBI_HEARTBEAT seconds (300)
69# and kills a command silent for UBI_IDLE_TIMEOUT seconds (1800) or older than
70# UBI_RUN_TIMEOUT seconds (0, unlimited), exiting 124 with the last 40 lines,
71# so a hung verify fails fast instead of holding the VM until its TTL.
72#
73# Detached phases and reattach: `run` and `verify` start their setup, command
74# and tier phases on the VM detached from the ssh connection (setsid; stdout to
75# ~/.ubirun-<phase>.out, stderr to .err, the session leader's pid in .pid, and
76# the exit status in .exit, written by rename once the streams are closed) and
77# stream both files back. Before creating a VM they compute a run key locally
78# (the checkout's content tree, the command, the setup script's content, every
79# --env entry, each --pass name with a sha256 of its value, size, location,
80# image, storage, origin/HEAD, tiers and this script's own sha256) and look for
81# a run record (<state dir>/<vm>/run) with the same key whose VM still exists:
82# its runner alive (lease pid and start time) refuses with UBI_ATTACH_BUSY,
83# otherwise the run reattaches, printing `reattached to <vm> created_at=<epoch>
84# key=<k12>`, and continues from the first phase without an exit file. After
85# unpack the VM recomputes the tree and refuses a mismatch (UBI_VM_TREE_MISMATCH).
86# A leader gone with no exit file exits 125 (UBI_ATTACH_LOST). Signals: INT
87# and TERM tear the VM down as before; HUP once a remote phase is live detaches
88# (UBI_DETACHED, exit 129) and leaves the VM under its TTL; SIGKILL or a machine
89# stop leaves the record and the VM, and the next run of the same command
90# reattaches. `attach NAME` resumes a recorded run explicitly. Reattach never
91# extends a TTL, and the reaper still applies.
92#
93# Tier checkpoints: a .gstack/ubicloud.json with `tiers` runs each tier as its
94# own detached phase after setup, and records start/pass/fail per tier under
95# UBI_CHECKPOINT_DIR/<run key>/ (write-once JSON plus the tier's log, published
96# under a lock with a monotonic generation). A rerun reuses a pass younger than
97# UBI_CHECKPOINT_MAX_AGE whose log re-hashes, so a resume on a fresh VM reruns
98# only setup and the tiers without a pass.
99#
100# `verify` is the one entry point for a repository's full remote verify: it
101# reads .gstack/ubicloud.json (setup, command, size, location, env, pass, pull)
102# and runs it through `run`, so lanes share one committed definition instead of
103# per-lane wrappers.
104set -euo pipefail
105# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
106BASH_COMPAT=50
108API="${UBICLOUD_API_URL:-https://api.ubicloud.com}"
109STATE_ROOT="${UBI_RUNNER_STATE:-${XDG_STATE_HOME:-$HOME/.local/state}/ubi-runner}"
110DEFAULT_SIZE="${UBI_SIZE:-standard-16}"
111DEFAULT_LOCATION="${UBI_LOCATION:-eu-central-h1}"
112GC_HOURS="${UBI_GC_HOURS:-0}"
113API_TIMEOUT="${UBI_API_TIMEOUT:-120}"
114CREATE_GRACE="${UBI_CREATE_GRACE:-180}"
115DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"
116POLL="${UBI_POLL_SECONDS:-5}"
117ALLOWED_STORAGE="${UBI_ALLOWED_STORAGE:-320 640}"
118TTL_HOURS="${UBI_TTL_HOURS:-3}"
119LEGACY_HOURS="${UBI_LEGACY_HOURS:-4}"
120REAP="${UBI_REAP:-1}"
121REAP_GRACE=600
122IDLE_TIMEOUT="${UBI_IDLE_TIMEOUT:-1800}"
123RUN_TIMEOUT="${UBI_RUN_TIMEOUT:-0}"
124HEARTBEAT="${UBI_HEARTBEAT:-300}"
125QUOTA_VCPUS="${UBI_QUOTA_VCPUS:-}"
126QUOTA_WAIT="${UBI_QUOTA_WAIT:-0}"
127PREFIX="ubirun"
128if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
129elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
130else VM_RECORD="$STATE_ROOT/vms.tsv"; fi
131if [ -n "${UBI_CHECKPOINT_DIR:-}" ]; then CHECKPOINT_DIR="$UBI_CHECKPOINT_DIR"
132elif [ -d "$HOME/.capy/work" ]; then CHECKPOINT_DIR="$HOME/.capy/work/ubi-runner/checkpoints"
133else CHECKPOINT_DIR="$STATE_ROOT/checkpoints"; fi
134CHECKPOINT_MAX_AGE="${UBI_CHECKPOINT_MAX_AGE:-86400}"
135REATTACH_MARGIN="${UBI_REATTACH_MARGIN:-900}"
137die() { echo "ubi-runner: $*" >&2; exit 1; }
138log() { echo "ubi-runner: $*" >&2; }
140for bin in curl python3 ssh ssh-keygen tar; do
141 command -v "$bin" >/dev/null || die "$bin is required"
142done
143UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
144[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
145for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT CHECKPOINT_MAX_AGE REATTACH_MARGIN; do
146 [[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
147done
148[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
150owner_id() {
151 local raw=${UBI_OWNER:-} file="$STATE_ROOT/owner-id" tmp
152 if [ -z "$raw" ]; then
153 if [ ! -s "$file" ]; then
154 mkdir -p "$STATE_ROOT"
155 tmp="$file.$$"
156 echo "m$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')" >"$tmp"
157 mv -n "$tmp" "$file"
158 rm -f "$tmp"
159 fi
160 raw=$(cat "$file")
161 fi
162 raw=$(printf %s "$raw" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9')
163 [ -n "$raw" ] || die "UBI_OWNER must contain letters or digits"
164 [[ "$raw" =~ ^[a-z] ]] || raw="u$raw"
165 echo "${raw:0:12}"
166}
167OWNER=$(owner_id)
169# require_owner: up, run and verify create VMs, and a VM must name its owner
170# and carry a TTL. Only creates check UBI_TTL_HOURS, so an old value never
171# blocks down, reap or list.
172require_owner() {
173 [ -n "${UBI_OWNER:-}" ] || die "UBI_OWNER is not set: every VM must carry its owner; export UBI_OWNER=<your thread or session code> (for example gsta164) and rerun (UBI_OWNER_REQUIRED)"
174 [[ "$TTL_HOURS" =~ ^[1-9][0-9]{0,2}$ ]] || die "UBI_TTL_HOURS must be a whole number of hours from 1 to 999 with no leading zero (every VM carries a TTL; set it above your longest run)"
175}
177# leftover FIELDS...: print the UBICLOUD: leftover line, and append it to
178# UBI_LEFT_FILE when set, so a caller can read it apart from the command's output.
179leftover() {
180 echo "UBICLOUD: $*" >&2
181 [ -z "${UBI_LEFT_FILE:-}" ] || echo "UBICLOUD: $*" >>"$UBI_LEFT_FILE"
182}
184# api OUT ARGS...: send one Ubicloud CLI command, write the body to OUT, and
185# print the HTTP status (000 when no response arrived).
186api() {
187 local out=$1 code
188 shift
189 code=$(python3 -c 'import json,sys; print(json.dumps({"argv": sys.argv[1:]}))' "$@" \
190 | curl -sS -o "$out" -w '%{http_code}' --max-time "$API_TIMEOUT" -X POST \
191 -H "Authorization: Bearer $UBICLOUD_API_KEY" \
192 -H 'Accept: text/plain' -H 'Content-Type: application/json' \
193 -H 'X-Ubi-Version: 1.0.0' --data @- "$API/cli") || true
194 echo "${code:-000}"
195}
197cli() {
198 local out code
199 out=$(mktemp)
200 code=$(api "$out" "$@")
201 if [ "$code" = 000 ]; then
202 rm -f "$out"
203 die "request failed: ubi $*"
204 fi
205 if [ "$code" != 200 ]; then
206 cat "$out" >&2
207 rm -f "$out"
208 return 1
209 fi
210 cat "$out"
211 rm -f "$out"
212}
214state_dir() { echo "$STATE_ROOT/$1"; }
216load() {
217 local dir
218 dir=$(state_dir "$1")
219 [ -f "$dir/env" ] || die "no local state for VM '$1' (created on another machine?)"
220 # shellcheck disable=SC1091
221 . "$dir/env"
222 KEY="$dir/key"
223}
225ssh_opts() {
226 echo -i "$KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
227 -o LogLevel=ERROR -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ConnectTimeout=10 \
228 -o ControlMaster=auto -o "ControlPath=$(dirname "$KEY")/ctl" -o ControlPersist=300
229}
231# Ubuntu's default umask (002) leaves new directories group-writable, which
232# permission-checking code under test rejects; every remote command uses 022.
233remote() {
234 local name=$1; shift
235 load "$name"
236 # shellcheck disable=SC2046
237 ssh $(ssh_opts) "ubi@$IP" "umask 022; $*"
238}
240mtime() { stat -c %Y "$1" 2>/dev/null || stat -f %m "$1"; }
242# sq STRING: STRING as one single-quoted shell word.
243sq() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; }
245# remote_cancel NAME SIDFILE: end the session a watched command started on the
246# VM (its leader pid is in SIDFILE there), bounded to 20 seconds. Killing the
247# local ssh client alone leaves a silent remote command running until it next
248# writes to the closed channel.
249remote_cancel() {
250 local name=$1 sidf=$2 cpid i=0
251 remote "$name" "sid=\$(cat $sidf 2>/dev/null); rm -f $sidf; [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 </dev/null &
252 cpid=$!
253 while kill -0 "$cpid" 2>/dev/null && [ "$i" -lt 20 ]; do sleep 1; i=$(( i + 1 )); done
254 kill "$cpid" 2>/dev/null || true
255 wait "$cpid" 2>/dev/null || true
256}
258# watched NAME PHASE COMMAND: run COMMAND on the VM, streaming its output and
259# keeping a copy of both streams in the VM's state dir, with a heartbeat every HEARTBEAT
260# seconds and a watchdog that kills it after IDLE_TIMEOUT seconds of silence or
261# RUN_TIMEOUT seconds in total: exit 124, reason and the last 40 lines on
262# stderr. The ssh runs in its own session (python3's setsid) so the watchdog
263# kills it and anything it started, not just the shell that launched it; on
264# the VM the command runs in its own session too (setsid where the image has
265# it), whose leader pid is written to ~/.ubirun-PHASE-<runner pid>.sid so
266# remote_cancel can end it when the client is killed.
267watched() {
268 local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps inner sidf
269 sidf="~/.ubirun-$phase-$$.sid"
270 load "$name"
271 out="$(state_dir "$name")/$phase.log"
272 : >"$out"
273 inner=$(sq "printf %s \$\$ >$sidf; $cmd")
274 # The sid file outlives the command on purpose: a background child it left
275 # behind keeps the ssh channel open after the wrapper exits, and remote_cancel
276 # still needs the session id to end it. remote_cancel removes the file.
277 cmd="if command -v setsid >/dev/null 2>&1; then setsid -w bash -c $inner; else bash -c $inner; fi"
278 # The caller's stdin is handed to the background ssh explicitly (a background
279 # job in a non-interactive shell would otherwise read /dev/null), so piped
280 # input reaches the remote command as it did before the watchdog.
281 exec 3<&0
282 # shellcheck disable=SC2046
283 ( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
284 ssh $(ssh_opts) "ubi@$IP" "umask 022; $cmd" ) <&3 > >(tee -a "$out") 2> >(tee -a "$out" >&2) &
285 pid=$!
286 exec 3<&-
287 # A signal to this process takes the ssh session down with it, so Ctrl-C on
288 # `ssh NAME COMMAND` leaves no orphaned client behind.
289 old_traps=$(trap -p INT TERM)
290 trap 'kill -TERM -- "-$pid" 2>/dev/null; exit 130' INT TERM
291 start=$(date +%s)
292 last_beat=$start
293 while :; do
294 sleep 1
295 kill -0 "$pid" 2>/dev/null || break
296 now=$(date +%s)
297 elapsed=$(( now - start ))
298 idle=$(( now - $(mtime "$out") ))
299 if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
300 why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
301 elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
302 why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
303 fi
304 if [ -n "$why" ]; then
305 # The command may have finished while the clock was read; a finished
306 # command is never reported as a timeout.
307 kill -0 "$pid" 2>/dev/null || break
308 log "watchdog: $phase $why; killing it. Last output:"
309 tail -n 40 "$out" >&2
310 kill -TERM -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true
311 wait "$pid" 2>/dev/null || rc=$?
312 trap - INT TERM; eval "$old_traps"
313 # A command killed by the watchdog never exits 0; a 0 here means it
314 # finished in the instant before the kill, and that result stands.
315 if [ "$rc" -eq 0 ]; then
316 log "watchdog: $phase finished before it could be killed; its exit 0 stands"
317 return 0
318 fi
319 remote_cancel "$name" "$sidf"
320 return 124
321 fi
322 if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
323 log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
324 last_beat=$now
325 fi
326 done
327 wait "$pid" || rc=$?
328 trap - INT TERM; eval "$old_traps"
329 return "$rc"
330}
332proc_start() { ps -o lstart= -p "$1" 2>/dev/null | tr -s ' ' || true; }
333fsize() { if [ -f "$1" ]; then wc -c <"$1" | tr -d ' '; else echo 0; fi; }
334sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | cut -d' ' -f1; else shasum -a 256 | cut -d' ' -f1; fi; }
336# TREE_SH: print the content tree of the current directory, the same way here
337# and on the VM: at a checkout's root, `git add -A` into a copy of its index
338# (tracked plus untracked files that are not ignored, the files pack uploads);
339# elsewhere, the same into a scratch repository over the directory.
340TREE_SH='idx=$(mktemp -d "${TMPDIR:-/tmp}/ubirun-tree.XXXXXX") || exit 1
341if [ "$(git rev-parse --show-toplevel 2>/dev/null)" = "$(pwd -P)" ]; then
342 gi=$(git rev-parse --git-path index)
343 if [ "$(git config --bool core.splitIndex)" != true ] && [ -f "$gi" ]; then cp "$gi" "$idx/index"; fi
344else
345 git init -q --bare "$idx/git" && export GIT_DIR="$idx/git" GIT_WORK_TREE=.
346fi
347GIT_INDEX_FILE="$idx/index" git add -A >/dev/null 2>&1 && GIT_INDEX_FILE="$idx/index" git write-tree
348rc=$?; rm -rf "$idx"; exit $rc'
350# vm_tree NAME QDEST: the content tree of QDEST on the VM.
351vm_tree() {
352 remote "$1" "command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git; } >/dev/null 2>&1; cd $2 && bash -c $(sq "$TREE_SH")" </dev/null
353}
355base_sha() { git -C "$1" rev-parse -q --verify 'refs/remotes/origin/HEAD^{commit}' 2>/dev/null || echo none; }
357# phase_status NAME PHASE: "exit <rc>" once the phase wrote its exit file,
358# "alive" while its session leader runs, "lost" when the leader is gone with no
359# exit file, "none" when it never started.
360phase_status() {
361 local p="~/.ubirun-$2"
362 remote "$1" "if [ -f $p.exit ]; then echo exit \$(cat $p.exit); elif [ -s $p.pid ] && kill -0 \$(cat $p.pid) 2>/dev/null; then echo alive; elif [ -s $p.pid ]; then echo lost; else echo none; fi" </dev/null
363}
365# phase_start NAME PHASE INNER: start INNER on the VM in its own session,
366# detached from this ssh connection. The leader records its pid, runs INNER with
367# stdout and stderr in files, and writes the exit status by rename only after
368# INNER's streams are closed.
369phase_start() {
370 local p="~/.ubirun-$2" inner
371 inner=$(sq "echo \$\$ >$p.pid.tmp && mv $p.pid.tmp $p.pid; ( $3 ) >$p.out 2>$p.err </dev/null; echo \$? >$p.exit.tmp && mv $p.exit.tmp $p.exit")
372 remote "$1" "rm -f $p.exit $p.pid; : >$p.out; : >$p.err; if command -v setsid >/dev/null 2>&1; then setsid -f bash -c $inner </dev/null >/dev/null 2>&1; else nohup bash -c $inner </dev/null >/dev/null 2>&1 & fi" </dev/null >/dev/null
373}
375# stream_cmd PHASE OUT_OFFSET ERR_OFFSET: follow both output files from the
376# given byte offsets until the session leader exits.
377stream_cmd() {
378 local p="~/.ubirun-$1"
379 printf '%s' "i=0; while [ ! -s $p.pid ] && [ ! -f $p.exit ] && [ \$i -lt 600 ]; do sleep 0.1; i=\$((i + 1)); done; pid=\$(cat $p.pid 2>/dev/null); f=; [ -z \"\$pid\" ] || f=\"-f --pid=\$pid\"; tail -c +$(( $3 + 1 )) \$f $p.err >&2 & tail -c +$(( $2 + 1 )) \$f $p.out; wait"
380}
382PHASE_LIVE="" STREAM_PID="" RUN_KEY="" TREE=""
384# on_hup: once a remote phase is live, HUP detaches: the VM keeps running under
385# its TTL and the next run of the same command reattaches. Before that, HUP
386# tears down like INT and TERM.
387on_hup() {
388 [ -n "$PHASE_LIVE" ] || exit 130
389 [ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null || true
390 trap - EXIT
391 rm -f "$(state_dir "$RUN_VM")/lease"
392 leftover "left=1 owner=$OWNER vm=$RUN_VM detached=1"
393 echo "UBI_DETACHED: vm=$RUN_VM key=${RUN_KEY:0:12} reattach: rerun the same command" >&2
394 exit 129
395}
397# detached NAME PHASE INNER: run INNER as a detached phase (or reattach to it),
398# streaming its output to this process and to the VM's state dir
399# (<phase>.out, <phase>.err and the combined <phase>.log) under the same
400# heartbeat and watchdog as watched. RUN_TIMEOUT counts from the phase's
401# recorded start, not from a reattach. Returns the phase's exit status, 124
402# after a watchdog kill, 125 when the session is gone without an exit status.
403detached() {
404 local name=$1 phase=$2 inner=$3 dir st rc=0 tries=0 start now idle elapsed last_beat why old_traps spid sst oo oe
405 load "$name"
406 dir=$(state_dir "$name")
407 local lout="$dir/$phase.out" lerr="$dir/$phase.err" out="$dir/$phase.log" p="~/.ubirun-$phase"
408 st=none
409 if [ -f "$dir/phase.$phase.start" ]; then
410 st=$(phase_status "$name" "$phase") || st=""
411 fi
412 case $st in
413 none)
414 : >"$lout"; : >"$lerr"; : >"$out"
415 date +%s >"$dir/phase.$phase.start"
416 phase_start "$name" "$phase" "$inner" || { log "could not start $phase on $name"; return 255; } ;;
417 alive) log "$phase is still running on $name; following it" ;;
418 "exit "*) ;;
419 lost) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; return 125 ;;
420 *) log "could not read the state of $phase on $name"; return 255 ;;
421 esac
422 PHASE_LIVE=1
423 start=$(cat "$dir/phase.$phase.start")
424 # A runner killed on this machine leaves its stream client behind; it would
425 # append to the same local copies.
426 if [ -f "$dir/stream" ] && read -r spid sst <"$dir/stream" && [ -n "$spid" ] && [ "$(proc_start "$spid")" = "$sst" ]; then
427 kill -TERM -- "-$spid" 2>/dev/null || true
428 fi
429 oo=$(fsize "$lout") oe=$(fsize "$lerr")
430 if [ "$st" != none ]; then head -c "$oo" "$lout"; head -c "$oe" "$lerr" >&2; fi
431 old_traps=$(trap -p INT TERM)
432 trap '[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null; exit 130' INT TERM
433 last_beat=$(date +%s)
434 while :; do
435 touch "$out"
436 # shellcheck disable=SC2046
437 ( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
438 ssh $(ssh_opts) "ubi@$IP" "umask 022; $(stream_cmd "$phase" "$oo" "$oe")" ) \
439 </dev/null > >(tee -a "$lout" | tee -a "$out") 2> >(tee -a "$lerr" | tee -a "$out" >&2) &
440 STREAM_PID=$!
441 echo "$STREAM_PID $(proc_start "$STREAM_PID")" >"$dir/stream"
442 why=""
443 while :; do
444 sleep 1
445 kill -0 "$STREAM_PID" 2>/dev/null || break
446 now=$(date +%s)
447 elapsed=$(( now - start ))
448 idle=$(( now - $(mtime "$out") ))
449 if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
450 why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
451 elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
452 why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
453 fi
454 [ -z "$why" ] || break
455 if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
456 log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
457 last_beat=$now
458 fi
459 done
460 if [ -n "$why" ]; then
461 log "watchdog: $phase $why; killing it. Last output:"
462 tail -n 40 "$out" >&2
463 kill -TERM -- "-$STREAM_PID" 2>/dev/null || kill "$STREAM_PID" 2>/dev/null || true
464 wait "$STREAM_PID" 2>/dev/null || true
465 STREAM_PID=""
466 # A phase that finished in the instant before the kill keeps its exit 0.
467 if [ "$(phase_status "$name" "$phase" 2>/dev/null)" = "exit 0" ]; then
468 log "watchdog: $phase finished before it could be killed; its exit 0 stands"
469 rc=0
470 else
471 remote_cancel "$name" "$p.pid"
472 rc=124
473 fi
474 break
475 fi
476 wait "$STREAM_PID" 2>/dev/null || true
477 STREAM_PID=""
478 st=$(phase_status "$name" "$phase") || st=""
479 case $st in
480 "exit "*) rc=${st#exit }; break ;;
481 lost|none) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; rc=125; break ;;
482 esac
483 tries=$(( tries + 1 ))
484 if [ "$tries" -ge 5 ]; then log "lost the connection to $phase on $name 5 times; giving up (the VM keeps it until teardown)"; rc=255; break; fi
485 log "the stream of $phase on $name dropped; following it again"
486 sleep "$POLL"
487 oo=$(fsize "$lout") oe=$(fsize "$lerr")
488 done
489 rm -f "$dir/stream"
490 trap - INT TERM; eval "$old_traps"
491 return "$rc"
492}
494# write_record DIR: the run record a later run of the same command reattaches
495# by (key, tree, and this invocation for `attach NAME`). Remote files of each
496# phase: ~/.ubirun-<phase>.{out,err,exit,pid} (setup, command, tier-<name>).
497qwords() { local x; for x in "$@"; do printf '%q ' "$x"; done; }
498write_record() {
499 {
500 printf 'RUN_KEY=%q\nTREE=%q\nPULL_ROOT=%q\nRESUME=%q\nRERUN_FAILED=%q\n' "$RUN_KEY" "$TREE" "$PULL_ROOT" "$RESUME" "$RERUN_FAILED"
501 printf 'RUN_ARGS=(%s)\n' "$(qwords "${RUN_ARGS[@]}")"
502 printf 'TIER_NAMES=(%s)\n' "$(qwords ${TIER_NAMES[@]+"${TIER_NAMES[@]}"})"
503 printf 'TIER_CMDS=(%s)\n' "$(qwords ${TIER_CMDS[@]+"${TIER_CMDS[@]}"})"
504 echo "# remote files: ~/.ubirun-<phase>.{out,err,exit,pid}; phase starts: phase.<phase>.start; lease: lease (pid start time)"
505 } >"$1/run.$$" && mv "$1/run.$$" "$1/run"
506}
508lease_take() { echo "$$ $(proc_start $$)" >"$1/lease.$$" && mv "$1/lease.$$" "$1/lease"; }
509lease_busy() {
510 local pid st
511 [ -f "$1/lease" ] || return 1
512 read -r pid st <"$1/lease" || true
513 [ -n "$pid" ] && [ "$pid" != "$$" ] && kill -0 "$pid" 2>/dev/null && [ "$(proc_start "$pid")" = "$st" ]
514}
516# vm_alive NAME: the recorded VM still answers show. One confirmed gone is
517# cleaned up (state dir and scratch record), so its record is never tried again.
518vm_alive() {
519 local n=$1 out code
520 ( load "$n"; [ "$CREATED" = yes ] && [ -n "$IP" ] ) 2>/dev/null || return 1
521 out=$(mktemp)
522 code=$(load "$n"; api "$out" vm "$LOCATION/$n" show)
523 rm -f "$out"
524 case $code in
525 200) return 0 ;;
526 404) log "run record for $n: the VM is gone; starting a new one"; cmd_down "$n" >/dev/null 2>&1 || true; return 1 ;;
527 *) return 1 ;;
528 esac
529}
531# ttl_short NAME SETUP: true when NAME's TTL (its created_at plus the name's
532# -t<hours>, minus now) does not cover the run still to do plus
533# REATTACH_MARGIN, with the numbers in TTL_WHY. The run still to do is the
534# UBI_RUN_TIMEOUT bound of each phase that has not finished (setup when SETUP is
535# set, then the command or every tier the plan does not reuse), counted from a
536# started phase's recorded start; it is 0 when UBI_RUN_TIMEOUT is 0.
537TTL_WHY=""
538ttl_short() {
539 local name=$1 dir created left now need=0 p i phases=()
540 [[ "$name" =~ -t([0-9]{1,3})$ ]] || return 1
541 dir=$(state_dir "$name")
542 created=$(load "$name"; echo "$CREATE_AT")
543 now=$(date +%s)
544 left=$(( created + 10#${BASH_REMATCH[1]} * 3600 - now ))
545 [ -z "$2" ] || phases+=(setup)
546 if [ ${#TIER_NAMES[@]} -gt 0 ]; then
547 for i in "${!TIER_NAMES[@]}"; do [ "${TIER_ACT[i]}" = reuse ] || phases+=("tier-${TIER_NAMES[i]}"); done
548 else
549 phases+=(command)
550 fi
551 if [ "$RUN_TIMEOUT" -gt 0 ]; then
552 for p in "${phases[@]}"; do
553 if [ -f "$dir/phase.$p.start" ]; then i=$(( RUN_TIMEOUT - (now - $(cat "$dir/phase.$p.start")) )); [ "$i" -gt 0 ] || i=0
554 else i=$RUN_TIMEOUT; fi
555 need=$(( need + i ))
556 done
557 fi
558 TTL_WHY="ttl_left=${left}s needs=$(( need + REATTACH_MARGIN ))s (run still to do ${need}s + UBI_REATTACH_MARGIN=${REATTACH_MARGIN}s)"
559 [ "$left" -lt $(( need + REATTACH_MARGIN )) ]
560}
562# find_record: the VM of a run record with this RUN_KEY whose VM still exists.
563find_record() {
564 local f n
565 for f in "$STATE_ROOT"/*/run; do
566 [ -f "$f" ] || continue
567 grep -qx "RUN_KEY=$RUN_KEY" "$f" || continue
568 n=$(basename "$(dirname "$f")")
569 if vm_alive "$n"; then echo "$n"; return 0; fi
570 done
571}
573# Tier checkpoints: write-once records <tier>.<epoch>.<pid>.json beside a copy
574# of the tier's log, under CHECKPOINT_DIR/<run key>/, published under a lock
575# with a monotonic generation; the newest generation per tier decides.
576CKPT_PY='
577import fcntl, hashlib, json, os, shutil, sys, time
578op, d = sys.argv[1], sys.argv[2]
579def records():
580 out = []
581 for f in (os.listdir(d) if os.path.isdir(d) else []):
582 if not f.endswith(".json") or f.startswith("."):
583 continue
584 try:
585 r = json.load(open(os.path.join(d, f)))
586 except (OSError, ValueError):
587 continue
588 if isinstance(r, dict) and isinstance(r.get("generation"), int):
589 r["_file"] = f
590 out.append(r)
591 return out
592def newest(rs, tier):
593 rs = [r for r in rs if r.get("tier") == tier]
594 return max(rs, key=lambda r: r["generation"]) if rs else None
595def sha(p):
596 h = hashlib.sha256()
597 with open(p, "rb") as fh:
598 for b in iter(lambda: fh.read(65536), b""):
599 h.update(b)
600 return h.hexdigest()
601if op == "decide":
602 now, maxage, resume, rerun = int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] == "1", sys.argv[6] == "1"
603 rs = records()
604 for tier in sys.argv[7:]:
605 r = newest(rs, tier)
606 f = r["_file"] if r else "-"
607 age = now - int(r.get("finished_at") or r.get("started_at") or now) if r else 0
608 dur = int(r.get("duration_s") or 0) if r else 0
609 code = str(r.get("exit", "-")) if r else "-"
610 log = os.path.join(d, str(r.get("log"))) if r and r.get("log") else ""
611 if not resume:
612 act, why = "run", "--no-resume"
613 elif r is None:
614 act, why = "run", "no record"
615 elif r.get("status") == "start":
616 act, why = "run", "interrupted (only a start record, %s)" % f
617 elif r.get("status") == "pass":
618 if age >= maxage:
619 act, why = "run", "pass %s is %ds old (UBI_CHECKPOINT_MAX_AGE=%d)" % (f, age, maxage)
620 elif not log or not os.path.isfile(log) or sha(log) != r.get("log_sha256"):
621 act, why = "run", "pass %s: its log does not re-hash" % f
622 else:
623 act, why = "reuse", "-"
624 elif r.get("status") == "fail":
625 act, why = ("rerun", "prior fail %s (exit %s; --rerun-failed)" % (f, code)) if rerun else ("blocked", "-")
626 else:
627 act, why = "run", "unreadable record %s" % f
628 print("\t".join([tier, act, f, str(age), str(dur), code, why]))
629elif op == "publish":
630 tier, status, vm, started, finished, code, log, startgen, key, tree, pid = sys.argv[3:14]
631 os.makedirs(d, exist_ok=True)
632 with open(os.path.join(d, ".lock"), "a") as lk:
633 fcntl.flock(lk, fcntl.LOCK_EX)
634 rs = records()
635 gen = max([r["generation"] for r in rs] + [0]) + 1
636 if status != "start":
637 r = newest(rs, tier)
638 if r is None or str(r["generation"]) != startgen:
639 print("stale writer: its start record is generation %s, the newest for %s is %s" % (startgen, tier, r["generation"] if r else "none"), file=sys.stderr)
640 sys.exit(2)
641 rec = {"tier": tier, "status": status, "generation": gen, "vm": vm, "key": key, "tree": tree, "started_at": int(started)}
642 if status != "start":
643 rec.update(finished_at=int(finished), duration_s=int(finished) - int(started), exit=int(code))
644 epoch = int(time.time())
645 while os.path.exists(os.path.join(d, "%s.%d.%s.json" % (tier, epoch, pid))) or os.path.exists(os.path.join(d, "%s.%d.%s.log" % (tier, epoch, pid))):
646 epoch += 1
647 stem = "%s.%d.%s" % (tier, epoch, pid)
648 if status != "start":
649 shutil.copyfile(log, os.path.join(d, stem + ".log"))
650 rec["log"], rec["log_sha256"] = stem + ".log", sha(os.path.join(d, stem + ".log"))
651 tmp = os.path.join(d, ".%s.tmp" % stem)
652 with open(tmp, "w") as fh:
653 json.dump(rec, fh, sort_keys=True)
654 fh.write("\n")
655 os.link(tmp, os.path.join(d, stem + ".json"))
656 os.unlink(tmp)
657 print(gen)
658'
660# ckpt_publish TIER STATUS VM STARTED FINISHED EXIT LOG STARTGEN: print the new
661# record's generation; fails (nothing published) for a stale writer.
662ckpt_publish() { python3 -c "$CKPT_PY" publish "$CHECKPOINT_DIR/$RUN_KEY" "$@" "$RUN_KEY" "$TREE" "$$"; }
664TIER_ACT=() TIER_TO_RUN=0 TIER_PASSED=0 TIER_REUSED="" TIER_FAILED="" TIER_PRIOR="" TIER_SAVED=0 TIER_BLOCKED="" TIER_BLOCKED_RC=1
666# tier_plan: decide each tier from its newest checkpoint and print one
667# UBI_VERIFY_RESUME line per tier before anything runs.
668tier_plan() {
669 local plan t act f age dur code why
670 plan=$(python3 -c "$CKPT_PY" decide "$CHECKPOINT_DIR/$RUN_KEY" "$(date +%s)" "$CHECKPOINT_MAX_AGE" "$RESUME" "$RERUN_FAILED" "${TIER_NAMES[@]}") \
671 || die "verify: could not read the tier checkpoints in $CHECKPOINT_DIR/$RUN_KEY"
672 while IFS=$'\t' read -r t act f age dur code why; do
673 case $act in
674 reuse)
675 echo "UBI_VERIFY_RESUME: tier=$t reuse $f (${age}s)" >&2
676 TIER_REUSED=${TIER_REUSED:+$TIER_REUSED,}$t TIER_SAVED=$(( TIER_SAVED + dur )) TIER_PASSED=$(( TIER_PASSED + 1 )) ;;
677 blocked)
678 echo "UBI_VERIFY_RESUME: tier=$t fail $f (${age}s, exit $code): not rerun without --rerun-failed" >&2
679 if [ -z "$TIER_BLOCKED" ]; then TIER_BLOCKED=$t; if [[ "$code" =~ ^[0-9]+$ ]]; then TIER_BLOCKED_RC=$code; fi; fi ;;
680 rerun)
681 echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
682 TIER_PRIOR=${TIER_PRIOR:+$TIER_PRIOR,}$t:$code TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
683 *)
684 echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
685 TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
686 esac
687 TIER_ACT+=("$act")
688 done <<<"$plan"
689}
691verify_line() {
692 echo "UBI_VERIFY: tree=${TREE:0:12} key=${RUN_KEY:0:12} tiers=${#TIER_NAMES[@]} passed=$TIER_PASSED reused=${TIER_REUSED:-none} failed=${TIER_FAILED:-none} prior_failures=${TIER_PRIOR:-none} saved_s=$TIER_SAVED"
693}
695# run_tiers NAME QDEST: run every tier the plan did not reuse, in order, each
696# as its own detached phase; stop at the first failure and return its status.
697# A pass is recorded only when the VM's tree after the tier equals the tree
698# before it.
699run_tiers() {
700 local name=$1 qdest=$2 dir i t phase before after trc started finished gen
701 dir=$(state_dir "$name")
702 for i in "${!TIER_NAMES[@]}"; do
703 [ "${TIER_ACT[i]}" != reuse ] || continue
704 t=${TIER_NAMES[i]} phase="tier-${TIER_NAMES[i]}"
705 if [ ! -f "$dir/phase.$phase.start" ]; then
706 before=$(vm_tree "$name" "$qdest") || before=unavailable
707 echo "$before" >"$dir/phase.$phase.tree"
708 if gen=$(ckpt_publish "$t" start "$name" "$(date +%s)" 0 0 "" 0); then echo "$gen" >"$dir/phase.$phase.gen"
709 else echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t start record (the tier still runs)" >&2; fi
710 fi
711 log "running tier $t on $name: ${TIER_CMDS[i]}"
712 trc=0
713 detached "$name" "$phase" "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "${TIER_CMDS[i]}")" || trc=$?
714 started=$(cat "$dir/phase.$phase.start" 2>/dev/null || date +%s) finished=$(date +%s)
715 log "tier $t exited $trc after $(( finished - started ))s"
716 case $trc in 125|255) TIER_FAILED=$t; return "$trc" ;; esac
717 gen=$(cat "$dir/phase.$phase.gen" 2>/dev/null || echo 0)
718 if [ "$trc" -ne 0 ]; then
719 ckpt_publish "$t" fail "$name" "$started" "$finished" "$trc" "$dir/$phase.log" "$gen" >/dev/null \
720 || echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t fail record (the result stands)" >&2
721 TIER_FAILED=$t
722 return "$trc"
723 fi
724 TIER_PASSED=$(( TIER_PASSED + 1 ))
725 after=$(vm_tree "$name" "$qdest") || after=unavailable
726 before=$(cat "$dir/phase.$phase.tree" 2>/dev/null || echo unavailable)
727 if [ "$after" != "$before" ] || [ "$after" = unavailable ]; then
728 echo "UBI_VERIFY_TREE_MOVED: tier=$t before=${before:0:12} after=${after:0:12}: a tier must leave the source tree as it found it; no pass record" >&2
729 continue
730 fi
731 ckpt_publish "$t" pass "$name" "$started" "$finished" 0 "$dir/$phase.log" "$gen" >/dev/null \
732 || echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t pass record (the result stands)" >&2
733 done
734}
736# vm_rows: "location name owner expires created" for every VM in the project;
737# owner is "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner
738# did not name; expires is the epoch the name's -t<hours> TTL ends at, or "-";
739# created is the name's epoch, or "-".
740vm_rows() {
741 local loc name owner created expires
742 cli vm list -N -f location,name | while read -r loc name; do
743 owner=. created="" expires=-
744 if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-([0-9]{10})- ]]; then
745 owner=${BASH_REMATCH[1]} created=${BASH_REMATCH[2]}
746 elif [[ "$name" =~ ^${PREFIX}-([0-9]{10})- ]]; then
747 owner=- created=${BASH_REMATCH[1]}
748 fi
749 if [ -n "$created" ] && [[ "$name" =~ -t([0-9]{1,3})$ ]]; then
750 expires=$(( 10#$created + 10#${BASH_REMATCH[1]} * 3600 ))
751 fi
752 echo "$loc $name $owner $expires ${created:--}"
753 done
754}
756# reap: destroy every VM whose name carries a TTL that ended more than
757# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
758# own declaration), and the caller's own TTL-less ubirun-<UBI_OWNER>-* names at
759# least LEGACY_HOURS old. Another owner's TTL-less VM, an untagged one, and names
760# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.
761cmd_reap() {
762 local now rows loc name owner expires created n=0 failed=0
763 [[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"
764 now=$(date +%s)
765 rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
766 while read -r loc name owner expires created; do
767 [ -n "$name" ] || continue
768 if [ "$expires" != - ]; then
769 [ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
770 log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
771 else
772 [ -n "${UBI_OWNER:-}" ] && [ "$owner" = "$OWNER" ] && [ "$created" != - ] || continue
773 [ $(( now - 10#$created )) -ge $(( LEGACY_HOURS * 3600 )) ] || continue
774 log "reaping TTL-less $loc/$name (owner $owner; $(( (now - 10#$created) / 3600 ))h old, legacy sweep at ${LEGACY_HOURS}h)"
775 fi
776 n=$(( n + 1 ))
777 cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
778 done <<<"$rows"
779 [ "$n" -gt 0 ] || log "reap: no expired VMs"
780 [ "$failed" -eq 0 ] || { log "reap: $failed of $n expired VMs could not be destroyed"; return 1; }
781}
783cmd_gc() {
784 local hours=${1:-$GC_HOURS} now loc name owner
785 [[ "$hours" =~ ^[0-9]+$ ]] || die "gc: HOURS must be a whole number (or set UBI_GC_HOURS)"
786 if [ "$hours" -eq 0 ]; then log "gc: no sweep (give HOURS > 0 or set UBI_GC_HOURS); nothing listed, nothing destroyed"; return 0; fi
787 now=$(date +%s)
788 vm_rows | while read -r loc name owner _; do
789 [ "$owner" = "$OWNER" ] || continue
790 [[ "$name" =~ -([0-9]{10})- ]] || continue
791 if [ $(( (now - BASH_REMATCH[1]) / 3600 )) -ge "$hours" ]; then
792 log "destroying stale $loc/$name (owner $OWNER, older than ${hours}h)"
793 cmd_down "$name" -l "$loc" || log "failed to destroy $loc/$name"
794 fi
795 done
796}
798cmd_list() {
799 case ${1:-} in
800 "") cli vm list ;;
801 --mine) vm_rows | awk -v o="$OWNER" '$3==o {print $1 " " $2}' ;;
802 *) die "list: unknown option $1 (use --mine)" ;;
803 esac
804}
806# usage: VMs and vCPUs per owner, from each VM's size (standard-16 = 16 vCPUs).
807cmd_usage() {
808 local tmp loc name owner i=0
809 tmp=$(mktemp -d)
810 while read -r loc name owner _; do
811 i=$(( i + 1 ))
812 cli vm "$loc/$name" show 2>/dev/null | sed -n "s/^size: /$owner /p" >"$tmp/$i" &
813 [ $(( i % 8 )) -ne 0 ] || wait
814 done < <(vm_rows)
815 wait
816 cat "$tmp"/* 2>/dev/null | awk -v me="$OWNER" '
817 { n = $2; sub(/^.*-/, "", n); o = $1 == "-" ? "(untagged)" : $1 == "." ? "(other)" : $1
818 vms[o]++; cpu[o] += n; total += n; count++ }
819 END {
820 printf "%-14s %4s %6s\n", "OWNER", "VMS", "VCPUS"
821 for (o in cpu) printf "%-14s %4d %6d%s\n", o, vms[o], cpu[o], o == me ? " (you)" : "" | "sort -k3,3nr"
822 close("sort -k3,3nr")
823 printf "%-14s %4d %6d\n", "total", count, total
824 }'
825 rm -rf "$tmp"
826}
828write_state() {
829 printf 'NAME=%q\nLOCATION=%q\nSIZE=%q\nIP=%q\nCREATED=%q\nCREATE_AT=%q\nCREATE_PID=%q\n' \
830 "$1" "$2" "$3" "$4" "$5" "$6" "$$" >"$(state_dir "$1")/env"
831}
833# record_vm NAME LOCATION: append to the scratch VM-id file (record-before-create).
834record_vm() {
835 mkdir -p "$(dirname "$VM_RECORD")" 2>/dev/null || return 0
836 printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$OWNER" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$$" >>"$VM_RECORD"
837}
839# forget_vm NAME: drop the VM from the scratch file once it is confirmed gone.
840forget_vm() {
841 [ -f "$VM_RECORD" ] || return 0
842 local tmp="$VM_RECORD.$$"
843 awk -F'\t' -v n="$1" '$1!=n' "$VM_RECORD" >"$tmp" && mv "$tmp" "$VM_RECORD"
844}
846# records: print the scratch file (VMs this machine created and has not confirmed gone).
847cmd_records() {
848 echo "record: $VM_RECORD"
849 [ -s "$VM_RECORD" ] || { echo "(no VMs recorded)"; return 0; }
850 printf '%-40s %-14s %-12s %-20s %s\n' NAME LOCATION OWNER CREATED_AT PID
851 awk -F'\t' '{ printf "%-40s %-14s %-12s %-20s %s\n", $1, $2, $3, $4, $5 }' "$VM_RECORD"
852}
854new_name() {
855 echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$TTL_HOURS"
856}
858# vcpus SIZE: the vCPU count a Ubicloud size name carries (standard-16 -> 16).
859vcpus() { [[ "$1" =~ -([0-9]+)$ ]] && echo "${BASH_REMATCH[1]}"; }
861# quota_preflight LOCATION NAME SIZE: refuse (or wait UBI_QUOTA_WAIT seconds)
862# before the create request when the project's running VMs plus this size
863# exceed the known vCPU ceiling. The ceiling comes from UBI_QUOTA_VCPUS or the
864# maximum the last refusal printed; without either the API's own refusal rules.
865QUOTA_CACHE_MAX_AGE=86400
866quota_preflight() {
867 local location=$1 name=$2 size=$3 need max used remaining step source="UBI_QUOTA_VCPUS" cache="$STATE_ROOT/quota-max"
868 need=$(vcpus "$size") || return 0
869 max=$QUOTA_VCPUS
870 if [ -z "$max" ] && [ -f "$cache" ]; then
871 # A refusal's maximum is remembered for a day: after that (a raised quota,
872 # another project's key) the API's own answer decides again.
873 if [ $(( $(date +%s) - $(mtime "$cache") )) -lt "$QUOTA_CACHE_MAX_AGE" ]; then
874 max=$(cat "$cache" 2>/dev/null || true); source="the last refusal, cached in $cache"
875 else
876 rm -f "$cache"
877 fi
878 fi
879 [[ "$max" =~ ^[0-9]+$ ]] || return 0
880 # The budget counts seconds spent waiting for room, not the usage calls
881 # between them: a slow first usage read must not shave the printed budget.
882 remaining=$QUOTA_WAIT
883 while :; do
884 used=$(cmd_usage 2>/dev/null | awk '$1=="total" {print $3}')
885 [[ "$used" =~ ^[0-9]+$ ]] || return 0
886 [ $(( used + need )) -gt "$max" ] || return 0
887 if [ "$remaining" -le 0 ]; then
888 log "quota preflight refused $location/$name ($size): it needs $need vCPUs and the project's VMs already use $used of $max (ceiling from $source; other usage such as managed runners counts on top)"
889 cmd_usage >&2 || true
890 die "create refused before any request: vCPU quota exhausted (quota full); wait for running VMs to finish (UBI_QUOTA_WAIT=<seconds> waits here), run $0 reap, use a smaller size, or set UBI_QUOTA_VCPUS to the project's real ceiling"
891 fi
892 log "quota: $used of $max vCPUs in use and $need needed; waiting up to ${remaining}s more for room"
893 step=$(( remaining < 30 ? remaining : 30 ))
894 sleep "$step"
895 remaining=$(( remaining - step ))
896 done
897}
899valid_name() { [[ "$1" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; }
901# check_storage GIB: Ubicloud accepts only these disk sizes; refuse before any API call.
902check_storage() {
903 case " $ALLOWED_STORAGE " in
904 *" $1 "*) [ -n "$1" ] && return 0 ;;
905 esac
906 die "invalid storage '$1' GiB (allowed: $ALLOWED_STORAGE; set UBI_ALLOWED_STORAGE when Ubicloud adds sizes)"
907}
909UP_NAME=""
910UP_READY=""
911cmd_up() {
912 require_owner
913 local size=$DEFAULT_SIZE location=$DEFAULT_LOCATION name="" storage="" image=ubuntu-noble
914 while [ $# -gt 0 ]; do
915 case $1 in
916 -s|--size) size=$2; shift 2 ;;
917 -l|--location) location=$2; shift 2 ;;
918 -n|--name) name=$2; shift 2 ;;
919 -S|--storage) check_storage "$2"; storage=$2; shift 2 ;;
920 -b|--image) image=$2; shift 2 ;;
921 *) die "up: unknown option $1" ;;
922 esac
923 done
924 [ -n "$name" ] || name=$(new_name)
925 valid_name "$name" || die "invalid VM name '$name' (Ubicloud allows 1-63 of a-z, 0-9 and '-', starting and ending with a letter or digit)"
926 if [ "$REAP" != 0 ]; then
927 cmd_reap || log "reap failed; continuing"
928 fi
929 if [ "$GC_HOURS" -gt 0 ]; then
930 cmd_gc "$GC_HOURS" || log "stale-VM cleanup failed; continuing"
931 fi
932 quota_preflight "$location" "$name" "$size"
934 local dir create_at
935 dir=$(state_dir "$name")
936 mkdir -p "$dir"
937 chmod 700 "$dir"
938 ssh-keygen -q -t ed25519 -N '' -C "$name" -f "$dir/key"
940 local args=(vm "$location/$name" create -s "$size" -b "$image")
941 [ -z "$storage" ] || args+=(-S "$storage")
942 args+=("$(cat "$dir/key.pub")")
943 create_at=$(date +%s)
944 write_state "$name" "$location" "$size" "" unknown "$create_at"
945 record_vm "$name" "$location"
946 UP_NAME=$name
947 log "creating $location/$name ($size)"
948 # Hold signals until the create answer is recorded, so teardown knows
949 # whether the VM exists instead of waiting out UBI_CREATE_GRACE.
950 local interrupted="" created=yes refusal need used max
951 trap 'interrupted=1' INT TERM HUP QUIT
952 cli "${args[@]}" >/dev/null 2>"$dir/create.err" || created=no
953 write_state "$name" "$location" "$size" "" "$created" "$create_at"
954 trap 'exit 130' INT TERM HUP QUIT
955 [ -z "$interrupted" ] || exit 130
956 if [ "$created" = no ]; then
957 refusal=$(sed -n 's/.*Requested vCPU count: \([0-9]*\), currently used vCPU count: \([0-9]*\), maximum allowed vCPU count: \([0-9]*\).*/\1 \2 \3/p' "$dir/create.err")
958 if [ -z "$refusal" ]; then
959 cat "$dir/create.err" >&2
960 die "create failed"
961 fi
962 read -r need used max <<<"$refusal"
963 echo "$max" >"$STATE_ROOT/quota-max"
964 log "quota refused $location/$name ($size): it needs $need vCPUs and the project already uses $used of $max"
965 log "VMs in the project by owner (the rest of the used count is other usage, such as managed GitHub runners):"
966 cmd_usage >&2 || true
967 die "create failed: vCPU quota exhausted; wait for running VMs to finish, or use fewer or smaller VMs (ci:ubicloud --vms N)"
968 fi
970 local deadline=$(( $(date +%s) + 600 )) show state ip
971 while :; do
972 show=$(cli vm "$location/$name" show 2>/dev/null || true)
973 state=$(sed -n 's/^state: //p' <<<"$show")
974 ip=$(sed -n 's/^ip4: //p' <<<"$show")
975 [ "$state" = running ] && [ -n "$ip" ] && break
976 [ "$(date +%s)" -lt "$deadline" ] || die "VM did not reach running in 10 minutes (last state: ${state:-unknown})"
977 sleep "$POLL"
978 done
979 write_state "$name" "$location" "$size" "$ip" yes "$create_at"
981 load "$name"
982 deadline=$(( $(date +%s) + 300 ))
983 # shellcheck disable=SC2046
984 until ssh $(ssh_opts) "ubi@$IP" true 2>/dev/null; do
985 [ "$(date +%s)" -lt "$deadline" ] || die "SSH did not come up on $IP"
986 sleep "$POLL"
987 done
988 remote "$name" 'cloud-init status --wait >/dev/null 2>&1 || true'
989 UP_READY=1
990 log "ready: $name ($IP)"
991 echo "$name"
992}
994# down NAME [-l LOCATION]: destroy NAME and return 0 only once it is confirmed
995# gone or never existed. While this machine's create request for NAME may
996# still be in flight (its `up` process is alive, or the create was sent less
997# than UBI_CREATE_GRACE seconds ago with no answer), "not found" is not final.
998cmd_down() {
999 local name=${1:-} loc="" dir
1000 [ -n "$name" ] || die "down: missing NAME"
1001 shift
1002 while [ $# -gt 0 ]; do
1003 case $1 in
1004 -l|--location) loc=$2; shift 2 ;;
1005 *) die "down: unknown option $1" ;;
1006 esac
1007 done
1008 valid_name "$name" || die "invalid VM name '$name'"
1009 dir=$(state_dir "$name")
1010 local CREATED=none CREATE_AT=0 CREATE_PID="" IP="" LOCATION="" KEY=""
1011 if [ -f "$dir/env" ]; then
1012 load "$name"
1013 loc=$LOCATION
1014 if [ -S "$dir/ctl" ] && [ -n "$IP" ]; then
1015 # shellcheck disable=SC2046
1016 ssh $(ssh_opts) -O exit "ubi@$IP" >/dev/null 2>&1 || true
1017 fi
1018 fi
1020 local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now
1021 out=$(mktemp)
1022 while :; do
1023 now=$(date +%s)
1024 if [ -z "$loc" ]; then
1025 loc=$(vm_rows 2>/dev/null | awk -v n="$name" '$2==n {print $1}') || loc=""
1026 code=404
1027 [ -z "$loc" ] || code=$(api "$out" vm "$loc/$name" show)
1028 else
1029 code=$(api "$out" vm "$loc/$name" show)
1030 fi
1031 if [ "$code" = 200 ]; then
1032 seen=1
1033 if [ $(( now - asked )) -ge 60 ]; then
1034 code=$(api "$out" vm "$loc/$name" destroy -f)
1035 if [ "$code" != 200 ]; then
1036 cat "$out" >&2
1037 rm -f "$out"
1038 log "FAILED to destroy $loc/$name (HTTP $code); retry: $0 down $name"
1039 return 1
1040 fi
1041 asked=$now
1042 log "destroy requested: $loc/$name"
1043 fi
1044 elif [ "$code" = 404 ]; then
1045 if [ "$seen" = 1 ] || [ "$CREATED" != unknown ] || { [ $(( now - CREATE_AT )) -ge "$CREATE_GRACE" ] \
1046 && { [ "$CREATE_PID" = "$$" ] || ! kill -0 "$CREATE_PID" 2>/dev/null; }; }; then
1047 rm -f "$out"
1048 rm -rf "$dir"
1049 forget_vm "$name"
1050 if [ "$seen" = 1 ]; then log "destroyed ${loc:-?}/$name"; else log "gone: ${loc:-?}/$name (never existed or already destroyed)"; fi
1051 return 0
1052 fi
1053 log "waiting for an in-flight create of $name to resolve"
1054 fi
1055 if [ "$now" -ge "$deadline" ]; then
1056 rm -f "$out"
1057 log "WARNING: could not confirm ${loc:-?}/$name is gone after ${DOWN_TIMEOUT}s; rerun: $0 down $name"
1058 return 1
1059 fi
1060 sleep "$POLL"
1061 done
1062}
1064# pack SRC: write a gzipped checkout tarball to stdout (tracked + untracked
1065# files that are not ignored, plus a .git snapshot; a non-git directory is
1066# taken whole). The live .git is never tarred: another process running git in
1067# the checkout (a review subagent's diff, a trial merge's worktree add) writes
1068# index.lock, packed-refs and objects while tar reads, and GNU tar exits 1
1069# on a file that changed under it (GSTA-121's PREGATE_REMOTE_INFRA). Instead
1070# the refs are fetched into a fresh repository through git's own transport,
1071# which reads a consistent view, the remote.* config is replayed so
1072# origin/<base> still resolves, and the index is copied (git writes it
1073# atomically through a rename). A linked worktree's .git is a one-line
1074# pointer into the main repository, useless on the VM; the same snapshot
1075# replaces it, on the worktree's own branch and index. Not carried: submodule
1076# repositories (.git/modules) and a partial clone's promisor (the fetch pulls
1077# every missing blob first); gstack has neither.
1078cmd_pack() {
1079 local src tmp gitdir branch head entry
1080 src=$(cd "${1:-.}" && pwd)
1081 if git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
1082 head=$(git -C "$src" rev-parse --verify -q HEAD) || die "pack: $src has no commits; the VM needs a HEAD to fingerprint"
1083 branch=$(git -C "$src" symbolic-ref -q --short HEAD || true)
1084 tmp=$(mktemp -d)
1085 # The path is baked in: a set -e failure unwinds the function before the trap runs, and a local $tmp is unset there.
1086 trap "rm -rf $(printf %q "$tmp")" EXIT
1087 # No template hooks from a global init.templateDir, and no auto-gc under the tar.
1088 git -c gc.auto=0 init -q --bare --template= "$tmp/.git"
1089 git -C "$tmp/.git" -c gc.auto=0 fetch -q --no-tags --update-shallow "$src" '+refs/*:refs/*' \
1090 || die "pack: could not snapshot the refs of $src"
1091 if [ -z "$branch" ]; then
1092 branch="detached-${head:0:12}"
1093 git -C "$tmp/.git" fetch -q --no-tags --update-shallow "$src" "+HEAD:refs/heads/$branch"
1094 fi
1095 git -C "$tmp/.git" symbolic-ref HEAD "refs/heads/$branch"
1096 git -C "$tmp/.git" config core.bare false
1097 gitdir=$(cd "$src" && cd "$(git rev-parse --git-common-dir)" && pwd)
1098 while IFS= read -r -d '' entry; do
1099 # A key with no value (`[remote "x"] mirror`) is boolean true.
1100 case $entry in *$'\n'*) ;; *) entry="$entry"$'\n'true ;; esac
1101 git -C "$tmp/.git" config --add "${entry%%$'\n'*}" "${entry#*$'\n'}"
1102 done < <(git -C "$gitdir" config --local --null --get-regexp '^(remote|branch)\.' 2>/dev/null || true)
1103 # The checkout's index rides along so the VM's status matches, except a
1104 # split index, whose shared half lives only in the original git dir. The
1105 # index's blobs that HEAD's tree lacks (staged, intent-to-add, the stages
1106 # of an unmerged entry) are behind no ref, so they travel as a pack of
1107 # their own; without them the VM's write-tree (gstack-wtree) cannot read a
1108 # staged file.
1109 if [ "$(git -C "$src" config --bool core.splitIndex 2>/dev/null)" != true ]; then
1110 cp "$(cd "$src" && cd "$(dirname "$(git rev-parse --git-path index)")" && pwd)/index" "$tmp/.git/index"
1111 # Read the copy, not the live index, so the pack matches it; a gitlink's
1112 # sha (mode 160000) is a submodule commit this object store lacks.
1113 GIT_INDEX_FILE="$tmp/.git/index" git -C "$src" ls-files -s | awk '$1 != "160000" {print $2}' | sort -u \
1114 | comm -23 - <(git -C "$src" ls-tree -r HEAD | awk '{print $3}' | sort -u) \
1115 | git -C "$src" pack-objects -q --stdout \
1116 | git -C "$tmp/.git" unpack-objects -q \
1117 || die "pack: could not snapshot the staged blobs of $src"
1118 fi
1119 (
1120 cd "$src"
1121 git ls-files -z -co --exclude-standard \
1122 | while IFS= read -r -d '' f; do
1123 if [ -e "$f" ] || [ -L "$f" ]; then printf '%s\0' "$f"; fi
1124 done \
1125 | tar --null -T - -c -C "$tmp" .git -zf -
1126 )
1127 rm -rf "$tmp"; trap - EXIT
1128 else
1129 tar -C "$src" -czf - .
1130 fi
1131}
1133# unpack NAME DEST: extract a tarball read from stdin into DEST on the VM.
1134cmd_unpack() {
1135 local qdest
1136 qdest=$(printf %q "$2")
1137 remote "$1" "mkdir -p $qdest && tar -xzf - -C $qdest"
1138}
1140cmd_sync() {
1141 local name=$1 src=${2:-.} dest=${3:-}
1142 src=$(cd "$src" && pwd)
1143 [ -n "$dest" ] || dest="work/$(basename "$src")"
1144 cmd_pack "$src" | cmd_unpack "$name" "$dest"
1145 log "synced $src -> $name:$dest"
1146}
1148# pull NAME REMOTE_GLOB LOCAL_DIR: copy matching remote entries into LOCAL_DIR.
1149# Relative remote paths start at /home/ubi. A glob that matches nothing (for
1150# example an optional flake ledger) is reported and skipped, not a failure.
1151cmd_pull() {
1152 local name=$1 from=$2 to=$3 dir
1153 dir=$(printf %q "$(dirname "$from")")
1154 if ! remote "$name" "cd $dir 2>/dev/null && ls -d -- $(basename "$from") >/dev/null 2>&1"; then
1155 log "pull: nothing matches $from"
1156 return 0
1157 fi
1158 mkdir -p "$to"
1159 remote "$name" "cd $dir && tar -czf - $(basename "$from")" | tar -xzf - -C "$to"
1160}
1162# pass_name NAME: a variable this runner may forward to a VM. Its own
1163# credentials (UBICLOUD_*) never leave this machine, whoever asks.
1164pass_name() {
1165 [[ "$1" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || die "--pass $1: not a variable name"
1166 case $1 in UBICLOUD_*) die "--pass $1: the runner never forwards its own Ubicloud credentials to a VM" ;; esac
1167}
1169# inside_dir DIR ROOT: DIR resolves, symlinks followed, to ROOT or below it.
1170# Its nearest existing ancestor is checked first, so a refused destination
1171# creates nothing (not even outside the root); only then is DIR created and
1172# checked again.
1173inside_dir() {
1174 local a b near=$1
1175 b=$(cd "$2" 2>/dev/null && pwd -P) || return 1
1176 while [ ! -d "$near" ]; do
1177 [ "$near" != "$(dirname "$near")" ] || return 1
1178 near=$(dirname "$near")
1179 done
1180 a=$(cd "$near" 2>/dev/null && pwd -P) || return 1
1181 [ "$a" = "$b" ] || [[ "$a" = "$b"/* ]] || return 1
1182 mkdir -p "$1" 2>/dev/null || return 1
1183 a=$(cd "$1" 2>/dev/null && pwd -P) || return 1
1184 [ "$a" = "$b" ] || [[ "$a" = "$b"/* ]]
1185}
1187# PULL_ROOT: set by verify; every pull destination must resolve inside it at
1188# pull time (an earlier pull may have placed a symlink where a later one lands).
1189PULL_ROOT=""
1191# run_teardown RC: run's EXIT handler. Destroys RUN_VM, ignoring further
1192# signals, then checks the project's VM list for it and prints the leftover
1193# line; a VM still listed, or a list that cannot be read, fails the run.
1194run_teardown() {
1195 local rc=$1 rows left others
1196 trap "" INT TERM HUP QUIT
1197 cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"
1198 if rows=$(vm_rows 2>/dev/null); then
1199 left=$(awk -v n="$RUN_VM" '$2==n' <<<"$rows" | grep -c . || true)
1200 others=$(awk -v n="$RUN_VM" -v o="$OWNER" '$3==o && $2!=n' <<<"$rows" | grep -c . || true)
1201 else
1202 left=unknown others=unknown
1203 fi
1204 leftover "left=$left owner=$OWNER vm=$RUN_VM others=$others"
1205 [ "$left" = 0 ] && return 0
1206 log "VM cleanup not confirmed for $RUN_VM (left=$left); destroy it with: $0 down $RUN_VM"
1207 [ "$rc" -ne 0 ] || exit 1
1208}
1210TIER_NAMES=() TIER_CMDS=() RESUME=1 RERUN_FAILED=0 FORCE_ATTACH="" RUN_ARGS=()
1211cmd_run() {
1212 require_owner
1213 local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=() passes=() key_inputs=() probe=0
1214 RUN_ARGS=("$@")
1215 while [ $# -gt 0 ]; do
1216 case $1 in
1217 -S|--storage) check_storage "$2"; up_args+=("$1" "$2"); shift 2 ;;
1218 -s|--size|-l|--location|-n|--name|-b|--image) up_args+=("$1" "$2"); shift 2 ;;
1219 --src) src=$2; shift 2 ;;
1220 --dest) dest=$2; shift 2 ;;
1221 --setup) setup=$2; shift 2 ;;
1222 --env) envs+=("$2"); shift 2 ;;
1223 --pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; passes+=("$2"); shift 2 ;;
1224 --pull) pulls+=("$2"); shift 2 ;;
1225 --keep) keep=1; shift ;;
1226 --key-input) key_inputs+=("$2"); shift 2 ;;
1227 --probe) probe=1; shift ;;
1228 --resume) RESUME=1; shift ;;
1229 --no-resume) RESUME=0; shift ;;
1230 --rerun-failed) RERUN_FAILED=1; shift ;;
1231 --) shift; break ;;
1232 *) die "run: unknown option $1" ;;
1233 esac
1234 done
1235 [ $# -gt 0 ] || [ ${#TIER_NAMES[@]} -gt 0 ] || die "run: missing command after --"
1236 [ -z "$setup" ] || [ -f "$setup" ] || die "setup script not found: $setup"
1237 command -v git >/dev/null || die "git is required: the run key fingerprints the checkout with it"
1238 src=$(cd "$src" && pwd)
1239 [ -n "$dest" ] || dest="work/$(basename "$src")"
1241 local i name="" size=$DEFAULT_SIZE location=$DEFAULT_LOCATION image=ubuntu-noble storage="" e n
1242 for (( i = 0; i < ${#up_args[@]}; i += 2 )); do
1243 case ${up_args[i]} in
1244 -n|--name) name=${up_args[i+1]} ;;
1245 -s|--size) size=${up_args[i+1]} ;;
1246 -l|--location) location=${up_args[i+1]} ;;
1247 -b|--image) image=${up_args[i+1]} ;;
1248 -S|--storage) storage=${up_args[i+1]} ;;
1249 esac
1250 done
1251 # The run key: everything that decides what the VM runs, computed before any
1252 # VM exists. Pass values enter only as their sha256.
1253 if [ -z "$FORCE_ATTACH" ]; then
1254 TREE=$(cd "$src" && bash -c "$TREE_SH") || die "could not fingerprint $src (git add -A into a temporary index failed)"
1255 RUN_KEY=$({
1256 echo "ubi-runner run key v1"
1257 echo "tree $TREE"
1258 printf 'command %q\n' "$*"
1259 if [ -n "$setup" ]; then echo "setup $(sha256 <"$setup")"; else echo "setup none"; fi
1260 for e in ${envs[@]+"${envs[@]}"}; do printf 'env %q\n' "$e"; done
1261 for n in ${passes[@]+"${passes[@]}"}; do printf 'pass %s %s\n' "$n" "$(printf %s "${!n}" | sha256)"; done
1262 printf 'vm %q %q %q %q\n' "$size" "$location" "$image" "$storage"
1263 echo "base $(base_sha "$src")"
1264 echo "runner $(sha256 <"${BASH_SOURCE[0]}")"
1265 for e in ${key_inputs[@]+"${key_inputs[@]}"}; do printf 'input %q\n' "$e"; done
1266 for i in "${!TIER_NAMES[@]}"; do printf 'tier %q %q\n' "${TIER_NAMES[i]}" "${TIER_CMDS[i]}"; done
1267 } | sha256)
1268 fi
1269 [ ${#TIER_NAMES[@]} -eq 0 ] || tier_plan
1271 local attach=""
1272 if [ -n "$FORCE_ATTACH" ]; then
1273 vm_alive "$FORCE_ATTACH" || die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"
1274 attach=$FORCE_ATTACH
1275 else
1276 attach=$(find_record)
1277 fi
1278 if [ "$probe" = 1 ]; then
1279 if [ -z "$attach" ]; then echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12}"
1280 else
1281 local busy=0
1282 ! lease_busy "$(state_dir "$attach")" || busy=1
1283 if [ "$busy" = 0 ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
1284 echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12} ttl_short vm=$attach $TTL_WHY"
1285 else
1286 echo "UBI_ATTACH_INFO: vm=$attach created_at=$(load "$attach"; echo "$CREATE_AT") key=${RUN_KEY:0:12} busy=$busy"
1287 fi
1288 fi
1289 return 0
1290 fi
1291 if [ ${#TIER_NAMES[@]} -gt 0 ] && { [ -n "$TIER_BLOCKED" ] || [ "$TIER_TO_RUN" -eq 0 ]; } && [ -n "$attach" ] && ! lease_busy "$(state_dir "$attach")"; then
1292 log "verify: no tier needs $attach, which a dead runner left behind; destroying it"
1293 cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
1294 fi
1295 if [ ${#TIER_NAMES[@]} -gt 0 ] && [ -n "$TIER_BLOCKED" ]; then
1296 TIER_FAILED=$TIER_BLOCKED
1297 verify_line
1298 return "$TIER_BLOCKED_RC"
1299 fi
1300 if [ ${#TIER_NAMES[@]} -gt 0 ] && [ "$TIER_TO_RUN" -eq 0 ]; then
1301 log "verify: every tier passed under this key; no VM needed"
1302 verify_line
1303 return 0
1304 fi
1305 if [ -n "$attach" ] && lease_busy "$(state_dir "$attach")"; then
1306 die "UBI_ATTACH_BUSY: vm=$attach key=${RUN_KEY:0:12}: runner pid $(cut -d' ' -f1 <"$(state_dir "$attach")/lease") on this machine still owns it; wait for it or stop it, then rerun"
1307 fi
1308 if [ -n "$attach" ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
1309 log "UBI_REATTACH_TTL_SHORT: vm=$attach key=${RUN_KEY:0:12} $TTL_WHY: starting a fresh VM (tier checkpoints carry over) and destroying $attach"
1310 cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
1311 attach=""
1312 fi
1313 [ -z "$attach" ] || name=$attach
1314 [ -n "$name" ] || { name=$(new_name); up_args+=(-n "$name"); }
1315 RUN_VM=$name
1316 # Armed before the create request: an interrupted `up` still destroys its VM.
1317 # A second signal must not cut teardown short, so the EXIT handler ignores them.
1318 trap 'run_teardown "$?"' EXIT
1319 trap 'exit 130' INT TERM QUIT
1320 trap on_hup HUP
1321 if [ -n "$attach" ]; then
1322 load "$name"
1323 log "reattached to $name created_at=$CREATE_AT key=${RUN_KEY:0:12}"
1324 else
1325 cmd_up "${up_args[@]}" >/dev/null
1326 trap 'exit 130' INT TERM QUIT
1327 trap on_hup HUP
1328 if [[ "$name" =~ -t([0-9]+)$ ]]; then
1329 log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
1330 elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
1331 log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
1332 else
1333 log "$name has no TTL and no reapable name, so nothing reaps it; if this process dies before its teardown line, destroy it with: $0 down $name"
1334 fi
1335 fi
1336 local dir
1337 dir=$(state_dir "$name")
1338 write_record "$dir"
1339 lease_take "$dir"
1340 if [ "$keep" = 1 ]; then
1341 # A kept VM's finished run is not reattached by the next run; a detach (HUP) keeps the record.
1342 trap 'rm -f "$(state_dir "$RUN_VM")/run"' EXIT
1343 leftover "left=1 owner=$OWNER vm=$name kept=1"
1344 log "--keep: leaving $name running; destroy with: $0 down $name"
1345 fi
1347 local qdest rc=0 vmt
1348 qdest=$(printf %q "$dest")
1349 if [ ! -f "$dir/synced" ]; then
1350 { for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done
1351 for n in ${passes[@]+"${passes[@]}"}; do printf 'export %s=%q\n' "$n" "${!n}"; done; } \
1352 | remote "$name" 'umask 077 && cat > ~/.ubirun-env'
1353 cmd_sync "$name" "$src" "$dest"
1354 # A subdirectory of a checkout uploads the whole .git with only its own
1355 # files, so its tree is compared VM to VM (tiers) but not to this side.
1356 if [ -z "$(git -C "$src" rev-parse --show-prefix 2>/dev/null)" ]; then
1357 vmt=$(vm_tree "$name" "$qdest") || vmt=unavailable
1358 if [ "$vmt" != "$TREE" ]; then
1359 log "UBI_VM_TREE_MISMATCH: vm=$name tree=${vmt:0:12} expected=${TREE:0:12}: the unpacked checkout is not the one this run was keyed on"
1360 exit 3
1361 fi
1362 fi
1363 [ -z "$setup" ] || remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
1364 : >"$dir/synced"
1365 fi
1366 if [ -n "$setup" ]; then
1367 log "running setup $(basename "$setup")"
1368 detached "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
1369 fi
1371 if [ ${#TIER_NAMES[@]} -gt 0 ]; then
1372 run_tiers "$name" "$qdest" || rc=$?
1373 else
1374 log "running on $name: $*"
1375 detached "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
1376 log "command exited $rc after $(( $(date +%s) - $(cat "$dir/phase.command.start" 2>/dev/null || date +%s) ))s"
1377 fi
1379 local p
1380 for p in ${pulls[@]+"${pulls[@]}"}; do
1381 if [ -n "$PULL_ROOT" ] && ! inside_dir "${p#*:}" "$PULL_ROOT"; then
1382 log "pull refused: ${p#*:} resolves outside $PULL_ROOT"
1383 [ "$rc" -ne 0 ] || rc=1
1384 continue
1385 fi
1386 cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
1387 done
1388 [ ${#TIER_NAMES[@]} -eq 0 ] || verify_line
1389 return "$rc"
1390}
1392# attach NAME: resume the run recorded for NAME on this machine (the same
1393# invocation, key and tiers), as a rerun of the same command would.
1394cmd_attach() {
1395 local name=${1:-}
1396 [ -n "$name" ] || die "attach: missing NAME"
1397 valid_name "$name" || die "invalid VM name '$name'"
1398 [ -f "$(state_dir "$name")/run" ] || die "attach: no run record for $name on this machine ($(state_dir "$name")/run); a machine replacement loses it"
1399 # shellcheck disable=SC1091
1400 . "$(state_dir "$name")/run"
1401 FORCE_ATTACH=$name
1402 cmd_run "${RUN_ARGS[@]}"
1403}
1405# verify [run opts] [--trust-config] [-- COMMAND]: run the repository's
1406# committed remote verify. .gstack/ubicloud.json under --src (default .)
1407# supplies setup, command, size, location, storage, image, env, pass and pull;
1408# flags given here and a trailing -- COMMAND override it. Without the file it
1409# refuses, so every lane runs the one committed definition instead of its own
1410# wrapper.
1411#
1412# The file is repository content and gets no reach beyond the checkout on its
1413# own: its setup script and pull destinations must resolve inside --src, and a
1414# local variable it lists under pass is forwarded only when this command line
1415# also names it (--pass NAME) or grants every listed name (--trust-config).
1416cmd_verify() {
1417 local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=() cfg_tier_names=() cfg_tier_cmds=()
1418 for (( i = 1; i <= $#; i++ )); do
1419 [ "${!i}" = -- ] && break
1420 if [ "${!i}" = --src ]; then j=$(( i + 1 )); src=${!j}; fi
1421 done
1422 cfg="$src/.gstack/ubicloud.json"
1423 [ -f "$cfg" ] || die "verify: no $cfg; commit one ({\"setup\": \"scripts/ubicloud/bootstrap.sh\", \"command\": \"bash scripts/ubicloud/verify.sh\"}) so every lane runs the same verify instead of its own wrapper"
1424 err=$(mktemp)
1425 parsed=$(python3 -c '
1426import json, os, re, shlex, sys
1427cfg, src = sys.argv[1], sys.argv[2]
1428try:
1429 c = json.load(open(cfg))
1430except ValueError:
1431 sys.exit(3)
1432if not isinstance(c, dict):
1433 sys.exit(3)
1434base, root = os.path.abspath(src), os.path.realpath(src)
1435def inside(rel, what):
1436 if not isinstance(rel, str) or not rel or os.path.isabs(rel) or rel.startswith("~") or ".." in rel.replace("\\", "/").split("/"):
1437 print("%s %r must be a relative path inside the checkout" % (what, rel), file=sys.stderr); sys.exit(2)
1438 full = os.path.realpath(os.path.join(root, rel))
1439 if full != root and not full.startswith(root + os.sep):
1440 print("%s %r resolves outside the checkout (%s)" % (what, rel, full), file=sys.stderr); sys.exit(2)
1441 return os.path.join(base, rel)
1442q = lambda s: shlex.quote(str(s))
1443args = []
1444for key, flag in (("size", "-s"), ("location", "-l"), ("storage", "-S"), ("image", "-b")):
1445 if key in c: args += [flag, q(c[key])]
1446for item in c.get("env", []): args += ["--env", q(item)]
1447for item in c.get("pull", []):
1448 remote, sep, local = str(item).partition(":")
1449 if not sep or not remote:
1450 print("pull entry %r must be REMOTE_GLOB:LOCAL_DIR" % (item,), file=sys.stderr); sys.exit(2)
1451 args += ["--pull", q(remote + ":" + inside(local, "pull destination"))]
1452tiers = c.get("tiers")
1453names = []
1454if tiers is not None:
1455 if "command" in c:
1456 print("UBI_CONFIG_TIERS_AND_COMMAND: give either command or tiers, not both", file=sys.stderr); sys.exit(2)
1457 if not isinstance(tiers, list) or not tiers:
1458 print("UBI_CONFIG_TIER_NAME: tiers must be a non-empty list of {\"name\", \"command\"}", file=sys.stderr); sys.exit(2)
1459 for t in tiers:
1460 name = t.get("name") if isinstance(t, dict) else None
1461 if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,31}", name):
1462 print("UBI_CONFIG_TIER_NAME: tier name %r must match ^[a-z0-9][a-z0-9_-]{0,31}$" % (name,), file=sys.stderr); sys.exit(2)
1463 if name in names:
1464 print("UBI_CONFIG_TIER_NAME: tier name %r appears twice" % name, file=sys.stderr); sys.exit(2)
1465 if not isinstance(t.get("command"), str) or not t["command"].strip():
1466 print("UBI_CONFIG_TIER_NAME: tier %r needs a command" % name, file=sys.stderr); sys.exit(2)
1467 names.append(name)
1468setup = c.get("setup", "")
1469print("cfg_tier_names=(" + " ".join(q(n) for n in names) + ")")
1470print("cfg_tier_cmds=(" + " ".join(q(t["command"]) for t in (tiers or [])) + ")")
1471print("cfg_args=(" + " ".join(args) + ")")
1472print("cfg_pass=(" + " ".join(q(x) for x in c.get("pass", [])) + ")")
1473print("cfg_setup=" + q(inside(setup, "setup") if setup else ""))
1474print("cfg_cmd=" + q(c.get("command", "")))
1475' "$cfg" "$src" 2>"$err") || rc=$?
1476 case $rc in
1477 0) ;;
1478 2) local why; why=$(cat "$err"); rm -f "$err"; die "verify: $cfg: $why" ;;
1479 *) rm -f "$err"; die "verify: $cfg is not valid JSON" ;;
1480 esac
1481 rm -f "$err"
1482 eval "$parsed"
1483 while [ $# -gt 0 ]; do
1484 case $1 in
1485 --) shift; user_cmd=("$@"); break ;;
1486 --trust-config) trust=1; shift ;;
1487 --keep|--resume|--no-resume|--rerun-failed|--probe) user_pre+=("$1"); shift ;;
1488 --pass|-S|--storage|-s|--size|-l|--location|-n|--name|-b|--image|--src|--dest|--setup|--env|--pull)
1489 [ $# -ge 2 ] || die "verify: $1 needs a value"
1490 [ "$1" != --pass ] || granted+=("$2")
1491 user_pre+=("$1" "$2"); shift 2 ;;
1492 *) die "verify: unknown option $1" ;;
1493 esac
1494 done
1495 for n in ${cfg_pass[@]+"${cfg_pass[@]}"}; do
1496 pass_name "$n"
1497 if [ "$trust" = 1 ]; then cfg_args+=(--pass "$n"); continue; fi
1498 for g in ${granted[@]+"${granted[@]}"}; do [ "$g" = "$n" ] && continue 2; done
1499 missing+=("$n")
1500 done
1501 [ ${#missing[@]} -eq 0 ] || die "verify: $cfg asks to forward local ${missing[*]}; grant each on this command line with --pass NAME, or every name it lists with --trust-config"
1502 # A trailing -- COMMAND replaces the configured tiers as it replaces command.
1503 if [ ${#user_cmd[@]} -eq 0 ] && [ ${#cfg_tier_names[@]} -gt 0 ]; then
1504 TIER_NAMES=("${cfg_tier_names[@]}") TIER_CMDS=("${cfg_tier_cmds[@]}")
1505 else
1506 [ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
1507 [ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
1508 fi
1509 log "verify: $cfg"
1510 PULL_ROOT=$(cd "$src" && pwd)
1511 cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- ${user_cmd[@]+"${user_cmd[@]}"}
1512}
1514usage() {
1515 cat <<EOF
1516usage: ubi-runner.sh <command> [args]
1518 run [up opts] [--src DIR] [--dest PATH] [--setup FILE] [--env K=V]...
1519 [--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] [--key-input S]... [--probe] -- COMMAND
1520 up + sync + setup + command + pull + destroy; exits with COMMAND's status.
1521 A rerun of the same command (same run key) reattaches to its live VM;
1522 --probe prints UBI_ATTACH_INFO (and the tier plan) and exits
1523 verify [run opts] [--trust-config] [--resume|--no-resume] [--rerun-failed] [-- COMMAND]
1524 run the repository's committed verify (.gstack/ubicloud.json under --src);
1525 its setup script and every pull destination stay inside --src (checked
1526 again at pull time), and its pass names are forwarded only when granted
1527 here (--pass NAME, or all: --trust-config). With tiers, a pass checkpoint
1528 under the same run key is reused (resume is the default)
1529 attach NAME resume the run recorded for NAME on this machine, as a rerun would
1530 up [-s SIZE] [-l LOCATION] [-n NAME] [-S STORAGE_GIB] [-b IMAGE]
1531 create a VM and wait for SSH; prints its name
1532 (STORAGE_GIB: one of $ALLOWED_STORAGE)
1533 ssh NAME [COMMAND] shell or login-shell command on the VM (user ubi, passwordless sudo)
1534 sync NAME [SRC] [DEST] stream a checkout (tracked + untracked-unignored + .git)
1535 pack [SRC] write that checkout tarball to stdout
1536 unpack NAME DEST extract a tarball from stdin into DEST on the VM
1537 pull NAME REMOTE_GLOB LOCAL_DIR
1538 copy matching remote entries into LOCAL_DIR
1539 down NAME [-l LOC] destroy the VM; returns once it is confirmed gone or never existed
1540 list [--mine] list all VMs in the project, or only this owner's
1541 usage VMs and vCPUs per owner across the project
1542 owner print this caller's owner tag ($OWNER)
1543 records VMs this machine created and has not confirmed gone ($VM_RECORD)
1544 gc HOURS destroy this owner's VMs older than HOURS (never other owners')
1545 reap destroy every VM whose -t<hours> TTL has ended (any owner) and this
1546 owner's TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS
1547 cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
1549signals: INT and TERM tear the VM down. HUP, once a remote phase is live, detaches
1550(UBI_DETACHED, exit 129; the VM runs on under its TTL). SIGKILL or a machine stop leaves
1551the run record and the VM; rerun the same command to reattach (UBI_ATTACH_BUSY while
1552another runner owns it; exit 125 UBI_ATTACH_LOST when the remote session left no status).
1554VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; up, run and verify need
1555UBI_OWNER (e.g. your thread code); run and verify end with a UBICLOUD: left=<n> line.
1556defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
1557env: UBI_TTL_HOURS (${TTL_HOURS}h in the name, 1-999), UBI_REAP (0: up skips the reap),
1558 UBI_LEGACY_HOURS (${LEGACY_HOURS}h: reap's bound for this owner's TTL-less ubirun-* names),
1559 UBI_IDLE_TIMEOUT (${IDLE_TIMEOUT}s of silence kills the remote command, exit 124; 0 disables),
1560 UBI_RUN_TIMEOUT (${RUN_TIMEOUT}s wall clock; 0 unlimited), UBI_HEARTBEAT (${HEARTBEAT}s between heartbeat lines),
1561 UBI_QUOTA_VCPUS (project vCPU ceiling for the preflight; else the last refusal's maximum),
1562 UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
1563 UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
1564 UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
1565 UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640"),
1566 UBI_CHECKPOINT_DIR (tier checkpoints; $CHECKPOINT_DIR), UBI_CHECKPOINT_MAX_AGE (${CHECKPOINT_MAX_AGE}s a pass is reused)
1567EOF
1568}
1570cmd=${1:-help}
1571[ $# -eq 0 ] || shift
1572case $cmd in
1573 owner|pack|records|help|-h|--help) ;;
1574 *) [ -n "${UBICLOUD_API_KEY:-}" ] || die "UBICLOUD_API_KEY is not set (create a token under your Ubicloud project's Tokens page)" ;;
1575esac
1576case $cmd in
1577 run) cmd_run "$@" ;;
1578 verify) cmd_verify "$@" ;;
1579 attach) cmd_attach "$@" ;;
1580 up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
1581 trap 'exit 130' INT TERM HUP QUIT
1582 cmd_up "$@" ;;
1583 ssh) name=$1; shift; load "$name"
1584 # shellcheck disable=SC2046
1585 if [ $# -eq 0 ]; then exec ssh -t $(ssh_opts) "ubi@$IP"; else watched "$name" ssh "bash -lc $(printf %q "$*")"; fi ;;
1586 sync) cmd_sync "$@" ;;
1587 pack) cmd_pack "$@" ;;
1588 unpack) cmd_unpack "$@" ;;
1589 pull) cmd_pull "$@" ;;
1590 down) cmd_down "$@" ;;
1591 list) cmd_list "$@" ;;
1592 usage) cmd_usage ;;
1593 owner) echo "$OWNER" ;;
1594 records) cmd_records ;;
1595 gc) cmd_gc "$@" ;;
1596 reap) cmd_reap ;;
1597 cli) cli "$@" ;;
1598 help|-h|--help) usage ;;
1599 *) usage >&2; exit 2 ;;
1600esac