Skills › Compound Engineering › ce-optimize › measure.sh
measure.sh
140 lines · skills/ce-optimize/scripts/measure.sh at 3449230
1#!/bin/bash
3# Measurement Runner
4# Runs a measurement command, captures JSON output, and handles timeouts.
5# The orchestrating agent (not this script) evaluates gates and handles
6# stability repeats.
7#
8# Usage: measure.sh <command> <timeout_seconds> [working_directory] [KEY=VALUE ...]
9#
10# Arguments:
11# command - Shell command to run (e.g., "python evaluate.py")
12# timeout_seconds - Maximum seconds before killing the command
13# working_directory - Directory to run the command in (default: .)
14# KEY=VALUE - Optional environment variables to set before running
15#
16# Output:
17# stdout: Raw JSON output from the measurement command
18# stderr: Passed through from the measurement command
19# exit code: Same as the measurement command (124 for timeout, 125 when
20# CE_OPTIMIZE_CENSOR_AFTER fires before timeout_seconds)
22set -euo pipefail
24# Parse arguments
25COMMAND="${1:?Error: command argument required}"
26TIMEOUT="${2:?Error: timeout_seconds argument required}"
27shift 2
29WORKDIR="."
30if [[ $# -gt 0 ]] && [[ "$1" != *=* ]]; then
31 WORKDIR="$1"
32 shift
33fi
35# Set any KEY=VALUE environment variables
36for arg in "$@"; do
37 if [[ "$arg" == *=* ]]; then
38 export "$arg"
39 fi
40done
42# Change to working directory
43cd "$WORKDIR" || {
44 echo "Error: cannot cd to $WORKDIR" >&2
45 exit 1
46}
48run_timed_command() {
49 local timeout_bin="$1"
50 if [[ -n "${CENSOR_STATUS_FILE:-}" ]]; then
51 "$timeout_bin" "$TIMEOUT" bash -c 'bash -c "$1"; printf "%s\n" "$?" > "$2"; exit 0' _ "$COMMAND" "$CENSOR_STATUS_FILE"
52 return
53 fi
54 "$timeout_bin" "$TIMEOUT" bash -c "$COMMAND"
55}
57run_with_timeout() {
58 if command -v timeout >/dev/null 2>&1; then
59 run_timed_command timeout
60 return
61 fi
63 if command -v gtimeout >/dev/null 2>&1; then
64 run_timed_command gtimeout
65 return
66 fi
68 PY=""
69 for c in python3 python py; do
70 if command -v "$c" >/dev/null 2>&1 && "$c" -c '' >/dev/null 2>&1; then
71 PY="$c"
72 break
73 fi
74 done
75 if [ -n "$PY" ]; then
76 "$PY" - "$TIMEOUT" "$COMMAND" "${CENSOR_STATUS_FILE:-}" <<'PY'
77import os
78import signal
79import subprocess
80import sys
82timeout_seconds = float(sys.argv[1])
83command = sys.argv[2]
84status_file = sys.argv[3] if len(sys.argv) > 3 and sys.argv[3] else ""
85proc = subprocess.Popen(["bash", "-c", command], start_new_session=True)
87try:
88 rc = proc.wait(timeout=timeout_seconds)
89 if status_file:
90 with open(status_file, "w", encoding="utf-8") as fh:
91 fh.write(f"{rc}\n")
92 sys.exit(0)
93 sys.exit(rc)
94except subprocess.TimeoutExpired:
95 os.killpg(proc.pid, signal.SIGTERM)
96 try:
97 proc.wait(timeout=5)
98 except subprocess.TimeoutExpired:
99 os.killpg(proc.pid, signal.SIGKILL)
100 proc.wait()
101 sys.exit(124)
102PY
103 return
104 fi
106 echo "Error: no timeout implementation available (tried timeout, gtimeout, and a working Python 3 interpreter)" >&2
107 exit 1
108}
110# Optional futility bound: CE_OPTIMIZE_CENSOR_AFTER=<seconds> kills a live
111# run that has already exceeded a predeclared noncompetitive bound. Distinct
112# from timeout_seconds (the spec's hard cap). Exit 125 means censored; 124
113# still means the configured timeout fired.
114CENSOR_AFTER="${CE_OPTIMIZE_CENSOR_AFTER:-}"
115CENSORING=0
116CENSOR_STATUS_FILE=""
117if [[ -n "$CENSOR_AFTER" ]] && awk -v a="$CENSOR_AFTER" -v t="$TIMEOUT" 'BEGIN { exit !(a ~ /^[0-9]+(\.[0-9]+)?$/ && t+0 == t && a+0 > 0 && a+0 < t+0) }'; then
118 TIMEOUT="$CENSOR_AFTER"
119 CENSORING=1
120 CENSOR_STATUS_FILE=$(mktemp "${TMPDIR:-/tmp}/ce-optimize-censor-XXXXXX")
121fi
123# Run the measurement command with timeout
124# timeout returns 124 if the command times out
125# We pass stdout and stderr through directly
126set +e
127run_with_timeout
128status=$?
129set -e
130if [[ $CENSORING -eq 1 ]]; then
131 if [[ -s "$CENSOR_STATUS_FILE" ]]; then
132 status=$(cat "$CENSOR_STATUS_FILE")
133 elif [[ $status -eq 124 ]]; then
134 echo "Error: measurement censored after ${CENSOR_AFTER}s (noncompetitive bound)" >&2
135 rm -f "$CENSOR_STATUS_FILE"
136 exit 125
137 fi
138 rm -f "$CENSOR_STATUS_FILE"
139fi
140exit "$status"