Skills › HyperFrames › remotion-to-hyperframes › run.sh
run.sh
246 lines · skills/remotion-to-hyperframes/assets/test-corpus/run.sh at a595755
1#!/usr/bin/env bash
2# run.sh — corpus orchestrator. Runs every tier and prints a pass/fail summary.
3#
4# Tiers 1-3: render Remotion baseline + HF translation, run SSIM diff,
5# assert mean >= ssim_threshold from each fixture's expected.json.
6# Tier 4: runs cases/validate.sh which lints each case and asserts against
7# expected.json.
8#
9# Usage:
10# ./run.sh run all tiers
11# ./run.sh tier-1-title-card run a single tier
12#
13# Requirements:
14# - ffmpeg, ffprobe, python3 on PATH
15# - node 22 (for the HF CLI)
16# - npm (for Remotion installs)
17# - HF CLI built at packages/cli/dist/cli.js (run `bun run --filter @hyperframes/cli build`
18# in the repo root if missing)
19#
20# Output:
21# <fixture>/diff/summary.json per-fixture SSIM summary
22# <fixture>/diff/worst.jpg per-fixture sheet of the worst moments
23# ./run-report.json aggregate report
25set -euo pipefail
27THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
28SKILL_DIR="$(cd "$THIS_DIR/../.." && pwd)"
29REPO_ROOT="$(cd "$SKILL_DIR/../.." && pwd)"
31LINT="$SKILL_DIR/scripts/lint_source.py"
32HF_CLI="$REPO_ROOT/packages/cli/dist/cli.js"
33REPORT="$THIS_DIR/run-report.json"
35# Per-fixture results land here as one JSON file each, then the aggregator
36# globs them. This is safer than building JSON via bash string concatenation
37# (a fixture name containing a quote would break the previous approach).
38RESULTS_DIR="$(mktemp -d)"
39trap 'rm -rf "$RESULTS_DIR"' EXIT
41# T4 is lint-only — no ffmpeg or HF CLI needed. Defer the render-tier
42# toolchain checks until run_render_tier() actually runs, so
43# `./run.sh tier-4-escape-hatch` works on a clean checkout.
44require_render_tier_tools() {
45 if [[ ! -f "$HF_CLI" ]]; then
46 echo "error: HF CLI not built at $HF_CLI" >&2
47 echo " Run 'bun run --filter @hyperframes/cli build' in $REPO_ROOT" >&2
48 return 2
49 fi
50 if ! command -v ffmpeg >/dev/null 2>&1; then
51 echo "error: ffmpeg not on PATH" >&2
52 return 2
53 fi
54 return 0
55}
57# Write one fixture's result as a JSON file. Values are passed via argv so
58# bash string interpolation can't corrupt the JSON or inject Python source.
59write_result() {
60 local fixture_name="$1"
61 local status="$2"
62 shift 2
63 python3 - "$RESULTS_DIR/$fixture_name.json" "$fixture_name" "$status" "$@" <<'PY'
64import json
65import sys
67out_path, fixture_name, status, *kvs = sys.argv[1:]
68result = {"fixture": fixture_name, "status": status}
69for i in range(0, len(kvs), 2):
70 k, v = kvs[i], kvs[i + 1]
71 try:
72 result[k] = float(v) if "." in v or v.lstrip("-").isdigit() else v
73 except ValueError:
74 result[k] = v
75with open(out_path, "w") as f:
76 json.dump(result, f)
77PY
78}
80# Read a top-level scalar value from a JSON file. Falls back to $3 if the
81# key is missing (used to default composition_id for older fixtures).
82read_json_value() {
83 local file="$1"
84 local key="$2"
85 local default="${3:-}"
86 python3 - "$file" "$key" "$default" <<'PY'
87import json
88import sys
90path, key, default = sys.argv[1], sys.argv[2], sys.argv[3]
91with open(path) as f:
92 data = json.load(f)
93val = data.get(key, default)
94print(val if val is not None else "")
95PY
96}
98run_render_tier() {
99 local fixture_dir="$1"
100 local fixture_name
101 fixture_name=$(basename "$fixture_dir")
102 local expected="$fixture_dir/expected.json"
104 if ! require_render_tier_tools; then
105 echo " ⚠ $fixture_name: render toolchain unavailable, skipping"
106 write_result "$fixture_name" "skipped" reason "render toolchain unavailable"
107 return 0
108 fi
110 local threshold composition_id
111 threshold=$(read_json_value "$expected" "ssim_threshold")
112 composition_id=$(read_json_value "$expected" "composition_id" "Composition")
114 echo " ▶ $fixture_name (threshold $threshold, composition $composition_id)"
116 if [[ -x "$fixture_dir/setup.sh" ]]; then
117 "$fixture_dir/setup.sh" >/dev/null
118 fi
120 if ! python3 "$LINT" "$fixture_dir/remotion-src/src/" >/dev/null; then
121 echo " ✗ lint failed (blockers in Remotion source)"
122 write_result "$fixture_name" "fail" stage "lint"
123 return 0
124 fi
126 if [[ ! -d "$fixture_dir/remotion-src/node_modules" ]]; then
127 echo " ⏳ npm install (first run)"
128 (cd "$fixture_dir/remotion-src" && npm install --silent --no-progress >/dev/null 2>&1)
129 fi
131 echo " ⏳ render Remotion baseline"
132 if ! (cd "$fixture_dir/remotion-src" && \
133 npx --no-install remotion render "$composition_id" out/baseline.mp4 >/dev/null 2>&1); then
134 echo " ✗ Remotion render failed"
135 write_result "$fixture_name" "fail" stage "remotion-render"
136 return 0
137 fi
139 echo " ⏳ render HF translation"
140 if ! (cd "$fixture_dir" && \
141 node "$HF_CLI" render hf-src/ --output hf.mp4 --quiet >/dev/null 2>&1); then
142 echo " ✗ HF render failed"
143 write_result "$fixture_name" "fail" stage "hf-render"
144 return 0
145 fi
147 # Gate on the tier's mean threshold only, as the calibration runs did; the
148 # worst-moment sheet lands in diff/worst.jpg either way.
149 mkdir -p "$fixture_dir/diff"
150 local mean
151 if node "$HF_CLI" diff \
152 "$fixture_dir/remotion-src/out/baseline.mp4" \
153 "$fixture_dir/hf.mp4" \
154 --min-mean "$threshold" --min-p05 0 --min-cell 0 \
155 --out "$fixture_dir/diff" --json >"$fixture_dir/diff/summary.json" 2>/dev/null; then
156 mean=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['ssim']['mean'])" "$fixture_dir/diff/summary.json")
157 echo " ✓ pass (mean SSIM $mean, threshold $threshold)"
158 write_result "$fixture_name" "pass" mean_ssim "$mean" threshold "$threshold"
159 else
160 mean=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1])).get('ssim', {}).get('mean', ''))" "$fixture_dir/diff/summary.json")
161 echo " ✗ fail (mean SSIM $mean, threshold $threshold; sheet $fixture_dir/diff/worst.jpg)"
162 write_result "$fixture_name" "fail" stage "ssim" mean_ssim "$mean" threshold "$threshold"
163 fi
164}
166run_lint_tier() {
167 local fixture_dir="$1"
168 local fixture_name
169 fixture_name=$(basename "$fixture_dir")
171 echo " ▶ $fixture_name (lint-only)"
172 if "$fixture_dir/validate.sh" >/dev/null 2>&1; then
173 echo " ✓ pass (8/8 cases)"
174 write_result "$fixture_name" "pass" mode "lint"
175 else
176 echo " ✗ fail (some cases mismatched expected.json)"
177 write_result "$fixture_name" "fail" mode "lint"
178 fi
179}
181echo "remotion-to-hyperframes corpus run"
182echo "=================================="
184for tier in tier-1-title-card tier-2-multi-scene tier-3-data-driven; do
185 if [[ -n "${1:-}" && "$1" != "$tier" ]]; then
186 continue
187 fi
188 if [[ -d "$THIS_DIR/$tier" ]]; then
189 run_render_tier "$THIS_DIR/$tier"
190 fi
191done
193if [[ -z "${1:-}" || "$1" == "tier-4-escape-hatch" ]]; then
194 if [[ -d "$THIS_DIR/tier-4-escape-hatch" ]]; then
195 run_lint_tier "$THIS_DIR/tier-4-escape-hatch"
196 fi
197fi
199# Aggregate the per-fixture JSON files into one report.
200#
201# Skipped fixtures are *not* a pass — they mean a tier didn't run because
202# tooling or fixtures were unavailable. The orchestrator exits non-zero on
203# any skip so a clean checkout that lacks the HF CLI doesn't accidentally
204# report "passed 1/4" (T4 alone) and look like the corpus is healthy.
205#
206# Single-tier mode (`./run.sh tier-N`) only writes a result file for the
207# selected tier; tiers that weren't run aren't counted as skips.
208python3 - "$RESULTS_DIR" "$REPORT" <<'PY'
209import json
210import sys
211from pathlib import Path
213results_dir, out_path = Path(sys.argv[1]), Path(sys.argv[2])
214results = sorted(
215 (json.loads(p.read_text()) for p in results_dir.glob("*.json")),
216 key=lambda r: r["fixture"],
217)
219total = len(results)
220passed = sum(1 for r in results if r["status"] == "pass")
221failed = sum(1 for r in results if r["status"] == "fail")
222skipped = sum(1 for r in results if r["status"] == "skipped")
223report = {
224 "total": total,
225 "passed": passed,
226 "failed": failed,
227 "skipped": skipped,
228 "results": results,
229}
230out_path.write_text(json.dumps(report, indent=2))
232print()
233print("=" * 50)
234print(f" passed {passed}/{total}, failed {failed}, skipped {skipped}")
235print(f" report → {out_path}")
236if skipped > 0:
237 skipped_fixtures = [r["fixture"] for r in results if r["status"] == "skipped"]
238 skipped_reasons = sorted({r.get("reason", "unknown") for r in results if r["status"] == "skipped"})
239 print()
240 print(f" ⚠ {skipped} skipped: {', '.join(skipped_fixtures)}")
241 for reason in skipped_reasons:
242 print(f" reason: {reason}")
243 print(" Skipped fixtures count as failures for the aggregate.")
244print("=" * 50)
245sys.exit(0 if failed == 0 and skipped == 0 else 1)
246PY