Skills › HyperFrames › remotion-to-hyperframes › validate.sh
validate.sh
110 lines · skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/validate.sh at a595755
1#!/usr/bin/env bash
2# validate.sh — assert lint_source.py output matches expected.json for every T4 case.
3#
4# T4 has no renders to diff. The skill is graded on whether it correctly
5# refuses to translate each case (or drops only the lambda config in case 5,
6# or warns appropriately in cases 6 and 7).
7#
8# Usage:
9# ./validate.sh
10# Exit 0 on pass.
12set -euo pipefail
14THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
15SCRIPTS_DIR="$(cd "$THIS_DIR/../../../scripts" && pwd)"
16EXPECTED="$THIS_DIR/expected.json"
18if [[ ! -f "$SCRIPTS_DIR/lint_source.py" ]]; then
19 echo "error: lint_source.py not found at $SCRIPTS_DIR/lint_source.py" >&2
20 exit 2
21fi
22if [[ ! -f "$EXPECTED" ]]; then
23 echo "error: expected.json not found at $EXPECTED" >&2
24 exit 2
25fi
27# Drive lint_file() in-process so the per-case overhead is one Python startup,
28# not N (8 cases × ~80 ms forking python3 was the dominant cost).
29SCRIPTS_DIR="$SCRIPTS_DIR" \
30THIS_DIR="$THIS_DIR" \
31EXPECTED="$EXPECTED" \
32python3 <<'PY'
33import json
34import os
35import sys
36from collections import Counter
37from pathlib import Path
39scripts_dir = Path(os.environ["SCRIPTS_DIR"])
40this_dir = Path(os.environ["THIS_DIR"])
41expected_path = Path(os.environ["EXPECTED"])
42cases_dir = this_dir / "cases"
44sys.path.insert(0, str(scripts_dir))
45from lint_source import BLOCKER, WARNING, lint_file # noqa: E402
47expected = json.loads(expected_path.read_text())
49fails: list[str] = []
50passes: list[str] = []
52for case in expected["cases"]:
53 file_name = case["file"]
54 fixture = cases_dir / file_name
55 if not fixture.exists():
56 fails.append(f"{file_name}: fixture missing at {fixture}")
57 continue
59 findings = lint_file(fixture)
60 rule_counts: Counter[str] = Counter()
61 severity_by_rule: dict[str, str] = {}
62 for f in findings:
63 rule_counts[f.rule] += 1
64 severity_by_rule[f.rule] = f.severity
66 case_failed = False
68 def assert_rule(expected_entry, expected_severity_floor, kind):
69 global case_failed
70 rule = expected_entry["rule"]
71 min_count = expected_entry["min_count"]
72 actual = rule_counts[rule]
73 actual_severity = severity_by_rule.get(rule)
74 if actual < min_count:
75 fails.append(f"{file_name}: expected >={min_count} {kind} findings of rule {rule}, got {actual}")
76 case_failed = True
77 elif actual_severity not in expected_severity_floor:
78 fails.append(
79 f"{file_name}: rule {rule} found but severity={actual_severity!r} (expected {kind})"
80 )
81 case_failed = True
83 for entry in case["expected"]["blockers"]:
84 assert_rule(entry, {BLOCKER}, "blocker")
85 for entry in case["expected"]["warnings"]:
86 assert_rule(entry, {WARNING, BLOCKER}, "warning")
88 # Implied lint exit code: 1 when blockers are expected, 0 otherwise.
89 has_blockers = any(f.severity == BLOCKER for f in findings)
90 expected_has_blockers = bool(case["expected"]["blockers"])
91 if has_blockers != expected_has_blockers:
92 fails.append(
93 f"{file_name}: implied lint exit {1 if has_blockers else 0}, "
94 f"expected {1 if expected_has_blockers else 0} (blockers expected: {expected_has_blockers})"
95 )
96 case_failed = True
98 if not case_failed:
99 passes.append(file_name)
101print(f"Passed: {len(passes)}")
102for name in passes:
103 print(f" ✓ {name}")
104if fails:
105 print(f"Failed: {len(fails)}")
106 for msg in fails:
107 print(f" ✗ {msg}")
108 sys.exit(1)
109sys.exit(0)
110PY