Skills › BMAD Method › bmad-eval › aggregate_benchmark.py
aggregate_benchmark.py
254 lines · skills/bmad-eval/scripts/aggregate_benchmark.py at bd70c64
1#!/usr/bin/env python3
2# /// script
3# requires-python = ">=3.11"
4# ///
5"""Variance benchmark: summarize a metric across N runs, and compare two configs.
7A single skill run is noisy. Running the same case N times and summarizing the
8spread tells you whether a difference between two versions is real or just noise.
9This script computes, per numeric metric, the mean, the sample standard deviation
10(n-1, the unbiased estimator for a sample), the min, and the max across N runs.
11Given two such config summaries it reports the delta on each shared metric so a
12"did the change help" question gets a number instead of a guess.
14Input shapes accepted for a single config:
15 - a list of run records, each a flat dict of metric -> number
16 [{"elapsed_s": 12.1, "total_tokens": 800}, {"elapsed_s": 11.4, ...}]
17 - {"runs": [ ...records... ]}
18 - a directory of run folders, each holding timing.json files written by
19 run_evals.py (the script reads every timing.json under the directory and
20 treats each as one run record)
22Usage:
23 Summarize one config across its runs:
24 uv run aggregate_benchmark.py --runs CONFIG_A.json
25 uv run aggregate_benchmark.py --runs RUN_DIR/ (reads timing.json files)
27 Compare two configs (each summarized, then delta = B - A):
28 uv run aggregate_benchmark.py --baseline A.json --variant B.json
30 Self-test on a known fixture (no external input needed):
31 uv run aggregate_benchmark.py --self-test
33Output is one JSON object on stdout.
34"""
36from __future__ import annotations
38import argparse
39import json
40import math
41import sys
42from pathlib import Path
44NUMERIC = (int, float)
47# --- statistics -------------------------------------------------------------
50def sample_stddev(values: list[float]) -> float:
51 """Sample standard deviation using n-1 (Bessel's correction).
53 Returns 0.0 for fewer than two values, where the sample variance is
54 undefined and reporting zero spread is the least surprising choice.
55 """
56 n = len(values)
57 if n < 2:
58 return 0.0
59 mean = sum(values) / n
60 var = sum((x - mean) ** 2 for x in values) / (n - 1)
61 return math.sqrt(var)
64def summarize_metric(values: list[float]) -> dict:
65 return {
66 "n": len(values),
67 "mean": (sum(values) / len(values)) if values else 0.0,
68 "stddev": sample_stddev(values),
69 "min": min(values) if values else 0.0,
70 "max": max(values) if values else 0.0,
71 }
74def collect_numeric_metrics(records: list[dict]) -> dict[str, list[float]]:
75 """Group every numeric field across records by metric name."""
76 by_metric: dict[str, list[float]] = {}
77 for rec in records:
78 if not isinstance(rec, dict):
79 continue
80 for key, val in rec.items():
81 if isinstance(val, bool):
82 continue # bools are ints in Python; not a metric
83 if isinstance(val, NUMERIC):
84 by_metric.setdefault(key, []).append(float(val))
85 return by_metric
88def summarize_config(records: list[dict]) -> dict:
89 by_metric = collect_numeric_metrics(records)
90 return {
91 "runs": len(records),
92 "metrics": {name: summarize_metric(vals) for name, vals in sorted(by_metric.items())},
93 }
96def delta_configs(baseline: dict, variant: dict) -> dict:
97 """Per shared metric, delta = variant.mean - baseline.mean, plus context."""
98 b_metrics = baseline.get("metrics", {})
99 v_metrics = variant.get("metrics", {})
100 shared = sorted(set(b_metrics) & set(v_metrics))
101 out: dict[str, dict] = {}
102 for name in shared:
103 b = b_metrics[name]
104 v = v_metrics[name]
105 diff = v["mean"] - b["mean"]
106 pct = (diff / b["mean"] * 100.0) if b["mean"] != 0 else None
107 out[name] = {
108 "baseline_mean": b["mean"],
109 "variant_mean": v["mean"],
110 "delta": diff,
111 "delta_pct": pct,
112 "baseline_stddev": b["stddev"],
113 "variant_stddev": v["stddev"],
114 }
115 return out
118# --- input loading ----------------------------------------------------------
121def load_records(path: Path) -> tuple[list[dict], int]:
122 """Run records from a JSON file, a {'runs': [...]} file, or a dir of timing.json files.
124 Returns (completed records, number excluded). A record whose `status` is anything but
125 "ok" did not finish, so its elapsed time would skew the mean; it is left out and counted.
126 """
127 if path.is_dir():
128 data: list = []
129 for f in sorted(path.rglob("timing.json")):
130 try:
131 data.append(json.loads(f.read_text(encoding="utf-8")))
132 except (OSError, json.JSONDecodeError):
133 continue
134 else:
135 data = json.loads(path.read_text(encoding="utf-8"))
136 if isinstance(data, dict) and "runs" in data:
137 data = data["runs"]
138 if not isinstance(data, list):
139 raise ValueError(f"expected a list of run records in {path}")
140 records = [r for r in data if isinstance(r, dict)]
141 completed = [r for r in records if r.get("status", "ok") == "ok"]
142 return completed, len(records) - len(completed)
145# --- self-test --------------------------------------------------------------
148def run_self_test() -> int:
149 """Verify mean/stddev/min/max/delta on a known fixture."""
150 config_a = [
151 {"elapsed_s": 10.0, "total_tokens": 100},
152 {"elapsed_s": 12.0, "total_tokens": 200},
153 {"elapsed_s": 14.0, "total_tokens": 300},
154 ]
155 summary_a = summarize_config(config_a)
156 el = summary_a["metrics"]["elapsed_s"]
157 # mean of 10,12,14 = 12; n-1 stddev = sqrt(((-2)^2+0+2^2)/2)=sqrt(4)=2
158 assert el["n"] == 3, el
159 assert abs(el["mean"] - 12.0) < 1e-9, el
160 assert abs(el["stddev"] - 2.0) < 1e-9, el
161 assert el["min"] == 10.0 and el["max"] == 14.0, el
162 tok = summary_a["metrics"]["total_tokens"]
163 # mean of 100,200,300 = 200; n-1 stddev = sqrt((10000+0+10000)/2)=100
164 assert abs(tok["mean"] - 200.0) < 1e-9, tok
165 assert abs(tok["stddev"] - 100.0) < 1e-9, tok
167 # single value -> stddev 0
168 one = summarize_config([{"x": 5}])
169 assert one["metrics"]["x"]["stddev"] == 0.0, one
171 # bools are not treated as metrics
172 with_bool = summarize_config([{"ok": True, "x": 1}, {"ok": False, "x": 3}])
173 assert "ok" not in with_bool["metrics"], with_bool
174 assert abs(with_bool["metrics"]["x"]["mean"] - 2.0) < 1e-9, with_bool
176 # delta: variant slower by 3s on mean, faster question answered by sign
177 config_b = [
178 {"elapsed_s": 13.0, "total_tokens": 90},
179 {"elapsed_s": 15.0, "total_tokens": 110},
180 {"elapsed_s": 17.0, "total_tokens": 100},
181 ]
182 summary_b = summarize_config(config_b)
183 d = delta_configs(summary_a, summary_b)
184 # elapsed mean: A=12, B=15 -> delta +3, pct +25%
185 assert abs(d["elapsed_s"]["delta"] - 3.0) < 1e-9, d
186 assert abs(d["elapsed_s"]["delta_pct"] - 25.0) < 1e-9, d
187 # tokens mean: A=200, B=100 -> delta -100, pct -50%
188 assert abs(d["total_tokens"]["delta"] + 100.0) < 1e-9, d
189 assert abs(d["total_tokens"]["delta_pct"] + 50.0) < 1e-9, d
191 print(
192 json.dumps(
193 {
194 "self_test": "passed",
195 "checked": [
196 "mean",
197 "stddev_n_minus_1",
198 "min",
199 "max",
200 "single_value_stddev",
201 "bool_excluded",
202 "delta",
203 "delta_pct",
204 ],
205 }
206 )
207 )
208 return 0
211# --- main -------------------------------------------------------------------
214def main(argv: list[str] | None = None) -> int:
215 p = argparse.ArgumentParser(
216 description=__doc__,
217 formatter_class=argparse.RawDescriptionHelpFormatter,
218 )
219 p.add_argument("--runs", type=Path, help="summarize one config (JSON file or dir of timing.json)")
220 p.add_argument("--baseline", type=Path, help="baseline config for a two-config comparison")
221 p.add_argument("--variant", type=Path, help="variant config for a two-config comparison")
222 p.add_argument("--self-test", action="store_true", help="run the built-in fixture self-test and exit")
223 args = p.parse_args(argv)
225 if args.self_test:
226 return run_self_test()
228 if args.baseline and args.variant:
229 b_records, b_excluded = load_records(args.baseline)
230 v_records, v_excluded = load_records(args.variant)
231 b = summarize_config(b_records)
232 v = summarize_config(v_records)
233 out = {
234 "baseline": b,
235 "variant": v,
236 "delta": delta_configs(b, v),
237 "excluded_incomplete": {"baseline": b_excluded, "variant": v_excluded},
238 }
239 print(json.dumps(out, indent=2))
240 return 0
242 if args.runs:
243 records, excluded = load_records(args.runs)
244 out = summarize_config(records)
245 out["excluded_incomplete"] = excluded
246 print(json.dumps(out, indent=2))
247 return 0
249 p.error("provide --runs, or both --baseline and --variant, or --self-test")
250 return 2
253if __name__ == "__main__":
254 sys.exit(main())