Skills › BMAD Method › bmad-eval › convert_cases.py
convert_cases.py
119 lines · skills/bmad-eval/scripts/convert_cases.py at bd70c64
1#!/usr/bin/env python3
2# /// script
3# requires-python = ">=3.11"
4# ///
5"""Convert eval cases between the skill-creator and the runner formats.
7skill-creator's evals.json:
8 {"skill_name": "...", "evals": [{"id": 1, "prompt": "...", "expected_output": "...",
9 "files": [...], "expectations": [...]}]}
11The runner's cases.json:
12 [{"id": "1", "input": "...", "rubric": [...], "state_prefix": null, "files": [...]}]
14Toward the runner, `expected_output` lands at the end of the rubric as
15"output matches: <text>". Toward skill-creator, that rubric line becomes
16`expected_output` again and a numeric id becomes an int.
18Usage:
19 uv run convert_cases.py EVALS.json [--to runner|skill-creator] [--output OUT.json]
20 [--skill-name NAME]
22Without --output the result goes to stdout.
23"""
25from __future__ import annotations
27import argparse
28import json
29import sys
30from pathlib import Path
32OUTPUT_PREFIX = "output matches: "
35def to_runner(data: dict) -> list[dict]:
36 cases = []
37 for ev in data.get("evals", []):
38 rubric = [str(e) for e in ev.get("expectations", [])]
39 expected = ev.get("expected_output")
40 if expected:
41 rubric.append(f"{OUTPUT_PREFIX}{expected}")
42 cases.append(
43 {
44 "id": str(ev["id"]),
45 "input": ev.get("prompt", ""),
46 "rubric": rubric,
47 "state_prefix": None,
48 "files": list(ev.get("files", [])),
49 }
50 )
51 return cases
54def to_skill_creator(cases: list[dict], skill_name: str) -> dict:
55 evals = []
56 for case in cases:
57 raw_id = str(case["id"])
58 expectations = []
59 expected_output = ""
60 for item in case.get("rubric", []):
61 text = str(item)
62 if text.startswith(OUTPUT_PREFIX) and not expected_output:
63 expected_output = text[len(OUTPUT_PREFIX) :]
64 else:
65 expectations.append(text)
66 evals.append(
67 {
68 "id": int(raw_id) if raw_id.isdigit() else raw_id,
69 "prompt": case.get("input", ""),
70 "expected_output": expected_output,
71 "files": list(case.get("files", [])),
72 "expectations": expectations,
73 }
74 )
75 return {"skill_name": skill_name, "evals": evals}
78def load_cases(data: object) -> list[dict]:
79 if isinstance(data, dict) and "cases" in data:
80 data = data["cases"]
81 if not isinstance(data, list):
82 raise ValueError("runner cases must be a list or {'cases': [...]}")
83 return data
86def default_skill_name(path: Path) -> str:
87 # <skill>/evals/cases.json names the skill two levels up.
88 parent = path.resolve().parent
89 return parent.parent.name if parent.name == "evals" else parent.name
92def main(argv: list[str] | None = None) -> int:
93 p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
94 p.add_argument("input", type=Path, help="evals.json (skill-creator) or cases.json (runner)")
95 p.add_argument("--to", choices=("runner", "skill-creator"), default="runner")
96 p.add_argument("--output", type=Path, default=None, help="write here instead of stdout")
97 p.add_argument("--skill-name", default=None, help="skill_name for --to skill-creator")
98 args = p.parse_args(argv)
100 data = json.loads(args.input.read_text(encoding="utf-8"))
101 if args.to == "runner":
102 if not isinstance(data, dict) or "evals" not in data:
103 print("error: skill-creator input must be an object with an 'evals' list", file=sys.stderr)
104 return 2
105 result: object = to_runner(data)
106 else:
107 result = to_skill_creator(load_cases(data), args.skill_name or default_skill_name(args.input))
109 text = json.dumps(result, indent=2, ensure_ascii=False) + "\n"
110 if args.output:
111 args.output.parent.mkdir(parents=True, exist_ok=True)
112 args.output.write_text(text, encoding="utf-8")
113 else:
114 sys.stdout.write(text)
115 return 0
118if __name__ == "__main__":
119 sys.exit(main())