Skills › Scientific Agent Skills › hypogenic › evaluate_local.py
evaluate_local.py
250 lines · skills/hypogenic/scripts/evaluate_local.py at 5f2bc03
1#!/usr/bin/env python3
2"""Create an evaluation plan or score saved predictions without model calls."""
4from __future__ import annotations
6import argparse
7from collections.abc import Sequence
9if __package__:
10 from ._common import (
11 MAX_CONFIG_BYTES,
12 MAX_JSON_BYTES,
13 SPLIT_NAMES,
14 CliError,
15 checked_input_file,
16 classification_metrics,
17 emit_error,
18 emit_json,
19 load_json_document,
20 load_structured_document,
21 sha256_file,
22 validate_dataset_manifest,
23 validate_result_document,
24 validate_run_config,
25 )
26else:
27 from _common import ( # type: ignore
28 MAX_CONFIG_BYTES,
29 MAX_JSON_BYTES,
30 SPLIT_NAMES,
31 CliError,
32 checked_input_file,
33 classification_metrics,
34 emit_error,
35 emit_json,
36 load_json_document,
37 load_structured_document,
38 sha256_file,
39 validate_dataset_manifest,
40 validate_result_document,
41 validate_run_config,
42 )
45def build_parser() -> argparse.ArgumentParser:
46 parser = argparse.ArgumentParser(
47 description=(
48 "Plan split-safe evaluation or compute metrics from saved strict JSON. "
49 "No HypoGeniC/provider package or model is imported or called."
50 )
51 )
52 subparsers = parser.add_subparsers(dest="command", required=True)
54 plan = subparsers.add_parser(
55 "plan",
56 help="Create a model-free, split-aware evaluation plan",
57 )
58 plan.add_argument("--config", required=True, help="Reviewed run policy")
59 plan.add_argument("--manifest", required=True, help="Dataset manifest JSON")
60 plan.add_argument("--root", default=".", help="Existing local I/O boundary")
62 report = subparsers.add_parser(
63 "report",
64 help="Compute deterministic classification metrics from saved predictions",
65 )
66 report.add_argument("--results", required=True, help="Strict local result JSON")
67 report.add_argument("--root", default=".", help="Existing local I/O boundary")
68 report.add_argument(
69 "--expected-split",
70 choices=SPLIT_NAMES,
71 default="test",
72 help="Require this split label (default test)",
73 )
74 return parser
77def make_evaluation_plan(
78 config: dict,
79 manifest: dict,
80 *,
81 config_sha256: str,
82 manifest_sha256: str,
83) -> dict:
84 split_names = [split["name"] for split in manifest["splits"]]
85 return {
86 "ok": split_names == list(SPLIT_NAMES),
87 "plan_kind": "hypogenic_model_free_evaluation",
88 "schema_version": config["schema_version"],
89 "config_sha256": config_sha256,
90 "dataset_manifest_sha256": manifest_sha256,
91 "dataset_source": manifest["source"],
92 "provider": {
93 "type": config["provider"]["type"],
94 "model": config["provider"]["model"],
95 "data_destination": config["provider"]["data_destination"],
96 },
97 "split_protocol": [
98 {
99 "split": "train",
100 "allowed_use": "candidate generation and iterative updates",
101 "locked": False,
102 },
103 {
104 "split": "validation",
105 "allowed_use": "method, threshold, or hypothesis selection",
106 "locked": False,
107 },
108 {
109 "split": "test",
110 "allowed_use": "one final evaluation after choices are frozen",
111 "locked_until_final": True,
112 },
113 ],
114 "planned_metrics": [
115 "coverage",
116 "accuracy_all_records",
117 "accuracy_covered_records",
118 "macro_f1_all_records",
119 "redacted_confusion_matrix",
120 ],
121 "required_provenance": [
122 "immutable dataset revision",
123 "dataset manifest SHA-256",
124 "hypothesis bank SHA-256",
125 "exact split",
126 "seeds",
127 "selection procedure",
128 "missing-prediction handling",
129 "all deviations from the plan",
130 ],
131 "limits": {
132 "train_examples": config["limits"]["train_examples"],
133 "validation_examples": config["limits"]["validation_examples"],
134 "test_examples": config["limits"]["test_examples"],
135 "max_hypotheses": config["limits"]["max_hypotheses"],
136 },
137 "execution": {
138 "external_calls_authorized": False,
139 "requires_separate_confirmation": True,
140 "network_access": False,
141 "model_called": False,
142 "package_imported": False,
143 },
144 "interpretation": {
145 "candidate_hypotheses_are_scientific_evidence": False,
146 "predictive_metrics_validate_mechanisms": False,
147 "independent_scientific_validation_required": True,
148 },
149 }
152def make_report(result: dict, *, result_sha256: str) -> dict:
153 metrics = classification_metrics(result["records"])
154 return {
155 "ok": True,
156 "report_kind": "hypogenic_local_prediction_evaluation",
157 "schema_version": result["schema_version"],
158 "result_file_sha256": result_sha256,
159 "dataset_manifest_sha256": result["dataset_manifest_sha256"],
160 "hypothesis_bank_sha256": result["hypothesis_bank_sha256"],
161 "split": result["split"],
162 "metrics": metrics,
163 "raw_labels_predictions_and_ids_included": False,
164 "network_access": False,
165 "model_called": False,
166 "package_imported": False,
167 "interpretation": {
168 "scope": "classification utility on the declared saved split",
169 "candidate_hypotheses_are_scientific_evidence": False,
170 "metrics_validate_causality_or_novelty": False,
171 "independent_scientific_validation_required": True,
172 },
173 }
176def _plan(args: argparse.Namespace) -> int:
177 config_path = checked_input_file(
178 args.config,
179 root=args.root,
180 suffixes={".json", ".yaml", ".yml"},
181 max_bytes=MAX_CONFIG_BYTES,
182 )
183 manifest_path = checked_input_file(
184 args.manifest,
185 root=args.root,
186 suffixes={".json"},
187 max_bytes=MAX_CONFIG_BYTES,
188 )
189 config = validate_run_config(
190 load_structured_document(
191 args.config,
192 root=args.root,
193 max_bytes=MAX_CONFIG_BYTES,
194 )
195 )
196 manifest = validate_dataset_manifest(
197 load_json_document(
198 args.manifest,
199 root=args.root,
200 max_bytes=MAX_CONFIG_BYTES,
201 )
202 )
203 report = make_evaluation_plan(
204 config,
205 manifest,
206 config_sha256=sha256_file(config_path, max_bytes=MAX_CONFIG_BYTES),
207 manifest_sha256=sha256_file(manifest_path, max_bytes=MAX_CONFIG_BYTES),
208 )
209 emit_json(report)
210 return 0 if report["ok"] else 3
213def _report(args: argparse.Namespace) -> int:
214 result_path = checked_input_file(
215 args.results,
216 root=args.root,
217 suffixes={".json"},
218 max_bytes=MAX_JSON_BYTES,
219 )
220 result = validate_result_document(
221 load_json_document(
222 args.results,
223 root=args.root,
224 max_bytes=MAX_JSON_BYTES,
225 )
226 )
227 if result["split"] != args.expected_split:
228 raise CliError(
229 f"result split {result['split']!r} does not match "
230 f"--expected-split {args.expected_split!r}"
231 )
232 emit_json(
233 make_report(
234 result,
235 result_sha256=sha256_file(result_path, max_bytes=MAX_JSON_BYTES),
236 )
237 )
238 return 0
241def main(argv: Sequence[str] | None = None) -> int:
242 args = build_parser().parse_args(argv)
243 try:
244 return _plan(args) if args.command == "plan" else _report(args)
245 except (CliError, OSError) as error:
246 return emit_error(error)
249if __name__ == "__main__":
250 raise SystemExit(main())