Skills › Scientific Agent Skills › clinical-decision-support › aggregate_model_evaluation_template.json
aggregate_model_evaluation_template.json
115 lines · skills/clinical-decision-support/assets/aggregate_model_evaluation_template.json at 5f2bc03
1{
2 "schema_version": "1.0",
3 "metadata": {
4 "evaluation_id": "SYNTHETIC-MODEL-EVAL-001",
5 "title": "Synthetic Aggregate Model Evaluation",
6 "evaluation_target": "Locked binary research model",
7 "target_version": "synthetic-v1",
8 "purpose": "Demonstrate aggregate performance and calibration reporting.",
9 "population": "Synthetic validation cohort; no real persons or records.",
10 "setting": "Synthetic offline evaluation",
11 "outcome": "Synthetic binary outcome",
12 "outcome_horizon": "Synthetic fixed horizon",
13 "threshold": "0.50, prespecified for this synthetic example",
14 "threshold_pre_specified": true,
15 "threshold_source_id": "SRC-PROTOCOL",
16 "validation_dataset": "Independent synthetic aggregate dataset",
17 "external_validation": true,
18 "data_cut_date": "2026-07-23",
19 "data_level": "synthetic",
20 "person_level_output": false,
21 "human_review_required": true
22 },
23 "intended_use": "Research evaluation only; not for patient care or live clinical use.",
24 "prohibited_uses": [
25 "No person-level prediction or classification.",
26 "No diagnosis, treatment recommendation, dosing, triage, alarm, or autonomous decision.",
27 "No claim of clinical validity, utility, safety, fairness, authorization, or compliance."
28 ],
29 "limitations": [
30 "All counts are synthetic.",
31 "Aggregate bins cannot estimate calibration slope.",
32 "Subgroup differences are descriptive and do not determine fairness."
33 ],
34 "human_review": {
35 "required": true,
36 "roles": [
37 "prediction-model methodologist",
38 "domain expert",
39 "fairness and governance reviewer"
40 ],
41 "completed": false
42 },
43 "governance": {
44 "owner": "research model governance owner",
45 "version_and_change_control": "Lock model, threshold, data, code, and report versions; independently validate every change.",
46 "monitoring": "Define aggregate calibration, discrimination, subgroup, drift, failure, and incident review.",
47 "auditability": "Retain provenance, commands, versions, reviewer decisions, and unresolved findings without sensitive data.",
48 "human_factors": "Document intended users and offline usability evidence; live evaluation is outside this skill.",
49 "rollback_and_retirement": "Define rollback, suspension, and retirement triggers before release."
50 },
51 "sources": [
52 {
53 "id": "SRC-PROTOCOL",
54 "citation": "Synthetic locked evaluation protocol, version 1.",
55 "url": "not_applicable_for_local_synthetic_source",
56 "accessed": "2026-07-23"
57 }
58 ],
59 "groups": [
60 {
61 "name": "Synthetic Group A",
62 "n": 120,
63 "confusion": {
64 "tp": 36,
65 "fp": 18,
66 "tn": 50,
67 "fn": 16
68 },
69 "calibration_bins": [
70 {
71 "n": 40,
72 "mean_predicted_probability": 0.2,
73 "observed_events": 12
74 },
75 {
76 "n": 40,
77 "mean_predicted_probability": 0.5,
78 "observed_events": 17
79 },
80 {
81 "n": 40,
82 "mean_predicted_probability": 0.75,
83 "observed_events": 23
84 }
85 ]
86 },
87 {
88 "name": "Synthetic Group B",
89 "n": 120,
90 "confusion": {
91 "tp": 28,
92 "fp": 22,
93 "tn": 56,
94 "fn": 14
95 },
96 "calibration_bins": [
97 {
98 "n": 40,
99 "mean_predicted_probability": 0.2,
100 "observed_events": 11
101 },
102 {
103 "n": 40,
104 "mean_predicted_probability": 0.45,
105 "observed_events": 14
106 },
107 {
108 "n": 40,
109 "mean_predicted_probability": 0.7,
110 "observed_events": 17
111 }
112 ]
113 }
114 ]
115}