Skills › Scientific Agent Skills › scholar-evaluation › generate_report_scaffold.py
generate_report_scaffold.py
243 lines · skills/scholar-evaluation/scripts/generate_report_scaffold.py at 5f2bc03
1#!/usr/bin/env python3
2"""Generate a minimized local JSON scaffold for qualified human review."""
4from __future__ import annotations
6import argparse
7from pathlib import Path
8from typing import Any
10import _common
13def _load_companion(
14 report: dict[str, Any] | None,
15 *,
16 report_type: str,
17 rubric_id: str | None = None,
18 evaluation_id: str | None = None,
19 work_id: str | None = None,
20) -> dict[str, Any] | None:
21 if report is None:
22 return None
23 if not isinstance(report, dict) or report.get("schema_version") != _common.SCHEMA_VERSION:
24 raise _common.ValidationError("COMPANION_REPORT_INVALID")
25 if report.get("report_type") != report_type:
26 raise _common.ValidationError("COMPANION_REPORT_TYPE_MISMATCH")
27 if rubric_id is not None and report.get("rubric_id") != rubric_id:
28 raise _common.ValidationError("COMPANION_RUBRIC_ID_MISMATCH")
29 if evaluation_id is not None and report.get("evaluation_id") != evaluation_id:
30 raise _common.ValidationError("COMPANION_EVALUATION_ID_MISMATCH")
31 if work_id is not None and report.get("work_id") != work_id:
32 raise _common.ValidationError("COMPANION_WORK_ID_MISMATCH")
33 return report
36def generate_scaffold(
37 rubric: dict[str, Any],
38 evaluation: dict[str, Any],
39 *,
40 traceability: dict[str, Any] | None = None,
41 agreement: dict[str, Any] | None = None,
42 sensitivity: dict[str, Any] | None = None,
43 process: dict[str, Any] | None = None,
44) -> dict[str, Any]:
45 rubric_issues = _common.validate_rubric(rubric)
46 _common.require_valid(rubric_issues)
47 evaluation_issues = _common.validate_evaluation(evaluation, rubric)
48 _common.require_valid(evaluation_issues)
49 score_report = _common.score_evaluation(rubric, evaluation)
50 traceability = _load_companion(
51 traceability,
52 report_type="evidence_traceability_check",
53 evaluation_id=evaluation["evaluation_id"],
54 work_id=evaluation["work_id"],
55 )
56 agreement = _load_companion(
57 agreement,
58 report_type="inter_rater_agreement_summary",
59 rubric_id=rubric["rubric_id"],
60 )
61 sensitivity = _load_companion(
62 sensitivity,
63 report_type="weight_sensitivity_and_rank_instability",
64 rubric_id=rubric["rubric_id"],
65 )
66 process = _load_companion(process, report_type="bias_and_process_check")
67 expected_record = {
68 "evaluation_id": evaluation["evaluation_id"],
69 "work_id": evaluation["work_id"],
70 }
71 for companion in (agreement, sensitivity):
72 if companion is not None and (
73 not isinstance(companion.get("evaluation_records"), list)
74 or expected_record not in companion["evaluation_records"]
75 ):
76 raise _common.ValidationError("COMPANION_EVALUATION_NOT_INCLUDED")
77 if traceability and (
78 traceability.get("status") not in ("pass", "fail")
79 or not isinstance(traceability.get("error_count"), int)
80 ):
81 raise _common.ValidationError("COMPANION_REPORT_INVALID")
82 agreement_overall = agreement.get("overall") if agreement else None
83 if agreement and (
84 not isinstance(agreement_overall, dict)
85 or not isinstance(agreement_overall.get("pair_observations"), int)
86 ):
87 raise _common.ValidationError("COMPANION_REPORT_INVALID")
88 if sensitivity and not isinstance(
89 sensitivity.get("rank_instability_detected"), bool
90 ):
91 raise _common.ValidationError("COMPANION_REPORT_INVALID")
92 if process and process.get("status") not in (
93 "invalid",
94 "blocked",
95 "incomplete",
96 "complete_for_low_stakes_process",
97 ):
98 raise _common.ValidationError("COMPANION_REPORT_INVALID")
99 criterion_scaffold = []
100 for result in score_report["criteria"]:
101 criterion_scaffold.append(
102 {
103 "criterion_id": result["criterion_id"],
104 "status": result["status"],
105 "score": result["score"],
106 "uncertainty": result["uncertainty"],
107 "evidence_finding_refs": [],
108 "strength_refs": [],
109 "limitation_refs": [],
110 "improvement_option_refs": [],
111 "qualified_reviewer_comment_ref": "",
112 }
113 )
114 return {
115 "schema_version": _common.SCHEMA_VERSION,
116 "report_type": "developmental_scholarly_work_report_scaffold",
117 "notice": _common.NOTICE,
118 "evaluation_id": evaluation["evaluation_id"],
119 "work_id": evaluation["work_id"],
120 "rubric_id": rubric["rubric_id"],
121 "purpose": _common.ALLOWED_PURPOSE,
122 "unit_of_assessment": _common.ALLOWED_UNIT,
123 "construct": {
124 "rubric_record_ref": rubric["rubric_id"],
125 "content_validity_status": rubric["provenance"][
126 "content_validity_status"
127 ],
128 "rubric_source_count": len(rubric["provenance"]["source_ids"]),
129 },
130 "descriptive_score_summary": score_report["aggregates"],
131 "criterion_scaffold": criterion_scaffold,
132 "quality_assurance": {
133 "traceability_status": (
134 traceability.get("status") if traceability else "not_provided"
135 ),
136 "traceability_error_count": (
137 traceability.get("error_count") if traceability else None
138 ),
139 "agreement_pair_observations": (
140 agreement_overall.get("pair_observations") if agreement_overall else None
141 ),
142 "agreement_status": (
143 "provided" if agreement else "not_provided"
144 ),
145 "inter_rater_reliability_status": rubric["rater_protocol"][
146 "inter_rater_reliability_status"
147 ],
148 "weight_sensitivity_status": (
149 "provided" if sensitivity else "not_provided"
150 ),
151 "rank_instability_detected": (
152 sensitivity.get("rank_instability_detected")
153 if sensitivity
154 else None
155 ),
156 "process_check_status": (
157 process.get("status") if process else "not_provided"
158 ),
159 },
160 "human_review": {
161 "committee_finding_refs": [],
162 "conflict_and_recusal_record_ref": "",
163 "disciplinary_context_ref": "",
164 "accessibility_accommodation_record_ref": "",
165 "subgroup_bias_review_ref": rubric["governance"][
166 "subgroup_review_ref"
167 ],
168 "appeal_process_ref": rubric["governance"]["appeal_process_ref"],
169 "final_human_review_record_ref": "",
170 },
171 "required_limitations": [
172 "ScholarEval is an experimental literature-grounded research-idea framework, not validated psychometrics.",
173 "This rubric requires separate validity, reliability, fairness, and intended-use evidence.",
174 "Missing and not-applicable ratings limit comparability.",
175 "Criterion uncertainty intervals are bounded judgment ranges, not confidence intervals.",
176 "Journal, citation, prestige, institution, venue, and attention indicators do not establish quality.",
177 "Companion identifiers establish membership, not unchanged input content; regenerate checks after edits.",
178 "The process checklist is a separate self-attestation, not verification of this work or rubric.",
179 ],
180 "private_source_content_included": False,
181 "person_ranking_provided": False,
182 "decision_recommendation_provided": False,
183 }
186def build_parser() -> argparse.ArgumentParser:
187 parser = argparse.ArgumentParser(
188 description=(
189 "Generate a local JSON report scaffold containing bounded scores, "
190 "quality-control summaries, and empty reference slots for qualified "
191 "human findings. No source-document text is copied."
192 )
193 )
194 parser.add_argument("--rubric", required=True, type=Path, help="Local rubric JSON")
195 parser.add_argument(
196 "--evaluation", required=True, type=Path, help="Local evaluation JSON"
197 )
198 parser.add_argument(
199 "--traceability", type=Path, help="Optional traceability report JSON"
200 )
201 parser.add_argument(
202 "--agreement", type=Path, help="Optional agreement report JSON"
203 )
204 parser.add_argument(
205 "--sensitivity", type=Path, help="Optional sensitivity report JSON"
206 )
207 parser.add_argument(
208 "--process", type=Path, help="Optional process-check report JSON"
209 )
210 parser.add_argument("--output", type=Path, help="Optional local JSON report")
211 parser.add_argument(
212 "--force", action="store_true", help="Replace an existing output JSON file"
213 )
214 return parser
217def main() -> int:
218 args = build_parser().parse_args()
219 try:
220 optional_paths = {
221 "traceability": args.traceability,
222 "agreement": args.agreement,
223 "sensitivity": args.sensitivity,
224 "process": args.process,
225 }
226 companions = {
227 name: _common.read_json(path) if path else None
228 for name, path in optional_paths.items()
229 }
230 report = generate_scaffold(
231 _common.read_json(args.rubric),
232 _common.read_json(args.evaluation),
233 **companions,
234 )
235 _common.write_json(report, args.output, force=args.force)
236 return 0
237 except _common.ValidationError as error:
238 _common.write_json(_common.failure_report(error), None)
239 return 2
242if __name__ == "__main__":
243 raise SystemExit(main())