Skills › Scientific Agent Skills › scholar-evaluation › summarize_agreement.py
summarize_agreement.py
272 lines · skills/scholar-evaluation/scripts/summarize_agreement.py at 5f2bc03
1#!/usr/bin/env python3
2"""Summarize inter-rater agreement from pseudonymous local CSV ratings."""
4from __future__ import annotations
6import argparse
7import csv
8import io
9import itertools
10import math
11from collections import Counter, defaultdict
12from pathlib import Path
13from typing import Any
15import _common
17CSV_FIELDS = [
18 "evaluation_id",
19 "work_id",
20 "rater_id",
21 "criterion_id",
22 "status",
23 "score",
24]
27def read_rows(path: Path, rubric: dict[str, Any]) -> list[dict[str, Any]]:
28 text = _common.read_csv_text(path)
29 reader = csv.DictReader(io.StringIO(text))
30 if reader.fieldnames != CSV_FIELDS:
31 raise _common.ValidationError("CSV_HEADER_INVALID")
32 accepted_scores = _common.scale_values(rubric)
33 criterion_ids = {criterion["criterion_id"] for criterion in rubric["criteria"]}
34 rows: list[dict[str, Any]] = []
35 seen: set[tuple[str, str, str, str]] = set()
36 evaluation_works: dict[str, str] = {}
37 for index, row in enumerate(reader, start=2):
38 if len(rows) >= _common.MAX_CSV_ROWS:
39 raise _common.ValidationError("CSV_TOO_MANY_ROWS")
40 path_prefix = f"$.rows[{index}]"
41 if set(row) != set(CSV_FIELDS) or any(
42 not isinstance(row.get(field), str) for field in CSV_FIELDS
43 ):
44 raise _common.ValidationError("CSV_ROW_SHAPE_INVALID", path_prefix)
45 for field in ("evaluation_id", "work_id", "rater_id", "criterion_id"):
46 if not _common.is_identifier(row.get(field)):
47 raise _common.ValidationError(
48 "CSV_IDENTIFIER_INVALID", f"{path_prefix}.{field}"
49 )
50 if row["criterion_id"] not in criterion_ids:
51 raise _common.ValidationError(
52 "CRITERION_UNKNOWN", f"{path_prefix}.criterion_id"
53 )
54 if row["status"] not in _common.RATING_STATUSES:
55 raise _common.ValidationError(
56 "RATING_STATUS_INVALID", f"{path_prefix}.status"
57 )
58 key = (
59 row["evaluation_id"],
60 row["work_id"],
61 row["rater_id"],
62 row["criterion_id"],
63 )
64 if key in seen:
65 raise _common.ValidationError("CSV_RATING_DUPLICATE", path_prefix)
66 seen.add(key)
67 previous_work = evaluation_works.setdefault(row["evaluation_id"], row["work_id"])
68 if previous_work != row["work_id"]:
69 raise _common.ValidationError("EVALUATION_WORK_MISMATCH", path_prefix)
70 score: float | None
71 if row["status"] == "rated":
72 try:
73 score = float(row["score"])
74 except ValueError as error:
75 raise _common.ValidationError(
76 "CSV_SCORE_INVALID", f"{path_prefix}.score"
77 ) from error
78 if not math.isfinite(score) or score not in accepted_scores:
79 raise _common.ValidationError(
80 "SCORE_NOT_ON_SCALE", f"{path_prefix}.score"
81 )
82 else:
83 if row["score"].strip():
84 raise _common.ValidationError(
85 "UNRATED_SCORE_MUST_BE_BLANK", f"{path_prefix}.score"
86 )
87 score = None
88 rows.append({**row, "score": score})
89 if not rows:
90 raise _common.ValidationError("CSV_NO_ROWS")
91 return rows
94def _summary(differences: Counter[float], step: float) -> dict[str, Any]:
95 if not differences:
96 return {
97 "pair_observations": 0,
98 "exact_agreement_rate": None,
99 "within_one_scale_step_rate": None,
100 "mean_absolute_difference": None,
101 }
102 pair_count = sum(differences.values())
103 return {
104 "pair_observations": pair_count,
105 "exact_agreement_rate": _common.rounded(
106 sum(count for difference, count in differences.items() if difference == 0)
107 / pair_count
108 ),
109 "within_one_scale_step_rate": _common.rounded(
110 sum(
111 count for difference, count in differences.items()
112 if difference <= step or math.isclose(difference, step, rel_tol=1e-9, abs_tol=0.0)
113 )
114 / pair_count
115 ),
116 "mean_absolute_difference": _common.rounded(
117 sum(difference * count for difference, count in differences.items()) / pair_count
118 ),
119 }
122def summarize(rubric: dict[str, Any], rows: list[dict[str, Any]]) -> dict[str, Any]:
123 rubric_issues = _common.validate_rubric(rubric)
124 _common.require_valid(rubric_issues)
125 grouped: dict[str, dict[tuple[str, str], dict[str, float]]] = defaultdict(
126 lambda: defaultdict(dict)
127 )
128 status_counts: dict[str, dict[str, int]] = defaultdict(
129 lambda: {"rated": 0, "missing": 0, "not_applicable": 0}
130 )
131 raters_by_criterion: dict[str, set[str]] = defaultdict(set)
132 works_by_criterion: dict[str, set[str]] = defaultdict(set)
133 all_raters: set[str] = set()
134 all_works: set[str] = set()
135 evaluation_works: dict[str, str] = {}
136 seen: set[tuple[str, str, str, str]] = set()
137 for row in rows:
138 key = tuple(row[field] for field in CSV_FIELDS[:4])
139 if key in seen:
140 raise _common.ValidationError("CSV_RATING_DUPLICATE")
141 seen.add(key)
142 previous_work = evaluation_works.setdefault(row["evaluation_id"], row["work_id"])
143 if previous_work != row["work_id"]:
144 raise _common.ValidationError("EVALUATION_WORK_MISMATCH")
145 criterion_id = row["criterion_id"]
146 status_counts[criterion_id][row["status"]] += 1
147 raters_by_criterion[criterion_id].add(row["rater_id"])
148 works_by_criterion[criterion_id].add(row["work_id"])
149 all_raters.add(row["rater_id"])
150 all_works.add(row["work_id"])
151 if row["status"] == "rated":
152 unit = (row["evaluation_id"], row["work_id"])
153 grouped[criterion_id][unit][row["rater_id"]] = row["score"]
155 minimum_raters = int(rubric["rater_protocol"]["minimum_raters"])
156 if len(all_raters) < minimum_raters:
157 raise _common.ValidationError("MINIMUM_RATERS_NOT_MET")
158 step = float(rubric["scale"]["step"])
159 criterion_reports: list[dict[str, Any]] = []
160 overall_differences: Counter[float] = Counter()
161 insufficient: list[str] = []
162 for criterion in rubric["criteria"]:
163 criterion_id = criterion["criterion_id"]
164 differences: Counter[float] = Counter()
165 overlap_works: set[str] = set()
166 overlap_evaluation_count = 0
167 below_minimum_count = 0
168 for (_, work_id), work_ratings in grouped.get(criterion_id, {}).items():
169 if len(work_ratings) >= 2:
170 overlap_works.add(work_id)
171 overlap_evaluation_count += 1
172 if len(work_ratings) < minimum_raters:
173 below_minimum_count += 1
174 counts = Counter(work_ratings.values())
175 # At most 21 scale anchors: do not materialize O(raters**2) pairs.
176 for score, count in counts.items():
177 if count >= 2:
178 differences[0.0] += count * (count - 1) // 2
179 for (first, first_count), (second, second_count) in itertools.combinations(counts.items(), 2):
180 differences[abs(first - second)] += first_count * second_count
181 if not differences:
182 insufficient.append(criterion_id)
183 overall_differences.update(differences)
184 criterion_reports.append(
185 {
186 "criterion_id": criterion_id,
187 "rater_count": len(raters_by_criterion.get(criterion_id, set())),
188 "work_count": len(works_by_criterion.get(criterion_id, set())),
189 "works_with_rater_overlap": len(overlap_works),
190 "evaluations_with_rater_overlap": overlap_evaluation_count,
191 "rated_evaluations_below_minimum_raters": below_minimum_count,
192 "rated_rows": status_counts[criterion_id]["rated"],
193 "missing_rows": status_counts[criterion_id]["missing"],
194 "not_applicable_rows": status_counts[criterion_id][
195 "not_applicable"
196 ],
197 **_summary(differences, step),
198 }
199 )
200 warnings = []
201 if insufficient:
202 warnings.append("INSUFFICIENT_OVERLAP_FOR_SOME_CRITERIA")
203 if any(item["rated_evaluations_below_minimum_raters"] for item in criterion_reports):
204 warnings.append("MINIMUM_RATERS_NOT_MET_FOR_SOME_CRITERIA_EVALUATIONS")
205 if any(
206 counts["missing"] or counts["not_applicable"]
207 for counts in status_counts.values()
208 ):
209 warnings.append("MISSING_OR_NOT_APPLICABLE_RATINGS_PRESENT")
210 return {
211 "schema_version": _common.SCHEMA_VERSION,
212 "report_type": "inter_rater_agreement_summary",
213 "notice": _common.NOTICE,
214 "rubric_id": rubric["rubric_id"],
215 "rater_count": len(all_raters),
216 "work_count": len(all_works),
217 "row_count": len(rows),
218 "evaluation_records": [
219 {"evaluation_id": evaluation_id, "work_id": work_id}
220 for evaluation_id, work_id in sorted(evaluation_works.items())
221 ],
222 "aggregation_unit": "criterion within one evaluation_id and work_id",
223 "aggregation_weighting": "pooled rater pairs; units with more rated raters contribute more pairs",
224 "scale_step": step,
225 "overall": _summary(overall_differences, step),
226 "criteria": criterion_reports,
227 "criteria_with_insufficient_overlap": insufficient,
228 "warnings": warnings,
229 "rater_identifiers_in_output": False,
230 "limitations": [
231 "Percent agreement and mean absolute difference are descriptive.",
232 "These summaries are not a psychometric validation or reliability coefficient.",
233 "Select a construct- and design-appropriate reliability model with qualified measurement expertise.",
234 "Repeated rounds are kept separate; pooled pairs are dependent observations, not independent samples.",
235 "Omitted CSV rows have unknown status and are not counted as missing.",
236 ],
237 }
240def build_parser() -> argparse.ArgumentParser:
241 parser = argparse.ArgumentParser(
242 description=(
243 "Summarize exact agreement, within-step agreement, and mean "
244 "absolute difference from pseudonymous local CSV ratings."
245 )
246 )
247 parser.add_argument("--rubric", required=True, type=Path, help="Local rubric JSON")
248 parser.add_argument(
249 "--ratings", required=True, type=Path, help="Local ratings CSV"
250 )
251 parser.add_argument("--output", type=Path, help="Optional local JSON report")
252 parser.add_argument(
253 "--force", action="store_true", help="Replace an existing output JSON file"
254 )
255 return parser
258def main() -> int:
259 args = build_parser().parse_args()
260 try:
261 rubric = _common.read_json(args.rubric)
262 _common.require_valid(_common.validate_rubric(rubric))
263 report = summarize(rubric, read_rows(args.ratings, rubric))
264 _common.write_json(report, args.output, force=args.force)
265 return 0
266 except _common.ValidationError as error:
267 _common.write_json(_common.failure_report(error), None)
268 return 2
271if __name__ == "__main__":
272 raise SystemExit(main())