Skills › Scientific Agent Skills › iso-standards-readiness › validate_evidence_manifest.py
validate_evidence_manifest.py
246 lines · skills/iso-standards-readiness/scripts/validate_evidence_manifest.py at 5f2bc03
1#!/usr/bin/env python3
2"""Validate a bounded audit/readiness evidence manifest and optional local files."""
4from __future__ import annotations
6import argparse
7import hashlib
8from pathlib import Path
9from typing import Any
11from _catalog import StandardProfile
12from _common import (
13 ALLOWED_STATUSES,
14 MAX_INPUT_BYTES,
15 InputError,
16 Review,
17 finish,
18 guarded_main,
19 load_json,
20 profile_for,
21 require_root_object,
22 standard_parser,
23)
25PURPOSES = {
26 "internal-audit",
27 "iso-certification-readiness",
28 "accreditation-assessment-readiness",
29 "fda-inspection-readiness",
30 "national-regulatory-inspection-readiness",
31 "mdsap-audit-readiness",
32 "eu-conformity-assessment-readiness",
33}
34CLASSIFICATIONS = {"public", "internal", "confidential", "restricted"}
37def _safe_evidence_file(base: Path, relative_text: str) -> Path:
38 relative = Path(relative_text)
39 if relative.is_absolute() or ".." in relative.parts:
40 raise InputError(f"evidence path must be relative and contained: {relative}")
41 if relative.suffix.lower() not in {".json", ".md", ".markdown"}:
42 raise InputError(f"evidence file must be JSON or Markdown: {relative}")
43 candidate = base / relative
44 if candidate.is_symlink():
45 raise InputError(f"symbolic-link evidence is refused: {relative}")
46 try:
47 resolved_base = base.resolve(strict=True)
48 resolved = candidate.resolve(strict=True)
49 resolved.relative_to(resolved_base)
50 stat = resolved.stat()
51 except (OSError, ValueError) as exc:
52 raise InputError(f"invalid evidence path {relative}: {exc}") from exc
53 if not resolved.is_file():
54 raise InputError(f"evidence is not a regular file: {relative}")
55 if stat.st_size > MAX_INPUT_BYTES:
56 raise InputError(
57 f"evidence exceeds {MAX_INPUT_BYTES} bytes: {relative}"
58 )
59 return resolved
62def _text_array(
63 review: Review,
64 value: Any,
65 path: str,
66 allowed: set[str],
67) -> list[str]:
68 raw = review.list(value, path, min_items=1)
69 if raw is None:
70 return []
71 result: list[str] = []
72 for index, item in enumerate(raw):
73 item_path = f"{path}[{index}]"
74 if not isinstance(item, str) or item not in allowed:
75 review.add(
76 "VALUE_UNKNOWN",
77 item_path,
78 f"must be one of: {', '.join(sorted(allowed))}",
79 )
80 else:
81 result.append(item)
82 if len(result) != len(set(result)):
83 review.add("VALUE_DUPLICATE", path, "must not contain duplicates")
84 return result
87def validate(
88 data: dict[str, Any],
89 *,
90 base_dir: str | None,
91 verify_files: bool,
92 profile: StandardProfile,
93) -> tuple[Review, dict[str, int]]:
94 review = Review()
95 domains = set(profile.process_domains)
96 metadata = review.object(data.get("metadata"), "metadata")
97 if metadata is not None:
98 review.controlled_item(metadata, "metadata", require_approved=True)
99 review.text(metadata, "manifest_id", "metadata", max_chars=120)
100 review.date(metadata, "review_date", "metadata")
102 context = review.object(data.get("audit_context"), "audit_context")
103 if context is not None:
104 review.choice(context, "purpose", PURPOSES, "audit_context")
105 review.text(context, "scope", "audit_context", max_chars=3_000)
106 review.text(context, "sampling_plan", "audit_context", max_chars=3_000)
107 review.text(context, "limitations", "audit_context", max_chars=3_000)
108 review.text(context, "owner", "audit_context", max_chars=200)
109 review.evidence(context, "audit_context")
110 review.approval(context, "audit_context", require_approved=True)
111 review.source_refs(context, "audit_context")
113 expected = _text_array(
114 review,
115 data.get("expected_domains"),
116 "expected_domains",
117 domains,
118 )
120 if verify_files and base_dir is None:
121 raise InputError("--verify-files requires --base-dir")
122 base = Path(base_dir) if base_dir is not None else None
124 entries = review.list(data.get("entries"), "entries", min_items=1) or []
125 covered: set[str] = set()
126 verified_files = 0
127 for index, value in enumerate(entries):
128 path = f"entries[{index}]"
129 entry = review.object(value, path)
130 if entry is None:
131 continue
132 review.text(entry, "id", path, max_chars=120)
133 domain = review.choice(
134 entry,
135 "domain",
136 domains,
137 path,
138 )
139 if domain:
140 covered.add(domain)
141 review.text(entry, "title", path, max_chars=300)
142 review.text(entry, "owner", path, max_chars=200)
143 status = review.choice(entry, "status", ALLOWED_STATUSES, path)
144 review.text(entry, "revision_or_date", path, max_chars=120)
145 review.choice(entry, "classification", CLASSIFICATIONS, path)
146 local_path = review.text(entry, "local_path", path, max_chars=1_000)
147 review.evidence(entry, path)
148 review.source_refs(entry, path)
149 review.approval(
150 entry,
151 path,
152 require_approved=status in {"approved", "implemented", "verified", "closed"},
153 )
154 if verify_files and local_path is not None and base is not None:
155 evidence_path = _safe_evidence_file(base, local_path)
156 digest = hashlib.sha256(evidence_path.read_bytes()).hexdigest()
157 expected_digest = entry.get("sha256")
158 if expected_digest is not None:
159 if (
160 not isinstance(expected_digest, str)
161 or len(expected_digest) != 64
162 or any(char not in "0123456789abcdef" for char in expected_digest)
163 ):
164 review.add(
165 "HASH_INVALID",
166 f"{path}.sha256",
167 "must be a lowercase SHA-256 hex digest",
168 )
169 elif expected_digest != digest:
170 review.add(
171 "HASH_MISMATCH",
172 f"{path}.sha256",
173 "does not match the local evidence file",
174 "blocker",
175 )
176 verified_files += 1
177 review.unique_ids(entries, "entries")
179 for domain in sorted(set(expected) - covered):
180 review.add(
181 "DOMAIN_EVIDENCE_MISSING",
182 "entries",
183 f"no manifest entry covers expected domain: {domain}",
184 )
186 gaps = review.list(data.get("open_gaps"), "open_gaps", min_items=0) or []
187 for index, value in enumerate(gaps):
188 path = f"open_gaps[{index}]"
189 gap = review.object(value, path)
190 if gap is None:
191 continue
192 review.text(gap, "id", path, max_chars=120)
193 review.text(gap, "description", path, max_chars=2_000)
194 review.text(gap, "owner", path, max_chars=200)
195 review.choice(gap, "status", {"open", "accepted", "closed"}, path)
196 review.evidence(gap, path, min_items=0)
197 review.approval(
198 gap,
199 path,
200 require_approved=gap.get("status") in {"accepted", "closed"},
201 )
202 review.unique_ids(gaps, "open_gaps")
204 return review, {
205 "entries": len(entries),
206 "expected_domains": len(expected),
207 "open_gaps": len(gaps),
208 "verified_files": verified_files,
209 }
212def main() -> int:
213 parser = standard_parser(
214 "Validate a local audit/readiness evidence manifest.",
215 "Path to the local evidence-manifest JSON file",
216 with_standard=True,
217 )
218 parser.add_argument(
219 "--base-dir",
220 help="Base directory for contained relative evidence paths",
221 )
222 parser.add_argument(
223 "--verify-files",
224 action="store_true",
225 help="Verify bounded local JSON/Markdown evidence paths and optional hashes",
226 )
227 args: argparse.Namespace = parser.parse_args()
228 profile = profile_for(args)
229 data = require_root_object(load_json(args.input))
230 review, metrics = validate(
231 data,
232 base_dir=args.base_dir,
233 verify_files=args.verify_files,
234 profile=profile,
235 )
236 return finish(
237 "validate_evidence_manifest",
238 review,
239 args,
240 metrics=metrics,
241 standard=profile.key,
242 )
245if __name__ == "__main__":
246 guarded_main(main)