Skills › Scientific Agent Skills › exploratory-data-analysis › _capabilities.py
_capabilities.py
576 lines · skills/exploratory-data-analysis/scripts/_capabilities.py at 5f2bc03
1#!/usr/bin/env python3
2"""Closed capability registry and lightweight format checks for EDA tools."""
4from __future__ import annotations
6import itertools
7import zipfile
8from pathlib import Path
9from typing import Any
11from _common import (
12 MAX_COMPRESSION_RATIO,
13 MAX_NPZ_MEMBERS,
14 MAX_NPZ_UNCOMPRESSED_BYTES,
15 CliError,
16)
19AUTOMATED_FORMATS: dict[str, dict[str, Any]] = {
20 ".csv": {
21 "format": "CSV",
22 "tier": "automated_core",
23 "depth": "bounded schema, aggregate profile, missingness, and sensitivity audit",
24 "dependency": "Python standard library",
25 "reference": "references/general_scientific_formats.md",
26 },
27 ".tsv": {
28 "format": "TSV",
29 "tier": "automated_core",
30 "depth": "bounded schema, aggregate profile, missingness, and sensitivity audit",
31 "dependency": "Python standard library",
32 "reference": "references/general_scientific_formats.md",
33 },
34 ".json": {
35 "format": "JSON",
36 "tier": "automated_core",
37 "depth": "bounded strict parse and structural aggregate profile",
38 "dependency": "Python standard library",
39 "reference": "references/general_scientific_formats.md",
40 },
41 ".npy": {
42 "format": "NumPy NPY",
43 "tier": "automated_optional",
44 "depth": "header, shape, dtype, and bounded numeric sample",
45 "dependency": "numpy==2.5.3",
46 "reference": "references/general_scientific_formats.md",
47 },
48 ".npz": {
49 "format": "NumPy NPZ",
50 "tier": "automated_optional",
51 "depth": "ZIP bomb preflight plus per-array bounded inspection",
52 "dependency": "numpy==2.5.3",
53 "reference": "references/general_scientific_formats.md",
54 },
55 ".h5": {
56 "format": "HDF5",
57 "tier": "automated_optional",
58 "depth": "bounded hierarchy and dataset metadata; no dataset values or external links",
59 "dependency": "h5py==3.16.0",
60 "reference": "references/general_scientific_formats.md",
61 },
62 ".hdf5": {
63 "format": "HDF5",
64 "tier": "automated_optional",
65 "depth": "bounded hierarchy and dataset metadata; no dataset values or external links",
66 "dependency": "h5py==3.16.0",
67 "reference": "references/general_scientific_formats.md",
68 },
69 ".fasta": {
70 "format": "FASTA",
71 "tier": "automated_optional",
72 "depth": "bounded record sample with length, alphabet, and GC aggregates",
73 "dependency": "biopython==1.88",
74 "reference": "references/bioinformatics_genomics_formats.md",
75 },
76 ".fa": {
77 "format": "FASTA",
78 "tier": "automated_optional",
79 "depth": "bounded record sample with length, alphabet, and GC aggregates",
80 "dependency": "biopython==1.88",
81 "reference": "references/bioinformatics_genomics_formats.md",
82 },
83 ".fna": {
84 "format": "FASTA",
85 "tier": "automated_optional",
86 "depth": "bounded record sample with length, alphabet, and GC aggregates",
87 "dependency": "biopython==1.88",
88 "reference": "references/bioinformatics_genomics_formats.md",
89 },
90 ".fastq": {
91 "format": "FASTQ",
92 "tier": "automated_optional",
93 "depth": "bounded record sample with length, GC, and Phred+33 aggregates",
94 "dependency": "biopython==1.88",
95 "reference": "references/bioinformatics_genomics_formats.md",
96 },
97 ".fq": {
98 "format": "FASTQ",
99 "tier": "automated_optional",
100 "depth": "bounded record sample with length, GC, and Phred+33 aggregates",
101 "dependency": "biopython==1.88",
102 "reference": "references/bioinformatics_genomics_formats.md",
103 },
104 ".png": {
105 "format": "PNG",
106 "tier": "automated_optional",
107 "depth": "container metadata only; pixels are not decoded",
108 "dependency": "pillow==12.3.0",
109 "reference": "references/microscopy_imaging_formats.md",
110 },
111 ".jpg": {
112 "format": "JPEG",
113 "tier": "automated_optional",
114 "depth": "container metadata only; pixels are not decoded",
115 "dependency": "pillow==12.3.0",
116 "reference": "references/microscopy_imaging_formats.md",
117 },
118 ".jpeg": {
119 "format": "JPEG",
120 "tier": "automated_optional",
121 "depth": "container metadata only; pixels are not decoded",
122 "dependency": "pillow==12.3.0",
123 "reference": "references/microscopy_imaging_formats.md",
124 },
125 ".tif": {
126 "format": "TIFF",
127 "tier": "automated_optional",
128 "depth": "bounded TIFF series/page metadata only; pixels are not decoded",
129 "dependency": "tifffile==2026.9.20",
130 "reference": "references/microscopy_imaging_formats.md",
131 },
132 ".tiff": {
133 "format": "TIFF",
134 "tier": "automated_optional",
135 "depth": "bounded TIFF series/page metadata only; pixels are not decoded",
136 "dependency": "tifffile==2026.9.20",
137 "reference": "references/microscopy_imaging_formats.md",
138 },
139 ".ome.tif": {
140 "format": "OME-TIFF",
141 "tier": "automated_optional",
142 "depth": "bounded TIFF/OME structural metadata only; OME-XML and pixels are not emitted",
143 "dependency": "tifffile==2026.9.20",
144 "reference": "references/microscopy_imaging_formats.md",
145 },
146 ".ome.tiff": {
147 "format": "OME-TIFF",
148 "tier": "automated_optional",
149 "depth": "bounded TIFF/OME structural metadata only; OME-XML and pixels are not emitted",
150 "dependency": "tifffile==2026.9.20",
151 "reference": "references/microscopy_imaging_formats.md",
152 },
153}
156def _reference(
157 format_name: str,
158 reference: str,
159 note: str,
160) -> dict[str, str]:
161 return {
162 "format": format_name,
163 "tier": "reference_only",
164 "depth": note,
165 "dependency": "manual or separately validated domain tooling",
166 "reference": reference,
167 }
170REFERENCE_ONLY_FORMATS: dict[str, dict[str, str]] = {
171 # General tabular/array containers
172 ".parquet": _reference(
173 "Apache Parquet",
174 "references/general_scientific_formats.md",
175 "documented pandas/Polars/Arrow workflow; no bundled parser",
176 ),
177 ".feather": _reference(
178 "Apache Feather",
179 "references/general_scientific_formats.md",
180 "documented columnar workflow; no bundled parser",
181 ),
182 ".xlsx": _reference(
183 "Excel OOXML",
184 "references/general_scientific_formats.md",
185 "manual workbook review required, including formulas and hidden content",
186 ),
187 ".xls": _reference(
188 "Legacy Excel",
189 "references/general_scientific_formats.md",
190 "manual workbook review required; macros are never executed",
191 ),
192 ".zarr": _reference(
193 "Zarr",
194 "references/general_scientific_formats.md",
195 "directory/store format is not accepted by bundled file inspectors",
196 ),
197 ".nc": _reference(
198 "NetCDF",
199 "references/general_scientific_formats.md",
200 "documented xarray/netCDF workflow; no bundled parser",
201 ),
202 ".mat": _reference(
203 "MATLAB MAT",
204 "references/general_scientific_formats.md",
205 "manual version-aware workflow; no bundled parser",
206 ),
207 ".fits": _reference(
208 "FITS",
209 "references/general_scientific_formats.md",
210 "documented Astropy workflow; no bundled parser",
211 ),
212 # Bioinformatics/genomics
213 ".sam": _reference(
214 "SAM",
215 "references/bioinformatics_genomics_formats.md",
216 "use a pinned SAM/BAM validator; no bundled parser",
217 ),
218 ".bam": _reference(
219 "BAM",
220 "references/bioinformatics_genomics_formats.md",
221 "use a pinned SAM/BAM validator; no bundled parser",
222 ),
223 ".cram": _reference(
224 "CRAM",
225 "references/bioinformatics_genomics_formats.md",
226 "reference-aware validation required; no bundled parser",
227 ),
228 ".vcf": _reference(
229 "VCF",
230 "references/bioinformatics_genomics_formats.md",
231 "use a pinned VCF validator; no bundled parser",
232 ),
233 ".bcf": _reference(
234 "BCF",
235 "references/bioinformatics_genomics_formats.md",
236 "use a pinned VCF/BCF validator; no bundled parser",
237 ),
238 ".bed": _reference(
239 "BED",
240 "references/bioinformatics_genomics_formats.md",
241 "assembly-aware interval validation required; no bundled parser",
242 ),
243 ".gff": _reference(
244 "GFF",
245 "references/bioinformatics_genomics_formats.md",
246 "version-aware annotation validation required; no bundled parser",
247 ),
248 ".gff3": _reference(
249 "GFF3",
250 "references/bioinformatics_genomics_formats.md",
251 "version-aware annotation validation required; no bundled parser",
252 ),
253 ".gtf": _reference(
254 "GTF",
255 "references/bioinformatics_genomics_formats.md",
256 "annotation-specific validation required; no bundled parser",
257 ),
258 ".h5ad": _reference(
259 "AnnData H5AD",
260 "references/bioinformatics_genomics_formats.md",
261 "generic HDF5 metadata is not AnnData semantic validation",
262 ),
263 ".loom": _reference(
264 "Loom",
265 "references/bioinformatics_genomics_formats.md",
266 "generic HDF5 metadata is not Loom semantic validation",
267 ),
268 ".mtx": _reference(
269 "Matrix Market",
270 "references/bioinformatics_genomics_formats.md",
271 "matrix and sidecar alignment require domain tooling",
272 ),
273 # Chemistry/molecular
274 ".pdb": _reference(
275 "Legacy PDB",
276 "references/chemistry_molecular_formats.md",
277 "wwPDB-aware structural validation required; no bundled parser",
278 ),
279 ".cif": _reference(
280 "CIF or PDBx/mmCIF",
281 "references/chemistry_molecular_formats.md",
282 "extension is ambiguous; dictionary-aware validation required",
283 ),
284 ".mmcif": _reference(
285 "PDBx/mmCIF",
286 "references/chemistry_molecular_formats.md",
287 "wwPDB dictionary-aware validation required; no bundled parser",
288 ),
289 ".mol": _reference(
290 "MDL Molfile",
291 "references/chemistry_molecular_formats.md",
292 "chemistry-aware validation required; no bundled parser",
293 ),
294 ".sdf": _reference(
295 "Structure Data File",
296 "references/chemistry_molecular_formats.md",
297 "chemistry-aware validation required; no bundled parser",
298 ),
299 ".smi": _reference(
300 "SMILES table",
301 "references/chemistry_molecular_formats.md",
302 "line notation requires chemistry-aware parsing; no bundled parser",
303 ),
304 ".xyz": _reference(
305 "XYZ coordinates",
306 "references/chemistry_molecular_formats.md",
307 "units and record boundaries require explicit confirmation",
308 ),
309 ".dcd": _reference(
310 "DCD trajectory",
311 "references/chemistry_molecular_formats.md",
312 "topology-dependent trajectory tooling required",
313 ),
314 ".xtc": _reference(
315 "XTC trajectory",
316 "references/chemistry_molecular_formats.md",
317 "topology-dependent trajectory tooling required",
318 ),
319 ".trr": _reference(
320 "TRR trajectory",
321 "references/chemistry_molecular_formats.md",
322 "topology-dependent trajectory tooling required",
323 ),
324 # Imaging beyond the bounded metadata inspectors
325 ".nd2": _reference(
326 "Nikon ND2",
327 "references/microscopy_imaging_formats.md",
328 "vendor-aware reader required; no bundled parser",
329 ),
330 ".czi": _reference(
331 "Zeiss CZI",
332 "references/microscopy_imaging_formats.md",
333 "vendor-aware reader required; no bundled parser",
334 ),
335 ".lif": _reference(
336 "Leica LIF",
337 "references/microscopy_imaging_formats.md",
338 "vendor-aware reader required; no bundled parser",
339 ),
340 ".dcm": _reference(
341 "DICOM",
342 "references/microscopy_imaging_formats.md",
343 "PHI-aware DICOM tooling and policy required",
344 ),
345 ".nii": _reference(
346 "NIfTI",
347 "references/microscopy_imaging_formats.md",
348 "orientation-aware neuroimaging tooling required",
349 ),
350 ".nii.gz": _reference(
351 "Compressed NIfTI",
352 "references/microscopy_imaging_formats.md",
353 "compressed content is not decompressed by bundled tools",
354 ),
355 ".mrc": _reference(
356 "MRC",
357 "references/microscopy_imaging_formats.md",
358 "electron-microscopy-aware tooling required",
359 ),
360 ".svs": _reference(
361 "Aperio SVS",
362 "references/microscopy_imaging_formats.md",
363 "whole-slide reader and privacy review required",
364 ),
365 ".ndpi": _reference(
366 "Hamamatsu NDPI",
367 "references/microscopy_imaging_formats.md",
368 "whole-slide reader and privacy review required",
369 ),
370 # Spectroscopy/proteomics/metabolomics
371 ".mzml": _reference(
372 "HUPO-PSI mzML",
373 "references/spectroscopy_analytical_formats.md",
374 "schema/CV-aware mass-spectrometry tooling required",
375 ),
376 ".mzxml": _reference(
377 "mzXML",
378 "references/spectroscopy_analytical_formats.md",
379 "legacy MS tooling required; no bundled parser",
380 ),
381 ".jdx": _reference(
382 "JCAMP-DX",
383 "references/spectroscopy_analytical_formats.md",
384 "technique/version-aware JCAMP parser required",
385 ),
386 ".dx": _reference(
387 "JCAMP-DX",
388 "references/spectroscopy_analytical_formats.md",
389 "technique/version-aware JCAMP parser required",
390 ),
391 ".spc": _reference(
392 "SPC",
393 "references/spectroscopy_analytical_formats.md",
394 "vendor/variant-aware reader required",
395 ),
396 ".mgf": _reference(
397 "Mascot Generic Format",
398 "references/spectroscopy_analytical_formats.md",
399 "MS-specific parser required; no bundled parser",
400 ),
401 ".raw": _reference(
402 "Ambiguous vendor RAW",
403 "references/spectroscopy_analytical_formats.md",
404 "extension alone cannot identify the vendor format",
405 ),
406 ".mzid": _reference(
407 "mzIdentML",
408 "references/proteomics_metabolomics_formats.md",
409 "schema/CV-aware identification tooling required",
410 ),
411 ".mzidentml": _reference(
412 "mzIdentML",
413 "references/proteomics_metabolomics_formats.md",
414 "schema/CV-aware identification tooling required",
415 ),
416 ".pepxml": _reference(
417 "pepXML",
418 "references/proteomics_metabolomics_formats.md",
419 "search-engine-aware parser required",
420 ),
421 ".protxml": _reference(
422 "protXML",
423 "references/proteomics_metabolomics_formats.md",
424 "protein-inference-aware parser required",
425 ),
426 ".mztab": _reference(
427 "mzTab or mzTab-M",
428 "references/proteomics_metabolomics_formats.md",
429 "version-aware PSI validation required; generic TSV parsing is insufficient",
430 ),
431 ".featurexml": _reference(
432 "OpenMS featureXML",
433 "references/proteomics_metabolomics_formats.md",
434 "OpenMS-aware parser required",
435 ),
436}
439def suffix_key(path: Path) -> str:
440 """Return a registered compound or simple suffix, failing closed otherwise."""
442 name = path.name.casefold()
443 keys = sorted(
444 {*AUTOMATED_FORMATS, *REFERENCE_ONLY_FORMATS},
445 key=len,
446 reverse=True,
447 )
448 for key in keys:
449 if name.endswith(key):
450 return key
451 raise CliError("unknown format; no content sniffing or generic fallback is allowed")
454def capability_for_path(path: Path) -> dict[str, Any]:
455 """Return a copy of the closed capability entry for a path."""
457 key = suffix_key(path)
458 entry = AUTOMATED_FORMATS.get(key) or REFERENCE_ONLY_FORMATS[key]
459 return {"suffix": key, **entry}
462def _first_nonempty_text_byte(path: Path, *, limit: int = 8192) -> bytes | None:
463 with path.open("rb") as handle:
464 chunk = handle.read(limit)
465 for line in chunk.splitlines():
466 stripped = line.lstrip()
467 if stripped:
468 return stripped[:1]
469 return None
472def validate_magic(path: Path, suffix: str) -> None:
473 """Check unambiguous signatures without guessing an unsupported format."""
475 try:
476 with path.open("rb") as handle:
477 prefix = handle.read(16)
478 except OSError as exc:
479 raise CliError("the input header could not be read") from exc
481 if suffix == ".npy" and not prefix.startswith(b"\x93NUMPY"):
482 raise CliError("the NPY signature does not match the declared suffix")
483 if suffix == ".npz" and not prefix.startswith(b"PK"):
484 raise CliError("the NPZ ZIP signature does not match the declared suffix")
485 if suffix in {".h5", ".hdf5"}:
486 signature = b"\x89HDF\r\n\x1a\n"
487 found = False
488 with path.open("rb") as handle:
489 for offset in itertools.takewhile(
490 lambda value: value < max(path.stat().st_size, 1),
491 (0, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536),
492 ):
493 handle.seek(offset)
494 if handle.read(8) == signature:
495 found = True
496 break
497 if not found:
498 raise CliError("the HDF5 signature does not match the declared suffix")
499 if suffix in {".fasta", ".fa", ".fna"}:
500 if _first_nonempty_text_byte(path) != b">":
501 raise CliError("the FASTA record marker does not match the declared suffix")
502 if suffix in {".fastq", ".fq"}:
503 if _first_nonempty_text_byte(path) != b"@":
504 raise CliError("the FASTQ record marker does not match the declared suffix")
505 if suffix == ".png" and not prefix.startswith(b"\x89PNG\r\n\x1a\n"):
506 raise CliError("the PNG signature does not match the declared suffix")
507 if suffix in {".jpg", ".jpeg"} and not prefix.startswith(b"\xff\xd8\xff"):
508 raise CliError("the JPEG signature does not match the declared suffix")
509 if suffix in {".tif", ".tiff", ".ome.tif", ".ome.tiff"}:
510 valid = (
511 prefix.startswith(b"II*\x00")
512 or prefix.startswith(b"MM\x00*")
513 or prefix.startswith(b"II+\x00")
514 or prefix.startswith(b"MM\x00+")
515 )
516 if not valid:
517 raise CliError("the TIFF signature does not match the declared suffix")
520def preflight_npz(path: Path) -> dict[str, Any]:
521 """Reject encrypted, traversing, oversized, or high-ratio NPZ members."""
523 try:
524 with zipfile.ZipFile(path) as archive:
525 members = archive.infolist()
526 except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile) as exc:
527 raise CliError("the NPZ container is not a valid bounded ZIP archive") from exc
528 if not members or len(members) > MAX_NPZ_MEMBERS:
529 raise CliError(
530 f"the NPZ member count must be between 1 and {MAX_NPZ_MEMBERS}"
531 )
532 total_uncompressed = 0
533 highest_ratio = 0.0
534 member_names: set[str] = set()
535 for member in members:
536 member_path = Path(member.filename)
537 normalized_name = member.filename.replace("\\", "/")
538 normalized_parts = Path(normalized_name).parts
539 if (
540 member.is_dir()
541 or member_path.is_absolute()
542 or normalized_name.startswith("/")
543 or ":" in normalized_name
544 or ".." in normalized_parts
545 ):
546 raise CliError("the NPZ contains an invalid member path")
547 if normalized_name in member_names:
548 raise CliError("the NPZ contains duplicate member names")
549 member_names.add(normalized_name)
550 if member.flag_bits & 0x1:
551 raise CliError("encrypted NPZ members are not accepted")
552 if not member.filename.casefold().endswith(".npy"):
553 raise CliError("every NPZ member must be an NPY array")
554 total_uncompressed += member.file_size
555 denominator = max(member.compress_size, 1)
556 highest_ratio = max(highest_ratio, member.file_size / denominator)
557 if total_uncompressed > MAX_NPZ_UNCOMPRESSED_BYTES:
558 raise CliError(
559 "the NPZ declared uncompressed size exceeds the safety limit"
560 )
561 if highest_ratio > MAX_COMPRESSION_RATIO:
562 raise CliError("the NPZ compression ratio exceeds the safety limit")
563 return {
564 "member_count": len(members),
565 "declared_uncompressed_bytes": total_uncompressed,
566 "maximum_compression_ratio": round(highest_ratio, 3),
567 }
570def automated_capability_rows() -> list[dict[str, Any]]:
571 """Return deterministic rows for the public capability matrix."""
573 return [
574 {"suffix": suffix, **AUTOMATED_FORMATS[suffix]}
575 for suffix in sorted(AUTOMATED_FORMATS)
576 ]