Skills › Scientific Agent Skills › exploratory-data-analysis › _structured.py
_structured.py
392 lines · skills/exploratory-data-analysis/scripts/_structured.py at 5f2bc03
1#!/usr/bin/env python3
2"""Bounded inspectors for JSON, NumPy containers, and HDF5 metadata."""
4from __future__ import annotations
6import itertools
7import math
8from pathlib import Path
9from typing import Any
11from _capabilities import preflight_npz
12from _common import (
13 MAX_JSON_BYTES,
14 CliError,
15 display_identifier,
16 load_strict_json,
17)
18from _tabular import profile_json_structure
21MAX_ARRAY_SAMPLE = 4096
22MAX_HDF5_OBJECTS = 1000
23MAX_HDF5_DEPTH = 16
26def inspect_json(
27 path: Path,
28 *,
29 reveal_identifiers: bool = False,
30) -> dict[str, Any]:
31 """Strictly parse bounded JSON and emit structure, never scalar values."""
33 document = load_strict_json(path, max_bytes=MAX_JSON_BYTES)
34 return profile_json_structure(
35 document,
36 reveal_identifiers=reveal_identifiers,
37 )
40def _dtype_report(
41 dtype: Any,
42 *,
43 reveal_identifiers: bool,
44) -> dict[str, Any]:
45 fields = getattr(dtype, "fields", None)
46 report: dict[str, Any] = {
47 "kind": str(dtype.kind),
48 "itemsize": int(dtype.itemsize),
49 "byteorder": str(dtype.byteorder),
50 "contains_python_objects": bool(dtype.hasobject),
51 "structured": fields is not None,
52 }
53 if fields is not None:
54 names = list(dtype.names or ())
55 report["field_count"] = len(names)
56 report["field_ids"] = [
57 display_identifier(
58 name,
59 kind="field",
60 reveal_identifiers=reveal_identifiers,
61 )
62 for name in names[:128]
63 ]
64 report["field_ids_truncated"] = len(names) > 128
65 else:
66 report["dtype"] = str(dtype)
67 return report
70def _sample_array(array: Any, np: Any) -> Any:
71 size = int(array.size)
72 if size <= MAX_ARRAY_SAMPLE:
73 return np.asarray(array).reshape(-1)
74 positions = np.linspace(
75 0,
76 size - 1,
77 num=MAX_ARRAY_SAMPLE,
78 dtype=np.int64,
79 )
80 # ndarray.flat indexes in C order without materializing a full C-order
81 # copy of a Fortran-contiguous memory map.
82 return np.asarray(array.flat[positions])
85def _array_report(
86 array: Any,
87 np: Any,
88 *,
89 array_id: str | None,
90 reveal_identifiers: bool,
91) -> dict[str, Any]:
92 if bool(array.dtype.hasobject):
93 raise CliError("NumPy object arrays are not accepted because they require pickle")
94 report: dict[str, Any] = {
95 "shape": [int(value) for value in array.shape],
96 "dimension_count": int(array.ndim),
97 "element_count": int(array.size),
98 "dtype": _dtype_report(
99 array.dtype,
100 reveal_identifiers=reveal_identifiers,
101 ),
102 }
103 if array_id is not None:
104 report["array_id"] = array_id
105 if int(array.size) == 0:
106 report["numeric_sample"] = {"sample_count": 0}
107 return report
108 if np.issubdtype(array.dtype, np.bool_):
109 sample = _sample_array(array, np)
110 report["boolean_sample"] = {
111 "sample_count": int(sample.size),
112 "true_count": int(np.count_nonzero(sample)),
113 "sample_is_bounded": int(sample.size) < int(array.size),
114 }
115 return report
116 if np.issubdtype(array.dtype, np.number):
117 sample = _sample_array(array, np)
118 if np.iscomplexobj(sample):
119 finite = np.isfinite(sample.real) & np.isfinite(sample.imag)
120 magnitudes = np.abs(sample[finite])
121 report["numeric_sample"] = {
122 "sample_count": int(sample.size),
123 "finite_count": int(np.count_nonzero(finite)),
124 "complex_values_summarized_by_magnitude": True,
125 "magnitude_mean": (
126 float(np.mean(magnitudes)) if magnitudes.size else None
127 ),
128 "magnitude_minimum": (
129 float(np.min(magnitudes)) if magnitudes.size else None
130 ),
131 "magnitude_maximum": (
132 float(np.max(magnitudes)) if magnitudes.size else None
133 ),
134 "sample_is_bounded": int(sample.size) < int(array.size),
135 }
136 else:
137 finite = np.isfinite(sample)
138 finite_values = sample[finite]
139 report["numeric_sample"] = {
140 "sample_count": int(sample.size),
141 "finite_count": int(np.count_nonzero(finite)),
142 "nan_count": int(np.count_nonzero(np.isnan(sample)))
143 if np.issubdtype(sample.dtype, np.inexact)
144 else 0,
145 "infinite_count": int(np.count_nonzero(np.isinf(sample)))
146 if np.issubdtype(sample.dtype, np.inexact)
147 else 0,
148 "mean": (
149 float(np.mean(finite_values)) if finite_values.size else None
150 ),
151 "minimum": (
152 float(np.min(finite_values)) if finite_values.size else None
153 ),
154 "maximum": (
155 float(np.max(finite_values)) if finite_values.size else None
156 ),
157 "sample_is_bounded": int(sample.size) < int(array.size),
158 }
159 return report
162def inspect_numpy(
163 path: Path,
164 *,
165 suffix: str,
166 reveal_identifiers: bool = False,
167) -> dict[str, Any]:
168 """Inspect NPY/NPZ with pickle disabled and bounded decompression."""
170 try:
171 import numpy as np
172 except ImportError as exc:
173 raise CliError(
174 'optional dependency missing; install with: uv pip install "numpy==2.5.3"'
175 ) from exc
176 if suffix == ".npy":
177 try:
178 array = np.load(
179 path,
180 mmap_mode="r",
181 allow_pickle=False,
182 max_header_size=10_000,
183 )
184 except (OSError, ValueError, TypeError, MemoryError) as exc:
185 raise CliError(
186 "the NPY file could not be inspected safely; object arrays are rejected"
187 ) from exc
188 return {
189 "profile_type": "numpy_npy_bounded_profile",
190 "pickle_allowed": False,
191 "array": _array_report(
192 array,
193 np,
194 array_id=None,
195 reveal_identifiers=reveal_identifiers,
196 ),
197 "raw_values_emitted": False,
198 }
199 if suffix != ".npz":
200 raise CliError("the NumPy inspector received an unsupported suffix")
201 preflight = preflight_npz(path)
202 arrays: list[dict[str, Any]] = []
203 try:
204 with np.load(
205 path,
206 allow_pickle=False,
207 max_header_size=10_000,
208 ) as archive:
209 for name in archive.files:
210 array = archive[name]
211 arrays.append(
212 _array_report(
213 array,
214 np,
215 array_id=display_identifier(
216 name,
217 kind="array",
218 reveal_identifiers=reveal_identifiers,
219 ),
220 reveal_identifiers=reveal_identifiers,
221 )
222 )
223 del array
224 except (OSError, ValueError, TypeError, MemoryError) as exc:
225 raise CliError(
226 "the NPZ arrays could not be inspected safely; object arrays are rejected"
227 ) from exc
228 return {
229 "profile_type": "numpy_npz_bounded_profile",
230 "pickle_allowed": False,
231 "archive_preflight": preflight,
232 "arrays": arrays,
233 "raw_values_emitted": False,
234 }
237def _shape_elements(shape: Any) -> int | None:
238 if shape is None:
239 return None
240 total = 1
241 for value in shape:
242 total *= int(value)
243 if total > 2**63 - 1:
244 return None
245 return total
248def inspect_hdf5(
249 path: Path,
250 *,
251 reveal_identifiers: bool = False,
252) -> dict[str, Any]:
253 """Inspect HDF5 links and metadata without reading dataset values."""
255 try:
256 import h5py
257 except ImportError as exc:
258 raise CliError(
259 'optional dependency missing; install with: uv pip install "h5py==3.16.0"'
260 ) from exc
261 objects: list[dict[str, Any]] = []
262 link_counts = {"hard": 0, "soft_not_followed": 0, "external_not_followed": 0}
263 object_limit_reached = False
264 depth_limit_reached = False
265 seen_addresses: set[int] = set()
266 try:
267 with h5py.File(path, "r") as handle:
268 stack: list[tuple[Any, str, int]] = [(handle, "/", 0)]
269 while stack:
270 group, logical_path, depth = stack.pop()
271 if len(objects) >= MAX_HDF5_OBJECTS:
272 object_limit_reached = True
273 break
274 for name in itertools.islice(
275 group.keys(),
276 MAX_HDF5_OBJECTS - len(objects) + 1,
277 ):
278 if len(objects) >= MAX_HDF5_OBJECTS:
279 object_limit_reached = True
280 break
281 link = group.get(name, getlink=True)
282 child_path = (
283 f"/{name}" if logical_path == "/" else f"{logical_path}/{name}"
284 )
285 object_id = display_identifier(
286 child_path,
287 kind="hdf_object",
288 reveal_identifiers=reveal_identifiers,
289 )
290 if isinstance(link, h5py.ExternalLink):
291 link_counts["external_not_followed"] += 1
292 objects.append(
293 {
294 "object_id": object_id,
295 "type": "external_link",
296 "followed": False,
297 }
298 )
299 continue
300 if isinstance(link, h5py.SoftLink):
301 link_counts["soft_not_followed"] += 1
302 objects.append(
303 {
304 "object_id": object_id,
305 "type": "soft_link",
306 "followed": False,
307 }
308 )
309 continue
310 if not isinstance(link, h5py.HardLink):
311 raise CliError("the HDF5 file contains an unsupported link type")
312 link_counts["hard"] += 1
313 child = group.get(name)
314 if child is None:
315 raise CliError("the HDF5 file contains an unresolved hard link")
316 address = int(h5py.h5o.get_info(child.id).addr)
317 already_seen = address in seen_addresses
318 seen_addresses.add(address)
319 if isinstance(child, h5py.Dataset):
320 compression = child.compression
321 external_count = len(child.external or ())
322 objects.append(
323 {
324 "object_id": object_id,
325 "type": "dataset",
326 "hard_link_alias": already_seen,
327 "shape": [int(value) for value in child.shape]
328 if child.shape is not None
329 else None,
330 "element_count": _shape_elements(child.shape),
331 "dtype": _dtype_report(
332 child.dtype,
333 reveal_identifiers=reveal_identifiers,
334 ),
335 "attribute_count": len(child.attrs),
336 "chunked": child.chunks is not None,
337 "chunk_shape": [int(value) for value in child.chunks]
338 if child.chunks is not None
339 else None,
340 "compressed": compression is not None,
341 "compression_id": display_identifier(
342 str(compression),
343 kind="compression",
344 reveal_identifiers=reveal_identifiers,
345 )
346 if compression is not None
347 else None,
348 "external_storage_file_count": external_count,
349 "virtual_dataset": bool(child.is_virtual),
350 "values_read": False,
351 }
352 )
353 elif isinstance(child, h5py.Group):
354 objects.append(
355 {
356 "object_id": object_id,
357 "type": "group",
358 "hard_link_alias": already_seen,
359 "attribute_count": len(child.attrs),
360 }
361 )
362 if already_seen:
363 continue
364 if depth >= MAX_HDF5_DEPTH:
365 depth_limit_reached = True
366 else:
367 stack.append((child, child_path, depth + 1))
368 else:
369 raise CliError("the HDF5 file contains an unsupported object type")
370 except CliError:
371 raise
372 except (OSError, RuntimeError, ValueError, TypeError, MemoryError) as exc:
373 raise CliError("the HDF5 metadata could not be inspected safely") from exc
374 return {
375 "profile_type": "hdf5_metadata_only",
376 "object_count_reported": len(objects),
377 "object_limit_reached": object_limit_reached,
378 "depth_limit_reached": depth_limit_reached,
379 "links": link_counts,
380 "objects": objects,
381 "dataset_values_read": False,
382 "attributes_values_read": False,
383 "soft_links_followed": False,
384 "external_links_followed": False,
385 "external_dataset_storage_read": False,
386 "filter_plugins_invoked_for_data": False,
387 "raw_values_emitted": False,
388 "limitations": [
389 "Generic hierarchy inspection is not semantic validation of H5AD, Loom, or other HDF5 conventions.",
390 "Dataset payloads are intentionally not read, so value-level statistics are unavailable.",
391 ],
392 }