Skills › Scientific Agent Skills › fluidsim › output_inventory.py
output_inventory.py
353 lines · skills/fluidsim/scripts/output_inventory.py at 5f2bc03
1#!/usr/bin/env python3
2"""Inventory bounded FluidSim output and HDF5/netCDF4 metadata lazily."""
4from __future__ import annotations
6import argparse
7import os
8from pathlib import Path
9from typing import Any
11try:
12 from ._common import (
13 MAX_ATTRIBUTES,
14 MAX_DATASETS,
15 MAX_FILES,
16 MAX_HDF5_BYTES,
17 ToolError,
18 bounded_int,
19 checked_input,
20 checked_root,
21 emit_json,
22 fail_json,
23 iter_local_files,
24 relative_display,
25 )
26except ImportError: # Direct script execution.
27 from _common import (
28 MAX_ATTRIBUTES,
29 MAX_DATASETS,
30 MAX_FILES,
31 MAX_HDF5_BYTES,
32 ToolError,
33 bounded_int,
34 checked_input,
35 checked_root,
36 emit_json,
37 fail_json,
38 iter_local_files,
39 relative_display,
40 )
43TOOL = "fluidsim-output-inventory"
44HDF5_SUFFIXES = (".h5", ".hdf5", ".nc")
45SAFE_SCALAR_ATTRIBUTES = {
46 "Lx",
47 "Ly",
48 "Lz",
49 "class_name",
50 "fluidfft",
51 "fluidsim",
52 "it",
53 "module_name",
54 "name_run",
55 "nx",
56 "ny",
57 "nz",
58 "solver",
59 "time",
60 "version",
61}
64def _clean_name(value: str) -> str:
65 cleaned = "".join(
66 character if character.isprintable() else "?" for character in value
67 )
68 return cleaned[:300]
71def _scalar(value: Any) -> Any:
72 """Convert a known scalar attribute without retaining arbitrary arrays."""
74 if hasattr(value, "item"):
75 try:
76 value = value.item()
77 except (TypeError, ValueError):
78 return None
79 if isinstance(value, bytes):
80 return value.decode("utf-8", errors="replace")[:300]
81 if isinstance(value, str):
82 return value[:300]
83 if value is None or isinstance(value, (bool, int, float)):
84 return value
85 return None
88def _attributes(
89 obj: Any, *, count: list[int], max_attributes: int
90) -> list[dict[str, Any]]:
91 result: list[dict[str, Any]] = []
92 for name in sorted(obj.attrs):
93 count[0] += 1
94 if count[0] > max_attributes:
95 raise ToolError("HDF5 attribute-count limit exceeded")
96 identifier = obj.attrs.get_id(name)
97 entry: dict[str, Any] = {
98 "dtype": str(identifier.dtype)[:100],
99 "name": _clean_name(str(name)),
100 "shape": list(identifier.shape),
101 "value": None,
102 }
103 if name in SAFE_SCALAR_ATTRIBUTES and identifier.shape == ():
104 entry["value"] = _scalar(obj.attrs[name])
105 result.append(entry)
106 return result
109def _object_address(h5py: Any, obj: Any) -> int:
110 try:
111 return int(h5py.h5o.get_info(obj.id).addr)
112 except (AttributeError, TypeError, ValueError):
113 return hash(obj.id)
116def inspect_hdf5_metadata(
117 path: Path,
118 *,
119 max_datasets: int,
120 max_attributes: int,
121 max_depth: int,
122) -> dict[str, Any]:
123 """Read only object/link metadata; never index a dataset."""
125 try:
126 import h5py # Lazy optional dependency.
127 except ImportError as exc:
128 raise ToolError(
129 "HDF5 inspection requires optional h5py; install the pinned FluidSim environment"
130 ) from exc
132 datasets: list[dict[str, Any]] = []
133 groups: list[dict[str, Any]] = []
134 links = {"external_not_followed": 0, "soft_not_followed": 0}
135 attribute_count = [0]
136 try:
137 with h5py.File(path, "r") as handle:
138 stack: list[tuple[Any, str, int]] = [(handle, "/", 0)]
139 seen_groups: set[int] = set()
140 while stack:
141 group, group_path, depth = stack.pop()
142 address = _object_address(h5py, group)
143 if address in seen_groups:
144 continue
145 seen_groups.add(address)
146 groups.append(
147 {
148 "attributes": _attributes(
149 group,
150 count=attribute_count,
151 max_attributes=max_attributes,
152 ),
153 "path": _clean_name(group_path),
154 }
155 )
156 if depth >= max_depth:
157 continue
158 for name in sorted(group.keys(), reverse=True):
159 link = group.get(name, getlink=True)
160 child_path = (
161 f"/{name}" if group_path == "/" else f"{group_path}/{name}"
162 )
163 if isinstance(link, h5py.ExternalLink):
164 links["external_not_followed"] += 1
165 continue
166 if isinstance(link, h5py.SoftLink):
167 links["soft_not_followed"] += 1
168 continue
169 child = group.get(name, getlink=False)
170 if isinstance(child, h5py.Dataset):
171 if len(datasets) >= max_datasets:
172 raise ToolError("HDF5 dataset-count limit exceeded")
173 datasets.append(
174 {
175 "attributes": _attributes(
176 child,
177 count=attribute_count,
178 max_attributes=max_attributes,
179 ),
180 "chunks": (
181 list(child.chunks)
182 if child.chunks is not None
183 else None
184 ),
185 "compression": child.compression,
186 "dtype": str(child.dtype)[:100],
187 "path": _clean_name(child_path),
188 "shape": list(child.shape),
189 "storage_bytes": int(child.id.get_storage_size()),
190 }
191 )
192 elif isinstance(child, h5py.Group):
193 stack.append((child, child_path, depth + 1))
194 except OSError as exc:
195 return {
196 "attributes": 0,
197 "datasets": [],
198 "groups": [],
199 "hdf5_readable": False,
200 "links": links,
201 "message": str(exc)[:300],
202 }
203 return {
204 "attributes": attribute_count[0],
205 "datasets": datasets,
206 "groups": groups,
207 "hdf5_readable": True,
208 "links": links,
209 }
212def _kind(path: Path) -> str:
213 name = path.name.casefold()
214 if name.startswith("state_phys"):
215 return "physical_state"
216 if name.startswith(("spectra", "spect_energy", "spectra_multidim")):
217 return "spectral_output"
218 if name.startswith("spatial_means"):
219 return "scalar_output"
220 if name in {"params_simul.xml", "info_solver.xml", "stdout.txt"}:
221 return "provenance_or_log"
222 if name.endswith(HDF5_SUFFIXES):
223 return "hdf5_or_netcdf"
224 return "other"
227def inventory(
228 path: Path,
229 *,
230 root: Path,
231 max_files: int,
232 max_hdf5_files: int,
233 max_datasets: int,
234 max_attributes: int,
235 max_depth: int,
236) -> dict[str, Any]:
237 files = iter_local_files(
238 path, suffixes=None, max_files=max_files, recursive=True
239 )
240 entries: list[dict[str, Any]] = []
241 hdf5_entries: list[dict[str, Any]] = []
242 total_bytes = 0
243 for file_path in files:
244 size = file_path.stat().st_size
245 if size > MAX_HDF5_BYTES:
246 raise ToolError("a file exceeds the hard inspection-size bound")
247 total_bytes += size
248 kind = _kind(file_path)
249 entries.append(
250 {
251 "kind": kind,
252 "path": relative_display(file_path, root),
253 "size_bytes": size,
254 }
255 )
256 if (
257 file_path.name.casefold().endswith(HDF5_SUFFIXES)
258 and len(hdf5_entries) < max_hdf5_files
259 ):
260 hdf5_entries.append(
261 {
262 "metadata": inspect_hdf5_metadata(
263 file_path,
264 max_datasets=max_datasets,
265 max_attributes=max_attributes,
266 max_depth=max_depth,
267 ),
268 "path": relative_display(file_path, root),
269 "size_bytes": size,
270 }
271 )
272 counts: dict[str, int] = {}
273 for entry in entries:
274 counts[entry["kind"]] = counts.get(entry["kind"], 0) + 1
275 return {
276 "arrays_loaded": False,
277 "external_links_followed": False,
278 "file_count": len(entries),
279 "files": entries,
280 "hdf5_files_inspected": len(hdf5_entries),
281 "hdf5_metadata": hdf5_entries,
282 "kind_counts": dict(sorted(counts.items())),
283 "network_used": False,
284 "ok": True,
285 "root": ".",
286 "total_bytes": total_bytes,
287 "tool": TOOL,
288 }
291def build_parser() -> argparse.ArgumentParser:
292 parser = argparse.ArgumentParser(
293 description=(
294 "Inventory local FluidSim output. HDF5/netCDF4 datasets are described "
295 "by shape/dtype/chunks only; array values and external links are not read."
296 )
297 )
298 parser.add_argument("--path", required=True, help="Run directory or one file.")
299 parser.add_argument("--root", default=".", help="Local I/O boundary.")
300 parser.add_argument("--max-files", type=int, default=256)
301 parser.add_argument("--max-hdf5-files", type=int, default=32)
302 parser.add_argument("--max-datasets", type=int, default=2_000)
303 parser.add_argument("--max-attributes", type=int, default=5_000)
304 parser.add_argument("--max-depth", type=int, default=16)
305 return parser
308def main(argv: list[str] | None = None) -> int:
309 args = build_parser().parse_args(argv)
310 try:
311 max_files = bounded_int(
312 args.max_files, name="max_files", minimum=1, maximum=MAX_FILES
313 )
314 max_hdf5_files = bounded_int(
315 args.max_hdf5_files,
316 name="max_hdf5_files",
317 minimum=0,
318 maximum=MAX_FILES,
319 )
320 max_datasets = bounded_int(
321 args.max_datasets,
322 name="max_datasets",
323 minimum=1,
324 maximum=MAX_DATASETS,
325 )
326 max_attributes = bounded_int(
327 args.max_attributes,
328 name="max_attributes",
329 minimum=1,
330 maximum=MAX_ATTRIBUTES,
331 )
332 max_depth = bounded_int(
333 args.max_depth, name="max_depth", minimum=1, maximum=64
334 )
335 root = checked_root(args.root)
336 path = checked_input(args.path, root=root, kind="any")
337 report = inventory(
338 path,
339 root=root,
340 max_files=max_files,
341 max_hdf5_files=max_hdf5_files,
342 max_datasets=max_datasets,
343 max_attributes=max_attributes,
344 max_depth=max_depth,
345 )
346 emit_json(report)
347 return 0
348 except (OSError, ToolError, UnicodeError) as exc:
349 return fail_json(TOOL, exc)
352if __name__ == "__main__":
353 raise SystemExit(main())