Skills › Scientific Agent Skills › pathml › plan_pipeline.py
plan_pipeline.py
276 lines · skills/pathml/scripts/plan_pipeline.py at 5f2bc03
1#!/usr/bin/env python3
2"""Plan bounded PathML tiling and pipeline work without opening a slide."""
4from __future__ import annotations
6import argparse
7import math
8from typing import Any
10from _common import CliError, emit_json, parse_name_list, run_cli
13TRANSFORM_KINDS = {
14 "AdaptiveHistogramEqualization": "image",
15 "BinaryThreshold": "mask",
16 "BoxBlur": "image",
17 "CollapseRunsCODEX": "image",
18 "CollapseRunsVectra": "image",
19 "ForegroundDetection": "mask",
20 "GaussianBlur": "image",
21 "HistogramEqualization": "image",
22 "LabelArtifactTileHE": "label",
23 "LabelWhiteSpaceHE": "label",
24 "MedianBlur": "image",
25 "MorphClose": "mask",
26 "MorphOpen": "mask",
27 "NucleusDetectionHE": "mask",
28 "QuantifyMIF": "counts",
29 "RescaleIntensity": "image",
30 "SegmentMIF": "two_instance_masks_deprecated",
31 "SegmentMIFRemote": "two_instance_masks_network_download",
32 "StainNormalizationHE": "image",
33 "SuperpixelInterpolation": "image",
34 "TissueDetectionHE": "mask",
35}
38def _stable_pathml_count(
39 dimension: int,
40 tile_extent: int,
41 stride: int,
42 pad: bool,
43) -> int:
44 """Match PathML 3.0.8 OpenSlide/Bio-Formats tile-count arithmetic."""
46 if pad and dimension % stride != 0:
47 return dimension // stride + 1
48 if dimension < tile_extent:
49 return 0
50 return (dimension - tile_extent) // stride + 1
53def _parse_pipeline(value: str | None) -> list[str]:
54 stages = parse_name_list(value, name="--pipeline")
55 unknown = sorted(set(stages) - set(TRANSFORM_KINDS))
56 if unknown:
57 raise CliError(
58 "unknown PathML 3.0.8 transform names: " + ", ".join(unknown)
59 )
60 return stages
63def make_plan(args: argparse.Namespace) -> dict[str, Any]:
64 if not 1 <= args.width <= 1_000_000_000:
65 raise CliError("--width must be between 1 and 1000000000")
66 if not 1 <= args.height <= 1_000_000_000:
67 raise CliError("--height must be between 1 and 1000000000")
68 if not 1 <= args.tile_size <= 8192:
69 raise CliError("--tile-size must be between 1 and 8192")
70 if not 1 <= args.stride <= 8192:
71 raise CliError("--stride must be between 1 and 8192")
72 if not math.isfinite(args.level_downsample) or not (
73 1.0 <= args.level_downsample <= 1_000_000.0
74 ):
75 raise CliError("--level-downsample must be finite and in [1, 1000000]")
76 if not 1 <= args.channels <= 4096:
77 raise CliError("--channels must be between 1 and 4096")
78 if args.bytes_per_channel not in {1, 2, 4, 8}:
79 raise CliError("--bytes-per-channel must be one of 1, 2, 4, or 8")
80 if not 1 <= args.max_tiles <= 100_000_000:
81 raise CliError("--max-tiles must be between 1 and 100000000")
82 if not math.isfinite(args.max_output_gib) or not (
83 0 < args.max_output_gib <= 1_000_000
84 ):
85 raise CliError("--max-output-gib must be finite and positive")
86 if args.mpp_x is not None and (
87 not math.isfinite(args.mpp_x) or not 0 < args.mpp_x <= 1000
88 ):
89 raise CliError("--mpp-x must be finite and in (0, 1000]")
90 if args.mpp_y is not None and (
91 not math.isfinite(args.mpp_y) or not 0 < args.mpp_y <= 1000
92 ):
93 raise CliError("--mpp-y must be finite and in (0, 1000]")
95 stages = _parse_pipeline(args.pipeline)
96 if (args.level_width is None) != (args.level_height is None):
97 raise CliError("provide both --level-width and --level-height")
98 measured_shape = args.level_width is not None
99 if measured_shape:
100 if not 1 <= args.level_width <= args.width or not 1 <= args.level_height <= args.height:
101 raise CliError("selected-level dimensions must be positive and no larger than level 0")
102 level_width, level_height = args.level_width, args.level_height
103 else:
104 level_width = math.ceil(args.width / args.level_downsample)
105 level_height = math.ceil(args.height / args.level_downsample)
106 tiles_i = _stable_pathml_count(
107 level_height, args.tile_size, args.stride, args.pad
108 )
109 tiles_j = _stable_pathml_count(
110 level_width, args.tile_size, args.stride, args.pad
111 )
112 tile_count = tiles_i * tiles_j
113 if tile_count > args.max_tiles:
114 raise CliError(
115 f"planned tile count {tile_count} exceeds --max-tiles={args.max_tiles}"
116 )
118 image_bytes_per_tile = (
119 args.tile_size
120 * args.tile_size
121 * args.channels
122 * args.bytes_per_channel
123 )
124 mask_outputs = sum(
125 2 if TRANSFORM_KINDS[stage].startswith("two_instance_masks") else 1
126 for stage in stages
127 if TRANSFORM_KINDS[stage] in {"mask", "two_instance_masks_deprecated",
128 "two_instance_masks_network_download"}
129 )
130 label_outputs = sum(
131 TRANSFORM_KINDS[stage] == "label" for stage in stages
132 )
133 count_outputs = sum(
134 TRANSFORM_KINDS[stage] == "counts" for stage in stages
135 )
136 # h5pathManager.add_tile casts both tile arrays and masks to float16.
137 # The input dtype can differ from the stored dtype.
138 storage_image_bytes_per_tile = args.tile_size * args.tile_size * args.channels * 2
139 mask_bytes_per_tile = args.tile_size * args.tile_size * mask_outputs * 2
140 estimated_payload_bytes = tile_count * (
141 storage_image_bytes_per_tile + mask_bytes_per_tile
142 )
143 estimated_payload_gib = estimated_payload_bytes / 1024**3
144 if estimated_payload_gib > args.max_output_gib:
145 raise CliError(
146 f"estimated uncompressed payload {estimated_payload_gib:.3f} GiB "
147 f"exceeds --max-output-gib={args.max_output_gib}"
148 )
150 overlap = max(0, args.tile_size - args.stride)
151 gap = max(0, args.stride - args.tile_size)
152 warnings: list[str] = []
153 if args.level_downsample != 1 and not measured_shape:
154 warnings.append(
155 "Selected-level shape is estimated from downsample; provide exact "
156 "--level-width and --level-height from the backend for tile counts."
157 )
158 if not args.level_downsample.is_integer():
159 warnings.append(
160 "PathML 3.0.8 OpenSlideBackend truncates the region-origin downsample "
161 "to int; use level 0 or an integral-downsample level for extraction."
162 )
163 if mask_outputs:
164 warnings.append(
165 "h5path casts masks to float16; instance IDs above 2048 can lose "
166 "identity. Keep authoritative integer masks separately. Mask-stage "
167 "counts may overestimate stored masks when stages reuse a name."
168 )
169 if count_outputs:
170 warnings.append("AnnData/count storage is not included in the payload estimate.")
171 if args.pad:
172 warnings.append(
173 "PathML 3.0.8 padding uses backend-specific count arithmetic and "
174 "zero-filled edges; verify a synthetic case."
175 )
176 if overlap:
177 warnings.append(
178 "Overlapping tiles duplicate pixels/objects; define stitching and "
179 "deduplication before counting."
180 )
181 if gap:
182 warnings.append("Stride exceeds tile size, leaving unprocessed gaps.")
183 if "SegmentMIF" in stages:
184 warnings.append("SegmentMIF is deprecated and needs a separate DeepCell stack.")
185 if "SegmentMIFRemote" in stages:
186 warnings.append(
187 "SegmentMIFRemote downloads Mesmer ONNX from Hugging Face at construction; "
188 "explicit network consent is required."
189 )
190 if args.level_downsample != 1:
191 warnings.append(
192 "Area thresholds and morphology kernels are in selected-level pixels."
193 )
195 physical_tile_um: list[float] | None = None
196 if args.mpp_x is not None and args.mpp_y is not None:
197 physical_tile_um = [
198 args.tile_size * args.level_downsample * args.mpp_y,
199 args.tile_size * args.level_downsample * args.mpp_x,
200 ]
202 return {
203 "pathml_version_modelled": "3.0.8",
204 "input_level0_shape_hw": [args.height, args.width],
205 "level_downsample": args.level_downsample,
206 "planned_level_shape_hw": [level_height, level_width],
207 "level_shape_source": "supplied_backend_dimensions" if measured_shape else "downsample_estimate",
208 "tile_size_hw": [args.tile_size, args.tile_size],
209 "stride_ij": [args.stride, args.stride],
210 "pad": args.pad,
211 "tiles_ij": [tiles_i, tiles_j],
212 "tile_count": tile_count,
213 "overlap_pixels_per_axis": overlap,
214 "gap_pixels_per_axis": gap,
215 "pipeline": [
216 {"name": stage, "kind": TRANSFORM_KINDS[stage]} for stage in stages
217 ],
218 "expected_mask_outputs_per_tile": mask_outputs,
219 "expected_label_outputs_per_tile": label_outputs,
220 "expected_count_outputs_per_tile": count_outputs,
221 "input_image_bytes_per_tile": image_bytes_per_tile,
222 "storage_image_bytes_per_tile": storage_image_bytes_per_tile,
223 "storage_mask_bytes_per_tile": mask_bytes_per_tile,
224 "storage_dtype": "float16",
225 "estimated_uncompressed_payload_gib": round(estimated_payload_gib, 6),
226 "physical_tile_size_um_yx": physical_tile_um,
227 "within_bounds": True,
228 "warnings": warnings,
229 "note": (
230 "This is a dry-run estimate. It does not open a slide, import PathML, "
231 "measure compression, or predict transform/model workspace memory. "
232 "Payload models h5path float16 images/masks at the supplied channel "
233 "count; counts, labels, metadata, and channel/shape changes are excluded."
234 ),
235 }
238def build_parser() -> argparse.ArgumentParser:
239 parser = argparse.ArgumentParser(
240 description=(
241 "Estimate PathML 3.0.8 tile counts and payload bounds without "
242 "opening a slide or importing PathML."
243 )
244 )
245 parser.add_argument("--width", type=int, required=True, help="level-0 width")
246 parser.add_argument("--height", type=int, required=True, help="level-0 height")
247 parser.add_argument("--level-downsample", type=float, default=1.0)
248 parser.add_argument("--level-width", type=int, help="exact selected-level backend width")
249 parser.add_argument("--level-height", type=int, help="exact selected-level backend height")
250 parser.add_argument("--tile-size", type=int, default=256)
251 parser.add_argument("--stride", type=int, default=256)
252 parser.add_argument("--pad", action="store_true")
253 parser.add_argument("--channels", type=int, default=3)
254 parser.add_argument("--bytes-per-channel", type=int, default=1)
255 parser.add_argument(
256 "--pipeline",
257 help="comma-separated stable transform class names in execution order",
258 )
259 parser.add_argument("--mpp-x", type=float)
260 parser.add_argument("--mpp-y", type=float)
261 parser.add_argument("--max-tiles", type=int, default=1_000_000)
262 parser.add_argument("--max-output-gib", type=float, default=1024.0)
263 parser.add_argument("--output")
264 parser.add_argument("--root", default=".")
265 parser.add_argument("--force", action="store_true")
266 return parser
269def main() -> None:
270 args = build_parser().parse_args()
271 report = make_plan(args)
272 emit_json(report, output=args.output, root=args.root, force=args.force)
275if __name__ == "__main__":
276 raise SystemExit(run_cli(main))