Skills › Scientific Agent Skills › hypogenic › _common.py
_common.py
1312 lines · skills/hypogenic/scripts/_common.py at 5f2bc03
1"""Shared, dependency-light helpers for the HypoGeniC skill CLIs.
3The helpers perform bounded local file I/O only. They never import HypoGeniC,
4contact a network service, enumerate environment variables, load ``.env``
5files, or execute text from configs, datasets, hypotheses, or results.
6"""
8from __future__ import annotations
10import hashlib
11import json
12import math
13import os
14import re
15import stat
16import sys
17from collections import Counter
18from collections.abc import Iterable, Mapping, Sequence
19from datetime import date
20from pathlib import Path
21from typing import Any
24SCHEMA_VERSION = "1.0"
25HYPOGENIC_VERSION = "0.3.5"
26HYPOGENIC_COMMIT = "8c3800ccae155e333fac5b530afa8abdaac38300"
27HYPOGENIC_WHEEL_SHA256 = (
28 "f4ee8d7fa433cd59c58e0a8fe7df2f481ae29e7465a1b30ccbdac2c216a1b755"
29)
30PY_YAML_VERSION = "6.0.3"
32MAX_CONFIG_BYTES = 2 * 1024 * 1024
33MAX_JSON_BYTES = 128 * 1024 * 1024
34MAX_REPORT_BYTES = 8 * 1024 * 1024
35MAX_DOCUMENT_NODES = 100_000
36MAX_DOCUMENT_DEPTH = 32
37MAX_ROWS = 1_000_000
38MAX_HYPOTHESES = 10_000
39MAX_STRING_CHARS = 100_000
40MAX_CORRECT_EXAMPLES = 1_000_000
42SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
43REVISION_RE = re.compile(r"^[0-9a-f]{40,64}$")
44NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,127}$")
45TASK_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_./-]{0,127}$")
46ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]{1,127}$")
47DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
48SECRET_VALUE_RE = re.compile(
49 r"^(?:sk-(?:proj-)?|sk-ant-|sess-|api[_-]?key[:=])[A-Za-z0-9_-]{8,}$",
50 re.IGNORECASE,
51)
53PROVIDER_ENV = {
54 "gpt": "OPENAI_API_KEY",
55 "claude": "ANTHROPIC_API_KEY",
56 "huggingface": None,
57 "vllm": None,
58}
59PROVIDER_DESTINATION = {
60 "gpt": "openai_api",
61 "claude": "anthropic_api",
62 "huggingface": "local_process",
63 "vllm": "local_process",
64}
65REMOTE_PROVIDERS = frozenset({"gpt", "claude"})
66LOCAL_PROVIDERS = frozenset({"huggingface", "vllm"})
67SPLIT_NAMES = ("train", "validation", "test")
68MISSING_PREDICTION = "<missing>"
71class CliError(ValueError):
72 """A concise, user-correctable CLI validation error."""
75def _reject_constant(value: str) -> None:
76 raise CliError(f"non-standard JSON constant is forbidden: {value}")
79def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
80 result: dict[str, Any] = {}
81 for key, value in pairs:
82 if key in result:
83 raise CliError("duplicate object key is forbidden (key redacted)")
84 result[key] = value
85 return result
88def _reject_url_or_secret_path(value: str) -> None:
89 lowered = value.strip().lower()
90 if "\x00" in value:
91 raise CliError("paths must not contain a NUL byte")
92 if "://" in lowered or lowered.startswith(
93 ("http:", "https:", "ftp:", "s3:", "gs:", "file:")
94 ):
95 raise CliError("URLs are not accepted as local paths")
96 if any(part.lower().startswith(".env") for part in Path(value).parts):
97 raise CliError(".env files and directories are not accepted")
100def _absolute_lexical(path: Path) -> Path:
101 return Path(os.path.abspath(os.fspath(path)))
104def _reject_symlink_components(path: Path) -> None:
105 absolute = _absolute_lexical(path)
106 current = Path(absolute.anchor)
107 for part in absolute.parts[1:]:
108 current /= part
109 try:
110 if current.is_symlink():
111 raise CliError(f"symlink paths are forbidden: {current.name!r}")
112 except OSError as exc:
113 raise CliError(f"cannot inspect path component {current.name!r}: {exc}") from exc
116def _within_root(candidate: Path, root: Path) -> None:
117 try:
118 candidate.relative_to(root)
119 except ValueError as exc:
120 raise CliError("path escapes the declared root directory") from exc
123def checked_root(value: str | os.PathLike[str]) -> Path:
124 """Resolve an existing local directory used as the I/O boundary."""
126 raw = os.fspath(value)
127 _reject_url_or_secret_path(raw)
128 supplied = _absolute_lexical(Path(raw).expanduser())
129 try:
130 resolved = supplied.resolve(strict=True)
131 info = resolved.stat()
132 except OSError as exc:
133 raise CliError(f"cannot access root directory: {exc}") from exc
134 if not stat.S_ISDIR(info.st_mode):
135 raise CliError("root must be an existing directory")
136 _reject_symlink_components(resolved)
137 return resolved
140def checked_input_file(
141 value: str | os.PathLike[str],
142 *,
143 root: str | os.PathLike[str] = ".",
144 suffixes: Iterable[str],
145 max_bytes: int,
146) -> Path:
147 """Return a bounded regular file inside root, rejecting traversal/symlinks."""
149 raw = os.fspath(value)
150 _reject_url_or_secret_path(raw)
151 path_value = Path(raw).expanduser()
152 if ".." in path_value.parts:
153 raise CliError("parent-directory traversal is forbidden")
154 root_path = checked_root(root)
155 path = path_value if path_value.is_absolute() else root_path / path_value
156 path = _absolute_lexical(path)
157 _within_root(path, root_path)
158 _reject_symlink_components(path)
159 if path.is_symlink():
160 raise CliError(f"input must not be a symlink: {path.name!r}")
161 try:
162 resolved = path.resolve(strict=True)
163 info = resolved.stat()
164 except OSError as exc:
165 raise CliError(f"cannot access input file {path.name!r}: {exc}") from exc
166 _within_root(resolved, root_path)
167 if not stat.S_ISREG(info.st_mode):
168 raise CliError(f"input is not a regular file: {path.name!r}")
169 allowed = {suffix.lower() for suffix in suffixes}
170 if resolved.suffix.lower() not in allowed:
171 raise CliError(f"input suffix must be one of: {', '.join(sorted(allowed))}")
172 if info.st_size > max_bytes:
173 raise CliError(
174 f"input {path.name!r} is {info.st_size} bytes; limit is {max_bytes}"
175 )
176 return resolved
179def safe_relative_path(
180 value: Any,
181 *,
182 name: str,
183 suffixes: Iterable[str] | None = None,
184 allow_dot: bool = False,
185) -> str:
186 """Validate a lexical relative path without touching the filesystem."""
188 if not isinstance(value, str) or not value or len(value) > 1024:
189 raise CliError(f"{name} must be a nonempty relative path")
190 _reject_url_or_secret_path(value)
191 path = Path(value)
192 if path.is_absolute() or ".." in path.parts:
193 raise CliError(f"{name} must stay relative and contain no '..' segments")
194 normalized = path.as_posix()
195 if normalized in {"", "."} and not allow_dot:
196 raise CliError(f"{name} must name a path below the root")
197 if suffixes is not None:
198 allowed = {suffix.lower() for suffix in suffixes}
199 if path.suffix.lower() not in allowed:
200 raise CliError(f"{name} suffix must be one of: {', '.join(sorted(allowed))}")
201 return normalized
204def resolve_manifest_path(
205 value: str,
206 *,
207 data_root: Path,
208 suffixes: Iterable[str],
209 max_bytes: int,
210) -> Path:
211 """Resolve a manifest-owned file beneath an already checked data root."""
213 relative = safe_relative_path(value, name="manifest split path", suffixes=suffixes)
214 return checked_input_file(
215 relative,
216 root=data_root,
217 suffixes=suffixes,
218 max_bytes=max_bytes,
219 )
222def _validate_plain_document(value: Any) -> Any:
223 """Accept only JSON-compatible scalar/container types with global bounds."""
225 nodes = 0
227 def visit(item: Any, depth: int) -> Any:
228 nonlocal nodes
229 nodes += 1
230 if nodes > MAX_DOCUMENT_NODES:
231 raise CliError(f"document exceeds {MAX_DOCUMENT_NODES} nodes")
232 if depth > MAX_DOCUMENT_DEPTH:
233 raise CliError(f"document exceeds depth {MAX_DOCUMENT_DEPTH}")
234 if item is None or isinstance(item, (str, bool)):
235 if isinstance(item, str) and len(item) > MAX_STRING_CHARS:
236 raise CliError(f"string exceeds {MAX_STRING_CHARS} characters")
237 return item
238 if isinstance(item, int):
239 return item
240 if isinstance(item, float):
241 if not math.isfinite(item):
242 raise CliError("non-finite numbers are forbidden")
243 return item
244 if isinstance(item, list):
245 return [visit(child, depth + 1) for child in item]
246 if isinstance(item, dict):
247 result: dict[str, Any] = {}
248 for key, child in item.items():
249 if not isinstance(key, str):
250 raise CliError("object keys must be strings")
251 if key in result:
252 raise CliError("duplicate object key is forbidden (key redacted)")
253 result[key] = visit(child, depth + 1)
254 return result
255 raise CliError(f"unsupported scalar type: {type(item).__name__}")
257 return visit(value, 0)
260def _load_yaml(text: str) -> Any:
261 """Parse YAML with one reviewed PyYAML version and a restricted loader."""
263 try:
264 import yaml
265 except ImportError as exc:
266 raise CliError(
267 f"YAML requires PyYAML=={PY_YAML_VERSION}; JSON needs no dependency"
268 ) from exc
269 if getattr(yaml, "__version__", None) != PY_YAML_VERSION:
270 raise CliError(
271 f"YAML parser must be exactly PyYAML=={PY_YAML_VERSION}; "
272 f"found {getattr(yaml, '__version__', 'unknown')}"
273 )
275 forbidden_tokens = (
276 yaml.tokens.AliasToken,
277 yaml.tokens.AnchorToken,
278 yaml.tokens.TagToken,
279 )
280 try:
281 for token in yaml.scan(text):
282 if isinstance(token, forbidden_tokens):
283 raise CliError("YAML aliases, anchors, and explicit tags are forbidden")
284 except yaml.YAMLError as exc:
285 raise CliError("invalid YAML token stream (source content redacted)") from exc
287 class UniqueSafeLoader(yaml.SafeLoader):
288 pass
290 def construct_mapping(loader: Any, node: Any, deep: bool = False) -> dict[str, Any]:
291 loader.flatten_mapping(node)
292 mapping: dict[str, Any] = {}
293 for key_node, value_node in node.value:
294 key = loader.construct_object(key_node, deep=deep)
295 if not isinstance(key, str):
296 raise CliError("YAML mapping keys must be strings")
297 if key in mapping:
298 raise CliError("duplicate YAML key is forbidden (key redacted)")
299 mapping[key] = loader.construct_object(value_node, deep=deep)
300 return mapping
302 UniqueSafeLoader.add_constructor(
303 yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
304 construct_mapping,
305 )
306 try:
307 return yaml.load(text, Loader=UniqueSafeLoader)
308 except CliError:
309 raise
310 except yaml.YAMLError as exc:
311 raise CliError("invalid safe YAML (source content redacted)") from exc
314def load_structured_document(
315 value: str | os.PathLike[str],
316 *,
317 root: str | os.PathLike[str] = ".",
318 max_bytes: int = MAX_CONFIG_BYTES,
319) -> Any:
320 """Load strict JSON or restricted YAML from a bounded local file."""
322 path = checked_input_file(
323 value,
324 root=root,
325 suffixes={".json", ".yaml", ".yml"},
326 max_bytes=max_bytes,
327 )
328 try:
329 text = path.read_text(encoding="utf-8")
330 except (OSError, UnicodeError) as exc:
331 raise CliError(f"cannot read UTF-8 input {path.name!r}: {exc}") from exc
332 try:
333 if path.suffix.lower() == ".json":
334 document = json.loads(
335 text,
336 object_pairs_hook=_unique_object,
337 parse_constant=_reject_constant,
338 )
339 else:
340 document = _load_yaml(text)
341 except CliError:
342 raise
343 except json.JSONDecodeError as exc:
344 raise CliError(
345 f"invalid JSON at line {exc.lineno}, column {exc.colno}"
346 ) from exc
347 return _validate_plain_document(document)
350def load_json_document(
351 value: str | os.PathLike[str],
352 *,
353 root: str | os.PathLike[str] = ".",
354 max_bytes: int = MAX_JSON_BYTES,
355) -> Any:
356 path = checked_input_file(
357 value,
358 root=root,
359 suffixes={".json"},
360 max_bytes=max_bytes,
361 )
362 try:
363 with path.open("r", encoding="utf-8") as handle:
364 document = json.load(
365 handle,
366 object_pairs_hook=_unique_object,
367 parse_constant=_reject_constant,
368 )
369 except CliError:
370 raise
371 except (OSError, UnicodeError, json.JSONDecodeError) as exc:
372 raise CliError(f"cannot read strict JSON from {path.name!r}: {exc}") from exc
373 return _validate_plain_document(document)
376def validate_keys(
377 value: Any,
378 *,
379 required: Iterable[str],
380 optional: Iterable[str] = (),
381 context: str,
382) -> dict[str, Any]:
383 if not isinstance(value, dict):
384 raise CliError(f"{context} must be an object")
385 required_set = set(required)
386 allowed = required_set | set(optional)
387 missing = sorted(required_set - set(value))
388 unknown = sorted(set(value) - allowed)
389 if missing:
390 raise CliError(f"{context} is missing keys: {', '.join(missing)}")
391 if unknown:
392 raise CliError(f"{context} has unknown keys: {', '.join(unknown)}")
393 return value
396def bounded_int(value: Any, *, name: str, minimum: int, maximum: int) -> int:
397 if isinstance(value, bool) or not isinstance(value, int):
398 raise CliError(f"{name} must be an integer")
399 if value < minimum or value > maximum:
400 raise CliError(f"{name} must be between {minimum} and {maximum}")
401 return value
404def finite_number(
405 value: Any,
406 *,
407 name: str,
408 minimum: float = 0.0,
409 maximum: float = 1_000_000_000.0,
410) -> float:
411 if isinstance(value, bool) or not isinstance(value, (int, float)):
412 raise CliError(f"{name} must be numeric")
413 number = float(value)
414 if not math.isfinite(number) or not minimum <= number <= maximum:
415 raise CliError(f"{name} must be finite and between {minimum} and {maximum}")
416 return number
419def nonempty_text(value: Any, *, name: str, maximum: int = 1024) -> str:
420 if (
421 not isinstance(value, str)
422 or not value.strip()
423 or len(value) > maximum
424 or "\x00" in value
425 ):
426 raise CliError(f"{name} must be a nonempty string of at most {maximum} characters")
427 if SECRET_VALUE_RE.match(value.strip()):
428 raise CliError(f"{name} appears to contain a secret value")
429 return value
432def strict_bool(value: Any, *, name: str, expected: bool | None = None) -> bool:
433 if not isinstance(value, bool):
434 raise CliError(f"{name} must be true or false")
435 if expected is not None and value is not expected:
436 raise CliError(f"{name} must be {str(expected).lower()}")
437 return value
440def normalized_sha256(value: Any, *, name: str) -> str:
441 if not isinstance(value, str) or SHA256_RE.fullmatch(value.lower()) is None:
442 raise CliError(f"{name} must be a 64-character SHA-256 hex digest")
443 return value.lower()
446def normalized_date(value: Any, *, name: str) -> str:
447 if not isinstance(value, str) or DATE_RE.fullmatch(value) is None:
448 raise CliError(f"{name} must use YYYY-MM-DD")
449 try:
450 date.fromisoformat(value)
451 except ValueError as exc:
452 raise CliError(f"{name} is not a calendar date") from exc
453 return value
456def _validate_no_secret_fields(value: Any, *, context: str = "config") -> None:
457 if isinstance(value, dict):
458 for key, child in value.items():
459 lowered = key.lower()
460 if lowered in {
461 "api_key",
462 "api_key_value",
463 "secret",
464 "token",
465 "password",
466 "credentials",
467 }:
468 raise CliError(
469 f"{context} contains forbidden secret-bearing field {key!r}; "
470 "use credential_env"
471 )
472 _validate_no_secret_fields(child, context=f"{context}.{key}")
473 elif isinstance(value, list):
474 for index, child in enumerate(value):
475 _validate_no_secret_fields(child, context=f"{context}[{index}]")
476 elif isinstance(value, str) and SECRET_VALUE_RE.match(value.strip()):
477 raise CliError(f"{context} appears to contain a secret value")
480def validate_run_config(document: Any) -> dict[str, Any]:
481 """Validate the skill-local, plan-only run policy schema."""
483 top = validate_keys(
484 document,
485 required=(
486 "schema_version",
487 "data",
488 "provider",
489 "limits",
490 "pricing",
491 "execution",
492 "logging",
493 ),
494 context="run config",
495 )
496 if top["schema_version"] != SCHEMA_VERSION:
497 raise CliError(f"schema_version must be {SCHEMA_VERSION!r}")
498 _validate_no_secret_fields(top)
500 raw_data = validate_keys(
501 top["data"],
502 required=(
503 "task_config",
504 "dataset_manifest",
505 "output_directory",
506 "test_split_policy",
507 ),
508 context="data",
509 )
510 if raw_data["test_split_policy"] != "locked_until_final":
511 raise CliError("data.test_split_policy must be 'locked_until_final'")
512 data = {
513 "task_config": safe_relative_path(
514 raw_data["task_config"],
515 name="data.task_config",
516 suffixes={".json", ".yaml", ".yml"},
517 ),
518 "dataset_manifest": safe_relative_path(
519 raw_data["dataset_manifest"],
520 name="data.dataset_manifest",
521 suffixes={".json"},
522 ),
523 "output_directory": safe_relative_path(
524 raw_data["output_directory"],
525 name="data.output_directory",
526 ),
527 "test_split_policy": "locked_until_final",
528 }
530 raw_provider = validate_keys(
531 top["provider"],
532 required=(
533 "type",
534 "model",
535 "credential_env",
536 "data_destination",
537 "local_model_path",
538 ),
539 context="provider",
540 )
541 provider_type = raw_provider["type"]
542 if provider_type not in PROVIDER_ENV:
543 raise CliError(f"provider.type must be one of {sorted(PROVIDER_ENV)}")
544 model = nonempty_text(raw_provider["model"], name="provider.model", maximum=256)
545 expected_env = PROVIDER_ENV[provider_type]
546 credential_env = raw_provider["credential_env"]
547 if credential_env != expected_env:
548 raise CliError(
549 f"provider.credential_env for {provider_type!r} must be {expected_env!r}"
550 )
551 if isinstance(credential_env, str) and ENV_NAME_RE.fullmatch(credential_env) is None:
552 raise CliError("provider.credential_env is not a valid named environment variable")
553 destination = raw_provider["data_destination"]
554 if destination != PROVIDER_DESTINATION[provider_type]:
555 raise CliError(
556 f"provider.data_destination for {provider_type!r} must be "
557 f"{PROVIDER_DESTINATION[provider_type]!r}"
558 )
559 local_model_path = raw_provider["local_model_path"]
560 if provider_type in LOCAL_PROVIDERS:
561 local_model_path = safe_relative_path(
562 local_model_path,
563 name="provider.local_model_path",
564 )
565 elif local_model_path is not None:
566 raise CliError("provider.local_model_path must be null for hosted providers")
567 provider = {
568 "type": provider_type,
569 "model": model,
570 "credential_env": credential_env,
571 "data_destination": destination,
572 "local_model_path": local_model_path,
573 }
575 raw_limits = validate_keys(
576 top["limits"],
577 required=(
578 "max_requests",
579 "max_input_tokens_per_request",
580 "max_output_tokens_per_request",
581 "max_total_tokens",
582 "max_cost_usd",
583 "max_concurrent",
584 "train_examples",
585 "validation_examples",
586 "test_examples",
587 "max_hypotheses",
588 ),
589 context="limits",
590 )
591 limits = {
592 "max_requests": bounded_int(
593 raw_limits["max_requests"], name="limits.max_requests", minimum=1, maximum=1_000_000
594 ),
595 "max_input_tokens_per_request": bounded_int(
596 raw_limits["max_input_tokens_per_request"],
597 name="limits.max_input_tokens_per_request",
598 minimum=1,
599 maximum=10_000_000,
600 ),
601 "max_output_tokens_per_request": bounded_int(
602 raw_limits["max_output_tokens_per_request"],
603 name="limits.max_output_tokens_per_request",
604 minimum=1,
605 maximum=10_000_000,
606 ),
607 "max_total_tokens": bounded_int(
608 raw_limits["max_total_tokens"],
609 name="limits.max_total_tokens",
610 minimum=1,
611 maximum=10_000_000_000,
612 ),
613 "max_cost_usd": finite_number(
614 raw_limits["max_cost_usd"],
615 name="limits.max_cost_usd",
616 minimum=0.01,
617 maximum=1_000_000.0,
618 ),
619 "max_concurrent": bounded_int(
620 raw_limits["max_concurrent"],
621 name="limits.max_concurrent",
622 minimum=1,
623 maximum=128,
624 ),
625 "train_examples": bounded_int(
626 raw_limits["train_examples"],
627 name="limits.train_examples",
628 minimum=1,
629 maximum=MAX_ROWS,
630 ),
631 "validation_examples": bounded_int(
632 raw_limits["validation_examples"],
633 name="limits.validation_examples",
634 minimum=1,
635 maximum=MAX_ROWS,
636 ),
637 "test_examples": bounded_int(
638 raw_limits["test_examples"],
639 name="limits.test_examples",
640 minimum=1,
641 maximum=MAX_ROWS,
642 ),
643 "max_hypotheses": bounded_int(
644 raw_limits["max_hypotheses"],
645 name="limits.max_hypotheses",
646 minimum=1,
647 maximum=MAX_HYPOTHESES,
648 ),
649 }
651 raw_pricing = validate_keys(
652 top["pricing"],
653 required=(
654 "input_usd_per_million_tokens",
655 "output_usd_per_million_tokens",
656 "reviewed_on",
657 "source",
658 ),
659 context="pricing",
660 )
661 input_rate = raw_pricing["input_usd_per_million_tokens"]
662 output_rate = raw_pricing["output_usd_per_million_tokens"]
663 if (input_rate is None) != (output_rate is None):
664 raise CliError("both pricing token rates must be null or both must be numeric")
665 if input_rate is None:
666 if raw_pricing["reviewed_on"] is not None or raw_pricing["source"] is not None:
667 raise CliError("pricing date/source must be null when rates are null")
668 pricing = {
669 "input_usd_per_million_tokens": None,
670 "output_usd_per_million_tokens": None,
671 "reviewed_on": None,
672 "source": None,
673 }
674 else:
675 pricing = {
676 "input_usd_per_million_tokens": finite_number(
677 input_rate,
678 name="pricing.input_usd_per_million_tokens",
679 minimum=0.0,
680 maximum=100_000.0,
681 ),
682 "output_usd_per_million_tokens": finite_number(
683 output_rate,
684 name="pricing.output_usd_per_million_tokens",
685 minimum=0.0,
686 maximum=100_000.0,
687 ),
688 "reviewed_on": normalized_date(
689 raw_pricing["reviewed_on"], name="pricing.reviewed_on"
690 ),
691 "source": nonempty_text(
692 raw_pricing["source"], name="pricing.source", maximum=2048
693 ),
694 }
696 raw_execution = validate_keys(
697 top["execution"],
698 required=(
699 "mode",
700 "external_calls_authorized",
701 "require_separate_confirmation",
702 "send_test_split",
703 ),
704 context="execution",
705 )
706 if raw_execution["mode"] != "plan_only":
707 raise CliError("execution.mode must be 'plan_only'")
708 execution = {
709 "mode": "plan_only",
710 "external_calls_authorized": strict_bool(
711 raw_execution["external_calls_authorized"],
712 name="execution.external_calls_authorized",
713 expected=False,
714 ),
715 "require_separate_confirmation": strict_bool(
716 raw_execution["require_separate_confirmation"],
717 name="execution.require_separate_confirmation",
718 expected=True,
719 ),
720 "send_test_split": strict_bool(
721 raw_execution["send_test_split"],
722 name="execution.send_test_split",
723 expected=False,
724 ),
725 }
727 raw_logging = validate_keys(
728 top["logging"],
729 required=(
730 "level",
731 "redact_prompts",
732 "redact_responses",
733 "include_credentials",
734 ),
735 context="logging",
736 )
737 if raw_logging["level"] not in {"INFO", "WARNING", "ERROR", "CRITICAL"}:
738 raise CliError("logging.level must be INFO, WARNING, ERROR, or CRITICAL")
739 logging = {
740 "level": raw_logging["level"],
741 "redact_prompts": strict_bool(
742 raw_logging["redact_prompts"],
743 name="logging.redact_prompts",
744 expected=True,
745 ),
746 "redact_responses": strict_bool(
747 raw_logging["redact_responses"],
748 name="logging.redact_responses",
749 expected=True,
750 ),
751 "include_credentials": strict_bool(
752 raw_logging["include_credentials"],
753 name="logging.include_credentials",
754 expected=False,
755 ),
756 }
757 return {
758 "schema_version": SCHEMA_VERSION,
759 "data": data,
760 "provider": provider,
761 "limits": limits,
762 "pricing": pricing,
763 "execution": execution,
764 "logging": logging,
765 }
768def named_env_presence(config: Mapping[str, Any]) -> dict[str, Any]:
769 """Check only the validated provider-specific environment variable name."""
771 name = config["provider"]["credential_env"]
772 if name is None:
773 return {"required": False, "name": None, "present": None, "value_included": False}
774 if name not in {value for value in PROVIDER_ENV.values() if value is not None}:
775 raise CliError("credential environment name is not allowlisted")
776 return {
777 "required": True,
778 "name": name,
779 "present": bool(os.getenv(name)),
780 "value_included": False,
781 }
784def _validate_prompt_value(value: Any, *, context: str, depth: int = 0) -> None:
785 if depth > 16:
786 raise CliError(f"{context} exceeds prompt-template depth 16")
787 if isinstance(value, str):
788 if len(value) > MAX_STRING_CHARS:
789 raise CliError(f"{context} exceeds {MAX_STRING_CHARS} characters")
790 return
791 if isinstance(value, list):
792 if len(value) > 256:
793 raise CliError(f"{context} has more than 256 list entries")
794 for index, child in enumerate(value):
795 _validate_prompt_value(child, context=f"{context}[{index}]", depth=depth + 1)
796 return
797 if isinstance(value, dict):
798 if len(value) > 256:
799 raise CliError(f"{context} has more than 256 object entries")
800 for key, child in value.items():
801 nonempty_text(key, name=f"{context} key", maximum=128)
802 _validate_prompt_value(child, context=f"{context}.{key}", depth=depth + 1)
803 return
804 raise CliError(f"{context} must contain only strings, lists, and objects")
807def validate_task_config(document: Any) -> dict[str, Any]:
808 """Validate the pinned source's task-config surface without interpreting prompts."""
810 top = validate_keys(
811 document,
812 required=(
813 "task_name",
814 "train_data_path",
815 "val_data_path",
816 "test_data_path",
817 "prompt_templates",
818 ),
819 optional=("label_name", "ood_data_path"),
820 context="task config",
821 )
822 task_name = nonempty_text(top["task_name"], name="task_name", maximum=128)
823 if TASK_NAME_RE.fullmatch(task_name) is None:
824 raise CliError(f"task_name must match {TASK_NAME_RE.pattern}")
825 label_name = top.get("label_name", "label")
826 if not isinstance(label_name, str) or NAME_RE.fullmatch(label_name) is None:
827 raise CliError(f"label_name must match {NAME_RE.pattern}")
828 paths = {
829 key: safe_relative_path(
830 top[key],
831 name=key,
832 suffixes={".json"},
833 )
834 for key in ("train_data_path", "val_data_path", "test_data_path")
835 }
836 if len(set(paths.values())) != 3:
837 raise CliError("train, validation, and test paths must be distinct")
838 ood_path = None
839 if "ood_data_path" in top:
840 ood_path = safe_relative_path(
841 top["ood_data_path"],
842 name="ood_data_path",
843 suffixes={".json"},
844 )
845 if ood_path in set(paths.values()):
846 raise CliError("ood_data_path must be distinct from train/validation/test")
848 templates = top["prompt_templates"]
849 if not isinstance(templates, dict) or not templates or len(templates) > 256:
850 raise CliError("prompt_templates must be a nonempty object with at most 256 keys")
851 required_templates = {"observations", "batched_generation", "inference"}
852 missing_templates = sorted(required_templates - set(templates))
853 if missing_templates:
854 raise CliError(
855 "prompt_templates is missing basic keys: " + ", ".join(missing_templates)
856 )
857 _validate_prompt_value(templates, context="prompt_templates")
858 return {
859 "schema_version": SCHEMA_VERSION,
860 "task_name": task_name,
861 "label_name": label_name,
862 **paths,
863 "ood_data_path": ood_path,
864 "prompt_template_keys": sorted(templates),
865 "prompt_text_interpreted": False,
866 }
869def validate_dataset_manifest(document: Any) -> dict[str, Any]:
870 top = validate_keys(
871 document,
872 required=(
873 "schema_version",
874 "source",
875 "root",
876 "label_field",
877 "identity_fields",
878 "splits",
879 ),
880 context="dataset manifest",
881 )
882 if top["schema_version"] != SCHEMA_VERSION:
883 raise CliError(f"schema_version must be {SCHEMA_VERSION!r}")
884 source_obj = validate_keys(
885 top["source"],
886 required=("repository", "revision", "retrieved_on"),
887 context="source",
888 )
889 repository = nonempty_text(
890 source_obj["repository"], name="source.repository", maximum=2048
891 )
892 if not repository.startswith(
893 ("https://github.com/", "https://huggingface.co/")
894 ):
895 raise CliError("source.repository must be an HTTPS GitHub or Hugging Face URL")
896 revision = source_obj["revision"]
897 if not isinstance(revision, str) or REVISION_RE.fullmatch(revision.lower()) is None:
898 raise CliError("source.revision must be an immutable 40-64 character hex revision")
899 root = safe_relative_path(
900 top["root"],
901 name="manifest root",
902 allow_dot=True,
903 )
904 label_field = top["label_field"]
905 if not isinstance(label_field, str) or NAME_RE.fullmatch(label_field) is None:
906 raise CliError(f"label_field must match {NAME_RE.pattern}")
907 identity_fields = top["identity_fields"]
908 if (
909 not isinstance(identity_fields, list)
910 or not identity_fields
911 or len(identity_fields) > 128
912 ):
913 raise CliError("identity_fields must contain 1 to 128 field names")
914 normalized_fields: list[str] = []
915 for index, field in enumerate(identity_fields):
916 if not isinstance(field, str) or NAME_RE.fullmatch(field) is None:
917 raise CliError(f"identity_fields[{index}] must match {NAME_RE.pattern}")
918 normalized_fields.append(field)
919 if len(set(normalized_fields)) != len(normalized_fields):
920 raise CliError("identity_fields must not contain duplicates")
921 if label_field in normalized_fields:
922 raise CliError("label_field must not be included in identity_fields")
924 splits = top["splits"]
925 if not isinstance(splits, list) or len(splits) != 3:
926 raise CliError("splits must contain exactly train, validation, and test")
927 normalized_splits: list[dict[str, str]] = []
928 names: list[str] = []
929 paths: list[str] = []
930 for index, raw_split in enumerate(splits):
931 split = validate_keys(
932 raw_split,
933 required=("name", "path", "sha256"),
934 context=f"splits[{index}]",
935 )
936 name = split["name"]
937 if name not in SPLIT_NAMES:
938 raise CliError(f"splits[{index}].name must be one of {SPLIT_NAMES}")
939 path = safe_relative_path(
940 split["path"],
941 name=f"splits[{index}].path",
942 suffixes={".json"},
943 )
944 names.append(name)
945 paths.append(path)
946 normalized_splits.append(
947 {
948 "name": name,
949 "path": path,
950 "sha256": normalized_sha256(
951 split["sha256"], name=f"splits[{index}].sha256"
952 ),
953 }
954 )
955 if set(names) != set(SPLIT_NAMES) or len(set(names)) != 3:
956 raise CliError("splits must name train, validation, and test exactly once")
957 if len(set(paths)) != 3:
958 raise CliError("split paths must be distinct")
959 normalized_splits.sort(key=lambda item: SPLIT_NAMES.index(item["name"]))
960 return {
961 "schema_version": SCHEMA_VERSION,
962 "source": {
963 "repository": repository,
964 "revision": revision.lower(),
965 "retrieved_on": normalized_date(
966 source_obj["retrieved_on"], name="source.retrieved_on"
967 ),
968 },
969 "root": root,
970 "label_field": label_field,
971 "identity_fields": normalized_fields,
972 "splits": normalized_splits,
973 }
976def validate_hypothesis_bank(document: Any) -> list[dict[str, Any]]:
977 if not isinstance(document, dict) or not document:
978 raise CliError("hypothesis bank root must be a nonempty object")
979 if len(document) > MAX_HYPOTHESES:
980 raise CliError(f"hypothesis bank exceeds {MAX_HYPOTHESES} entries")
981 entries: list[dict[str, Any]] = []
982 allowed = {
983 "hypothesis",
984 "acc",
985 "reward",
986 "num_visits",
987 "correct_examples",
988 "num_select",
989 }
990 required = {
991 "hypothesis",
992 "acc",
993 "reward",
994 "num_visits",
995 "correct_examples",
996 }
997 for index, (hypothesis, raw_info) in enumerate(document.items()):
998 text = nonempty_text(
999 hypothesis,
1000 name=f"hypothesis key {index}",
1001 maximum=MAX_STRING_CHARS,
1002 )
1003 info = validate_keys(
1004 raw_info,
1005 required=required,
1006 optional=allowed - required,
1007 context=f"hypothesis[{index}]",
1008 )
1009 if info["hypothesis"] != text:
1010 raise CliError(f"hypothesis[{index}].hypothesis must equal its object key")
1011 accuracy = finite_number(
1012 info["acc"], name=f"hypothesis[{index}].acc", minimum=0.0, maximum=1.0
1013 )
1014 reward = finite_number(
1015 info["reward"],
1016 name=f"hypothesis[{index}].reward",
1017 minimum=-1_000_000_000.0,
1018 maximum=1_000_000_000.0,
1019 )
1020 visits = bounded_int(
1021 info["num_visits"],
1022 name=f"hypothesis[{index}].num_visits",
1023 minimum=0,
1024 maximum=MAX_ROWS,
1025 )
1026 correct_examples = info["correct_examples"]
1027 if (
1028 not isinstance(correct_examples, list)
1029 or len(correct_examples) > MAX_CORRECT_EXAMPLES
1030 ):
1031 raise CliError(
1032 f"hypothesis[{index}].correct_examples must be a bounded list"
1033 )
1034 for example_index, example in enumerate(correct_examples):
1035 if not isinstance(example, list) or len(example) != 2:
1036 raise CliError(
1037 f"hypothesis[{index}].correct_examples[{example_index}] "
1038 "must be [row_index, label]"
1039 )
1040 bounded_int(
1041 example[0],
1042 name=(
1043 f"hypothesis[{index}].correct_examples[{example_index}][0]"
1044 ),
1045 minimum=0,
1046 maximum=MAX_ROWS,
1047 )
1048 nonempty_text(
1049 example[1],
1050 name=(
1051 f"hypothesis[{index}].correct_examples[{example_index}][1]"
1052 ),
1053 maximum=256,
1054 )
1055 if "num_select" in info:
1056 bounded_int(
1057 info["num_select"],
1058 name=f"hypothesis[{index}].num_select",
1059 minimum=0,
1060 maximum=MAX_ROWS,
1061 )
1062 normalized = " ".join(text.split()).casefold()
1063 entries.append(
1064 {
1065 "text": text,
1066 "text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
1067 "normalized_sha256": hashlib.sha256(
1068 normalized.encode("utf-8")
1069 ).hexdigest(),
1070 "characters": len(text),
1071 "words": len(text.split()),
1072 "accuracy": accuracy,
1073 "reward": reward,
1074 "num_visits": visits,
1075 "correct_example_count": len(correct_examples),
1076 }
1077 )
1078 return entries
1081def validate_result_document(document: Any) -> dict[str, Any]:
1082 top = validate_keys(
1083 document,
1084 required=(
1085 "schema_version",
1086 "dataset_manifest_sha256",
1087 "hypothesis_bank_sha256",
1088 "split",
1089 "records",
1090 ),
1091 context="result document",
1092 )
1093 if top["schema_version"] != SCHEMA_VERSION:
1094 raise CliError(f"schema_version must be {SCHEMA_VERSION!r}")
1095 split = top["split"]
1096 if split not in SPLIT_NAMES:
1097 raise CliError(f"split must be one of {SPLIT_NAMES}")
1098 records = top["records"]
1099 if not isinstance(records, list) or not records or len(records) > MAX_ROWS:
1100 raise CliError(f"records must contain 1 to {MAX_ROWS} objects")
1101 normalized_records: list[dict[str, str | None]] = []
1102 identifiers: set[str] = set()
1103 for index, raw_record in enumerate(records):
1104 record = validate_keys(
1105 raw_record,
1106 required=("id", "label", "prediction"),
1107 context=f"records[{index}]",
1108 )
1109 identifier = nonempty_text(
1110 record["id"], name=f"records[{index}].id", maximum=256
1111 )
1112 if identifier in identifiers:
1113 raise CliError(f"records[{index}].id duplicates an earlier result id (value redacted)")
1114 identifiers.add(identifier)
1115 label = nonempty_text(
1116 record["label"], name=f"records[{index}].label", maximum=256
1117 )
1118 if label == MISSING_PREDICTION:
1119 raise CliError(f"records[{index}].label uses a reserved value")
1120 prediction = record["prediction"]
1121 if prediction is not None:
1122 prediction = nonempty_text(
1123 prediction,
1124 name=f"records[{index}].prediction",
1125 maximum=256,
1126 )
1127 if prediction == MISSING_PREDICTION:
1128 raise CliError(f"records[{index}].prediction uses a reserved value")
1129 normalized_records.append(
1130 {"id": identifier, "label": label, "prediction": prediction}
1131 )
1132 return {
1133 "schema_version": SCHEMA_VERSION,
1134 "dataset_manifest_sha256": normalized_sha256(
1135 top["dataset_manifest_sha256"], name="dataset_manifest_sha256"
1136 ),
1137 "hypothesis_bank_sha256": normalized_sha256(
1138 top["hypothesis_bank_sha256"], name="hypothesis_bank_sha256"
1139 ),
1140 "split": split,
1141 "records": normalized_records,
1142 }
1145def label_fingerprint(label: str) -> str:
1146 return "label-" + hashlib.sha256(label.encode("utf-8")).hexdigest()[:12]
1149def classification_metrics(records: Sequence[Mapping[str, str | None]]) -> dict[str, Any]:
1150 """Compute deterministic local classification metrics without dependencies."""
1152 labels = {str(record["label"]) for record in records}
1153 labels.update(
1154 str(record["prediction"])
1155 for record in records
1156 if record["prediction"] is not None
1157 )
1158 ordered_labels = sorted(labels)
1159 total = len(records)
1160 covered = sum(record["prediction"] is not None for record in records)
1161 correct = sum(record["prediction"] == record["label"] for record in records)
1162 covered_correct = sum(
1163 record["prediction"] == record["label"]
1164 for record in records
1165 if record["prediction"] is not None
1166 )
1168 f1_values: list[float] = []
1169 for label in ordered_labels:
1170 true_positive = sum(
1171 record["label"] == label and record["prediction"] == label
1172 for record in records
1173 )
1174 false_positive = sum(
1175 record["label"] != label and record["prediction"] == label
1176 for record in records
1177 )
1178 false_negative = sum(
1179 record["label"] == label and record["prediction"] != label
1180 for record in records
1181 )
1182 denominator = 2 * true_positive + false_positive + false_negative
1183 f1_values.append(0.0 if denominator == 0 else 2 * true_positive / denominator)
1185 prediction_labels = ordered_labels + [MISSING_PREDICTION]
1186 confusion: dict[str, dict[str, int]] = {}
1187 for true_label in ordered_labels:
1188 true_key = label_fingerprint(true_label)
1189 confusion[true_key] = {}
1190 for predicted_label in prediction_labels:
1191 predicted_key = (
1192 MISSING_PREDICTION
1193 if predicted_label == MISSING_PREDICTION
1194 else label_fingerprint(predicted_label)
1195 )
1196 count = sum(
1197 record["label"] == true_label
1198 and (
1199 record["prediction"]
1200 if record["prediction"] is not None
1201 else MISSING_PREDICTION
1202 )
1203 == predicted_label
1204 for record in records
1205 )
1206 confusion[true_key][predicted_key] = count
1208 return {
1209 "record_count": total,
1210 "covered_count": covered,
1211 "missing_prediction_count": total - covered,
1212 "label_count": len(ordered_labels),
1213 "label_fingerprints": [label_fingerprint(label) for label in ordered_labels],
1214 "coverage": round(covered / total, 12),
1215 "accuracy_all_records": round(correct / total, 12),
1216 "accuracy_covered_records": (
1217 None if covered == 0 else round(covered_correct / covered, 12)
1218 ),
1219 "macro_f1_all_records": round(sum(f1_values) / len(f1_values), 12),
1220 "confusion_matrix_redacted": confusion,
1221 }
1224def canonical_json_bytes(value: Any) -> bytes:
1225 try:
1226 return json.dumps(
1227 value,
1228 sort_keys=True,
1229 separators=(",", ":"),
1230 ensure_ascii=False,
1231 allow_nan=False,
1232 ).encode("utf-8")
1233 except (TypeError, ValueError) as exc:
1234 raise CliError(f"value cannot be represented as strict JSON: {exc}") from exc
1237def sha256_file(path: Path, *, max_bytes: int = MAX_JSON_BYTES) -> str:
1238 try:
1239 size = path.stat().st_size
1240 except OSError as exc:
1241 raise CliError(f"cannot stat input file: {exc}") from exc
1242 if size > max_bytes:
1243 raise CliError(f"file is {size} bytes; hashing limit is {max_bytes}")
1244 digest = hashlib.sha256()
1245 try:
1246 with path.open("rb") as handle:
1247 for chunk in iter(lambda: handle.read(1024 * 1024), b""):
1248 digest.update(chunk)
1249 except OSError as exc:
1250 raise CliError(f"cannot hash input file: {exc}") from exc
1251 return digest.hexdigest()
1254def summarize_numbers(values: Sequence[float | int]) -> dict[str, float | int | None]:
1255 if not values:
1256 return {"minimum": None, "maximum": None, "mean": None}
1257 return {
1258 "minimum": min(values),
1259 "maximum": max(values),
1260 "mean": round(sum(values) / len(values), 12),
1261 }
1264def count_values(values: Iterable[str]) -> list[dict[str, Any]]:
1265 """Return redacted value counts ordered by descending count then digest."""
1267 counts = Counter(values)
1268 rows = [
1269 {"fingerprint": label_fingerprint(value), "count": count}
1270 for value, count in counts.items()
1271 ]
1272 return sorted(rows, key=lambda row: (-row["count"], row["fingerprint"]))
1275def strict_json_bytes(document: Any) -> bytes:
1276 try:
1277 payload = (
1278 json.dumps(
1279 document,
1280 indent=2,
1281 sort_keys=True,
1282 ensure_ascii=False,
1283 allow_nan=False,
1284 )
1285 + "\n"
1286 ).encode("utf-8")
1287 except (TypeError, ValueError) as exc:
1288 raise CliError(f"report is not strict JSON: {exc}") from exc
1289 if len(payload) > MAX_REPORT_BYTES:
1290 raise CliError(f"report exceeds {MAX_REPORT_BYTES} bytes")
1291 return payload
1294def emit_json(document: Any, *, stream: Any = None) -> None:
1295 if stream is None:
1296 stream = sys.stdout
1297 print(strict_json_bytes(document).decode("utf-8"), end="", file=stream)
1300def emit_error(error: Exception) -> int:
1301 emit_json(
1302 {
1303 "ok": False,
1304 "error": type(error).__name__,
1305 "message": str(error),
1306 "network_access": False,
1307 "model_called": False,
1308 "secret_values_included": False,
1309 },
1310 stream=sys.stderr,
1311 )
1312 return 2