Skills › Compound Engineering › ce-brainstorm › packs-resolve.py
packs-resolve.py
737 lines · skills/ce-brainstorm/scripts/packs-resolve.py at 3449230
1#!/usr/bin/env python3
2"""Resolve the Compound Packs declared in this repo's CE config into pack roots.
4Reads the `packs:` list from `<repo-root>/.compound-engineering/config.yaml`
5and `config.local.yaml` (both layers concatenate; local adds, never replaces),
6validates each entry, resolves path and git sources, enumerates the packs each
7source publishes, applies selection, and prints one JSON object to stdout:
9 {"roots": [{"id": "...", "dir": "/abs/path", "nested_rule_shaped": 0}],
10 "warnings": [...], "errors": [...], "entries": <number of parsed packs entries>}
12`nested_rule_shaped` counts the rule-shaped `.md` files one level below the
13pack's top level. Discovery never reads them (subdirectories are storage), so
14the count is informational and is reported by the health check, not warned.
16Exit 0 whenever resolution ran (per-entry failures are data in `errors` /
17`warnings`); non-zero only when the resolver itself cannot run. Consumers treat
18`errors` as loud per-entry configuration problems and `warnings` as degraded
19availability (e.g. an unreachable git source skipped per the warn-and-continue
20contract).
22`--declared-only` answers the config-only question without touching git or the
23cache: it parses both config layers, shape-checks each entry, and prints
25 {"declared": <bool|null>, "entries": <n>, "errors": [...]}
27where `declared` is true when any entry parsed or the `packs:` block is
28malformed (a broken declaration is still a declaration the consumer should
29surface), false when neither config layer names packs, and null when no
30repository -- and so no config -- could be located.
32Entry shape (documented subset -- anything else under `packs:` is a loud error):
34 packs:
35 - source: packs/local-rules # repo-relative path
36 - source: ~/packs/kk-style # ~ or absolute path
37 - source: https://github.com/o/r # git URL: ref required
38 ref: v1.2.0 # tag, sha, or branch
39 path: packs # optional subfolder (git only)
40 pack: [rails, inertia] # one id, a list, or omit = all
41 id: rails-core # rename (single-pack entries)
42 - source: https://github.com/o/r/tree/main/packs # tree-URL sugar
44Git sources cache under `<scratch-root>/ce-packs/<sha256(url\\nref)>` with an
45atomic temp-clone-then-rename, so a keyed path's existence proves a complete
46clone. All git subprocesses run non-interactively (GIT_TERMINAL_PROMPT=0, ssh
47BatchMode, bounded timeout): missing credentials degrade to a warning, never a
48hang. A missing `git` binary degrades git sources only (each warns and is
49skipped); path sources still resolve, with the repository located by walking up
50from the working directory to the nearest `.git` entry. Environment overrides:
51CE_PACKS_CACHE_ROOT (cache base for tests), CE_PACKS_GIT_TIMEOUT (seconds,
52default 60).
54A published pack is a directory a consumer lists and reads itself, so nothing
55in it may link outside its source: a pack whose tree holds such a link is not
56published (a loud per-entry error), never trimmed a file at a time.
57"""
59from __future__ import annotations
61import argparse
62import functools
63import hashlib
64import json
65import os
66import re
67import shutil
68import stat
69import subprocess
70import sys
71import tempfile
73IS_WINDOWS = os.name == "nt"
74_uid_getter = getattr(os, "geteuid", None) or getattr(os, "getuid", None)
75_EFFECTIVE_UID = _uid_getter() if _uid_getter is not None else None
76GIT_TIMEOUT = float(os.environ.get("CE_PACKS_GIT_TIMEOUT") or 60)
78CONFIG_FILES = ("config.yaml", "config.local.yaml")
79KNOWN_KEYS = {"source", "ref", "path", "pack", "id"}
80SCALAR_KEYS = ("source", "ref", "path", "id") # every known key but the list-valued `pack`
81_TREE_URL_RE = re.compile(
82 r"^(?P<base>https?://github\.com/[^/\s]+/[^/\s]+?)(?:\.git)?/tree/(?P<ref>[^/\s]+)(?:/(?P<path>[^\s]*))?/?$"
83)
86def _is_git_url(source: str) -> bool:
87 return bool(
88 re.match(r"^(https?|ssh|git|file)://", source) or re.match(r"^[\w.-]+@[\w.-]+:", source)
89 )
92def _within(path: str, parent: str) -> bool:
93 """True when `path` is `parent` or lies below it (both already realpath'd)."""
94 return path == parent or path.startswith(parent + os.sep)
97# --- scratch root (peer-job-runner shape: probe /tmp, fall back to TMPDIR) ---
99def _owned_dir(path: str) -> bool:
100 """Directory, not a symlink, owned by the effective uid (POSIX)."""
101 try:
102 st = os.lstat(path)
103 except OSError:
104 return False
105 if not stat.S_ISDIR(st.st_mode):
106 return False
107 if _EFFECTIVE_UID is not None and st.st_uid != _EFFECTIVE_UID:
108 return False
109 return True
112def _private_root_usable(path: str) -> bool:
113 """Create or repair `path` as a private (0700) root owned by this user."""
114 try:
115 os.mkdir(path, 0o700)
116 except FileExistsError:
117 pass
118 except OSError:
119 return False
120 if not IS_WINDOWS:
121 if not _owned_dir(path):
122 return False
123 try:
124 os.chmod(path, 0o700)
125 except OSError:
126 return False
127 return os.path.isdir(path) and os.access(path, os.W_OK)
130def _trusted_checkout(path: str) -> bool:
131 """A real directory (never a symlink) that this user owns; ownership is POSIX-only."""
132 if os.path.islink(path) or not os.path.isdir(path):
133 return False
134 return IS_WINDOWS or _owned_dir(path)
137@functools.lru_cache(maxsize=None)
138def cache_base() -> str | None:
139 configured = os.environ.get("CE_PACKS_CACHE_ROOT")
140 if configured:
141 root = os.path.abspath(configured)
142 os.makedirs(root, exist_ok=True)
143 return root
144 if IS_WINDOWS:
145 base = os.environ.get("LOCALAPPDATA") or tempfile.gettempdir()
146 root = os.path.join(base, "compound-engineering-packs")
147 return root if _private_root_usable(root) else None
148 if _EFFECTIVE_UID is None:
149 return None
150 for base in ("/tmp", os.environ.get("TMPDIR") or "/tmp"):
151 root = os.path.join(base, f"compound-engineering-{_EFFECTIVE_UID}")
152 if _private_root_usable(root):
153 packs = os.path.join(root, "ce-packs")
154 if _private_root_usable(packs):
155 return packs
156 return None
159# --- minimal YAML reader for the documented packs: subset --------------------
161def _strip_comment(line: str) -> str:
162 """Drop a trailing comment (a # preceded by whitespace, outside quotes).
164 A quote toggles quoted state only when it opens a value (start of line or
165 after `: `/`- `/`[`/`,`) or closes one it opened -- a mid-word apostrophe
166 (``it's``) is ordinary content and must not absorb a later comment.
167 """
168 out, quote = [], ""
169 for i, ch in enumerate(line):
170 prev = line[i - 1] if i else " "
171 if quote:
172 if ch == quote:
173 quote = ""
174 elif ch in "'\"" and prev in " \t[,:":
175 quote = ch
176 elif ch == "#" and prev in " \t":
177 break
178 out.append(ch)
179 return "".join(out).rstrip()
182def _scalar(raw: str):
183 raw = raw.strip()
184 if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in "'\"":
185 return raw[1:-1]
186 if raw.lower() in ("true", "false"):
187 return raw.lower() == "true"
188 return raw
191def _parse_value(raw: str):
192 raw = raw.strip()
193 if raw.startswith("[") and raw.endswith("]"):
194 inner = raw[1:-1].strip()
195 return [] if not inner else [_scalar(part) for part in inner.split(",")]
196 return _scalar(raw)
199def parse_packs_block(path: str, errors: list) -> list:
200 """Return the entry dicts under this file's top-level `packs:` key."""
201 if not os.path.isfile(path):
202 return []
203 with open(path, encoding="utf-8-sig", errors="replace") as fh:
204 lines = fh.read().splitlines()
205 entries, in_packs, current, pending_list_key = [], False, None, None
206 for lineno, raw in enumerate(lines, 1):
207 line = _strip_comment(raw)
208 if not line.strip():
209 continue
210 indent = len(line) - len(line.lstrip())
211 if indent == 0 and not (in_packs and line.lstrip().startswith("-")):
212 # A new top-level key ends the packs block; a zero-indent list item
213 # (`- source: ...`) is still part of it -- YAML allows both styles.
214 key, sep, rest = line.partition(":")
215 in_packs = bool(sep) and key.strip() == "packs" and rest.strip() in ("", "[]")
216 if sep and key.strip() == "packs" and not in_packs:
217 # `packs: <inline value>` is malformed, not absent: say so rather
218 # than letting a flow list or a bare path declare nothing.
219 errors.append(
220 f"{os.path.basename(path)}:{lineno}: `packs:` must be a block list of"
221 f" `- source: ...` entries (got `{line.strip()}`)"
222 )
223 current, pending_list_key = None, None
224 continue
225 if not in_packs:
226 continue
227 stripped = line.strip()
228 loc = f"{os.path.basename(path)}:{lineno}"
229 if stripped.startswith("- ") or stripped == "-":
230 body = stripped[1:].strip()
231 if pending_list_key and current is not None and ":" not in body:
232 current[pending_list_key].append(_scalar(body))
233 continue
234 current, pending_list_key = {"_origin": os.path.basename(path), "_line": lineno}, None
235 entries.append(current)
236 if body:
237 if ":" not in body:
238 errors.append(f"{loc}: unrecognized packs entry `{stripped}` -- expected `key: value`")
239 continue
240 key, _, val = body.partition(":")
241 _set_key(current, key.strip(), val, loc, errors)
242 continue
243 if current is None:
244 errors.append(f"{loc}: unrecognized line under packs: `{stripped}` -- expected a `- source: ...` entry")
245 continue
246 if ":" not in stripped:
247 errors.append(f"{loc}: unrecognized line under packs: `{stripped}` -- expected `key: value`")
248 continue
249 key, _, val = stripped.partition(":")
250 key = key.strip()
251 if val.strip() == "" and key == "pack":
252 current[key] = []
253 pending_list_key = key
254 continue
255 pending_list_key = None
256 _set_key(current, key, val, loc, errors)
257 return entries
260def _set_key(entry: dict, key: str, raw_val: str, loc: str, errors: list) -> None:
261 if key not in KNOWN_KEYS:
262 errors.append(f"{loc}: unknown packs entry key `{key}:` -- accepted keys: {', '.join(sorted(KNOWN_KEYS))}")
263 return
264 entry[key] = _parse_value(raw_val)
267# --- git ---------------------------------------------------------------------
269@functools.lru_cache(maxsize=None)
270def _git_env() -> dict:
271 env = dict(os.environ)
272 env["GIT_TERMINAL_PROMPT"] = "0"
273 env["GIT_ASKPASS"] = env.get("GIT_ASKPASS") or "true"
274 ssh = env.get("GIT_SSH_COMMAND") or "ssh"
275 if "BatchMode" not in ssh:
276 env["GIT_SSH_COMMAND"] = ssh + " -o BatchMode=yes"
277 return env
280def _run_git(args: list, cwd: str | None = None):
281 return subprocess.run(
282 ["git", *args], cwd=cwd, env=_git_env(), timeout=GIT_TIMEOUT,
283 capture_output=True, text=True,
284 )
287def resolve_git_source(url: str, ref: str, warnings: list, label: str) -> str | None:
288 """Return the cached checkout dir for url@ref, cloning on miss. None = warn+skip."""
289 if shutil.which("git") is None:
290 warnings.append(f"{label}: git binary not found; source skipped")
291 return None
292 base = cache_base()
293 if base is None:
294 warnings.append(f"{label}: no writable cache root for git sources; source skipped")
295 return None
296 key = hashlib.sha256(f"{url}\n{ref}".encode()).hexdigest()
297 dest = os.path.join(base, key)
298 if os.path.lexists(dest):
299 if _trusted_checkout(dest):
300 return dest
301 warnings.append(f"{label}: cached checkout {dest} is a symlink or not owned by this user; refetching")
302 # rmtree refuses to follow a symlink (and ignores a plain file); unlink
303 # those explicitly so a planted link is removed, never its target.
304 shutil.rmtree(dest, ignore_errors=True)
305 if os.path.islink(dest) or os.path.isfile(dest):
306 try:
307 os.unlink(dest)
308 except OSError:
309 pass
310 if os.path.lexists(dest):
311 warnings.append(f"{label}: cannot replace untrusted cached checkout {dest}; source skipped")
312 return None
313 tmp = tempfile.mkdtemp(prefix=f"{key}.part-", dir=base)
314 try:
315 try:
316 proc = _run_git(["clone", "--quiet", "--depth", "1", "--no-recurse-submodules",
317 "--branch", ref, "--end-of-options", url, tmp])
318 except subprocess.TimeoutExpired:
319 warnings.append(f"{label}: git clone timed out after {int(GIT_TIMEOUT)}s; source skipped")
320 return None
321 if proc.returncode != 0:
322 # tag/branch clone failed -- retry treating ref as a commit sha
323 try:
324 fetched = (
325 _run_git(["init", "--quiet", tmp]).returncode == 0
326 and _run_git(["fetch", "--quiet", "--depth", "1", "--end-of-options", url, ref], cwd=tmp).returncode == 0
327 and _run_git(["checkout", "--quiet", "FETCH_HEAD"], cwd=tmp).returncode == 0
328 )
329 if not fetched:
330 warnings.append(f"{label}: cannot fetch `{ref}` from {url}; source skipped")
331 return None
332 except subprocess.TimeoutExpired:
333 warnings.append(f"{label}: git fetch timed out after {int(GIT_TIMEOUT)}s; source skipped")
334 return None
335 if not os.path.lexists(dest):
336 try:
337 os.replace(tmp, dest)
338 except OSError:
339 # Expected when another resolver published the same key
340 # concurrently; anything else leaves no checkout to return.
341 if not os.path.lexists(dest):
342 warnings.append(f"{label}: could not publish the clone to {dest}; source skipped")
343 return None
344 if _trusted_checkout(dest):
345 return dest
346 warnings.append(
347 f"{label}: cached checkout {dest} appeared during the clone and is a symlink or not owned by this user; source skipped"
348 )
349 return None
350 finally:
351 if os.path.isdir(tmp):
352 shutil.rmtree(tmp, ignore_errors=True)
355# --- pack enumeration --------------------------------------------------------
357_FRONTMATTER_KEYS = ("title:", "applies_when:")
358_FRONTMATTER_CAP = 64 * 1024 # frontmatter must close within this many characters
359# A pack's README is its description, never a rule, whatever frontmatter it
360# carries: it is not published, not reported as skipped, and not counted.
361_README = "readme.md"
364@functools.lru_cache(maxsize=None)
365def _is_knowledge_file(path: str) -> bool:
366 """True when the file opens with a `---` frontmatter block, closed by a `---`
367 line within the cap, that carries `title:` and `applies_when:`. A UTF-8 BOM
368 is not part of the content."""
369 try:
370 with open(path, encoding="utf-8-sig", errors="replace") as fh:
371 lines = fh.read(_FRONTMATTER_CAP).splitlines()
372 except OSError:
373 return False
374 if not lines or lines[0].strip() != "---":
375 return False
376 for end, line in enumerate(lines[1:], 1):
377 if line.strip() == "---":
378 fm = "\n".join(lines[1:end])
379 break
380 else:
381 return False
382 return all(re.search(rf"^\s*{re.escape(k)}", fm, re.MULTILINE) for k in _FRONTMATTER_KEYS)
385def _contained_md_files(directory: str, boundary: str, escaped: list) -> list:
386 """Paths of the `.md` entries directly under `directory`, minus its README,
387 whose real path stays within `boundary` (a realpath). An entry that links
388 outside it is appended to `escaped` and never opened, so a pack cannot read
389 files off the user's machine."""
390 try:
391 names = sorted(os.listdir(directory))
392 except OSError:
393 return []
394 files = []
395 for name in names:
396 if not name.endswith(".md") or name.lower() == _README:
397 continue
398 child = os.path.join(directory, name)
399 if _within(os.path.realpath(child), boundary):
400 files.append(child)
401 else:
402 escaped.append(child)
403 return files
406def _contained_child_dirs(directory: str, boundary: str, escaped: list) -> list:
407 """`(name, path)` of the non-hidden directories directly under `directory`
408 whose real path stays within `boundary`; one that links outside it is
409 appended to `escaped` and never entered."""
410 try:
411 names = sorted(os.listdir(directory))
412 except OSError:
413 return []
414 dirs = []
415 for name in names:
416 child = os.path.join(directory, name)
417 if name.startswith(".") or not os.path.isdir(child):
418 continue
419 if _within(os.path.realpath(child), boundary):
420 dirs.append((name, child))
421 else:
422 escaped.append(child)
423 return dirs
426def _has_knowledge_files(directory: str, boundary: str, escaped: list) -> bool:
427 return any(_is_knowledge_file(f) for f in _contained_md_files(directory, boundary, escaped))
430def _escaping_links(root: str, boundary: str) -> list:
431 """Symlinks anywhere under `root` (walked without following links) whose real
432 path leaves `boundary`, sorted. Only a link can leave: every other entry sits
433 under `root`, which is already inside the boundary."""
434 leaks = []
435 for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
436 for name in dirnames + filenames:
437 child = os.path.join(dirpath, name)
438 if os.path.islink(child) and not _within(os.path.realpath(child), boundary):
439 leaks.append(child)
440 return sorted(leaks)
443def enumerate_packs(source_root: str, boundary: str, escaped: list, self_name: str | None = None) -> dict:
444 """Map published pack id -> dir. Immediate children only; self = single pack.
446 `boundary` is the realpath every child directory and rule file must stay
447 within (the git checkout, the repository, or the source root itself); a
448 child that links outside it is recorded in `escaped` and skipped. Symlinks
449 that stay inside the boundary are ordinary content.
450 """
451 if _has_knowledge_files(source_root, boundary, escaped):
452 name = self_name or os.path.basename(os.path.abspath(source_root))
453 return {name: source_root}
454 return {
455 name: child
456 for name, child in _contained_child_dirs(source_root, boundary, escaped)
457 if _has_knowledge_files(child, boundary, escaped)
458 }
461def nested_rule_shaped(pack_dir: str, boundary: str) -> tuple:
462 """`(total, subdirs)`: how many rule-shaped files sit in the immediate
463 subdirectories of `pack_dir`, and which subdirectories hold them. Scans one
464 level down only, inside `boundary`, so the pack's data is never walked.
465 Subdirectories are storage, so these files are counted, never published."""
466 hits, total = [], 0
467 for name, child in _contained_child_dirs(pack_dir, boundary, []):
468 count = sum(1 for f in _contained_md_files(child, boundary, []) if _is_knowledge_file(f))
469 if count:
470 hits.append(name)
471 total += count
472 return total, hits
475def nested_rules_warning(pack_id: str, pack_dir: str, boundary: str) -> str | None:
476 """The warning for a pack directory with no rule at its top level whose
477 subdirectories hold rule-shaped files: it registers, yet nothing can ever be
478 discovered. None when the subdirectories hold none. The caller establishes
479 that the top level is empty of rules; a pack with top-level rules keeps its
480 nested files as storage and gets `nested_rule_shaped` instead."""
481 total, hits = nested_rule_shaped(pack_dir, boundary)
482 if not hits:
483 return None
484 where = ", ".join(f"`{name}/`" for name in hits)
485 return (
486 f"pack `{pack_id}` has {total} rule-shaped file(s) under {where} that discovery"
487 " never reads -- move rules to the pack's top level (see https://everyinc.github.io/compound-engineering-plugin/guides/packs/, Pack layout)"
488 )
491# --- entry resolution --------------------------------------------------------
493def _entry_label(entry: dict) -> str:
494 return f"{entry.get('_origin', 'config')}:{entry.get('_line', '?')}"
497def _entry_shape_ok(entry: dict, label: str, errors: list) -> bool:
498 """I/O-free checks on one parsed entry: `source:` present, and the scalar
499 keys actually scalar. False after appending the error."""
500 for key in SCALAR_KEYS:
501 val = entry.get(key)
502 if val is not None and not isinstance(val, str):
503 errors.append(f"{label}: `{key}:` must be a single string (got {val!r})")
504 return False
505 if not entry.get("source"):
506 errors.append(f"{label}: entry has no `source:`")
507 return False
508 return True
511def resolve_entry(entry: dict, repo_root: str, roots: list, warnings: list, errors: list) -> None:
512 label = _entry_label(entry)
513 if not _entry_shape_ok(entry, label, errors):
514 return
515 source, ref, sub_path = entry["source"], entry.get("ref"), entry.get("path")
517 tree = _TREE_URL_RE.match(source)
518 if tree:
519 t_ref, t_path = tree.group("ref"), tree.group("path") or ""
520 if isinstance(ref, str) and ref != t_ref:
521 errors.append(f"{label}: tree URL pins ref `{t_ref}` but entry says `ref: {ref}` -- remove one")
522 return
523 if isinstance(sub_path, str) and sub_path.strip("/") != t_path.strip("/"):
524 errors.append(f"{label}: tree URL path `{t_path}` conflicts with `path: {sub_path}` -- remove one")
525 return
526 source, ref, sub_path = tree.group("base"), t_ref, t_path or None
528 if _is_git_url(source):
529 if not isinstance(ref, str) or not ref:
530 errors.append(f"{label}: git source `{source}` requires `ref:` (tag, sha, or branch)")
531 return
532 if ref.startswith("-") or source.startswith("-"):
533 errors.append(f"{label}: git source/ref may not begin with `-`")
534 return
535 checkout = resolve_git_source(source, ref, warnings, label)
536 if checkout is None:
537 if tree:
538 warnings.append(
539 f"{label}: if the branch name contains `/`, tree-URL parsing splits it wrong -- use explicit `ref:` and `path:` fields"
540 )
541 return
542 git_meta = {"url": source, "ref": ref}
543 source_root = os.path.join(checkout, sub_path) if sub_path else checkout
544 real_root, real_checkout = os.path.realpath(source_root), os.path.realpath(checkout)
545 if not _within(real_root, real_checkout):
546 errors.append(f"{label}: path `{sub_path}` escapes the source checkout")
547 return
548 source_root, boundary = real_root, real_checkout
549 if not os.path.isdir(source_root):
550 errors.append(f"{label}: path `{sub_path}` does not exist in {source}@{ref}")
551 return
552 else:
553 git_meta = None
554 if ref is not None:
555 errors.append(f"{label}: `ref:` is only valid on git sources; path sources are read live")
556 return
557 if sub_path is not None:
558 errors.append(f"{label}: `path:` is only valid on git sources; point `source:` at the directory instead")
559 return
560 expanded = os.path.expanduser(source)
561 if os.path.isabs(expanded):
562 source_root = os.path.realpath(expanded)
563 boundary = source_root
564 else:
565 source_root = os.path.realpath(os.path.join(repo_root, expanded))
566 repo_real = os.path.realpath(repo_root)
567 if not _within(source_root, repo_real) or _within(source_root, os.path.join(repo_real, ".git")):
568 errors.append(f"{label}: repo-relative source `{source}` resolves outside the repository")
569 return
570 boundary = repo_real
571 if not os.path.isdir(source_root):
572 errors.append(f"{label}: source directory `{source}` does not exist")
573 return
575 if git_meta:
576 # Display name for a single-pack git source: the path: subfolder's
577 # basename, else the URL's last path segment (never the cache key).
578 tail = (sub_path or source).rstrip("/").rsplit("/", 1)[-1]
579 self_name = re.sub(r"\.git$", "", tail.split(":")[-1]) or None
580 else:
581 self_name = None
582 escaped = []
583 published = enumerate_packs(source_root, boundary, escaped, self_name)
584 for link in escaped:
585 warnings.append(
586 f"{label}: skipped `{os.path.relpath(link, source_root)}` in `{source}` -- it links outside the source"
587 )
588 if not published:
589 warnings.append(f"{label}: source `{source}` publishes no packs (no directories with valid knowledge files)")
590 # Each child directory is a would-be pack with no top-level rule (one
591 # with a rule would have published); say when its rules sit one level
592 # too deep, so the author learns why nothing published.
593 for name, child in _contained_child_dirs(source_root, boundary, []):
594 nested = nested_rules_warning(name, child, boundary)
595 if nested:
596 warnings.append(f"{label}: {nested}")
597 return
599 selection = entry.get("pack")
600 if selection is None:
601 selected = dict(published)
602 else:
603 wanted = selection if isinstance(selection, list) else [selection]
604 if not wanted:
605 warnings.append(f"{label}: `pack:` lists no ids; nothing installed from `{source}`")
606 return
607 missing = [w for w in wanted if w not in published]
608 if missing:
609 errors.append(
610 f"{label}: pack id(s) {', '.join(map(str, missing))} not published by `{source}`"
611 f" -- available: {', '.join(sorted(published)) or 'none'}"
612 )
613 return
614 selected = {w: published[w] for w in wanted}
616 override = entry.get("id")
617 if override is not None:
618 if len(selected) != 1:
619 errors.append(f"{label}: `id:` override requires the entry to install exactly one pack")
620 return
621 selected = {str(override): next(iter(selected.values()))}
623 for pack_id, pack_dir in selected.items():
624 # Consumers list the pack directory themselves, so a link that leaves
625 # the source anywhere inside it would let pack content read files off
626 # the user's machine: refuse the whole pack. (Top-level escapes were
627 # already named as skipped during enumeration.)
628 leaks = _escaping_links(pack_dir, boundary)
629 if leaks:
630 names = ", ".join(f"`{os.path.relpath(p, pack_dir)}`" for p in leaks)
631 errors.append(f"{label}: pack `{pack_id}` not published -- {names} link(s) outside the source")
632 continue
633 for child in _contained_md_files(pack_dir, boundary, []):
634 name = os.path.basename(child)
635 if os.path.isfile(child) and not _is_knowledge_file(child):
636 warnings.append(
637 f"{label}: skipped pack file `{pack_id}/{name}` (missing `title`/`applies_when` frontmatter)"
638 )
639 # A published pack has a rule at its top level, so rule-shaped files in
640 # its subdirectories are storage: counted for the health report, not warned.
641 nested_total, _ = nested_rule_shaped(pack_dir, boundary)
642 root = {"id": pack_id, "dir": pack_dir, "nested_rule_shaped": nested_total, "_label": label}
643 if git_meta:
644 root.update(git_meta)
645 roots.append(root)
648def _emit(declared_only: bool, entries: list, roots: list, warnings: list, errors: list,
649 declared: bool | None = None) -> int:
650 if declared_only:
651 print(json.dumps({"declared": declared, "entries": len(entries), "errors": errors}))
652 else:
653 print(json.dumps({
654 "roots": [{k: v for k, v in r.items() if not k.startswith("_")} for r in roots],
655 "warnings": warnings,
656 "errors": errors,
657 "entries": len(entries),
658 }))
659 return 0
662def _repo_root() -> str | None:
663 """The enclosing checkout's top level: git's answer when git is on PATH,
664 otherwise the nearest ancestor of the working directory holding a `.git`
665 entry (a directory, or the file a worktree or submodule leaves). None when
666 the working directory is not inside a checkout."""
667 if shutil.which("git") is not None:
668 proc = subprocess.run(["git", "rev-parse", "--show-toplevel"],
669 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
670 return proc.stdout.strip() if proc.returncode == 0 else None
671 current = os.getcwd()
672 while True:
673 if os.path.lexists(os.path.join(current, ".git")):
674 return current
675 parent = os.path.dirname(current)
676 if parent == current:
677 return None
678 current = parent
681def _main(argv: list) -> int:
682 parser = argparse.ArgumentParser(description="Resolve the Compound Packs declared in CE config.")
683 parser.add_argument(
684 "--declared-only", action="store_true",
685 help="parse and shape-check the packs: entries only; no git or cache work",
686 )
687 args = parser.parse_args(argv)
688 warnings, errors, roots, entries = [], [], [], []
690 repo_root = _repo_root()
691 if repo_root is None:
692 warnings.append("not inside a git repository; no CE config to read")
693 return _emit(args.declared_only, entries, roots, warnings, errors)
694 cfg_dir = os.path.join(repo_root, ".compound-engineering")
695 for name in CONFIG_FILES:
696 entries.extend(parse_packs_block(os.path.join(cfg_dir, name), errors))
698 if args.declared_only:
699 for entry in entries:
700 _entry_shape_ok(entry, _entry_label(entry), errors)
701 return _emit(True, entries, roots, warnings, errors, declared=bool(entries or errors))
703 for entry in entries:
704 try:
705 resolve_entry(entry, repo_root, roots, warnings, errors)
706 except Exception as exc: # one entry's surprise is that entry's error, not everyone's
707 errors.append(f"{_entry_label(entry)}: unexpected error resolving entry: {exc}")
709 # First declaration wins: config.yaml entries precede config.local.yaml in
710 # CONFIG_FILES, so a personal pack can never displace the team's.
711 by_id = {}
712 final = []
713 for root in roots:
714 prev = by_id.get(root["id"])
715 if prev is not None:
716 errors.append(
717 f"duplicate pack id `{root['id']}`: {root['_label']} ignored, {prev['_label']} kept"
718 " -- rename one with `id:`"
719 )
720 continue
721 by_id[root["id"]] = root
722 final.append(root)
723 return _emit(False, entries, final, warnings, errors)
726def main() -> int:
727 try:
728 return _main(sys.argv[1:])
729 except Exception as exc: # never a traceback: consumers need valid JSON
730 print(json.dumps({
731 "roots": [], "warnings": [], "errors": [f"packs resolver failed unexpectedly: {exc}"], "entries": 0,
732 }))
733 return 0
736if __name__ == "__main__":
737 sys.exit(main())