Skills › BMAD Method › bmad › migrate_v6.py
migrate_v6.py
1052 lines · skills/bmad/scripts/migrate_v6.py at bd70c64
1#!/usr/bin/env python3
2# /// script
3# requires-python = ">=3.11"
4# ///
5"""Report or remove the v6 traces in `_bmad/` that v7 does not read, or fold pre-release spec folders.
7With no flag it lists the traces: the classic installer's files, the v6 install answers
8v7 has no key for, the customization keys of retired skills, and customizations of
9skills that are not installed. With `--clean` it copies `_bmad/` to
10`.v6-v7-migration-backup-<datetime>/`, which the method migration reuses, then removes them.
12With `--fold-specs <root>` it folds every 6.13 pre-release spec folder (`spec-<name>/` holding
13`spec-<name>.md`) under the output folder into the ticket that holds it, or into a new `backlog/` story when the folder
14sits at the root; the method migration's guide says when to run it.
16Usage:
17 uv run migrate_v6.py --project-root P --skill <the bmad skill> [--root R ...] [--clean]
18 uv run migrate_v6.py --project-root P --skill <the bmad skill> --fold-specs <output folder>
19"""
21from __future__ import annotations
23import argparse
24import datetime
25import importlib.util
26import os
27import re
28import shutil
29import sys
30import tomllib
31from pathlib import Path, PurePosixPath
33sys.dont_write_bytecode = True
36def _load_setup():
37 spec = importlib.util.spec_from_file_location("bmad_setup", Path(__file__).with_name("setup.py"))
38 module = importlib.util.module_from_spec(spec)
39 assert spec.loader is not None
40 spec.loader.exec_module(module)
41 return module
44_setup = _load_setup()
45Installation = _setup.Installation
46Retirement = _setup.Retirement
47TABLE_HEADER = _setup.TABLE_HEADER
48TEAM_CONFIG = _setup.TEAM_CONFIG
49USER_CONFIG = _setup.USER_CONFIG
50_MISSING = _setup._MISSING
51discover_installation = _setup.discover_installation
52header_path = _setup.header_path
53lookup = _setup.lookup
54present = _setup.present
55reject_unusable_bmad = _setup.reject_unusable_bmad
56render_toml = _setup.render_toml
57retired_skills = _setup.retired_skills
58retirement_report = _setup.retirement_report
59write_text = _setup.write_text
61# Traces the classic installer leaves under _bmad.
62LEGACY_LEFTOVERS = (
63 "_config/manifest.yaml",
64 "_config/files-manifest.csv",
65 "_config/skill-manifest.csv",
66 "_config/bmad-help.csv",
67 "config.user.toml",
68 "core/config.yaml",
69 "core/module-help.csv",
70 "core/v6-shims",
71 "bmm/config.yaml",
72 "bmm/module-help.csv",
73 "bmm/v6-shims",
74)
75# The v6.12 install answers v7 has no key for. Only these are removed, so a key added later is never touched.
76V6_KEYS = (
77 ("core", "user_name"),
78 ("core", "communication_language"),
79 ("core", "document_output_language"),
80 ("modules", "bmm", "user_skill_level"),
81 ("modules", "bmm", "planning_artifacts"),
82 ("modules", "bmm", "implementation_artifacts"),
83 ("modules", "bmm", "project_knowledge"),
84)
85CENTRAL_CONFIGS = (TEAM_CONFIG, "_bmad/custom/config.toml", USER_CONFIG)
86BACKUP_PREFIX = ".v6-v7-migration-backup-"
87# `[workflow]` keys the retired spec and ticket skills read that the agent they retired into does not. Only
88# these are removed from that skill's `_bmad/custom/` files, and the note names what replaced them.
89RETIRED_CUSTOM_KEYS = {
90 "bmad-agent-po": (
91 "initiative_template",
92 "epic_template",
93 "story_template",
94 "spec_template",
95 "spec_output_path",
96 "run_folder_pattern",
97 "on_complete",
98 ),
99}
100RETIRED_CUSTOM_NOTE = (
101 "{file}: {keys} are keys of retired skills and were dropped; bmad-agent-po reads ticket_template, "
102 "the one template for an initiative, epic or story, and <capability>_after (spec_after, tickets_after, ...) "
103 "in place of on_complete."
104)
105# `[workflow]` keys of the retired skills that the agent reads under `[agent]`; they move there, values kept.
106MOVED_CUSTOM_KEYS = {
107 "bmad-agent-po": ("activation_steps_prepend", "activation_steps_append", "persistent_facts"),
108}
109MOVED_CUSTOM_NOTE = "{file}: {keys} moved from [workflow] to [agent], where bmad-agent-po reads them."
112def legacy_leftovers(project_root: Path) -> list[str]:
113 bmad = project_root / "_bmad"
114 return [
115 relative
116 for relative in LEGACY_LEFTOVERS
117 if (path := bmad.joinpath(*PurePosixPath(relative).parts)).exists() and inside_bmad(path, project_root)
118 ]
121def inside_bmad(path: Path, project_root: Path) -> bool:
122 """A link along the way could point a cleanup outside the project; such a path is not cleaned."""
123 return path.resolve().is_relative_to((project_root / "_bmad").resolve())
126def stale_config_keys(project_root: Path) -> list[tuple[str, tuple[str, ...]]]:
127 """The V6_KEYS set in the central config files."""
128 stale: list[tuple[str, tuple[str, ...]]] = []
129 for relative in CENTRAL_CONFIGS:
130 path = project_root / relative
131 if path.is_symlink() or not path.is_file() or not inside_bmad(path, project_root):
132 continue
133 try:
134 data = tomllib.loads(path.read_text(encoding="utf-8"))
135 except (OSError, UnicodeError, tomllib.TOMLDecodeError):
136 continue
137 stale += [(relative, key) for key in V6_KEYS if lookup(data, key) is not _MISSING]
138 return stale
141def stale_custom_keys(project_root: Path, retirement: Retirement) -> list[tuple[str, tuple[str, ...]]]:
142 """The RETIRED_CUSTOM_KEYS and MOVED_CUSTOM_KEYS set in a skill's `_bmad/custom/` files, a retired name's file
143 carried over included."""
144 custom = project_root / "_bmad" / "custom"
145 stale: list[tuple[str, tuple[str, ...]]] = []
146 for skill in dict.fromkeys((*RETIRED_CUSTOM_KEYS, *MOVED_CUSTOM_KEYS)):
147 keys = (*RETIRED_CUSTOM_KEYS.get(skill, ()), *MOVED_CUSTOM_KEYS.get(skill, ()))
148 names = [f"{skill}.toml", f"{skill}.user.toml"]
149 names += [old for old, new in (*retirement.renames, *retirement.unmoved) if new.split(".")[0] == skill]
150 for name in dict.fromkeys(names):
151 path = custom / name
152 if path.is_symlink() or not path.is_file() or not inside_bmad(path, project_root):
153 continue
154 try:
155 data = tomllib.loads(path.read_text(encoding="utf-8"))
156 except (OSError, UnicodeError, tomllib.TOMLDecodeError):
157 continue
158 stale += [
159 (f"_bmad/custom/{name}", ("workflow", key))
160 for key in keys
161 if lookup(data, ("workflow", key)) is not _MISSING
162 ]
163 return stale
166def is_moved(key: tuple[str, ...]) -> bool:
167 return any(key[-1] in keys for keys in MOVED_CUSTOM_KEYS.values())
170def custom_key_notes(stale: list[tuple[str, tuple[str, ...]]]) -> list[str]:
171 """Per file in a `stale_custom_keys` result, a note on the keys dropped and one on the keys moved."""
172 notes: list[str] = []
173 for file in dict.fromkeys(file for file, _key in stale):
174 for note, moved in ((RETIRED_CUSTOM_NOTE, False), (MOVED_CUSTOM_NOTE, True)):
175 keys = [key[-1] for f, key in stale if f == file and is_moved(key) == moved]
176 if keys:
177 notes.append(note.format(file=file, keys=", ".join(keys)))
178 return notes
181def unused_customizations(project_root: Path, installation: Installation, retirement: Retirement) -> list[str]:
182 """`_bmad/custom/` files of skills that are not installed. Files setup is about to rename are not unused."""
183 custom = project_root / "_bmad" / "custom"
184 if custom.is_symlink() or not custom.is_dir():
185 return []
186 moving = {old for old, _new in (*retirement.renames, *retirement.unmoved)}
187 unused: list[str] = []
188 for path in sorted(custom.iterdir(), key=lambda entry: entry.name):
189 name = path.name
190 stem = name.removesuffix(".toml").removesuffix(".user")
191 if not name.startswith("bmad-") or not name.endswith(".toml") or name in moving:
192 continue
193 if stem not in installation.folders:
194 unused.append(name)
195 return unused
198def traces_json(project_root: Path, installation: Installation, retirement: Retirement) -> dict[str, object]:
199 custom_keys = stale_custom_keys(project_root, retirement)
200 return {
201 "legacy_leftovers": legacy_leftovers(project_root),
202 "stale_config_keys": [
203 {"file": relative, "key": ".".join(path)}
204 for relative, path in (*stale_config_keys(project_root), *custom_keys)
205 ],
206 "unused_customizations": [
207 f"_bmad/custom/{name}" for name in unused_customizations(project_root, installation, retirement)
208 ],
209 "notes": custom_key_notes(custom_keys),
210 }
213def clean(project_root: Path, skill_root: Path, *, roots: tuple[Path, ...] = ()) -> dict[str, object]:
214 """Back up `_bmad/`, then remove the v6 traces v7 does not read: the classic installer's files, v6
215 config keys, and customizations of skills that are not installed."""
216 reject_unusable_bmad(project_root)
217 bmad = project_root / "_bmad"
218 installation = discover_installation(skill_root, roots)
219 if installation.missing_records:
220 # Without every module record, a renamed skill's customizations would look unused.
221 raise Exception("a module record is missing; run setup to repair the installation before cleaning up")
222 retirement = retirement_report(
223 project_root, skill_root, installation, retired_skills(installation.modules, installation.modules)
224 )
225 leftovers = legacy_leftovers(project_root)
226 custom_keys = stale_custom_keys(project_root, retirement)
227 stale = [*stale_config_keys(project_root), *custom_keys]
228 unused = unused_customizations(project_root, installation, retirement)
229 moved = [(file, key) for file, key in custom_keys if is_moved(key)]
230 removed = [(file, key) for file, key in stale if (file, key) not in moved]
231 edits = {
232 relative: text_without_keys(
233 (project_root / relative).read_text(encoding="utf-8"),
234 [key for file, key in removed if file == relative],
235 [key for file, key in moved if file == relative],
236 )
237 for relative in dict.fromkeys(file for file, _path in stale)
238 }
239 report: dict[str, object] = {
240 "mode": "clean",
241 "backup": None,
242 "removed_files": [],
243 "removed_keys": [],
244 "moved_keys": [],
245 "removed_customizations": [],
246 "notes": [],
247 }
248 if not (leftovers or stale or unused):
249 return report
250 backup = backup_bmad(project_root)
251 report["backup"] = backup.relative_to(project_root).as_posix()
252 for relative in leftovers:
253 path = bmad.joinpath(*PurePosixPath(relative).parts)
254 if path.is_dir() and not path.is_symlink():
255 shutil.rmtree(path)
256 else:
257 path.unlink()
258 prune_empty_parents(path.parent, bmad)
259 for relative, text in edits.items():
260 write_text(project_root / relative, text)
261 for name in unused:
262 (bmad / "custom" / name).unlink()
263 report["removed_files"] = [f"_bmad/{relative}" for relative in leftovers]
264 report["removed_keys"] = [{"file": relative, "key": ".".join(path)} for relative, path in removed]
265 report["moved_keys"] = [
266 {"file": relative, "key": ".".join(path), "to": f"agent.{path[-1]}"} for relative, path in moved
267 ]
268 report["removed_customizations"] = [f"_bmad/custom/{name}" for name in unused]
269 report["notes"] = custom_key_notes(custom_keys)
270 return report
273def backup_bmad(project_root: Path) -> Path:
274 """Copy `_bmad/` into a new `.v6-v7-migration-backup-<datetime>/`, which the method migration reuses.
275 Its `.gitignore` keeps the copy, personal settings included, out of every commit."""
276 now = datetime.datetime.now()
277 backup = project_root / f"{BACKUP_PREFIX}{now:%Y%m%d-%H%M}"
278 if present(backup):
279 backup = project_root / f"{BACKUP_PREFIX}{now:%Y%m%d-%H%M%S}"
280 shutil.copytree(project_root / "_bmad", backup / "_bmad", symlinks=True)
281 write_text(backup / ".gitignore", "*")
282 return backup
285def prune_empty_parents(folder: Path, stop: Path) -> None:
286 while folder != stop and folder.is_dir() and not folder.is_symlink() and not any(folder.iterdir()):
287 folder.rmdir()
288 folder = folder.parent
291def text_without_keys(text: str, paths: list[tuple[str, ...]], moves: list[tuple[str, ...]] = ()) -> str:
292 """The file's text with these keys removed, and `moves` moved to the same key under `[agent]` (a list joined
293 after one already there), so comments and layout survive.
295 A key written on one line in its table's section is cut by text, and a table left
296 empty loses its header. When the result does not parse to the expected values,
297 the whole file is rendered instead.
298 """
299 original = tomllib.loads(text)
300 expected = tomllib.loads(text)
301 moved = {path[-1]: value for path in moves if (value := lookup(original, path)) is not _MISSING}
302 for path in (*paths, *moves):
303 drop_path(expected, path)
304 if moved:
305 agent = expected.setdefault("agent", {})
306 for leaf, value in moved.items():
307 current = agent.get(leaf)
308 if current is None:
309 agent[leaf] = value
310 elif isinstance(current, list) and isinstance(value, list):
311 agent[leaf] = current + value
312 lines = text.split("\n")
313 for path in (*paths, *moves):
314 lines = lines_without_key(lines, path[:-1], path[-1])
315 headers = [index for index, line in enumerate(lines) if TABLE_HEADER.match(line)]
316 emptied = {
317 index
318 for position, index in enumerate(headers)
319 if (name := header_path(lines[index])) is not None
320 and lookup(expected, name) is _MISSING
321 and not any(
322 line.strip() and not line.lstrip().startswith("#")
323 for line in lines[index + 1 : headers[position + 1] if position + 1 < len(headers) else len(lines)]
324 )
325 }
326 candidate = "\n".join(line for index, line in enumerate(lines) if index not in emptied)
327 if moved and "agent" not in original:
328 candidate = candidate.rstrip("\n") + "\n\n" + render_toml({"agent": moved})
329 try:
330 if tomllib.loads(candidate) == expected:
331 return candidate
332 except tomllib.TOMLDecodeError:
333 pass
334 return render_toml(expected)
337def drop_path(data: dict, path: tuple[str, ...]) -> None:
338 """Delete the key at path, then any table the deletion left empty."""
339 parents = [data]
340 for part in path[:-1]:
341 child = parents[-1].get(part)
342 if not isinstance(child, dict):
343 return
344 parents.append(child)
345 parents[-1].pop(path[-1], None)
346 for depth in range(len(path) - 1, 0, -1):
347 if parents[depth]:
348 break
349 parents[depth - 1].pop(path[depth - 1], None)
352def lines_without_key(lines: list[str], table: tuple[str, ...], leaf: str) -> list[str]:
353 start, end = 0, len(lines)
354 if table:
355 headers = [(index, header_path(line)) for index, line in enumerate(lines) if TABLE_HEADER.match(line)]
356 for position, (index, name) in enumerate(headers):
357 if name == table:
358 start = index + 1
359 end = headers[position + 1][0] if position + 1 < len(headers) else len(lines)
360 break
361 else:
362 return lines
363 else:
364 end = next((index for index, line in enumerate(lines) if TABLE_HEADER.match(line)), len(lines))
365 key = re.compile(rf"\s*(?:{re.escape(leaf)}|\"{re.escape(leaf)}\"|'{re.escape(leaf)}')\s*=")
366 for index in range(start, end):
367 if not key.match(lines[index]):
368 continue
369 # A multi-line value ends at the first line after which the file parses again.
370 for stop in range(index + 1, end + 1):
371 remaining = [*lines[:index], *lines[stop:]]
372 try:
373 tomllib.loads("\n".join(remaining))
374 except tomllib.TOMLDecodeError:
375 continue
376 return remaining
377 return lines
378 return lines
381# --- Pre-release spec folders ---------------------------------------------------------------------------------
382# A 6.13 pre-release `spec-<name>/` holds `spec-<name>.md` (frontmatter lists `companions` and `sources`, sections Why,
383# Capabilities with `CAP-<n>` ids, Constraints, Non-goals, Success signal, Assumptions, Open Questions), `.memlog.md`
384# and companions. The ticket file is the spec now, so the folder folds into the ticket that holds it.
386MEMLOG = ".memlog.md"
387SPEC_FOLDER = re.compile(r"\Aspec-(?P<slug>[A-Za-z0-9][A-Za-z0-9._-]*)\Z")
388CAP_ID = re.compile(r"\bCAP-(\d+)\b")
389R_LINE = re.compile(r"^\s*-\s+R(\d+)\b", re.MULTILINE)
390R_SOURCE = re.compile(r"^- R\d+ \((.*?)\)", re.MULTILINE)
391CAPABILITY = re.compile(r"\A-\s+\*\*(CAP-(\d+))\*\*\s*(.*)\Z")
392CAPABILITY_FIELD = re.compile(r"\A\s+-\s+\*\*(intent|success):\*\*\s*(.*)\Z")
393HEADING = re.compile(r"(?m)^## (.+?)\s*$")
394SECTION_ORDER = (
395 "Description",
396 "Outcome",
397 "Requirements",
398 "Constraints",
399 "Done when",
400 "Boundaries",
401 "References",
402 "Notes",
403)
404UNTOUCHED_FOLDERS = ("archive", "archive-v6")
405COMPANION_TYPES = {"DESIGN.md": "design", "EXPERIENCE.md": "experience", "ARCHITECTURE-SPINE.md": "architecture"}
406TEXT_SUFFIXES = (".md", ".toml")
409def fold_specs(project_root: Path, store: Path) -> dict[str, object]:
410 """Fold every pre-release spec folder under `store` into its ticket, parents before children."""
411 if not store.is_dir():
412 raise Exception(f"{store} is not a folder")
413 report: dict[str, object] = {"mode": "fold-specs", "folded": [], "problems": []}
414 for folder in spec_folders(store):
415 try:
416 report["folded"].append(fold_spec(project_root, store, folder))
417 except FoldProblem as problem:
418 report["problems"].append({"spec": relative(folder, project_root), "problem": str(problem)})
419 return report
422class FoldProblem(Exception):
423 pass
426def spec_folders(store: Path) -> list[Path]:
427 found: list[Path] = []
428 for current, dirs, _files in os.walk(store):
429 dirs[:] = sorted(d for d in dirs if not d.startswith(".") and d not in UNTOUCHED_FOLDERS)
430 for name in dirs:
431 if SPEC_FOLDER.match(name) and (Path(current) / name / f"{name}.md").is_file():
432 found.append(Path(current) / name)
433 return sorted(found, key=lambda path: (len(path.parts), path.as_posix()))
436def relative(path: Path, root: Path) -> str:
437 """From the root, or absolute for a store outside it."""
438 return path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix()
441def fold_spec(project_root: Path, store: Path, folder: Path) -> dict[str, object]:
442 slug = SPEC_FOLDER.match(folder.name)["slug"]
443 spec = parse_spec(read_for_fold(folder / f"{folder.name}.md", project_root))
444 loose = folder.parent == store
445 if loose:
446 home = store / "backlog"
447 ticket = home / f"story-{slug}.md"
448 if ticket.exists():
449 raise FoldProblem(f"{relative(ticket, project_root)} already exists")
450 else:
451 home = folder.parent
452 ticket = home / f"{home.name}.md"
453 if not ticket.is_file():
454 raise FoldProblem("not inside a ticket folder (one holding a same-named .md) and not at the root")
455 moving = [
456 path
457 for path in sorted(folder.iterdir(), key=lambda p: p.name)
458 if path.name not in (f"{folder.name}.md", MEMLOG)
459 and not path.name.endswith(".tmp")
460 and not path.name.startswith(".")
461 ]
462 taken = [path.name for path in moving if (home / path.name).exists()]
463 if taken:
464 raise FoldProblem(f"{', '.join(taken)} already exist(s) beside the ticket")
466 existing = read_for_fold(ticket, project_root) if ticket.is_file() else ""
467 existing_requirements = section_body(existing, "Requirements")
468 offset = max((int(n) for n in R_LINE.findall(existing_requirements)), default=0)
469 ids = {f"CAP-{n}": f"R{n + offset}" for n in spec["capabilities"]}
471 def map_cap(match: re.Match[str]) -> str:
472 return f"R{int(match.group(1)) + offset}"
474 def map_caps(text: str) -> str:
475 """CAP ids mapped everywhere but inside a Requirements line's parentheses, where they are a folded spec's source ids."""
476 out, last = [], 0
477 for span in R_SOURCE.finditer(text):
478 out += [CAP_ID.sub(map_cap, text[last : span.start()]), span.group(0)]
479 last = span.end()
480 return "".join(out) + CAP_ID.sub(map_cap, text[last:])
482 # Lines the ticket wrote itself stay, less any that name the spec folder, the mapped ones after them; prose that
483 # only pointed at the folder is replaced.
484 own = "\n".join(line for line in existing_requirements.splitlines() if folder.name not in line)
485 own_lines = map_caps(own).strip("\n").splitlines()
486 if R_LINE.search(existing_requirements):
487 replace_requirements = True
488 else:
489 own_lines = []
490 replace_requirements = not existing_requirements.strip() or folder.name in existing_requirements
492 for key in ("constraints", "non_goals", "assumptions", "questions"):
493 spec[key] = [CAP_ID.sub(map_cap, line) for line in spec[key]]
494 spec["why"] = CAP_ID.sub(map_cap, spec["why"])
495 spec["kept"] = [(name, CAP_ID.sub(map_cap, value)) for name, value in spec["kept"]]
496 spec["capabilities"] = {n: CAP_ID.sub(map_cap, line) for n, line in spec["capabilities"].items()}
498 # References to the folder, wherever they are: `spec-<name>/spec-<name>.md` becomes the ticket, `spec-<name>/x` the
499 # moved companion. A suffix rewrite keeps relative and absolute forms right, since the folder's parent is unchanged.
500 new_prefix = "backlog/" if loose else ""
501 rewrites = [(f"{folder.name}/{folder.name}.md", f"{new_prefix}{ticket.name}")]
502 rewrites += [(f"{folder.name}/{path.name}", f"{new_prefix}{path.name}") for path in moving]
503 # Every rewrite is read and computed first, so a file that cannot be read stops the fold before anything changes.
504 writes: dict[Path, str] = {}
505 for path in text_files(store, skip=(folder,)):
506 text = read_for_fold(path, project_root)
507 changed = text
508 for old, new in rewrites:
509 changed = rewrite_reference(changed, old, new, path, folder.parent, project_root)
510 if changed != text:
511 writes[path] = changed
512 for path in companion_files(moving):
513 text = read_for_fold(path, project_root)
514 changed = rebase_links(text, path, folder, home, ticket)
515 if changed != text:
516 writes[path] = changed
517 for path in cap_scope(home, folder, moving) if not loose else companion_files(moving):
518 text = writes[path] if path in writes else read_for_fold(path, project_root)
519 changed = map_caps(text)
520 if changed != text:
521 writes[path] = changed
522 for path in (home / MEMLOG, folder / MEMLOG):
523 if path.is_file():
524 read_for_fold(path, project_root)
525 for path, text in writes.items():
526 path.write_text(text, encoding="utf-8")
527 rewritten = [relative(path, project_root) for path in writes]
529 references, unresolved = companion_references(spec["companions"], folder, home, project_root, store)
530 sources = [rebase(item, folder, home, project_root, store) for item in spec["sources"]]
531 companions = [companion_path(item, folder, home, project_root, store) for item in spec["companions"]]
532 requirements = own_lines + [f"- {ids[f'CAP-{n}']} (CAP-{n}): {line}" for n, line in spec["capabilities"].items()]
533 if loose:
534 text = story_text(spec, home, slug, requirements, references, sources, companions)
535 else:
536 text = folded_ticket_text(
537 ticket.read_text(encoding="utf-8"),
538 ticket,
539 spec,
540 requirements,
541 references,
542 sources,
543 companions,
544 replace_requirements,
545 )
546 home.mkdir(parents=True, exist_ok=True)
547 write_text(ticket, text)
549 memlog = home / MEMLOG
550 spec_memlog = folder / MEMLOG
551 if spec_memlog.is_file():
552 if memlog.is_file():
553 append_text(memlog, memlog_body(spec_memlog))
554 spec_memlog.unlink()
555 else:
556 shutil.move(spec_memlog, memlog)
557 elif not memlog.is_file():
558 write_text(memlog, f"---\ntopic: {spec['title']}\nupdated: {datetime.datetime.now():%Y-%m-%dT%H:%M}\n---\n")
559 entries = [f"- (note) {name}, from {folder.name}: {value}" for name, value in spec["kept"]]
560 entries.append(
561 f"- (event) folded {folder.name}/ into {ticket.name}"
562 + ("; " + ", ".join(f"{old} is {new}" for old, new in ids.items()) if ids else "")
563 )
564 append_text(memlog, "\n".join(entries) + "\n")
566 for path in moving:
567 shutil.move(path, home / path.name)
568 (folder / f"{folder.name}.md").unlink()
569 shutil.rmtree(folder)
570 return {
571 "spec": relative(folder, project_root),
572 "ticket": relative(ticket, project_root),
573 "ids": ids,
574 "companions": [relative(home / path.name, project_root) for path in moving],
575 "unresolved": unresolved,
576 "memlog": relative(memlog, project_root),
577 "rewritten": rewritten,
578 }
581PATH_DELIMITERS = set(" \t\n()[]<>\"'`,|=:")
584def rewrite_reference(text: str, old: str, new: str, file: Path, base: Path, project_root: Path) -> str:
585 """`old` replaced by `new` where the path it ends resolves to `base / old`: from the file's folder or one above
586 it (the project root among them), from `{project-root}`, or absolute. Another folder's spec of the same name
587 keeps its references."""
588 target = (base / old).resolve()
589 out, last = [], 0
590 for match in re.finditer(re.escape(old), text):
591 start = match.start()
592 while start > 0 and text[start - 1] not in PATH_DELIMITERS:
593 start -= 1
594 written = text[start : match.end()]
595 if written.startswith("{project-root}/"):
596 candidates = [project_root / written.removeprefix("{project-root}/")]
597 elif written.startswith("/"):
598 candidates = [Path(written)]
599 else:
600 candidates = [above / written for above in (file.parent, *file.parent.parents)]
601 if any(candidate.resolve() == target for candidate in candidates):
602 out += [text[last : match.start()], new]
603 last = match.end()
604 return "".join(out) + text[last:]
607LINK_TOKEN = re.compile(r"[^\s()\[\]<>\"'`,|=:]+")
610def rebase_links(text: str, file: Path, folder: Path, home: Path, ticket: Path) -> str:
611 """A companion's links as they read once it moves from the spec folder to `home`: a `./` or `../` path that
612 leaves the folder is recomputed from the new place, and a path to the spec itself names the ticket."""
613 spec = (folder / f"{folder.name}.md").resolve()
614 new_parent = (home / file.relative_to(folder)).parent.resolve()
616 def swap(match: re.Match[str]) -> str:
617 path, hash_, anchor = match.group(0).partition("#")
618 if not path.startswith(("./", "../")) and path != spec.name:
619 return match.group(0)
620 target = (file.parent / path).resolve()
621 if target == spec:
622 target = ticket.resolve()
623 elif target.is_relative_to(folder.resolve()) or not target.exists():
624 return match.group(0)
625 return Path(os.path.relpath(target, new_parent)).as_posix() + hash_ + anchor
627 return LINK_TOKEN.sub(swap, text)
630def read_for_fold(path: Path, project_root: Path) -> str:
631 try:
632 return path.read_text(encoding="utf-8")
633 except (OSError, UnicodeError) as error:
634 raise FoldProblem(f"{relative(path, project_root)} cannot be read: {error}") from error
637def text_files(root: Path, *, skip: tuple[Path, ...] = ()) -> list[Path]:
638 """The .md and .toml files under root, hidden files, archives and `skip` folders left out."""
639 found: list[Path] = []
640 for current, dirs, files in os.walk(root):
641 dirs[:] = sorted(
642 d for d in dirs if not d.startswith(".") and d not in UNTOUCHED_FOLDERS and (Path(current) / d) not in skip
643 )
644 found += [Path(current) / f for f in sorted(files) if f.endswith(TEXT_SUFFIXES) and not f.startswith(".")]
645 return found
648def companion_files(moving: list[Path]) -> list[Path]:
649 files: list[Path] = []
650 for path in moving:
651 files += text_files(path) if path.is_dir() else [path] if path.name.endswith(TEXT_SUFFIXES) else []
652 return files
655def cap_scope(home: Path, folder: Path, moving: list[Path]) -> list[Path]:
656 """Where a spec's `CAP` ids are cited: the ticket folder and its children, except a child container with a
657 spec of its own, folded or not, whose own fold maps its ids; only that child's ticket file, which cites the
658 parent's, is in."""
659 files: list[Path] = []
660 for current, dirs, names in os.walk(home):
661 here = Path(current)
662 if here == folder:
663 dirs[:] = []
664 continue
665 files += [here / f for f in sorted(names) if f.endswith(TEXT_SUFFIXES) and not f.startswith(".")]
666 kept: list[str] = []
667 for d in sorted(dirs):
668 if d.startswith(".") or d in UNTOUCHED_FOLDERS:
669 continue
670 child = here / d
671 if has_own_spec(child):
672 files.append(child / f"{d}.md")
673 continue
674 kept.append(d)
675 dirs[:] = kept
676 return files + companion_files(moving)
679def has_own_spec(child: Path) -> bool:
680 ticket = child / f"{child.name}.md"
681 if not ticket.is_file():
682 return False
683 if any(SPEC_FOLDER.match(p.name) and (p / f"{p.name}.md").is_file() for p in child.iterdir() if p.is_dir()):
684 return True
685 requirements = section_body(ticket.read_text(encoding="utf-8"), "Requirements")
686 return any(CAP_ID.search(span.group(1)) for span in R_SOURCE.finditer(requirements))
689def parse_spec(text: str) -> dict[str, object]:
690 meta, body = split_frontmatter(text)
691 title = next((line[2:].strip() for line in body.splitlines() if line.startswith("# ")), "")
692 sections = {name.lower(): content for name, content in split_sections(body)[1]}
693 capabilities: dict[int, str] = {}
694 current: dict[str, str] | None = None
695 key = "rest"
696 blocks: list[tuple[int, dict[str, str]]] = []
697 for line in sections.get("capabilities", "").splitlines():
698 if match := CAPABILITY.match(line):
699 current, key = {"rest": match.group(3).strip()}, "rest"
700 blocks.append((int(match.group(2)), current))
701 elif current is not None and (field := CAPABILITY_FIELD.match(line)):
702 key = field.group(1)
703 current[key] = field.group(2).strip()
704 elif current is not None and line.strip():
705 # A wrapped line continues the field above it; another bullet does not.
706 if line.lstrip().startswith("- "):
707 key = "rest"
708 current[key] = f"{current.get(key, '')} {line.strip()}".strip()
709 for number, block in blocks:
710 intent, success = block.get("intent", ""), block.get("success", "")
711 line = intent or block["rest"]
712 if success:
713 line = f"{line.rstrip('.')}. Success: {success}"
714 capabilities[number] = line
715 kept = [(name, prose(sections[name])) for name in ("why", "success signal") if sections.get(name, "").strip()]
716 return {
717 "title": title,
718 "companions": frontmatter_list(meta, "companions"),
719 "sources": frontmatter_list(meta, "sources"),
720 "capabilities": capabilities,
721 "constraints": bullets(sections.get("constraints", "")),
722 "non_goals": bullets(sections.get("non-goals", "")),
723 "assumptions": bullets(sections.get("assumptions", "")),
724 "questions": bullets(sections.get("open questions", "")),
725 "why": prose(sections.get("why", "")),
726 "kept": kept,
727 }
730def prose(section: str) -> str:
731 """A section as one line, bullet markers dropped."""
732 lines = [line.strip() for line in section.splitlines() if line.strip()]
733 return " ".join(line[2:].strip() if line.startswith("- ") else line for line in lines)
736def split_frontmatter(text: str) -> tuple[list[str], str]:
737 lines = text.splitlines()
738 if not lines or lines[0].strip() != "---":
739 return [], text
740 end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
741 if end is None:
742 return [], text
743 return lines[1:end], "\n".join(lines[end + 1 :])
746def frontmatter_list(meta: list[str], key: str) -> list[str]:
747 """`key: [a, b]`, or `key:` followed by `- a` lines. Comments after the value are dropped."""
748 items: list[str] = []
749 for index, line in enumerate(meta):
750 if not re.match(rf"\A{re.escape(key)}\s*:", line):
751 continue
752 value = line.split(":", 1)[1].split("#", 1)[0].strip()
753 if value.startswith("["):
754 items = [item.strip().strip("'\"") for item in value.strip("[]").split(",")]
755 else:
756 for following in meta[index + 1 :]:
757 if not following.lstrip().startswith("- "):
758 break
759 items.append(following.lstrip()[2:].split("#", 1)[0].strip().strip("'\""))
760 break
761 return [item for item in items if item]
764def bullets(section: str) -> list[str]:
765 """Bullet lines as plain text, a wrapped line joined to its bullet, without placeholder comments."""
766 items: list[str] = []
767 open_item = False
768 for line in section.splitlines():
769 stripped = line.strip()
770 if stripped.startswith("- "):
771 items.append(stripped[2:].strip())
772 open_item = True
773 elif open_item and stripped and not stripped.startswith("<!--"):
774 items[-1] = f"{items[-1]} {stripped}"
775 else:
776 open_item = False
777 return items
780def split_sections(text: str) -> tuple[str, list[tuple[str, str]]]:
781 """(the text before the first `## `, [(name, body)]) with each body stripped of surrounding blank lines."""
782 parts = HEADING.split(text)
783 return parts[0], [(parts[i], parts[i + 1].strip("\n")) for i in range(1, len(parts), 2)]
786def join_sections(head: str, sections: list[tuple[str, str]]) -> str:
787 out = [head.rstrip("\n")] if head.strip() else []
788 out += [f"## {name}\n\n{body}" if body else f"## {name}" for name, body in sections]
789 return "\n\n".join(out) + "\n"
792def section_body(text: str, name: str) -> str:
793 return next((body for heading, body in split_sections(text)[1] if heading.lower() == name.lower()), "")
796def add_lines(sections: list[tuple[str, str]], name: str, lines: list[str], *, replace: bool = False) -> None:
797 """Append lines to the section, replacing its body when `replace`; a missing section is created in template order."""
798 if not lines:
799 return
800 for index, (heading, body) in enumerate(sections):
801 if heading.lower() == name.lower():
802 if replace or not body:
803 body = "\n".join(lines)
804 else:
805 # A list after prose needs a blank line to be one.
806 separator = "\n" if body.splitlines()[-1].startswith("- ") else "\n\n"
807 body = f"{body}{separator}{chr(10).join(lines)}"
808 sections[index] = (heading, body)
809 return
810 rank = SECTION_ORDER.index(name)
811 position = next(
812 (
813 i
814 for i, (heading, _b) in enumerate(sections)
815 if heading in SECTION_ORDER and SECTION_ORDER.index(heading) > rank
816 ),
817 len(sections),
818 )
819 sections.insert(position, (name, "\n".join(lines)))
822def folded_ticket_text(
823 text: str,
824 ticket: Path,
825 spec: dict,
826 requirements: list[str],
827 references: list[str],
828 sources: list[str],
829 companions: list[str],
830 replace_requirements: bool,
831) -> str:
832 meta, body = split_frontmatter(text)
833 head, sections = split_sections(body)
834 add_lines(sections, "Requirements", requirements, replace=replace_requirements)
835 add_lines(sections, "Constraints", [f"- {line}" for line in spec["constraints"]])
836 add_lines(sections, "Boundaries", [f"- Non-goal: {line}" for line in spec["non_goals"]])
837 existing = section_body(body, "References")
838 self_reference = re.compile(rf"(^|/){re.escape(ticket.name)}(,|\s|$)")
839 kept = [
840 line
841 for line in existing.splitlines()
842 if not (line.strip().startswith("- ") and self_reference.search(line.split(" — ", 1)[-1]))
843 ]
844 kept += [line for line in references if line not in existing]
845 index = next((i for i, (heading, _b) in enumerate(sections) if heading.lower() == "references"), None)
846 if index is None:
847 add_lines(sections, "References", kept)
848 else:
849 sections[index] = (sections[index][0], "\n".join(kept))
850 add_lines(
851 sections,
852 "Notes",
853 [f"- Assumption: {line}" for line in spec["assumptions"]]
854 + [f"- Open question: {line}" for line in spec["questions"]],
855 )
856 meta = with_list(meta, "sources", sources)
857 meta = with_list(meta, "companions", companions)
858 if any(re.match(r"\Atype\s*:\s*initiative\b", line) for line in meta):
859 meta = [line for line in meta if not re.match(r"\Acovers\s*:", line)]
860 front = "---\n" + "\n".join(meta) + "\n---\n" if meta else ""
861 return front + join_sections(head, sections)
864def with_list(meta: list[str], key: str, items: list[str]) -> list[str]:
865 """The frontmatter with `key` holding its existing items plus `items`, the line added after `sources:` or at the end."""
866 if not items:
867 return meta
868 index = next((i for i, line in enumerate(meta) if re.match(rf"\A{key}\s*:", line)), None)
869 merged = list(dict.fromkeys((frontmatter_list(meta, key) if index is not None else []) + items))
870 line = f"{key}: [{', '.join(merged)}]"
871 if index is not None:
872 # A block list's `- item` lines go with the key line they continue.
873 end = index + 1
874 while end < len(meta) and meta[end].lstrip().startswith("- "):
875 end += 1
876 return [*meta[:index], line, *meta[end:]]
877 after = next((i for i, line in enumerate(meta) if re.match(r"\Asources\s*:", line)), None)
878 if after is None or key == "sources":
879 return [*meta, line]
880 return [*meta[: after + 1], line, *meta[after + 1 :]]
883def story_text(
884 spec: dict,
885 home: Path,
886 slug: str,
887 requirements: list[str],
888 references: list[str],
889 sources: list[str],
890 companions: list[str],
891) -> str:
892 used = [
893 int(match.group(1))
894 for path in sorted(home.glob("*.md"))
895 if (match := re.search(r"(?m)^id:\s*(\d+)\s*$", path.read_text(encoding="utf-8")))
896 ]
897 # `next_id` at the top of the folder's tickets.toml keeps a moved-out id from being reused.
898 floor = 0
899 breakdown = home / "tickets.toml"
900 if breakdown.is_file():
901 try:
902 floor = tomllib.loads(breakdown.read_text(encoding="utf-8")).get("next_id", 0)
903 except (OSError, UnicodeError, tomllib.TOMLDecodeError):
904 floor = 0
905 if isinstance(floor, bool) or not isinstance(floor, int):
906 floor = 0
907 title = spec["title"] or slug.replace("-", " ")
908 meta = [
909 f"id: {max(max(used, default=0) + 1, floor)}",
910 "type: story",
911 f'title: "{title.replace(chr(34), chr(39))}"',
912 "parent: none",
913 "covers: []",
914 f"sources: [{', '.join(sources)}]",
915 f"companions: [{', '.join(companions)}]",
916 "after: []",
917 "refined: false",
918 "hitl: false",
919 "risk: low",
920 ]
921 sections: list[tuple[str, str]] = [("Description", spec["why"] or title)]
922 add_lines(sections, "Requirements", requirements)
923 add_lines(sections, "Constraints", [f"- {line}" for line in spec["constraints"]])
924 add_lines(sections, "Boundaries", [f"- Non-goal: {line}" for line in spec["non_goals"]])
925 add_lines(sections, "References", references)
926 add_lines(
927 sections,
928 "Notes",
929 [f"- Assumption: {line}" for line in spec["assumptions"]]
930 + [f"- Open question: {line}" for line in spec["questions"]],
931 )
932 return "---\n" + "\n".join(meta) + "\n---\n\n" + join_sections(f"# {title}", sections)
935def companion_references(
936 items: list[str], folder: Path, home: Path, project_root: Path, store: Path
937) -> tuple[list[str], list[str]]:
938 """A References line per companion: `companion — <name>` for one moving beside the ticket, else typed by its
939 file name with the path rebased to the ticket's folder. A path that resolves nowhere is kept as written."""
940 lines: list[str] = []
941 unresolved: list[str] = []
942 for item in items:
943 target = resolve_companion(item, folder, project_root, store)
944 if target is None:
945 unresolved.append(item)
946 lines.append(f"- companion — {item}")
947 continue
948 if target.is_relative_to(folder.resolve()):
949 lines.append(f"- companion — {target.relative_to(folder.resolve()).as_posix()}")
950 continue
951 kind = COMPANION_TYPES.get(target.name) or re.split(r"[-_.]", target.stem, maxsplit=1)[0].lower() or "companion"
952 lines.append(f"- {kind} — {rebase(item, folder, home, project_root, store)}")
953 return lines, unresolved
956def companion_path(item: str, folder: Path, home: Path, project_root: Path, store: Path) -> str:
957 """The companion as `companions:` lists it: a path from the project root when it resolves there, else as rebased."""
958 target = resolve_companion(item, folder, project_root, store)
959 if target is None:
960 return item
961 target = moved_path(target, folder, home)
962 if target.is_relative_to(project_root.resolve()):
963 return target.relative_to(project_root.resolve()).as_posix()
964 return rebase(item, folder, home, project_root, store)
967def resolve_companion(item: str, folder: Path, project_root: Path, store: Path) -> Path | None:
968 for base in (folder, project_root, store):
969 candidate = (base / item).resolve()
970 if candidate.exists():
971 return candidate
972 return None
975def moved_path(target: Path, folder: Path, home: Path) -> Path:
976 """Where a file in the spec folder, at any depth, sits once the fold moves the folder's contents beside the ticket."""
977 if target.is_relative_to(folder.resolve()):
978 return home.resolve() / target.relative_to(folder.resolve())
979 return target
982def rebase(item: str, folder: Path, home: Path, project_root: Path, store: Path) -> str:
983 """The path as the ticket's folder sees it: a path from beside it, a path from the project root, else relative."""
984 target = resolve_companion(item, folder, project_root, store)
985 if target is None:
986 return item
987 target = moved_path(target, folder, home)
988 if target.is_relative_to(home.resolve()):
989 return target.relative_to(home.resolve()).as_posix()
990 if target.is_relative_to(project_root.resolve()):
991 return target.relative_to(project_root.resolve()).as_posix()
992 return Path(os.path.relpath(target, home.resolve())).as_posix()
995def memlog_body(path: Path) -> str:
996 _meta, body = split_frontmatter(path.read_text(encoding="utf-8"))
997 body = body.strip("\n")
998 return body + "\n" if body else ""
1001def append_text(path: Path, text: str) -> None:
1002 """Add text at the end of the file, on its own line. The memlog is append-only, so nothing above is touched."""
1003 if not text:
1004 return
1005 with path.open("r+", encoding="utf-8") as handle:
1006 current = handle.read()
1007 handle.write(("" if not current or current.endswith("\n") else "\n") + text)
1010def report(project_root: Path, skill_root: Path, *, roots: tuple[Path, ...] = ()) -> dict[str, object]:
1011 """The traces `--clean` would remove. Reads only."""
1012 reject_unusable_bmad(project_root)
1013 installation = discover_installation(skill_root, roots)
1014 retirement = retirement_report(
1015 project_root, skill_root, installation, retired_skills(installation.modules, installation.modules)
1016 )
1017 return {"mode": "report", **traces_json(project_root, installation, retirement)}
1020def main(argv: list[str] | None = None) -> int:
1021 parser = argparse.ArgumentParser(description="Report or remove the v6 traces in _bmad that v7 does not read.")
1022 parser.add_argument("--project-root", type=Path, required=True)
1023 parser.add_argument("--skill", type=Path, required=True)
1024 parser.add_argument("--root", type=Path, action="append", default=[], help="an active skills folder, repeated")
1025 parser.add_argument("--clean", action="store_true", help="back up _bmad, then remove the traces")
1026 parser.add_argument(
1027 "--fold-specs", type=Path, metavar="FOLDER", help="fold the pre-release spec folders under this output folder"
1028 )
1029 args = parser.parse_args(argv)
1030 if args.fold_specs and args.clean:
1031 parser.error("--fold-specs cannot be combined with --clean")
1032 project_root = args.project_root.resolve()
1033 skill_root = args.skill.resolve()
1034 roots = tuple(root.resolve() for root in args.root)
1035 if args.fold_specs:
1036 _setup.print_json(fold_specs(project_root, args.fold_specs.resolve()))
1037 return 0
1038 run = clean if args.clean else report
1039 _setup.print_json(run(project_root, skill_root, roots=roots))
1040 return 0
1043if __name__ == "__main__":
1044 if sys.platform == "win32":
1045 # Piped output on Windows defaults to a legacy code page, not UTF-8.
1046 sys.stdout.reconfigure(encoding="utf-8")
1047 sys.stderr.reconfigure(encoding="utf-8")
1048 try:
1049 raise SystemExit(main())
1050 except Exception as error:
1051 sys.stderr.write(f"error: {error}\n")
1052 raise SystemExit(1) from None