Skills › BMAD Method › bmad › knowledge.py
knowledge.py
462 lines · skills/bmad/scripts/knowledge.py at bd70c64
1#!/usr/bin/env python3
2# /// script
3# requires-python = ">=3.11"
4# ///
5"""Report the knowledge documents the installed modules offer.
7A folder whose `bmod.toml` has a `[bmod]` table is a module record, whatever
8the folder is called. The record names the module's skills and holds each
9knowledge document once. `help/help.md` in the record's folder covers every
10skill of the module and needs no entry. A `[[bmod.knowledge]]` entry adds a further document
11and says which of the module's skills it covers: `"*"` or no `skills` key means
12all of them, a list means the named ones.
14Every other `help/*.md` is a topic: detail that `help/help.md` points to and a
15reader opens only when a question needs it. Topics are listed with their
16file path and never with their text.
18A `migration-<n>.toml` file in the record's folder is a migration the
19module ships: the rules for moving a project from one major version of the
20module to the next. `<n>` is a whole number that sets the order in which a
21module's migrations are listed, checked and run. One is listed only when its
22`[migration]` table names the record's `module` and has `from`, `to`,
23`title`, `summary`, `detect`, `guide`, and a `checklist`; the listing carries
24`from`, `to`, `title` and the file path, never the text. A `[migration]`
25table in a file with any other name, and two files of one module with the
26same number, are problems. `bmad migrate` reads the file.
28A file this script cannot use becomes an entry in `problems`, never an
29exception.
31Usage:
32 uv run knowledge.py --root .claude/skills [--root ...] [--content]
33"""
35from __future__ import annotations
37import argparse
38import json
39import re
40import stat
41import sys
42import tomllib
43from pathlib import Path, PurePosixPath
44from typing import NamedTuple
46sys.dont_write_bytecode = True
48MANIFEST_NAME = "bmod.toml"
49TOPICS_DIR = "help"
50HELP_NAME = f"{TOPICS_DIR}/help.md"
51ROSTER_NAME = "roster.toml"
52RETIRED_NAME = "retired.toml"
53MIGRATION_TABLE = "migration"
54MIGRATION_FIELDS = ("module", "from", "to", "title", "summary", "detect", "guide")
55MIGRATION_NAME = re.compile(r"migration-([0-9]+)\.toml")
56READ_LIMIT = 1024 * 1024
59class Module(NamedTuple):
60 code: str
61 folder: Path
62 table: dict[str, object]
63 skills: list[str]
66class Scan(NamedTuple):
67 folders: dict[str, Path]
68 modules: list[Module]
69 skills: list[dict[str, object]]
70 problems: list[dict[str, object]]
73def main(argv: list[str] | None = None) -> int:
74 parser = argparse.ArgumentParser(description="Report the knowledge documents the installed modules offer.")
75 parser.add_argument("--root", type=Path, action="append", required=True, help="a skills root to scan")
76 parser.add_argument("--content", action="store_true", help="include each document's text")
77 args = parser.parse_args(argv)
78 print(json.dumps(collect(args.root, include_content=args.content), ensure_ascii=False))
79 return 0
82def collect(roots: list[Path], *, include_content: bool = False) -> dict[str, object]:
83 found = scan(roots)
84 problems = found.problems
85 documents: dict[tuple[str, str], dict[str, object]] = {}
87 for module in found.modules:
88 entries = module.table.get("knowledge", [])
89 if not isinstance(entries, list):
90 problems.append(
91 {"kind": "knowledge", "skill": module.folder.name, "problem": "[bmod] 'knowledge' is not a list"}
92 )
93 continue
94 if (module.folder / HELP_NAME).exists():
95 entries = [{"path": HELP_NAME}, *entries]
96 for entry in entries:
97 record_document(documents, problems, found.folders, module, entry, include_content=include_content)
99 topics = [
100 topic
101 for module in found.modules
102 for topic in module_topics(module, problems)
103 if (topic["module"], topic["path"]) not in documents
104 ]
106 migrations = [migration for module in found.modules for migration in module_migrations(module, problems)]
108 return {
109 "roots": [str(root) for root in roots],
110 "skills": sorted(found.skills, key=lambda item: str(item["skill"])),
111 "documents": sorted(documents.values(), key=lambda item: (str(item["module"]), str(item["path"]))),
112 "topics": sorted(topics, key=lambda item: (str(item["module"]), str(item["path"]))),
113 "migrations": sorted(migrations, key=lambda item: (str(item["module"]), migration_number(str(item["path"])))),
114 "problems": problems,
115 }
118def scan(roots: list[Path]) -> Scan:
119 """Find every module record and every module skill in the roots."""
120 folders: dict[str, Path] = {}
121 modules: list[Module] = []
122 problems: list[dict[str, object]] = []
123 record_codes: dict[str, str] = {}
124 pending: list[tuple[Path, Path, dict[str, object], bool]] = []
126 for root in roots:
127 try:
128 found = sorted(path for path in root.iterdir() if path.is_dir())
129 except OSError as error:
130 problems.append({"kind": "root", "root": str(root), "problem": f"cannot read root {root}: {error}"})
131 continue
132 for folder in found:
133 # The first root wins a folder name outright: a project copy shadows a
134 # user copy even when the project copy carries no bmod.toml.
135 if folder.name in folders:
136 continue
137 folders[folder.name] = folder
138 manifest = folder / MANIFEST_NAME
139 try:
140 if not manifest.is_file():
141 continue
142 data = tomllib.loads(manifest.read_text(encoding="utf-8"))
143 except (OSError, UnicodeError, tomllib.TOMLDecodeError) as error:
144 problems.append(manifest_problem(folder, f"cannot use {manifest}: {error}"))
145 continue
146 if "bmod" not in data and "skill" not in data:
147 problems.append(manifest_problem(folder, f"{manifest} has neither a [bmod] nor a [skill] table"))
148 continue
150 skill = data.get("skill")
151 if "skill" in data and not isinstance(skill, dict):
152 problems.append(manifest_problem(folder, f"{manifest}: 'skill' is not a table"))
153 skill = None
154 if "bmod" in data:
155 module = read_record(folder, data["bmod"], skill is not None, problems)
156 if module is not None:
157 record_codes[folder.name] = module.code
158 first = next((other for other in modules if other.code.casefold() == module.code.casefold()), None)
159 if first is None:
160 modules.append(module)
161 else:
162 problems.append(
163 {
164 "kind": "module",
165 "skill": folder.name,
166 "problem": f"{folder.name}: module {module.code!r} is already recorded by "
167 f"{first.folder.name}; {first.folder.name} is used",
168 }
169 )
170 if skill is not None:
171 pending.append((root, folder, skill, "bmod" in data))
173 skills = [
174 resolve_skill(root, folder, table, own_record, record_codes) for root, folder, table, own_record in pending
175 ]
176 problems.extend(absent_records(skills, folders))
177 for entry in skills:
178 entry.pop("source")
179 return Scan(folders, modules, skills, problems)
182def manifest_problem(folder: Path, problem: str) -> dict[str, object]:
183 return {"kind": "manifest", "skill": folder.name, "manifest": str(folder / MANIFEST_NAME), "problem": problem}
186def read_record(folder: Path, table: object, has_skill: bool, problems: list[dict[str, object]]) -> Module | None:
187 manifest = folder / MANIFEST_NAME
188 if not isinstance(table, dict):
189 problems.append(manifest_problem(folder, f"{manifest}: 'bmod' is not a table"))
190 return None
191 code = table.get("code")
192 if not isinstance(code, str) or not code:
193 problems.append(manifest_problem(folder, f"{manifest}: [bmod] has no usable 'code'"))
194 return None
195 listed = table.get("skills")
196 if listed is None:
197 # A record that is also a skill, with no list, is its own one member.
198 members = [folder.name] if has_skill else []
199 elif isinstance(listed, list) and all(isinstance(name, str) and name for name in listed):
200 members = list(dict.fromkeys(listed))
201 else:
202 problems.append(manifest_problem(folder, f"{manifest}: [bmod] 'skills' is not a list of skill names"))
203 members = []
204 return Module(code, folder, table, members)
207def resolve_skill(
208 root: Path, folder: Path, table: dict[str, object], own_record: bool, record_codes: dict[str, str]
209) -> dict[str, object]:
210 bmod = folder.name if own_record else table.get("bmod")
211 if not isinstance(bmod, str) or not bmod:
212 bmod = None
213 return {
214 "skill": folder.name,
215 "module": record_codes.get(bmod) if bmod else None,
216 "bmod": bmod,
217 "root": str(root),
218 "source": table.get("source"),
219 }
222def absent_records(skills: list[dict[str, object]], folders: dict[str, Path]) -> list[dict[str, object]]:
223 """One problem per module record that skills name and no root holds."""
224 problems: list[dict[str, object]] = []
225 by_bmod: dict[str, list[dict[str, object]]] = {}
226 for entry in skills:
227 if entry["module"] is not None:
228 continue
229 if entry["bmod"] is None:
230 problems.append(
231 {
232 "kind": "manifest",
233 "skill": entry["skill"],
234 "problem": f"{entry['skill']}: [skill] does not name its module record under 'bmod'",
235 }
236 )
237 continue
238 by_bmod.setdefault(str(entry["bmod"]), []).append(entry)
239 for bmod, entries in sorted(by_bmod.items()):
240 names = sorted(str(entry["skill"]) for entry in entries)
241 state = "has no usable module record" if bmod in folders else "is not installed"
242 problem: dict[str, object] = {
243 "kind": "module",
244 "bmod": bmod,
245 "skills": names,
246 "problem": f"module record {bmod} {state}; it is named by {', '.join(names)}",
247 }
248 command = None if bmod in folders else install_command(entries[0]["source"], bmod)
249 if command:
250 problem["install"] = command
251 problem["problem"] = f"{problem['problem']}; install it with `{command}`"
252 problems.append(problem)
253 return problems
256def install_command(source: object, skill: str) -> str | None:
257 if not isinstance(source, str) or not source.startswith("github:"):
258 return None
259 parts = source.removeprefix("github:").split("/")
260 if len(parts) < 2 or not all(parts[:2]):
261 return None
262 return f"npx skills add {parts[0]}/{parts[1]} --skill {skill}"
265def record_document(
266 documents: dict[tuple[str, str], dict[str, object]],
267 problems: list[dict[str, object]],
268 folders: dict[str, Path],
269 module: Module,
270 entry: object,
271 *,
272 include_content: bool,
273) -> None:
274 folder = module.folder
275 name = entry.get("path") if isinstance(entry, dict) else None
276 if not isinstance(name, str):
277 problems.append(
278 {"kind": "knowledge", "skill": folder.name, "problem": f"knowledge entry {entry!r} has no path"}
279 )
280 return
281 relative = safe_skill_relative(name)
282 if relative is None:
283 problems.append({"kind": "knowledge", "skill": folder.name, "problem": f"knowledge names unsafe path {name!r}"})
284 return
285 covered = entry.get("skills", "*")
286 if covered == "*":
287 skills = list(module.skills)
288 elif isinstance(covered, list) and all(isinstance(skill, str) and skill for skill in covered):
289 skills = list(dict.fromkeys(covered))
290 else:
291 problems.append(
292 {
293 "kind": "knowledge",
294 "skill": folder.name,
295 "problem": f"knowledge entry {name!r}: 'skills' is neither \"*\" nor a list of skill names",
296 }
297 )
298 return
299 key = (module.code, relative.as_posix())
300 if key in documents:
301 problems.append({"kind": "knowledge", "skill": folder.name, "problem": f"knowledge names {name!r} twice"})
302 return
304 path = folder.joinpath(*relative.parts)
305 try:
306 raw = read_document(path, folder)
307 except (OSError, ValueError) as error:
308 problems.append(
309 {"kind": "document", "skill": folder.name, "document": str(path), "problem": f"{path}: {error}"}
310 )
311 return
312 try:
313 text = raw.decode("utf-8")
314 except UnicodeError as error:
315 problems.append(
316 {
317 "kind": "document",
318 "skill": folder.name,
319 "document": str(path),
320 "problem": f"{path}: not valid UTF-8, so it is not a knowledge document: {error}",
321 }
322 )
323 return
325 document: dict[str, object] = {
326 "module": module.code,
327 "path": relative.as_posix(),
328 "skills": skills,
329 "installed_skills": [skill for skill in skills if skill in folders],
330 "reported_from": folder.name,
331 }
332 if include_content:
333 document["content"] = text
334 documents[key] = document
337def module_topics(module: Module, problems: list[dict[str, object]]) -> list[dict[str, object]]:
338 folder = module.folder
339 try:
340 found = sorted(path for path in (folder / TOPICS_DIR).glob("*.md") if path != folder / HELP_NAME)
341 except OSError:
342 return []
343 topics: list[dict[str, object]] = []
344 for path in found:
345 try:
346 read_document(path, folder).decode("utf-8")
347 except (OSError, ValueError) as error:
348 problems.append(
349 {"kind": "document", "skill": folder.name, "document": str(path), "problem": f"{path}: {error}"}
350 )
351 continue
352 topics.append(
353 {"module": module.code, "topic": path.stem, "path": f"{TOPICS_DIR}/{path.name}", "file": str(path)}
354 )
355 return topics
358def module_migrations(module: Module, problems: list[dict[str, object]]) -> list[dict[str, object]]:
359 """The migrations a record ships: its `migration-<n>.toml` files, in number order."""
360 folder = module.folder
361 try:
362 found = sorted(
363 path for path in folder.glob("*.toml") if path.name not in (MANIFEST_NAME, ROSTER_NAME, RETIRED_NAME)
364 )
365 except OSError:
366 return []
367 migrations: list[dict[str, object]] = []
368 for path in found:
369 try:
370 data = tomllib.loads(read_document(path, folder).decode("utf-8"))
371 except (OSError, ValueError) as error:
372 problems.append(migration_problem(folder, path, str(error)))
373 continue
374 if MIGRATION_TABLE not in data:
375 continue
376 table = data[MIGRATION_TABLE]
377 if not isinstance(table, dict):
378 problems.append(migration_problem(folder, path, "'migration' is not a table"))
379 continue
380 fields = {name: table.get(name) for name in MIGRATION_FIELDS}
381 missing = [name for name, value in fields.items() if not isinstance(value, str) or not value.strip()]
382 checklist = table.get("checklist")
383 if (
384 not isinstance(checklist, list)
385 or not checklist
386 or not all(isinstance(item, str) and item.strip() for item in checklist)
387 ):
388 missing.append("checklist")
389 if missing:
390 problems.append(migration_problem(folder, path, f"[migration] needs non-empty {', '.join(missing)}"))
391 continue
392 if fields["module"] != module.code:
393 problems.append(
394 migration_problem(
395 folder, path, f"[migration] module {fields['module']!r} is not this record's {module.code!r}"
396 )
397 )
398 continue
399 if migration_number(path.name) is None:
400 problems.append(migration_problem(folder, path, "a migration file must be named migration-<n>.toml"))
401 continue
402 listed = {name: fields[name] for name in ("from", "to", "title")}
403 migrations.append({"module": module.code, "path": path.name, "file": str(path), **listed})
404 numbers = [migration_number(str(item["path"])) for item in migrations]
405 duplicated = {number for number in numbers if numbers.count(number) > 1}
406 for item in migrations:
407 if migration_number(str(item["path"])) in duplicated:
408 problems.append(
409 migration_problem(
410 folder, Path(str(item["file"])), "another migration of this module has the same number"
411 )
412 )
413 kept = [item for item in migrations if migration_number(str(item["path"])) not in duplicated]
414 return sorted(kept, key=lambda item: migration_number(str(item["path"])))
417def migration_number(name: str) -> int | None:
418 match = MIGRATION_NAME.fullmatch(name)
419 return int(match.group(1)) if match else None
422def migration_problem(folder: Path, path: Path, problem: str) -> dict[str, object]:
423 return {"kind": "migration", "skill": folder.name, "document": str(path), "problem": f"{path}: {problem}"}
426def read_document(path: Path, folder: Path) -> bytes:
427 """Read a knowledge document, refusing anything that is not a plain file inside the folder."""
428 resolved = path.resolve()
429 if not resolved.is_relative_to(folder.resolve()):
430 raise ValueError("resolves outside the skill folder")
431 status = resolved.stat()
432 if not stat.S_ISREG(status.st_mode):
433 raise ValueError("is not a regular file")
434 with resolved.open("rb") as handle:
435 raw = handle.read(READ_LIMIT + 1)
436 if len(raw) > READ_LIMIT:
437 raise ValueError(f"is larger than {READ_LIMIT} bytes")
438 return raw
441def safe_skill_relative(entry: str) -> PurePosixPath | None:
442 """A bmod.toml path that cannot escape the skill folder, or None if it can.
444 Mirrors safe_skill_relative in setup.py. A URL parses as an ordinary
445 relative path and a Windows drive prefix makes a later join discard the
446 skill folder, so both are refused by name. pathlib drops "." components
447 itself, so only ".." and an empty final component need checking.
448 """
449 if not entry or "://" in entry or "\\" in entry or ":" in entry:
450 return None
451 relative = PurePosixPath(entry)
452 if relative.is_absolute() or ".." in relative.parts or not relative.name:
453 return None
454 return relative
457if __name__ == "__main__":
458 if sys.platform == "win32":
459 # Piped output on Windows defaults to a legacy code page, not UTF-8.
460 sys.stdout.reconfigure(encoding="utf-8")
461 sys.stderr.reconfigure(encoding="utf-8")
462 sys.exit(main())