Skills › BMAD Method › bmad-deep-recon › recon_kit.py
recon_kit.py
663 lines · skills/bmad-deep-recon/scripts/recon_kit.py at bd70c64
1#!/usr/bin/env python3
2# /// script
3# requires-python = ">=3.11"
4# ///
5"""recon_kit — deterministic helpers for bmad-deep-recon.
7The mechanical half of the research workflow: everything here is exact,
8repeatable work the LLM should never re-derive by hand. All subcommands
9print one JSON object to stdout; diagnostics go to stderr. Exit codes:
100 = pass, 1 = findings that need attention, 2 = usage/parse error.
12Subcommands:
13 citations RESEARCH_MD
14 Cross-check inline [n] markers against the source-appendix table:
15 dangling markers (no appendix row) and orphaned rows (never cited).
16 tally MEMLOG_MD
17 Count memlog entries by type, and claim entries by status.
18 Claim lines carry `status=<word>` and optionally `ref=[n]`; for a
19 given ref the LAST status wins, so status changes are appends.
20 staleness CLAIMS_JSON --windows JSON [--today YYYY-MM-DD]
21 Given claims [{claim, class, pub_date}] and a months-per-class map
22 (e.g. '{"size/growth": 18, "pricing": 3}'), compute each claim's
23 re-check date, flag stale ones, and report the earliest re-check.
24 slug TOPIC --type TYPE [--pattern P] [--date YYYY-MM-DD]
25 Expand the run-folder pattern deterministically so the same topic
26 always lands in the same folder across draft -> process -> refresh.
27 escape-sources RESEARCH_MD
28 Emit the source-appendix table as HTML with every cell escaped and
29 only validated http(s) URLs turned into links, for the briefing.
30 anchors RESEARCH_MD --repo PATH [--at REV]
31 Re-find each `path:line (symbol) @ sha` and `sha "subject" #pr`
32 citation in git. A code claim is anchored, moved, incomplete,
33 missing or unknown-commit; a history claim is anchored,
34 subject-mismatch, pr-mismatch or unknown-commit.
35 moved means the file changed between the cited commit and --at (default HEAD), not that the line shifted; head_line says where the cited text now sits, or null when it is gone.
36 incomplete means the symbol is empty; pr-mismatch means the subject ends in (#n) and the citation names no PR or another one; unparsed lists citation-shaped text no claim matched. All three fail the run. symbol_found says whether the symbol's last identifier is in the cited lines or the 80 lines above them; it never fails the run.
37"""
39from __future__ import annotations
41import argparse
42import calendar
43import html
44import json
45import os
46import posixpath
47import re
48import subprocess
49import sys
50import unicodedata
51from collections import Counter
52from datetime import date, datetime
53from pathlib import Path
54from urllib.parse import urlparse
56MARKER_RE = re.compile(r"\[(\d+)\](?!\()") # [3] but not a [3](url) link
57# URLs may hold one level of balanced parentheses, as Wikipedia's often do.
58MD_LINK_RE = re.compile(r"\[([^\]]*)\]\(((?:[^\s()]|\([^\s()]*\))+)\)")
59BARE_URL_RE = re.compile(r"https?://(?:[^\s()|\]]|\([^\s()]*\))+")
60ROW_ID_RE = re.compile(r"\[(\d+)\]") # appendix row id: [n], never a bare number
61FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})") # any indent: fences nest in list items
64def out(payload: dict, exit_code: int) -> int:
65 print(json.dumps(payload, indent=2, ensure_ascii=False, default=str))
66 return exit_code
69def read_text(path_arg: str) -> str:
70 if path_arg == "-":
71 # Decode the bytes ourselves: the locale's encoding may not be UTF-8.
72 buffer = getattr(sys.stdin, "buffer", None)
73 return buffer.read().decode("utf-8") if buffer is not None else sys.stdin.read()
74 return Path(path_arg).read_text(encoding="utf-8")
77def strip_fences(text: str) -> str:
78 """Blank out fenced code blocks so their contents never count as markers or rows.
80 Fences pair CommonMark-style: a block closes only on a line of the opener's
81 character, at least as long, with nothing after it.
82 """
83 lines, open_fence = [], None # (char, length) while inside a fenced block
84 for ln in text.splitlines():
85 fence = FENCE_RE.match(ln)
86 if open_fence is None:
87 if fence:
88 open_fence = (fence.group(1)[0], len(fence.group(1)))
89 lines.append("" if fence else ln)
90 continue
91 marker = fence.group(1) if fence else ""
92 if marker[:1] == open_fence[0] and len(marker) >= open_fence[1] and ln.strip() == marker:
93 open_fence = None
94 lines.append("")
95 return "\n".join(lines)
98def table_cells(line: str) -> list[str]:
99 return [c.strip() for c in line.strip().strip("|").split("|")]
102def appendix_rows(text: str) -> dict[int, list[str]]:
103 """Source-appendix rows: markdown table rows whose first cell is [n]."""
104 rows: dict[int, list[str]] = {}
105 for ln in text.splitlines():
106 stripped = ln.strip()
107 if not stripped.startswith("|"):
108 continue
109 cells = table_cells(stripped)
110 if not cells or len(cells) < 2:
111 continue
112 m = ROW_ID_RE.fullmatch(cells[0])
113 if m:
114 rows[int(m.group(1))] = cells
115 return rows
118# --- citations ---------------------------------------------------------------
121def cmd_citations(args) -> int:
122 text = strip_fences(read_text(args.file))
123 rows = appendix_rows(text)
124 markers: set[int] = set()
125 for ln in text.splitlines():
126 stripped = ln.strip()
127 if stripped.startswith("|"):
128 cells = table_cells(stripped)
129 if cells and ROW_ID_RE.fullmatch(cells[0]):
130 continue # an appendix row is not a citation of itself
131 markers.update(int(n) for n in MARKER_RE.findall(ln))
132 dangling = sorted(markers - set(rows))
133 orphaned = sorted(set(rows) - markers)
134 ok = not dangling and not orphaned
135 return out(
136 {
137 "markers": sorted(markers),
138 "appendix_rows": sorted(rows),
139 "dangling_markers": dangling,
140 "orphaned_rows": orphaned,
141 "ok": ok,
142 },
143 0 if ok else 1,
144 )
147# --- tally -------------------------------------------------------------------
149ENTRY_RE = re.compile(r"^- (?:\(([\w-]+)(?: by [^)]*)?\)\s*)?(.*)$")
152def cmd_tally(args) -> int:
153 text = read_text(args.file)
154 body = text.split("---", 2)[-1] if text.startswith("---") else text
155 by_type: dict[str, int] = {}
156 by_ref: dict[int, str] = {}
157 unref_status: dict[str, int] = {}
158 entries = 0
159 for ln in body.splitlines():
160 m = ENTRY_RE.match(ln)
161 if not m or not ln.startswith("- "):
162 continue
163 entries += 1
164 etype = m.group(1) or "note"
165 by_type[etype] = by_type.get(etype, 0) + 1
166 if etype == "claim":
167 status_m = re.search(r"status=([\w-]+)", m.group(2))
168 status = status_m.group(1) if status_m else "unknown"
169 ref_m = re.search(r"ref=\[?(\d+)\]?", m.group(2))
170 if ref_m:
171 by_ref[int(ref_m.group(1))] = status # last status wins per ref
172 else:
173 unref_status[status] = unref_status.get(status, 0) + 1
174 claims: dict[str, int] = dict(unref_status)
175 for status in by_ref.values():
176 claims[status] = claims.get(status, 0) + 1
177 return out(
178 {
179 "entries": entries,
180 "by_type": dict(sorted(by_type.items())),
181 "claims": dict(sorted(claims.items())),
182 "claims_total": sum(claims.values()),
183 },
184 0,
185 )
188# --- staleness ---------------------------------------------------------------
191def parse_date(raw: str) -> date:
192 raw = raw.strip()
193 for fmt in ("%Y-%m-%d", "%Y-%m", "%Y"):
194 try:
195 return datetime.strptime(raw, fmt).date()
196 except ValueError:
197 continue
198 raise ValueError(f"unparseable date: {raw!r} (want YYYY[-MM[-DD]])")
201def add_months(d: date, months: int) -> date:
202 total = d.month - 1 + months
203 year, month = d.year + total // 12, total % 12 + 1
204 return date(year, month, min(d.day, calendar.monthrange(year, month)[1]))
207def cmd_staleness(args) -> int:
208 try:
209 payload = json.loads(read_text(args.file))
210 raw_windows = json.loads(args.windows)
211 if not isinstance(raw_windows, dict):
212 raise ValueError("--windows must be a JSON object of class -> months")
213 windows = {k.lower(): int(v) for k, v in raw_windows.items()}
214 today = parse_date(args.today) if args.today else date.today()
215 except (TypeError, ValueError) as e:
216 print(f"error: {e}", file=sys.stderr)
217 return 2
218 claims = payload.get("claims") if isinstance(payload, dict) else payload
219 if not isinstance(claims, list) or not all(isinstance(c, dict) for c in claims):
220 print('error: claims must be a JSON array of objects, or {"claims": [...]}', file=sys.stderr)
221 return 2
222 results, no_window, stale_count = [], set(), 0
223 earliest: date | None = None
224 for c in claims:
225 cls = str(c.get("class", "")).lower()
226 try:
227 pub = parse_date(str(c["pub_date"]))
228 except (KeyError, ValueError) as e:
229 print(f"error in claim {c!r}: {e}", file=sys.stderr)
230 return 2
231 months = windows.get(cls)
232 if months is None:
233 no_window.add(cls)
234 results.append({**c, "recheck": None, "stale": None})
235 continue
236 recheck = add_months(pub, months)
237 stale = recheck <= today
238 stale_count += stale
239 earliest = recheck if earliest is None or recheck < earliest else earliest
240 results.append({**c, "recheck": recheck.isoformat(), "stale": stale})
241 return out(
242 {
243 "today": today.isoformat(),
244 "claims": results,
245 "stale_count": stale_count,
246 "earliest_recheck": earliest.isoformat() if earliest else None,
247 "no_window_classes": sorted(no_window),
248 },
249 1 if stale_count else 0,
250 )
253# --- slug --------------------------------------------------------------------
256def slugify(text: str, max_len: int = 40) -> str:
257 text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
258 text = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
259 return re.sub(r"-{2,}", "-", text)[:max_len].rstrip("-")
262def cmd_slug(args) -> int:
263 slug = slugify(args.topic)
264 if not slug:
265 print("error: topic slugified to an empty string", file=sys.stderr)
266 return 2
267 folder = (
268 args.pattern.replace("{research_type}", args.type)
269 .replace("{topic_slug}", slug)
270 .replace("{date}", args.date or date.today().isoformat())
271 )
272 return out({"topic_slug": slug, "folder": folder}, 0)
275# --- escape-sources ----------------------------------------------------------
278def safe_url(raw: str) -> str | None:
279 parsed = urlparse(raw)
280 return raw if parsed.scheme in ("http", "https") and parsed.netloc else None
283def cell_html(cell: str, invalid: list[str]) -> str:
284 """Escape a cell; a markdown link or bare URL becomes an <a> only when http(s)."""
285 link = MD_LINK_RE.search(cell)
286 if link:
287 url = safe_url(link.group(2))
288 label = html.escape(link.group(1) or link.group(2))
289 if url:
290 return (
291 html.escape(cell[: link.start()])
292 + f'<a href="{html.escape(url, quote=True)}" target="_blank" rel="noopener">{label}</a>'
293 + html.escape(cell[link.end() :])
294 )
295 invalid.append(link.group(2))
296 return html.escape(cell.replace(link.group(0), link.group(1) or link.group(2)))
297 bare = BARE_URL_RE.search(cell)
298 if bare:
299 url = safe_url(bare.group(0))
300 if url:
301 escaped = html.escape(url, quote=True)
302 return (
303 html.escape(cell[: bare.start()])
304 + f'<a href="{escaped}" target="_blank" rel="noopener">{escaped}</a>'
305 + html.escape(cell[bare.end() :])
306 )
307 invalid.append(bare.group(0))
308 return html.escape(cell)
311def cmd_escape_sources(args) -> int:
312 text = strip_fences(read_text(args.file))
313 rows = appendix_rows(text)
314 if not rows:
315 print("error: no source-appendix table rows found", file=sys.stderr)
316 return 2
317 invalid: list[str] = []
318 body_rows = []
319 for n in sorted(rows):
320 cells = rows[n]
321 tds = "".join(f"<td>{cell_html(c, invalid)}</td>" for c in cells[1:])
322 body_rows.append(f'<tr id="src-{n}"><td>[{n}]</td>{tds}</tr>')
323 table = '<table class="sources"><tbody>' + "".join(body_rows) + "</tbody></table>"
324 return out({"rows": len(rows), "invalid_urls": invalid, "html": table}, 1 if invalid else 0)
327# --- anchors -----------------------------------------------------------------
329# A path starts a token, so a URL is never read as one, and may hold a balanced (group) as Next.js routes do.
330PATH_CHAR = r"(?:[^\s:`'\"()<>|*]|\([^\s:`'\"()<>|*/]*\))"
331CITED_PATH = rf"(?<![\w./:\\)-])({PATH_CHAR}+)"
332# A path may hold spaces, so this takes up to five words before it; spaced_path keeps those the commit's tree has.
333CODE_CLAIM_RE = re.compile(
334 rf"(?<![\w./:\\)-])({PATH_CHAR}+(?: {PATH_CHAR}+){{0,5}})"
335 + r":(\d+)(?:-(\d+))?[ \t]*\(([^()\n]*)\)[ \t]*@[ \t]*([0-9a-f]{7,40})\b"
336)
337# Not after "@ ": that sha ends a code claim, and a quote after it is not a subject.
338# A subject may hold quotes (Revert "x"): it ends at a quote not followed by a word or another quote.
339HISTORY_CLAIM_RE = re.compile(r"(?<![\w./:\\-])(?<!@ )([0-9a-f]{7,40})[ \t]+\"(.+?)\"(?![\w\"])(?:[ \t]+#(\d+))?")
340PR_SUFFIX_RE = re.compile(r"\s*\(#(\d+)\)$")
341SUBJECT_END_RE = re.compile(r"\"(?:[ \t]+#(\d+))?")
342# Citation-shaped text: what no claim consumed is reported as unparsed.
343LOOSE_CODE_RE = re.compile(
344 CITED_PATH + r":\d+(?:[-:]\d+)*\b(?:[ \t]*\((?:[^()\n]|\([^()\n]*\))*\))?(?:[ \t]*@[ \t]*[0-9a-fA-F]{7,40}\b)?"
345)
346FILE_EXT_RE = re.compile(r"\.\w*[A-Za-z]\w*$")
347SHA_TAIL_RE = re.compile(r"(?<![\w.@/-])@[ \t]*[0-9a-fA-F]{7,40}\b") # not user@host or action@sha
348CURLY_SUBJECT_RE = re.compile(
349 r"(?<![\w./:\\-])[0-9a-fA-F]{7,40}[ \t]+[\u2018\u2019\u201c\u201d][^\n]*?(?:[\u2018\u2019\u201c\u201d]|$)", re.M
350)
351RANGE_DASH_RE = re.compile(r"(:\d+)[\u2013\u2014](?=\d)")
352CURLY_DOUBLE = str.maketrans("\u201c\u201d", '""')
353CURLY_ANY = str.maketrans("\u201c\u201d\u2018\u2019", "\"\"''")
354# What git prints when the object or path is not there; any other failure is an error.
355GIT_MISSING_RE = re.compile(
356 r"not a valid object name|does not exist in|exists on disk, but not in|bad file|outside repository", re.I
357)
360class GitError(Exception):
361 """git failed for a reason other than a missing object or path."""
363 def __init__(self, proc: subprocess.CompletedProcess[str]):
364 super().__init__(f"git {' '.join(proc.args[1:])} exited {proc.returncode}: {proc.stderr.strip()}")
367def git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]:
368 # Drop GIT_DIR and the rest: a git hook exports them, and they would override --repo.
369 env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
370 env["LC_ALL"] = "C" # untranslated messages, which GIT_MISSING_RE reads
371 return subprocess.run(
372 ["git", *args], cwd=repo, env=env, capture_output=True, text=True, encoding="utf-8", errors="replace"
373 )
376def raise_unless_missing(proc: subprocess.CompletedProcess[str]) -> None:
377 if not GIT_MISSING_RE.search(proc.stderr):
378 raise GitError(proc)
381def repo_root(path: Path) -> Path | None:
382 if not path.exists():
383 return None
384 proc = git(path if path.is_dir() else path.parent, "rev-parse", "--show-toplevel")
385 top = proc.stdout.strip()
386 return Path(top) if proc.returncode == 0 and top else None
389def blob_lines(repo: Path, rev: str, path: str) -> list[str] | None:
390 """The file's lines at rev, or None when rev holds no file at path."""
391 proc = git(repo, "cat-file", "blob", f"{rev}:{path}") # not git show, which lists a directory with exit 0
392 if proc.returncode:
393 raise_unless_missing(proc)
394 return None
395 lines = proc.stdout.split("\n") # as git counts lines; splitlines() also breaks on form feeds
396 return lines[:-1] if lines[-1] == "" else lines
399def is_commit(repo: Path, sha: str) -> bool:
400 proc = git(repo, "cat-file", "-e", f"{sha}^{{commit}}")
401 if proc.returncode:
402 raise_unless_missing(proc)
403 return proc.returncode == 0
406def tree_paths(repo: Path, sha: str, cache: dict[str, set[str]]) -> set[str]:
407 """Every file path at sha, or none when sha is not a commit."""
408 if sha not in cache:
409 proc = git(repo, "ls-tree", "-r", "--name-only", "-z", sha) if is_commit(repo, sha) else None
410 if proc is not None and proc.returncode:
411 raise_unless_missing(proc)
412 cache[sha] = set(proc.stdout.split("\0")) if proc is not None and not proc.returncode else set()
413 return cache[sha]
416def spaced_path(repo: Path, m: re.Match, cache: dict[str, set[str]]) -> tuple[str, int]:
417 """The cited path and where it starts: the most trailing words of the match's path the commit's tree has, else the last."""
418 words = m[1].split(" ")
419 paths = tree_paths(repo, m[5], cache) if len(words) > 1 else set()
420 for i in range(len(words) - 1):
421 if posixpath.normpath(" ".join(words[i:])) in paths:
422 break
423 else:
424 i = len(words) - 1
425 return " ".join(words[i:]), m.start(1) + len(" ".join(words[:i])) + (1 if i else 0)
428def history_claim(repo: Path, m: re.Match) -> tuple[str, str, int | None, tuple[int, int]]:
429 """sha, subject, pr and span of a history match, its subject run on to the last quote on the line that keeps it a prefix of git's own."""
430 sha, subject, pr, span = m[1], m[2], int(m[3]) if m[3] else None, m.span()
431 text = m.string
432 quotes = [m.end(2) + 1 + i for i, c in enumerate(text[m.end(2) + 1 :].split("\n", 1)[0]) if c == '"']
433 if quotes and is_commit(repo, sha):
434 log = git(repo, "log", "-1", "--no-show-signature", "--format=%s", sha, "--")
435 if log.returncode:
436 raise GitError(log)
437 actual = norm_subject(log.stdout)
438 for q in reversed(quotes):
439 if actual.startswith(norm_subject(text[m.start(2) : q])):
440 end = SUBJECT_END_RE.match(text, q)
441 subject, pr, span = text[m.start(2) : q], int(end[1]) if end[1] else None, (m.start(), end.end())
442 break
443 return sha, subject, pr, span
446def find_block(lines: list[str], block: list[str]) -> int | None:
447 """The 1-based line where block starts in lines, or None."""
448 return next((i + 1 for i in range(len(lines) - len(block) + 1) if lines[i : i + len(block)] == block), None)
451def symbol_found(symbol: str, lines: list[str]) -> bool:
452 """Whether the symbol's last identifier stands as a whole word in lines."""
453 parts = [part for part in re.split(r"::|[.#\s]", symbol) if part]
454 return bool(parts) and re.search(rf"(?<!\w){re.escape(parts[-1])}(?!\w)", "\n".join(lines)) is not None
457def check_code(repo: Path, at: str, sha: str, path: str, line: int, end_line: int | None, symbol: str) -> dict:
458 unread = {"line_text": None, "changed_since": None, "head_line": None, "symbol_found": None}
459 if not is_commit(repo, sha):
460 return {"status": "unknown-commit", **unread}
461 path = posixpath.normpath(path) # git reads rev:path literally, so src/../x.md would need a folder named ".."
462 lines = blob_lines(repo, sha, path)
463 if lines is None or not all(1 <= n <= len(lines) for n in (line, end_line) if n is not None):
464 return {"status": "missing", **unread}
465 last = max(line, end_line or line)
466 block = [text.strip() for text in lines[line - 1 : last]]
467 # Literal, so a path such as app/[slug]/page.tsx is not read as a glob.
468 diff = git(repo, "--literal-pathspecs", "diff", "--quiet", sha, at, "--", path)
469 if diff.returncode not in (0, 1):
470 raise GitError(diff)
471 if diff.returncode == 0:
472 status, head_line = "anchored", line
473 else:
474 head = [text.strip() for text in blob_lines(repo, at, path) or []]
475 status, head_line = "moved", find_block(head, block) or find_block(head, block[:1])
476 return {
477 "status": status if symbol else "incomplete",
478 "line_text": block[0],
479 "changed_since": diff.returncode == 1,
480 "head_line": head_line,
481 "symbol_found": symbol_found(symbol, lines[max(0, line - 81) : last]),
482 }
485def subject_pr(subject: str) -> int | None:
486 m = PR_SUFFIX_RE.search(" ".join(subject.split()))
487 return int(m.group(1)) if m else None
490def norm_subject(subject: str) -> str:
491 return PR_SUFFIX_RE.sub("", " ".join(subject.translate(CURLY_ANY).split())).casefold()
494def check_history(repo: Path, sha: str, subject: str, pr: int | None) -> dict:
495 if not is_commit(repo, sha):
496 return {"status": "unknown-commit", "actual_subject": None, "actual_pr": None}
497 log = git(repo, "log", "-1", "--no-show-signature", "--format=%s", sha, "--")
498 if log.returncode:
499 raise GitError(log)
500 actual = log.stdout.strip()
501 actual_pr = subject_pr(actual)
502 named = {n for n in (pr, subject_pr(subject)) if n is not None}
503 if not norm_subject(actual).startswith(norm_subject(subject)):
504 status = "subject-mismatch"
505 elif actual_pr is not None and named != {actual_pr}:
506 status = "pr-mismatch"
507 else:
508 status = "anchored"
509 return {"status": status, "actual_subject": actual, "actual_pr": actual_pr}
512def normalize(text: str) -> str:
513 """Fold en and em dash ranges and curly double quotes to ASCII, one character for one, so offsets still match."""
514 return RANGE_DASH_RE.sub(r"\1-", text.translate(CURLY_DOUBLE))
517def unparsed_spans(text: str, norm: str, consumed: list[tuple[int, int]]) -> list[dict]:
518 """Citation-shaped spans no claim consumed, overlapping ones merged, as written in text."""
519 taken = bytearray(len(text))
520 for start, end in consumed:
521 taken[start:end] = b"\1" * (end - start)
522 spans = [m.span() for m in LOOSE_CODE_RE.finditer(norm) if "/" in m[1] or FILE_EXT_RE.search(m[1])]
523 spans += [m.span() for m in SHA_TAIL_RE.finditer(norm)]
524 spans += [m.span() for m in CURLY_SUBJECT_RE.finditer(text)] # text, since norm has no curly double quotes
525 merged: list[list[int]] = []
526 for start, end in sorted(spans):
527 if taken.find(1, start, end) >= 0:
528 continue
529 if merged and start < merged[-1][1]:
530 merged[-1][1] = max(merged[-1][1], end)
531 else:
532 merged.append([start, end])
533 found, line, pos = [], 1, 0
534 for start, end in merged:
535 line += text.count("\n", pos, start)
536 pos = start
537 found.append({"text": text[start:end], "line": line})
538 return found
541def cmd_anchors(args) -> int:
542 text = read_text(args.file)
543 repo = repo_root(Path(args.repo))
544 if repo is None:
545 print(f"error: --repo {args.repo} is not inside a git working tree", file=sys.stderr)
546 return 2
547 at = git(repo, "rev-parse", "--verify", "--quiet", f"{args.at}^{{commit}}").stdout.strip()
548 if not at:
549 print(f"error: --at {args.at} does not name a commit", file=sys.stderr)
550 return 2
551 norm = normalize(text)
552 trees: dict[str, set[str]] = {}
553 code_matches = [(m, *spaced_path(repo, m, trees)) for m in CODE_CLAIM_RE.finditer(norm)]
554 code_claims = Counter(
555 (path, int(m[2]), int(m[3]) if m[3] else None, m[4].strip(), m[5]) for m, path, _ in code_matches
556 )
557 history_matches: list[tuple[str, str, int | None, tuple[int, int]]] = []
558 for m in HISTORY_CLAIM_RE.finditer(norm):
559 if not history_matches or m.start() >= history_matches[-1][3][1]: # not inside a subject run on past its quote
560 history_matches.append(history_claim(repo, m))
561 history_claims = Counter((sha, subject, pr) for sha, subject, pr, _ in history_matches)
562 code = [
563 {
564 "path": path,
565 "line": line,
566 "end_line": end_line,
567 "symbol": symbol,
568 "commit": sha,
569 **check_code(repo, at, sha, path, line, end_line, symbol),
570 "occurrences": n,
571 }
572 for (path, line, end_line, symbol, sha), n in code_claims.items()
573 ]
574 history = [
575 {"commit": sha, "subject": subject, "pr": pr, **check_history(repo, sha, subject, pr), "occurrences": n}
576 for (sha, subject, pr), n in history_claims.items()
577 ]
578 consumed = [(start, m.end()) for m, _, start in code_matches] + [span for *_, span in history_matches]
579 unparsed = unparsed_spans(text, norm, consumed)
580 statuses = Counter(c["status"] for c in code + history)
581 failing = ("incomplete", "missing", "unknown-commit", "subject-mismatch", "pr-mismatch")
582 counts = {"code": len(code), "history": len(history)}
583 for status in ("anchored", "moved", *failing):
584 counts[status.replace("-", "_")] = statuses[status]
585 counts["unparsed"] = len(unparsed)
586 failed = sum(statuses[status] for status in failing) + len(unparsed)
587 return out(
588 {
589 "file": args.file,
590 "repo": str(repo),
591 "at": at,
592 "code": code,
593 "history": history,
594 "unparsed": unparsed,
595 "counts": counts,
596 },
597 1 if failed else 0,
598 )
601# --- entry point -------------------------------------------------------------
604def main(argv: list[str] | None = None) -> int:
605 p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
606 sub = p.add_subparsers(dest="cmd", required=True)
608 pc = sub.add_parser("citations", help="cross-check [n] markers vs the source appendix")
609 pc.add_argument("file", help="path to the research report (or - for stdin)")
610 pc.set_defaults(func=cmd_citations)
612 pt = sub.add_parser("tally", help="count memlog entries by type and claims by status")
613 pt.add_argument("file", help="path to .memlog.md (or - for stdin)")
614 pt.set_defaults(func=cmd_tally)
616 ps = sub.add_parser("staleness", help="compute re-check dates from freshness windows")
617 ps.add_argument("file", help="claims JSON: [{claim, class, pub_date}] (or - for stdin)")
618 ps.add_argument("--windows", required=True, help="JSON months-per-class map, e.g. '{\"pricing\": 3}'")
619 ps.add_argument("--today", help="override today's date (YYYY-MM-DD)")
620 ps.set_defaults(func=cmd_staleness)
622 pg = sub.add_parser("slug", help="expand the run-folder pattern deterministically")
623 pg.add_argument("topic", help="research topic text")
624 pg.add_argument("--type", required=True, help="research type code (e.g. market)")
625 pg.add_argument(
626 "--pattern",
627 default="research-{topic_slug}",
628 help="folder pattern (default: research-{topic_slug})",
629 )
630 pg.add_argument("--date", help="override date (YYYY-MM-DD; default today)")
631 pg.set_defaults(func=cmd_slug)
633 pe = sub.add_parser("escape-sources", help="source appendix as escaped HTML with validated links")
634 pe.add_argument("file", help="path to the research report (or - for stdin)")
635 pe.set_defaults(func=cmd_escape_sources)
637 pa = sub.add_parser(
638 "anchors",
639 help="re-find each code and history citation in git",
640 description=(
641 "moved means the file changed between the cited commit and --at (default HEAD), not that the line shifted; head_line says where the cited text now sits, or null when it is gone. "
642 "incomplete means the symbol is empty; pr-mismatch means the subject ends in (#n) and the citation names no PR or another one; unparsed lists citation-shaped text no claim matched. All three fail the run. symbol_found says whether the symbol's last identifier is in the cited lines or the 80 lines above them; it never fails the run."
643 ),
644 )
645 pa.add_argument("file", help="path to the research report (or - for stdin)")
646 pa.add_argument("--repo", required=True, help="the git working tree, or any path inside it")
647 pa.add_argument("--at", default="HEAD", help="revision to compare against (default: HEAD)")
648 pa.set_defaults(func=cmd_anchors)
650 args = p.parse_args(argv)
651 try:
652 return args.func(args)
653 except (FileNotFoundError, GitError) as e:
654 print(f"error: {e}", file=sys.stderr)
655 return 2
658if __name__ == "__main__":
659 if sys.platform == "win32":
660 # Piped output on Windows defaults to a legacy code page, not UTF-8.
661 sys.stdout.reconfigure(encoding="utf-8")
662 sys.stderr.reconfigure(encoding="utf-8")
663 sys.exit(main())