Skills › Compound Engineering › ce-compound › validate-doc-claims.py
validate-doc-claims.py
498 lines · skills/ce-compound/scripts/validate-doc-claims.py at 3449230
1#!/usr/bin/env python3
2"""Validate cited claims in a solution doc against the git tree.
4Usage:
5 python3 validate-doc-claims.py <doc-path>
7Exit codes:
8 0 — nothing flagged
9 1 — one or more flags need adjudication (report on stdout)
10 2 — usage error (bad arguments, missing file)
12Scope: mechanical grounding checks on a written doc's *body*. Complements
13validate-frontmatter.py (parser-safety) — this script checks the body's
14citations against the repository:
16 1. Cited repo-relative paths (backticked, containing at least one '/')
17 exist in the working tree, including already-absolute citations that
18 fall inside the repo (rewritten to repo-relative before candidacy).
19 Tokens containing '../' resolve from the doc's directory (those
20 escaping the repo are skipped). Misses tracked at HEAD or the
21 upstream default branch still count as real paths and are classified
22 (deleted/uncommitted vs stale checkout). Tokens missing everywhere
23 are flagged only when path-shaped; slash-delimited identifiers
24 (branch names, git refs, provider/model IDs) and slash-prefixed
25 URL routes are skipped.
26 2. Cited commit SHAs (7-40 hex chars with at least one digit and one
27 a-f letter) resolve to commits, classified by reachability from
28 HEAD and the upstream default branch. Session ids, content hashes
29 and blob hashes are hex too, so an unresolvable hex word is
30 reported in one of two tiers rather than asserted to be fabricated:
31 FLAG when the text right before it presents it as a commit (a
32 likely fabricated citation), NOTE otherwise (an identifier this
33 script cannot classify). Only FLAG affects the exit code. The
34 cue vocabulary therefore ranks confidence; it does not decide
35 whether an item is surfaced, so a phrasing it misses is reported
36 one tier down instead of disappearing.
37 3. Relative markdown link targets resolve from the doc's location.
38 4. Dangling drafting scaffold: "Learning(s) N" numbering and
39 unresolved {{...}} placeholder tokens. Inline code spans and fenced
40 code blocks are masked first, so a {{...}} shown as documented syntax
41 (Handlebars, a CI variable, a GitHub ruleset placeholder) is not
42 mistaken for a leaked scaffold; only a bare token in prose is flagged.
44Flags are adjudication input, NOT hard failures — a doc may legitimately
45cite a path deleted by the very fix it documents. The calling agent
46decides per flag: fix, annotate as historical, or confirm intentional.
47Only the summary exit code distinguishes "clean" from "needs a look".
49The script never touches the network (no fetch); classification uses
50whatever refs exist locally. Run a best-effort `git fetch --quiet` first
51when freshness matters. Pure stdlib (no third-party deps).
52"""
53import os
54import re
55import subprocess
56import sys
58# Tokens containing these are placeholders/examples, not real citations.
59PLACEHOLDER_CHARS = set("<>{}*$")
60PLACEHOLDER_SUBSTRINGS = ("path/to", "...", "…")
62SHA_RE = re.compile(r"\b[0-9a-f]{7,40}\b")
63# Words that name a commit or a commit operation, so a hex token right after
64# one is a commit citation. Words naming some other git object ("blob", "tree"),
65# any hash ("sha"), and the bare tool name ("git", which precedes every object
66# kind equally) are deliberately absent — they are what the flag used to mistake
67# for commits.
68COMMIT_WORDS = frozenset(
69 "commit commits committed committing revision revisions rev revs "
70 "revert reverts reverted cherry-pick cherry-picked rebase rebased "
71 "bisect bisected".split()
72)
73# A hex token is also a citation when the sentence attributes a change landing
74# in this repository to it: "landed in <sha>", "resolved by <sha>". Both halves
75# are needed. The preposition alone attributes without saying what to, so it
76# would read "recorded at <digest>" as a commit; the verb alone does not point
77# at the token. Membership below is that condition, not a tally of phrasings
78# seen so far: a verb belongs when it says a change landed, and does not when
79# it says an identifier was assigned or a value stored.
80CITATION_VERBS = frozenset(
81 "fixed fix fixes landed lands land introduced introduces introduce "
82 "shipped ships ship merged merges merge resolved resolves resolve "
83 "reverted reverts broke breaks broken caused causes regressed "
84 "added adds removed removes released releases".split()
85)
86CITATION_PREPS = frozenset(("in", "by", "at", "with"))
87# The pin form that names a commit is owner/repo@<sha>. A bare "@" is not it:
88# it also prefixes account names and image tags, whose identifiers are hex too.
89REPO_PIN_RE = re.compile(r"(?<![\w./@-])[\w.-]+/[\w.-]+@$")
90BACKTICK_RE = re.compile(r"`([^`\n]+)`")
91MD_LINK_RE = re.compile(r"\[[^\]]*\]\(([^)\s]+)\)")
92FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$")
93SCAFFOLD_RES = (
94 re.compile(r"\bLearnings?\s+#?\d"),
95 re.compile(r"\{\{[^}\n]*\}\}"),
96)
99def usage_fail(msg: str) -> "NoReturn":
100 sys.stderr.write(f"validate-doc-claims: {msg}\n")
101 sys.exit(2)
104def git(args: list[str], cwd: str) -> tuple[int, str]:
105 try:
106 result = subprocess.run(
107 ["git", *args],
108 cwd=cwd,
109 capture_output=True,
110 text=True,
111 timeout=30,
112 )
113 return result.returncode, result.stdout.strip()
114 except (OSError, subprocess.TimeoutExpired):
115 return 1, ""
118def split_body(text: str) -> tuple[str, int]:
119 """Return (body, 1-indexed line number the body starts on).
121 Skips YAML frontmatter when present so frontmatter fields are not
122 scanned as body citations.
123 """
124 lines = text.split("\n")
125 if lines and lines[0].rstrip() == "---":
126 for i in range(1, len(lines)):
127 if lines[i].rstrip() == "---":
128 return "\n".join(lines[i + 1 :]), i + 2
129 return text, 1
132def is_path_candidate(token: str, *, known_path: bool = False) -> bool:
133 if any(ch.isspace() for ch in token):
134 return False
135 if not known_path and "/" not in token:
136 return False
137 if "://" in token or token.startswith(("http", "#", "/", "~")):
138 return False
139 if token.startswith(("origin/", "upstream/", "refs/")):
140 return False # git refs, not repo paths
141 if PLACEHOLDER_CHARS & set(token):
142 return False
143 if any(sub in token for sub in PLACEHOLDER_SUBSTRINGS):
144 return False
145 return True
148def is_path_shaped(token: str, base: str) -> bool:
149 """Distinguish a path citation from a slash-delimited identifier
150 (branch name, provider/model ID) among tokens found nowhere in git."""
151 segments = token.split("/")
152 if re.search(r"\.[A-Za-z0-9]{1,8}$", segments[-1]):
153 return True
154 if token.endswith("/"):
155 return True
156 return os.path.isdir(os.path.join(base, segments[0]))
159def mask_code(lines: list[str]) -> list[str]:
160 """Blank out fenced code blocks and inline code spans, preserving line
161 count and length. Illustrative {{...}} in quoted code must not read as a
162 leaked drafting scaffold; only bare tokens in prose should."""
163 masked: list[str] = []
164 fence: str | None = None # active fence run (e.g. "```"), or None
165 for line in lines:
166 m = FENCE_RE.match(line)
167 if fence is None and m:
168 fence = m.group(1)
169 masked.append(" " * len(line))
170 continue
171 if fence is not None:
172 # CommonMark: a closing fence is the same char, at least as long,
173 # and followed only by whitespace — an info string (```json) opens
174 # but never closes, so it stays block content.
175 if (
176 m
177 and m.group(1)[0] == fence[0]
178 and len(m.group(1)) >= len(fence)
179 and not m.group(2).strip()
180 ):
181 fence = None
182 masked.append(" " * len(line))
183 continue
184 masked.append(BACKTICK_RE.sub(lambda x: " " * len(x.group(0)), line))
185 return masked
188def cites_a_commit(prefix: str) -> bool:
189 """True when the text just before a hex word presents it as a commit.
191 Only the last few words on the same line count: a cue further away
192 ("the git history shows session 7e6861b4") describes the surroundings,
193 not the token. Digits are word characters so a hash named after its
194 algorithm ("SHA256") stays one non-cue word, and command flags drop out
195 first so `git show --format=%H <sha>` reads like `git show <sha>`.
196 """
197 if REPO_PIN_RE.search(prefix):
198 return True # owner/repo@<sha> pins a commit
199 unflagged = " ".join(w for w in prefix.split() if not w.startswith("-"))
200 words = [
201 w.lower() for w in re.findall(r"[A-Za-z][A-Za-z0-9-]*", unflagged)[-3:]
202 ]
203 if any(word in COMMIT_WORDS for word in words):
204 return True
205 return (
206 len(words) >= 2
207 and words[-1] in CITATION_PREPS
208 and words[-2] in CITATION_VERBS
209 )
212def normalize_path(token: str) -> str:
213 token = token.strip().rstrip(".,;")
214 token = re.sub(r":\d+(-\d+)?$", "", token) # strip `:line` / `:a-b` refs
215 if token.startswith("./"):
216 token = token[2:]
217 return token
220def strip_repo_prefix(token: str, base: str) -> str:
221 """Rewrite an already-absolute path inside the repo to repo-relative.
223 Relative tokens, URL routes, and out-of-repo absolute paths are
224 unchanged so the existing candidacy guard still drops them. Realpath
225 both sides so a host where /tmp is a symlink still matches. A
226 successful rewrite is slash-normalized so Windows relpath output
227 stays a candidate.
228 """
229 if not os.path.isabs(token):
230 return token
231 try:
232 rel = os.path.relpath(os.path.realpath(token), os.path.realpath(base))
233 except ValueError:
234 return token
235 if rel == ".." or rel.startswith(".." + os.sep):
236 return token
237 return rel.replace("\\", "/")
240def main(argv: list[str]) -> int:
241 if len(argv) != 2:
242 usage_fail(f"usage: {os.path.basename(argv[0])} <doc-path>")
244 doc_path = argv[1]
245 if not os.path.isfile(doc_path):
246 usage_fail(f"file not found: {doc_path}")
248 with open(doc_path) as f:
249 text = f.read()
251 doc_dir = os.path.dirname(os.path.abspath(doc_path))
252 body, body_start = split_body(text)
253 body_lines = body.split("\n")
255 def loc_suffix(needle: str) -> str:
256 for i, line in enumerate(body_lines):
257 if needle in line:
258 return f" (line {body_start + i})"
259 return ""
261 infos: list[str] = []
262 notes: list[str] = []
263 flags: list[str] = []
265 # --- Repo context -----------------------------------------------------
266 code, repo_root = git(["rev-parse", "--show-toplevel"], doc_dir)
267 in_git = code == 0 and bool(repo_root)
268 upstream: str | None = None
269 if in_git:
270 code, ref = git(["rev-parse", "--abbrev-ref", "origin/HEAD"], repo_root)
271 if code == 0 and ref:
272 upstream = ref
273 else:
274 for candidate in ("origin/main", "origin/master"):
275 code, _ = git(
276 ["rev-parse", "--verify", "--quiet", candidate], repo_root
277 )
278 if code == 0:
279 upstream = candidate
280 break
281 if upstream:
282 code, behind = git(
283 ["rev-list", "--count", f"HEAD..{upstream}"], repo_root
284 )
285 if code == 0 and behind.isdigit() and int(behind) > 0:
286 infos.append(
287 f"INFO: worktree is {behind} commits behind {upstream} — "
288 "verify merge-state claims against remote truth (gh pr view), "
289 "not this checkout"
290 )
291 else:
292 infos.append(
293 "INFO: no upstream default branch found — "
294 "path/SHA classification limited to HEAD"
295 )
296 else:
297 infos.append(
298 "INFO: not a git repository — path and SHA classification skipped "
299 "(scaffold and link checks still apply)"
300 )
302 def upstream_has_path(path: str) -> bool:
303 if not (in_git and upstream):
304 return False
305 code, _ = git(["cat-file", "-e", f"{upstream}:{path}"], repo_root)
306 return code == 0
308 def head_has_path(path: str) -> bool:
309 if not in_git:
310 return False
311 code, _ = git(["cat-file", "-e", f"HEAD:{path}"], repo_root)
312 return code == 0
314 # --- 1. Cited repo paths ----------------------------------------------
315 checked_paths = 0
316 seen_paths: set[str] = set()
317 base = repo_root if in_git else os.getcwd()
318 for raw in BACKTICK_RE.findall(body):
319 token = normalize_path(raw)
320 rewritten_abs = False
321 if in_git:
322 before = token
323 token = strip_repo_prefix(token, base)
324 rewritten_abs = token != before
325 if not is_path_candidate(token, known_path=rewritten_abs):
326 continue
327 check = token
328 if token.startswith("../") or "/../" in token:
329 # A `../` citation is doc-relative (matching how markdown links
330 # resolve), so map it to a repo-root path before checking.
331 if not in_git:
332 continue
333 resolved = os.path.realpath(os.path.join(doc_dir, token))
334 check = os.path.relpath(resolved, os.path.realpath(base))
335 if check.startswith(".."):
336 continue # escapes the repo — not checkable as a repo path
337 if check in seen_paths:
338 continue
339 seen_paths.add(check)
340 if os.path.exists(os.path.join(base, check)):
341 checked_paths += 1
342 continue
343 tracked_head = head_has_path(check)
344 tracked_upstream = upstream_has_path(check)
345 if not (tracked_head or tracked_upstream) and not is_path_shaped(
346 check, base
347 ):
348 continue # branch name / provider ID, not a path citation
349 checked_paths += 1
350 loc = loc_suffix(raw)
351 if tracked_head:
352 flags.append(
353 f"FLAG path `{token}`{loc} — tracked at HEAD but missing from "
354 "the working tree: deleted or uncommitted removal? Annotate as "
355 "historical (e.g. removed by this fix) or restore it."
356 )
357 elif tracked_upstream:
358 flags.append(
359 f"FLAG path `{token}`{loc} — not in working tree but exists at "
360 f"{upstream}: stale checkout? Annotate or verify against upstream."
361 )
362 else:
363 flags.append(
364 f"FLAG path `{token}`{loc} — not found under {base}"
365 + (f" or {upstream}" if upstream else "")
366 + ". Only that location was searched; check other repos or "
367 "stores before treating the citation as wrong. Fix the "
368 "citation, or annotate it as historical (e.g. removed by "
369 "this fix)."
370 )
372 # --- 2. Cited commit SHAs ----------------------------------------------
373 checked_shas = 0
374 if in_git:
375 # One entry per distinct hex word: the line to report it at, and
376 # whether any occurrence of it is presented as a commit. A doc often
377 # quotes a token in a transcript before citing it, so a later citing
378 # occurrence upgrades the tier and supplies the line.
379 seen_shas: dict[str, tuple[int, bool]] = {}
380 order: list[str] = []
381 for m in SHA_RE.finditer(body):
382 sha = m.group(0)
383 if not (any(c.isdigit() for c in sha) and any(c in "abcdef" for c in sha)):
384 continue # dates and decimal ids are not SHAs
385 line_start = body.rfind("\n", 0, m.start()) + 1
386 cited = cites_a_commit(body[line_start : m.start()])
387 line_no = body_start + body.count("\n", 0, m.start())
388 if sha not in seen_shas:
389 seen_shas[sha] = (line_no, cited)
390 order.append(sha)
391 elif cited and not seen_shas[sha][1]:
392 seen_shas[sha] = (line_no, True)
393 for sha in order:
394 line_no, cited = seen_shas[sha]
395 code, _ = git(["cat-file", "-e", f"{sha}^{{commit}}"], repo_root)
396 resolved = code == 0
397 loc = f" (line {line_no})"
398 if not resolved:
399 if not cited:
400 # Nothing here says "commit", and hex is also how session
401 # ids and content hashes are written. Surface it without
402 # claiming to know which it is; the reader adjudicates.
403 notes.append(
404 f"NOTE sha {sha}{loc} — an unresolved hex identifier "
405 "with no commit reference around it. This script cannot "
406 "tell a session id or content hash from a commit; verify "
407 "it if it was meant as one."
408 )
409 continue
410 checked_shas += 1
411 flags.append(
412 f"FLAG sha {sha}{loc} — does not resolve to a commit in this "
413 "repository. Replace with the PR number, or drop it."
414 )
415 continue
416 checked_shas += 1
417 in_head = (
418 git(["merge-base", "--is-ancestor", sha, "HEAD"], repo_root)[0] == 0
419 )
420 in_up = (
421 upstream is not None
422 and git(["merge-base", "--is-ancestor", sha, upstream], repo_root)[0]
423 == 0
424 )
425 if in_head and (in_up or upstream is None):
426 continue
427 if in_head and not in_up:
428 flags.append(
429 f"FLAG sha {sha}{loc} — reachable from HEAD but not {upstream}: "
430 "local-only commit whose SHA may be rewritten on merge "
431 "(rebase/squash). Prefer citing the PR number."
432 )
433 elif in_up:
434 flags.append(
435 f"FLAG sha {sha}{loc} — not reachable from HEAD but reachable "
436 f"from {upstream}: this checkout predates the merge. Add a "
437 "temporal qualifier or verify the claim via gh."
438 )
439 else:
440 flags.append(
441 f"FLAG sha {sha}{loc} — exists but unreachable from HEAD"
442 + (f" or {upstream}" if upstream else "")
443 + ": likely a rebased-away commit. Prefer citing the PR number."
444 )
446 # --- 3. Relative markdown links -----------------------------------------
447 checked_links = 0
448 seen_links: set[str] = set()
449 for target in MD_LINK_RE.findall(body):
450 if re.match(r"^[a-z][a-z0-9+.-]*:", target, re.IGNORECASE):
451 continue # URL scheme
452 if target.startswith("#"):
453 continue # intra-doc anchor
454 bare = target.split("#", 1)[0]
455 if not bare or bare in seen_links:
456 continue
457 seen_links.add(bare)
458 checked_links += 1
459 if not os.path.exists(os.path.normpath(os.path.join(doc_dir, bare))):
460 loc = loc_suffix(target)
461 flags.append(
462 f"FLAG link ({target}){loc} — relative target does not resolve "
463 "from the doc's location. Fix the path."
464 )
466 # --- 4. Dangling drafting scaffold ---------------------------------------
467 for i, line_text in enumerate(mask_code(body_lines)):
468 for pattern in SCAFFOLD_RES:
469 m = pattern.search(line_text)
470 if m:
471 flags.append(
472 f'FLAG scaffold "{m.group(0)}" (line {body_start + i}) — '
473 "drafting-context reference leaked into the doc. Rewrite it "
474 "as a real path or link."
475 )
477 # --- Report ---------------------------------------------------------------
478 for info in infos:
479 print(info)
480 for note in notes:
481 print(note)
482 for flag in flags:
483 print(flag)
484 summary = (
485 f"checked {checked_paths} paths, {checked_shas} SHAs, "
486 f"{checked_links} links; {len(flags)} flags"
487 )
488 if notes:
489 summary += f", {len(notes)} notes"
490 print(summary)
491 if flags:
492 return 1
493 print(f"OK: {doc_path}")
494 return 0
497if __name__ == "__main__":
498 sys.exit(main(sys.argv))