Skills › gstack › gstack › pr-dependencies.ts

pr-dependencies.ts

622 lines · scripts/pr-dependencies.ts at 3f79d81

Open on GitHub

1/**

2 * Derived PR-lane dependencies (W1 follow-up): which paid cases consume a

3 * repository file that no touchfile names, computed from real references on

4 * the checked-out tree instead of hand lists.

5 *

6 * A case consumes the transitive reference closure of its owning paid test

7 * file plus every tracked file its touchfiles declare. References are:

8 * - static and literal dynamic imports of code (resolved relative paths);

9 * - repository paths written in code, shell or skill text (bin/, lib/,

10 * scripts/, browse/, design/, make-pdf/, hosts/, test/helpers/ ...),

11 * including `path.join(..., 'lib', 'x.ts')` segments;

12 * - bin script names (`gstack-*`) a skill, test or script invokes;

13 * - compiled binaries (scripts/build.sh `--outfile`): text naming

14 * `design/dist/design` consumes the whole `design/src/**` tree;

15 * - package scripts invoked as `bun run <name>`;

16 * - template placeholders: a skill whose template uses `{{NAME}}` consumes

17 * the resolver module behind RESOLVERS.NAME and that module's imports;

18 * - host configs: `hosts/<id>.ts` and `hosts/<id>/**` feed cases whose test

19 * names that host id.

20 * Selection data modules are opaque (their literals classify files rather

21 * than consume them), and the walk never enters another paid test file.

22 *

23 * A file the derivation cannot place is not guessed: PR selection restores

24 * the full gate for it (scripts/test-pr-profile.ts), and PR_FULL_GATE_FILES

25 * there lists the inputs every case consumes that must always do so.

26 */

27import * as fs from 'node:fs';

28import * as path from 'node:path';

29import { spawnSync } from 'node:child_process';

30import { matchGlob } from '../test/helpers/test-selection';

31import { isPaidTestFile } from '../test/helpers/paid-test-set';

33export interface DependencyMaps {

34 e2eTouchfiles: Record<string, string[]>;

35 judgeTouchfiles: Record<string, string[]>;

36 globalTouchfiles: readonly string[];

37}

39export interface DerivedDependencies {

40 /** Repository file -> E2E case ids whose reference closure contains it. */

41 e2e: Map<string, Set<string>>;

42 /** Repository file -> LLM-judge ids whose reference closure contains it. */

43 judges: Map<string, Set<string>>;

44 /** Files every E2E case's closure contains: an edit there is as broad as a global touchfile. */

45 everyCase: Set<string>;

46 /** The global touchfiles and their code-import closure (the paid runner and harness every case runs through). */

47 global: Set<string>;

48 /** Every tracked file at derivation time. */

49 tracked: Set<string>;

50 /** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */

51 trace(caseId: string, file: string): string[] | null;

52 /**

53 * Files whose import scan failed for a reason other than a leading `#!`

54 * line (their imports are unknown). PR selection restores the full gate

55 * while this is non-empty; it fills as the derivation reads files.

56 */

57 scanErrors: Map<string, string>;

58 /**

59 * Tracked files whose reference text still names `missing`, a path absent

60 * from the tree (deleted in the diff): its path or extensionless import

61 * target, a relative import resolving to it, its bin name, an ancestor

62 * directory below the top two levels, or the outfile of a binary built from

63 * its tree. Whole-line comments and selection data never count.

64 */

65 referencers(missing: string): string[];

66}

68const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/;

69/** An extensionless script Bun or Node runs: its imports are code, not shell. */

70const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/;

71const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/;

73/** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */

75const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g;

76const PLACEHOLDER = /\{\{([A-Z][A-Z0-9_]*)/g;

77const PACKAGE_RUN = /\b(?:bun|npm) run ([a-z][\w:.-]*)/g;

78const OWNER = /^test\/[^/]+\.test\.ts$/;

79/**

80 * Selection and inventory data: their string literals name files they

81 * classify, not files they consume, so references inside them are not

82 * followed (the same rule as test/helpers/touchfile-closure.ts).

83 */

84export const SELECTION_DATA_MODULES: readonly string[] = [

85 'test/helpers/touchfiles-data.ts', 'test/helpers/touchfiles.ts', 'test/helpers/test-selection.ts',

86 'test/helpers/touchfile-closure.ts', 'test/helpers/free-fixtures-data.ts', 'test/helpers/paid-test-set.ts',

87 'test/helpers/periodic-exclude-data.ts', 'test/helpers/eval-budgets.ts', 'scripts/test-pr-profile.ts',

88 'scripts/pr-dependencies.ts', 'scripts/lib/paid-cases.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/paid-types.ts',

89 // The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.

90 'scripts/build.sh',

91 // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.

92 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts',

93 // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).

94 'test/helpers/exercised-commands.json',

95];

96const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts';

98const transpilers = {

99 ts: new Bun.Transpiler({ loader: 'ts' }), tsx: new Bun.Transpiler({ loader: 'tsx' }),

100 js: new Bun.Transpiler({ loader: 'js' }), jsx: new Bun.Transpiler({ loader: 'jsx' }),

101};

102

103/** Code by extension, or an extensionless file whose shebang runs Bun or Node. */

104export function isCodeFile(file: string, text: string): boolean {

105 return CODE.test(file) || (!/\.[^/]+$/.test(path.posix.basename(file)) && CODE_SHEBANG.test(text));

106}

107

108/**

109 * Relative import specifiers of a code file. A leading `#!` line is blanked

110 * first (`scanImports` rejects it), and each extension gets its own loader

111 * (the TSX loader misreads `.ts` generics and casts). Any other parse error

112 * throws, so callers fail closed.

113 */

114export function scanCodeImports(file: string, text: string): string[] {

115 const loader = /\.tsx$/.test(file) ? 'tsx' : /\.jsx$/.test(file) ? 'jsx' : /\.[cm]?js$/.test(file) ? 'js' : 'ts';

116 return transpilers[loader].scanImports(text.replace(/^#![^\n]*/, '')).map(item => item.path);

117}

118

119/**

120 * Text a reference scan reads: whole-line comments are prose about other

121 * files, not consumption (block comments that open a line, `//` lines, and

122 * shell `#` lines). Trailing comments stay, so a stripped reference is never

123 * one the code executes.

124 */

125export function referenceText(file: string, text: string): string {

126 if (CODE.test(file)) return text.replace(/^[ \t]*\/\*[\s\S]*?\*\//gm, '').replace(/^[ \t]*\/\/.*$/gm, '');

127 if (/\.json$/.test(file)) return '';

128 if (!/\.(?:md|tmpl|txt)$/.test(file)) return text.replace(/^[ \t]*#(?!!).*$/gm, '');

129 return text;

130}

131

132function trackedFiles(root: string): string[] {

133 const listed = spawnSync('git', ['ls-files', '-z'], { cwd: root, encoding: 'utf8', timeout: 20_000, maxBuffer: 64 * 1024 * 1024 });

134 if (listed.status !== 0) throw new Error('git ls-files failed; dependency derivation needs a git checkout');

135 return listed.stdout.split('\0').filter(Boolean);

136}

137

138/** Resolver module (repo path) -> placeholder names whose RESOLVERS entry calls into it. */

139function placeholderModules(root: string, read: (file: string) => string): Map<string, string[]> {

140 const index = 'scripts/resolvers/index.ts';

141 const source = read(index);

142 const fnModule = new Map<string, string>();

143 for (const match of source.matchAll(/import\s*\{([^}]*)\}\s*from\s*'\.\/([\w./-]+)'/g)) {

144 const file = resolveSpecifier(root, `./${match[2]}`, index);

145 if (!file) continue;

146 for (const name of match[1]!.split(',').map(part => part.trim().replace(/^type\s+/, '').split(/\s+as\s+/).pop()!).filter(Boolean)) fnModule.set(name, file);

147 }

148 const body = source.slice(source.indexOf('export const RESOLVERS'));

149 const byModule = new Map<string, string[]>();

150 for (const entry of body.matchAll(/^\s+([A-Z][A-Z0-9_]*):\s*(.+?),?\s*$/gm)) {

151 for (const ident of entry[2]!.matchAll(/\b([A-Za-z_$][\w$]*)\b/g)) {

152 const file = fnModule.get(ident[1]!);

153 if (file) byModule.set(file, [...new Set([...(byModule.get(file) ?? []), entry[1]!])]);

154 }

155 }

156 return byModule;

157}

158

159function resolveSpecifier(root: string, specifier: string, from: string): string | null {

160 if (!specifier.startsWith('.')) return null;

161 const base = path.posix.normalize(path.posix.join(path.posix.dirname(from), specifier));

162 for (const candidate of [base, `${base}.ts`, `${base}.tsx`, `${base}.js`, `${base}.mjs`, `${base}/index.ts`, `${base}/index.js`]) {

163 try { if (fs.statSync(path.join(root, candidate)).isFile()) return candidate; } catch { /* next */ }

164 }

165 return null;

166}

167

168/** Binary outfile path -> the source tree it is compiled from (scripts/build.sh). */

169function compiledBinaries(read: (file: string) => string): Array<{ outfile: string; tree: string }> {

170 return [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)]

171 .map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? match[1]! : `${path.posix.dirname(match[1]!)}/**` }));

172}

173

174export function deriveDependencies(maps: DependencyMaps, root: string): DerivedDependencies {

175 const tracked = trackedFiles(root);

176 const trackedSet = new Set(tracked);

177 const sources = new Map<string, string>();

178 const read = (file: string): string => {

179 if (!sources.has(file)) {

180 let text = '';

181 try { if (trackedSet.has(file) && (TEXT.test(file) || file.startsWith('bin/') || file === 'setup')) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ }

182 sources.set(file, text);

183 }

184 return sources.get(file)!;

185 };

186 const binNames = new Map(tracked.filter(file => file.startsWith('bin/')).map(file => [path.posix.basename(file).replace(/\.ts$/, ''), file]));

187 const binaries = compiledBinaries(read);

188 let scripts: Record<string, string> = {};

189 try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ }

190 const placeholders = placeholderModules(root, read);

191 const placeholderOwners = new Map<string, string>();

192 for (const [file, names] of placeholders) for (const name of names) placeholderOwners.set(name, file);

193 const expanded = new Map<string, string[]>();

194 const expand = (pattern: string): string[] => {

195 if (!expanded.has(pattern)) expanded.set(pattern, pattern.includes('*') ? tracked.filter(file => matchGlob(file, pattern)) : trackedSet.has(pattern) ? [pattern] : []);

196 return expanded.get(pattern)!;

197 };

198 const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));

199

200 const scanErrors = new Map<string, string>();

201 const importsOf = (file: string): string[] => {

202 try { return scanCodeImports(file, read(file)); } catch (error) { scanErrors.set(file, String(error)); return []; }

203 };

204 const edges = new Map<string, string[]>();

205 const refs = (file: string): string[] => {

206 if (edges.has(file)) return edges.get(file)!;

207 const raw = read(file);

208 const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw);

209 const out = new Set<string>();

210 if (isCodeFile(file, raw)) for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); }

211 for (const match of text.matchAll(REPO_PATH)) {

212 const ref = match[1]!.replace(/[.]+$/, '');

213 if (trackedSet.has(ref)) out.add(ref);

214 else if (dirs.has(ref)) for (const child of expand(`${ref}/**`)) out.add(child);

215 }

216 for (const match of text.matchAll(JOINED_PATH)) {

217 const ref = `${match[1]}/${match[2]}`;

218 if (trackedSet.has(ref)) out.add(ref);

219 else if (dirs.has(ref)) for (const child of expand(`${ref}/**`)) out.add(child);

220 }

221 for (const match of text.matchAll(BIN_NAME)) { const bin = binNames.get(match[0]); if (bin) out.add(bin); }

222 for (const { outfile, tree } of binaries) if (text.includes(outfile)) for (const child of expand(tree)) out.add(child);

223 for (const match of text.matchAll(PACKAGE_RUN)) {

224 const command = scripts[match[1]!];

225 if (command) for (const ref of command.matchAll(REPO_PATH)) for (const child of expand(ref[1]!)) out.add(child);

226 }

227 if (/\.(?:md|tmpl)$/.test(file)) for (const match of text.matchAll(PLACEHOLDER)) { const owner = placeholderOwners.get(match[1]!); if (owner) out.add(owner); }

228 out.delete(file);

229 const list = [...out];

230 edges.set(file, list);

231 return list;

232 };

233 const closure = (entries: Iterable<string>, parents?: Map<string, string>): Set<string> => {

234 const seen = new Set<string>();

235 const queue = [...entries].filter(file => trackedSet.has(file)).reverse();

236 for (let head = 0; head < queue.length; head++) {

237 const file = queue[head]!;

238 if (seen.has(file)) continue;

239 seen.add(file);

240 for (const next of refs(file)) if (!seen.has(next) && !/\.test\.tsx?$/.test(next)) { queue.push(next); if (parents && !parents.has(next)) parents.set(next, file); }

241 }

242 return seen;

243 };

244

245 const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts'));

246 const consumers = (table: Record<string, string[]>, defaultOwner: string | null): Map<string, Set<string>> => {

247 const byFile = new Map<string, Set<string>>();

248 for (const [id, patterns] of Object.entries(table)) {

249 const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern));

250 const declared = [...(owners.length ? owners : defaultOwner ? [defaultOwner] : []), ...patterns.flatMap(expand)];

251 const files = closure(declared);

252 const ownerText = (owners.length ? owners : defaultOwner ? [defaultOwner] : []).map(read).join('\n');

253 for (const host of hostIds) {

254 if (new RegExp(`['"\`]${host}['"\`]|--host[ =]${host}\\b`).test(ownerText)) for (const file of closure([`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)])) files.add(file);

255 }

256 for (const file of files) {

257 if (!byFile.has(file)) byFile.set(file, new Set());

258 byFile.get(file)!.add(id);

259 }

260 }

261 return byFile;

262 };

263 const e2e = consumers(maps.e2eTouchfiles, null);

264 const total = Object.keys(maps.e2eTouchfiles).length;

265 const everyCase = new Set([...e2e].filter(([, ids]) => ids.size === total).map(([file]) => file));

266 const global = new Set<string>();

267 const pending = maps.globalTouchfiles.flatMap(expand);

268 while (pending.length) {

269 const file = pending.pop()!;

270 if (global.has(file)) continue;

271 global.add(file);

272 if (!isCodeFile(file, read(file))) continue;

273 for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit && !global.has(hit)) pending.push(hit); }

274 }

275 const importTargets = new Map<string, string[]>();

276 const targetsOf = (file: string): string[] => {

277 if (!importTargets.has(file)) {

278 const targets: string[] = [];

279 if (isCodeFile(file, read(file)) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {

280 for (const specifier of importsOf(file)) {

281 if (specifier.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), specifier)));

282 }

283 }

284 importTargets.set(file, targets);

285 }

286 return importTargets.get(file)!;

287 };

288 const referencerCache = new Map<string, string[]>();

289 const referencers = (missing: string): string[] => {

290 if (!referencerCache.has(missing)) referencerCache.set(missing, findReferencers(missing));

291 return referencerCache.get(missing)!;

292 };

293 const findReferencers = (missing: string): string[] => {

294 const bare = missing.replace(/\.(?:[cm]?[jt]s|tsx|jsx)$/, '');

295 const parts = missing.split('/');

296 const ancestors = parts.slice(0, -1).map((_, i) => parts.slice(0, i + 1).join('/')).filter(dir => dir.split('/').length >= 3);

297 const binName = missing.startsWith('bin/') ? path.posix.basename(bare) : null;

298 const outfiles = binaries.filter(({ tree }) => matchGlob(missing, tree)).map(({ outfile }) => outfile);

299 const needles = [missing, ...(bare !== missing ? [bare] : []), ...ancestors.map(dir => `${dir}/`), ...outfiles];

300 return tracked.filter(file => {

301 if (file === missing || (SELECTION_DATA_MODULES as readonly string[]).includes(file)) return false;

302 const raw = read(file);

303 if (!raw) return false;

304 const text = referenceText(file, raw);

305 if (needles.some(needle => text.includes(needle))) return true;

306 if (binName && new RegExp(`\\b${binName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`).test(text)) return true;

307 return targetsOf(file).some(target => target === missing || target === bare || `${target}/index.ts` === missing);

308 });

309 };

310 const trace = (caseId: string, file: string): string[] | null => {

311 const patterns = maps.e2eTouchfiles[caseId] ?? [];

312 const parents = new Map<string, string>();

313 const reached = closure([...patterns.filter(pattern => OWNER.test(pattern)), ...patterns.flatMap(expand)], parents);

314 if (!reached.has(file)) return null;

315 const chain = [file];

316 while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!);

317 return chain;

318 };

319 return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers, scanErrors };

320}

321

322const cache = new Map<string, DerivedDependencies>();

323

324/** Derived dependencies of the checkout at `root`, computed once per process and map set. */

325export function derivedDependencies(maps: DependencyMaps, root: string): DerivedDependencies {

326 const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}\0${Object.keys(maps.judgeTouchfiles).length}`;

327 if (!cache.has(key)) cache.set(key, deriveDependencies(maps, root));

328 return cache.get(key)!;

329}

330

331// ─── Exercised-path consumer map (W2-A2, shadow mode) ──────────────────────

332

333/** The committed per-case observed commands (generated by scripts/exercised-map.ts; union-only refreshes). */

334export const EXERCISED_MAP_PATH = 'test/helpers/exercised-commands.json';

335

336export interface ExercisedCase {

337 /** Recorded trials (run, case) the commands were taken from. */

338 observations: number;

339 /** Of those, trials that recorded at least one command. */

340 withCommands: number;

341 /** Bins and compiled outfiles the case invoked (binsInCommand over every recorded command). */

342 commands: string[];

343 /** Mean known trial cost per run, for the shadow cost estimate. */

344 costUsd?: number;

345 costRuns?: number;

346}

347

348export interface ExercisedMap {

349 schema: 1;

350 generatedAt: string;

351 observedThrough: string;

352 sourceRuns: string[];

353 cases: Record<string, ExercisedCase>;

354}

355

356export function isExercisedMap(value: unknown): value is ExercisedMap {

357 const map = value as ExercisedMap;

358 return !!map && map.schema === 1 && typeof map.generatedAt === 'string' && typeof map.observedThrough === 'string'

359 && Array.isArray(map.sourceRuns) && map.sourceRuns.every(run => typeof run === 'string')

360 && !!map.cases && typeof map.cases === 'object' && Object.values(map.cases).every(entry => !!entry

361 && Number.isSafeInteger(entry.observations) && Number.isSafeInteger(entry.withCommands)

362 && Array.isArray(entry.commands) && entry.commands.every(command => typeof command === 'string'));

363}

364

365/** Parse an exercised map's text; throws on malformed input so callers fail closed. */

366export function parseExercisedMap(text: string): ExercisedMap {

367 const parsed = JSON.parse(text);

368 if (!isExercisedMap(parsed)) throw new Error(`${EXERCISED_MAP_PATH}: not a schema-1 exercised map`);

369 return parsed;

370}

371

372export interface ExercisedConsumers {

373 /** Repository file -> case ids whose exercised closure contains it. */

374 e2e: Map<string, Set<string>>;

375 /** One case's exercised closure: file -> how it is reached (touchfile, import, skill bash via <file>, observed <command>, declared via <file>, host). */

376 closure(caseId: string): Map<string, string>;

377 /** Bins (repo paths) and compiled outfiles one recorded command invokes. */

378 binsInCommand(command: string): string[];

379 /** The tracked files a bin or compiled outfile runs. */

380 commandFiles(command: string): string[];

381 /** The observed map the consumers were built with. */

382 map: ExercisedMap | null;

383}

384

385const SHELL_SHEBANG = /^#![^\n]*\b(?:ba|z|da)?sh\b/;

387const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/;

388const CMD_PREFIX = /(?:"?(?:\$\{?\w+\}?|\$$dirname\s+"?\$0"?$|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/;

390

391/** Text between `$(`…`)` and backticks in a span: substitutions run even inside a message. */

392function substitutions(span: string): string[] {

393 const out: string[] = [];

394 for (let i = 0; i < span.length; i++) {

395 if (span[i] === '\\') { i++; continue; }

396 if (span[i] === '\x60') { const end = span.indexOf('\x60', i + 1); if (end < 0) break; out.push(`$(${span.slice(i + 1, end)})`); i = end; continue; }

397 if (span[i] !== '$' || span[i + 1] !== '(') continue;

398 let depth = 0, j = i + 1;

399 for (; j < span.length; j++) { if (span[j] === '(') depth++; else if (span[j] === ')' && --depth === 0) break; }

400 out.push(span.slice(i, j + 1));

401 i = j;

402 }

403 return out;

404}

405

406/** Message calls (echo, printf, die …) are prose; their command substitutions still execute and are kept. */

407function stripMessages(text: string): string {

408 let out = '', i = 0;

409 while (i < text.length) {

410 const prev = i === 0 ? '\n' : text[i - 1]!;

411 const call = /[\s;|&(\x60{]/.test(prev) ? MESSAGE_CALL.exec(text.slice(i, i + 8)) : null;

412 if (!call) { out += text[i]; i++; continue; }

413 let j = i + call[0].length, quote: string | null = null, depth = 0;

414 while (j < text.length) {

415 const c = text[j]!;

416 if (c === '\\') { j += 2; continue; }

417 if (quote) { if (c === quote) quote = null; else if (quote === '"' && c === '$' && text[j + 1] === '(') { depth++; j++; } else if (c === ')' && depth) depth--; j++; continue; }

418 if (c === '"' || c === "'") { quote = c; j++; continue; }

419 if (c === '$' && text[j + 1] === '(') { depth++; j += 2; continue; }

420 if (c === ')' && depth) { depth--; j++; continue; }

421 if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break;

422 j++;

423 }

424 out += ` ${substitutions(text.slice(i, j)).join('; ')} `;

425 i = j;

426 }

427 return out;

428}

429

430/**

431 * Executable shell text: comments, case patterns and message calls are

432 * prose. A heredoc an interpreter reads stays whole; an expanding (unquoted)

433 * heredoc keeps its command substitutions; a quoted one fed to anything else

434 * is dropped.

435 */

436export function executableShell(text: string): string {

437 return stripMessages(text

438 .replace(/^(.*?)<<-?\s*(['"]?)(\w+)\2([^\n]*)\n([\s\S]*?)\n\s*\3\s*$/gm, (whole, head: string, quote: string, _word: string, rest: string, body: string) =>

439 INTERPRETER.test(head + rest) ? whole : quote ? head : [head, ...substitutions(body)].join('\n'))

440 .split('\n').filter(line => !/^\s*#(?!!)/.test(line))

441 .map(line => line.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, ''))

442 .join('\n'));

443}

444

445function commandPosition(before: string): boolean {

446 const head = before.replace(CMD_PREFIX, '');

447 return CMD_START.test(head) || (/\/$/.test(before) && /\b\w+="?$/.test(head));

448}

449

450/**

451 * Consumers by exercised paths: a case consumes its touchfile matches, the

452 * closure of its test file, the executable shell of its loaded skill

453 * (template and generated SKILL.md) and of every bin, lib and script that

454 * shell runs, and the bins its recorded trials invoked (`observed`). The

455 * closure stops at `stopAt` (the full-gate files) and the global set,

456 * which stay full.

457 */

458export function exercisedConsumers(maps: DependencyMaps, root: string, options: { observed: ExercisedMap | null; stopAt: readonly string[] }): ExercisedConsumers {

459 const base = derivedDependencies(maps, root);

460 const tracked = [...base.tracked];

461 const texts = new Map<string, string>();

462 const read = (file: string): string => {

463 if (!texts.has(file)) {

464 let text = '';

465 try { if (base.tracked.has(file) && fs.statSync(path.join(root, file)).size < 3_000_000) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ }

466 texts.set(file, text);

467 }

468 return texts.get(file)!;

469 };

470 const isShell = (file: string) => !isCodeFile(file, read(file)) && (/\.sh$/.test(file) || file === 'setup' || SHELL_SHEBANG.test(read(file)));

471 const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));

472 const expandRef = (ref: string): string[] => {

473 const clean = ref.replace(/[.]+$/, '');

474 if (base.tracked.has(clean)) return [clean];

475 return dirs.has(clean) ? tracked.filter(file => file.startsWith(`${clean}/`)) : [];

476 };

477 const expanded = new Map<string, string[]>();

478 const expand = (pattern: string): string[] => {

479 if (!pattern.includes('*')) return base.tracked.has(pattern) ? [pattern] : [];

480 if (!expanded.has(pattern)) expanded.set(pattern, tracked.filter(file => matchGlob(file, pattern)));

481 return expanded.get(pattern)!;

482 };

483 const binByName = new Map<string, string>();

484 for (const file of tracked.filter(file => file.startsWith('bin/') || file.startsWith('browse/bin/'))) {

485 binByName.set(path.posix.basename(file), file);

486 binByName.set(path.posix.basename(file).replace(/\.(?:ts|sh|js)$/, ''), file);

487 }

488 const binaries = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)]

489 .map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? [match[1]!] : tracked.filter(file => matchGlob(file, `${path.posix.dirname(match[1]!)}/**`)) }));

490 const outfileHit = (outfile: string, text: string) => text.includes(outfile)

491 || (outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(text)) || (outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(text));

492 let scripts: Record<string, string> = {};

493 try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ }

494

495 const shellEdges = (text: string, from: string, out: Set<string>) => {

496 for (const match of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) {

497 const start = text.lastIndexOf('\n', match.index! - 1) + 1;

498 const bin = binByName.get(match[0]);

499 if (bin && commandPosition(text.slice(start, match.index))) out.add(bin);

500 }

501 for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) {

502 const before = text.slice(text.lastIndexOf('\n', match.index! - 1) + 1, match.index);

503 if (/[\w.@-]$/.test(before)) continue;

504 if (commandPosition(before) || /\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) for (const file of expandRef(match[0])) out.add(file);

505 }

507 for (const match of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) {

508 for (const ref of match[1]!.matchAll(/(?:bin|lib|scripts)\/[\w.@-]+(?:\/[\w.@-]+)*\.sh\b/g)) for (const file of expandRef(ref[0])) out.add(file);

509 const name = [...match[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(found => found[1]!).at(-1);

510 if (!name) continue;

511 const sibling = path.posix.join(path.posix.dirname(from), name);

512 if (base.tracked.has(sibling)) out.add(sibling);

513 else if (binByName.has(name)) out.add(binByName.get(name)!);

514 }

515 if (/(?:^|[\s;&|(])"?(?:\.|\$\{?\w+\}?)\/setup\b/m.test(text) && base.tracked.has('setup')) out.add('setup');

516 for (const { outfile, tree } of binaries) if (outfileHit(outfile, text)) for (const file of tree) out.add(file);

517 for (const match of text.matchAll(PACKAGE_RUN)) {

518 const command = scripts[match[1]!];

519 if (command) for (const ref of command.matchAll(REPO_PATH)) for (const file of expandRef(ref[1]!)) out.add(file);

520 }

521 };

522 const edges = new Map<string, string[]>();

523 const execRefs = (file: string): string[] => {

524 if (edges.has(file)) return edges.get(file)!;

525 const raw = read(file);

526 const out = new Set<string>();

527 if (raw && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {

528 if (isCodeFile(file, raw)) {

529 try { for (const specifier of scanCodeImports(file, raw)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } }

530 catch (error) { base.scanErrors.set(file, String(error)); }

531 const code = raw.replace(/^#![^\n]*/, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, '');

532 for (const match of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const hit of expandRef(match[1]!)) out.add(hit);

533 for (const match of code.matchAll(JOINED_PATH)) for (const hit of expandRef(`${match[1]}/${match[2]}`)) out.add(hit);

534 if (/join$[^)\n]*['"]setup['"]\s*$/.test(code) && base.tracked.has('setup')) out.add('setup');

535 for (const { outfile, tree } of binaries) if (code.includes(`'${outfile}'`) || code.includes(`"${outfile}"`)) for (const hit of tree) out.add(hit);

536 } else if (isShell(file)) {

537 shellEdges(executableShell(raw), file, out);

538 } else if (/\.(?:md|tmpl)$/.test(file)) {

539 shellEdges(executableShell([...raw.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(match => match[1]!).join('\n')), file, out);

540 }

541 }

542 out.delete(file);

543 edges.set(file, [...out]);

544 return edges.get(file)!;

545 };

546 const stopped = new Map<string, boolean>();

547 const stops = (file: string): boolean => {

548 if (!stopped.has(file)) stopped.set(file, base.global.has(file) || options.stopAt.some(pattern => matchGlob(file, pattern)));

549 return stopped.get(file)!;

550 };

551 const binsInCommand = (command: string): string[] => {

552 const out = new Set<string>();

553 const pty = command.startsWith('PTY:');

554 for (const match of command.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) {

555 let token = match[0];

556 if (pty) while (token.length > 7 && !binByName.has(token)) { const cut = token.lastIndexOf('-'); token = cut > 6 ? token.slice(0, cut) : ''; }

557 const at = match.index!;

558 if (command[at - 1] === '/' && at >= 2 && /[\s"'\x60(]/.test(command[at - 2]!)) continue;

559 const bin = binByName.get(token);

560 if (bin) out.add(bin);

561 }

562 for (const match of command.matchAll(/browse\/bin\/([a-z-]+)/g)) { const bin = binByName.get(match[1]!); if (bin) out.add(bin); }

563 for (const { outfile } of binaries) if (outfileHit(outfile, command) || (outfile === 'make-pdf/dist/pdf' && command.includes(outfile))) out.add(outfile);

564 return [...out];

565 };

566 const commandFiles = (command: string) => base.tracked.has(command) ? [command] : binaries.find(binary => binary.outfile === command)?.tree ?? [];

567

568 const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts'));

569 const closures = new Map<string, Map<string, string>>();

570 const caseClosure = (id: string): Map<string, string> => {

571 if (closures.has(id)) return closures.get(id)!;

572 const patterns = maps.e2eTouchfiles[id] ?? [];

573 const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern));

574 const declared = patterns.flatMap(expand);

575 const generated = declared.filter(file => file.endsWith('.tmpl')).map(file => file.replace(/\.tmpl$/, '')).filter(file => base.tracked.has(file));

576 const touchfiles = new Set([...declared, ...generated]);

577 const skillDoc = (file: string) => /\.(?:md|tmpl)$/.test(file);

578 const seeds: Array<[string, string]> = [

579 ...owners.map(file => [file, 'import'] as [string, string]),

580 ...[...declared, ...generated].map(file => [file, skillDoc(file) ? 'skill' : `declared via ${file}`] as [string, string]),

581 ...(options.observed?.cases[id]?.commands ?? []).flatMap(command => commandFiles(command).map(file => [file, `observed ${command}`] as [string, string])),

582 ];

583 const ownerText = owners.map(read).join('\n');

584 for (const host of hostIds) {

585 if (new RegExp(`['"\x60]${host}['"\x60]|--host[ =]${host}\\b`).test(ownerText)) for (const file of [`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)]) seeds.push([file, 'host']);

586 }

587 const reached = new Map<string, string>();

588 const queue = seeds.filter(([file]) => base.tracked.has(file));

589 for (let head = 0; head < queue.length; head++) {

590 const [file, label] = queue[head]!;

591 if (reached.has(file)) continue;

592 reached.set(file, touchfiles.has(file) ? 'touchfile' : label);

593 if (stops(file)) continue;

594 for (const next of execRefs(file)) {

595 if (reached.has(next) || (/\.test\.tsx?$/.test(next) && isPaidTestFile(next))) continue;

596 queue.push([next, label === 'skill' ? `skill bash via ${next}` : label]);

597 }

598 }

599 closures.set(id, reached);

600 return reached;

601 };

602 const e2e = new Map<string, Set<string>>();

603 for (const id of Object.keys(maps.e2eTouchfiles)) {

604 for (const file of caseClosure(id).keys()) {

605 if (!e2e.has(file)) e2e.set(file, new Set());

606 e2e.get(file)!.add(id);

607 }

608 }

609 return { e2e, closure: caseClosure, binsInCommand, commandFiles, map: options.observed };

610}

611

612const exercisedCache = new Map<string, ExercisedConsumers>();

613

614/** The exercised consumers of the checkout at `root` with its committed map, computed once per process; throws when the map is malformed. */

615export function checkoutExercisedConsumers(maps: DependencyMaps, root: string, stopAt: readonly string[]): ExercisedConsumers {

616 const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}`;

617 if (!exercisedCache.has(key)) {

618 const observed = parseExercisedMap(fs.readFileSync(path.join(root, EXERCISED_MAP_PATH), 'utf8'));

619 exercisedCache.set(key, exercisedConsumers(maps, root, { observed, stopAt }));

620 }

621 return exercisedCache.get(key)!;

622}