gen-llms-txt.ts
270 lines · scripts/gen-llms-txt.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/**
3 * Generate gstack/llms.txt — a single discoverable index of every gstack
4 * capability for AI agents.
5 *
6 * Inputs:
7 * - Skill SKILL.md.tmpl frontmatter (name, description) at root and one
8 * level deep, via scripts/discover-skills.ts
9 * - browse/src/commands.ts COMMAND_DESCRIPTIONS
10 * - design/src/commands.ts COMMAND_DESCRIPTIONS (if present)
11 *
12 * Output: gstack/llms.txt at repo root.
13 *
14 * Skill links point at the Claude host's generated SKILL.md (the files this
15 * checkout ships). Skills the Claude host skips (claude-code) have no such
16 * file, so they are left out of the index rather than linked to a template.
17 *
18 * Refresh: invoked from scripts/gen-skill-docs.ts after SKILL.md generation
19 * so it regenerates automatically on every skill change.
20 *
21 * Convention: https://llmstxt.org/ (single-file index agents can crawl).
22 */
24import * as fs from 'fs';
25import * as path from 'path';
26import { mkdirpSync } from '../lib/fs-utils';
27import { discoverTemplates, includesSkill } from './discover-skills';
28import claudeHost from '../hosts/claude';
29import { COMMAND_DESCRIPTIONS as BROWSE_COMMANDS } from '../browse/src/commands';
31const ROOT = path.resolve(import.meta.dir, '..');
32const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt');
34interface SkillEntry {
35 name: string;
36 description: string;
37 output: string;
38}
40/**
41 * Parse YAML frontmatter at the top of a SKILL.md.tmpl file. We only need
42 * `name` and `description`. description: | followed by indented lines is
43 * the gstack convention; we collapse those into a single paragraph.
44 */
45function parseSkillFrontmatter(filePath: string): Omit<SkillEntry, 'output'> | null {
46 const content = fs.readFileSync(filePath, 'utf-8');
47 if (!content.startsWith('---')) return null;
48 const end = content.indexOf('\n---', 3);
49 if (end < 0) return null;
50 const frontmatter = content.slice(3, end).split('\n');
52 let name = '';
53 let description = '';
54 let inDescription = false;
55 let descriptionLines: string[] = [];
57 for (const rawLine of frontmatter) {
58 const line = rawLine.replace(/\r$/, '');
59 if (inDescription) {
60 // Block-scalar continues until a non-indented (or differently-keyed) line.
61 if (line.startsWith(' ') || line === '') {
62 descriptionLines.push(line.replace(/^ /, ''));
63 continue;
64 }
65 inDescription = false;
66 // Fall through to normal key parsing for this line.
67 }
68 const m = line.match(/^([a-zA-Z_-]+):\s*(.*)$/);
69 if (!m) continue;
70 const key = m[1];
71 const value = m[2];
72 if (key === 'name') {
73 name = value.trim();
74 } else if (key === 'description') {
75 if (value === '|' || value === '|-' || value === '>' || value === '>-') {
76 inDescription = true;
77 descriptionLines = [];
78 } else {
79 description = value.trim();
80 }
81 }
82 }
84 if (!description && descriptionLines.length) {
85 description = descriptionLines
86 .map((l) => l.trim())
87 .filter(Boolean)
88 .join(' ')
89 .trim();
90 }
92 if (!name) return null;
93 if (!description) return null;
94 return { name, description };
95}
97/**
98 * Best-effort import of the design CLI's COMMAND_DESCRIPTIONS. Only present
99 * in a full gstack checkout; absent on minimal installs. Returns {} if the
100 * module isn't found rather than throwing.
101 */
102async function readDesignCommands(): Promise<Record<string, { category: string; description: string; usage?: string }>> {
103 const designCommandsPath = path.join(ROOT, 'design', 'src', 'commands.ts');
104 if (!fs.existsSync(designCommandsPath)) return {};
105 try {
106 const mod: unknown = await import(designCommandsPath);
107 const m = mod as { COMMAND_DESCRIPTIONS?: Record<string, { category: string; description: string; usage?: string }> };
108 return m.COMMAND_DESCRIPTIONS ?? {};
109 } catch {
110 return {};
111 }
112}
114/**
115 * Render a one-line summary from a multi-paragraph description: take the
116 * first sentence (up to '.', '!', or '?') and trim. Keeps llms.txt scannable.
117 */
118function oneLine(text: string): string {
119 const first = text.split(/(?<=[.!?])\s/)[0] ?? text;
120 return first.replace(/\s+/g, ' ').trim();
121}
123interface GenerateOptions {
124 /** Override repo root (for tests). */
125 root?: string;
126 /** When true, missing skill description should fail the build. */
127 strict?: boolean;
128 /** Skill directories a module loadout leaves out (scripts/modules.ts). */
129 excludedSkills?: string[];
130}
132export interface GenerateResult {
133 content: string;
134 skills: SkillEntry[];
135 browseCommands: string[];
136 designCommands: string[];
137 warnings: string[];
138}
140export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<GenerateResult> {
141 const root = opts.root ?? ROOT;
142 const warnings: string[] = [];
144 const templates = discoverTemplates(root);
145 const skills: SkillEntry[] = [];
146 for (const t of templates) {
147 if (!includesSkill(claudeHost, path.dirname(t.tmpl))) continue;
148 if (opts.excludedSkills?.includes(path.dirname(t.tmpl))) continue;
149 const filePath = path.join(root, t.tmpl);
150 const entry = parseSkillFrontmatter(filePath);
151 if (!entry) {
152 warnings.push(`skill ${t.tmpl}: missing name or description in frontmatter`);
153 if (opts.strict) {
154 throw new Error(`gen-llms-txt: ${t.tmpl} is missing name or description in frontmatter`);
155 }
156 continue;
157 }
158 skills.push({ ...entry, output: t.output });
159 }
160 skills.sort((a, b) => a.name.localeCompare(b.name));
162 const browseCommands = Object.keys(BROWSE_COMMANDS).sort();
163 const designCommands = Object.keys(await readDesignCommands()).sort();
165 const lines: string[] = [];
166 lines.push('# gstack');
167 lines.push('');
168 lines.push("> gstack is Garry's Stack: AI coding skills + a fast headless browser binary + a design CLI. This file indexes every capability so agents can discover and invoke them without crawling individual SKILL.md files.");
169 lines.push('');
170 lines.push('Conventions:');
171 lines.push('- Skills are invoked by name (e.g. `/ship`, `/plan-ceo-review`).');
172 lines.push('- Browse commands run as `browse <command> [args]` (or `$B` shorthand).');
173 lines.push('- Design commands run as `design <command> [args]` (or `$D`).');
174 lines.push('- Project-specific config lives in `CLAUDE.md`. Always read it first.');
175 lines.push('');
177 lines.push('## Skills');
178 lines.push('');
179 for (const skill of skills) {
180 const summary = oneLine(skill.description);
181 lines.push(`- [/${skill.name}](${skill.output}): ${summary}`);
182 }
183 lines.push('');
185 lines.push('## Browse Commands');
186 lines.push('');
187 lines.push('Run with `browse <command> [args]`. Full reference: `browse/SKILL.md`.');
188 lines.push('');
189 const byCategory: Record<string, Array<{ name: string; description: string; usage?: string }>> = {};
190 for (const cmd of browseCommands) {
191 const meta = BROWSE_COMMANDS[cmd];
192 const cat = meta.category || 'Other';
193 if (!byCategory[cat]) byCategory[cat] = [];
194 byCategory[cat].push({ name: cmd, description: meta.description, usage: meta.usage });
195 }
196 for (const cat of Object.keys(byCategory).sort()) {
197 lines.push(`### ${cat}`);
198 for (const cmd of byCategory[cat]) {
199 const usage = cmd.usage ? `\`${cmd.usage}\`` : `\`${cmd.name}\``;
200 lines.push(`- ${usage}: ${oneLine(cmd.description)}`);
201 }
202 lines.push('');
203 }
205 if (designCommands.length > 0) {
206 lines.push('## Design Commands');
207 lines.push('');
208 lines.push('Run with `design <command> [args]`. Full reference: `design/SKILL.md`.');
209 lines.push('');
210 const designMeta = await readDesignCommands();
211 for (const cmd of designCommands) {
212 const meta = designMeta[cmd];
213 lines.push(`- \`${cmd}\`: ${oneLine(meta.description)}`);
214 }
215 lines.push('');
216 }
218 lines.push('## More');
219 lines.push('');
220 lines.push('- Repository: https://github.com/garrytan/gstack');
221 lines.push('- Top-level guide: `SKILL.md`');
222 lines.push('- Project ethos: `ETHOS.md`');
223 lines.push('- This file is auto-generated by `bun run gen:skill-docs`.');
224 lines.push('');
226 return {
227 content: lines.join('\n'),
228 skills,
229 browseCommands,
230 designCommands,
231 warnings,
232 };
233}
235export async function writeLlmsTxt(opts: GenerateOptions & { outputPath?: string } = {}): Promise<GenerateResult> {
236 const result = await generateLlmsTxt(opts);
237 const outputPath = opts.outputPath ?? OUTPUT;
238 mkdirpSync(path.dirname(outputPath));
239 fs.writeFileSync(outputPath, result.content, { encoding: 'utf-8' });
240 return result;
241}
243// ─── CLI entry ──────────────────────────────────────────────
244// Wrapped in an IIFE so top-level await doesn't make this module async-by-
245// import (which would break require() consumers like
246// test/gen-skill-docs.test.ts that pull writeLlmsTxt indirectly via
247// gen-skill-docs).
248if (import.meta.main) {
249 void (async () => {
250 const strict = process.argv.includes('--strict');
251 const dryRun = process.argv.includes('--dry-run');
252 const result = dryRun
253 ? await generateLlmsTxt({ strict })
254 : await writeLlmsTxt({ strict });
256 for (const w of result.warnings) console.error(`[gen-llms-txt] WARN: ${w}`);
258 if (dryRun) {
259 const existing = fs.existsSync(OUTPUT) ? fs.readFileSync(OUTPUT, 'utf-8') : '';
260 if (existing !== result.content) {
261 console.error('[gen-llms-txt] OUT OF DATE — run `bun run gen:skill-docs` to regenerate gstack/llms.txt');
262 process.exit(1);
263 }
264 console.log('[gen-llms-txt] up to date');
265 } else {
266 console.log(`[gen-llms-txt] wrote ${OUTPUT}`);
267 console.log(`[gen-llms-txt] skills=${result.skills.length} browse=${result.browseCommands.length} design=${result.designCommands.length}`);
268 }
269 })();
270}