skill-check.ts
202 lines · scripts/skill-check.ts at 3f79d81
1#!/usr/bin/env -S bun --no-env-file --config=/dev/null
2/** Validate all host renders and the freshness of repository-owned outputs.
3 * Optional installed host caches are never read or changed. */
5import * as fs from 'fs';
6import * as os from 'os';
7import * as path from 'path';
8import { spawnSync } from 'child_process';
9import { validateSkill, externalHostPathLeaks } from '../test/helpers/skill-parser';
10import { ALL_HOST_CONFIGS } from '../hosts/index';
11import {
12 runGeneration,
13 type GeneratedArtifact,
14} from './gen-skill-docs';
16const ROOT = path.resolve(import.meta.dir, '..');
18export interface CheckDiagnostic {
19 kind: 'error' | 'invalid' | 'stale' | 'untracked';
20 relativePath?: string;
21 message: string;
22}
24/** Match the strict YAML contract used by hosts, rather than accepting fields
25 * that merely look present to the generator's extraction regexes. */
26export function validateSkillFrontmatter(content: string): string[] {
27 const block = content.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
28 if (!block) return ['frontmatter must have opening and closing delimiters'];
29 let parsed: unknown;
30 try {
31 parsed = Bun.YAML.parse(block[1]);
32 } catch (error) {
33 return [`frontmatter must be valid YAML: ${error instanceof Error ? error.message : String(error)}`];
34 }
35 const fields = parsed as Record<string, unknown> | null;
36 return ['name', 'description'].flatMap(field =>
37 typeof fields?.[field] === 'string' && (fields[field] as string).trim()
38 ? [] : [`frontmatter ${field} must be a nonempty string`]);
39}
41/** Reuse the command/snapshot validator and host smoke-test content rules. */
42export function validateGeneratedArtifact(renderRoot: string, artifact: GeneratedArtifact): CheckDiagnostic[] {
43 const diagnostics: CheckDiagnostic[] = [];
44 const invalid = (message: string) => diagnostics.push({ kind: 'invalid', relativePath: artifact.relativePath, message });
45 const file = path.join(renderRoot, artifact.relativePath);
46 try {
47 const content = fs.readFileSync(file, 'utf-8');
48 if (!content.trim()) invalid('generated file is empty');
49 if (artifact.kind === 'skill') {
50 for (const message of validateSkillFrontmatter(content)) invalid(message);
51 }
52 if (artifact.kind === 'skill' || artifact.kind === 'section' || artifact.kind === 'preamble') {
53 const validation = validateSkill(file);
54 for (const command of validation.invalid) invalid(`line ${command.line}: unknown command '${command.command}'`);
55 for (const error of validation.snapshotFlagErrors) invalid(`line ${error.command.line}: ${error.error}`);
56 if (artifact.host && artifact.host !== 'claude') {
57 // Host smoke tests permit legitimate fallback paths in bash examples.
58 if (externalHostPathLeaks(content).length) invalid('contains .claude/skills reference outside a bash block');
59 }
60 }
61 if (artifact.kind === 'metadata') {
62 for (const field of ['display_name:', 'short_description:', 'default_prompt:', 'allow_implicit_invocation: true']) {
63 if (!content.includes(field)) invalid(`metadata is missing ${field}`);
64 }
65 }
66 } catch (error) {
67 diagnostics.push({ kind: 'error', relativePath: artifact.relativePath, message: (error as Error).message });
68 }
69 return diagnostics;
70}
72function git(root: string, args: string[], input?: string, accepted = [0]): string {
73 const result = spawnSync('git', args, { cwd: root, encoding: 'utf-8', input, timeout: 30_000, maxBuffer: 16 * 1024 * 1024 });
74 if (result.error || result.status === null || !accepted.includes(result.status)) {
75 throw new Error(`git ${args.join(' ')} failed: ${result.error?.message || result.stderr.trim() || `exit ${result.status}`}`);
76 }
77 return result.stdout;
78}
80/** Git refuses to check paths beneath an installed symlink. Evaluate the
81 * expected generated tree in an empty worktree view, retaining the repository's
82 * Git excludes and applicable source .gitignore rules. Never follow cache links
83 * while finding those rules, and never change the render or source checkout. */
84function ignoredGeneratedPaths(repoRoot: string, paths: string[]): Set<string> {
85 if (!paths.length) return new Set();
86 const view = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-ignore-check-'));
87 try {
88 const gitDir = git(repoRoot, ['rev-parse', '--absolute-git-dir']).trim();
89 const excludesFile = git(repoRoot, ['config', '--path', '--get', 'core.excludesFile'], undefined, [0, 1]).trim();
90 const directories = new Map<string, boolean>([['', true]]);
91 for (const file of paths) {
92 let dir = '';
93 for (const segment of file.split('/').slice(0, -1)) {
94 dir = dir ? `${dir}/${segment}` : segment;
95 if (!directories.has(dir)) {
96 try {
97 directories.set(dir, fs.lstatSync(path.join(repoRoot, dir)).isDirectory());
98 } catch (error) {
99 if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
100 directories.set(dir, false);
101 }
102 }
103 if (!directories.get(dir)) break;
104 }
105 }
106 for (const [dir, reachable] of directories) {
107 if (!reachable) continue;
108 const source = path.join(repoRoot, dir, '.gitignore');
109 try {
110 // Git also ignores symlinked .gitignore files.
111 if (!fs.lstatSync(source).isFile()) continue;
112 const destination = path.join(view, dir, '.gitignore');
113 fs.mkdirSync(path.dirname(destination), { recursive: true });
114 fs.copyFileSync(source, destination);
115 } catch (error) {
116 if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
117 }
118 }
119 return new Set(git(view, ['--git-dir', gitDir, '--work-tree', view,
120 ...(excludesFile ? ['-c', `core.excludesFile=${path.resolve(repoRoot, excludesFile)}`] : []),
121 'check-ignore', '--no-index', '-z', '--stdin'], `${paths.join('\0')}\0`, [0, 1]).split('\0').filter(Boolean));
122 } finally {
123 fs.rmSync(view, { recursive: true, force: true });
124 }
125}
127/** Compare only generated targets: unrelated dirty files cannot fail this check.
128 * An expected nonignored output must also be tracked, catching new skills whose
129 * generated file was never committed. Tracked files win over ignore rules. */
130export function checkGeneratedFreshness(repoRoot: string, renderRoot: string, artifacts: GeneratedArtifact[]): { checked: number; diagnostics: CheckDiagnostic[] } {
131 const diagnostics: CheckDiagnostic[] = [];
132 let checked = 0;
133 try {
134 const tracked = new Set(git(repoRoot, ['ls-files', '-z']).split('\0').filter(Boolean));
135 const untracked = artifacts.map(a => a.relativePath).filter(file => !tracked.has(file));
136 const ignored = ignoredGeneratedPaths(repoRoot, untracked);
137 for (const artifact of artifacts) {
138 const { relativePath } = artifact;
139 if (!tracked.has(relativePath)) {
140 if (!ignored.has(relativePath)) {
141 diagnostics.push({ kind: 'untracked', relativePath, message: 'generated output is not tracked; regenerate and add it to git' });
142 }
143 continue;
144 }
145 checked++;
146 try {
147 const generated = fs.readFileSync(path.join(renderRoot, relativePath));
148 const existing = fs.readFileSync(path.join(repoRoot, relativePath));
149 if (!generated.equals(existing)) diagnostics.push({ kind: 'stale', relativePath, message: 'generated output differs from the tracked file' });
150 } catch (error) {
151 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
152 diagnostics.push({ kind: 'stale', relativePath, message: 'generated or tracked output is missing' });
153 } else {
154 diagnostics.push({ kind: 'error', relativePath, message: (error as Error).message });
155 }
156 }
157 }
158 } catch (error) {
159 diagnostics.push({ kind: 'error', message: (error as Error).message });
160 }
161 return { checked, diagnostics };
162}
164export async function main(): Promise<number> {
165 const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-skill-check-'));
166 try {
167 console.log('Generating all hosts into scratch with canonical settings...');
168 const result = await runGeneration({ host: 'all', outputRoot: scratch, contentLinkRoot: null });
169 for (const diagnostic of result.diagnostics) {
170 console.log(`${diagnostic.kind.toUpperCase()}: ${diagnostic.host ? `${diagnostic.host}: ` : ''}${diagnostic.message}`);
171 }
172 if (result.exitCode !== 0) {
173 console.error('Generation failed; freshness and content checks require a complete render.');
174 return 1;
175 }
177 const diagnostics = result.artifacts.flatMap(artifact => validateGeneratedArtifact(scratch, artifact));
178 for (const host of ALL_HOST_CONFIGS) {
179 const skills = result.artifacts.filter(a => a.host === host.name && a.kind === 'skill').length;
180 console.log(` ${host.displayName}: ${skills} generated skills checked`);
181 }
182 const freshness = checkGeneratedFreshness(ROOT, scratch, result.artifacts);
183 diagnostics.push(...freshness.diagnostics);
184 for (const diagnostic of diagnostics) {
185 console.error(`${diagnostic.kind.toUpperCase()}: ${diagnostic.relativePath ? `${diagnostic.relativePath}: ` : ''}${diagnostic.message}`);
186 }
187 console.log(`\nChecked ${result.artifacts.length} generated artifacts; ${freshness.checked} tracked outputs compared.`);
188 if (diagnostics.some(d => d.kind === 'stale' || d.kind === 'untracked')) {
189 console.error('Run: bun run gen:skill-docs --host all, then commit the generated outputs.');
190 }
191 return diagnostics.length ? 1 : 0;
192 } finally {
193 fs.rmSync(scratch, { recursive: true, force: true });
194 }
195}
197if (import.meta.main) {
198 void main().then(code => { process.exitCode = code; }).catch(error => {
199 console.error(`ERROR: ${error instanceof Error ? error.message : String(error)}`);
200 process.exitCode = 1;
201 });
202}