Skills › gstack › gstack › cso-scanner-catalog.ts

cso-scanner-catalog.ts

173 lines · scripts/cso-scanner-catalog.ts at 3f79d81

Open on GitHub

1#!/usr/bin/env -S bun --no-env-file --config=/dev/null

2/** Assemble and validate a complete source-controlled scanner catalog proposal. */

3import * as fs from 'node:fs';

4import { createHash } from 'node:crypto';

5import { basename, join, relative, resolve, sep } from 'node:path';

6import { canonical, sha256 } from '../lib/cso/contracts';

7import { QualifiedScanner, ScannerCatalog, scannerVersionHash, validateQualifiedScanner, validateScannerCatalog } from '../lib/cso/scanner-catalog';

8import { SCANNER_IDS, scannerPlans } from '../lib/cso/scanners';

10const PLATFORMS = ['linux/amd64', 'linux/arm64'] as const;

11const REVISION = /^[a-z0-9][a-z0-9._-]{0,100}$/;

12const WORKFLOW = /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/actions\/runs\/[0-9]+$/;

13const MAX_ARTIFACT = 1_048_576;

14const MAX_TREE_BYTES = 10 * 1024 * 1024 * 1024;

15const MAX_TREE_ENTRIES = 100_000;

17function sameFile(left: fs.Stats, right: fs.Stats): boolean {

18 return left.dev === right.dev && left.ino === right.ino && left.mode === right.mode && left.nlink === right.nlink && left.size === right.size;

19}

20function hashRegularFile(path: string, expected: fs.Stats): string {

21 const hash = createHash('sha256'), fd = fs.openSync(path, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));

22 try {

23 const before = fs.fstatSync(fd);

24 if (!before.isFile() || before.nlink !== 1 || !sameFile(expected, before)) throw new Error('UNSAFE_SCANNER_ASSET');

25 const buffer = Buffer.alloc(1024 * 1024); let count = 0, bytes = 0;

26 while ((count = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) {

27 bytes += count;

28 if (bytes > expected.size || bytes > MAX_TREE_BYTES) throw new Error('SCANNER_ASSET_LIMIT');

29 hash.update(buffer.subarray(0, count));

30 }

31 const after = fs.fstatSync(fd), pathname = fs.lstatSync(path);

32 if (bytes !== expected.size || !sameFile(before, after) || !sameFile(after, pathname)) throw new Error('UNSAFE_SCANNER_ASSET');

33 return hash.digest('hex');

34 } finally { fs.closeSync(fd); }

35}

37function sameStrings(left: string[], right: string[]): boolean {

38 return canonical([...left].sort()) === canonical([...right].sort());

39}

40function exactKeys(value: Record<string, unknown>, allowed: string[], label: string): void {

41 if (Object.keys(value).some(key => !allowed.includes(key))) throw new Error(`UNEXPECTED_${label.toUpperCase()}_FIELD`);

42}

43function strictProfile(value: unknown): QualifiedScanner {

44 if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('INVALID_SCANNER_FRAGMENT');

45 const profile = value as QualifiedScanner & Record<string, unknown>;

46 exactKeys(profile, ['id', 'scanner', 'state', 'platform', 'image', 'entrypoint', 'executable', 'version', 'versionOutputSha256', 'helperAbi', 'isolationPolicyHash', 'capabilities', 'assets', 'qualifiedAt', 'qualification'], 'scanner_profile');

47 if (!profile.qualification || typeof profile.qualification !== 'object' || Array.isArray(profile.qualification)) throw new Error('INVALID_SCANNER_QUALIFICATION');

48 exactKeys(profile.qualification as unknown as Record<string, unknown>, ['sourceCommit', 'workflow', 'sbomDigest', 'provenanceDigest', 'verifiedProvenance', 'containmentPassed', 'adapterContractPassed', 'offlineAssetsPassed'], 'scanner_qualification');

49 if (profile.assets !== undefined) {

50 if (!profile.assets || typeof profile.assets !== 'object' || Array.isArray(profile.assets)) throw new Error('INVALID_SCANNER_ASSETS');

51 exactKeys(profile.assets as unknown as Record<string, unknown>, ['semgrepRules', 'advisoryDatabase'], 'scanner_assets');

52 for (const [name, asset] of Object.entries(profile.assets)) {

53 if (!asset || typeof asset !== 'object' || Array.isArray(asset)) throw new Error('INVALID_SCANNER_ASSETS');

54 exactKeys(asset as unknown as Record<string, unknown>, name === 'semgrepRules' ? ['path', 'sha256'] : ['path', 'contentSha256', 'updatedAt', 'ecosystems'], 'scanner_asset');

55 }

56 }

57 validateQualifiedScanner(profile);

58 const required = scannerPlans({ snapshotRoot: '/source', offline: true, selected: [profile.scanner] })[0].requiredFeatures;

59 if (!sameStrings(profile.capabilities, required)) throw new Error(`CAPABILITY_CONTRACT_MISMATCH: ${profile.scanner}`);

60 return profile;

61}

63export function scannerCatalogProposal(current: ScannerCatalog, fragments: unknown[], revision: string, expected: { sourceCommit: string; workflow: string; imagePrefix: string }): ScannerCatalog {

64 validateScannerCatalog(current);

65 if (!REVISION.test(revision) || revision === current.revision) throw new Error('INVALID_SCANNER_CATALOG_REVISION');

66 if (!expected || !/^[a-f0-9]{40}$/.test(expected.sourceCommit)) throw new Error('INVALID_EXPECTED_SOURCE_COMMIT');

67 if (!WORKFLOW.test(expected.workflow) || !expected.workflow.startsWith('https://github.com/garrytan/gstack/actions/runs/')) throw new Error('INVALID_EXPECTED_WORKFLOW');

68 if (expected.imagePrefix !== 'ghcr.io/garrytan/gstack/cso-scanners/') throw new Error('INVALID_EXPECTED_IMAGE_PREFIX');

69 const scanners = fragments.map(strictProfile), identities = new Set<string>();

70 if (scanners.length !== SCANNER_IDS.length * PLATFORMS.length) throw new Error('INCOMPLETE_SCANNER_CATALOG_MATRIX');

71 for (const profile of scanners) {

72 const identity = `${profile.scanner}:${profile.platform}`;

73 if (identities.has(identity)) throw new Error(`DUPLICATE_SCANNER_CATALOG_PROFILE: ${identity}`);

74 identities.add(identity);

75 if (profile.qualification.sourceCommit !== expected.sourceCommit) throw new Error(`SOURCE_COMMIT_MISMATCH: ${identity}`);

76 if (profile.qualification.workflow !== expected.workflow) throw new Error(`WORKFLOW_IDENTITY_MISMATCH: ${identity}`);

77 if (!profile.image.startsWith(expected.imagePrefix)) throw new Error(`IMAGE_REPOSITORY_MISMATCH: ${identity}`);

78 }

79 for (const scanner of SCANNER_IDS) for (const platform of PLATFORMS) if (!identities.has(`${scanner}:${platform}`)) throw new Error(`MISSING_SCANNER_CATALOG_PROFILE: ${scanner}:${platform}`);

80 scanners.sort((a, b) => a.scanner.localeCompare(b.scanner) || a.platform.localeCompare(b.platform));

81 const proposal: ScannerCatalog = {

82 schemaVersion: 1,

83 revision,

84 previousRevision: current.revision,

85 helperAbi: 3,

86 promotion: { sourceCommit: expected.sourceCommit, workflow: expected.workflow, evidenceDigest: `sha256:${sha256(canonical(scanners))}` },

87 scanners,

88 };

89 validateScannerCatalog(proposal);

90 return proposal;

91}

93/** Promotion is a compare-and-swap against the catalog revision reviewed by qualification. */

94export function validateScannerCatalogTransition(current: ScannerCatalog, proposed: ScannerCatalog): void {

95 validateScannerCatalog(current); validateScannerCatalog(proposed);

96 if (proposed.revision === current.revision || proposed.previousRevision !== current.revision)

97 throw new Error('SCANNER_CATALOG_BASE_REVISION_MISMATCH');

98}

100function readJson(path: string): unknown {

101 const stat = fs.lstatSync(path);

102 if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1 || stat.size > MAX_ARTIFACT) throw new Error(`UNSAFE_CATALOG_ARTIFACT: ${basename(path)}`);

103 return JSON.parse(fs.readFileSync(path, 'utf8'));

104}

105function fragments(directory: string): unknown[] {

106 const root = fs.realpathSync(directory), values: unknown[] = [];

107 for (const name of fs.readdirSync(root).sort()) {

108 if (!/^[a-z0-9._-]+\.json$/.test(name)) throw new Error(`UNSAFE_FRAGMENT_NAME: ${name}`);

109 values.push(readJson(join(root, name)));

110 }

111 return values;

112}

113

114/** A file hashes as its bytes; a directory hashes a canonical regular-file inventory. */

115export function scannerAssetHash(path: string): string {

116 const supplied = fs.lstatSync(path);

117 if (supplied.isSymbolicLink()) throw new Error('UNSAFE_SCANNER_ASSET');

118 const root = fs.realpathSync(path), initial = fs.lstatSync(root);

119 if (initial.isFile()) {

120 if (initial.nlink !== 1 || initial.size > MAX_TREE_BYTES) throw new Error('UNSAFE_SCANNER_ASSET');

121 return hashRegularFile(root, initial);

122 }

123 if (!initial.isDirectory()) throw new Error('UNSAFE_SCANNER_ASSET');

124 const entries: Array<[string, number, string]> = [], pending = [root]; let bytes = 0, objects = 0;

125 while (pending.length) {

126 const directory = pending.pop()!;

127 for (const item of fs.readdirSync(directory).sort()) {

128 const full = join(directory, item), stat = fs.lstatSync(full), rel = relative(root, full).split(sep).join('/');

129 if (!rel || rel.startsWith('../') || stat.isSymbolicLink() || ++objects > MAX_TREE_ENTRIES) throw new Error('UNSAFE_SCANNER_ASSET');

130 if (stat.isDirectory()) { pending.push(full); continue; }

131 if (!stat.isFile() || stat.nlink !== 1) throw new Error('UNSAFE_SCANNER_ASSET');

132 bytes += stat.size;

133 if (bytes > MAX_TREE_BYTES) throw new Error('SCANNER_ASSET_LIMIT');

134 entries.push([rel, stat.size, hashRegularFile(full, stat)]);

135 }

136 }

137 if (!entries.length) throw new Error('EMPTY_SCANNER_ASSET');

138 entries.sort(([left], [right]) => left.localeCompare(right));

139 return sha256(canonical(entries));

140}

141

142function option(args: string[], name: string): string | undefined {

143 const at = args.indexOf(name); if (at < 0) return undefined;

144 const value = args[at + 1]; if (!value || value.startsWith('--')) throw new Error(`MISSING_${name.slice(2).toUpperCase().replaceAll('-', '_')}`);

145 args.splice(at, 2); return value;

146}

147

148if (import.meta.main) {

149 try {

150 const args = process.argv.slice(2), command = args.shift();

151 if (command === 'validate') {

152 const path = args.shift(); if (!path || args.length) throw new Error('Usage: validate CATALOG.json');

153 const value = readJson(resolve(path)); validateScannerCatalog(value); process.stdout.write('VALID\n');

154 } else if (command === 'validate-transition') {

155 const current = args.shift(), proposed = args.shift(); if (!current || !proposed || args.length) throw new Error('Usage: validate-transition CURRENT.json PROPOSED.json');

156 validateScannerCatalogTransition(readJson(resolve(current)) as ScannerCatalog, readJson(resolve(proposed)) as ScannerCatalog); process.stdout.write('VALID TRANSITION\n');

157 } else if (command === 'assemble') {

158 const directory = args.shift(), currentPath = args.shift(), output = args.shift(), revision = option(args, '--revision'), sourceCommit = option(args, '--source-commit'), workflow = option(args, '--workflow'), imagePrefix = option(args, '--image-prefix');

159 if (!directory || !currentPath || !output || !revision || !sourceCommit || !workflow || !imagePrefix || args.length) throw new Error('Usage: assemble FRAGMENTS CURRENT OUTPUT --revision ID --source-commit SHA --workflow URL --image-prefix PREFIX');

160 const proposal = scannerCatalogProposal(readJson(resolve(currentPath)) as ScannerCatalog, fragments(resolve(directory)), revision, { sourceCommit, workflow, imagePrefix });

161 fs.writeFileSync(resolve(output), JSON.stringify(proposal, null, 2) + '\n', { flag: 'wx', mode: 0o600 });

162 process.stdout.write(`${proposal.scanners.length} QUALIFIED PROFILES\n`);

163 } else if (command === 'hash-asset') {

164 const path = args.shift(); if (!path || args.length) throw new Error('Usage: hash-asset PATH'); process.stdout.write(scannerAssetHash(resolve(path)) + '\n');

165 } else if (command === 'version-hash') {

166 const stdout = args.shift(), stderr = args.shift(); if (!stdout || !stderr || args.length) throw new Error('Usage: version-hash STDOUT STDERR');

167 const read = (path: string) => { const stat = fs.lstatSync(path); if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 8192) throw new Error('UNSAFE_VERSION_OUTPUT'); return fs.readFileSync(path, 'utf8'); };

168 process.stdout.write(scannerVersionHash(read(resolve(stdout)), read(resolve(stderr))) + '\n');

169 } else throw new Error('Usage: cso-scanner-catalog <validate|validate-transition|assemble|hash-asset|version-hash> ...');

170 } catch (error) {

171 process.stderr.write((error instanceof Error ? error.message : 'SCANNER_CATALOG_ERROR') + '\n'); process.exitCode = 1;

172 }

173}