sources.mjs
318 lines · archify/explainer/sources.mjs at a406594
1import fs from 'node:fs';
2import path from 'node:path';
3import { spawnSync } from 'node:child_process';
4import { createHash } from 'node:crypto';
5import { throwDiagnosticError } from '../renderers/shared/diagnostics.mjs';
6import { verifyRepositoryEvidence } from '../renderers/shared/repository-evidence.mjs';
7import { parseRepositoryRemote } from '../renderers/shared/repository-location.mjs';
8import { chartFromData, compileExplainerChart } from './charts.mjs';
10const MAX_CODE_BYTES = 16 * 1024 * 1024;
11const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
12const MAX_DATA_BYTES = 2 * 1024 * 1024;
13const MAX_ASSET_BYTES = 32 * 1024 * 1024;
14const SECRET_NAME = /^(?:\.env(?:\..*)?|\.npmrc|\.netrc|\.pypirc|\.(?:ssh|aws|gnupg|kube)|(?:id_(?:rsa|dsa|ecdsa|ed25519))(?:\..*)?|(?:credentials?|secrets?)(?:[._-].*)?|.*\.(?:pem|key|p12|pfx|keystore))$/i;
15const SECRET_CONTENT = /-----BEGIN (?:[A-Z ]*PRIVATE KEY|PGP PRIVATE KEY BLOCK)-----|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(?:proj-)?[A-Za-z0-9_-]{20,})\b/i;
16const CREDENTIAL_ASSIGNMENT = /\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token|client[_-]?secret)\b["']?[ \t]*[:=][ \t]*([^\r\n]*)/gi;
17const ENVIRONMENT_REFERENCE = /^(?:\$(?:[A-Za-z_][\w]*|\{[A-Za-z_][\w]*\})|(?:process\.env|os\.environ|env)(?:\.[A-Za-z_][\w]*|\[['"][A-Za-z_][\w]*['"]\])|(?:os\.getenv|getenv|env)\(\s*['"][A-Za-z_][\w]*['"]\s*\))$/;
19export function containsExplainerSecret(text, sourcePath = '') {
20 if (SECRET_CONTENT.test(text)) return true;
21 const configuration = /\.(?:ya?ml|toml|ini|conf|config|properties|cfg|sh|bash)$/i.test(sourcePath);
22 for (const match of text.matchAll(CREDENTIAL_ASSIGNMENT)) {
23 const value = match[1].trim().replace(/\s+(?:#|\/\/).*$/, '').replace(/[;,]\s*$/, '').trim();
24 if (!value || /^(?:null|nil|none|undefined|~)$/i.test(value) || ENVIRONMENT_REFERENCE.test(value)) continue;
25 const quoted = value.match(/^(['"`])([\s\S]*?)\1/);
26 if (quoted) {
27 if (quoted[2].length >= 4 && !ENVIRONMENT_REFERENCE.test(quoted[2])) return true;
28 } else if (configuration) {
29 // Configuration scalars are values, whereas an unquoted JS/Python identifier can be a code reference.
30 return true;
31 }
32 }
33 return false;
34}
36function location(document, pointer, component) {
37 const map = document?.sourceMap;
38 const entry = map instanceof Map ? map.get(pointer) : map?.[pointer];
39 return { path: pointer, component, ...(document?.sourcePath ? { input: document.sourcePath } : {}), line: Number.isInteger(entry?.line) && entry.line > 0 ? entry.line : 1,
40 ...(Number.isInteger(entry?.column) ? { column: entry.column } : {}) };
41}
43function fail(document, pointer, component, reason, message, fix, example) {
44 const examples = {
45 source: '{"id":"router","title":"Router","path":"src/router.js","lines":[1,3]}',
46 code: '{"id":"code","type":"code","source":"router"}',
47 image: '{"id":"image","type":"image","path":"images/figure.png","alt":"Figure"}',
48 };
49 throwDiagnosticError(message, [{ code: `explainer/${reason}`, severity: 'error', message,
50 subject: location(document, pointer, component),
51 evidence: { reason, example: example || examples[component] || examples.source }, supportedFixes: [fix] }]);
52}
54function safePath(document, value, pointer, component) {
55 if (typeof value !== 'string' || !value || /^[\/\\]/.test(value)
56 || /[\\:\u0000-\u001f\u007f?#]/.test(value)
57 || value.split('/').some(part => !part || part === '.' || part === '..' || part.toLowerCase() === '.git')) {
58 fail(document, pointer, component, 'source-path', 'Resource paths must stay inside their root directory.',
59 'Use a relative path with forward slashes and no dot segments.', 'src/router.js');
60 }
61 if (value.split('/').some(part => SECRET_NAME.test(part))) {
62 fail(document, pointer, component, 'secret-path', 'This resource belongs to a sensitive file class.',
63 'Reference a public source file or remove this resource.');
64 }
65 return value;
66}
68function gitBytes(document, repoRoot, args, pointer, limit = MAX_CODE_BYTES) {
69 const result = spawnSync('git', ['--no-replace-objects', '-C', repoRoot, ...args], { maxBuffer: limit });
70 if (result.error || result.status !== 0 || !Buffer.isBuffer(result.stdout) || result.stdout.length > limit) {
71 fail(document, pointer, 'source', 'source-read', 'The pinned repository resource could not be read within the size limit.',
72 'Use an available, readable source file smaller than 16 MiB.');
73 }
74 return result.stdout;
75}
77function deriveRepository(document, repoRoot) {
78 if (!repoRoot) fail(document, '/meta/repository', 'source', 'repository-root', 'Repository sources require a local repository.',
79 'Pass --repo-root with the matching local Git checkout.', '--repo-root /absolute/path/to/repository');
80 const raw = gitBytes(document, repoRoot, ['remote', 'get-url', 'origin'], '/meta/repository').toString('utf8').trim();
81 const remote = parseRepositoryRemote(raw);
82 if (!remote) fail(document, '/meta/repository', 'source', 'repository-origin', 'The repository origin cannot be represented safely.',
83 'Configure a credential-free repository origin without query or fragment.');
84 const revision = gitBytes(document, repoRoot, ['rev-parse', '--verify', 'HEAD^{commit}'], '/meta/repository').toString('utf8').trim();
85 return { url: remote.url, revision,
86 ...(remote.provider && remote.protocol === 'https:' && remote.endpoint === 'standard' ? {} : { link_mode: 'local-only' }) };
87}
89function resources(document) {
90 const result = [];
91 for (const [panelIndex, panel] of (document.panels || []).entries()) {
92 for (const [index, block] of (panel.blocks || []).entries()) result.push({ block, pointer: `/panels/${panelIndex}/blocks/${index}` });
93 }
94 return result;
95}
97function digest(bytes) {
98 return { sha256: createHash('sha256').update(bytes).digest('hex'), bytes: bytes.length,
99 contentBase64: bytes.toString('base64') };
100}
102function imageMime(bytes) {
103 if (bytes.length >= 24 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))
104 && bytes.toString('ascii', 12, 16) === 'IHDR') return 'image/png';
105 if (bytes.length >= 4 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return 'image/jpeg';
106 if (bytes.length >= 13 && /^(GIF87a|GIF89a)$/.test(bytes.toString('ascii', 0, 6))) return 'image/gif';
107 if (bytes.length >= 16 && bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP') return 'image/webp';
108 return null;
109}
111function sameFile(left, right) {
112 return left.dev === right.dev && left.ino === right.ino && left.size === right.size && left.mtimeMs === right.mtimeMs;
113}
115function readImage(document, baseDir, authoredPath, pointer) {
116 if (!baseDir) fail(document, pointer, 'image', 'asset-root', 'Local images require an input directory.',
117 'Provide the original input directory as baseDir.');
118 const relative = safePath(document, authoredPath, `${pointer}/path`, 'image');
119 const expected = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif' }[path.extname(relative).toLowerCase()];
120 if (!expected) fail(document, `${pointer}/path`, 'image', 'asset-format', 'Only local PNG, JPEG, WebP and GIF images are supported.',
121 'Use a PNG, JPEG, WebP or GIF image.');
122 const bytes = readContained(document, baseDir, relative, pointer, 'image', MAX_IMAGE_BYTES, '8 MiB');
123 const mime = imageMime(bytes);
124 if (mime !== expected) fail(document, pointer, 'image', 'asset-format', 'The image bytes do not match a supported image format.',
125 'Use a correctly encoded image whose extension matches its bytes.');
126 return { path: relative, mime, ...digest(bytes), dataUrl: `data:${mime};base64,${bytes.toString('base64')}` };
127}
129// Read a regular file inside baseDir without following links, and confirm it did not change meanwhile.
130function readContained(document, baseDir, relative, pointer, component, limit, limitText) {
131 const noun = component === 'image' ? 'image' : 'data file', Noun = noun[0].toUpperCase() + noun.slice(1);
132 let fd;
133 try {
134 const root = fs.realpathSync(baseDir);
135 const ancestors = [[root, fs.lstatSync(root)]];
136 let candidate = root;
137 for (const part of relative.split('/')) {
138 candidate = path.join(candidate, part);
139 const stat = fs.lstatSync(candidate);
140 if (stat.isSymbolicLink()) fail(document, `${pointer}/path`, component, 'asset-symlink', `${Noun} paths may not traverse symbolic links.`,
141 `Copy the ${noun} into the input directory as a regular file.`);
142 ancestors.push([candidate, stat]);
143 }
144 if (!ancestors.at(-1)[1].isFile()) fail(document, pointer, component, 'asset-file', `The ${noun} must be a regular file.`, `Use a regular ${noun}.`);
145 fd = fs.openSync(candidate, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0));
146 const before = fs.fstatSync(fd);
147 if (!sameFile(before, ancestors.at(-1)[1]) || before.size > limit) fail(document, pointer, component, 'asset-size',
148 `The ${noun} changed or exceeds the ${limitText} size limit.`, `Use a stable ${noun} smaller than ${limitText}.`);
149 const bytes = Buffer.alloc(before.size);
150 let offset = 0;
151 while (offset < bytes.length) {
152 const count = fs.readSync(fd, bytes, offset, bytes.length - offset, offset);
153 if (!count) break;
154 offset += count;
155 }
156 const unchanged = offset === bytes.length && sameFile(before, fs.fstatSync(fd))
157 && ancestors.every(([file, stat]) => { const current = fs.lstatSync(file); return !current.isSymbolicLink() && sameFile(current, stat); });
158 if (!unchanged) fail(document, pointer, component, 'asset-changed', `The ${noun} path or bytes changed while being frozen.`,
159 `Finish ${noun} updates before rendering again.`);
160 return bytes;
161 } catch (error) {
162 if (error.archifyDiagnostics) throw error;
163 fail(document, pointer, component, 'asset-read', `The ${noun} cannot be read safely inside the input directory.`,
164 `Use an accessible regular ${noun} inside the input directory.`);
165 } finally {
166 if (fd !== undefined) fs.closeSync(fd);
167 }
168}
170/** Prepare source locations and freeze code/image bytes; never mutate author input. */
171export async function resolveExplainerSources(document, { repoRoot, baseDir, sourceMap, sourcePath } = {}) {
172 const context = { sourceMap, sourcePath };
173 const sources = Object.create(null);
174 const assets = Object.create(null);
175 const dependencies = [];
176 const entries = document.sources || [];
177 const blocks = resources(document);
178 const codeIds = new Set();
179 const blockIds = new Set();
180 for (const { block, pointer } of blocks) {
181 if (!['code', 'image'].includes(block.type)) continue;
182 if (typeof block.id !== 'string' || !block.id || blockIds.has(block.id)) fail(context, pointer, block.type, 'resource-id',
183 'Resource blocks require a unique stable id.', 'Give each code or image block its own id.');
184 blockIds.add(block.id);
185 if (block.type === 'code') codeIds.add(block.source);
186 }
187 const repositorySources = [];
188 for (const [index, entry] of entries.entries()) {
189 const pointer = `/sources/${index}`;
190 if (typeof entry.id !== 'string' || !entry.id || Object.hasOwn(sources, entry.id)) fail(context, pointer, 'source', 'source-id',
191 'Sources require unique ids.', 'Give each source a unique id.');
192 sources[entry.id] = null;
193 if (entry.path) {
194 safePath(context, entry.path, `${pointer}/path`, 'source');
195 if (entry.lines && (!Array.isArray(entry.lines) || entry.lines.length !== 2 || !entry.lines.every(Number.isSafeInteger)
196 || entry.lines[0] < 1 || entry.lines[1] < entry.lines[0])) fail(context, `${pointer}/lines`, 'source', 'source-lines',
197 'Repository line ranges must be ordered and positive.', 'Use lines: [start, end].', '[1, 3]');
198 }
199 if (entry.url) {
200 let url; try { url = new URL(entry.url); } catch {}
201 if (!url || !['https:', 'http:'].includes(url.protocol) || url.username || url.password || /[\u0000-\u0020\u007f]/.test(entry.url)) fail(context, `${pointer}/url`, 'source', 'source-url',
202 'Source links require credential-free HTTP(S) URLs.', 'Use https://example.com/article.');
203 }
204 sources[entry.id] = Object.freeze({id:entry.id,title:entry.title,...(entry.url?{url:entry.url}:{}),...(entry.path?{path:entry.path}:{}),...(entry.lines?{lines:[...entry.lines]}:{}),...(entry.revision?{revision:entry.revision}:{})});
205 if (codeIds.has(entry.id)) {
206 if (!entry.path || !entry.lines) fail(context,pointer,'code','code-source','Code requires a Git file and line range.','Add path and lines to the source.');
207 repositorySources.push({entry,index});
208 }
209 }
211 let repository = null;
212 if (repositorySources.length) {
213 const metadata = document.meta?.repository ? { ...document.meta.repository } : deriveRepository(context, repoRoot);
214 for (const { entry, index } of repositorySources) {
215 if (entry.revision !== undefined && String(entry.revision).toLowerCase() !== String(metadata.revision).toLowerCase()) fail(context, `/sources/${index}/revision`, 'source', 'source-revision',
216 'All repository sources must use the document’s pinned revision.', 'Use the document repository revision or omit the source revision.');
217 }
218 let verified;
219 try {
220 verified = verifyRepositoryEvidence('architecture', { meta: { repository: metadata }, components: repositorySources.map(({ entry }) => ({
221 id: entry.id, sources: [{ path: entry.path, line: entry.lines[0], end_line: entry.lines[1] }],
222 })) }, repoRoot);
223 } catch (error) {
224 const original = error.archifyDiagnostics?.[0];
225 const match = original?.subject?.path?.match(/^\/components\/(\d+)/);
226 const index = match ? repositorySources[Number(match[1])]?.index : undefined;
227 const examples = {
228 'repository-evidence/root-required': '--repo-root /absolute/path/to/repository',
229 'repository-evidence/root-not-top-level': '--repo-root /absolute/path/to/repository',
230 'repository-evidence/root-unreadable': '--repo-root /absolute/path/to/repository',
231 'repository-evidence/revision-invalid': 'git rev-parse HEAD',
232 'repository-evidence/revision-unavailable': 'git rev-parse HEAD',
233 'repository-evidence/url-invalid': '{"url":"https://github.com/owner/repository"}',
234 'repository-evidence/origin-mismatch': 'git remote get-url origin',
235 'repository-evidence/line-out-of-range': '{"path":"src/router.js","lines":[1,3]}',
236 'repository-evidence/file-missing': '{"path":"src/router.js","lines":[1,3]}',
237 };
238 fail(context, index === undefined ? '/meta/repository' : `/sources/${index}`, 'source', 'repository-location',
239 'The declared repository location could not be verified.', 'Correct the repository origin, pinned commit, file path and line range.',
240 examples[original?.code]);
241 }
242 repository = Object.freeze({ ...verified.repository });
243 for (const { entry, index } of repositorySources) {
244 const reference = verified.nodes[entry.id][0];
245 const result = { id: entry.id, ...(entry.title === undefined ? {} : { title: entry.title }),
246 revision: repository.revision, path: reference.path, lines: Object.freeze([...entry.lines]),
247 ...(reference.href ? { url: reference.href } : {}) };
248 if (codeIds.has(entry.id)) {
249 const bytes = gitBytes(context, repoRoot, ['show', `${repository.revision}:${entry.path}`], `/sources/${index}`);
250 let text;
251 try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch {
252 fail(context, `/sources/${index}`, 'code', 'source-encoding', 'Code sources must contain UTF-8 text.', 'Use a UTF-8 text source file.');
253 }
254 if (text.includes('\0') || containsExplainerSecret(text, entry.path)) fail(context, `/sources/${index}`, 'code', 'secret-content',
255 'This source cannot be displayed because it contains sensitive or binary content.', 'Use a public source file without secrets.');
256 result.text = text.split(/\r\n|\n|\r/).slice(entry.lines[0] - 1, entry.lines[1]).join('\n');
257 const frozen = digest(bytes);
258 Object.assign(result, { sha256: frozen.sha256, bytes: frozen.bytes });
259 dependencies.push(Object.freeze({ kind: 'repository', id: entry.id, path: entry.path, revision: repository.revision, ...frozen }));
260 }
261 sources[entry.id] = Object.freeze(result);
262 }
263 }
264 // A cited repository file also carries its excerpt when the repository is at hand, so a reader can
265 // check the claim in place. A citation that cannot be verified or read stays a plain reference.
266 const cited = document.sources.map((entry,index)=>({entry,index})).filter(({entry})=>entry.path && entry.lines && !codeIds.has(entry.id));
267 if (cited.length && repoRoot) {
268 let metadata = null;
269 try { metadata = document.meta?.repository ? { ...document.meta.repository } : deriveRepository(context, repoRoot); } catch {}
270 for (const { entry, index } of metadata ? cited : []) {
271 try {
272 const verified = verifyRepositoryEvidence('architecture', { meta: { repository: metadata }, components: [{ id: entry.id, sources: [{ path: entry.path, line: entry.lines[0], end_line: entry.lines[1] }] }] }, repoRoot);
273 const reference = verified.nodes[entry.id][0];
274 const bytes = gitBytes(context, repoRoot, ['show', `${verified.repository.revision}:${entry.path}`], `/sources/${index}`);
275 const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
276 if (text.includes('\0') || containsExplainerSecret(text, entry.path)) continue;
277 const frozen = digest(bytes);
278 sources[entry.id] = Object.freeze({ id: entry.id, ...(entry.title === undefined ? {} : { title: entry.title }), revision: verified.repository.revision, path: reference.path,
279 lines: Object.freeze([...entry.lines]), ...(reference.href ? { url: reference.href } : {}), text: text.split(/\r\n|\n|\r/).slice(entry.lines[0] - 1, entry.lines[1]).join('\n'), sha256: frozen.sha256, bytes: frozen.bytes });
280 dependencies.push(Object.freeze({ kind: 'repository', id: entry.id, path: entry.path, revision: verified.repository.revision, ...frozen }));
281 repository ||= Object.freeze({ ...verified.repository });
282 } catch {}
283 }
284 }
285 for (const { block, pointer } of blocks) {
286 if (block.type === 'code') {
287 const source = sources[block.source];
288 if (!source || !source.path || typeof source.text !== 'string') fail(context, `${pointer}/source`, 'code', 'code-source',
289 'Code blocks require a Git code excerpt.', 'Reference a repository source id with a valid line range.');
290 }
291 }
292 let assetBytes = 0;
293 for (const { block, pointer } of blocks) {
294 if (block.type !== 'image') continue;
295 const asset = readImage(context, baseDir, block.path, pointer);
296 assetBytes += asset.bytes;
297 if (assetBytes > MAX_ASSET_BYTES) fail(context, pointer, 'image', 'asset-budget', 'Embedded images exceed the 32 MiB document limit.',
298 'Reduce the total embedded image bytes below 32 MiB.');
299 assets[block.id] = Object.freeze({ id: block.id, alt: block.alt, ...asset });
300 dependencies.push(Object.freeze({ kind: 'image', id: block.id, path: asset.path, sha256: asset.sha256, bytes: asset.bytes, contentBase64: asset.contentBase64 }));
301 }
302 // A data chart reads its CSV or JSON once, freezes the bytes and keeps the chart text built from them.
303 const charts = Object.create(null);
304 for (const { block, pointer } of blocks) {
305 if (block.type !== 'chart' || !block.data) continue;
306 if (!baseDir) fail(context, pointer, 'chart', 'asset-root', 'A data chart requires an input directory.', 'Run finalize on the draft file next to its data.');
307 const relative = safePath(context, block.data.path, `${pointer}/data/path`, 'chart');
308 if (!/\.(?:csv|json)$/iu.test(relative)) fail(context, `${pointer}/data/path`, 'chart', 'chart-data-format', 'Chart data is a .csv or .json file.', 'data=latency.csv');
309 const bytes = readContained(context, baseDir, relative, pointer, 'chart', MAX_DATA_BYTES, '2 MiB');
310 let built;
311 try { built = chartFromData(bytes, relative, block.data); } catch { built = { error: 'The data file is not UTF-8 text.' }; }
312 const chart = built.error ? built : compileExplainerChart(built.text);
313 if (chart.error) fail(context, pointer, 'chart', 'chart-data', chart.error, '@chart type=bar data=latency.csv x=day y=p50,p99');
314 charts[block.id] = built.text;
315 dependencies.push(Object.freeze({ kind: 'data', id: block.id, path: relative, ...digest(bytes) }));
316 }
317 return { sources: Object.freeze(sources), assets: Object.freeze(assets), charts: Object.freeze(charts), dependencies: Object.freeze(dependencies), repository };
318}