pdftotext.ts
318 lines · make-pdf/src/pdftotext.ts at 3f79d81
1/**
2 * pdftotext wrapper — the tool behind the copy-paste CI gate.
3 *
4 * Codex round 2 surfaced two real problems we address here:
5 *
6 * #18: pdftotext (Poppler) vs pdftotext (Xpdf) vs pdftotext-next vary on
7 * whitespace, line wrap, Unicode normalization, form feeds, and
8 * extraction order. Cross-platform exact diffing is a non-starter.
9 * We normalize aggressively and diff the normalized form.
10 *
11 * #19: the regex /(?:\b\w\s){4,}/ only catches one failure shape (letters
12 * spaced out). It misses word-order corruption, missing whitespace
13 * between paragraphs, and homoglyph substitution. We add a word-token
14 * diff and a paragraph-boundary assertion on top.
15 *
16 * Resolution order for the pdftotext binary (v1.24-aligned):
17 * 1. $GSTACK_PDFTOTEXT_BIN env override (preferred, matches v1.24 GSTACK_*_BIN pattern)
18 * 2. $PDFTOTEXT_BIN env override (back-compat alias)
19 * 3. PATH lookup via Bun.which('pdftotext') — handles Windows PATHEXT natively
20 * 4. standard POSIX paths (Homebrew + distro) — no Windows candidates because
21 * Poppler scatters across Scoop / Chocolatey / oschwartz10612-poppler-windows
22 * and guessing causes false positives. Set GSTACK_PDFTOTEXT_BIN explicitly.
23 * 5. throws a friendly "install poppler" error
24 *
25 * The wrapper is *optional at runtime*: production renders don't need it.
26 * Only the CI gate and unit tests invoke pdftotext.
27 */
29import { execFileSync, spawnSync } from "node:child_process";
30import * as fs from "node:fs";
31import * as os from "node:os";
32import * as path from "node:path";
34export class PdftotextUnavailableError extends Error {
35 constructor(message: string) {
36 super(message);
37 this.name = "PdftotextUnavailableError";
38 }
39}
41export interface PdftotextInfo {
42 bin: string;
43 version: string; // "pdftotext version 24.02.0" or similar
44 flavor: "poppler" | "xpdf" | "unknown";
45}
47/**
48 * Probe a base path for executability, honoring Windows extension suffixes.
49 */
50export function findExecutable(base: string): string | null {
51 if (isExecutable(base)) return base;
52 if (process.platform === "win32") {
53 for (const ext of [".exe", ".cmd", ".bat"]) {
54 const withExt = base + ext;
55 if (isExecutable(withExt)) return withExt;
56 }
57 }
58 return null;
59}
61function resolveOverride(value: string | undefined, env: NodeJS.ProcessEnv): string | null {
62 if (!value?.trim()) return null;
63 const trimmed = value.trim().replace(/^"(.*)"$/, '$1');
64 if (path.isAbsolute(trimmed)) return findExecutable(trimmed);
65 const PATH = env.PATH ?? env.Path ?? '';
66 return Bun.which(trimmed, { PATH }) ?? null;
67}
69/**
70 * Locate pdftotext. Throws PdftotextUnavailableError if none is found.
71 */
72export function resolvePdftotext(env: NodeJS.ProcessEnv = process.env): PdftotextInfo {
73 // 1 + 2: env overrides (GSTACK_PDFTOTEXT_BIN preferred, PDFTOTEXT_BIN back-compat).
74 const overrideRaw = env.GSTACK_PDFTOTEXT_BIN ?? env.PDFTOTEXT_BIN;
75 const override = resolveOverride(overrideRaw, env);
76 if (override) return describeBinary(override);
78 // 3: PATH lookup via Bun.which — handles Windows PATHEXT natively.
79 const PATH = env.PATH ?? env.Path ?? '';
80 const onPath = Bun.which('pdftotext', { PATH });
81 if (onPath) return describeBinary(onPath);
83 // 4: POSIX-only standard locations. No Windows candidates — Poppler installs
84 // scatter across Scoop/Chocolatey/portable zips and guessing causes false
85 // positives. Windows users set GSTACK_PDFTOTEXT_BIN explicitly.
86 const posixCandidates = [
87 "/opt/homebrew/bin/pdftotext", // Apple Silicon Homebrew
88 "/usr/local/bin/pdftotext", // Intel Mac or Linuxbrew
89 "/usr/bin/pdftotext", // distro package
90 ];
91 for (const candidate of posixCandidates) {
92 if (isExecutable(candidate)) return describeBinary(candidate);
93 }
95 throw new PdftotextUnavailableError([
96 "pdftotext not found.",
97 "",
98 "make-pdf needs pdftotext to run the copy-paste CI gate.",
99 "(Runtime rendering does NOT need it. This only affects tests.)",
100 "",
101 "To install:",
102 " macOS: brew install poppler",
103 " Ubuntu: sudo apt-get install poppler-utils",
104 " Fedora: sudo dnf install poppler-utils",
105 " Windows: scoop install poppler (or download from",
106 " https://github.com/oschwartz10612/poppler-windows)",
107 "",
108 "Or set GSTACK_PDFTOTEXT_BIN to an explicit path:",
109 process.platform === "win32"
110 ? ' setx GSTACK_PDFTOTEXT_BIN "C:\\path\\to\\pdftotext.exe"'
111 : " export GSTACK_PDFTOTEXT_BIN=/path/to/pdftotext",
112 ].join("\n"));
113}
115/**
116 * Locate a poppler companion tool (pdffonts, pdfimages, pdftoppm, pdfinfo,
117 * pdftotext) used by the e2e render gates. Mirrors resolvePdftotext's resolution order:
118 * 1. $GSTACK_<TOOL>_BIN env override (e.g. GSTACK_PDFFONTS_BIN)
119 * 2. PATH via Bun.which
120 * 3. standard POSIX locations (Homebrew + distro)
121 *
122 * Returns null (does NOT throw) when the tool is missing — the emoji gate skips
123 * cleanly rather than failing on a box without full poppler-utils.
124 */
125export function resolvePopplerTool(
126 tool: "pdffonts" | "pdfimages" | "pdftoppm" | "pdfinfo" | "pdftotext",
127 env: NodeJS.ProcessEnv = process.env,
128): string | null {
129 const override = resolveOverride(env[`GSTACK_${tool.toUpperCase()}_BIN`], env);
130 if (override) return override;
132 const PATH = env.PATH ?? env.Path ?? "";
133 const onPath = Bun.which(tool, { PATH });
134 if (onPath) return onPath;
136 for (const dir of ["/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"]) {
137 const candidate = findExecutable(path.join(dir, tool));
138 if (candidate) return candidate;
139 }
140 return null;
141}
143function isExecutable(p: string): boolean {
144 try {
145 // access(X_OK) is true for directories (the traverse bit); only regular files count.
146 if (!fs.statSync(p).isFile()) return false;
147 fs.accessSync(p, fs.constants.X_OK);
148 return true;
149 } catch {
150 return false;
151 }
152}
154function describeBinary(bin: string): PdftotextInfo {
155 let version = "unknown";
156 let flavor: PdftotextInfo["flavor"] = "unknown";
158 // spawnSync, not execFileSync: poppler writes -v output to STDERR and exits 0,
159 // so execFileSync neither returns it (it returns stdout, which is empty) nor
160 // throws (which is what made the stderr fallback below reachable). The result
161 // was version="unknown" flavor="unknown" on every poppler install. spawnSync
162 // hands back both streams regardless of exit status, which also covers the
163 // xpdf builds that exit 99.
164 const res = spawnSync(bin, ["-v"], { encoding: "utf8" });
165 const raw = `${res.stdout ?? ""}\n${res.stderr ?? ""}`;
167 // The version banner is not reliably the first line once both streams are in
168 // play, so match it rather than taking line 0.
169 const lines = raw.split("\n").map(l => l.trim()).filter(Boolean);
170 version = lines.find(l => /pdftotext\s+version/i.test(l)) ?? lines[0] ?? "unknown";
172 // Flavor comes from the WHOLE banner, never the version line alone: poppler
173 // prints "pdftotext version 26.06.0" on line 1 and only identifies itself on
174 // line 2, "Copyright ... The Poppler Developers".
175 const v = raw.toLowerCase();
176 if (v.includes("poppler")) flavor = "poppler";
177 else if (v.includes("xpdf")) flavor = "xpdf";
178 return { bin, version, flavor };
179}
181/**
182 * Run pdftotext on a PDF and return the extracted text.
183 *
184 * Uses `-layout` by default because that's what downstream normalization
185 * expects. Callers that need raw text can pass layout=false.
186 */
187export function pdftotext(pdfPath: string, opts?: { layout?: boolean }): string {
188 const info = resolvePdftotext();
189 const layout = opts?.layout ?? true;
190 const args: string[] = [];
191 if (layout) args.push("-layout");
192 args.push(pdfPath, "-"); // "-" = stdout
193 try {
194 return execFileSync(info.bin, args, {
195 encoding: "utf8",
196 maxBuffer: 32 * 1024 * 1024,
197 });
198 } catch (err: any) {
199 throw new Error(`pdftotext failed on ${pdfPath}: ${err.message}`);
200 }
201}
203/**
204 * Normalize extracted text for cross-platform, cross-flavor diffing.
205 *
206 * What we strip / normalize:
207 * - Unicode: NFC canonical composition (macOS emits NFD; Linux emits NFC;
208 * this dodges the fundamental encoding diff).
209 * - CR and CRLF → LF (Windows Xpdf emits CRLF).
210 * - Form feeds (\f) → double newline (Poppler emits \f at page breaks).
211 * - Trailing spaces on every line.
212 * - Runs of 3+ blank lines → 2 blank lines.
213 * - Leading/trailing whitespace on the whole string.
214 * - Non-breaking space (U+00A0) → regular space.
215 * - Zero-width space (U+200B) and zero-width non-joiner (U+200C) → empty.
216 * - Soft hyphen (U+00AD) → empty (pdftotext -layout sometimes emits these
217 * for hyphens: auto breaks).
218 */
219export function normalize(raw: string): string {
220 let s = raw;
221 s = s.normalize("NFC");
222 s = s.replace(/\r\n/g, "\n");
223 s = s.replace(/\r/g, "\n");
224 s = s.replace(/\f/g, "\n\n");
225 s = s.replace(/\u00a0/g, " ");
226 s = s.replace(/[\u200b\u200c\u00ad]/g, "");
227 s = s.replace(/[ \t]+$/gm, "");
228 s = s.replace(/\n{3,}/g, "\n\n");
229 s = s.trim();
230 return s;
231}
233/**
234 * The canonical copy-paste gate used in the E2E tests.
235 *
236 * Returns { ok: true } when all three assertions pass; returns
237 * { ok: false, reasons: [...] } with one or more failure reasons otherwise.
238 */
239export interface GateResult {
240 ok: boolean;
241 reasons: string[];
242 extracted: string;
243}
245export function copyPasteGate(pdfPath: string, expected: string): GateResult {
246 const extracted = normalize(pdftotext(pdfPath, { layout: true }));
247 const expectedNorm = normalize(expected);
248 const reasons: string[] = [];
250 // Assertion 1: every expected paragraph appears as a whole line or
251 // contiguous block in the extracted text.
252 const expectedParagraphs = splitParagraphs(expectedNorm);
253 for (const paragraph of expectedParagraphs) {
254 const compact = collapseWhitespace(paragraph);
255 const extractedCompact = collapseWhitespace(extracted);
256 if (!extractedCompact.includes(compact)) {
257 reasons.push(
258 `expected paragraph not found in extracted text: ${truncate(paragraph, 80)}`,
259 );
260 }
261 }
263 // Assertion 2: no "S a i l i n g"-style single-char runs.
264 // Count groups of 4+ consecutive letter-then-space tokens. False positive
265 // risk on things like "A B C D" (initials) — mitigate by requiring the
266 // letters spell a known-word substring of the expected text.
267 const fragRegex = /((?:\b\w\s){4,})/g;
268 let fragMatch: RegExpExecArray | null;
269 while ((fragMatch = fragRegex.exec(extracted)) !== null) {
270 const letters = fragMatch[1].replace(/\s/g, "");
271 // Only flag if the reassembled letters appear in the expected text.
272 if (expectedNorm.toLowerCase().includes(letters.toLowerCase()) && letters.length >= 4) {
273 reasons.push(
274 `per-glyph emission detected (the "S ai li ng" bug): "${fragMatch[1].trim()}" reassembles to "${letters}"`,
275 );
276 }
277 }
279 // Assertion 3: paragraph boundaries preserved. Count double-newlines
280 // in both; they should differ by no more than ±2 (header/footer noise).
281 const expectedBreaks = (expectedNorm.match(/\n\n/g) || []).length;
282 const extractedBreaks = (extracted.match(/\n\n/g) || []).length;
283 if (Math.abs(expectedBreaks - extractedBreaks) > 4) {
284 reasons.push(
285 `paragraph boundary count drift: expected ~${expectedBreaks}, got ${extractedBreaks}`,
286 );
287 }
289 return { ok: reasons.length === 0, reasons, extracted };
290}
292function splitParagraphs(s: string): string[] {
293 return s.split(/\n\n+/).map(p => p.trim()).filter(p => p.length > 0);
294}
296function collapseWhitespace(s: string): string {
297 return s.replace(/\s+/g, " ").trim();
298}
300function truncate(s: string, n: number): string {
301 return s.length > n ? s.slice(0, n) + "..." : s;
302}
304/**
305 * Emit diagnostic info to stderr — useful for CI failure debugging.
306 * Call this once before running any gate in a CI log.
307 */
308export function logDiagnostics(): void {
309 try {
310 const info = resolvePdftotext();
311 process.stderr.write(
312 `[pdftotext] bin=${info.bin} flavor=${info.flavor} version="${info.version}" ` +
313 `os=${os.platform()}-${os.arch()} node=${process.version}\n`,
314 );
315 } catch (err: any) {
316 process.stderr.write(`[pdftotext] unavailable: ${err.message}\n`);
317 }
318}