sanitize.ts
49 lines · browse/src/sanitize.ts at 3f79d81
1// Lone Unicode surrogate sanitization.
2//
3// Lone surrogates (\uD800-\uDFFF without a matching pair) are valid UTF-16
4// but invalid UTF-8, so JSON.stringify produces output the Claude API rejects
5// with HTTP 400 "no low surrogate in string". Page captures from real-world
6// HTML hit this when content contains broken emoji bytes or mid-emoji splits.
7//
8// Two sanitizers are needed because both forms appear in browse responses:
9// - Raw UTF-16 surrogates in text/plain bodies (pre-stringify state).
10// - JSON \uXXXX escape sequences after JSON.stringify already ran.
11// Both replace lone surrogates with U+FFFD (replacement character).
13const LONE_SURROGATE_HIGH = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])/g;
14const LONE_SURROGATE_LOW = /(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
16export function stripLoneSurrogates(s: string): string {
17 return s.replace(LONE_SURROGATE_HIGH, '�').replace(LONE_SURROGATE_LOW, '�');
18}
20/**
21 * JSON.stringify replacer that strips lone UTF-16 surrogates from string
22 * values before they get escape-encoded. Pair with stringify when the
23 * consumer will JSON.parse the payload back into JS strings (SSE clients
24 * do this). Required at every JSON/SSE egress that ships page-content-derived
25 * fields — see CLAUDE.md "Unicode sanitization at server egress".
26 *
27 * The replacer must run INSIDE the encoding pipeline: post-stringify regex
28 * is a no-op because JSON.stringify has already converted \uD800 into the
29 * literal escape text "\\ud800" before a regex could see the surrogate.
30 */
31export function sanitizeReplacer(_key: string, value: unknown): unknown {
32 return typeof value === 'string' ? stripLoneSurrogates(value) : value;
33}
35// Matches \uD8XX-\uDFXX escape text where the pair is not completed by an
36// adjacent \uDC00-\uDFFF (high) or preceded by \uD800-\uDBFF (low).
37const LONE_SURROGATE_HIGH_ESCAPE = /\\u[Dd][89ABab][0-9A-Fa-f]{2}(?!\\u[Dd][C-Fc-f][0-9A-Fa-f]{2})/g;
38const LONE_SURROGATE_LOW_ESCAPE = /(?<!\\u[Dd][89ABab][0-9A-Fa-f]{2})\\u[Dd][C-Fc-f][0-9A-Fa-f]{2}/g;
40export function stripLoneSurrogateEscapes(s: string): string {
41 return s.replace(LONE_SURROGATE_HIGH_ESCAPE, '\\uFFFD').replace(LONE_SURROGATE_LOW_ESCAPE, '\\uFFFD');
42}
44// Pick the right sanitizer based on whether the body has already been JSON-stringified.
45// For application/json bodies, run both passes: raw first (in case the JSON encoder
46// emitted surrogates as-is rather than escaping), then escape-text.
47export function sanitizeBody(body: string, isJson: boolean): string {
48 return isJson ? stripLoneSurrogateEscapes(stripLoneSurrogates(body)) : stripLoneSurrogates(body);
49}