published-text.ts
41 lines · scripts/lib/published-text.ts at 3f79d81
1/**
2 * The one sanitizer for model- or fixture-written text that reaches a public
3 * issue, PR comment or committed file (census report, tracking issue,
4 * eval:pass-rates output). Control and bidi characters are stripped, the
5 * repo's credential-scan patterns (lib/redact-engine, the engine the quality
6 * gate's credential scan runs) are redacted, @-mentions are neutralized and
7 * length is capped. Inline text also has HTML and Markdown link brackets
8 * escaped; blocks go inside a code fence longer than any backtick run they
9 * contain, so their content stays literal and cannot close the fence.
10 */
11import { redactFindingSpans } from '../../lib/redact-engine';
13export const PUBLISHED_TEXT_MAX = 1000;
15const cap = (text: string, max: number) => (text.length > max ? `${text.slice(0, max - 1)}…` : text);
17function clean(text: string): string {
18 // eslint-disable-next-line no-control-regex
19 const stripped = text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069]/g, ' ').slice(0, 64 * 1024);
20 const redacted = redactFindingSpans(stripped) ?? '[redacted: credential-shaped text that could not be isolated]';
21 return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
22}
24/** One inline line: cleaned, HTML, Markdown link brackets and backticks neutralized, capped. */
25export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string {
26 return cap(clean(text).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
27 .replace(/\[/g, '[').replace(/\]/g, ']').replace(/`/g, "'"), max);
28}
30/** One line for a consumer that wraps it in a fixed three-backtick fence (the PR comment): backticks become quotes. */
31export function sanitizeFixedFenceLine(text: string, max = PUBLISHED_TEXT_MAX): string {
32 return cap(clean(text).replace(/`/g, "'"), max);
33}
35/** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */
36export function publishedFence(lines: readonly string[], max = PUBLISHED_TEXT_MAX): string[] {
37 const body = lines.map(line => cap(clean(line), max));
38 const longest = Math.max(0, ...body.flatMap(line => (line.match(/`+/g) ?? []).map(run => run.length)));
39 const fence = '`'.repeat(Math.max(3, longest + 1));
40 return [fence, ...body, fence];
41}