write-commands.ts
1438 lines · browse/src/write-commands.ts at 3f79d81
1/**
2 * Write commands — navigate and interact with pages (side effects)
3 *
4 * goto, back, forward, reload, click, fill, select, hover, type,
5 * press, scroll, wait, viewport, cookie, header, useragent
6 */
8import type { TabSession } from './tab-session';
9import type { BrowserManager } from './browser-manager';
10import { CookieImportError, cookieDomainMatches, findInstalledBrowsers, listSupportedBrowserNames } from './cookie-import-browser';
11import { generatePickerCode } from './cookie-picker-routes';
12import { formatCookieImportResult, parseCookieImportArgs, runCookieImport, validateCookieTarget } from './cookie-import-operation';
13import { validateCookieAuthOptions, validateCookieStorageSupport } from './cookie-auth-verification';
14import { validateNavigationUrl } from './url-validation';
15import { validateOutputPath, validateReadPath } from './path-security';
16import { guardScreenshotPath } from './screenshot-size-guard';
17import * as fs from 'fs';
18import * as path from 'path';
19import type { SetContentWaitUntil } from './tab-session';
20import { TEMP_DIR } from './platform';
21import { SAFE_DIRECTORIES } from './path-security';
22import { modifyStyle, undoModification, resetModifications, getModificationHistory } from './cdp-inspector';
23import { withCdpSession } from './cdp-bridge';
25/**
26 * Aggressive page cleanup selectors and heuristics.
27 * Goal: make the page readable and clean while keeping it recognizable.
28 * Inspired by uBlock Origin filter lists, Readability.js, and reader mode heuristics.
29 */
30const CLEANUP_SELECTORS = {
31 ads: [
32 // Google Ads
33 'ins.adsbygoogle', '[id^="google_ads"]', '[id^="div-gpt-ad"]',
34 'iframe[src*="doubleclick"]', 'iframe[src*="googlesyndication"]',
35 '[data-google-query-id]', '.google-auto-placed',
36 // Generic ad patterns (uBlock Origin common filters)
37 '[class*="ad-banner"]', '[class*="ad-wrapper"]', '[class*="ad-container"]',
38 '[class*="ad-slot"]', '[class*="ad-unit"]', '[class*="ad-zone"]',
39 '[class*="ad-placement"]', '[class*="ad-holder"]', '[class*="ad-block"]',
40 '[class*="adbox"]', '[class*="adunit"]', '[class*="adwrap"]',
41 '[id*="ad-banner"]', '[id*="ad-wrapper"]', '[id*="ad-container"]',
42 '[id*="ad-slot"]', '[id*="ad_banner"]', '[id*="ad_container"]',
43 '[data-ad]', '[data-ad-slot]', '[data-ad-unit]', '[data-adunit]',
44 '[class*="sponsored"]', '[class*="Sponsored"]',
45 '.ad', '.ads', '.advert', '.advertisement',
46 '#ad', '#ads', '#advert', '#advertisement',
47 // Common ad network iframes
48 'iframe[src*="amazon-adsystem"]', 'iframe[src*="outbrain"]',
49 'iframe[src*="taboola"]', 'iframe[src*="criteo"]',
50 'iframe[src*="adsafeprotected"]', 'iframe[src*="moatads"]',
51 // Promoted/sponsored content
52 '[class*="promoted"]', '[class*="Promoted"]',
53 '[data-testid*="promo"]', '[class*="native-ad"]',
54 // Empty ad placeholders (divs with only ad classes, no real content)
55 'aside[class*="ad"]', 'section[class*="ad-"]',
56 ],
57 cookies: [
58 // Cookie consent frameworks
59 '[class*="cookie-consent"]', '[class*="cookie-banner"]', '[class*="cookie-notice"]',
60 '[id*="cookie-consent"]', '[id*="cookie-banner"]', '[id*="cookie-notice"]',
61 '[class*="consent-banner"]', '[class*="consent-modal"]', '[class*="consent-wall"]',
62 '[class*="gdpr"]', '[id*="gdpr"]', '[class*="GDPR"]',
63 '[class*="CookieConsent"]', '[id*="CookieConsent"]',
64 // OneTrust (very common)
65 '#onetrust-consent-sdk', '.onetrust-pc-dark-filter', '#onetrust-banner-sdk',
66 // Cookiebot
67 '#CybotCookiebotDialog', '#CybotCookiebotDialogBodyUnderlay',
68 // TrustArc / TRUSTe
69 '#truste-consent-track', '.truste_overlay', '.truste_box_overlay',
70 // Quantcast
71 '.qc-cmp2-container', '#qc-cmp2-main',
72 // Generic patterns
73 '[class*="cc-banner"]', '[class*="cc-window"]', '[class*="cc-overlay"]',
74 '[class*="privacy-banner"]', '[class*="privacy-notice"]',
75 '[id*="privacy-banner"]', '[id*="privacy-notice"]',
76 '[class*="accept-cookies"]', '[id*="accept-cookies"]',
77 ],
78 overlays: [
79 // Paywall / subscription overlays
80 '[class*="paywall"]', '[class*="Paywall"]', '[id*="paywall"]',
81 '[class*="subscribe-wall"]', '[class*="subscription-wall"]',
82 '[class*="meter-wall"]', '[class*="regwall"]', '[class*="reg-wall"]',
83 // Newsletter / signup popups
84 '[class*="newsletter-popup"]', '[class*="newsletter-modal"]',
85 '[class*="signup-modal"]', '[class*="signup-popup"]',
86 '[class*="email-capture"]', '[class*="lead-capture"]',
87 '[class*="popup-modal"]', '[class*="modal-overlay"]',
88 // Interstitials
89 '[class*="interstitial"]', '[id*="interstitial"]',
90 // Push notification prompts
91 '[class*="push-notification"]', '[class*="notification-prompt"]',
92 '[class*="web-push"]',
93 // Survey / feedback popups
94 '[class*="survey-"]', '[class*="feedback-modal"]',
95 '[id*="survey-"]', '[class*="nps-"]',
96 // App download banners
97 '[class*="app-banner"]', '[class*="smart-banner"]', '[class*="app-download"]',
98 '[id*="branch-banner"]', '.smartbanner',
99 // Cross-promotion / "follow us" / "preferred source" widgets
100 '[class*="promo-banner"]', '[class*="cross-promo"]', '[class*="partner-promo"]',
101 '[class*="preferred-source"]', '[class*="google-promo"]',
102 ],
103 clutter: [
104 // Audio/podcast player widgets (not part of the article text)
105 '[class*="audio-player"]', '[class*="podcast-player"]', '[class*="listen-widget"]',
106 '[class*="everlit"]', '[class*="Everlit"]',
107 'audio', // bare audio elements
108 // Sidebar games/puzzles widgets
109 '[class*="puzzle"]', '[class*="daily-game"]', '[class*="games-widget"]',
110 '[class*="crossword-promo"]', '[class*="mini-game"]',
111 // "Most Popular" / "Trending" sidebar recirculation (not the top nav trending bar)
112 'aside [class*="most-popular"]', 'aside [class*="trending"]',
113 'aside [class*="most-read"]', 'aside [class*="recommended"]',
114 // Related articles / recirculation at bottom
115 '[class*="related-articles"]', '[class*="more-stories"]',
116 '[class*="recirculation"]', '[class*="taboola"]', '[class*="outbrain"]',
117 // Hearst-specific (SF Chronicle, etc.)
118 '[class*="nativo"]', '[data-tb-region]',
119 ],
120 sticky: [
121 // Handled via JavaScript evaluation, not pure selectors
122 ],
123 social: [
124 '[class*="social-share"]', '[class*="share-buttons"]', '[class*="share-bar"]',
125 '[class*="social-widget"]', '[class*="social-icons"]', '[class*="share-tools"]',
126 'iframe[src*="facebook.com/plugins"]', 'iframe[src*="platform.twitter"]',
127 '[class*="fb-like"]', '[class*="tweet-button"]',
128 '[class*="addthis"]', '[class*="sharethis"]',
129 // Follow prompts
130 '[class*="follow-us"]', '[class*="social-follow"]',
131 ],
132};
134export async function handleWriteCommand(
135 command: string,
136 args: string[],
137 session: TabSession,
138 bm: BrowserManager
139): Promise<string> {
140 const page = session.getPage();
141 // Frame-aware target for locator-based operations (click, fill, etc.)
142 const target = session.getActiveFrameOrPage();
143 const inFrame = session.getFrame() !== null;
145 switch (command) {
146 case 'goto': {
147 if (inFrame) throw new Error('Cannot use goto inside a frame. Run \'frame main\' first.');
148 const url = args[0];
149 if (!url) throw new Error('Usage: browse goto <url>');
150 // Clear loadedHtml BEFORE navigation — a timeout after the main-frame commit
151 // must not leave stale content that could resurrect on a later context recreation.
152 session.clearLoadedHtml();
153 const normalizedUrl = await validateNavigationUrl(url);
154 const response = await page.goto(normalizedUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
155 const status = response?.status() || 'unknown';
156 return `Navigated to ${normalizedUrl} (${status})`;
157 }
159 case 'back': {
160 if (inFrame) throw new Error('Cannot use back inside a frame. Run \'frame main\' first.');
161 session.clearLoadedHtml();
162 await page.goBack({ waitUntil: 'domcontentloaded', timeout: 15000 });
163 return `Back → ${page.url()}`;
164 }
166 case 'forward': {
167 if (inFrame) throw new Error('Cannot use forward inside a frame. Run \'frame main\' first.');
168 session.clearLoadedHtml();
169 await page.goForward({ waitUntil: 'domcontentloaded', timeout: 15000 });
170 return `Forward → ${page.url()}`;
171 }
173 case 'reload': {
174 if (inFrame) throw new Error('Cannot use reload inside a frame. Run \'frame main\' first.');
175 session.clearLoadedHtml();
176 await page.reload({ waitUntil: 'domcontentloaded', timeout: 15000 });
177 return `Reloaded ${page.url()}`;
178 }
180 case 'load-html': {
181 if (inFrame) throw new Error('Cannot use load-html inside a frame. Run \'frame main\' first.');
183 // --from-file <path.json>: read inline HTML from a JSON payload. Used by
184 // make-pdf to dodge Windows argv size limits on large rendered HTML.
185 // The JSON shape is { html: string, waitUntil?: "load"|"domcontentloaded"|"networkidle" }.
186 // The safe-dirs + magic-byte + size-cap checks below still apply to the
187 // INLINE HTML content, not to the payload file path itself.
188 let fromFilePayload: { html: string; waitUntil?: SetContentWaitUntil } | null = null;
189 let filePath: string | undefined;
190 let waitUntil: SetContentWaitUntil = 'domcontentloaded';
191 for (let i = 0; i < args.length; i++) {
192 if (args[i] === '--from-file') {
193 const payloadPath = args[++i];
194 if (!payloadPath) throw new Error('load-html: --from-file requires a path');
195 // Parity with the sibling `load-html <file>` path below (line 249):
196 // that branch runs every `file://` target through validateReadPath
197 // so the safe-dirs policy can't be side-stepped. Same policy must
198 // apply here — otherwise --from-file becomes a read-anywhere escape
199 // hatch for any caller that can pick the payload path (e.g., an
200 // MCP caller issuing load-html with an attacker-influenced path).
201 try {
202 validateReadPath(path.resolve(payloadPath));
203 } catch {
204 throw new Error(
205 `load-html: --from-file ${payloadPath} must be under ${SAFE_DIRECTORIES.join(' or ')} (security policy). Copy the payload into the project tree or /tmp first.`
206 );
207 }
208 const raw = fs.readFileSync(payloadPath, 'utf8');
209 let json: any;
210 try { json = JSON.parse(raw); }
211 catch (e: any) { throw new Error(`load-html: --from-file JSON parse failed: ${e.message}`); }
212 if (typeof json.html !== 'string') {
213 throw new Error('load-html: --from-file JSON must have a "html" string field');
214 }
215 if (json.waitUntil && json.waitUntil !== 'load'
216 && json.waitUntil !== 'domcontentloaded' && json.waitUntil !== 'networkidle') {
217 throw new Error(`load-html: --from-file waitUntil '${json.waitUntil}' invalid`);
218 }
219 fromFilePayload = { html: json.html, waitUntil: json.waitUntil };
220 } else if (args[i] === '--wait-until') {
221 const val = args[++i];
222 if (val !== 'load' && val !== 'domcontentloaded' && val !== 'networkidle') {
223 throw new Error(`Invalid --wait-until '${val}'. Must be one of: load, domcontentloaded, networkidle.`);
224 }
225 waitUntil = val;
226 } else if (args[i].startsWith('--')) {
227 throw new Error(`Unknown flag: ${args[i]}`);
228 } else if (!filePath) {
229 filePath = args[i];
230 }
231 }
233 // Inline HTML path: validate size + magic byte, then setContent directly.
234 if (fromFilePayload) {
235 const MAX_BYTES = parseInt(process.env.GSTACK_BROWSE_MAX_HTML_BYTES || '', 10) || (50 * 1024 * 1024);
236 if (Buffer.byteLength(fromFilePayload.html, 'utf8') > MAX_BYTES) {
237 throw new Error(
238 `load-html: --from-file html too large (> ${MAX_BYTES} bytes). ` +
239 'Raise with GSTACK_BROWSE_MAX_HTML_BYTES=<N>.'
240 );
241 }
242 const peek = fromFilePayload.html.trimStart();
243 if (!/^<[a-zA-Z!?]/.test(peek)) {
244 throw new Error('load-html: --from-file html does not start with a valid markup opener');
245 }
246 const finalWaitUntil = fromFilePayload.waitUntil ?? waitUntil;
247 await session.setTabContent(fromFilePayload.html, { waitUntil: finalWaitUntil });
248 return `Loaded HTML: (inline from --from-file, ${fromFilePayload.html.length} chars)`;
249 }
251 if (!filePath) throw new Error('Usage: browse load-html <file> [--wait-until load|domcontentloaded|networkidle] [--tab-id <N>] | load-html --from-file <payload.json> [--tab-id <N>]');
253 // Extension allowlist
254 const ALLOWED_EXT = ['.html', '.htm', '.xhtml'];
255 const ext = path.extname(filePath).toLowerCase();
256 if (!ALLOWED_EXT.includes(ext)) {
257 throw new Error(
258 `load-html: file does not appear to be HTML. Expected .html/.htm/.xhtml, got ${ext || '(no extension)'}. Rename the file if it's really HTML.`
259 );
260 }
262 const absolutePath = path.resolve(filePath);
264 // Safe-dirs check (reuses canonical read-side policy)
265 try {
266 validateReadPath(absolutePath);
267 } catch (e: any) {
268 throw new Error(
269 `load-html: ${absolutePath} must be under ${SAFE_DIRECTORIES.join(' or ')} (security policy). Copy the file into the project tree or /tmp first.`
270 );
271 }
273 // stat check — reject non-file targets with actionable error
274 let stat: fs.Stats;
275 try {
276 stat = await fs.promises.stat(absolutePath);
277 } catch (e: any) {
278 if (e.code === 'ENOENT') {
279 throw new Error(
280 `load-html: file not found at ${absolutePath}. Check spelling or copy the file under ${process.cwd()} or ${TEMP_DIR}.`
281 );
282 }
283 throw e;
284 }
285 if (stat.isDirectory()) {
286 throw new Error(`load-html: ${absolutePath} is a directory, not a file. Pass a .html file.`);
287 }
288 if (!stat.isFile()) {
289 throw new Error(`load-html: ${absolutePath} is not a regular file.`);
290 }
292 // Size cap
293 const MAX_BYTES = parseInt(process.env.GSTACK_BROWSE_MAX_HTML_BYTES || '', 10) || (50 * 1024 * 1024);
294 if (stat.size > MAX_BYTES) {
295 throw new Error(
296 `load-html: file too large (${stat.size} bytes > ${MAX_BYTES} cap). Raise with GSTACK_BROWSE_MAX_HTML_BYTES=<N> or split the HTML.`
297 );
298 }
300 // Single read: Buffer → magic-byte peek → utf-8 string
301 const buf = await fs.promises.readFile(absolutePath);
303 // Magic-byte check: strip UTF-8 BOM + leading whitespace, then verify the first
304 // non-whitespace byte starts a markup construct. Accepts any <tag, <!doctype,
305 // <!-- comment, <?xml prolog — including bare HTML fragments like `<div>...</div>`
306 // which setContent wraps in a full document. Rejects binary files mis-renamed .html
307 // (first byte won't be `<`).
308 let peek = buf.slice(0, 200);
309 if (peek[0] === 0xEF && peek[1] === 0xBB && peek[2] === 0xBF) {
310 peek = peek.slice(3);
311 }
312 const peekStr = peek.toString('utf8').trimStart();
313 // Valid markup opener: '<' followed by alpha (tag), '!' (doctype/comment), or '?' (xml prolog)
314 const looksLikeMarkup = /^<[a-zA-Z!?]/.test(peekStr);
315 if (!looksLikeMarkup) {
316 const hexDump = Array.from(buf.slice(0, 16)).map(b => b.toString(16).padStart(2, '0')).join(' ');
317 throw new Error(
318 `load-html: ${absolutePath} has ${ext} extension but content does not look like HTML. First bytes: ${hexDump}`
319 );
320 }
322 const html = buf.toString('utf8');
323 await session.setTabContent(html, { waitUntil });
324 return `Loaded HTML: ${absolutePath} (${stat.size} bytes)`;
325 }
327 case 'click': {
328 const selector = args[0];
329 if (!selector) throw new Error('Usage: browse click <selector>');
331 // Auto-route: if ref points to a real <option> inside a <select>, use selectOption
332 const role = session.getRefRole(selector);
333 if (role === 'option') {
334 const resolved = await session.resolveRef(selector);
335 if ('locator' in resolved) {
336 const optionInfo = await resolved.locator.evaluate(el => {
337 if (el.tagName !== 'OPTION') return null; // custom [role=option], not real <option>
338 const option = el as HTMLOptionElement;
339 const select = option.closest('select');
340 if (!select) return null;
341 return { value: option.value, text: option.text };
342 });
343 if (optionInfo) {
344 await resolved.locator.locator('xpath=ancestor::select').selectOption(optionInfo.value, { timeout: 5000 });
345 return `Selected "${optionInfo.text}" (auto-routed from click on <option>) → now at ${page.url()}`;
346 }
347 // Real <option> with no parent <select> or custom [role=option] — fall through to normal click
348 }
349 }
351 const resolved = await session.resolveRef(selector);
352 try {
353 if ('locator' in resolved) {
354 await resolved.locator.click({ timeout: 5000 });
355 } else {
356 await target.locator(resolved.selector).click({ timeout: 5000 });
357 }
358 } catch (err: any) {
359 // Enhanced error guidance: clicking <option> elements always fails (not visible / timeout)
360 const isOption = 'locator' in resolved
361 ? await resolved.locator.evaluate(el => el.tagName === 'OPTION').catch(() => false)
362 : await target.locator(resolved.selector).evaluate(
363 el => el.tagName === 'OPTION'
364 ).catch(() => false);
365 if (isOption) {
366 throw new Error(
367 `Cannot click <option> elements. Use 'browse select <parent-select> <value>' instead of 'click' for dropdown options.`
368 );
369 }
370 throw err;
371 }
372 // Wait for network to settle (catches XHR/fetch triggered by clicks)
373 await page.waitForLoadState('networkidle', { timeout: 2000 }).catch(() => {});
374 return `Clicked ${selector} → now at ${page.url()}`;
375 }
377 case 'fill': {
378 const [selector, ...valueParts] = args;
379 const value = valueParts.join(' ');
380 if (!selector || !value) throw new Error('Usage: browse fill <selector> <value>');
381 const resolved = await session.resolveRef(selector);
382 const locator = 'locator' in resolved ? resolved.locator : target.locator(resolved.selector);
383 await locator.fill(value, { timeout: 5000 });
384 // Playwright's fill() only dispatches an `input` event. Frameworks that
385 // validate on `change` (AngularJS ng-change, debounced strength/match
386 // checks — e.g. cPanel's Jupiter theme) never see the update, so a value
387 // that's correct in the DOM can still fail the framework's own
388 // validation. Dispatch `change` too so those listeners fire.
389 await locator.dispatchEvent('change');
390 // Wait for network to settle (form validation XHRs)
391 await page.waitForLoadState('networkidle', { timeout: 2000 }).catch(() => {});
392 return `Filled ${selector}`;
393 }
395 case 'select': {
396 const [selector, ...valueParts] = args;
397 const value = valueParts.join(' ');
398 if (!selector || !value) throw new Error('Usage: browse select <selector> <value>');
399 const resolved = await session.resolveRef(selector);
400 if ('locator' in resolved) {
401 await resolved.locator.selectOption(value, { timeout: 5000 });
402 } else {
403 await target.locator(resolved.selector).selectOption(value, { timeout: 5000 });
404 }
405 // Wait for network to settle (dropdown-triggered requests)
406 await page.waitForLoadState('networkidle', { timeout: 2000 }).catch(() => {});
407 return `Selected "${value}" in ${selector}`;
408 }
410 case 'hover': {
411 const selector = args[0];
412 if (!selector) throw new Error('Usage: browse hover <selector>');
413 const resolved = await session.resolveRef(selector);
414 if ('locator' in resolved) {
415 await resolved.locator.hover({ timeout: 5000 });
416 } else {
417 await target.locator(resolved.selector).hover({ timeout: 5000 });
418 }
419 return `Hovered ${selector}`;
420 }
422 case 'type': {
423 // #2936: bare `type` sends keystrokes to whatever has focus, so a selector
424 // given as the first word was typed as text. --selector targets an
425 // element; `--` ends flags so literal text may start with "--".
426 const usage = 'Usage: browse type [--selector <sel>] [--] <text>';
427 let rest = args;
428 let selector: string | undefined;
429 if (rest[0] === '--selector') {
430 selector = rest[1];
431 if (!selector) throw new Error(usage);
432 rest = rest.slice(2);
433 }
434 if (rest[0] === '--') rest = rest.slice(1);
435 const text = rest.join(' ');
436 if (!text) throw new Error(usage);
437 if (selector) {
438 const resolved = await session.resolveRef(selector);
439 const locator = 'locator' in resolved ? resolved.locator : target.locator(resolved.selector);
440 await locator.pressSequentially(text, { timeout: 5000 });
441 return `Typed ${text.length} characters into ${selector}`;
442 }
443 await page.keyboard.type(text);
444 const first = rest[0];
445 const selectorLike = /^([#.][A-Za-z_-]|\[[^\]]+\]$|[A-Za-z][\w-]*[#.[][A-Za-z_-])/.test(first) || rest.includes('>');
446 return selectorLike
447 ? `Typed ${text.length} characters into the focused element\nhint: "${first}" looks like a CSS selector. To type into that element: browse type --selector '${first}' <text>`
448 : `Typed ${text.length} characters`;
449 }
451 case 'press': {
452 const key = args[0];
453 if (!key) throw new Error('Usage: browse press <key> (e.g., Enter, Tab, Escape)');
454 await page.keyboard.press(key);
455 return `Pressed ${key}`;
456 }
458 case 'scroll': {
459 // Parse --times N and --wait ms flags
460 const timesIdx = args.indexOf('--times');
461 const times = timesIdx >= 0 ? parseInt(args[timesIdx + 1], 10) || 1 : 0;
462 const waitIdx = args.indexOf('--wait');
463 const waitMs = waitIdx >= 0 ? parseInt(args[waitIdx + 1], 10) || 1000 : 1000;
464 const selector = args.find(a => !a.startsWith('--') && args.indexOf(a) !== timesIdx + 1 && args.indexOf(a) !== waitIdx + 1);
466 if (times > 0) {
467 // Repeated scroll mode
468 for (let i = 0; i < times; i++) {
469 if (selector) {
470 const resolved = await bm.resolveRef(selector);
471 if ('locator' in resolved) {
472 await resolved.locator.scrollIntoViewIfNeeded({ timeout: 5000 });
473 } else {
474 await target.locator(resolved.selector).scrollIntoViewIfNeeded({ timeout: 5000 });
475 }
476 } else {
477 await target.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
478 }
479 if (i < times - 1) await new Promise(r => setTimeout(r, waitMs));
480 }
481 return `Scrolled ${times} times${selector ? ` (${selector})` : ''} with ${waitMs}ms delay`;
482 }
484 // Single scroll (original behavior)
485 if (selector) {
486 const resolved = await session.resolveRef(selector);
487 if ('locator' in resolved) {
488 await resolved.locator.scrollIntoViewIfNeeded({ timeout: 5000 });
489 } else {
490 await target.locator(resolved.selector).scrollIntoViewIfNeeded({ timeout: 5000 });
491 }
492 return `Scrolled ${selector} into view`;
493 }
494 await target.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
495 return 'Scrolled to bottom';
496 }
498 case 'wait': {
499 const selector = args[0];
500 if (!selector) throw new Error('Usage: browse wait <selector|--networkidle|--load|--domcontentloaded>');
501 if (selector === '--networkidle') {
502 const MAX_WAIT_MS = 300_000;
503 const MIN_WAIT_MS = 1_000;
504 const timeout = Math.min(Math.max(args[1] ? parseInt(args[1], 10) || MIN_WAIT_MS : 15000, MIN_WAIT_MS), MAX_WAIT_MS);
505 await page.waitForLoadState('networkidle', { timeout });
506 return 'Network idle';
507 }
508 if (selector === '--load') {
509 await page.waitForLoadState('load');
510 return 'Page loaded';
511 }
512 if (selector === '--domcontentloaded') {
513 await page.waitForLoadState('domcontentloaded');
514 return 'DOM content loaded';
515 }
516 const MAX_WAIT_MS = 300_000;
517 const MIN_WAIT_MS = 1_000;
518 const timeout = Math.min(Math.max(args[1] ? parseInt(args[1], 10) || MIN_WAIT_MS : 15000, MIN_WAIT_MS), MAX_WAIT_MS);
519 const resolved = await session.resolveRef(selector);
520 if ('locator' in resolved) {
521 await resolved.locator.waitFor({ state: 'visible', timeout });
522 } else {
523 await target.locator(resolved.selector).waitFor({ state: 'visible', timeout });
524 }
525 return `Element ${selector} appeared`;
526 }
528 case 'viewport': {
529 // Parse args: [<WxH>] [--scale <n>]. Either may be omitted, but NOT both.
530 let sizeArg: string | undefined;
531 let scaleArg: number | undefined;
532 for (let i = 0; i < args.length; i++) {
533 if (args[i] === '--scale') {
534 const val = args[++i];
535 if (val === undefined || val === '') {
536 throw new Error('viewport --scale: missing value. Usage: viewport [WxH] --scale <n>');
537 }
538 const parsed = Number(val);
539 if (!Number.isFinite(parsed)) {
540 throw new Error(`viewport --scale: value '${val}' is not a finite number.`);
541 }
542 scaleArg = parsed;
543 } else if (args[i].startsWith('--')) {
544 throw new Error(`Unknown viewport flag: ${args[i]}`);
545 } else if (sizeArg === undefined) {
546 sizeArg = args[i];
547 } else {
548 throw new Error(`Unexpected positional arg: ${args[i]}. Usage: viewport [WxH|auto] [--scale <n>]`);
549 }
550 }
552 if (sizeArg === undefined && scaleArg === undefined) {
553 throw new Error('Usage: browse viewport [<WxH>|auto] [--scale <n>] (e.g. 375x812, auto to unpin, or --scale 2 to keep current size)');
554 }
556 if (sizeArg !== undefined && ['auto', 'reset', 'unpin'].includes(sizeArg.toLowerCase())) {
557 if (scaleArg !== undefined) throw new Error('viewport auto cannot be combined with --scale or a size.');
558 return (await bm.resetViewport()) === 'window'
559 ? 'Viewport unpinned: the page follows the browser window again (refs cleared)'
560 : 'Viewport reset to default 1280x720';
561 }
563 // Resolve width/height: either from sizeArg or from current viewport if --scale-only.
564 let w: number, h: number;
565 if (sizeArg) {
566 if (!sizeArg.includes('x')) throw new Error('Usage: browse viewport [<WxH>|auto] [--scale <n>] (e.g., 375x812)');
567 const [rawW, rawH] = sizeArg.split('x').map(Number);
568 w = Math.min(Math.max(Math.round(rawW) || 1280, 1), 16384);
569 h = Math.min(Math.max(Math.round(rawH) || 720, 1), 16384);
570 } else {
571 // --scale without WxH → use BrowserManager's tracked viewport (source of truth
572 // since setViewport + launchContext keep it in sync). Falls back reliably on
573 // headed → headless transitions or contexts with viewport:null.
574 const current = bm.getCurrentViewport();
575 w = current.width;
576 h = current.height;
577 }
579 if (scaleArg !== undefined) {
580 const err = await bm.setDeviceScaleFactor(scaleArg, w, h);
581 if (err) return `Viewport partially set: ${err}`;
582 return `Viewport set to ${w}x${h} @ ${scaleArg}x (context recreated; refs and load-html content replayed)`;
583 }
585 await bm.setViewport(w, h);
586 return `Viewport set to ${w}x${h}`;
587 }
589 case 'cookie': {
590 const cookieStr = args[0];
591 if (!cookieStr || !cookieStr.includes('=')) throw new Error('Usage: browse cookie <name>=<value>');
592 const eq = cookieStr.indexOf('=');
593 const name = cookieStr.slice(0, eq);
594 const value = cookieStr.slice(eq + 1);
595 const url = new URL(page.url());
596 await page.context().addCookies([{
597 name,
598 value,
599 domain: url.hostname,
600 path: '/',
601 }]);
602 return `Cookie set: ${name}=****`;
603 }
605 case 'header': {
606 const headerStr = args[0];
607 if (!headerStr || !headerStr.includes(':')) throw new Error('Usage: browse header <name>:<value>');
608 const sep = headerStr.indexOf(':');
609 const name = headerStr.slice(0, sep).trim();
610 const value = headerStr.slice(sep + 1).trim();
611 await bm.setExtraHeader(name, value);
612 const sensitiveHeaders = ['authorization', 'cookie', 'set-cookie', 'x-api-key', 'x-auth-token'];
613 const redactedValue = sensitiveHeaders.includes(name.toLowerCase()) ? '****' : value;
614 return `Header set: ${name}: ${redactedValue}`;
615 }
617 case 'useragent': {
618 const ua = args.join(' ');
619 if (!ua) throw new Error('Usage: browse useragent <string>');
620 bm.setUserAgent(ua);
621 const error = await bm.recreateContext();
622 if (error) {
623 return `User agent set to "${ua}" but: ${error}`;
624 }
625 return `User agent set: ${ua}`;
626 }
628 case 'upload': {
629 const [selector, ...filePaths] = args;
630 if (!selector || filePaths.length === 0) throw new Error('Usage: browse upload <selector> <file1> [file2...]');
632 const validatedPaths = filePaths.map(fp => {
633 if (!fs.existsSync(fp)) throw new Error(`File not found: ${fp}`);
634 const realPath = fs.realpathSync(path.resolve(fp));
635 validateReadPath(realPath);
636 return realPath;
637 });
639 const resolved = await session.resolveRef(selector);
640 if ('locator' in resolved) {
641 await resolved.locator.setInputFiles(validatedPaths);
642 } else {
643 await target.locator(resolved.selector).setInputFiles(validatedPaths);
644 }
646 const fileInfo = validatedPaths.map(fp => {
647 const stat = fs.statSync(fp);
648 return `${path.basename(fp)} (${stat.size}B)`;
649 }).join(', ');
650 return `Uploaded: ${fileInfo}`;
651 }
653 case 'dialog-accept': {
654 const text = args.length > 0 ? args.join(' ') : null;
655 bm.setDialogAutoAccept(true);
656 bm.setDialogPromptText(text);
657 return text
658 ? `Dialogs will be accepted with text: "${text}"`
659 : 'Dialogs will be accepted';
660 }
662 case 'dialog-dismiss': {
663 bm.setDialogAutoAccept(false);
664 bm.setDialogPromptText(null);
665 return 'Dialogs will be dismissed';
666 }
668 case 'cookie-import': {
669 const filePath = args[0];
670 if (!filePath) throw new Error('Usage: browse cookie-import <json-file>');
671 // Path validation against safe dirs (#707: relative paths included),
672 // through the same realpath as SAFE_DIRECTORIES (#3104).
673 validateReadPath(filePath);
674 if (!fs.existsSync(filePath)) throw new Error(`File not found: ${filePath}`);
675 const raw = fs.readFileSync(filePath, 'utf-8');
676 let cookies: any[];
677 try { cookies = JSON.parse(raw); } catch (err: any) { throw new Error(`Invalid JSON in ${filePath}: ${err?.message || err}`); }
678 if (!Array.isArray(cookies)) throw new Error('Cookie file must contain a JSON array');
680 // Auto-fill domain from current page URL when missing (consistent with cookie command)
681 const pageUrl = new URL(page.url());
682 const defaultDomain = pageUrl.hostname;
684 for (const c of cookies) {
685 if (!c.name || c.value === undefined) throw new Error('Each cookie must have "name" and "value" fields');
686 if (!c.domain) {
687 c.domain = defaultDomain;
688 } else {
689 const cookieDomain = c.domain.startsWith('.') ? c.domain.slice(1) : c.domain;
690 if (cookieDomain !== defaultDomain && !defaultDomain.endsWith('.' + cookieDomain)) {
691 throw new Error(`Cookie domain "${c.domain}" does not match current page domain "${defaultDomain}". Use the target site first.`);
692 }
693 }
694 if (!c.path) c.path = '/';
695 }
697 await page.context().addCookies(cookies);
698 const importedDomains = [...new Set(cookies.map((c: any) => c.domain).filter(Boolean))];
699 if (importedDomains.length > 0) bm.trackCookieImportDomains(importedDomains);
700 return `Loaded ${cookies.length} cookies from ${filePath}`;
701 }
703 case 'cookie-import-browser': {
704 const options = parseCookieImportArgs(args);
705 const target = { page, url: page.url() };
706 const authOptions = {
707 identitySelector: process.env.GSTACK_COOKIE_AUTH_SELECTOR,
708 expectedIdentity: process.env.GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY,
709 };
710 if (options.domains || options.all) {
711 if (options.domains) {
712 const targetUrl = validateCookieTarget(target);
713 if (!options.domains.every(domain => cookieDomainMatches(targetUrl.hostname, '.' + domain))) {
714 throw new CookieImportError('The requested cookie domain does not match the current page. Navigate to the target site first.', 'target_mismatch');
715 }
716 }
717 const result = await runCookieImport(options, target, domains => bm.trackCookieImportDomains(domains), authOptions);
718 const message = formatCookieImportResult(result);
719 if (result.outcome === 'failed' || options.verifyAuth && !result.verification.verified) {
720 throw new CookieImportError(message, 'cookie_import_incomplete');
721 }
722 return message + (options.all ? ' Used --all: all source-profile cookie domains were selected.' : '');
723 }
725 const port = bm.serverPort;
726 if (!port) throw new CookieImportError('Server port not available', 'unavailable');
727 const browsers = findInstalledBrowsers();
728 if (browsers.length === 0) throw new CookieImportError(`No Chromium browsers found. Supported: ${listSupportedBrowserNames().join(', ')}`, 'not_installed');
729 if (options.clearStorage || options.verifyAuth) validateCookieTarget(target);
730 if (options.clearStorage) validateCookieStorageSupport(page);
731 if (options.verifyAuth) validateCookieAuthOptions(authOptions);
732 const code = generatePickerCode({
733 target,
734 browser: options.browser,
735 profile: options.profile,
736 clearStorage: options.clearStorage,
737 verifyAuth: options.verifyAuth,
738 });
739 const pickerUrl = `http://127.0.0.1:${port}/cookie-picker?code=${code}`;
740 const openCommand = process.platform === 'darwin' ? ['open', pickerUrl]
741 : process.platform === 'win32' ? ['cmd.exe', '/d', '/c', 'start', '', pickerUrl] : ['xdg-open', pickerUrl];
742 try {
743 Bun.spawn(openCommand, { stdout: 'ignore', stderr: 'ignore', windowsHide: true });
744 } catch {
745 throw new CookieImportError('Could not open the cookie picker in a local browser. Retry from a desktop session.', 'picker_open_failed');
746 }
747 return `Cookie picker opened at http://127.0.0.1:${port}/cookie-picker\nDetected browsers: ${browsers.map(b => b.name).join(', ')}\nSelect the source profile and domains. Cookies copied does not mean sign-in verified.`;
748 }
750 case 'style': {
751 // style --undo [N] → revert modification
752 if (args[0] === '--undo') {
753 const idx = args[1] ? parseInt(args[1], 10) : undefined;
754 await undoModification(page, idx);
755 return idx !== undefined ? `Reverted modification #${idx}` : 'Reverted last modification';
756 }
758 // style <selector> <property> <value>
759 const [selector, property, ...valueParts] = args;
760 const value = valueParts.join(' ');
761 if (!selector || !property || !value) {
762 throw new Error('Usage: browse style <sel> <prop> <value> | style --undo [N]');
763 }
765 // Validate CSS property name
766 if (!/^[a-zA-Z-]+$/.test(property)) {
767 throw new Error(`Invalid CSS property name: ${property}. Only letters and hyphens allowed.`);
768 }
770 // Validate CSS value — block data exfiltration patterns
771 const DANGEROUS_CSS = /url\s*\(|expression\s*\(|@import|javascript:|data:/i;
772 if (DANGEROUS_CSS.test(value)) {
773 throw new Error('CSS value rejected: contains potentially dangerous pattern.');
774 }
776 const mod = await modifyStyle(page, selector, property, value);
777 return `Style modified: ${selector} { ${property}: ${mod.oldValue || '(none)'} → ${value} } (${mod.method})`;
778 }
780 case 'cleanup': {
781 // Parse flags
782 let doAds = false, doCookies = false, doSticky = false, doSocial = false;
783 let doOverlays = false, doClutter = false;
784 let doAll = false;
786 // Default to --all if no args (most common use case from sidebar button)
787 if (args.length === 0) {
788 doAll = true;
789 }
791 for (const arg of args) {
792 switch (arg) {
793 case '--ads': doAds = true; break;
794 case '--cookies': doCookies = true; break;
795 case '--sticky': doSticky = true; break;
796 case '--social': doSocial = true; break;
797 case '--overlays': doOverlays = true; break;
798 case '--clutter': doClutter = true; break;
799 case '--all': doAll = true; break;
800 default:
801 throw new Error(`Unknown cleanup flag: ${arg}. Use: --ads, --cookies, --sticky, --social, --overlays, --clutter, --all`);
802 }
803 }
805 if (doAll) {
806 doAds = doCookies = doSticky = doSocial = doOverlays = doClutter = true;
807 }
809 const removed: string[] = [];
811 // Build selector list for categories to clean
812 const selectors: string[] = [];
813 if (doAds) selectors.push(...CLEANUP_SELECTORS.ads);
814 if (doCookies) selectors.push(...CLEANUP_SELECTORS.cookies);
815 if (doSocial) selectors.push(...CLEANUP_SELECTORS.social);
816 if (doOverlays) selectors.push(...CLEANUP_SELECTORS.overlays);
817 if (doClutter) selectors.push(...CLEANUP_SELECTORS.clutter);
819 if (selectors.length > 0) {
820 const count = await page.evaluate((sels: string[]) => {
821 let removed = 0;
822 for (const sel of sels) {
823 try {
824 const els = document.querySelectorAll(sel);
825 els.forEach(el => {
826 (el as HTMLElement).style.setProperty('display', 'none', 'important');
827 removed++;
828 });
829 } catch (err: any) {
830 // querySelectorAll throws DOMException on invalid CSS selectors — skip those
831 if (!(err instanceof DOMException)) throw err;
832 }
833 }
834 return removed;
835 }, selectors);
836 if (count > 0) {
837 if (doAds) removed.push('ads');
838 if (doCookies) removed.push('cookie banners');
839 if (doSocial) removed.push('social widgets');
840 if (doOverlays) removed.push('overlays/popups');
841 if (doClutter) removed.push('clutter');
842 }
843 }
845 // Sticky/fixed elements — handled separately with computed style check
846 if (doSticky) {
847 const stickyCount = await page.evaluate(() => {
848 let removed = 0;
849 // Collect all sticky/fixed elements, sort by vertical position
850 const stickyEls: Array<{ el: Element; top: number; width: number; height: number }> = [];
851 const allElements = document.querySelectorAll('*');
852 const viewportWidth = window.innerWidth;
853 for (const el of allElements) {
854 const style = getComputedStyle(el);
855 if (style.position === 'fixed' || style.position === 'sticky') {
856 const rect = el.getBoundingClientRect();
857 stickyEls.push({ el, top: rect.top, width: rect.width, height: rect.height });
858 }
859 }
860 // Sort by vertical position (topmost first)
861 stickyEls.sort((a, b) => a.top - b.top);
862 let preservedTopNav = false;
863 for (const { el, top, width, height } of stickyEls) {
864 const tag = el.tagName.toLowerCase();
865 // Always skip nav/header semantic elements
866 if (tag === 'nav' || tag === 'header') continue;
867 if (el.getAttribute('role') === 'navigation') continue;
868 // Skip the gstack control indicator
869 if ((el as HTMLElement).id === 'gstack-ctrl') continue;
870 // Preserve the FIRST full-width element near the top (site's main nav bar)
871 // This catches divs that act as navbars but aren't semantic <nav> elements
872 if (!preservedTopNav && top <= 50 && width > viewportWidth * 0.8 && height < 120) {
873 preservedTopNav = true;
874 continue;
875 }
876 (el as HTMLElement).style.setProperty('display', 'none', 'important');
877 removed++;
878 }
879 return removed;
880 });
881 if (stickyCount > 0) removed.push(`${stickyCount} sticky/fixed elements`);
882 }
884 // Unlock scrolling (many sites lock body scroll when modals are open)
885 const scrollFixed = await page.evaluate(() => {
886 let fixed = 0;
887 // Unlock body and html scroll
888 for (const el of [document.body, document.documentElement]) {
889 if (!el) continue;
890 const style = getComputedStyle(el);
891 if (style.overflow === 'hidden' || style.overflowY === 'hidden') {
892 (el as HTMLElement).style.setProperty('overflow', 'auto', 'important');
893 (el as HTMLElement).style.setProperty('overflow-y', 'auto', 'important');
894 fixed++;
895 }
896 // Remove height:100% + position:fixed that locks scroll
897 if (style.position === 'fixed' && (el === document.body || el === document.documentElement)) {
898 (el as HTMLElement).style.setProperty('position', 'static', 'important');
899 fixed++;
900 }
901 }
902 // Remove blur/filter effects (paywalls often blur the content)
903 const blurred = document.querySelectorAll('[style*="blur"], [style*="filter"]');
904 blurred.forEach(el => {
905 const s = (el as HTMLElement).style;
906 if (s.filter?.includes('blur') || s.webkitFilter?.includes('blur')) {
907 s.setProperty('filter', 'none', 'important');
908 s.setProperty('-webkit-filter', 'none', 'important');
909 fixed++;
910 }
911 });
912 // Remove max-height truncation (article truncation)
913 const truncated = document.querySelectorAll('[class*="truncat"], [class*="preview"], [class*="teaser"]');
914 truncated.forEach(el => {
915 const s = getComputedStyle(el);
916 if (s.maxHeight && s.maxHeight !== 'none' && parseInt(s.maxHeight) < 500) {
917 (el as HTMLElement).style.setProperty('max-height', 'none', 'important');
918 (el as HTMLElement).style.setProperty('overflow', 'visible', 'important');
919 fixed++;
920 }
921 });
922 return fixed;
923 });
924 if (scrollFixed > 0) removed.push('scroll unlocked');
926 // Remove "ADVERTISEMENT" / "Article continues below" text labels
927 const adLabelCount = await page.evaluate(() => {
928 let removed = 0;
929 const adTextPatterns = [
930 /^advertisement$/i, /^sponsored$/i, /^promoted$/i,
931 /article continues/i, /continues below/i,
932 /^ad$/i, /^paid content$/i, /^partner content$/i,
933 ];
934 // Walk text-heavy small elements looking for ad labels
935 const candidates = document.querySelectorAll('div, span, p, figcaption, label');
936 for (const el of candidates) {
937 const text = (el.textContent || '').trim();
938 if (text.length > 50) continue; // Too much text, probably real content
939 if (adTextPatterns.some(p => p.test(text))) {
940 // Also hide the parent if it's a wrapper with little else
941 const parent = el.parentElement;
942 if (parent && (parent.textContent || '').trim().length < 80) {
943 (parent as HTMLElement).style.setProperty('display', 'none', 'important');
944 } else {
945 (el as HTMLElement).style.setProperty('display', 'none', 'important');
946 }
947 removed++;
948 }
949 }
950 return removed;
951 });
952 if (adLabelCount > 0) removed.push(`${adLabelCount} ad labels`);
954 // Remove empty ad placeholder whitespace (divs that are now empty after ad removal)
955 const collapsedCount = await page.evaluate(() => {
956 let collapsed = 0;
957 const candidates = document.querySelectorAll(
958 'div[class*="ad"], div[id*="ad"], aside[class*="ad"], div[class*="sidebar"], ' +
959 'div[class*="rail"], div[class*="right-col"], div[class*="widget"]'
960 );
961 for (const el of candidates) {
962 const rect = el.getBoundingClientRect();
963 // If the element has significant height but no visible text content, collapse it
964 if (rect.height > 50 && rect.width > 0) {
965 const text = (el.textContent || '').trim();
966 const images = el.querySelectorAll('img:not([src*="logo"]):not([src*="icon"])');
967 const links = el.querySelectorAll('a');
968 // Empty or mostly empty: collapse
969 if (text.length < 20 && images.length === 0 && links.length < 2) {
970 (el as HTMLElement).style.setProperty('display', 'none', 'important');
971 collapsed++;
972 }
973 }
974 }
975 return collapsed;
976 });
977 if (collapsedCount > 0) removed.push(`${collapsedCount} empty placeholders`);
979 if (removed.length === 0) return 'No clutter elements found to remove.';
980 return `Cleaned up: ${removed.join(', ')}`;
981 }
983 case 'prettyscreenshot': {
984 // Parse flags
985 let scrollTo: string | undefined;
986 let doCleanup = false;
987 const hideSelectors: string[] = [];
988 let viewportWidth: number | undefined;
989 let outputPath: string | undefined;
991 for (let i = 0; i < args.length; i++) {
992 if (args[i] === '--scroll-to' && i + 1 < args.length) {
993 scrollTo = args[++i];
994 } else if (args[i] === '--cleanup') {
995 doCleanup = true;
996 } else if (args[i] === '--hide' && i + 1 < args.length) {
997 // Collect all following non-flag args as selectors to hide. A
998 // trailing image path is the output path, not a selector (#1419).
999 const values: string[] = [];
1000 i++;
1001 while (i < args.length && !args[i].startsWith('--')) {
1002 values.push(args[i]);
1003 i++;
1004 }
1005 i--; // Back up since the for loop will increment
1006 const last = values[values.length - 1];
1007 if (!outputPath && values.length > 1 && /^(\/|\.\.?\/|~)|\.(png|jpe?g|webp)$/i.test(last)) outputPath = values.pop();
1008 hideSelectors.push(...values);
1009 } else if (args[i] === '--width' && i + 1 < args.length) {
1010 viewportWidth = parseInt(args[++i], 10);
1011 if (isNaN(viewportWidth)) throw new Error('--width must be a number');
1012 } else if (!args[i].startsWith('--')) {
1013 outputPath = args[i];
1014 } else {
1015 throw new Error(`Unknown prettyscreenshot flag: ${args[i]}`);
1016 }
1017 }
1019 // Default output path
1020 if (!outputPath) {
1021 const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
1022 outputPath = `${TEMP_DIR}/browse-pretty-${timestamp}.png`;
1023 }
1024 validateOutputPath(outputPath);
1026 const originalViewport = page.viewportSize();
1028 // Set viewport width if specified
1029 if (viewportWidth && originalViewport) {
1030 await page.setViewportSize({ width: viewportWidth, height: originalViewport.height });
1031 }
1033 // Run cleanup if requested
1034 if (doCleanup) {
1035 const allSelectors = [
1036 ...CLEANUP_SELECTORS.ads,
1037 ...CLEANUP_SELECTORS.cookies,
1038 ...CLEANUP_SELECTORS.social,
1039 ];
1040 await page.evaluate((sels: string[]) => {
1041 for (const sel of sels) {
1042 try {
1043 document.querySelectorAll(sel).forEach(el => {
1044 (el as HTMLElement).style.display = 'none';
1045 });
1046 } catch (err: any) {
1047 if (!(err instanceof DOMException)) throw err;
1048 }
1049 }
1050 // Also hide fixed/sticky (except nav)
1051 for (const el of document.querySelectorAll('*')) {
1052 const style = getComputedStyle(el);
1053 if (style.position === 'fixed' || style.position === 'sticky') {
1054 const tag = el.tagName.toLowerCase();
1055 if (tag === 'nav' || tag === 'header') continue;
1056 if (el.getAttribute('role') === 'navigation') continue;
1057 (el as HTMLElement).style.display = 'none';
1058 }
1059 }
1060 }, allSelectors);
1061 }
1063 // Hide specific elements
1064 if (hideSelectors.length > 0) {
1065 await page.evaluate((sels: string[]) => {
1066 for (const sel of sels) {
1067 try {
1068 document.querySelectorAll(sel).forEach(el => {
1069 (el as HTMLElement).style.display = 'none';
1070 });
1071 } catch (err: any) {
1072 if (!(err instanceof DOMException)) throw err;
1073 }
1074 }
1075 }, hideSelectors);
1076 }
1078 // Scroll to target
1079 if (scrollTo) {
1080 // Try as CSS selector first, then as text content
1081 const scrolled = await page.evaluate((target: string) => {
1082 // Try CSS selector
1083 let el = document.querySelector(target);
1084 if (el) {
1085 el.scrollIntoView({ behavior: 'instant', block: 'center' });
1086 return true;
1087 }
1088 // Try text match
1089 const walker = document.createTreeWalker(
1090 document.body,
1091 NodeFilter.SHOW_TEXT,
1092 null,
1093 );
1094 let node: Node | null;
1095 while ((node = walker.nextNode())) {
1096 if (node.textContent?.includes(target)) {
1097 const parent = node.parentElement;
1098 if (parent) {
1099 parent.scrollIntoView({ behavior: 'instant', block: 'center' });
1100 return true;
1101 }
1102 }
1103 }
1104 return false;
1105 }, scrollTo);
1107 if (!scrolled) {
1108 // Restore viewport before throwing
1109 if (viewportWidth && originalViewport) {
1110 await page.setViewportSize(originalViewport);
1111 }
1112 throw new Error(`Could not find element or text to scroll to: ${scrollTo}`);
1113 }
1114 // Brief wait for scroll to settle
1115 await page.waitForTimeout(300);
1116 }
1118 // Take screenshot
1119 await page.screenshot({ path: outputPath, fullPage: !scrollTo });
1120 // Guard against Anthropic vision API >2000px brick (#1214). Only
1121 // applies to fullPage captures; scrollTo viewport-bound shots are
1122 // already capped by the viewport size.
1123 if (!scrollTo) await guardScreenshotPath(outputPath);
1125 // Restore viewport
1126 if (viewportWidth && originalViewport) {
1127 await page.setViewportSize(originalViewport);
1128 }
1130 const parts = ['Screenshot saved'];
1131 if (doCleanup) parts.push('(cleaned)');
1132 if (scrollTo) parts.push(`(scrolled to: ${scrollTo})`);
1133 parts.push(`: ${outputPath}`);
1134 return parts.join(' ');
1135 }
1137 case 'download': {
1138 if (args.length === 0) throw new Error('Usage: download <url|@ref> [path] [--base64] [--navigate]');
1139 const isBase64 = args.includes('--base64');
1140 const useNavigate = args.includes('--navigate');
1141 const filteredArgs = args.filter(a => a !== '--base64' && a !== '--navigate');
1142 let url = filteredArgs[0];
1143 const outputPath = filteredArgs[1];
1145 // Resolve @ref to element src
1146 if (url.startsWith('@')) {
1147 const resolved = await bm.resolveRef(url);
1148 if (!('locator' in resolved)) throw new Error(`Expected @ref, got CSS selector: ${url}`);
1149 const locator = resolved.locator;
1150 const tagName = await locator.evaluate(el => el.tagName.toLowerCase());
1151 if (tagName === 'img') {
1152 url = await locator.evaluate(el => {
1153 const img = el as HTMLImageElement;
1154 return img.currentSrc || img.src || img.getAttribute('data-src') || '';
1155 });
1156 } else if (tagName === 'video') {
1157 url = await locator.evaluate(el => (el as HTMLVideoElement).currentSrc || (el as HTMLVideoElement).src || '');
1158 } else if (tagName === 'audio') {
1159 url = await locator.evaluate(el => (el as HTMLAudioElement).currentSrc || (el as HTMLAudioElement).src || '');
1160 } else {
1161 // Try src attribute on any element
1162 url = await locator.evaluate(el => el.getAttribute('src') || '');
1163 }
1164 if (!url) throw new Error(`Could not extract URL from ${filteredArgs[0]} (${tagName})`);
1165 }
1167 // Check for HLS/DASH
1168 if (url.includes('.m3u8') || url.includes('.mpd')) {
1169 throw new Error('This is an HLS/DASH stream. Use yt-dlp or ffmpeg for adaptive stream downloads.');
1170 }
1172 // Determine output path and extension
1173 const page = bm.getPage();
1174 let contentType = 'application/octet-stream';
1175 let buffer: Buffer;
1177 if (url.startsWith('blob:')) {
1178 // Strategy 3: Blob URL -- in-page fetch + base64
1179 const dataUrl = await page.evaluate(async (blobUrl) => {
1180 try {
1181 const resp = await fetch(blobUrl);
1182 const blob = await resp.blob();
1183 if (blob.size > 100 * 1024 * 1024) return 'ERROR:TOO_LARGE';
1184 return new Promise<string>((resolve, reject) => {
1185 const reader = new FileReader();
1186 reader.onloadend = () => resolve(reader.result as string);
1187 reader.onerror = () => reject('Failed to read blob');
1188 reader.readAsDataURL(blob);
1189 });
1190 } catch (err: any) {
1191 return `ERROR:EXPIRED:${err?.message || 'unknown'}`;
1192 }
1193 }, url);
1195 if (dataUrl === 'ERROR:TOO_LARGE') throw new Error('Blob too large (>100MB). Use a different approach.');
1196 if (dataUrl.startsWith('ERROR:EXPIRED')) throw new Error(`Blob URL expired or inaccessible: ${dataUrl.slice('ERROR:EXPIRED:'.length)}`);
1198 const match = dataUrl.match(/^data:([^;]+);base64,(.+)$/);
1199 if (!match) throw new Error('Failed to decode blob data');
1200 contentType = match[1];
1201 buffer = Buffer.from(match[2], 'base64');
1202 } else if (useNavigate) {
1203 // Strategy 2: Navigate to URL and capture browser-triggered download.
1204 // Handles URLs that trigger file downloads via redirects,
1205 // Content-Disposition headers, or anti-bot CDN chains where
1206 // page.request.fetch() can't follow the auth/redirect chain.
1207 await validateNavigationUrl(url);
1208 const downloadPromise = page.waitForEvent('download', { timeout: 60000 });
1209 // Use goto with 'commit' wait — the page may redirect to trigger
1210 // the download, so 'domcontentloaded' may never fire.
1211 page.goto(url, { waitUntil: 'commit', timeout: 30000 }).catch(() => {
1212 // Navigation may "fail" because the response is a download,
1213 // not a page. The download event handles it.
1214 });
1215 const download = await downloadPromise;
1216 const failure = await download.failure();
1217 if (failure) {
1218 throw new Error(`Download failed: ${failure}`);
1219 }
1220 // Save to temp location first, then read into buffer
1221 const tempPath = path.join(TEMP_DIR, `browse-nav-download-${Date.now()}`);
1222 await download.saveAs(tempPath);
1223 buffer = fs.readFileSync(tempPath);
1224 // Try to infer content type from suggested filename
1225 const suggested = download.suggestedFilename();
1226 if (suggested) {
1227 const extMatch = suggested.match(/\.([a-z0-9]+)$/i);
1228 if (extMatch) {
1229 const extLower = extMatch[1].toLowerCase();
1230 const mimeMap: Record<string, string> = {
1231 epub: 'application/epub+zip', pdf: 'application/pdf',
1232 zip: 'application/zip', gz: 'application/gzip',
1233 mp3: 'audio/mpeg', mp4: 'video/mp4',
1234 jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png',
1235 txt: 'text/plain', html: 'text/html', json: 'application/json',
1236 };
1237 contentType = mimeMap[extLower] || 'application/octet-stream';
1238 }
1239 }
1240 // Clean up temp file if we're going to write elsewhere
1241 if (outputPath || isBase64) {
1242 try { fs.unlinkSync(tempPath); } catch { /* ignore */ }
1243 } else {
1244 // No explicit output path — rename temp file with inferred extension.
1245 const ext = contentType.split(';')[0].includes('/')
1246 ? mimeToExt(contentType.split(';')[0].trim())
1247 : '.bin';
1248 const finalPath = path.join(TEMP_DIR, `browse-download-${Date.now()}${ext}`);
1249 fs.renameSync(tempPath, finalPath);
1250 const sizeKB = Math.round(buffer.length / 1024);
1251 return `Downloaded: ${finalPath} (${sizeKB}KB, ${contentType.split(';')[0].trim()})${suggested ? ` [${suggested}]` : ''}`;
1252 }
1253 if (buffer.length > 200 * 1024 * 1024) {
1254 throw new Error('File too large (>200MB).');
1255 }
1256 } else {
1257 // Strategy 1: Direct URL via page.request.fetch().
1258 // Gate the URL through the same validator `goto` uses. Without
1259 // this check, download + scrape bypass the navigation
1260 // blocklist and a caller with write scope can read
1261 // http://169.254.169.254/latest/meta-data/ (AWS IMDSv1), the
1262 // GCP/Azure metadata equivalents, or any internal IPv4/IPv6
1263 // the server happens to route to. The response body is then
1264 // returned to the caller (base64) or written to disk where
1265 // GET /file serves it back.
1266 await validateNavigationUrl(url);
1267 const response = await page.request.fetch(url, { timeout: 30000 });
1268 const status = response.status();
1269 if (status >= 400) {
1270 throw new Error(`Download failed: HTTP ${status} ${response.statusText()}`);
1271 }
1272 contentType = response.headers()['content-type'] || 'application/octet-stream';
1273 buffer = Buffer.from(await response.body());
1274 if (buffer.length > 200 * 1024 * 1024) {
1275 throw new Error('File too large (>200MB).');
1276 }
1277 }
1279 // --base64 mode: return inline
1280 if (isBase64) {
1281 if (buffer.length > 10 * 1024 * 1024) {
1282 throw new Error('File too large for --base64 (>10MB). Use disk download + GET /file instead.');
1283 }
1284 const mimeType = contentType.split(';')[0].trim();
1285 return `data:${mimeType};base64,${buffer.toString('base64')}`;
1286 }
1288 // Write to disk
1289 const ext = contentType.split(';')[0].includes('/')
1290 ? mimeToExt(contentType.split(';')[0].trim())
1291 : '.bin';
1292 const destPath = outputPath || path.join(TEMP_DIR, `browse-download-${Date.now()}${ext}`);
1293 validateOutputPath(destPath);
1294 fs.writeFileSync(destPath, buffer);
1295 const sizeKB = Math.round(buffer.length / 1024);
1296 return `Downloaded: ${destPath} (${sizeKB}KB, ${contentType.split(';')[0].trim()})`;
1297 }
1299 case 'scrape': {
1300 if (args.length === 0) throw new Error('Usage: scrape <images|videos|media> [--selector sel] [--dir path] [--limit N]');
1301 const mediaType = args[0];
1302 if (!['images', 'videos', 'media'].includes(mediaType)) {
1303 throw new Error(`Invalid type: ${mediaType}. Use: images, videos, or media`);
1304 }
1306 // Parse flags
1307 const selectorIdx = args.indexOf('--selector');
1308 const selector = selectorIdx >= 0 ? args[selectorIdx + 1] : undefined;
1309 const dirIdx = args.indexOf('--dir');
1310 const requestedDir = dirIdx >= 0 ? args[dirIdx + 1] : path.join(TEMP_DIR, `browse-scrape-${Date.now()}`);
1311 const limitIdx = args.indexOf('--limit');
1312 const limit = Math.min(limitIdx >= 0 ? parseInt(args[limitIdx + 1], 10) || 50 : 50, 200);
1314 validateOutputPath(requestedDir);
1315 const dir = path.resolve(requestedDir);
1316 fs.mkdirSync(dir, { recursive: true });
1317 // Each file is validated too: a symlink planted in an existing --dir
1318 // would otherwise redirect the write outside the safe directories.
1319 const manifestPath = path.join(dir, 'manifest.json');
1320 validateOutputPath(manifestPath);
1322 const { extractMedia } = await import('./media-extract');
1323 const target = bm.getActiveFrameOrPage();
1324 const filter = mediaType === 'images' ? 'images' as const
1325 : mediaType === 'videos' ? 'videos' as const
1326 : undefined;
1327 const mediaResult = await extractMedia(target, { selector, filter });
1329 // Collect URLs to download
1330 const urls: Array<{ url: string; type: string }> = [];
1331 const seen = new Set<string>();
1333 for (const img of mediaResult.images) {
1334 const url = img.currentSrc || img.src || img.dataSrc;
1335 if (url && !seen.has(url) && !url.startsWith('data:')) {
1336 seen.add(url);
1337 urls.push({ url, type: 'image' });
1338 }
1339 }
1340 for (const vid of mediaResult.videos) {
1341 const url = vid.currentSrc || vid.src;
1342 if (url && !seen.has(url) && !url.startsWith('blob:') && !vid.isHLS && !vid.isDASH) {
1343 seen.add(url);
1344 urls.push({ url, type: 'video' });
1345 }
1346 }
1347 for (const bg of mediaResult.backgroundImages) {
1348 if (bg.url && !seen.has(bg.url)) {
1349 seen.add(bg.url);
1350 urls.push({ url: bg.url, type: 'image' });
1351 }
1352 }
1354 const toDownload = urls.slice(0, limit);
1355 const page = bm.getPage();
1356 const manifest: any = {
1357 url: page.url(),
1358 scraped_at: new Date().toISOString(),
1359 files: [] as any[],
1360 total_size: 0,
1361 succeeded: 0,
1362 failed: 0,
1363 };
1365 const lines: string[] = [];
1366 for (let i = 0; i < toDownload.length; i++) {
1367 const { url, type } = toDownload[i];
1368 try {
1369 // Same gate as the download command — page.request.fetch
1370 // must not reach cloud metadata, ULA ranges, or the rest of
1371 // the blocklist. See url-validation.ts for the full list.
1372 await validateNavigationUrl(url);
1373 const response = await page.request.fetch(url, { timeout: 30000 });
1374 if (response.status() >= 400) throw new Error(`HTTP ${response.status()}`);
1375 const ct = response.headers()['content-type'] || 'application/octet-stream';
1376 const ext = mimeToExt(ct.split(';')[0].trim());
1377 const filename = `${type}-${String(i + 1).padStart(3, '0')}${ext}`;
1378 const filePath = path.join(dir, filename);
1379 validateOutputPath(filePath);
1380 const body = Buffer.from(await response.body());
1381 try {
1382 fs.writeFileSync(filePath, body);
1383 } catch (writeErr: any) {
1384 throw new Error(`Disk write failed: ${writeErr.message}`);
1385 }
1386 manifest.files.push({ path: filename, src: url, size: body.length, type: ct.split(';')[0].trim() });
1387 manifest.total_size += body.length;
1388 manifest.succeeded++;
1389 lines.push(` [${i + 1}/${toDownload.length}] ${filename} (${Math.round(body.length / 1024)}KB)`);
1390 } catch (err: any) {
1391 manifest.files.push({ path: null, src: url, size: 0, type: '', error: err.message });
1392 manifest.failed++;
1393 lines.push(` [${i + 1}/${toDownload.length}] FAILED: ${err.message}`);
1394 }
1395 // 100ms delay between downloads
1396 if (i < toDownload.length - 1) await new Promise(r => setTimeout(r, 100));
1397 }
1399 // Write manifest
1400 fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
1402 return `Scraped ${toDownload.length} items to ${dir}/\n${lines.join('\n')}\n\nSummary: ${manifest.succeeded} succeeded, ${manifest.failed} failed, ${Math.round(manifest.total_size / 1024)}KB total`;
1403 }
1405 case 'archive': {
1406 const page = bm.getPage();
1407 const outputPath = args[0] || path.join(TEMP_DIR, `browse-archive-${Date.now()}.mhtml`);
1408 validateOutputPath(outputPath);
1410 try {
1411 const data = await withCdpSession(page, async (cdp) => {
1412 const result = await cdp.send('Page.captureSnapshot', { format: 'mhtml' });
1413 return (result as { data: string }).data;
1414 });
1415 fs.writeFileSync(outputPath, data);
1416 return `Archive saved: ${outputPath} (${Math.round(data.length / 1024)}KB, MHTML)`;
1417 } catch (err: any) {
1418 throw new Error(`MHTML archive requires Chromium CDP. Use 'text' or 'html' for raw page content. (${err.message})`);
1419 }
1420 }
1422 default:
1423 throw new Error(`Unknown write command: ${command}`);
1424 }
1425}
1427/** Map MIME type to file extension. */
1428function mimeToExt(mime: string): string {
1429 const map: Record<string, string> = {
1430 'image/png': '.png', 'image/jpeg': '.jpg', 'image/gif': '.gif',
1431 'image/webp': '.webp', 'image/svg+xml': '.svg', 'image/avif': '.avif',
1432 'video/mp4': '.mp4', 'video/webm': '.webm', 'video/quicktime': '.mov',
1433 'audio/mpeg': '.mp3', 'audio/wav': '.wav', 'audio/ogg': '.ogg',
1434 'application/pdf': '.pdf', 'application/json': '.json',
1435 'text/html': '.html', 'text/plain': '.txt',
1436 };
1437 return map[mime] || '.bin';
1438}