Skills › Marketing Skills › reverse-engineer-marketing › teardown.mjs

teardown.mjs

1227 lines · skills/reverse-engineer-marketing/scripts/teardown.mjs at 88c79f6

Open on GitHub

1#!/usr/bin/env node

2// Free-source collection and analysis for a growth teardown. Node 18+, no dependencies.

3// Run with no arguments for usage.

5import { gunzipSync } from 'node:zlib';

6import { readFileSync, writeFileSync } from 'node:fs';

7import { pathToFileURL } from 'node:url';

9const UA = 'Mozilla/5.0 (compatible; marketingskills-teardown/1.0)';

10const WAYBACK_DELAY_MS = 1500;

12export const SIGNATURES = [

13 { tool: 'Meta Pixel', group: 'ads', patterns: [/connect\.facebook\.net\/[^"']*fbevents\.js/, /\bfbq\(\s*['"]init/] },

14 { tool: 'Google Ads', group: 'ads', patterns: [/googleadservices\.com\/pagead\/conversion/, /['"]AW-\d{6,}/] },

15 { tool: 'Microsoft Ads (UET)', group: 'ads', patterns: [/bat\.bing\.com\/bat\.js/] },

16 { tool: 'X Ads', group: 'ads', patterns: [/static\.ads-twitter\.com\/uwt\.js/, /\btwq\(\s*['"]/] },

17 { tool: 'Reddit Ads', group: 'ads', patterns: [/redditstatic\.com\/ads\/pixel\.js/, /\brdt\(\s*['"]init/] },

18 { tool: 'TikTok Pixel', group: 'ads', patterns: [/analytics\.tiktok\.com\/i18n\/pixel/, /\bttq\.load\(/] },

19 { tool: 'LinkedIn Insight', group: 'ads', patterns: [/snap\.licdn\.com\/li\.lms-analytics/, /_linkedin_partner_id/] },

20 { tool: 'Quora Pixel', group: 'ads', patterns: [/a\.quora\.com\/qevents\.js/] },

21 { tool: 'Google Analytics', group: 'analytics', patterns: [/googletagmanager\.com\/gtag\/js\?id=G-/, /google-analytics\.com\/analytics\.js/] },

22 { tool: 'Google Tag Manager', group: 'analytics', patterns: [/googletagmanager\.com\/gtm\.js/, /['"]GTM-[A-Z0-9]{4,}/] },

23 { tool: 'Segment', group: 'analytics', patterns: [/cdn\.segment\.com\/analytics\.js/] },

24 { tool: 'PostHog', group: 'analytics', patterns: [/i\.posthog\.com/, /app\.posthog\.com/, /posthog\.init\(/] },

25 { tool: 'Mixpanel', group: 'analytics', patterns: [/cdn\.mxpnl\.com/, /mixpanel\.init\(/] },

26 { tool: 'Amplitude', group: 'analytics', patterns: [/cdn\.amplitude\.com/] },

27 { tool: 'Hotjar', group: 'analytics', patterns: [/static\.hotjar\.com/] },

28 { tool: 'Microsoft Clarity', group: 'analytics', patterns: [/clarity\.ms\/tag/] },

29 { tool: 'Optimizely', group: 'testing', patterns: [/cdn\.optimizely\.com/] },

30 { tool: 'VWO', group: 'testing', patterns: [/visualwebsiteoptimizer\.com/] },

31 { tool: 'Rewardful', group: 'referral', patterns: [/r\.wdfl\.co/] },

32 { tool: 'PartnerStack', group: 'referral', patterns: [/partnerstack\.com\/[^"']*\.js/] },

33 { tool: 'Tolt', group: 'referral', patterns: [/cdn\.tolt\.io/] },

34 { tool: 'FirstPromoter', group: 'referral', patterns: [/cdn\.firstpromoter\.com/] },

35 { tool: 'Dub', group: 'referral', patterns: [/dubcdn\.com\/analytics/] },

36 { tool: 'Intercom', group: 'lifecycle', patterns: [/widget\.intercom\.io/, /js\.intercomcdn\.com/] },

37 { tool: 'HubSpot', group: 'lifecycle', patterns: [/js\.hs-scripts\.com/, /js\.hsforms\.net/] },

38 { tool: 'Customer.io', group: 'lifecycle', patterns: [/assets\.customer\.io/] },

39 { tool: 'GrowthBook', group: 'testing', patterns: [/cdn\.growthbook\.io/, /growthbook/i] },

40 { tool: 'Statsig', group: 'testing', patterns: [/statsig/i] },

41 { tool: 'LaunchDarkly', group: 'testing', patterns: [/launchdarkly/i] },

42 { tool: 'AB Tasty', group: 'testing', patterns: [/abtasty/i] },

43 { tool: 'Convert', group: 'testing', patterns: [/convertexperiments\.com/] },

44 { tool: 'Kameleoon', group: 'testing', patterns: [/kameleoon/i] },

45 { tool: 'Shopify', group: 'ecommerce', patterns: [/cdn\.shopify\.com/, /Shopify\.theme/] },

46 { tool: 'WooCommerce', group: 'ecommerce', patterns: [/woocommerce/i] },

47 { tool: 'BigCommerce', group: 'ecommerce', patterns: [/bigcommerce\.com/] },

48 { tool: 'Klaviyo', group: 'ecommerce', patterns: [/klaviyo\.com/] },

49 { tool: 'Attentive', group: 'ecommerce', patterns: [/cdn\.attn\.tv/, /attentivemobile/i] },

50 { tool: 'Postscript', group: 'ecommerce', patterns: [/postscript\.io/] },

51 { tool: 'Yotpo', group: 'ecommerce', patterns: [/yotpo/i] },

52 { tool: 'Okendo', group: 'ecommerce', patterns: [/okendo/i] },

53 { tool: 'Judge.me', group: 'ecommerce', patterns: [/judge\.me/] },

54 { tool: 'Loox', group: 'ecommerce', patterns: [/loox\.io/] },

55 { tool: 'Rebuy', group: 'ecommerce', patterns: [/rebuyengine/i] },

56 { tool: 'Recharge', group: 'ecommerce', patterns: [/rechargeapps|rechargepayments|rechargecdn/i] },

57 { tool: 'Gorgias', group: 'ecommerce', patterns: [/gorgias/i] },

58 { tool: 'Smile.io', group: 'ecommerce', patterns: [/smile\.io/] },

59 { tool: 'Triple Whale', group: 'ecommerce', patterns: [/triplewhale|triple-whale/i] },

60 { tool: 'Northbeam', group: 'ecommerce', patterns: [/northbeam/i] },

61 { tool: 'Shop Pay', group: 'payments', patterns: [/shop\.app/] },

62 { tool: 'Klarna', group: 'payments', patterns: [/klarna/i] },

63 { tool: 'Afterpay', group: 'payments', patterns: [/afterpay/i] },

64 { tool: 'Affirm', group: 'payments', patterns: [/affirm\.com\/js|cdn1\.affirm\.com/] },

65 { tool: 'PriceSpider', group: 'retail', patterns: [/pricespider/i] },

66 { tool: 'Destini', group: 'retail', patterns: [/destini\.co/] },

67 { tool: 'CallRail', group: 'call-tracking', patterns: [/cdn\.callrail\.com/] },

68 { tool: 'CallTrackingMetrics', group: 'call-tracking', patterns: [/tctm\.co/] },

69 { tool: 'Invoca', group: 'call-tracking', patterns: [/invocacdn/i] },

70 { tool: 'WhatConverts', group: 'call-tracking', patterns: [/whatconverts/i] },

71 { tool: 'ServiceTitan', group: 'local', patterns: [/servicetitan/i] },

72 { tool: 'Housecall Pro', group: 'local', patterns: [/housecallpro/i] },

73 { tool: 'Jobber', group: 'local', patterns: [/getjobber\.com/] },

74 { tool: 'Podium', group: 'local', patterns: [/podium\.com/] },

75 { tool: 'Birdeye', group: 'local', patterns: [/birdeye/i] },

76 { tool: 'Zocdoc', group: 'healthcare', patterns: [/zocdoc/i] },

77 { tool: 'NexHealth', group: 'healthcare', patterns: [/nexhealth/i] },

78 { tool: 'Lawmatics', group: 'legal', patterns: [/lawmatics/i] },

79 { tool: 'Clio', group: 'legal', patterns: [/clio\.com|goclio/i] },

80 { tool: 'IDX Broker', group: 'real-estate', patterns: [/idxbroker/i] },

81 { tool: 'kvCORE', group: 'real-estate', patterns: [/kvcore/i] },

82 { tool: 'Follow Up Boss', group: 'real-estate', patterns: [/followupboss/i] },

83 { tool: 'Dealer.com', group: 'automotive', patterns: [/dealer\.com/] },

84 { tool: 'DealerInspire', group: 'automotive', patterns: [/dealerinspire/i] },

85 { tool: 'OpenTable', group: 'hospitality', patterns: [/opentable/i] },

86 { tool: 'Resy', group: 'hospitality', patterns: [/resy\.com/] },

87 { tool: 'SevenRooms', group: 'hospitality', patterns: [/sevenrooms/i] },

88 { tool: 'Toast', group: 'hospitality', patterns: [/toasttab/i] },

89 { tool: 'SynXis', group: 'hospitality', patterns: [/synxis/i] },

90 { tool: 'Classy', group: 'nonprofit', patterns: [/classy\.org/] },

91 { tool: 'Givebutter', group: 'nonprofit', patterns: [/givebutter/i] },

92 { tool: 'Donorbox', group: 'nonprofit', patterns: [/donorbox/i] },

93 { tool: 'Fundraise Up', group: 'nonprofit', patterns: [/fundraiseup/i] },

94 { tool: 'Piano', group: 'media', patterns: [/piano\.io|tinypass/i] },

95 { tool: 'Memberful', group: 'media', patterns: [/memberful/i] },

96 { tool: 'WalletConnect', group: 'crypto', patterns: [/walletconnect/i] },

97 { tool: 'Mindbody', group: 'fitness', patterns: [/mindbodyonline|healcode/i] },

98 { tool: 'ClassPass', group: 'fitness', patterns: [/classpass/i] },

99 { tool: 'Glofox', group: 'fitness', patterns: [/glofox/i] },

100 { tool: 'Mariana Tek', group: 'fitness', patterns: [/marianatek/i] },

101 { tool: 'Zen Planner', group: 'fitness', patterns: [/zenplanner/i] },

102 { tool: 'BackerKit', group: 'hardware', patterns: [/backerkit\.com/i] },

103 { tool: 'Eventbrite', group: 'events', patterns: [/eventbrite/i] },

104 { tool: 'Ticketmaster', group: 'events', patterns: [/ticketmaster/i] },

105 { tool: 'AXS', group: 'events', patterns: [/\baxs\.com/i] },

106 { tool: 'DICE', group: 'events', patterns: [/dice\.fm/i] },

107 { tool: 'Tixr', group: 'events', patterns: [/tixr\.com/i] },

108 { tool: 'Steam', group: 'games', patterns: [/store\.steampowered\.com/i] },

109 { tool: 'Kickstarter', group: 'hardware', patterns: [/kickstarter\.com/i] },

110 { tool: 'Indiegogo', group: 'hardware', patterns: [/indiegogo\.com/i] },

111];

112

113// Only executable scripts and resource URLs are scanned, so a tool named in page copy, JSON-LD, or a plain link

114// isn't mistaken for an installed one.

115export function scriptSurface(html) {

116 const scripts = [...html.matchAll(/<script\b([^>]*)>[\s\S]*?<\/script>/gi)]

117 .filter(m => !/\btype=["']?(?:application\/(?:ld\+)?json|text\/(?:template|html|x-template))/i.test(m[1]))

118 .map(m => m[0]);

119 const urls = [...html.matchAll(/<(?!a\b)[a-z][\w-]*\b[^>]*?\b(?:src|href|data-src|action)=["']([^"']+)["']/gi)].map(m => m[1]);

120 return [...scripts, ...urls].join('\n');

121}

122

123export function linkSurface(html) {

124 return [...html.matchAll(/<a\b[^>]*?\bhref=["']([^"']+)["']/gi)].map(m => m[1]).join('\n');

125}

126

127export function detectTools(html) {

128 const surface = scriptSurface(html);

129 return SIGNATURES.filter(s => s.patterns.some(p => p.test(surface))).map(s => s.tool);

130}

131

132// Tools reached only through ordinary links, such as a reservation or ticket button. A link is not an installation.

133export function detectLinkedTools(html) {

134 const installed = new Set(detectTools(html));

135 const surface = linkSurface(html);

136 return SIGNATURES.filter(s => !installed.has(s.tool) && s.patterns.some(p => p.test(surface))).map(s => s.tool);

137}

138

139// SPF includes and MX hosts name the services allowed to send or receive mail for the domain.

140export const MAIL_SIGNATURES = [

141 { tool: 'Google Workspace', pattern: /_spf\.google\.com|aspmx\.l\.google\.com|googlemail\.com/i },

142 { tool: 'Microsoft 365', pattern: /spf\.protection\.outlook\.com|mail\.protection\.outlook\.com/i },

143 { tool: 'SendGrid', pattern: /sendgrid\.net/i },

144 { tool: 'Mailgun', pattern: /mailgun\.org/i },

145 { tool: 'Amazon SES', pattern: /amazonses\.com/i },

146 { tool: 'Postmark', pattern: /spf\.mtasv\.net/i },

147 { tool: 'Mailchimp', pattern: /servers\.mcsv\.net|mandrillapp\.com/i },

148 { tool: 'HubSpot', pattern: /hubspotemail\.net/i },

149 { tool: 'Salesforce', pattern: /_spf\.salesforce\.com/i },

150 { tool: 'Marketo', pattern: /mktomail\.com/i },

151 { tool: 'Pardot', pattern: /pardot\.com/i },

152 { tool: 'Customer.io', pattern: /customer\.io/i },

153 { tool: 'Intercom', pattern: /intercom\.io/i },

154 { tool: 'Zendesk', pattern: /zendesk\.com/i },

155 { tool: 'Help Scout', pattern: /helpscoutemail\.com/i },

156 { tool: 'Freshdesk', pattern: /freshdesk\.com|freshemail\.io/i },

157 { tool: 'Brevo', pattern: /sendinblue\.com|brevo\.com/i },

158 { tool: 'Klaviyo', pattern: /klaviyo/i },

159];

160

161// Domain-verification TXT records left by tools the company has connected.

162export const VERIFICATION_SIGNATURES = [

163 { tool: 'Google (Search Console or Workspace)', pattern: /^google-site-verification=/i },

164 { tool: 'Meta Business (ads, commerce)', pattern: /^facebook-domain-verification=/i },

165 { tool: 'Microsoft 365', pattern: /^MS=/ },

166 { tool: 'Atlassian', pattern: /^atlassian-domain-verification=/i },

167 { tool: 'Stripe', pattern: /^stripe-verification=/i },

168 { tool: 'Apple', pattern: /^apple-domain-verification=/i },

169 { tool: 'DocuSign', pattern: /^docusign=/i },

170 { tool: 'Zoom', pattern: /^ZOOM_verify_/i },

171 { tool: 'Adobe', pattern: /^adobe-(idp-site|sign)-verification=/i },

172 { tool: 'OpenAI', pattern: /^openai-domain-verification=/i },

173 { tool: 'HubSpot', pattern: /^hubspot-developer-verification=/i },

174 { tool: 'Miro', pattern: /^miro-verification=/i },

175 { tool: 'Anthropic', pattern: /^anthropic-domain-verification/i },

176 { tool: 'TikTok (developer or ads)', pattern: /^tiktok-(developers-site-)?verification/i },

177 { tool: 'Slack', pattern: /^slack-domain-verification/i },

178 { tool: 'Notion', pattern: /^notion-domain-verification/i },

179 { tool: 'Cursor', pattern: /^cursor-domain-verification/i },

180 { tool: 'Postman', pattern: /^postman-domain-verification/i },

181 { tool: 'HackerOne', pattern: /^h1-domain-verification/i },

182 { tool: 'Pinterest', pattern: /^pinterest-site-verification/i },

183 { tool: 'LinkedIn', pattern: /^linkedin-domain-verification/i },

184];

185

186// CNAME targets on common marketing subdomains reveal the platform behind them.

187export const CNAME_SIGNATURES = [

188 { tool: 'HubSpot', pattern: /hubspot|hs-sites/i },

189 { tool: 'Marketo', pattern: /mktoweb|mktossl|marketo/i },

190 { tool: 'Pardot', pattern: /pardot/i },

191 { tool: 'Zendesk', pattern: /zendesk/i },

192 { tool: 'Intercom', pattern: /intercom/i },

193 { tool: 'Help Scout', pattern: /helpscout/i },

194 { tool: 'incident.io', pattern: /incident\.io/i },

195 { tool: 'Statuspage', pattern: /statuspage\.io/i },

196 { tool: 'GitBook', pattern: /gitbook/i },

197 { tool: 'Mintlify', pattern: /mintlify/i },

198 { tool: 'ReadMe', pattern: /readme\.io|readmessl/i },

199 { tool: 'Discourse', pattern: /discourse/i },

200 { tool: 'Circle', pattern: /circle\.so/i },

201 { tool: 'Webflow', pattern: /webflow/i },

202 { tool: 'Framer', pattern: /framer/i },

203 { tool: 'Ghost', pattern: /ghost\.io/i },

204 { tool: 'Substack', pattern: /substack/i },

205 { tool: 'beehiiv', pattern: /beehiiv/i },

206 { tool: 'Unbounce', pattern: /unbounce/i },

207 { tool: 'Instapage', pattern: /pageserve|instapage/i },

208 { tool: 'Leadpages', pattern: /leadpages/i },

209 { tool: 'Dub', pattern: /dub\.co/i },

210 { tool: 'Vercel', pattern: /vercel/i },

211 { tool: 'Netlify', pattern: /netlify/i },

212 { tool: 'Shopify', pattern: /myshopify|shops\.myshopify/i },

213 { tool: 'Teachable', pattern: /teachable/i },

214 { tool: 'Thinkific', pattern: /thinkific/i },

215 { tool: 'Kajabi', pattern: /kajabi/i },

216 { tool: 'Podia', pattern: /podia/i },

217 { tool: 'LearnWorlds', pattern: /learnworlds/i },

218 { tool: 'Eventbrite', pattern: /eventbrite/i },

219];

220

221export const MARKETING_SUBDOMAINS = [

222 'www', 'blog', 'go', 'info', 'learn', 'get', 'try', 'lp', 'pages', 'email', 'news', 'newsletter',

223 'help', 'support', 'docs', 'developers', 'community', 'forum', 'status', 'academy', 'partners',

224 'affiliates', 'events', 'app', 'link', 'links', 'shop', 'store', 'careers', 'jobs',

225 'courses', 'apply', 'admissions', 'franchise', 'ownafranchise', 'donate', 'give', 'tickets', 'book', 'order',

226];

227

228export function detectDns({ txt = [], mx = [], cnames = {} }) {

229 const spf = txt.filter(r => /^v=spf1/i.test(r)).join(' ');

230 const mail = MAIL_SIGNATURES.filter(s => s.pattern.test(spf) || mx.some(m => s.pattern.test(m))).map(s => s.tool);

231 const verifications = [];

232 const otherVerifications = [];

233 for (const r of txt) {

234 const known = VERIFICATION_SIGNATURES.find(s => s.pattern.test(r));

235 if (known) verifications.push(known.tool);

236 else if (/verification|verify/i.test(r)) otherVerifications.push(r.split('=')[0]);

237 }

238 const subdomains = Object.entries(cnames).map(([sub, target]) => ({

239 subdomain: sub,

240 cname: target,

241 tool: CNAME_SIGNATURES.find(s => s.pattern.test(target))?.tool || null,

242 }));

243 return {

244 mail: [...new Set(mail)],

245 verifications: [...new Set(verifications)],

246 other_verifications: [...new Set(otherVerifications)],

247 subdomains,

248 };

249}

250

251// crt.sh rows: name_value holds one or more names separated by newlines.

252export function parseCertificates(rows, domain) {

253 const firstSeen = {};

254 for (const row of rows) {

255 const date = String(row.not_before || row.entry_timestamp || '').slice(0, 10);

256 for (const raw of String(row.name_value || '').split('\n')) {

257 // A wildcard certificate doesn't show that any one host under it exists.

258 if (raw.trim().startsWith('*')) continue;

259 const name = raw.trim().toLowerCase();

260 if (!name.endsWith(`.${domain}`) || name === domain) continue;

261 if (!firstSeen[name] || date < firstSeen[name]) firstSeen[name] = date;

262 }

263 }

264 return Object.entries(firstSeen)

265 .map(([subdomain, first_seen]) => ({ subdomain, first_seen }))

266 .sort((a, b) => a.first_seen.localeCompare(b.first_seen));

267}

268

269// Whole labels or label prefixes followed by a digit or hyphen, so dev-2 is noise but developers is not.

271

272// Drops preview, staging, and other machine-generated names, and groups the rest by the label under the domain.

273export function filterSubdomains(subs, domain) {

274 const kept = [];

275 const groups = {};

276 let dropped = 0;

277 for (const s of subs) {

278 const labels = s.subdomain.slice(0, -(domain.length + 1)).split('.');

279 if (labels.some(l => NOISE_LABEL.test(l)) || labels.length >= 3) { dropped++; continue; }

280 const top = labels[labels.length - 1];

281 groups[top] ??= { label: top, names: 0, first_seen: s.first_seen };

282 groups[top].names++;

283 if (labels.length === 1) kept.push(s);

284 }

285 const multi = Object.values(groups).filter(g => g.names > 1).sort((a, b) => b.names - a.names);

286 return { subdomains: kept, groups: multi, dropped };

287}

288

289const decodeEntities = s => s.replace(/&/g, '&').replace(/"/g, '"').replace(/'|'/g, "'").replace(/</g, '<').replace(/>/g, '>').replace(/ /g, ' ');

290const stripTags = s => decodeEntities(s.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();

291

292export function extractHero(html) {

293 const title = stripTags(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '');

294 const description = decodeEntities(

295 html.match(/<meta[^>]+name=["']description["'][^>]*content=(["'])([\s\S]*?)\1/i)?.[2] ||

296 html.match(/<meta[^>]+content=(["'])([\s\S]*?)\1[^>]*name=["']description["']/i)?.[2] || ''

297 ).trim();

298 const h1 = stripTags(html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || '');

299 return { title, description, h1 };

300}

301

302export function countLocs(xml) {

303 return [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map(m => m[1].replace(/&/g, '&'));

304}

305

306const LOCALE = /^[a-z]{2}(-[a-z]{2,4})?$/i;

307

308export function localeOf(url) {

309 try {

310 const seg = new URL(url).pathname.split('/').filter(Boolean)[0];

311 return seg && LOCALE.test(seg) ? seg.toLowerCase() : null;

312 } catch {

313 return null;

314 }

315}

316

317// Locale prefixes (/de/, /pt-br/) are skipped so translated copies group with their family.

318export function pathFamily(url) {

319 try {

320 const segs = new URL(url).pathname.split('/').filter(Boolean);

321 if (segs.length && LOCALE.test(segs[0])) segs.shift();

322 return segs.length ? `/${segs[0].toLowerCase()}` : '/';

323 } catch {

324 return '(invalid)';

325 }

326}

327

328export function parseCsv(text) {

329 const rows = [];

330 let row = [];

331 let field = '';

332 let quoted = false;

333 for (let i = 0; i < text.length; i++) {

334 const c = text[i];

335 if (quoted) {

336 if (c === '"' && text[i + 1] === '"') { field += '"'; i++; }

337 else if (c === '"') quoted = false;

338 else field += c;

339 } else if (c === '"') quoted = true;

340 else if (c === ',') { row.push(field); field = ''; }

341 else if (c === '\n' || c === '\r') {

342 if (c === '\r' && text[i + 1] === '\n') i++;

343 row.push(field); field = '';

344 if (row.some(v => v !== '')) rows.push(row);

345 row = [];

346 } else field += c;

347 }

348 if (quoted) throw new Error(`CSV has an unclosed quote in row ${rows.length + 1}`);

349 row.push(field);

350 if (row.some(v => v !== '')) rows.push(row);

351 if (!rows.length) return [];

352 const [header, ...body] = rows;

353 return body.map(r => Object.fromEntries(header.map((h, i) => [h.trim(), (r[i] ?? '').trim()])));

354}

355

356export function toCsv(rows) {

357 if (!rows.length) return '';

358 const cols = Object.keys(rows[0]);

359 const esc = v => {

360 let s = v == null ? '' : String(v);

361 // Scraped text that starts like a formula is prefixed so spreadsheets show it as text.

362 if (/^[=+\-@\t\r]/.test(s) && !/^[-+]?\d[\d.,]*$/.test(s)) s = `'${s}`;

363 return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;

364 };

365 return [cols.join(','), ...rows.map(r => cols.map(c => esc(r[c])).join(','))].join('\n') + '\n';

366}

367

368const monthKey = d => d.slice(0, 7);

369const mean = xs => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);

370const round1 = x => (x == null ? null : Math.round(x * 10) / 10 + 0);

371

372// Blank cells are missing data, not zero.

373const cell = v => (v == null || String(v).trim() === '' ? NaN : Number(v));

374

375// One row per calendar month from first to last, so a missing month is a gap rather than two months glued together.

376export function fillMonths(traffic) {

377 const byMonth = new Map();

378 for (const t of traffic) {

379 const m = monthKey(t.month);

380 if (byMonth.has(m)) throw new Error(`traffic.csv has month ${m} more than once`);

381 byMonth.set(m, t);

382 }

383 const keys = [...byMonth.keys()].sort();

384 if (!keys.length) return [];

385 const out = [];

386 let [y, mo] = keys[0].split('-').map(Number);

387 const last = keys[keys.length - 1];

388 for (;;) {

389 const key = `${y}-${String(mo).padStart(2, '0')}`;

390 out.push(byMonth.get(key) || { month: key, visits: '', brand_searches: '' });

391 if (key >= last) break;

392 if (++mo > 12) { mo = 1; y++; }

393 }

394 return out;

395}

396

397function growthSeries(traffic, col) {

398 return traffic.map((t, i) => {

399 const prev = i > 0 ? cell(traffic[i - 1][col]) : NaN;

400 const cur = cell(t[col]);

401 return i > 0 && prev > 0 && Number.isFinite(cur) ? ((cur - prev) / prev) * 100 : null;

402 });

403}

404

405// Needs all three months on both sides; a partial window returns null instead of a misleading number.

406function windowLift(growth, i) {

407 if (i < 3 || i + 3 > growth.length) return null;

408 const before = growth.slice(i - 3, i);

409 const after = growth.slice(i, i + 3);

410 if (before.includes(null) || after.includes(null)) return null;

411 return round1(mean(after) - mean(before));

412}

413

414function ranks(xs) {

415 const order = xs.map((v, i) => [v, i]).sort((a, b) => a[0] - b[0]);

416 const r = new Array(xs.length);

417 for (let i = 0; i < order.length;) {

418 let j = i;

419 while (j + 1 < order.length && order[j + 1][0] === order[i][0]) j++;

420 for (let k = i; k <= j; k++) r[order[k][1]] = (i + j) / 2 + 1;

421 i = j + 1;

422 }

423 return r;

424}

425

426export function spearman(xs, ys) {

427 const pairs = xs.map((x, i) => [x, ys[i]]).filter(([x, y]) => x != null && y != null);

428 if (pairs.length < 3) return null;

429 const rx = ranks(pairs.map(p => p[0]));

430 const ry = ranks(pairs.map(p => p[1]));

431 const mx = mean(rx), my = mean(ry);

432 let num = 0, dx = 0, dy = 0;

433 for (let i = 0; i < rx.length; i++) {

434 num += (rx[i] - mx) * (ry[i] - my);

435 dx += (rx[i] - mx) ** 2;

436 dy += (ry[i] - my) ** 2;

437 }

438 return dx && dy ? Math.round((num / Math.sqrt(dx * dy)) * 100) / 100 : null;

439}

440

441// Moves within CLUSTER_DAYS of the previous move join its cluster.

442export const CLUSTER_DAYS = 42;

443export const MIN_MONTHS = 18;

444export const MIN_MOVES = 6;

445

446export function analyzeMoves(moves, traffic) {

447 traffic = fillMonths(traffic);

448 const months = traffic.map(t => monthKey(t.month));

449 const growth = growthSeries(traffic, 'visits');

450 const brandGrowth = traffic.some(t => Number.isFinite(cell(t.brand_searches))) ? growthSeries(traffic, 'brand_searches') : null;

451

452 const sorted = [...moves].sort((a, b) => a.date.localeCompare(b.date));

453 const clusters = [];

454 let last = null;

455 for (const m of sorted) {

456 const t = Date.parse(m.date);

457 if (last == null || (t - last) / 86400000 >= CLUSTER_DAYS) clusters.push({ start: m.date, moves: [] });

458 clusters[clusters.length - 1].moves.push(m);

459 last = t;

460 }

461

462 const clusterRows = clusters.map((c, n) => {

463 const i = months.indexOf(monthKey(c.start));

464 return {

465 cluster: n + 1,

466 start: c.start,

467 channels: [...new Set(c.moves.map(m => m.channel))].sort().join(' '),

468 moves: c.moves.map(m => m.move).join(' | '),

469 visit_lift_pts: i < 0 ? null : windowLift(growth, i),

470 brand_lift_pts: i < 0 || !brandGrowth ? null : windowLift(brandGrowth, i),

471 };

472 });

473

474 const channels = [...new Set(moves.map(m => m.channel))].sort();

475 const leading = channels.map(channel => {

476 const counts = months.map(mo => moves.filter(m => m.channel === channel && monthKey(m.date) === mo).length);

477 const total = counts.reduce((a, b) => a + b, 0);

478 if (months.length < MIN_MONTHS || total < MIN_MOVES) {

479 return { channel, moves: total, months: months.length, note: 'too few to call' };

480 }

481 const row = { channel, moves: total, months: months.length };

482 for (const k of [0, 1, 2]) row[`rho_lag${k}`] = spearman(counts.slice(0, counts.length - k), growth.slice(k));

483 return row;

484 });

485

486 const top = clusterRows

487 .filter(c => c.visit_lift_pts != null)

488 .sort((a, b) => b.visit_lift_pts - a.visit_lift_pts)

489 .slice(0, 3);

490 return { clusters: clusterRows, top, leading };

491}

492

493// The date is part of the key, so a second price change at the same /pricing URL is a new move.

494export function moveKey(m) {

495 return [m.date, m.channel, m.kind, (m.link || m.move || '').toLowerCase().replace(/\/+$/, '')].join('|');

496}

497

498export function diffMoves(oldMoves, newMoves) {

499 const seen = new Set(oldMoves.map(moveKey));

500 return newMoves.filter(m => !seen.has(moveKey(m)));

501}

502

503export function adLibraryLinks(name, domain) {

504 const q = encodeURIComponent(name);

505 return {

506 meta: `https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=ALL&q=${q}&search_type=keyword_unordered`,

507 google: `https://adstransparency.google.com/?domain=${encodeURIComponent(domain || name)}`,

508 linkedin: `https://www.linkedin.com/ad-library/search?companyName=${q}`,

509 tiktok: `https://library.tiktok.com/ads?adv_name=${q}`,

510 };

511}

512

513const sleep = ms => new Promise(r => setTimeout(r, ms));

514

515export function posInt(value, fallback, name) {

516 if (value == null) return fallback;

517 const n = typeof value === 'string' ? Number(value) : NaN;

518 if (!Number.isInteger(n) || n < 1) throw new Error(`--${name} needs a whole number of 1 or more`);

519 return n;

520}

521

522async function get(url, { binary = false } = {}) {

523 const res = await fetch(url, { headers: { 'user-agent': UA }, redirect: 'follow' });

524 if (!res.ok) throw new Error(`${res.status} ${url}`);

525 return binary ? Buffer.from(await res.arrayBuffer()) : res.text();

526}

527

528async function sitemapUrls(domain, limit) {

529 const base = `https://${domain}`;

530 let roots = [];

531 try {

532 const robots = await get(`${base}/robots.txt`);

533 roots = [...robots.matchAll(/^sitemap:\s*(\S+)/gim)].map(m => m[1]);

534 } catch {}

535 if (!roots.length) roots = [`${base}/sitemap.xml`];

536 const queue = [...roots];

537 const urls = new Set();

538 const visited = new Set();

539 while (queue.length && visited.size < 50 && urls.size < limit) {

540 const sm = queue.shift();

541 if (visited.has(sm)) continue;

542 visited.add(sm);

543 let xml;

544 try {

545 const buf = await get(sm, { binary: true });

546 xml = (sm.endsWith('.gz') ? gunzipSync(buf) : buf).toString('utf8');

547 } catch {

548 continue;

549 }

550 const locs = [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map(m => m[1].replace(/&/g, '&'));

551 if (/<sitemapindex/i.test(xml)) queue.push(...locs);

552 else for (const l of locs) urls.add(l);

553 }

554 return [...urls].slice(0, limit);

555}

556

557async function waybackMonthly(url, months) {

558 const from = new Date();

559 from.setMonth(from.getMonth() - months);

560 const cdx = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(url)}&output=json&from=${from.toISOString().slice(0, 7).replace('-', '')}&filter=statuscode:200&collapse=timestamp:6&fl=timestamp,original`;

561 const rows = JSON.parse((await get(cdx)) || '[]');

562 return rows.slice(1).map(([timestamp, original]) => ({ timestamp, original }));

563}

564

565async function cmdSite(domain, opts) {

566 const urls = await sitemapUrls(domain, posInt(opts.limit, 20000, 'limit'));

567 const families = {};

568 for (const u of urls) families[pathFamily(u)] = (families[pathFamily(u)] || 0) + 1;

569 const locales = [...new Set(urls.map(localeOf).filter(Boolean))].sort();

570 const rows = Object.entries(families).sort((a, b) => b[1] - a[1]).map(([family, pages]) => ({ family, pages }));

571 if (opts.out) writeFileSync(opts.out, toCsv(urls.map(url => ({ url, family: pathFamily(url), locale: localeOf(url) || '' }))));

572 return { domain, pulled: new Date().toISOString().slice(0, 10), urls: urls.length, locales, families: rows };

573}

574

575async function cmdStack(domain, opts) {

576 const months = posInt(opts.months, 24, 'months');

577 const snaps = await waybackMonthly(`${domain}/`, months);

578 const firstSeen = {};

579 const lastSeen = {};

580 const failed = [];

581 for (const s of snaps) {

582 try {

583 const html = await get(`https://web.archive.org/web/${s.timestamp}id_/${s.original}`);

584 const month = `${s.timestamp.slice(0, 4)}-${s.timestamp.slice(4, 6)}`;

585 for (const tool of detectTools(html)) {

586 firstSeen[tool] ??= month;

587 lastSeen[tool] = month;

588 }

589 } catch {

590 failed.push(s.timestamp);

591 }

592 await sleep(WAYBACK_DELAY_MS);

593 }

594 const firstMonth = snaps.length ? `${snaps[0].timestamp.slice(0, 4)}-${snaps[0].timestamp.slice(4, 6)}` : null;

595 let live = [];

596 try { live = detectTools(await get(`https://${domain}/`)); } catch {}

597 const tools = SIGNATURES.map(s => s.tool)

598 .filter(t => firstSeen[t] || live.includes(t))

599 .map(tool => ({

600 tool,

601 group: SIGNATURES.find(s => s.tool === tool).group,

602 first_seen: firstSeen[tool] || null,

603 present_at_start: Boolean(firstSeen[tool]) && firstSeen[tool] === firstMonth,

604 last_seen_archive: lastSeen[tool] || null,

605 live_now: live.includes(tool),

606 }));

607 return {

608 domain,

609 pulled: new Date().toISOString().slice(0, 10),

610 snapshots: snaps.length,

611 failed_snapshots: failed,

612 note: 'first_seen is the first monthly archive capture containing the script; the tool may predate it. present_at_start means it was already there at the start of the window. Scripts loaded only through a tag manager, or after consent, will not show, so live_now false is not proof of absence.',

613 tools,

614 };

615}

616

617async function cmdHistory(url, opts) {

618 const months = posInt(opts.months, 24, 'months');

619 const snaps = await waybackMonthly(url, months);

620 const changes = [];

621 const noPrices = [];

622 let prev = null;

623 for (const s of snaps) {

624 try {

625 const html = await get(`https://web.archive.org/web/${s.timestamp}id_/${s.original}`);

626 const text = html.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();

627 const prices = [...new Set(text.match(/[$€£]\s?\d[\d,.]*(?:\s?\/\s?(?:mo|month|yr|year|user|seat))?/gi) || [])];

628 const month = `${s.timestamp.slice(0, 4)}-${s.timestamp.slice(4, 6)}`;

629 const sig = prices.join(' ');

630 if (!prices.length) noPrices.push(month);

631 else if (sig !== prev) {

632 changes.push({ month, capture: `https://web.archive.org/web/${s.timestamp}/${s.original}`, prices: prices.slice(0, 20) });

633 prev = sig;

634 }

635 } catch {}

636 await sleep(WAYBACK_DELAY_MS);

637 }

638 return { url, pulled: new Date().toISOString().slice(0, 10), snapshots: snaps.length, note: 'Rows are months where the set of prices on the page changed. Open each capture to read what changed. Months in no_prices had no readable prices, often because the page renders prices with JavaScript.', changes, no_prices: noPrices };

639}

640

641async function cmdHero(url, opts) {

642 const months = posInt(opts.months, 24, 'months');

643 const snaps = await waybackMonthly(url, months);

644 const changes = [];

645 let prev = null;

646 for (const s of snaps) {

647 try {

648 const hero = extractHero(await get(`https://web.archive.org/web/${s.timestamp}id_/${s.original}`));

649 const sig = `${hero.h1}|${hero.title}|${hero.description}`;

650 if (hero.h1 || hero.title) {

651 if (sig !== prev) {

652 changes.push({ month: `${s.timestamp.slice(0, 4)}-${s.timestamp.slice(4, 6)}`, capture: `https://web.archive.org/web/${s.timestamp}/${s.original}`, ...hero });

653 prev = sig;

654 }

655 }

656 } catch {}

657 await sleep(WAYBACK_DELAY_MS);

658 }

659 return { url, pulled: new Date().toISOString().slice(0, 10), snapshots: snaps.length, note: 'Rows are months where the headline, title, or meta description changed. Open the capture to read the full hero and the logo wall.', changes };

660}

661

662async function cmdDns(domain, { resolver } = {}) {

663 const dns = resolver || (await import('node:dns')).promises;

664 const safe = async fn => { try { return await fn(); } catch { return []; } };

665 const txt = (await safe(() => dns.resolveTxt(domain))).map(parts => parts.join(''));

666 const dmarc = (await safe(() => dns.resolveTxt(`_dmarc.${domain}`))).map(parts => parts.join(''));

667 const mx = (await safe(() => dns.resolveMx(domain))).map(r => r.exchange);

668 const cnames = {};

669 await Promise.all(MARKETING_SUBDOMAINS.map(async sub => {

670 const target = (await safe(() => dns.resolveCname(`${sub}.${domain}`)))[0];

671 if (target) cnames[`${sub}.${domain}`] = target;

672 }));

673 return {

674 domain,

675 pulled: new Date().toISOString().slice(0, 10),

676 ...detectDns({ txt, mx, cnames }),

677 dmarc: dmarc[0] || null,

678 note: 'Mail tools come from SPF and MX records, verifications from TXT records, platforms from CNAMEs on common marketing subdomains. A record shows a tool was connected, not how it is used.',

679 };

680}

681

682async function cmdSubdomains(domain, opts = {}) {

683 const res = await fetch(`https://crt.sh/?q=${encodeURIComponent(`%.${domain}`)}&output=json`, { headers: { 'user-agent': UA }, signal: AbortSignal.timeout(90000) });

684 if (!res.ok) throw new Error(`crt.sh returned ${res.status}; it is often slow, so retry later`);

685 const all = parseCertificates(await res.json(), domain);

686 if (opts.all) return { domain, pulled: new Date().toISOString().slice(0, 10), count: all.length, subdomains: all };

687 const { subdomains, groups, dropped } = filterSubdomains(all, domain);

688 return {

689 domain,

690 pulled: new Date().toISOString().slice(0, 10),

691 count: subdomains.length,

692 filtered_out: dropped,

693 note: 'first_seen is the earliest certificate issued for the name, usually days before launch. Preview, staging, and deeply nested names are filtered out (use --all to keep them). groups lists labels with many names under them, such as per-customer or programmatic subdomains.',

694 groups,

695 subdomains,

696 };

697}

698

699async function cmdSitemapGrowth(domain, opts) {

700 const months = posInt(opts.months, 24, 'months');

701 const sitemapUrl = opts.sitemap || `${domain}/sitemap.xml`;

702 const snaps = await waybackMonthly(sitemapUrl.replace(/^https?:\/\//, ''), months);

703 const rows = [];

704 for (const s of snaps) {

705 const month = `${s.timestamp.slice(0, 4)}-${s.timestamp.slice(4, 6)}`;

706 try {

707 const xml = await get(`https://web.archive.org/web/${s.timestamp}id_/${s.original}`);

708 let urls = countLocs(xml);

709 let children = 0;

710 let failed = 0;

711 if (/<sitemapindex/i.test(xml)) {

712 const childUrls = urls.slice(0, 25);

713 children = childUrls.length;

714 urls = [];

715 for (const child of childUrls) {

716 try { urls.push(...countLocs(await get(`https://web.archive.org/web/${s.timestamp}id_/${child}`))); } catch { failed++; }

717 await sleep(WAYBACK_DELAY_MS);

718 }

719 }

720 if (children && failed === children) {

721 rows.push({ month, urls: null, child_sitemaps: children, error: 'no child sitemap readable' });

722 } else {

723 const families = {};

724 for (const u of urls) families[pathFamily(u)] = (families[pathFamily(u)] || 0) + 1;

725 // A partial count is a floor, not a drop in pages.

726 rows.push({ month, urls: urls.length, child_sitemaps: children, ...(failed ? { failed_children: failed, incomplete: true } : {}), families });

727 }

728 } catch {

729 rows.push({ month, urls: null, error: 'capture unreadable' });

730 }

731 await sleep(WAYBACK_DELAY_MS);

732 }

733 const firstSeen = {};

734 for (const r of rows) for (const f of Object.keys(r.families || {})) firstSeen[f] ??= r.month;

735 return {

736 domain,

737 pulled: new Date().toISOString().slice(0, 10),

738 note: 'URL counts per monthly capture of the sitemap. family_first_seen dates when each page family first appeared in an archived sitemap; families seen in the first month were already there. Large sitemap indexes are capped at 25 child sitemaps per month.',

739 family_first_seen: firstSeen,

740 months: rows.map(({ families, ...r }) => r),

741 };

742}

743

744const SC_BASE = 'https://api.scrapecreators.com';

745

746export function scAdsRequests(name, { domain, pageId, country } = {}) {

747 const q = new URLSearchParams();

748 const reqs = {};

749 if (pageId) {

750 q.set('pageId', pageId);

751 q.set('status', 'ACTIVE');

752 q.set('trim', 'true');

753 if (country) q.set('country', country);

754 reqs.meta_ads = `${SC_BASE}/v1/facebook/adLibrary/company/ads?${q}`;

755 } else {

756 reqs.meta_companies = `${SC_BASE}/v1/facebook/adLibrary/search/companies?${new URLSearchParams({ query: name })}`;

757 }

758 if (domain) reqs.google_ads = `${SC_BASE}/v1/google/company/ads?${new URLSearchParams({ domain })}`;

759 reqs.tiktok_ads = `${SC_BASE}/v1/tiktok/ad-library/search?${new URLSearchParams({ advertiser_name: name })}`;

760 return reqs;

761}

762

763export async function cmdScAds(name, opts, fetchImpl = fetch) {

764 const reqs = scAdsRequests(name, { domain: opts.domain, pageId: opts['page-id'], country: opts.country });

765 if (opts['dry-run']) return { dry_run: true, requests: reqs };

766 const key = process.env.SCRAPECREATORS_API_KEY;

767 if (!key) throw new Error('Set SCRAPECREATORS_API_KEY, or use the ads command for free library links');

768 const out = { name, pulled: new Date().toISOString().slice(0, 10) };

769 for (const [label, url] of Object.entries(reqs)) {

770 try {

771 const res = await fetchImpl(url, { headers: { 'x-api-key': key } });

772 out[label] = res.ok ? await res.json() : { error: `${res.status}` };

773 } catch (err) {

774 out[label] = { error: err.message };

775 }

776 }

777 if (out.meta_companies) out.next_step = 'Pick the matching Meta page from meta_companies and rerun with --page-id <id> for its active ads.';

778 return out;

779}

780

781export function schemaTypes(html) {

782 const types = new Set();

783 for (const m of html.matchAll(/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) {

784 for (const t of m[1].matchAll(/"@type"\s*:\s*(\[[^\]]*\]|"[^"]+")/g)) {

785 for (const v of t[1].matchAll(/"([^"]+)"/g)) types.add(v[1]);

786 }

787 }

788 return [...types];

789}

790

791// Each rule adds weight to an industry pack when its pattern appears in the page, its scripts, or its schema types.

792export const INDUSTRY_RULES = [

793 { pack: 'ecommerce-dtc', tools: ['Shopify', 'WooCommerce', 'BigCommerce', 'Klaviyo', 'Recharge', 'Yotpo', 'Okendo', 'Rebuy'], text: [/add to cart/i, /\/products\//, /free shipping/i], schema: [], weakSchema: ['Product', 'Offer'] },

794 { pack: 'saas', tools: ['Segment', 'Intercom', 'PostHog', 'Mixpanel', 'Amplitude'], text: [/start (your )?free trial/i, /book a demo/i, /sign up free/i, /per (user|seat)/i], weakText: [/integrations/i], schema: ['SoftwareApplication', 'WebApplication'] },

795 { pack: 'cpg-food-beverage', tools: ['PriceSpider', 'Destini'], text: [/where to buy/i, /find (us )?in stores/i, /nutrition facts/i, /ingredients/i], schema: [] },

796 { pack: 'retail', tools: [], text: [/store locator/i, /find a store/i, /store hours/i, /pick ?up in store/i], schema: ['Store', 'ClothingStore', 'DepartmentStore'] },

797 { pack: 'local-services', tools: ['CallRail', 'CallTrackingMetrics', 'ServiceTitan', 'Housecall Pro', 'Jobber', 'Podium', 'Birdeye'], text: [/free estimate/i, /service area/i, /licensed (and|&) insured/i, /24\/7 emergency/i, /financing available/i], schema: ['HomeAndConstructionBusiness', 'Plumber', 'HVACBusiness', 'RoofingContractor', 'Electrician', 'LocalBusiness'] },

798 { pack: 'healthcare', tools: ['Zocdoc', 'NexHealth'], text: [/book an appointment/i, /insurance (we )?accept/i, /new patients/i, /telehealth/i], schema: ['MedicalClinic', 'Dentist', 'Physician', 'MedicalBusiness', 'MedicalOrganization'] },

799 { pack: 'legal-professional-services', tools: ['Lawmatics', 'Clio'], text: [/free consultation/i, /attorney advertising/i, /practice areas/i, /\bCPA\b/], schema: ['LegalService', 'Attorney', 'AccountingService'] },

800 { pack: 'real-estate', tools: ['IDX Broker', 'kvCORE', 'Follow Up Boss'], text: [/\bMLS\b/, /homes for sale/i, /what'?s my home worth/i, /listings/i], schema: ['RealEstateAgent', 'RealEstateListing'] },

801 { pack: 'finance', tools: [], text: [/member FDIC/i, /\bNMLS\b/, /\bAPY\b/, /\bSIPC\b/, /credit score/i], schema: ['FinancialService', 'BankOrCreditUnion', 'InsuranceAgency'] },

802 { pack: 'crypto-web3', tools: ['WalletConnect'], text: [/connect wallet/i, /tokenomics/i, /governance token/i, /on-?chain/i, /\bstaking\b/i], schema: [] },

803 { pack: 'manufacturing-industrial', tools: [], text: [/request a quote/i, /\bRFQ\b/, /spec sheet/i, /\bCAD\b/, /ISO 9001/i, /find a distributor/i], schema: [] },

804 { pack: 'automotive', tools: ['Dealer.com', 'DealerInspire'], text: [/new inventory/i, /used inventory/i, /\bVIN\b/, /schedule service/i, /trade-?in/i], schema: ['AutoDealer', 'AutoRepair', 'Car'] },

805 { pack: 'agencies-b2b-services', tools: [], text: [/our work/i, /solutions partner/i, /get a proposal/i], weakText: [/case studies/i, /\bclients\b/i], schema: ['ProfessionalService'] },

806 { pack: 'nonprofit', tools: ['Classy', 'Givebutter', 'Donorbox', 'Fundraise Up'], text: [/donate/i, /501$c$$3$/i, /tax-deductible/i], schema: ['NGO', 'NonprofitOrganization'] },

807 { pack: 'marketplaces', tools: [], text: [/become a (seller|host|tasker|provider|driver)/i, /list your/i, /for (providers|sellers|hosts)/i], schema: [] },

808 { pack: 'mobile-apps-games', tools: [], text: [], weakText: [/apps\.apple\.com/, /play\.google\.com\/store/, /download (on|the) (the )?app/i], schema: ['MobileApplication'] },

809 { pack: 'education', tools: [], text: [/\benroll\b/i, /admissions/i, /\bcohort\b/i, /bootcamp/i, /tuition/i], schema: ['EducationalOrganization', 'CollegeOrUniversity', 'Course'] },

810 { pack: 'hospitality-travel', tools: ['OpenTable', 'Resy', 'SevenRooms', 'Toast', 'SynXis'], text: [/book (a table|your stay)/i, /make a reservation/i, /check-?in date/i, /order online/i, /view (our )?menu/i], schema: ['Hotel', 'LodgingBusiness', 'Restaurant', 'TouristTrip'] },

811 { pack: 'insurance', tools: [], text: [/underwritten by/i, /policyholders?/i, /file a claim/i, /find an agent/i, /compare (quotes|rates)/i, /Medicare Advantage/i], weakText: [/get a (free )?quote/i, /\bpremiums?\b/i], schema: ['InsuranceAgency'] },

812 { pack: 'fitness-wellness', tools: ['Mindbody', 'ClassPass', 'Glofox', 'Mariana Tek', 'Zen Planner'], text: [/unlimited (classes|membership)/i, /first class free/i, /free (first )?class/i, /class schedule/i, /personal training/i, /intro offer/i, /find a studio/i, /book a class/i], weakText: [/\bworkouts?\b/i], schema: ['ExerciseGym', 'HealthClub', 'SportsActivityLocation'] },

813 { pack: 'consumer-hardware', tools: ['Kickstarter', 'Indiegogo', 'BackerKit'], text: [/tech specs/i, /what'?s in the box/i, /backed on kickstarter/i, /warranty registration/i], weakText: [/pre-?order/i, /ships in/i], schema: [] },

814 { pack: 'solar-home-energy', tools: [], text: [/solar (panels?|energy|installation|installer)/i, /\$0 down/i, /net metering/i, /battery (storage|backup)/i, /heat pumps?/i, /\bPPA\b/], weakText: [/tax credit/i, /\bkWh\b/], schema: [], weakSchema: ['HomeAndConstructionBusiness'] },

815 { pack: 'care-services', tools: [], text: [/assisted living/i, /memory care/i, /home care/i, /in-home (care|assessment)/i, /senior living/i, /child ?care|daycare|preschool/i, /medicare-certified/i], weakText: [/schedule a tour/i], schema: ['ChildCare'], weakSchema: ['MedicalBusiness'] },

816 { pack: 'franchising', tools: [], text: [/own a franchise/i, /franchise opportunit/i, /franchise fee/i, /\bFDD\b/, /territories available/i, /discovery day/i, /initial investment/i], weakText: [/independently owned and operated/i], schema: [] },

817 { pack: 'logistics-transportation', tools: [], text: [/\bLTL\b/, /\bFTL\b/, /\b3PL\b/, /\bUSDOT\b/i, /\bMC[ #-]?\d{5,}/, /haul with us|drive for us|become a carrier/i, /track (a|your) shipment/i, /fulfillment center/i], weakText: [/freight/i, /\bCDL\b/], schema: [], weakSchema: ['MovingCompany'] },

818 { pack: 'govtech-public-sector', tools: [], text: [/\bFedRAMP\b/, /\b(StateRAMP|GovRAMP|CJIS)\b/, /cooperative (purchasing|contract)/i, /\bGSA (schedule|MAS)\b/i, /Sourcewell|OMNIA Partners|NASPO|BuyBoard/i, /state and local government/i], weakText: [/public sector/i, /for government/i, /\bVPAT\b|Section 508/i, /school districts/i], schema: [], weakSchema: ['GovernmentOrganization'] },

819 { pack: 'regulated-goods', tools: [], text: [/are you (21|18)( years of age)? or older/i, /contains nicotine/i, /\bdispensar(y|ies)\b/i, /dutchie|iheartjane|weedmaps|leafly/i, /\bFFL\b/, /ShipCompliant|Vinoshipper|Commerce7|WineDirect/i, /AgeChecker/i], weakText: [/must be 21/i, /age verification/i, /\bTHC\b/, /\bCBD\b/], schema: ['LiquorStore', 'Winery', 'Brewery', 'Distillery'] },

820 { pack: 'gambling-betting', tools: [], text: [/sportsbook/i, /deposit match/i, /1-800-GAMBLER/i, /gambling problem/i, /no purchase necessary/i, /odds boost/i], weakText: [/promo code/i, /\bsweepstakes\b/i, /\b21\+/], schema: [] },

821 { pack: 'games-pc-console', tools: ['Steam'], text: [/wishlist (it )?(on steam|now)/i, /store\.steampowered\.com\/app/, /\bplaytest\b/i], weakText: [/playstation|xbox|nintendo switch|eshop/i, /press kit/i, /discord\.gg/], schema: ['VideoGame', 'VideoGameSeries'] },

822 { pack: 'events-entertainment', tools: ['Eventbrite', 'Ticketmaster', 'AXS', 'DICE', 'Tixr'], text: [/(buy|get) tickets/i, /buy passes/i, /\blineup\b/i, /presale/i, /on sale (now|date)/i, /early bird/i], weakText: [/\bspeakers\b/i], schema: ['MusicEvent', 'Festival', 'SportsEvent', 'TheaterEvent'], weakSchema: ['Event'] },

823 { pack: 'media-creators', tools: ['Piano', 'Memberful'], text: [/become a (member|subscriber)/i, /\bepisodes?\b/i], weakText: [/subscribe to (our|the) newsletter/i, /\bpodcast\b/i], schema: ['NewsMediaOrganization', 'PodcastSeries'], weakSchema: ['NewsArticle'] },

824];

825

826export function detectIndustry({ html = '', tools = [], linked = [], cnames = [] }) {

827 const types = schemaTypes(html);

828 const text = html.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, ' ');

829 const scores = INDUSTRY_RULES.map(rule => {

830 const reasons = [];

831 for (const t of rule.tools) {

832 if (tools.includes(t)) reasons.push(`script: ${t}`);

833 else if (linked.includes(t)) reasons.push(`link: ${t}`);

834 }

835 const label = re => re.source.replace(/\\b|\\/g, '');

836 for (const re of rule.text) if (re.test(text) || re.test(html)) reasons.push(`page: ${label(re)}`);

837 for (const re of rule.weakText || []) if (re.test(text) || re.test(html)) reasons.push(`weak page: ${label(re)}`);

838 for (const s of rule.schema) if (types.includes(s)) reasons.push(`schema: ${s}`);

839 for (const s of rule.weakSchema || []) if (types.includes(s)) reasons.push(`weak schema: ${s}`);

840 // Scripts and specific schema types count double; generic words and types count half.

841 const weight = r => (r.startsWith('weak') ? 0.5 : r.startsWith('page') || r.startsWith('link') ? 1 : 2);

842 const score = reasons.reduce((n, r) => n + weight(r), 0);

843 return { pack: rule.pack, score, reasons };

844 });

845 if (cnames.some(c => /substack|beehiiv|ghost\.io/i.test(c))) {

846 const media = scores.find(s => s.pack === 'media-creators');

847 media.score += 2;

848 media.reasons.push('dns: newsletter platform');

849 }

850 return scores.filter(s => s.score >= 1).sort((a, b) => b.score - a.score).slice(0, 3);

851}

852

853export function extractPhones(html) {

854 const decode = s => { try { return decodeURIComponent(s); } catch { return s; } };

855 const tel = [...html.matchAll(/href=["']tel:([^"']+)["']/gi)].map(m => decode(m[1]).replace(/[^\d+]/g, ''));

856 const text = stripTags(html.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, ' '));

857 const shown = [...text.matchAll(/(?:\+?1[\s.-]?)?$?\b\d{3}$?[\s.-]\d{3}[\s.-]\d{4}\b/g)].map(m => m[0].replace(/[^\d+]/g, ''));

858 return [...new Set([...tel, ...shown])];

859}

860

861export function utmTestUrls(url) {

862 const variants = [

863 ['(none)', {}],

864 ['google-ads', { utm_source: 'google', utm_medium: 'cpc', gclid: 'test' }],

865 ['google-organic', { utm_source: 'google', utm_medium: 'organic' }],

866 ['meta-ads', { utm_source: 'facebook', utm_medium: 'paid', fbclid: 'test' }],

867 ['bing-ads', { utm_source: 'bing', utm_medium: 'cpc', msclkid: 'test' }],

868 ['gbp', { utm_source: 'gbp', utm_medium: 'organic' }],

869 ];

870 return variants.map(([label, params]) => {

871 const u = new URL(url);

872 for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);

873 return { label, url: u.toString() };

874 });

875}

876

877export function optimizelyNames(js) {

878 return [...new Set([...js.matchAll(/"name"\s*:\s*"([^"]{3,120})"/g)].map(m => m[1]))].slice(0, 150);

879}

880

881export function growthbookKeys(html) {

882 const key = html.match(/["'](sdk-[A-Za-z0-9]{8,})["']/)?.[1] || null;

883 const host = html.match(/apiHost["']?\s*[:=]\s*["'](https:\/\/[^"']+)["']/)?.[1] || 'https://cdn.growthbook.io';

884 return { key, host };

885}

886

887export function summarizeGrowthbook(json) {

888 if (json.encryptedFeatures) return { encrypted: true, features: [] };

889 const features = Object.entries(json.features || {}).map(([key, f]) => ({

890 feature: key,

891 experiments: (f.rules || []).filter(r => r.variations).map(r => r.key || r.name || '(unnamed)'),

892 }));

893 return { encrypted: false, features };

894}

895

896export function edgarDocUrl(cik, accession, doc) {

897 return `https://www.sec.gov/Archives/edgar/data/${Number(cik)}/${accession.replace(/-/g, '')}/${doc}`;

898}

899

900export function monthRanges(months, now = new Date()) {

901 const out = [];

902 for (let i = months - 1; i >= 0; i--) {

903 const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - i, 1));

904 // CFPB's date_received_max is inclusive, so the range ends on the month's last day.

905 const end = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth() + 1, 0));

906 out.push({ month: start.toISOString().slice(0, 7), min: start.toISOString().slice(0, 10), max: end.toISOString().slice(0, 10) });

907 }

908 return out;

909}

910

911// Sites that block scripted fetches are read from their latest Wayback capture instead.

912async function getPageOrArchive(url) {

913 try {

914 return { html: await get(url), source: 'live' };

915 } catch (err) {

916 const snap = await getJson(`https://archive.org/wayback/available?url=${encodeURIComponent(url)}`);

917 const closest = snap.archived_snapshots?.closest;

918 if (!closest?.available) throw err;

919 const ts = closest.timestamp;

920 return { html: await get(`https://web.archive.org/web/${ts}id_/${url}`), source: `archive ${ts.slice(0, 4)}-${ts.slice(4, 6)}-${ts.slice(6, 8)} (live fetch: ${err.message.split(' ')[0]})` };

921 }

922}

923

924async function getJson(url, headers = {}) {

925 const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json', ...headers }, redirect: 'follow' });

926 if (!res.ok) throw new Error(`${res.status} ${url}`);

927 return res.json();

928}

929

930async function cmdDetect(domain) {

931 const { html, source } = await getPageOrArchive(`https://${domain}/`);

932 const tools = detectTools(html);

933 const linked = detectLinkedTools(html);

934 const dnsInfo = await cmdDns(domain);

935 const packs = detectIndustry({ html, tools, linked, cnames: dnsInfo.subdomains.map(s => s.cname) });

936 return {

937 domain,

938 pulled: new Date().toISOString().slice(0, 10),

939 source,

940 note: 'Heuristic. Confirm the industry with the user, then load references/industries/<pack>.md. A company can fit two packs (a marketplace for local services).',

941 packs,

942 tools,

943 linked_tools: linked,

944 schema_types: schemaTypes(html),

945 };

946}

947

948async function cmdShopify(domain, opts) {

949 const maxPages = posInt(opts.pages, 20, 'pages');

950 const products = [];

951 let complete = false;

952 let stoppedAt = null;

953 for (let page = 1; page <= maxPages; page++) {

954 let json;

955 try { json = await getJson(`https://${domain}/products.json?limit=250&page=${page}`); } catch (err) {

956 if (page === 1) throw new Error(`products.json not available (${err.message}). The store may not be on Shopify or may block it.`);

957 stoppedAt = { page, error: err.message };

958 break;

959 }

960 if (!json.products?.length) { complete = true; break; }

961 products.push(...json.products);

962 await sleep(500);

963 }

964 const rows = products.map(p => {

965 const prices = (p.variants || []).map(v => Number(v.price)).filter(Number.isFinite);

966 return {

967 handle: p.handle,

968 title: p.title,

969 type: p.product_type || '',

970 vendor: p.vendor || '',

971 created: (p.created_at || '').slice(0, 10),

972 published: (p.published_at || '').slice(0, 10),

973 min_price: prices.length ? Math.min(...prices) : '',

974 max_price: prices.length ? Math.max(...prices) : '',

975 on_sale: (p.variants || []).some(v => Number(v.compare_at_price) > Number(v.price)),

976 variants: (p.variants || []).length,

977 tags: (Array.isArray(p.tags) ? p.tags : String(p.tags || '').split(',')).join('|'),

978 };

979 });

980 if (opts.out) writeFileSync(opts.out, toCsv(rows));

981 const byMonth = {};

982 for (const r of rows) if (r.created) byMonth[r.created.slice(0, 7)] = (byMonth[r.created.slice(0, 7)] || 0) + 1;

983 const types = {};

984 for (const r of rows) types[r.type || '(none)'] = (types[r.type || '(none)'] || 0) + 1;

985 let bestsellers = [];

986 try {

987 const html = await get(`https://${domain}/collections/all?sort_by=best-selling`);

988 bestsellers = [...new Set([...html.matchAll(/\/products\/([a-z0-9][a-z0-9-]*)/gi)].map(m => m[1].toLowerCase()))].slice(0, 30);

989 } catch {}

990 let apps = [];

991 try { apps = detectTools(await get(`https://${domain}/`)).filter(t => SIGNATURES.find(s => s.tool === t && ['ecommerce', 'payments', 'testing', 'lifecycle'].includes(s.group))); } catch {}

992 return {

993 domain,

994 pulled: new Date().toISOString().slice(0, 10),

995 products: rows.length,

996 complete,

997 ...(stoppedAt ? { stopped_at: stoppedAt } : {}),

998 ...(!complete && !stoppedAt ? { truncated_at_pages: maxPages } : {}),

999 on_sale: rows.filter(r => r.on_sale).length,

1000 created_by_month: Object.fromEntries(Object.entries(byMonth).sort()),

1001 product_types: Object.fromEntries(Object.entries(types).sort((a, b) => b[1] - a[1]).slice(0, 20)),

1002 bestsellers_in_order: bestsellers,

1003 apps,

1004 note: 'created is when the product was added to the store; published can be reset by relaunches. Bestseller order comes from the store\'s own best-selling sort, which some themes override. Hidden and unlisted products are not included.',

1005 };

1006}

1007

1008async function cmdExperiments(domain) {

1009 const { html, source } = await getPageOrArchive(`https://${domain}/`);

1010 const tools = detectTools(html).filter(t => SIGNATURES.find(s => s.tool === t && s.group === 'testing'));

1011 const out = { domain, pulled: new Date().toISOString().slice(0, 10), source, tools };

1012 const opt = html.match(/cdn\.optimizely\.com\/js\/(\d+)\.js/);

1013 if (opt) {

1014 try {

1015 out.optimizely = { snippet: `https://cdn.optimizely.com/js/${opt[1]}.js`, names: optimizelyNames(await get(`https://cdn.optimizely.com/js/${opt[1]}.js`)) };

1016 } catch (err) { out.optimizely = { error: err.message }; }

1017 }

1018 const gb = growthbookKeys(html);

1019 if (gb.key) {

1020 try { out.growthbook = { client_key: gb.key, ...summarizeGrowthbook(await getJson(`${gb.host}/api/features/${gb.key}`)) }; } catch (err) { out.growthbook = { error: err.message }; }

1021 }

1022 out.note = 'Optimizely names include experiments, pages, audiences, and events; read them for test themes. Other tools keep configs server-side or need a user context, so only their presence is reported. Many sites load these tools through a tag manager, so check in a browser too.';

1023 return out;

1024}

1025

1026async function cmdPhones(url) {

1027 const { html, source } = await getPageOrArchive(url);

1028 return {

1029 url,

1030 source,

1031 pulled: new Date().toISOString().slice(0, 10),

1032 numbers_in_html: extractPhones(html),

1033 call_tracking: detectTools(html).filter(t => SIGNATURES.find(s => s.tool === t && s.group === 'call-tracking')),

1034 browser_check: utmTestUrls(url),

1035 note: 'Call tracking swaps numbers in the browser after the page loads, so the HTML shows only the default. Open each browser_check URL in a fresh session and record the number shown. A different number per source means that channel is tracked separately.',

1036 };

1037}

1038

1039const SEC_FORMS = new Set(['10-K', '10-Q', '8-K', 'S-1', 'S-1/A', 'F-1', '424B4', 'DEF 14A', '20-F', '6-K', 'D']);

1040

1041async function cmdFilings(name, opts) {

1042 const ua = opts.ua || process.env.SEC_USER_AGENT;

1043 if (!ua) throw new Error('SEC requires a contact in the User-Agent. Pass --ua "Your Name you@company.com" or set SEC_USER_AGENT.');

1044 const headers = { 'user-agent': ua };

1045 const out = { query: name, pulled: new Date().toISOString().slice(0, 10) };

1046 const tickers = Object.values(await getJson('https://www.sec.gov/files/company_tickers.json', headers));

1047 const q = name.toLowerCase();

1048 const rank = t => (t.ticker.toLowerCase() === q ? 0 : t.title.toLowerCase().startsWith(q) ? 1 : 2);

1049 const matches = tickers.filter(t => t.ticker.toLowerCase() === q || t.title.toLowerCase().includes(q))

1050 .sort((a, b) => rank(a) - rank(b)).slice(0, 5);

1051 out.public_matches = matches.map(m => ({ cik: m.cik_str, ticker: m.ticker, name: m.title }));

1052 const cik = opts.cik || matches[0]?.cik_str;

1053 if (cik) {

1054 const sub = await getJson(`https://data.sec.gov/submissions/CIK${String(cik).padStart(10, '0')}.json`, headers);

1055 const r = sub.filings.recent;

1056 out.company = sub.name;

1057 out.filings = r.form.map((form, i) => ({ form, date: r.filingDate[i], url: edgarDocUrl(cik, r.accessionNumber[i], r.primaryDocument[i]) }))

1058 .filter(f => SEC_FORMS.has(f.form)).slice(0, posInt(opts.limit, 40, 'limit'));

1059 }

1060 try {

1061 const fts = await getJson(`https://efts.sec.gov/LATEST/search-index?q=${encodeURIComponent(`"${name}"`)}&forms=D`, headers);

1062 out.form_d = (fts.hits?.hits || []).slice(0, 20).map(h => {

1063 const [adsh, file] = h._id.split(':');

1064 return { filer: h._source.display_names?.[0], date: h._source.file_date, url: edgarDocUrl(h._source.ciks[0], adsh, file) };

1065 });

1066 } catch (err) { out.form_d = { error: err.message }; }

1067 out.note = 'Public companies: 10-K and 10-Q for customers, revenue, and sales and marketing spend; S-1 for the growth story; 8-K for events. Form D filings record private fundraises (often filed by the company or an SPV), with amounts and dates. Name matches can be wrong; check the filer.';

1068 return out;

1069}

1070

1071const CFPB = 'https://www.consumerfinance.gov/data-research/consumer-complaints/search/api/v1/';

1072

1073async function cmdCfpb(name, opts) {

1074 const months = posInt(opts.months, 24, 'months');

1075 let company = opts.exact;

1076 const candidates = (await getJson(`${CFPB}?${new URLSearchParams({ search_term: name, field: 'company', size: '0' })}`))

1077 .aggregations?.company?.company?.buckets?.map(b => ({ company: b.key, complaints: b.doc_count })) || [];

1078 company ||= candidates[0]?.company;

1079 if (!company) return { query: name, candidates, note: 'No company found in the CFPB database. Try another spelling, or --exact "Company Name".' };

1080 const ranges = monthRanges(months);

1081 const monthly = [];

1082 for (const r of ranges) {

1083 const json = await getJson(`${CFPB}?${new URLSearchParams({ company, size: '0', date_received_min: r.min, date_received_max: r.max })}`);

1084 monthly.push({ month: r.month, complaints: json.hits?.total?.value ?? null });

1085 await sleep(250);

1086 }

1087 const window = await getJson(`${CFPB}?${new URLSearchParams({ company, size: '0', date_received_min: ranges[0].min, date_received_max: ranges[ranges.length - 1].max })}`);

1088 const top = key => (window.aggregations?.[key]?.[key]?.buckets || []).slice(0, 10).map(b => ({ [key]: b.key, complaints: b.doc_count }));

1089 return {

1090 query: name,

1091 company,

1092 candidates,

1093 pulled: new Date().toISOString().slice(0, 10),

1094 monthly,

1095 top_products: top('product'),

1096 top_issues: top('issue'),

1097 note: 'Complaint volume tracks customer volume and problems together; read it next to growth, not as growth. The latest month is partial.',

1098 };

1099}

1100

1101async function cmdFdic(name, opts) {

1102 const base = 'https://api.fdic.gov/banks';

1103 const inst = await getJson(`${base}/institutions?${new URLSearchParams({ search: `NAME:${name}`, fields: 'NAME,CERT,ASSET,DEP,ACTIVE,CITY,STALP,ESTYMD', limit: '25', sort_by: 'ASSET', sort_order: 'DESC' })}`);

1104 const banks = (inst.data || []).map(d => d.data);

1105 // FDIC search also matches former names, so only an active bank whose current name starts with the query is picked.

1106 const norm = s => String(s).toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim();

1107 const named = banks.filter(b => b.ACTIVE === 1 && norm(b.NAME).startsWith(norm(name)));

1108 const cert = opts.cert || (named.find(b => norm(b.NAME) === norm(name)) || (named.length === 1 ? named[0] : null))?.CERT;

1109 const out = { query: name, pulled: new Date().toISOString().slice(0, 10), matches: banks };

1110 if (!cert) out.needs_cert = 'No single active bank matched the name. Pick a CERT from matches and rerun with --cert.';

1111 if (cert) {

1112 const fin = await getJson(`${base}/financials?${new URLSearchParams({ filters: `CERT:${cert}`, fields: 'REPDTE,ASSET,DEP,NUMEMP', sort_by: 'REPDTE', sort_order: 'DESC', limit: String(posInt(opts.quarters, 12, 'quarters')) })}`);

1113 out.cert = cert;

1114 out.quarterly = (fin.data || []).map(d => d.data).map(q => ({ quarter: q.REPDTE, assets_thousands: q.ASSET, deposits_thousands: q.DEP, employees: q.NUMEMP })).reverse();

1115 }

1116 out.note = 'Dollar figures are in thousands. Fintechs that are not banks hold deposits at partner banks; look up the partner bank named in their disclosures. Credit unions report to the NCUA, not the FDIC.';

1117 return out;

1118}

1119

1120function cmdMoves(opts) {

1121 if (!opts.moves || !opts.traffic) throw new Error('moves needs --moves moves.csv --traffic traffic.csv');

1122 const result = analyzeMoves(parseCsv(readFileSync(opts.moves, 'utf8')), parseCsv(readFileSync(opts.traffic, 'utf8')));

1123 if (opts.out) writeFileSync(opts.out, toCsv(result.clusters));

1124 return result;

1125}

1126

1127function cmdDiff(opts) {

1128 if (!opts.old || !opts.new) throw new Error('diff needs --old moves-previous.csv --new moves.csv');

1129 const added = diffMoves(parseCsv(readFileSync(opts.old, 'utf8')), parseCsv(readFileSync(opts.new, 'utf8')));

1130 if (opts.out) writeFileSync(opts.out, toCsv(added));

1131 return { new_moves: added.length, moves: added };

1132}

1133

1134function parseArgs(argv) {

1135 const pos = [];

1136 const opts = {};

1137 for (let i = 0; i < argv.length; i++) {

1138 if (argv[i].startsWith('--')) {

1139 const key = argv[i].slice(2);

1140 const next = argv[i + 1];

1141 if (next === undefined || next.startsWith('--')) opts[key] = true;

1142 else { opts[key] = next; i++; }

1143 } else pos.push(argv[i]);

1144 }

1145 return { pos, opts };

1146}

1147

1148const USAGE = `teardown.mjs: free-source collection and analysis for reverse-engineer-marketing

1149

1150Commands:

1151 site <domain> [--limit 20000] [--out urls.csv] Sitemap URLs grouped by path family

1152 stack <domain> [--months 24] Ad, analytics, testing, referral, and lifecycle

1153 scripts by first month seen in Wayback captures

1154 history <url> [--months 24] Months a page's prices changed (pricing history)

1155 hero <url> [--months 24] Months the headline, title, or meta description changed

1156 sitemap-growth <domain> [--months 24] [--sitemap url]

1157 Sitemap size and page families by month from Wayback

1158 dns <domain> Mail tools, domain verifications, and platforms behind

1159 common marketing subdomains, from DNS records

1160 subdomains <domain> [--all] Subdomains in certificate logs with first-seen dates,

1161 preview and staging names filtered out unless --all

1162 detect <domain> Likely industry packs, with the reasons

1163 shopify <domain> [--pages 20] [--out products.csv]

1164 Public Shopify catalog with dates, bestseller order, apps

1165 experiments <domain> Client-side A/B testing tools and readable test configs

1166 phones <url> Phone numbers, call-tracking tools, and UTM URLs to check

1167 filings <company> [--ua "Name email"] [--cik n] SEC filings for public companies and Form D raises

1168 cfpb <company> [--months 24] [--exact "Name"] CFPB consumer complaints by month, top products and issues

1169 fdic <bank> [--cert n] [--quarters 12] FDIC quarterly assets, deposits, and employees

1170 ads <name> [--domain example.com] Public ad library links for each network

1171 sc-ads <name> [--domain example.com] [--page-id id] [--country US] [--dry-run]

1172 Active Meta, Google, and TikTok ads via ScrapeCreators

1173 (needs SCRAPECREATORS_API_KEY)

1174 moves --moves moves.csv --traffic traffic.csv [--out moves-vs-growth.csv]

1175 Lift per cluster of moves and leading channels

1176 diff --old previous.csv --new moves.csv [--out new-moves.csv]

1177 Moves in the new log that were not in the old one

1178

1179CSV columns:

1180 moves.csv date,channel,kind,move,reach,link,evidence

1181 traffic.csv month,visits,brand_searches (one source for the whole series)

1182

1183Output is JSON on stdout.`;

1184

1185async function main() {

1186 const [cmd, ...rest] = process.argv.slice(2);

1187 const { pos, opts } = parseArgs(rest);

1188 const domain = (pos[0] || '').replace(/^https?:\/\//, '').replace(/\/.*$/, '');

1189 if (['site', 'stack', 'history', 'hero', 'sitemap-growth', 'dns', 'subdomains', 'ads', 'sc-ads', 'detect', 'shopify', 'experiments', 'phones', 'filings', 'cfpb', 'fdic'].includes(cmd) && !pos[0]) {

1190 console.log(USAGE);

1191 process.exitCode = 1;

1192 return;

1193 }

1194 let result;

1195 switch (cmd) {

1196 case 'site': result = await cmdSite(domain, opts); break;

1197 case 'stack': result = await cmdStack(domain, opts); break;

1198 case 'history': result = await cmdHistory(pos[0], opts); break;

1199 case 'hero': result = await cmdHero(pos[0], opts); break;

1200 case 'sitemap-growth': result = await cmdSitemapGrowth(domain, opts); break;

1201 case 'dns': result = await cmdDns(domain); break;

1202 case 'subdomains': result = await cmdSubdomains(domain, opts); break;

1203 case 'ads': result = adLibraryLinks(pos[0], opts.domain); break;

1204 case 'sc-ads': result = await cmdScAds(pos[0], opts); break;

1205 case 'detect': result = await cmdDetect(domain); break;

1206 case 'shopify': result = await cmdShopify(domain, opts); break;

1207 case 'experiments': result = await cmdExperiments(domain); break;

1208 case 'phones': result = await cmdPhones(pos[0].startsWith('http') ? pos[0] : `https://${pos[0]}`); break;

1209 case 'filings': result = await cmdFilings(pos[0], opts); break;

1210 case 'cfpb': result = await cmdCfpb(pos[0], opts); break;

1211 case 'fdic': result = await cmdFdic(pos[0], opts); break;

1212 case 'moves': result = cmdMoves(opts); break;

1213 case 'diff': result = cmdDiff(opts); break;

1214 default:

1215 console.log(USAGE);

1216 if (cmd) process.exitCode = 1;

1217 return;

1218 }

1219 console.log(JSON.stringify(result, null, 2));

1220}

1221

1222if (import.meta.url === pathToFileURL(process.argv[1] || '').href) {

1223 main().catch(err => {

1224 console.log(JSON.stringify({ error: err.message }));

1225 process.exitCode = 1;

1226 });

1227}