Skills › HyperFrames › media-use › media-fetch.mjs
media-fetch.mjs
85 lines · skills/media-use/scripts/lib/media-fetch.mjs at a595755
1// Media downloads use public HTTP(S) URLs. Validate every redirect target;
2// a provider result must meet the same host policy as a direct ingest URL.
3// Public HTTPS-to-HTTP redirects are allowed, matching direct HTTP support.
4// This is a literal-host policy, not DNS pinning: DNS resolution remains trusted.
6import { BlockList, isIP } from "node:net";
8const blocked = new BlockList();
9for (const [network, prefix] of [
10 ["0.0.0.0", 8],
11 ["10.0.0.0", 8],
12 ["100.64.0.0", 10],
13 ["127.0.0.0", 8],
14 ["169.254.0.0", 16],
15 ["172.16.0.0", 12],
16 ["192.0.0.0", 24],
17 ["192.0.2.0", 24],
18 ["192.88.99.0", 24],
19 ["192.168.0.0", 16],
20 ["198.18.0.0", 15],
21 ["198.51.100.0", 24],
22 ["203.0.113.0", 24],
23 ["224.0.0.0", 4],
24 ["240.0.0.0", 4],
25])
26 blocked.addSubnet(network, prefix, "ipv4");
27for (const [network, prefix] of [
28 ["::", 128],
29 ["::1", 128],
30 ["fc00::", 7],
31 ["fe80::", 10],
32 ["fec0::", 10],
33 ["ff00::", 8],
34 ["2001:db8::", 32],
35])
36 blocked.addSubnet(network, prefix, "ipv6");
38export function isPublicMediaUrl(value) {
39 try {
40 const url = new URL(value);
41 if (url.protocol !== "http:" && url.protocol !== "https:") return false;
42 const host = url.hostname.replace(/\.$/, "");
43 if (
44 host === "localhost" ||
45 host.endsWith(".localhost") ||
46 host.endsWith(".local") ||
47 host.endsWith(".internal")
48 )
49 return false;
50 const address = host.replace(/^\[|\]$/g, "");
51 const family = isIP(address);
52 return family === 0 || !blocked.check(address, family === 4 ? "ipv4" : "ipv6");
53 } catch {
54 return false;
55 }
56}
58export async function fetchMedia(url, { method = "GET", signal, fetchImpl = fetch } = {}) {
59 let current = String(url);
60 for (let hop = 0; hop <= 5; hop++) {
61 if (!isPublicMediaUrl(current))
62 throw new Error("Media download blocked: URL is not public HTTP(S)");
63 const response = await fetchImpl(current, { method, signal, redirect: "manual" });
64 if (!(response.status >= 300 && response.status < 400)) return response;
65 const location = response.headers.get("location");
66 if (!location) return response;
67 await response.body?.cancel();
68 current = new URL(location, current).href;
69 }
70 throw new Error("Media download exceeded redirect limit");
71}
73// Stream and abort once the cap is crossed, so a lying or chunked body can't buffer past it.
74export async function readCappedBody(res, maxBytes, label) {
75 const declared = Number(res.headers.get("content-length"));
76 if (declared > maxBytes) throw new Error(`${label}: ${declared} bytes exceeds ${maxBytes} cap`);
77 const chunks = [];
78 let total = 0;
79 for await (const chunk of res.body) {
80 total += chunk.length;
81 if (total > maxBytes) throw new Error(`${label}: stream exceeds ${maxBytes} cap`);
82 chunks.push(chunk);
83 }
84 return Buffer.concat(chunks, total);
85}