TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import { FetchaError } from "./errors";23/** Dependency-free IP literal detection (0 = not an IP, 4, 6). Keeps this module bundler-safe. */4export function isIP(s: string): 0 | 4 | 6 {5 if (/^(25[0-5]|2[0-4]\d|1?\d?\d)(\.(25[0-5]|2[0-4]\d|1?\d?\d)){3}$/.test(s)) return 4;6 if (s.includes(":") && /^[0-9a-f:.]+$/i.test(s)) {7 const parts = s.split("::");8 if (parts.length > 2) return 0;9 const groups = s.replace(/^::|::$/g, "").split(/::|:/).filter(Boolean);10 const hasV4 = /\d+\.\d+\.\d+\.\d+$/.test(s);11 const max = hasV4 ? 7 : 8;12 if (groups.length > max) return 0;13 if (parts.length === 1 && groups.length !== max) return 0;14 for (const g of groups) if (!/^[0-9a-f]{1,4}$/i.test(g) && !/^\d+\.\d+\.\d+\.\d+$/.test(g)) return 0;15 return 6;16 }17 return 0;18}1920/**21 * SSRF protection. Every externally supplied URL — and every redirect hop — must pass22 * `assertUrlAllowed()` before any network activity. We validate the scheme, the host23 * literal, and every resolved address (to defeat DNS rebinding we return the resolved24 * addresses so callers can pin them when connecting directly).25 */2627const BLOCKED_HOSTNAMES = new Set([28 "localhost",29 "localhost.localdomain",30 "ip6-localhost",31 "ip6-loopback",32 "metadata.google.internal",33 "metadata",34 "instance-data",35 "kubernetes.default",36 "kubernetes.default.svc",37]);3839const BLOCKED_SUFFIXES = [".localhost", ".local", ".internal", ".localdomain", ".home.arpa", ".in-addr.arpa", ".ip6.arpa", ".maclustr.io"];4041function ipv4ToInt(ip: string): number {42 const p = ip.split(".").map((x) => Number(x));43 return ((p[0]! << 24) >>> 0) + (p[1]! << 16) + (p[2]! << 8) + p[3]!;44}4546function inCidr4(ip: string, cidr: string): boolean {47 const [base, bitsStr] = cidr.split("/");48 const bits = Number(bitsStr);49 const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;50 return ((ipv4ToInt(ip) & mask) >>> 0) === ((ipv4ToInt(base!) & mask) >>> 0);51}5253const BLOCKED_V4 = [54 "0.0.0.0/8",55 "10.0.0.0/8",56 "100.64.0.0/10", // carrier-grade NAT (also Tailscale)57 "127.0.0.0/8",58 "169.254.0.0/16", // link-local + cloud metadata (169.254.169.254)59 "172.16.0.0/12",60 "192.0.0.0/24",61 "192.0.2.0/24",62 "192.168.0.0/16",63 "198.18.0.0/15",64 "198.51.100.0/24",65 "203.0.113.0/24",66 "224.0.0.0/4",67 "240.0.0.0/4",68 "255.255.255.255/32",69];7071export function isBlockedIPv4(ip: string): boolean {72 return BLOCKED_V4.some((c) => inCidr4(ip, c));73}7475export function isBlockedIPv6(ip: string): boolean {76 const lower = ip.toLowerCase();77 if (lower === "::" || lower === "::1") return true;78 // IPv4-mapped ::ffff:a.b.c.d79 const mapped = lower.match(/^(?:0*:)*ffff:(\d+\.\d+\.\d+\.\d+)$/);80 if (mapped) return isBlockedIPv4(mapped[1]!);81 const mappedHex = lower.match(/^(?:0*:)*ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);82 if (mappedHex) {83 const a = parseInt(mappedHex[1]!, 16);84 const b = parseInt(mappedHex[2]!, 16);85 return isBlockedIPv4(`${a >> 8}.${a & 255}.${b >> 8}.${b & 255}`);86 }87 if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb")) return true; // link-local fe80::/1088 if (lower.startsWith("fc") || lower.startsWith("fd")) return true; // unique local fc00::/789 if (lower.startsWith("ff")) return true; // multicast90 if (lower.startsWith("64:ff9b:")) return true; // NAT6491 if (lower.startsWith("2001:db8:")) return true; // documentation92 if (lower.startsWith("::ffff:0:")) return true;93 return false;94}9596export function isBlockedIP(ip: string): boolean {97 const v = isIP(ip);98 if (v === 4) return isBlockedIPv4(ip);99 if (v === 6) return isBlockedIPv6(ip);100 return true;101}102103export function isBlockedHostname(hostname: string): boolean {104 const h = hostname.toLowerCase().replace(/\.$/, "");105 if (BLOCKED_HOSTNAMES.has(h)) return true;106 if (BLOCKED_SUFFIXES.some((s) => h.endsWith(s))) return true;107 if (!h.includes(".") && isIP(h) === 0) return true; // bare single-label hosts108 return false;109}110111export interface AllowedUrl {112 url: URL;113 hostname: string;114 /** Resolved public addresses (empty when the host is a literal IP that passed). */115 addresses: string[];116 dns_ms: number;117}118119export interface UrlPolicyOptions {120 /** Resolve DNS and validate every address. Default true. */121 resolve?: boolean;122 allowedSchemes?: string[];123}124125/** Parse + validate a URL. Throws `FetchaError(URL_NOT_ALLOWED | INVALID_REQUEST)`. */126export async function assertUrlAllowed(raw: string, opts: UrlPolicyOptions = {}): Promise<AllowedUrl> {127 let url: URL;128 try {129 url = new URL(raw);130 } catch {131 throw new FetchaError("INVALID_REQUEST", "The URL is malformed.");132 }133 const schemes = opts.allowedSchemes ?? ["http:", "https:"];134 if (!schemes.includes(url.protocol)) {135 throw new FetchaError("URL_NOT_ALLOWED", `Only http and https URLs are supported (got ${url.protocol.replace(":", "")}).`);136 }137 if (url.username || url.password) {138 throw new FetchaError("URL_NOT_ALLOWED", "Credentials in the URL are not allowed.");139 }140 const hostname = url.hostname.replace(/^\[|\]$/g, "");141 if (!hostname) throw new FetchaError("INVALID_REQUEST", "The URL has no host.");142 if (isBlockedHostname(hostname)) {143 throw new FetchaError("URL_NOT_ALLOWED", "Requests to local, private or internal hosts are not allowed.");144 }145 const literal = isIP(hostname);146 if (literal) {147 if (isBlockedIP(hostname)) {148 throw new FetchaError("URL_NOT_ALLOWED", "Requests to private or reserved IP addresses are not allowed.");149 }150 return { url, hostname, addresses: [hostname], dns_ms: 0 };151 }152 // Numeric-looking hosts (e.g. 0x7f000001, 2130706433) — reject outright.153 if (/^[0-9x.]+$/i.test(hostname)) {154 throw new FetchaError("URL_NOT_ALLOWED", "Numeric host encodings are not allowed.");155 }156 if (opts.resolve === false) return { url, hostname, addresses: [], dns_ms: 0 };157158 const t0 = performance.now();159 let records: Array<{ address: string; family: number }>;160 try {161 records = await resolveAll(hostname);162 } catch {163 throw new FetchaError("TARGET_UNAVAILABLE", "The target hostname could not be resolved.");164 }165 const dns_ms = Math.round(performance.now() - t0);166 if (!records.length) throw new FetchaError("TARGET_UNAVAILABLE", "The target hostname has no address records.");167 for (const r of records) {168 if (isBlockedIP(r.address)) {169 throw new FetchaError("URL_NOT_ALLOWED", "The target resolves to a private or reserved address.");170 }171 }172 return { url, hostname, addresses: records.map((r) => r.address), dns_ms };173}174175const FALLBACK_DNS_SERVERS = (process.env.FETCHA_DNS_FALLBACK ?? "1.1.1.1,8.8.8.8,9.9.9.9").split(",").map((s) => s.trim()).filter(Boolean);176177/**178 * Resolve every address of a hostname. Uses the system resolver first, then falls back to179 * public resolvers (some hosts run split-horizon/MagicDNS resolvers that fail on unrelated names).180 */181export async function resolveAll(hostname: string): Promise<Array<{ address: string; family: number }>> {182 const dnsMod = await import("node:dns");183 try {184 const recs = await dnsMod.promises.lookup(hostname, { all: true, verbatim: true });185 if (recs.length) return recs;186 } catch {187 /* fall through to public resolvers */188 }189 const resolver = new dnsMod.promises.Resolver({ timeout: 4000, tries: 2 });190 resolver.setServers(FALLBACK_DNS_SERVERS);191 const [v4, v6] = await Promise.allSettled([resolver.resolve4(hostname), resolver.resolve6(hostname)]);192 const out: Array<{ address: string; family: number }> = [];193 if (v4.status === "fulfilled") out.push(...v4.value.map((address) => ({ address, family: 4 })));194 if (v6.status === "fulfilled") out.push(...v6.value.map((address) => ({ address, family: 6 })));195 if (!out.length) throw new Error(`Unable to resolve ${hostname}`);196 return out;197}198