TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import dns from "node:dns/promises";2import net from "node:net";34/**5 * SSRF protection. Every URL the engine fetches — seed, discovered or redirect hop —6 * must pass `assertUrlAllowed()` before any network activity. We validate the scheme,7 * the hostname, and every resolved address. `safeLookup` is used by the HTTP dispatcher8 * so the socket connects only to a validated address (defeats DNS rebinding).9 */1011const BLOCKED_HOSTNAMES = new Set([12 "localhost",13 "localhost.localdomain",14 "ip6-localhost",15 "ip6-loopback",16 "metadata.google.internal",17 "metadata",18 "instance-data",19 "kubernetes.default",20 "kubernetes.default.svc",21]);2223const BLOCKED_SUFFIXES = [".localhost", ".local", ".internal", ".localdomain", ".home.arpa", ".in-addr.arpa", ".ip6.arpa", ".maclustr.io", ".ts.net"];2425function ipv4ToInt(ip: string): number {26 const p = ip.split(".").map((x) => Number(x));27 return ((p[0]! << 24) >>> 0) + (p[1]! << 16) + (p[2]! << 8) + p[3]!;28}2930function inCidr4(ip: string, cidr: string): boolean {31 const [base, bitsStr] = cidr.split("/");32 const bits = Number(bitsStr);33 const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;34 return ((ipv4ToInt(ip) & mask) >>> 0) === ((ipv4ToInt(base!) & mask) >>> 0);35}3637const BLOCKED_V4 = [38 "0.0.0.0/8",39 "10.0.0.0/8",40 "100.64.0.0/10",41 "127.0.0.0/8",42 "169.254.0.0/16",43 "172.16.0.0/12",44 "192.0.0.0/24",45 "192.0.2.0/24",46 "192.168.0.0/16",47 "198.18.0.0/15",48 "198.51.100.0/24",49 "203.0.113.0/24",50 "224.0.0.0/4",51 "240.0.0.0/4",52 "255.255.255.255/32",53];5455export function isBlockedIPv4(ip: string): boolean {56 return BLOCKED_V4.some((c) => inCidr4(ip, c));57}5859export function isBlockedIPv6(ip: string): boolean {60 const lower = ip.toLowerCase();61 if (lower === "::" || lower === "::1") return true;62 const mapped = lower.match(/^(?:0*:)*ffff:(\d+\.\d+\.\d+\.\d+)$/);63 if (mapped) return isBlockedIPv4(mapped[1]!);64 const mappedHex = lower.match(/^(?:0*:)*ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);65 if (mappedHex) {66 const a = parseInt(mappedHex[1]!, 16);67 const b = parseInt(mappedHex[2]!, 16);68 return isBlockedIPv4(`${a >> 8}.${a & 255}.${b >> 8}.${b & 255}`);69 }70 if (/^fe[89ab]/.test(lower)) return true; // link-local71 if (lower.startsWith("fc") || lower.startsWith("fd")) return true; // unique local72 if (lower.startsWith("ff")) return true; // multicast73 // NAT64 (RFC 6052 well-known prefix 64:ff9b::/96, and the local-use 64:ff9b:1::/48): the last 32 bits74 // embed an IPv4 address — apply the IPv4 policy to it instead of blocking the whole prefix, so that75 // IPv6-only networks (464XLAT hotspots) can still reach public IPv4 hosts while private IPv4 stays blocked.76 const nat64 = lower.match(/^64:ff9b:(?:1:[0-9a-f]{1,4}:[0-9a-f]{1,4}:)?(?:0*:)*([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);77 if (nat64) {78 const a = parseInt(nat64[1]!, 16);79 const b = parseInt(nat64[2]!, 16);80 return isBlockedIPv4(`${a >> 8}.${a & 255}.${b >> 8}.${b & 255}`);81 }82 if (lower.startsWith("64:ff9b:")) return true; // malformed NAT64 form83 if (lower.startsWith("2001:db8:")) return true; // documentation84 return false;85}8687export function isBlockedIP(ip: string): boolean {88 const v = net.isIP(ip);89 if (v === 4) return isBlockedIPv4(ip);90 if (v === 6) return isBlockedIPv6(ip);91 return true;92}9394export function isBlockedHostname(hostname: string): boolean {95 const h = hostname.toLowerCase().replace(/\.$/, "");96 if (BLOCKED_HOSTNAMES.has(h)) return true;97 if (BLOCKED_SUFFIXES.some((s) => h.endsWith(s))) return true;98 if (!h.includes(".") && net.isIP(h) === 0) return true; // bare single-label hosts99 return false;100}101102export class UrlPolicyError extends Error {103 constructor(104 message: string,105 public readonly reason: string,106 ) {107 super(message);108 this.name = "UrlPolicyError";109 }110}111112export interface AllowedUrl {113 url: URL;114 hostname: string;115 addresses: string[];116}117118const DNS_FALLBACK_SERVERS = ["1.1.1.1", "8.8.8.8", "9.9.9.9"];119120/** Resolve all A/AAAA records, falling back to public resolvers when the system resolver fails. */121export async function resolveAll(hostname: string): Promise<string[]> {122 try {123 const res = await dns.lookup(hostname, { all: true, verbatim: true });124 const addrs = res.map((r) => r.address);125 if (addrs.length) return addrs;126 } catch {127 // fall through128 }129 const r = new dns.Resolver({ timeout: 4000, tries: 2 });130 r.setServers(DNS_FALLBACK_SERVERS);131 const out: string[] = [];132 const [a, aaaa] = await Promise.allSettled([r.resolve4(hostname), r.resolve6(hostname)]);133 if (a.status === "fulfilled") out.push(...a.value);134 if (aaaa.status === "fulfilled") out.push(...aaaa.value);135 return out;136}137138export interface UrlPolicyOptions {139 /** Resolve DNS and validate each address. Default true. */140 resolve?: boolean;141 /** Allow http:// (default true). */142 allowHttp?: boolean;143}144145/** Validate a URL (scheme, host, resolved addresses). Throws UrlPolicyError. */146export async function assertUrlAllowed(input: string | URL, opts: UrlPolicyOptions = {}): Promise<AllowedUrl> {147 let url: URL;148 try {149 url = typeof input === "string" ? new URL(input) : input;150 } catch {151 throw new UrlPolicyError("Malformed URL.", "malformed");152 }153 if (url.protocol !== "https:" && !(url.protocol === "http:" && (opts.allowHttp ?? true))) {154 throw new UrlPolicyError(`Scheme ${url.protocol} not allowed.`, "scheme");155 }156 if (url.username || url.password) throw new UrlPolicyError("Credentials in URL are not allowed.", "credentials");157 const hostname = url.hostname.replace(/^\[|\]$/g, "");158 if (!hostname) throw new UrlPolicyError("Missing host.", "host");159 if (isBlockedHostname(hostname)) throw new UrlPolicyError(`Host ${hostname} is not allowed.`, "blocked_host");160 if (net.isIP(hostname)) {161 if (isBlockedIP(hostname)) throw new UrlPolicyError(`Address ${hostname} is not allowed.`, "blocked_ip");162 return { url, hostname, addresses: [hostname] };163 }164 if (opts.resolve === false) return { url, hostname, addresses: [] };165 const addresses = await resolveAll(hostname);166 if (!addresses.length) throw new UrlPolicyError(`DNS resolution failed for ${hostname}.`, "dns");167 const bad = addresses.find((a) => isBlockedIP(a));168 if (bad) throw new UrlPolicyError(`Host ${hostname} resolves to a blocked address (${bad}).`, "blocked_ip");169 return { url, hostname, addresses };170}171172/**173 * `lookup` implementation for net.connect / undici connect options: only returns174 * addresses that pass the policy, so the TCP connection can never reach a private range175 * even if DNS answers differently than during validation (rebinding).176 */177export function safeLookup(178 hostname: string,179 options: unknown,180 callback: (err: NodeJS.ErrnoException | null, address: string | { address: string; family: number }[], family?: number) => void,181): void {182 const all = typeof options === "object" && options !== null && (options as { all?: boolean }).all === true;183 if (isBlockedHostname(hostname)) {184 callback(Object.assign(new Error(`blocked host ${hostname}`), { code: "EBLOCKED" }), all ? [] : "", 4);185 return;186 }187 resolveAll(hostname)188 .then((addrs) => {189 const ok = addrs.filter((a) => !isBlockedIP(a));190 if (!ok.length) {191 callback(Object.assign(new Error(`no allowed address for ${hostname}`), { code: "EBLOCKED" }), all ? [] : "", 4);192 return;193 }194 if (all) callback(null, ok.map((a) => ({ address: a, family: net.isIP(a) })));195 else callback(null, ok[0]!, net.isIP(ok[0]!));196 })197 .catch((err) => callback(err, all ? [] : "", 4));198}199