spb/tools-spboucher-ai
Public
1import { lookup } from "node:dns/promises";2import { isIP } from "node:net";3import { Agent } from "undici";45// Some vendors (e.g. Google properties) send response headers above undici's 16 KiB default → "Headers Overflow".6const dispatcher = new Agent({ maxHeaderSize: 256 * 1024, connect: { timeout: 10_000 } });78const TRACKING = /^(utm_\w+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref|ref_src|source|_hsenc|_hsmi|yclid|twclid|si)$/i;910/** Canonical form used for deduplication and storage: https, lower-case host, no default port, no tracking params, no fragment, sorted query, no trailing slash (except root). */11export function normalizeUrl(input: string): string {12 const u = new URL(input.trim());13 if (!/^https?:$/.test(u.protocol)) throw new Error("unsupported scheme");14 u.hash = "";15 u.hostname = u.hostname.toLowerCase().replace(/\.$/, "");16 if ((u.protocol === "https:" && u.port === "443") || (u.protocol === "http:" && u.port === "80")) u.port = "";17 const keep = [...u.searchParams.entries()].filter(([k]) => !TRACKING.test(k)).sort(([a], [b]) => a.localeCompare(b));18 u.search = keep.length ? "?" + new URLSearchParams(keep).toString() : "";19 u.username = ""; u.password = "";20 if (u.pathname !== "/" && u.pathname.endsWith("/")) u.pathname = u.pathname.replace(/\/+$/, "");21 return u.toString();22}23export function sameSite(a: string, b: string) {24 const h = (x: string) => new URL(x).hostname.replace(/^www\./, "");25 return h(a) === h(b);26}2728/** RFC1918, loopback, link-local, CGNAT, ULA, multicast, metadata — anything that must never be fetched server-side. */29export function isPrivateIp(ip: string): boolean {30 if (isIP(ip) === 4) {31 const [a, b] = ip.split(".").map(Number);32 return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;33 }34 const v6 = ip.toLowerCase();35 if (v6 === "::1" || v6 === "::") return true;36 if (v6.startsWith("fc") || v6.startsWith("fd") || v6.startsWith("fe8") || v6.startsWith("fe9") || v6.startsWith("fea") || v6.startsWith("feb") || v6.startsWith("ff")) return true;37 if (v6.startsWith("::ffff:")) return isPrivateIp(v6.slice(7));38 return false;39}40const BLOCKED_HOSTS = new Set(["localhost", "metadata.google.internal", "169.254.169.254"]);41export async function assertPublicHost(hostname: string) {42 const h = hostname.toLowerCase();43 if (BLOCKED_HOSTS.has(h) || h.endsWith(".local") || h.endsWith(".internal") || h.endsWith(".maclustr.io") || h.endsWith(".localhost") || !h.includes(".")) throw new Error(`blocked host ${h}`);44 if (isIP(h)) { if (isPrivateIp(h)) throw new Error(`private address ${h}`); return; }45 const addrs = await lookup(h, { all: true, verbatim: true }).catch(() => []);46 if (!addrs.length) throw new Error(`DNS resolution failed for ${h}`);47 for (const a of addrs) if (isPrivateIp(a.address)) throw new Error(`host ${h} resolves to private address`);48}4950export type SafeFetchResult = { ok: boolean; status: number; finalUrl: string; redirected: boolean; hops: string[]; contentType: string | null; body: string | null; error?: string; durationMs: number };51const UA = () => process.env.OUTBOUND_USER_AGENT ?? "tools-spboucher-ai-linkcheck/1.0"; // plain token: some CDNs (Apple) 404 on UAs containing URLs5253/** SSRF-safe fetch: validates every hop, caps redirects (5), body size (1 MiB) and time (15 s). Never sends credentials. */54export async function safeFetch(url: string, opts: { method?: "GET" | "HEAD"; maxBytes?: number; timeoutMs?: number } = {}): Promise<SafeFetchResult> {55 const started = Date.now(); const hops: string[] = []; let current = url; let redirected = false;56 const maxBytes = opts.maxBytes ?? 1_048_576; const timeoutMs = opts.timeoutMs ?? 20_000;57 try {58 for (let i = 0; i < 6; i++) {59 const u = new URL(current);60 if (!/^https?:$/.test(u.protocol)) throw new Error("unsupported scheme");61 await assertPublicHost(u.hostname);62 hops.push(current);63 const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs);64 let res: Response;65 try {66 res = await fetch(current, { method: opts.method ?? "GET", redirect: "manual", signal: ctrl.signal, dispatcher, headers: { "user-agent": UA(), accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8" }, credentials: "omit" as RequestCredentials } as RequestInit);67 } finally { clearTimeout(t); }68 if (res.status >= 300 && res.status < 400) {69 const loc = res.headers.get("location"); if (!loc) return { ok: false, status: res.status, finalUrl: current, redirected, hops, contentType: null, body: null, error: "redirect without location", durationMs: Date.now() - started };70 const next = new URL(loc, current).toString(); const nu = new URL(next);71 // Many sites bounce https → http (same host) for a trailing slash and immediately back to https. Allow same-host72 // downgrades inside the chain; a cross-host downgrade is rejected and a final http URL is reported as insecure below.73 if (nu.protocol === "http:" && u.protocol === "https:" && nu.hostname !== u.hostname) throw new Error("https→http downgrade redirect rejected");74 if (nu.toString() === current) { // self-redirect loop (bot challenge) — report as soft block75 return { ok: false, status: 403, finalUrl: current, redirected, hops, contentType: null, body: null, error: "self-redirect loop (likely bot challenge)", durationMs: Date.now() - started };76 }77 redirected = true; current = next; continue;78 }79 let body: string | null = null;80 if ((opts.method ?? "GET") === "GET" && res.body) {81 const reader = res.body.getReader(); const chunks: Uint8Array[] = []; let size = 0;82 while (true) { const { done, value } = await reader.read(); if (done) break; size += value.byteLength; if (size > maxBytes) { reader.cancel().catch(() => {}); break; } chunks.push(value); }83 body = new TextDecoder("utf-8", { fatal: false }).decode(Buffer.concat(chunks));84 }85 const insecure = new URL(url).protocol === "https:" && u.protocol === "http:";86 return { ok: res.status >= 200 && res.status < 300 && !insecure, status: res.status, finalUrl: current, redirected, hops, contentType: res.headers.get("content-type"), body, error: insecure ? "final URL is not https" : undefined, durationMs: Date.now() - started };87 }88 throw new Error("too many redirects");89 } catch (e) {90 const err = e as Error & { cause?: { code?: string; message?: string; errors?: { code?: string }[] } };91 const cause = err.cause ? (err.cause.code ?? err.cause.errors?.map((x) => x.code).join("|") ?? err.cause.message) : null;92 return { ok: false, status: 0, finalUrl: current, redirected, hops, contentType: null, body: null, error: err.name === "AbortError" ? "timeout" : cause ? `${err.message} (${String(cause).slice(0, 80)})` : err.message, durationMs: Date.now() - started };93 }94}9596/** Extract <title> and canonical link from HTML without a parser (cheap, best-effort). Input is untrusted. */97export function extractHtmlMeta(html: string): { title: string | null; canonical: string | null; description: string | null } {98 const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.replace(/\s+/g, " ").trim().slice(0, 300) ?? null;99 const canonical = html.match(/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/i)?.[1] ?? html.match(/<link[^>]+href=["']([^"']+)["'][^>]*rel=["']canonical["']/i)?.[1] ?? null;100 const description = html.match(/<meta[^>]+name=["']description["'][^>]*content=["']([^"']*)["']/i)?.[1]?.trim().slice(0, 500) ?? null;101 return { title: title ? decodeEntities(title) : null, canonical, description: description ? decodeEntities(description) : null };102}103function decodeEntities(s: string) { return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'|'/g, "'").replace(/ /g, " "); }104