// author: simon-pierre boucher import { Agent, request } from "undici"; import type { LookupFunction } from "node:net"; import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib"; import { err, normalizeUrl, ok, tendrilError, type FetchResult, type RedirectHop, type Result, } from "@tendril/shared"; import { validateEgress, type Resolver } from "@tendril/egress"; import { buildHeaders } from "./headers.js"; const MAX_REDIRECTS = 5; const DEFAULT_MAX_BYTES = 20 * 1024 * 1024; const pinnedAddresses = new Map(); const pinnedLookup: LookupFunction = (hostname, options, callback) => { const pinned = pinnedAddresses.get(hostname); if (pinned === undefined) { callback(new Error(`no pinned address for ${hostname}`) as NodeJS.ErrnoException, ""); return; } const family = pinned.includes(":") ? 6 : 4; if (options.all === true) { callback(null, [{ address: pinned, family }]); } else { callback(null, pinned, family); } }; const agent = new Agent({ connections: 64, pipelining: 1, keepAliveTimeout: 30_000, keepAliveMaxTimeout: 120_000, bodyTimeout: 20_000, headersTimeout: 10_000, connect: { timeout: 8_000, rejectUnauthorized: true, lookup: pinnedLookup, }, }); export interface HttpFetchOptions { readonly timeout?: number; readonly maxBytes?: number; readonly headers?: Readonly>; readonly userAgent?: string; readonly resolver?: Resolver; } function decompress(buf: Buffer, encoding: string | undefined): Buffer { try { switch ((encoding ?? "").toLowerCase()) { case "gzip": return gunzipSync(buf); case "deflate": return inflateSync(buf); case "br": return brotliDecompressSync(buf); default: return buf; } } catch { return buf; } } async function readCapped(body: AsyncIterable, max: number): Promise { const chunks: Buffer[] = []; let total = 0; for await (const chunk of body) { total += chunk.length; if (total > max) return null; chunks.push(chunk); } return Buffer.concat(chunks); } const META_REFRESH_RE = /]+http-equiv=["']?refresh["']?[^>]*content=["'][^"']*url=([^"'>\s]+)/i; function firstHeader(value: string | string[] | undefined): string | undefined { if (Array.isArray(value)) return value[0]; return value; } /** * Tier 0 HTTP fetch (§3). Handles redirects manually so it can re-validate SSRF * on every hop (§16.5), record the full chain, detect meta-refresh redirects the * transport cannot see, and stop on a redirect loop. Connections are pinned to * the SSRF-validated IP to defeat DNS rebinding between validation and connect. */ export async function httpFetch(rawUrl: string, options: HttpFetchOptions = {}): Promise> { const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; const redirects: RedirectHop[] = []; const seen = new Set(); let current = rawUrl; for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { const norm = normalizeUrl(current); if (!norm.ok) return norm; if (seen.has(norm.value)) return err(tendrilError("ERR_REDIRECT_LOOP", { details: { url: current } })); seen.add(norm.value); const safe = await validateEgress(current, options.resolver); if (!safe.ok) return safe; const address = safe.value.addresses[0]; if (address === undefined) return err(tendrilError("ERR_SSRF_BLOCKED", { details: { host: safe.value.host } })); pinnedAddresses.set(safe.value.host, address); let res: Awaited>; try { res = await request(current, { dispatcher: agent, method: "GET", headersTimeout: options.timeout ?? 10_000, bodyTimeout: options.timeout ?? 20_000, headers: buildHeaders(safe.value.host, options.headers, options.userAgent), }); } catch (cause) { return err(tendrilError("ERR_TARGET_5XX", { message: "Transport error", details: { url: current }, cause })); } const status = res.statusCode; const location = firstHeader(res.headers["location"]); if (status >= 300 && status < 400 && location !== undefined && location !== "") { res.body.dump().catch(() => undefined); if (hop === MAX_REDIRECTS) return err(tendrilError("ERR_REDIRECT_LOOP", { details: { url: current } })); let next: string; try { next = new URL(location, current).toString(); } catch { return err(tendrilError("ERR_INVALID_URL", { details: { location } })); } redirects.push({ from: current, to: next, status }); current = next; continue; } const rawBody = await readCapped(res.body, maxBytes); if (rawBody === null) return err(tendrilError("ERR_TOO_LARGE", { details: { max: maxBytes } })); const contentType = (firstHeader(res.headers["content-type"]) ?? "").toLowerCase(); const encoding = firstHeader(res.headers["content-encoding"]); const decoded = decompress(rawBody, encoding); const bodyText = decoded.toString("utf8"); const isHtml = contentType.includes("text/html") || contentType.includes("xhtml") || contentType === ""; if (isHtml && hop < MAX_REDIRECTS) { const meta = META_REFRESH_RE.exec(bodyText); if (meta?.[1] !== undefined) { let next: string; try { next = new URL(meta[1], current).toString(); } catch { next = ""; } if (next !== "" && !seen.has(normalizeUrl(next).ok ? (normalizeUrl(next) as { value: string }).value : next)) { redirects.push({ from: current, to: next, status: 200 }); current = next; continue; } } } const headers: Record = {}; for (const [key, value] of Object.entries(res.headers)) { headers[key] = Array.isArray(value) ? value.join(", ") : (value ?? ""); } const result: FetchResult = { tier: "http", status, finalUrl: current, contentType, body: bodyText, bodyBytes: decoded.length, redirects, headers, }; return ok(result); } return err(tendrilError("ERR_REDIRECT_LOOP", { details: { url: rawUrl } })); }