import { isIP } from "node:net"; import { lookup } from "node:dns/promises"; import { RateLimiter, backoffMs } from "./ratelimit.js"; import { redactUrl } from "./redact.js"; export interface HttpRequestOptions { method?: "GET" | "POST" | "HEAD"; headers?: Record; body?: string; timeoutMs?: number; /** Use conditional requests (ETag / If-Modified-Since) keyed by URL. Default true for GET. */ conditional?: boolean; retries?: number; /** Accept 304 as success and return cached body. */ acceptNotModified?: boolean; } export interface HttpResponse { status: number; ok: boolean; notModified: boolean; headers: Record; text: string; url: string; durationMs: number; fromCache: boolean; } export class HttpError extends Error { constructor( message: string, public status: number, public url: string, public retryAfterMs: number | null = null, ) { super(message); this.name = "HttpError"; } } interface CacheEntry { etag: string | null; lastModified: string | null; body: string; headers: Record; } export interface HttpClientOptions { userAgent: string; limiter?: RateLimiter; defaultTimeoutMs?: number; /** Called for every request (metrics). */ onRequest?: (info: { host: string; status: number; durationMs: number; ok: boolean }) => void; fetchImpl?: typeof fetch; } /** * Polite HTTP client: central per-host rate limiter, ETag/If-Modified-Since cache, * bounded retries with jittered backoff, secret-free URLs in errors, identified User-Agent. */ export class HttpClient { private cache = new Map(); readonly limiter: RateLimiter; private fetchImpl: typeof fetch; constructor(private opts: HttpClientOptions) { this.limiter = opts.limiter ?? new RateLimiter(); this.fetchImpl = opts.fetchImpl ?? fetch; } async request(url: string, options: HttpRequestOptions = {}): Promise { const method = options.method ?? "GET"; const u = new URL(url); const retries = options.retries ?? 2; const conditional = options.conditional ?? method === "GET"; let attempt = 0; for (;;) { await this.limiter.acquire(u.host); const started = Date.now(); const headers: Record = { "user-agent": this.opts.userAgent, accept: "application/json, text/xml, application/xml, text/html, text/csv, text/plain;q=0.9, */*;q=0.8", "accept-encoding": "gzip, br", ...(options.headers ?? {}), }; const cached = conditional ? this.cache.get(url) : undefined; if (cached?.etag) headers["if-none-match"] = cached.etag; if (cached?.lastModified) headers["if-modified-since"] = cached.lastModified; const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), options.timeoutMs ?? this.opts.defaultTimeoutMs ?? 20_000); try { const res = await this.fetchImpl(url, { method, headers, body: options.body, signal: ctrl.signal, redirect: "follow" }); const durationMs = Date.now() - started; const resHeaders: Record = {}; res.headers.forEach((v, k) => (resHeaders[k] = v)); this.opts.onRequest?.({ host: u.host, status: res.status, durationMs, ok: res.ok || res.status === 304 }); if (res.status === 304 && cached) { return { status: 304, ok: true, notModified: true, headers: resHeaders, text: cached.body, url, durationMs, fromCache: true }; } if (res.status === 429 || res.status >= 500) { const ra = res.headers.get("retry-after"); const retryAfterMs = ra ? (Number.isFinite(Number(ra)) ? Number(ra) * 1000 : Math.max(0, Date.parse(ra) - Date.now())) : null; if (attempt < retries) { attempt++; await sleep(retryAfterMs ?? backoffMs(attempt, 750, 30_000)); continue; } throw new HttpError(`HTTP ${res.status} from ${redactUrl(url)}`, res.status, redactUrl(url), retryAfterMs); } const text = await res.text(); if (!res.ok) throw new HttpError(`HTTP ${res.status} from ${redactUrl(url)}`, res.status, redactUrl(url)); if (conditional) { const etag = res.headers.get("etag"); const lastModified = res.headers.get("last-modified"); if (etag || lastModified) this.cache.set(url, { etag, lastModified, body: text, headers: resHeaders }); } return { status: res.status, ok: true, notModified: false, headers: resHeaders, text, url, durationMs, fromCache: false }; } catch (err) { if (err instanceof HttpError) throw err; if (attempt < Math.max(retries, 3)) { attempt++; await sleep(1000 + backoffMs(attempt, 1500, 20_000)); // DNS/TLS hiccups at boot need more than a few hundred ms continue; } const msg = err instanceof Error ? `${err.message}${(err as { cause?: { code?: string } }).cause?.code ? ` [${(err as { cause?: { code?: string } }).cause?.code}]` : ""}` : String(err); throw new HttpError(`request failed: ${msg} (${redactUrl(url)})`, 0, redactUrl(url)); } finally { clearTimeout(timer); } } } async getText(url: string, options?: HttpRequestOptions): Promise { return this.request(url, { ...options, method: "GET" }); } async getJson(url: string, options?: HttpRequestOptions): Promise<{ data: T; response: HttpResponse }> { const response = await this.request(url, { ...options, method: "GET", headers: { accept: "application/json", ...(options?.headers ?? {}) } }); try { return { data: JSON.parse(response.text) as T, response }; } catch { throw new HttpError(`invalid JSON from ${redactUrl(url)}`, response.status, redactUrl(url)); } } cacheSize(): number { return this.cache.size; } } export function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } const PRIVATE_V4 = [ /^10\./, /^127\./, /^0\./, /^169\.254\./, /^172\.(1[6-9]|2\d|3[01])\./, /^192\.168\./, /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./, /^22[4-9]\./, /^2[3-5]\d\./, ]; export function isPrivateAddress(ip: string): boolean { const v = isIP(ip); if (v === 4) return PRIVATE_V4.some((re) => re.test(ip)); if (v === 6) { const low = ip.toLowerCase(); return low === "::1" || low === "::" || low.startsWith("fc") || low.startsWith("fd") || low.startsWith("fe80") || low.startsWith("::ffff:"); } return true; } /** * SSRF guard for user-supplied URLs (discovery): public http(s) only, no private/link-local/metadata * targets, resolved addresses checked too. Throws on violation. */ export async function assertPublicUrl(input: string): Promise { const u = new URL(input); if (u.protocol !== "http:" && u.protocol !== "https:") throw new Error("only http(s) URLs are allowed"); if (u.username || u.password) throw new Error("credentials in URL are not allowed"); const host = u.hostname.toLowerCase(); if (host === "localhost" || host.endsWith(".local") || host.endsWith(".internal") || host.endsWith(".maclustr.io") || host === "metadata.google.internal") throw new Error("internal hostnames are not allowed"); if (isIP(host)) { if (isPrivateAddress(host)) throw new Error("private addresses are not allowed"); return u; } const addrs = await lookup(host, { all: true }); if (!addrs.length) throw new Error("hostname does not resolve"); for (const a of addrs) if (isPrivateAddress(a.address)) throw new Error("hostname resolves to a private address"); return u; }