import { Agent, ProxyAgent, request, type Dispatcher } from "undici"; import { CookieJar } from "./cookies"; import { TLS_CIPHERS, TLS_CURVES, TLS_SIGALGS, buildHeaders, pickProfile, type FingerprintProfile } from "./fingerprint"; import { ProviderError, type ProviderId, type ProviderRequest, type ProviderResponse, type ProviderTiming, type ProxyEndpoint } from "./types"; export type { ProxyEndpoint } from "./types"; const HTTP2_ENABLED = process.env.FETCHA_HTTP2 !== "0"; const agentCache = new Map(); interface DispatcherOpts { endpoint: ProxyEndpoint | null; tls: FingerprintProfile["tls"]; h2: boolean; timeoutMs: number; } function tlsOptions(tls: FingerprintProfile["tls"]) { return { ciphers: TLS_CIPHERS[tls], ecdhCurve: TLS_CURVES[tls], sigalgs: TLS_SIGALGS[tls], minVersion: "TLSv1.2" as const, maxVersion: "TLSv1.3" as const, honorCipherOrder: false, // Browsers never send the legacy renegotiation info in a way Node does by default; keep session reuse on. sessionTimeout: 300, }; } function dispatcherFor(o: DispatcherOpts): Dispatcher { const key = `${o.endpoint ? `${o.endpoint.host}:${o.endpoint.port}:${o.endpoint.username}` : "direct"}|${o.tls}|${o.h2 ? "h2" : "h1"}`; const hit = agentCache.get(key); if (hit) { hit.lastUsed = Date.now(); return hit.d; } const common = { headersTimeout: 120_000, bodyTimeout: 120_000, keepAliveTimeout: 15_000, connections: 128, allowH2: o.h2, // Browser-like ALPN and TLS parameters for the origin connection. connect: { timeout: Math.min(o.timeoutMs, 20_000), ...tlsOptions(o.tls), ALPNProtocols: o.h2 ? ["h2", "http/1.1"] : ["http/1.1"] }, }; let d: Dispatcher; if (!o.endpoint) { d = new Agent(common); } else { const token = "Basic " + Buffer.from(`${o.endpoint.username}:${o.endpoint.password}`).toString("base64"); d = new ProxyAgent({ uri: `http://${o.endpoint.host}:${o.endpoint.port}`, token, ...common, // TLS parameters used for the tunnelled origin connection (through CONNECT). requestTls: { ...tlsOptions(o.tls), ALPNProtocols: o.h2 ? ["h2", "http/1.1"] : ["http/1.1"] }, }); } // Bound the cache: sticky sessions create many usernames. Evict least recently used. if (agentCache.size >= 400) { let oldest: string | null = null; let t = Infinity; for (const [k, v] of agentCache) if (v.lastUsed < t) (t = v.lastUsed), (oldest = k); if (oldest) { agentCache.get(oldest)?.d.close().catch(() => {}); agentCache.delete(oldest); } } agentCache.set(key, { d, lastUsed: Date.now() }); return d; } const HOP_BY_HOP = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "host", "content-length", "cookie"]); /** Header names as browsers spell them in HTTP/1.1 (HTTP/2 lower-cases everything anyway). */ const SPECIAL_CASE: Record = { "sec-ch-ua": "Sec-CH-UA", "sec-ch-ua-mobile": "Sec-CH-UA-Mobile", "sec-ch-ua-platform": "Sec-CH-UA-Platform", te: "TE", dnt: "DNT", "x-requested-with": "X-Requested-With" }; function titleCase(name: string): string { if (SPECIAL_CASE[name]) return SPECIAL_CASE[name]!; return name .split("-") .map((p) => (p ? p[0]!.toUpperCase() + p.slice(1) : p)) .join("-"); } function flattenHeaders(h: Record): { flat: Record; setCookie: string[] } { const flat: Record = {}; let setCookie: string[] = []; for (const [k, v] of Object.entries(h)) { if (v === undefined) continue; const lk = k.toLowerCase(); if (lk === "set-cookie") { setCookie = Array.isArray(v) ? v : [v]; flat[lk] = setCookie.join(", "); continue; } flat[lk] = Array.isArray(v) ? v.join(", ") : v; } return { flat, setCookie }; } /** * Execute an HTTP request, optionally through an upstream proxy tunnel, emulating a real browser's * header order, client hints, TLS preferences and HTTP/2 usage. Redirects are followed manually so * every hop can be validated by the caller (SSRF) and cookies set along the way are replayed. */ export async function executeHttp(provider: ProviderId, endpoint: ProxyEndpoint | null, req: ProviderRequest): Promise { const started = performance.now(); const deadline = started + req.timeoutMs; const profile = req.profile ?? pickProfile({ device: null, seed: req.sessionKey ?? null }); const jar = req.jar ?? new CookieJar(); let h2 = HTTP2_ENABLED && req.http2 !== false; let url = new URL(req.url); let method = req.method; let body: string | Buffer | undefined = req.body; let redirects = 0; let bytesOut = 0; let bytesIn = 0; let originMs = 0; let referer: string | null = req.referer ?? null; let retriedH1 = false; // Caller-supplied cookies (explicit header) become host-only jar cookies for the first URL. const overrideHeaders: Record = {}; for (const [k, v] of Object.entries(req.headers)) { const lk = k.toLowerCase(); if (lk === "cookie") { jar.addRawCookieHeader(v, url); continue; } if (HOP_BY_HOP.has(lk)) continue; overrideHeaders[lk] = v; } for (;;) { const remaining = deadline - performance.now(); if (remaining <= 0) throw new ProviderError(provider, "timeout", "Timed out before the target responded."); const hasBody = body !== undefined && method !== "GET" && method !== "HEAD"; const ordered = buildHeaders(profile, { locale: req.locale, country: req.geo.country, referer, overrides: overrideHeaders, hasBody, contentType: hasBody ? (typeof body === "string" ? "application/json" : "application/octet-stream") : null, }); const cookie = jar.headerFor(url); if (cookie) { // Browsers send Cookie after Accept-Language / before Priority. const idx = ordered.findIndex(([k]) => k === "priority"); ordered.splice(idx === -1 ? ordered.length : idx, 0, ["cookie", cookie]); } const headers: Record = {}; for (const [k, v] of ordered) headers[titleCase(k)] = v; const dispatcher = dispatcherFor({ endpoint, tls: profile.tls, h2, timeoutMs: req.timeoutMs }); const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), remaining); const t0 = performance.now(); let res: Awaited>; try { res = await request(url, { method, headers, body: hasBody ? body : undefined, dispatcher, signal: ac.signal, }); } catch (e) { clearTimeout(timer); const err = classifyError(provider, e); // Some origins/proxies mis-handle h2 through CONNECT tunnels: fall back to HTTP/1.1 once. if (h2 && !retriedH1 && isH2Failure(e) && deadline - performance.now() > 1000) { retriedH1 = true; h2 = false; continue; } throw err; } const tFirstByte = performance.now(); bytesOut += approxRequestBytes(method, url, headers, hasBody ? body : undefined); const { flat: resHeaders, setCookie } = flattenHeaders(res.headers as Record); jar.storeFromHeaders(setCookie.length ? setCookie : resHeaders["set-cookie"], url); // Proxy-level auth/quota errors surface as 407 from the gateway before reaching the target. if (endpoint && res.statusCode === 407) { clearTimeout(timer); await res.body.dump().catch(() => {}); throw new ProviderError(provider, "auth", "Upstream proxy rejected the credentials.", { status: 407 }); } const isRedirect = [301, 302, 303, 307, 308].includes(res.statusCode) && !!resHeaders["location"]; if (isRedirect && req.followRedirects) { await res.body.dump().catch(() => {}); clearTimeout(timer); redirects += 1; originMs += tFirstByte - t0; if (redirects > req.maxRedirects) throw new ProviderError(provider, "redirect", `Exceeded ${req.maxRedirects} redirects.`); const next = new URL(resHeaders["location"]!, url); if (req.onRedirect) await req.onRedirect(next.toString()); // Browsers send the previous URL as referer on redirects (same-origin: full URL; cross-origin: origin only). referer = next.origin === url.origin ? url.toString() : url.protocol === "https:" && next.protocol !== "https:" ? null : url.origin + "/"; url = next; if (res.statusCode === 303 || ((res.statusCode === 301 || res.statusCode === 302) && method === "POST")) { method = "GET"; body = undefined; } continue; } // Read body with size cap. const chunks: Buffer[] = []; let total = 0; try { for await (const chunk of res.body) { const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); total += buf.length; if (total > req.maxResponseBytes) { ac.abort(); throw new ProviderError(provider, "too_large", `Response exceeded ${req.maxResponseBytes} bytes.`); } chunks.push(buf); } } catch (e) { clearTimeout(timer); if (e instanceof ProviderError) throw e; throw classifyError(provider, e); } clearTimeout(timer); const tEnd = performance.now(); const raw = Buffer.concat(chunks); bytesIn += total + approxHeaderBytes(resHeaders); originMs += tFirstByte - t0; const decoded = await decodeBody(raw, resHeaders["content-encoding"]); const tDecoded = performance.now(); const totalMs = Math.round(tDecoded - started); const timing: ProviderTiming = { dns_ms: 0, proxy_connect_ms: 0, tls_ms: 0, origin_ms: Math.round(originMs), processing_ms: Math.round(tDecoded - tEnd), total_ms: totalMs, }; delete resHeaders["content-encoding"]; return { status: res.statusCode, headers: resHeaders, body: decoded, finalUrl: url.toString(), redirects, bytesIn, bytesOut, timing, profileId: profile.id, protocol: h2 ? "h2?" : "http/1.1", }; } } function isH2Failure(e: unknown): boolean { const err = e as { code?: string; message?: string; cause?: { code?: string; message?: string } }; const code = err.code ?? err.cause?.code ?? ""; const msg = `${err.message ?? ""} ${err.cause?.message ?? ""}`; return /HTTP\/2|h2|ERR_HTTP2|NGHTTP2|GOAWAY|RST_STREAM|PROTOCOL_ERROR|UND_ERR_INVALID_ARG/i.test(code + " " + msg) || code === "ERR_HTTP2_ERROR" || code === "ECONNRESET" && /h2|http2/i.test(msg); } function approxRequestBytes(method: string, url: URL, headers: Record, body?: string | Buffer): number { let n = method.length + url.pathname.length + url.search.length + 12; for (const [k, v] of Object.entries(headers)) n += k.length + v.length + 4; if (body) n += typeof body === "string" ? Buffer.byteLength(body) : body.length; return n; } function approxHeaderBytes(headers: Record): number { let n = 16; for (const [k, v] of Object.entries(headers)) n += k.length + v.length + 4; return n; } async function decodeBody(raw: Buffer, encoding?: string): Promise { if (!encoding || raw.length === 0) return raw; const zlib = await import("node:zlib"); const { promisify } = await import("node:util"); // Handle stacked encodings ("gzip, br") right-to-left. const encodings = encoding.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean).reverse(); let buf = raw; for (const enc of encodings) { try { switch (enc) { case "gzip": case "x-gzip": buf = await promisify(zlib.gunzip)(buf); break; case "deflate": buf = await promisify(zlib.inflate)(buf).catch(() => promisify(zlib.inflateRaw)(buf)); break; case "br": buf = await promisify(zlib.brotliDecompress)(buf); break; case "zstd": { const z = zlib as unknown as { zstdDecompress?: (b: Buffer, cb: (e: Error | null, r: Buffer) => void) => void }; if (typeof z.zstdDecompress === "function") buf = await promisify(z.zstdDecompress)(buf); break; } case "identity": break; default: return buf; } } catch { return buf; } } return buf; } function classifyError(provider: ProviderId, e: unknown): ProviderError { const err = e as { name?: string; code?: string; message?: string; cause?: { code?: string; message?: string } }; const code = err.code ?? err.cause?.code ?? ""; const msg = err.message ?? err.cause?.message ?? String(e); if (err.name === "AbortError" || code === "UND_ERR_ABORTED" || code === "UND_ERR_HEADERS_TIMEOUT" || code === "UND_ERR_BODY_TIMEOUT" || code === "UND_ERR_CONNECT_TIMEOUT" || code === "ETIMEDOUT") { return new ProviderError(provider, "timeout", msg, { cause: e }); } if (/CERT_|SSL|TLS|ERR_TLS|EPROTO|handshake/i.test(code + " " + msg)) { return new ProviderError(provider, "tls", msg, { cause: e }); } if (code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "ECONNRESET" || code === "EPIPE" || code === "UND_ERR_SOCKET" || code === "EHOSTUNREACH" || code === "ENETUNREACH") { return new ProviderError(provider, "connect", msg, { cause: e }); } if (code === "UND_ERR_PRX_TLS" || /proxy/i.test(msg)) { return new ProviderError(provider, "proxy", msg, { cause: e }); } return new ProviderError(provider, "unknown", msg, { cause: e }); } export async function closeAllDispatchers(): Promise { await Promise.all([...agentCache.values()].map((v) => v.d.close().catch(() => {}))); agentCache.clear(); }