TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import { Agent, ProxyAgent, request, type Dispatcher } from "undici";2import { CookieJar } from "./cookies";3import { TLS_CIPHERS, TLS_CURVES, TLS_SIGALGS, buildHeaders, pickProfile, type FingerprintProfile } from "./fingerprint";4import { ProviderError, type ProviderId, type ProviderRequest, type ProviderResponse, type ProviderTiming, type ProxyEndpoint } from "./types";56export type { ProxyEndpoint } from "./types";78const HTTP2_ENABLED = process.env.FETCHA_HTTP2 !== "0";910const agentCache = new Map<string, { d: Dispatcher; lastUsed: number }>();1112interface DispatcherOpts {13 endpoint: ProxyEndpoint | null;14 tls: FingerprintProfile["tls"];15 h2: boolean;16 timeoutMs: number;17}1819function tlsOptions(tls: FingerprintProfile["tls"]) {20 return {21 ciphers: TLS_CIPHERS[tls],22 ecdhCurve: TLS_CURVES[tls],23 sigalgs: TLS_SIGALGS[tls],24 minVersion: "TLSv1.2" as const,25 maxVersion: "TLSv1.3" as const,26 honorCipherOrder: false,27 // Browsers never send the legacy renegotiation info in a way Node does by default; keep session reuse on.28 sessionTimeout: 300,29 };30}3132function dispatcherFor(o: DispatcherOpts): Dispatcher {33 const key = `${o.endpoint ? `${o.endpoint.host}:${o.endpoint.port}:${o.endpoint.username}` : "direct"}|${o.tls}|${o.h2 ? "h2" : "h1"}`;34 const hit = agentCache.get(key);35 if (hit) {36 hit.lastUsed = Date.now();37 return hit.d;38 }39 const common = {40 headersTimeout: 120_000,41 bodyTimeout: 120_000,42 keepAliveTimeout: 15_000,43 connections: 128,44 allowH2: o.h2,45 // Browser-like ALPN and TLS parameters for the origin connection.46 connect: { timeout: Math.min(o.timeoutMs, 20_000), ...tlsOptions(o.tls), ALPNProtocols: o.h2 ? ["h2", "http/1.1"] : ["http/1.1"] },47 };48 let d: Dispatcher;49 if (!o.endpoint) {50 d = new Agent(common);51 } else {52 const token = "Basic " + Buffer.from(`${o.endpoint.username}:${o.endpoint.password}`).toString("base64");53 d = new ProxyAgent({54 uri: `http://${o.endpoint.host}:${o.endpoint.port}`,55 token,56 ...common,57 // TLS parameters used for the tunnelled origin connection (through CONNECT).58 requestTls: { ...tlsOptions(o.tls), ALPNProtocols: o.h2 ? ["h2", "http/1.1"] : ["http/1.1"] },59 });60 }61 // Bound the cache: sticky sessions create many usernames. Evict least recently used.62 if (agentCache.size >= 400) {63 let oldest: string | null = null;64 let t = Infinity;65 for (const [k, v] of agentCache) if (v.lastUsed < t) (t = v.lastUsed), (oldest = k);66 if (oldest) {67 agentCache.get(oldest)?.d.close().catch(() => {});68 agentCache.delete(oldest);69 }70 }71 agentCache.set(key, { d, lastUsed: Date.now() });72 return d;73}7475const HOP_BY_HOP = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "host", "content-length", "cookie"]);7677/** Header names as browsers spell them in HTTP/1.1 (HTTP/2 lower-cases everything anyway). */78const SPECIAL_CASE: Record<string, string> = { "sec-ch-ua": "Sec-CH-UA", "sec-ch-ua-mobile": "Sec-CH-UA-Mobile", "sec-ch-ua-platform": "Sec-CH-UA-Platform", te: "TE", dnt: "DNT", "x-requested-with": "X-Requested-With" };79function titleCase(name: string): string {80 if (SPECIAL_CASE[name]) return SPECIAL_CASE[name]!;81 return name82 .split("-")83 .map((p) => (p ? p[0]!.toUpperCase() + p.slice(1) : p))84 .join("-");85}8687function flattenHeaders(h: Record<string, string | string[] | undefined>): { flat: Record<string, string>; setCookie: string[] } {88 const flat: Record<string, string> = {};89 let setCookie: string[] = [];90 for (const [k, v] of Object.entries(h)) {91 if (v === undefined) continue;92 const lk = k.toLowerCase();93 if (lk === "set-cookie") {94 setCookie = Array.isArray(v) ? v : [v];95 flat[lk] = setCookie.join(", ");96 continue;97 }98 flat[lk] = Array.isArray(v) ? v.join(", ") : v;99 }100 return { flat, setCookie };101}102103/**104 * Execute an HTTP request, optionally through an upstream proxy tunnel, emulating a real browser's105 * header order, client hints, TLS preferences and HTTP/2 usage. Redirects are followed manually so106 * every hop can be validated by the caller (SSRF) and cookies set along the way are replayed.107 */108export async function executeHttp(provider: ProviderId, endpoint: ProxyEndpoint | null, req: ProviderRequest): Promise<ProviderResponse> {109 const started = performance.now();110 const deadline = started + req.timeoutMs;111 const profile = req.profile ?? pickProfile({ device: null, seed: req.sessionKey ?? null });112 const jar = req.jar ?? new CookieJar();113 let h2 = HTTP2_ENABLED && req.http2 !== false;114115 let url = new URL(req.url);116 let method = req.method;117 let body: string | Buffer | undefined = req.body;118 let redirects = 0;119 let bytesOut = 0;120 let bytesIn = 0;121 let originMs = 0;122 let referer: string | null = req.referer ?? null;123 let retriedH1 = false;124125 // Caller-supplied cookies (explicit header) become host-only jar cookies for the first URL.126 const overrideHeaders: Record<string, string> = {};127 for (const [k, v] of Object.entries(req.headers)) {128 const lk = k.toLowerCase();129 if (lk === "cookie") {130 jar.addRawCookieHeader(v, url);131 continue;132 }133 if (HOP_BY_HOP.has(lk)) continue;134 overrideHeaders[lk] = v;135 }136137 for (;;) {138 const remaining = deadline - performance.now();139 if (remaining <= 0) throw new ProviderError(provider, "timeout", "Timed out before the target responded.");140 const hasBody = body !== undefined && method !== "GET" && method !== "HEAD";141 const ordered = buildHeaders(profile, {142 locale: req.locale,143 country: req.geo.country,144 referer,145 overrides: overrideHeaders,146 hasBody,147 contentType: hasBody ? (typeof body === "string" ? "application/json" : "application/octet-stream") : null,148 });149 const cookie = jar.headerFor(url);150 if (cookie) {151 // Browsers send Cookie after Accept-Language / before Priority.152 const idx = ordered.findIndex(([k]) => k === "priority");153 ordered.splice(idx === -1 ? ordered.length : idx, 0, ["cookie", cookie]);154 }155 const headers: Record<string, string> = {};156 for (const [k, v] of ordered) headers[titleCase(k)] = v;157158 const dispatcher = dispatcherFor({ endpoint, tls: profile.tls, h2, timeoutMs: req.timeoutMs });159 const ac = new AbortController();160 const timer = setTimeout(() => ac.abort(), remaining);161 const t0 = performance.now();162 let res: Awaited<ReturnType<typeof request>>;163 try {164 res = await request(url, {165 method,166 headers,167 body: hasBody ? body : undefined,168 dispatcher,169 signal: ac.signal,170 });171 } catch (e) {172 clearTimeout(timer);173 const err = classifyError(provider, e);174 // Some origins/proxies mis-handle h2 through CONNECT tunnels: fall back to HTTP/1.1 once.175 if (h2 && !retriedH1 && isH2Failure(e) && deadline - performance.now() > 1000) {176 retriedH1 = true;177 h2 = false;178 continue;179 }180 throw err;181 }182 const tFirstByte = performance.now();183 bytesOut += approxRequestBytes(method, url, headers, hasBody ? body : undefined);184 const { flat: resHeaders, setCookie } = flattenHeaders(res.headers as Record<string, string | string[] | undefined>);185 jar.storeFromHeaders(setCookie.length ? setCookie : resHeaders["set-cookie"], url);186187 // Proxy-level auth/quota errors surface as 407 from the gateway before reaching the target.188 if (endpoint && res.statusCode === 407) {189 clearTimeout(timer);190 await res.body.dump().catch(() => {});191 throw new ProviderError(provider, "auth", "Upstream proxy rejected the credentials.", { status: 407 });192 }193194 const isRedirect = [301, 302, 303, 307, 308].includes(res.statusCode) && !!resHeaders["location"];195 if (isRedirect && req.followRedirects) {196 await res.body.dump().catch(() => {});197 clearTimeout(timer);198 redirects += 1;199 originMs += tFirstByte - t0;200 if (redirects > req.maxRedirects) throw new ProviderError(provider, "redirect", `Exceeded ${req.maxRedirects} redirects.`);201 const next = new URL(resHeaders["location"]!, url);202 if (req.onRedirect) await req.onRedirect(next.toString());203 // Browsers send the previous URL as referer on redirects (same-origin: full URL; cross-origin: origin only).204 referer = next.origin === url.origin ? url.toString() : url.protocol === "https:" && next.protocol !== "https:" ? null : url.origin + "/";205 url = next;206 if (res.statusCode === 303 || ((res.statusCode === 301 || res.statusCode === 302) && method === "POST")) {207 method = "GET";208 body = undefined;209 }210 continue;211 }212213 // Read body with size cap.214 const chunks: Buffer[] = [];215 let total = 0;216 try {217 for await (const chunk of res.body) {218 const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);219 total += buf.length;220 if (total > req.maxResponseBytes) {221 ac.abort();222 throw new ProviderError(provider, "too_large", `Response exceeded ${req.maxResponseBytes} bytes.`);223 }224 chunks.push(buf);225 }226 } catch (e) {227 clearTimeout(timer);228 if (e instanceof ProviderError) throw e;229 throw classifyError(provider, e);230 }231 clearTimeout(timer);232 const tEnd = performance.now();233 const raw = Buffer.concat(chunks);234 bytesIn += total + approxHeaderBytes(resHeaders);235 originMs += tFirstByte - t0;236237 const decoded = await decodeBody(raw, resHeaders["content-encoding"]);238 const tDecoded = performance.now();239 const totalMs = Math.round(tDecoded - started);240 const timing: ProviderTiming = {241 dns_ms: 0,242 proxy_connect_ms: 0,243 tls_ms: 0,244 origin_ms: Math.round(originMs),245 processing_ms: Math.round(tDecoded - tEnd),246 total_ms: totalMs,247 };248 delete resHeaders["content-encoding"];249 return {250 status: res.statusCode,251 headers: resHeaders,252 body: decoded,253 finalUrl: url.toString(),254 redirects,255 bytesIn,256 bytesOut,257 timing,258 profileId: profile.id,259 protocol: h2 ? "h2?" : "http/1.1",260 };261 }262}263264function isH2Failure(e: unknown): boolean {265 const err = e as { code?: string; message?: string; cause?: { code?: string; message?: string } };266 const code = err.code ?? err.cause?.code ?? "";267 const msg = `${err.message ?? ""} ${err.cause?.message ?? ""}`;268 return /HTTP\/2|h2|ERR_HTTP2|NGHTTP2|GOAWAY|RST_STREAM|PROTOCOL_ERROR|UND_ERR_INVALID_ARG/i.test(code + " " + msg) || code === "ERR_HTTP2_ERROR" || code === "ECONNRESET" && /h2|http2/i.test(msg);269}270271function approxRequestBytes(method: string, url: URL, headers: Record<string, string>, body?: string | Buffer): number {272 let n = method.length + url.pathname.length + url.search.length + 12;273 for (const [k, v] of Object.entries(headers)) n += k.length + v.length + 4;274 if (body) n += typeof body === "string" ? Buffer.byteLength(body) : body.length;275 return n;276}277278function approxHeaderBytes(headers: Record<string, string>): number {279 let n = 16;280 for (const [k, v] of Object.entries(headers)) n += k.length + v.length + 4;281 return n;282}283284async function decodeBody(raw: Buffer, encoding?: string): Promise<Buffer> {285 if (!encoding || raw.length === 0) return raw;286 const zlib = await import("node:zlib");287 const { promisify } = await import("node:util");288 // Handle stacked encodings ("gzip, br") right-to-left.289 const encodings = encoding.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean).reverse();290 let buf = raw;291 for (const enc of encodings) {292 try {293 switch (enc) {294 case "gzip":295 case "x-gzip":296 buf = await promisify(zlib.gunzip)(buf);297 break;298 case "deflate":299 buf = await promisify(zlib.inflate)(buf).catch(() => promisify(zlib.inflateRaw)(buf));300 break;301 case "br":302 buf = await promisify(zlib.brotliDecompress)(buf);303 break;304 case "zstd": {305 const z = zlib as unknown as { zstdDecompress?: (b: Buffer, cb: (e: Error | null, r: Buffer) => void) => void };306 if (typeof z.zstdDecompress === "function") buf = await promisify(z.zstdDecompress)(buf);307 break;308 }309 case "identity":310 break;311 default:312 return buf;313 }314 } catch {315 return buf;316 }317 }318 return buf;319}320321function classifyError(provider: ProviderId, e: unknown): ProviderError {322 const err = e as { name?: string; code?: string; message?: string; cause?: { code?: string; message?: string } };323 const code = err.code ?? err.cause?.code ?? "";324 const msg = err.message ?? err.cause?.message ?? String(e);325 if (err.name === "AbortError" || code === "UND_ERR_ABORTED" || code === "UND_ERR_HEADERS_TIMEOUT" || code === "UND_ERR_BODY_TIMEOUT" || code === "UND_ERR_CONNECT_TIMEOUT" || code === "ETIMEDOUT") {326 return new ProviderError(provider, "timeout", msg, { cause: e });327 }328 if (/CERT_|SSL|TLS|ERR_TLS|EPROTO|handshake/i.test(code + " " + msg)) {329 return new ProviderError(provider, "tls", msg, { cause: e });330 }331 if (code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "ECONNRESET" || code === "EPIPE" || code === "UND_ERR_SOCKET" || code === "EHOSTUNREACH" || code === "ENETUNREACH") {332 return new ProviderError(provider, "connect", msg, { cause: e });333 }334 if (code === "UND_ERR_PRX_TLS" || /proxy/i.test(msg)) {335 return new ProviderError(provider, "proxy", msg, { cause: e });336 }337 return new ProviderError(provider, "unknown", msg, { cause: e });338}339340export async function closeAllDispatchers(): Promise<void> {341 await Promise.all([...agentCache.values()].map((v) => v.d.close().catch(() => {})));342 agentCache.clear();343}344