TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import {2 FetchaError,3 assertUrlAllowed,4 extractDomain,5 extractPageMetadata,6 htmlToMainText,7 htmlToMarkdown,8 htmlToText,9 isTransientStatus,10 looksBlocked,11 newId,12 normalizeGeo,13 scrubProviderText,14 type BlockVerdict,15 type ConcreteNetwork,16 type FetchRequest,17 type FetchResponseBody,18 type FetchTiming,19 type PageLink,20 type PageMetadata,21 type Plan,22} from "@fetcha/core";23import { CookieJar, ProviderError, pickProfile, retryReferer, type FingerprintProfile, type ProviderId, type ProviderResponse } from "@fetcha/providers";24import type { BrowserPool, RenderResult } from "@fetcha/browser";25import type { CircuitBreaker } from "./circuit";26import { routeKey, type DomainKnowledge, type RouteCandidate, type RoutingEngine } from "./engine";2728export type AttemptOutcome = "success" | "blocked" | "timeout" | "error" | "provider_error" | "too_large";29export type AttemptMode = "http" | "browser";3031/** Customer-facing aliases: upstream names are never revealed without provider visibility. */32export const PUBLIC_PROVIDER_ALIAS: Record<ProviderId, string> = {33 oxylabs: "network-a",34 decodo: "network-b",35 soax: "network-c",36 direct: "fetcha-edge",37};3839export interface AttemptRecord {40 attemptId: string;41 attemptNo: number;42 provider: ProviderId;43 network: ConcreteNetwork;44 mode: AttemptMode;45 country: string | null;46 sessionKey: string | null;47 outcome: AttemptOutcome;48 httpStatus: number | null;49 errorCode: string | null;50 errorDetail: string | null;51 blockReason: string | null;52 blockVendor: string | null;53 profileId: string | null;54 /** A captcha solver token was purchased and the attempt passed thanks to it. */55 captchaSolved: boolean;56 durationMs: number;57 bytesIn: number;58 bytesOut: number;59 unitPricePerGb: number;60 costUsd: number;61 routingScore: number;62 timing: FetchTiming | null;63}6465export type SerializedCookie = { name: string; value: string; domain: string; path: string; secure: boolean; expires: number | null };6667export interface ExecutionContext {68 requestId: string;69 plan: Plan;70 request: FetchRequest;71 /** Provider-agnostic sticky key resolved from a Fetcha session (if any). */72 sessionKey?: string | null;73 sessionProvider?: ProviderId | null;74 sessionNetwork?: ConcreteNetwork | null;75 /** Cookies persisted on the session from earlier requests. */76 sessionCookies?: SerializedCookie[] | null;77 knowledge?: DomainKnowledge | null;78 /** Called after every attempt (success or failure) for persistence & metrics. */79 onAttempt?: (a: AttemptRecord) => Promise<void> | void;80 /** Whether to include provider names in the debug payload. */81 providerVisibility?: boolean;82 maxResponseBytes?: number;83 /** Managed browser pool (null/undefined = browser mode unavailable). */84 browserPool?: BrowserPool | null;85 /** Whether the plan allows browser rendering. */86 browserAllowed?: boolean;87}8889export interface ExecutionResult {90 body: FetchResponseBody;91 attempts: AttemptRecord[];92 network: ConcreteNetwork | null;93 provider: ProviderId | null;94 mode: AttemptMode;95 costUsd: number;96 bytesIn: number;97 bytesOut: number;98 finalUrl: string;99 domain: string;100 /** Cookie jar state at the end of the request (for sticky-session persistence). */101 cookies: SerializedCookie[];102 /** True when the browser was needed after HTTP attempts were blocked (feeds domain intelligence). */103 browserRequired: boolean;104 /** Raw response for optional storage. */105 raw?: ProviderResponse;106}107108interface FinalResponse {109 status: number;110 headers: Record<string, string>;111 body: Buffer;112 finalUrl: string;113 timing: FetchTiming;114 screenshot?: Buffer;115}116117const HTML_CT = /text\/html|application\/xhtml/i;118119export class FetchExecutor {120 constructor(121 private readonly engine: RoutingEngine,122 private readonly circuit: CircuitBreaker,123 ) {}124125 async execute(ctx: ExecutionContext): Promise<ExecutionResult> {126 const { request, requestId } = ctx;127 const started = performance.now();128 const domain = extractDomain(request.url);129 const browserAvailable = Boolean(ctx.browserPool?.enabled) && ctx.browserAllowed !== false;130131 if (request.browser && !browserAvailable) {132 throw new FetchaError("BROWSER_UNAVAILABLE", undefined, { requestId });133 }134135 const allowed = await assertUrlAllowed(request.url);136 const geo = normalizeGeo({ country: request.country, region: request.region, city: request.city });137138 const plan = this.engine.plan({139 domain,140 network: request.network,141 geo,142 plan: ctx.plan,143 sessionRequired: Boolean(request.session),144 browser: request.browser,145 retries: request.retries,146 knowledge: ctx.knowledge ?? null,147 });148149 // A sticky session pins provider + network for its lifetime.150 let candidates: RouteCandidate[] = plan.candidates;151 if (ctx.sessionKey && ctx.sessionProvider) {152 const pinned = candidates.filter((c) => c.provider.id === ctx.sessionProvider && (!ctx.sessionNetwork || c.network === ctx.sessionNetwork));153 if (pinned.length) candidates = [pinned[0]!, ...candidates.filter((c) => c !== pinned[0])];154 }155156 if (!candidates.length) {157 if (request.network !== "auto") {158 throw new FetchaError("NETWORK_UNAVAILABLE", `The "${request.network}" network class is not available right now.`, { requestId });159 }160 throw new FetchaError("PROVIDER_UNAVAILABLE", undefined, { requestId });161 }162163 // Domain intelligence: sites that consistently need the browser go straight there.164 const knowledgeSaysBrowser = Boolean(ctx.knowledge && (ctx.knowledge.browserRequiredRate ?? 0) >= 0.5 && (ctx.knowledge.browserSamples ?? 0) >= 3);165 let browserMode = request.browser || (browserAvailable && request.browser_fallback && knowledgeSaysBrowser);166 let browserEscalated = false;167168 const attempts: AttemptRecord[] = [];169 let lastError: FetchaError | null = null;170 let lastBlocked: { res: FinalResponse; cand: RouteCandidate; mode: AttemptMode; verdict: BlockVerdict } | null = null;171 let lastTransient: { res: FinalResponse; cand: RouteCandidate; mode: AttemptMode } | null = null;172 const maxBytes = ctx.maxResponseBytes ?? request.max_response_bytes ?? 20_000_000;173 const deadline = started + request.timeout;174 const usedProfiles: string[] = [];175 const jar = CookieJar.fromSerialized(ctx.sessionCookies ?? null, allowed.hostname);176 if (request.cookies) for (const [k, v] of Object.entries(request.cookies)) jar.store(`${k}=${v}; Path=/`, allowed.url);177 let browserRequired = false;178 // The browser fallback is one extra attempt on top of the plan's HTTP budget.179 const maxAttempts = plan.maxAttempts + (browserAvailable && request.browser_fallback && !request.browser ? 1 : 0);180181 for (let i = 0; i < maxAttempts; i++) {182 const cand = candidates[Math.min(i, candidates.length - 1)]!;183 const remaining = deadline - performance.now();184 if (remaining < 800) {185 lastError = new FetchaError("TARGET_TIMEOUT", undefined, { requestId });186 break;187 }188 if (i > 0) await this.backoff(i, lastBlocked?.verdict.retryAfterMs, deadline);189190 const attemptId = newId("att");191 const key = routeKey(cand.provider.id, cand.network);192 const t0 = performance.now();193 const sessionKey = ctx.sessionKey ?? null;194 const profile: FingerprintProfile = pickProfile({ device: request.device ?? null, seed: sessionKey, attempt: i, exclude: sessionKey ? [] : usedProfiles });195 usedProfiles.push(profile.id);196 const headers = { ...(request.headers ?? {}) };197 if (request.device === "mobile" && headers["user-agent"]) delete headers["user-agent"]; // profile already mobile198 const referer = request.referer === "none" ? null : request.referer === "auto" ? (i === 0 && !browserEscalated ? null : retryReferer(i, geo.country)) : request.referer;199 const body = request.body === undefined ? undefined : typeof request.body === "string" ? request.body : JSON.stringify(request.body);200 const mode: AttemptMode = browserMode ? "browser" : "http";201 const timeoutMs = Math.max(1000, Math.min(deadline - performance.now(), request.timeout));202203 let res: FinalResponse | null = null;204 let verdict: BlockVerdict = { blocked: false };205 let captchaOutcome: RenderResult["captcha"] = null;206 let record: AttemptRecord;207 try {208 let bytesIn = 0;209 let bytesOut = 0;210 if (mode === "browser") {211 const render: RenderResult = await ctx.browserPool!.render({212 url: allowed.url.toString(),213 method: request.method,214 body,215 proxy: cand.provider.proxyEndpoint({ geo, sessionKey, sessionMinutes: 10 }),216 profile,217 locale: request.locale ?? null,218 country: geo.country,219 headers,220 cookies: jar.serialize(),221 referer,222 timeoutMs,223 waitUntil: request.wait_until,224 waitFor: request.wait_for ?? null,225 waitMs: request.wait_ms ?? null,226 javascript: request.javascript !== false,227 // A site that already blocked us (or is known to need the browser) gets every asset: anti-bot228 // verification checks that images/fonts really load.229 blockResources: request.block_resources && !browserEscalated && !knowledgeSaysBrowser,230 screenshot: request.screenshot,231 solveCaptcha: request.solve_captcha,232 maxResponseBytes: maxBytes,233 onRedirect: async (next) => {234 await assertUrlAllowed(next);235 },236 });237 for (const c of render.cookies) jar.store(`${c.name}=${c.value}; Domain=${c.domain}; Path=${c.path}${c.secure ? "; Secure" : ""}`, new URL(render.finalUrl));238 bytesIn = render.bytesIn;239 bytesOut = render.bytesOut;240 res = {241 status: render.status,242 headers: render.headers,243 body: render.body,244 finalUrl: render.finalUrl,245 screenshot: render.screenshot,246 timing: { dns_ms: allowed.dns_ms, proxy_connect_ms: 0, tls_ms: 0, origin_ms: render.timing.navigation_ms, processing_ms: render.timing.challenge_ms + render.timing.settle_ms + render.timing.capture_ms, total_ms: render.timing.total_ms },247 };248 verdict = render.block;249 if (render.challengeSolved) browserRequired = true;250 captchaOutcome = render.captcha;251 } else {252 const pr = await cand.provider.fetch({253 requestId,254 attemptId,255 url: allowed.url.toString(),256 method: request.method,257 headers,258 body,259 timeoutMs,260 network: cand.network,261 geo,262 sessionKey,263 sessionMinutes: 10,264 followRedirects: request.follow_redirects,265 maxRedirects: request.max_redirects,266 maxResponseBytes: maxBytes,267 profile,268 locale: request.locale ?? null,269 referer,270 jar,271 onRedirect: async (next) => {272 await assertUrlAllowed(next);273 },274 });275 bytesIn = pr.bytesIn;276 bytesOut = pr.bytesOut;277 res = { status: pr.status, headers: pr.headers, body: pr.body, finalUrl: pr.finalUrl, timing: { ...pr.timing, dns_ms: allowed.dns_ms } };278 const ct = pr.headers["content-type"] ?? "";279 verdict = looksBlocked(pr.status, !ct || HTML_CT.test(ct) || /json|xml|text/i.test(ct) ? pr.body.subarray(0, 40_000).toString("utf8") : "", pr.headers);280 }281 const durationMs = Math.round(performance.now() - t0);282 const transient = !verdict.blocked && isTransientStatus(res.status);283 const cost = cand.provider.estimateCost(cand.network, bytesIn + bytesOut) + (captchaOutcome?.costUsd ?? 0);284 record = {285 attemptId,286 attemptNo: i + 1,287 provider: cand.provider.id,288 network: cand.network,289 mode,290 country: geo.country,291 sessionKey,292 outcome: verdict.blocked ? "blocked" : transient ? "error" : "success",293 httpStatus: res.status,294 errorCode: verdict.blocked ? "TARGET_BLOCKED" : transient ? "TARGET_UNAVAILABLE" : null,295 errorDetail: null,296 blockReason: verdict.reason ?? null,297 blockVendor: verdict.vendor ?? null,298 profileId: profile.id,299 captchaSolved: Boolean(captchaOutcome?.solved),300 durationMs,301 bytesIn,302 bytesOut,303 unitPricePerGb: cand.provider.pricePerGb(cand.network),304 costUsd: cost,305 routingScore: cand.score,306 timing: res.timing,307 };308 attempts.push(record);309 await ctx.onAttempt?.(record);310 // Provider health: only proxy-side failures count against the route.311 this.circuit.record(key, !(verdict.blocked && verdict.reason === "http_407"));312313 if (!verdict.blocked && !transient) {314 if (mode === "browser" && browserEscalated) browserRequired = true;315 return this.finish(ctx, res, cand, mode, attempts, started, domain, jar, browserRequired);316 }317 if (transient) {318 lastTransient = { res, cand, mode };319 continue;320 }321 lastBlocked = { res, cand, mode, verdict };322 // Escalate to the browser after an HTTP block that a real browser can typically pass.323 if (mode === "http" && browserAvailable && request.browser_fallback && (verdict.challenge || i >= plan.maxAttempts - 1 || knowledgeSaysBrowser)) {324 browserMode = true;325 browserEscalated = true;326 }327 continue;328 } catch (e) {329 const durationMs = Math.round(performance.now() - t0);330 if (e instanceof FetchaError) {331 if (e.code === "URL_NOT_ALLOWED" || e.code === "RESPONSE_TOO_LARGE" || e.code === "TOO_MANY_REDIRECTS" || e.code === "INVALID_REQUEST") {332 // Policy violation or non-retryable: do not retry.333 record = this.errorRecord(attemptId, i, cand, mode, profile.id, geo.country, sessionKey, "error", e.code, e.message, durationMs);334 attempts.push(record);335 await ctx.onAttempt?.(record);336 throw e;337 }338 // Browser-side failure (timeout / unavailable): record and let the loop continue (HTTP or another route).339 const outcome: AttemptOutcome = e.code === "BROWSER_TIMEOUT" || e.code === "TARGET_TIMEOUT" ? "timeout" : e.code === "BROWSER_UNAVAILABLE" ? "provider_error" : "error";340 record = this.errorRecord(attemptId, i, cand, mode, profile.id, geo.country, sessionKey, outcome, e.code, e.message, durationMs);341 attempts.push(record);342 await ctx.onAttempt?.(record);343 lastError = new FetchaError(e.code, e.message, { requestId });344 if (e.code === "BROWSER_UNAVAILABLE" && !request.browser) browserMode = false; // fall back to HTTP for the remaining budget345 if (request.browser && e.code === "BROWSER_UNAVAILABLE") throw lastError;346 continue;347 }348 const pe = e instanceof ProviderError ? e : new ProviderError(cand.provider.id, "unknown", (e as Error).message ?? String(e), { cause: e });349 let outcome: AttemptOutcome = "provider_error";350 let code = "PROVIDER_UNAVAILABLE";351 if (pe.kind === "timeout") {352 outcome = "timeout";353 code = "TARGET_TIMEOUT";354 } else if (pe.kind === "too_large") {355 outcome = "too_large";356 code = "RESPONSE_TOO_LARGE";357 } else if (pe.kind === "redirect") {358 outcome = "error";359 code = "TOO_MANY_REDIRECTS";360 } else if (pe.kind === "connect" || pe.kind === "tls") {361 outcome = "error";362 code = "TARGET_UNAVAILABLE";363 }364 record = this.errorRecord(attemptId, i, cand, mode, profile.id, geo.country, sessionKey, outcome, code, pe.message, durationMs);365 attempts.push(record);366 await ctx.onAttempt?.(record);367 // Provider-side failures count against the circuit; target-side ones don't.368 this.circuit.record(key, !(pe.kind === "auth" || pe.kind === "proxy"));369 lastError = new FetchaError(code as FetchaError["code"], undefined, { requestId });370 if (pe.kind === "too_large" || pe.kind === "redirect") throw lastError;371 // A TLS failure through a proxy is often the route, not the site: escalate to the browser when we can.372 if (pe.kind === "tls" && browserAvailable && request.browser_fallback && !browserMode) {373 browserMode = true;374 browserEscalated = true;375 }376 continue;377 }378 }379380 // Exhausted: if the last route returned a blocked page, surface that page (customer still381 // gets the status) but mark success=false with TARGET_BLOCKED.382 if (lastBlocked) {383 const result = this.finish(ctx, lastBlocked.res, lastBlocked.cand, lastBlocked.mode, attempts, started, domain, jar, browserRequired);384 result.body.success = false;385 return result;386 }387 if (lastTransient) {388 const result = this.finish(ctx, lastTransient.res, lastTransient.cand, lastTransient.mode, attempts, started, domain, jar, browserRequired);389 result.body.success = false;390 return result;391 }392 throw lastError ?? new FetchaError("TARGET_UNAVAILABLE", undefined, { requestId });393 }394395 /** Jittered backoff between attempts; honours short Retry-After hints. */396 private async backoff(attempt: number, retryAfterMs: number | undefined, deadline: number): Promise<void> {397 let wait = 250 + Math.random() * 650 + Math.min(attempt, 3) * 200;398 if (retryAfterMs && retryAfterMs > 0) wait = Math.max(wait, Math.min(retryAfterMs, 6000));399 wait = Math.min(wait, Math.max(0, deadline - performance.now() - 1500));400 if (wait > 0) await new Promise((r) => setTimeout(r, wait));401 }402403 private errorRecord(404 attemptId: string,405 i: number,406 cand: RouteCandidate,407 mode: AttemptMode,408 profileId: string,409 country: string | null,410 sessionKey: string | null,411 outcome: AttemptOutcome,412 code: string,413 detail: string,414 durationMs: number,415 ): AttemptRecord {416 return {417 attemptId,418 attemptNo: i + 1,419 provider: cand.provider.id,420 network: cand.network,421 mode,422 country,423 sessionKey,424 outcome,425 httpStatus: null,426 errorCode: code,427 errorDetail: detail.slice(0, 500),428 blockReason: null,429 blockVendor: null,430 profileId,431 captchaSolved: false,432 durationMs,433 bytesIn: 0,434 bytesOut: 0,435 unitPricePerGb: cand.provider.pricePerGb(cand.network),436 costUsd: 0,437 routingScore: cand.score,438 timing: null,439 };440 }441442 private finish(ctx: ExecutionContext, res: FinalResponse, cand: RouteCandidate, mode: AttemptMode, attempts: AttemptRecord[], started: number, domain: string, jar: CookieJar, browserRequired: boolean): ExecutionResult {443 const { request, requestId } = ctx;444 const contentType = res.headers["content-type"] ?? null;445 const isText = !contentType || /text\/|json|xml|javascript|x-www-form-urlencoded/i.test(contentType);446 const isHtml = isText && (!contentType || HTML_CT.test(contentType) || (/^\s*<(!doctype|html)/i.test(res.body.subarray(0, 200).toString("utf8")) && !/json|xml|javascript/i.test(contentType ?? "")));447 const content = isText ? res.body.toString("utf8") : res.body.toString("base64");448 const tProc = performance.now();449450 let text: string | null | undefined;451 let markdown: string | null | undefined;452 let json: unknown;453 let page: PageMetadata | null = null;454 let links: PageLink[] | undefined;455 if (isHtml) {456 try {457 const meta = extractPageMetadata(content, res.finalUrl);458 page = meta.page;459 if (request.links) links = meta.links;460 } catch {461 page = null;462 }463 }464 if (request.format === "text") text = isText ? (isHtml ? htmlToMainText(content) || htmlToText(content) : content) : null;465 if (request.format === "markdown") markdown = isText ? (isHtml ? htmlToMarkdown(content, { baseUrl: res.finalUrl }) : content) : null;466 if (request.format === "json") {467 try {468 json = JSON.parse(content);469 } catch {470 json = undefined;471 }472 }473 const durationMs = Math.round(performance.now() - started);474 const bytesIn = attempts.reduce((s, a) => s + a.bytesIn, 0);475 const bytesOut = attempts.reduce((s, a) => s + a.bytesOut, 0);476 const costUsd = attempts.reduce((s, a) => s + a.costUsd, 0);477 const timing: FetchTiming = {478 ...res.timing,479 processing_ms: res.timing.processing_ms + Math.round(performance.now() - tProc),480 total_ms: durationMs,481 };482 const body: FetchResponseBody = {483 request_id: requestId,484 success: res.status >= 200 && res.status < 400,485 status: res.status,486 url: request.url,487 final_url: res.finalUrl,488 content: request.format === "raw" || request.format === "html" || request.format === "json" ? content : null,489 content_type: contentType,490 headers: res.headers,491 cookies: jar.publicList(),492 page,493 metadata: {494 network: cand.network,495 country: attempts.at(-1)?.country ?? null,496 mode,497 attempts: attempts.length,498 duration_ms: durationMs,499 bytes: bytesIn + bytesOut,500 session: request.session ?? null,501 cached: false,502 timing,503 },504 };505 if (request.format === "text") body.text = text ?? null;506 if (request.format === "markdown") body.markdown = markdown ?? null;507 if (request.format === "json") body.json = json;508 if (links) body.links = links;509 if (request.screenshot && mode === "browser") body.screenshot = res.screenshot ? res.screenshot.toString("base64") : null;510 if (request.debug) {511 body.metadata.debug = {512 attempts: attempts.map((a) => ({513 provider: ctx.providerVisibility ? a.provider : PUBLIC_PROVIDER_ALIAS[a.provider],514 network: a.network,515 mode: a.mode,516 country: a.country,517 outcome: a.outcome,518 block_reason: a.blockReason,519 ...(a.captchaSolved ? { captcha_solved: true } : {}),520 status: a.httpStatus,521 duration_ms: a.durationMs,522 ...(a.errorDetail && ctx.providerVisibility ? { error: scrubProviderText(a.errorDetail) } : {}),523 })),524 };525 }526 return {527 body,528 attempts,529 network: cand.network,530 provider: cand.provider.id,531 mode,532 costUsd,533 bytesIn,534 bytesOut,535 finalUrl: res.finalUrl,536 domain,537 cookies: jar.serialize(),538 browserRequired,539 raw: { status: res.status, headers: res.headers, body: res.body, finalUrl: res.finalUrl, redirects: 0, bytesIn, bytesOut, timing: res.timing },540 };541 }542}543544export function parseSetCookies(raw: string | undefined): Array<{ name: string; value: string; domain?: string; path?: string }> {545 if (!raw) return [];546 // Header was flattened with ", " — split conservatively on ", <token>=" boundaries.547 const parts = raw.split(/,(?=\s*[A-Za-z0-9_\-!#$%&'*+.^`|~]+=)/);548 const out: Array<{ name: string; value: string; domain?: string; path?: string }> = [];549 for (const part of parts) {550 const segs = part.split(";").map((s) => s.trim());551 const [nv, ...attrs] = segs;552 if (!nv) continue;553 const eq = nv.indexOf("=");554 if (eq <= 0) continue;555 const c: { name: string; value: string; domain?: string; path?: string } = { name: nv.slice(0, eq), value: nv.slice(eq + 1) };556 for (const a of attrs) {557 const [k, v] = a.split("=");558 if (!k) continue;559 if (k.toLowerCase() === "domain" && v) c.domain = v;560 if (k.toLowerCase() === "path" && v) c.path = v;561 }562 out.push(c);563 }564 return out;565}566