TypeScript 97.5%
SQL 1.4%
Python 0.8%
1/**2 * @fetcha/browser — managed headless Chromium for rendered fetches.3 *4 * One Chromium process (full Chromium in "new headless" mode, not the headless shell, driven by5 * Patchright — Playwright patched against the classic CDP leaks) hosts many isolated contexts; every6 * render gets its own context with the upstream proxy credentials of the chosen route, a consistent7 * fingerprint (UA, viewport, locale, timezone, platform) and stealth patches. Cloudflare/DataDome8 * style JavaScript challenges are given time to complete (and a best-effort Turnstile click) before9 * the DOM is captured. Set FETCHA_BROWSER_HEADLESS=0 on a node with a graphical session to run a10 * real (off-screen) window, which defeats more headless-detection heuristics.11 */12import { FetchaError, isBlockedHostname, isIP, isBlockedIP, looksBlocked, type HttpMethod } from "@fetcha/core";13import type { FingerprintProfile, ProxyEndpoint } from "@fetcha/providers";14import { acceptLanguage } from "@fetcha/providers";15import type { Browser, BrowserContext, Page, Response as PwResponse } from "patchright";16import { stealthScript, timezoneFor, webglFor } from "./stealth";17import { TURNSTILE_HOOK_KEY, TwoCaptchaSolver, turnstileHookScript, type TurnstileParams, type TurnstileSolution } from "./captcha";1819export * from "./captcha";2021export interface RenderRequest {22 url: string;23 method: HttpMethod;24 body?: string | Buffer;25 proxy: ProxyEndpoint | null;26 profile: FingerprintProfile;27 locale?: string | null;28 country?: string | null;29 headers?: Record<string, string>;30 cookies?: Array<{ name: string; value: string; domain: string; path: string; secure?: boolean; expires?: number | null }>;31 referer?: string | null;32 timeoutMs: number;33 waitUntil: "load" | "domcontentloaded" | "networkidle";34 waitFor?: string | null;35 waitMs?: number | null;36 javascript: boolean;37 blockResources: boolean;38 screenshot: boolean;39 maxResponseBytes: number;40 /** Use the configured captcha solver (2captcha) on Turnstile challenges (default true). */41 solveCaptcha?: boolean;42 /** Called for every main-frame navigation to a new URL (redirects). Throw to abort. */43 onRedirect?: (nextUrl: string) => Promise<void>;44}4546export interface CaptchaOutcome {47 provider: "2captcha";48 /** Token obtained and the page passed afterwards. */49 solved: boolean;50 ms: number;51 costUsd: number;52}5354export interface RenderResult {55 status: number;56 headers: Record<string, string>;57 /** Serialised DOM (UTF-8 HTML). */58 body: Buffer;59 finalUrl: string;60 redirects: number;61 bytesIn: number;62 bytesOut: number;63 timing: { navigation_ms: number; challenge_ms: number; settle_ms: number; capture_ms: number; total_ms: number };64 cookies: Array<{ name: string; value: string; domain: string; path: string; secure: boolean; expires: number | null }>;65 screenshot?: Buffer;66 /** True when a challenge page was detected and later replaced by real content. */67 challengeSolved: boolean;68 /** Block verdict on the captured DOM (after the challenge wait). */69 block: ReturnType<typeof looksBlocked>;70 /** Captcha solver involvement, when a token was purchased. */71 captcha: CaptchaOutcome | null;72}7374export interface BrowserPoolOptions {75 /** Max concurrent renders across the process. */76 maxConcurrency?: number;77 /** How long a render may wait for a slot before failing with BROWSER_UNAVAILABLE. */78 queueTimeoutMs?: number;79 /** Close the browser after this idle time. */80 idleCloseMs?: number;81 /** Playwright channel: "chromium" (full build, new headless) or "chrome". */82 channel?: string;83 enabled?: boolean;84 /** Maximum time to give a JS challenge to resolve. */85 challengeWaitMs?: number;86 /** Run a real (off-screen) window instead of headless mode. Requires a graphical session. */87 headless?: boolean;88 /** 2captcha API key (default: env TWOCAPTCHA_API_KEY). Empty = no solver. */89 captchaApiKey?: string | null;90 log?: { info: (m: string) => void; warn: (m: string) => void };91}9293export interface BrowserStatus {94 enabled: boolean;95 launched: boolean;96 running: number;97 capacity: number;98 queue: number;99 renders: number;100 failures: number;101 challengesSolved: number;102 lastError: string | null;103 version: string | null;104 headless: boolean;105 channel: string;106 captchaSolver: { provider: string; requested: number; solved: number; failed: number; spentUsd: number; lastError: string | null } | null;107}108109/** Assets that anti-bot vendors use to verify a real browser (trace pixels, widget assets): never block them. */110const ANTIBOT_ASSET_RE = /\/cdn-cgi\/|challenges\.cloudflare\.com|captcha-delivery\.com|datadome|perimeterx|px-cdn|px-cloud|kasada|kpsdk|imperva|incapsula|_Incapsula_Resource|hcaptcha\.com|recaptcha|gstatic\.com\/recaptcha|arkoselabs|funcaptcha|awswaf|distil|shape-security|vercel\.com\/_vercel|\/_vercel\/|akamaihd|akam\/|\/akam\//i;111112const CHALLENGE_SELECTORS = ["#challenge-running", "#challenge-form", "#challenge-stage", ".cf-turnstile", "iframe[src*='challenges.cloudflare.com']", "#px-captcha", "#datadome", "iframe[src*='captcha-delivery.com']", "#sec-cpt-if", "form#challenge", "#cmsg", "[data-testid='challenge']"];113114export class BrowserPool {115 private browser: Browser | null = null;116 private launching: Promise<Browser> | null = null;117 /** User agent reported by the real Chromium build (with "HeadlessChrome" normalised to "Chrome"). */118 private nativeUa: string | null = null;119 private readonly solver: TwoCaptchaSolver | null;120 private running = 0;121 private queue: Array<() => void> = [];122 private idleTimer: NodeJS.Timeout | null = null;123 private stats = { renders: 0, failures: 0, challengesSolved: 0, lastError: null as string | null };124 private readonly opts: Required<Omit<BrowserPoolOptions, "log">> & { log: NonNullable<BrowserPoolOptions["log"]> };125126 constructor(opts: BrowserPoolOptions = {}) {127 this.opts = {128 maxConcurrency: opts.maxConcurrency ?? Number(process.env.FETCHA_BROWSER_CONCURRENCY ?? 6),129 queueTimeoutMs: opts.queueTimeoutMs ?? 25_000,130 idleCloseMs: opts.idleCloseMs ?? 10 * 60_000,131 channel: opts.channel ?? process.env.FETCHA_BROWSER_CHANNEL ?? "chromium",132 enabled: opts.enabled ?? process.env.FETCHA_BROWSER_ENABLED !== "0",133 challengeWaitMs: opts.challengeWaitMs ?? 18_000,134 headless: opts.headless ?? process.env.FETCHA_BROWSER_HEADLESS !== "0",135 captchaApiKey: opts.captchaApiKey === undefined ? process.env.TWOCAPTCHA_API_KEY?.trim() || null : opts.captchaApiKey,136 log: opts.log ?? { info: (m) => console.log("[browser]", m), warn: (m) => console.warn("[browser]", m) },137 };138 this.solver = this.opts.captchaApiKey ? new TwoCaptchaSolver({ apiKey: this.opts.captchaApiKey, log: { info: (m) => this.opts.log.info(`2captcha: ${m}`), warn: (m) => this.opts.log.warn(`2captcha: ${m}`) } }) : null;139 }140141 get enabled(): boolean {142 return this.opts.enabled;143 }144145 get captchaSolver(): TwoCaptchaSolver | null {146 return this.solver;147 }148149 status(): BrowserStatus {150 return {151 enabled: this.opts.enabled,152 launched: Boolean(this.browser?.isConnected()),153 running: this.running,154 capacity: this.opts.maxConcurrency,155 queue: this.queue.length,156 renders: this.stats.renders,157 failures: this.stats.failures,158 challengesSolved: this.stats.challengesSolved,159 lastError: this.stats.lastError,160 version: this.browser?.version() ?? null,161 headless: this.opts.headless,162 channel: this.opts.channel,163 captchaSolver: this.solver ? this.solver.status() : null,164 };165 }166167 private async getBrowser(): Promise<Browser> {168 if (this.browser?.isConnected()) return this.browser;169 if (this.launching) return this.launching;170 this.launching = (async () => {171 // Patchright = Playwright patched against CDP leaks (Runtime.enable, command-line flags, console).172 const { chromium } = await import("patchright");173 const args = [174 "--no-first-run",175 "--no-default-browser-check",176 "--disable-dev-shm-usage",177 "--disable-background-timer-throttling",178 "--disable-renderer-backgrounding",179 "--disable-backgrounding-occluded-windows",180 "--disable-features=IsolateOrigins,site-per-process,Translate,MediaRouter,OptimizationHints",181 "--disable-infobars",182 "--no-service-autorun",183 "--password-store=basic",184 "--use-mock-keychain",185 "--export-tagged-pdf",186 "--force-color-profile=srgb",187 "--window-size=1920,1080",188 ...(this.opts.headless ? [] : ["--window-position=-32000,-32000"]),189 ];190 const launch = async (channel: string | undefined) =>191 chromium.launch({192 headless: this.opts.headless,193 channel,194 args,195 proxy: { server: "per-context" },196 timeout: 60_000,197 });198 let b: Browser;199 try {200 b = await launch(this.opts.channel);201 } catch (e) {202 this.opts.log.warn(`launch with channel=${this.opts.channel} failed (${(e as Error).message.split("\n")[0]}), retrying default build`);203 b = await launch(undefined);204 }205 b.on("disconnected", () => {206 this.opts.log.warn("browser disconnected");207 if (this.browser === b) this.browser = null;208 });209 this.browser = b;210 // Client hints (Sec-CH-UA*) come from the real build and cannot be spoofed coherently, so the UA211 // we present is the native one minus the headless marker — never a made-up version.212 try {213 const probe = await b.newContext({ proxy: { server: "http://127.0.0.1:1", bypass: "*" } });214 const pg = await probe.newPage();215 this.nativeUa = (await pg.evaluate(() => navigator.userAgent)).replace(/HeadlessChrome/g, "Chrome");216 await probe.close();217 } catch {218 this.nativeUa = null;219 }220 this.opts.log.info(`chromium ${b.version()} launched (channel=${this.opts.channel}, headless=${this.opts.headless}, concurrency=${this.opts.maxConcurrency})`);221 return b;222 })();223 try {224 return await this.launching;225 } finally {226 this.launching = null;227 }228 }229230 private touchIdle() {231 if (this.idleTimer) clearTimeout(this.idleTimer);232 this.idleTimer = setTimeout(() => {233 if (this.running === 0 && this.browser) {234 this.opts.log.info("closing idle browser");235 this.browser.close().catch(() => {});236 this.browser = null;237 }238 }, this.opts.idleCloseMs);239 this.idleTimer.unref?.();240 }241242 private async acquire(): Promise<() => void> {243 if (this.running < this.opts.maxConcurrency) {244 this.running++;245 return () => this.release();246 }247 return new Promise<() => void>((resolve, reject) => {248 const timer = setTimeout(() => {249 this.queue = this.queue.filter((f) => f !== wake);250 reject(new FetchaError("BROWSER_UNAVAILABLE", "The managed browser is busy. Retry in a few seconds."));251 }, this.opts.queueTimeoutMs);252 const wake = () => {253 clearTimeout(timer);254 this.running++;255 resolve(() => this.release());256 };257 this.queue.push(wake);258 });259 }260261 private release() {262 this.running = Math.max(0, this.running - 1);263 const next = this.queue.shift();264 if (next) next();265 else this.touchIdle();266 }267268 async close(): Promise<void> {269 if (this.idleTimer) clearTimeout(this.idleTimer);270 await this.browser?.close().catch(() => {});271 this.browser = null;272 }273274 async render(req: RenderRequest): Promise<RenderResult> {275 if (!this.opts.enabled) throw new FetchaError("BROWSER_UNAVAILABLE");276 const release = await this.acquire();277 const started = performance.now();278 const deadline = started + req.timeoutMs;279 let context: BrowserContext | null = null;280 try {281 const browser = await this.getBrowser();282 const profile = req.profile;283 const languages = acceptLanguage(req.locale, req.country)284 .split(",")285 .map((s) => s.split(";")[0]!.trim())286 .filter(Boolean);287 const isMobile = profile.device === "mobile";288 const extraHeaders: Record<string, string> = {};289 let customUa: string | null = null;290 for (const [k, v] of Object.entries(req.headers ?? {})) {291 const lk = k.toLowerCase();292 if (lk === "user-agent") {293 customUa = v;294 continue;295 }296 if (["cookie", "host", "content-length", "accept-encoding", "connection"].includes(lk)) continue;297 extraHeaders[lk] = v;298 }299 const userAgent = customUa ?? (isMobile ? profile.userAgent : this.nativeUa ?? profile.userAgent);300 if (!extraHeaders["accept-language"]) extraHeaders["accept-language"] = acceptLanguage(req.locale, req.country);301302 context = await browser.newContext({303 // The browser is launched with per-context proxying; a context without an upstream proxy304 // must still provide one, so we point at an unreachable local proxy and bypass every host (= direct).305 proxy: req.proxy ? { server: `http://${req.proxy.host}:${req.proxy.port}`, username: req.proxy.username, password: req.proxy.password } : { server: "http://127.0.0.1:1", bypass: "*" },306 userAgent,307 viewport: profile.viewport,308 deviceScaleFactor: isMobile ? 3 : profile.tls === "safari" ? 2 : 1,309 isMobile,310 hasTouch: isMobile || profile.device === "tablet",311 locale: languages[0] ?? "en-US",312 timezoneId: timezoneFor(req.country),313 javaScriptEnabled: req.javascript,314 extraHTTPHeaders: extraHeaders,315 ignoreHTTPSErrors: false,316 colorScheme: "light",317 serviceWorkers: "block",318 acceptDownloads: false,319 });320 const webgl = webglFor(customUa || isMobile ? profile.platform : /Windows/.test(userAgent) ? "Win32" : /Linux/.test(userAgent) ? "Linux x86_64" : "MacIntel", profile.tls);321 const platform = customUa || isMobile ? profile.platform : /Windows/.test(userAgent) ? "Win32" : /Linux/.test(userAgent) ? "Linux x86_64" : "MacIntel";322 await context.addInitScript(stealthScript({ platform, languages, hardwareConcurrency: isMobile ? 8 : 12, deviceMemory: 8, vendor: webgl.vendor, renderer: webgl.renderer, mobile: isMobile }));323 if (this.solver && req.solveCaptcha !== false) await context.addInitScript(turnstileHookScript());324 if (req.cookies?.length) {325 await context326 .addCookies(327 req.cookies.map((c) => ({328 name: c.name,329 value: c.value,330 domain: c.domain.startsWith(".") ? c.domain : c.domain,331 path: c.path || "/",332 secure: Boolean(c.secure),333 httpOnly: false,334 expires: c.expires ? Math.floor(c.expires / 1000) : -1,335 sameSite: "Lax" as const,336 })),337 )338 .catch(() => {});339 }340341 let bytesIn = 0;342 let bytesOut = 0;343 const pendingSizes: Promise<void>[] = [];344 const initialUrl = new URL(req.url);345 let redirects = 0;346 let policyError: FetchaError | null = null;347 let firstNav = true;348 // Resource blocking is lifted as soon as a challenge is detected: anti-bot scripts verify that349 // images/fonts actually load (measured: solved tokens were rejected with blocking on).350 let blockResources = req.blockResources;351352 const page = await context.newPage();353 page.setDefaultTimeout(Math.min(req.timeoutMs, 60_000));354355 await page.route("**/*", async (route) => {356 const r = route.request();357 const isMain = r.isNavigationRequest() && r.frame() === page.mainFrame();358 let target: URL;359 try {360 target = new URL(r.url());361 } catch {362 return route.abort("blockedbyclient");363 }364 // SSRF: never let the browser touch private hosts (even via subresources / redirects).365 const host = target.hostname.replace(/^\[|\]$/g, "");366 if (isBlockedHostname(host) || (isIP(host) && isBlockedIP(host)) || !/^https?:$/.test(target.protocol)) {367 if (isMain) policyError = new FetchaError("URL_NOT_ALLOWED", "The page redirected to a local, private or internal host.");368 return route.abort("blockedbyclient");369 }370 if (isMain) {371 if (firstNav) {372 firstNav = false;373 if (req.method !== "GET") {374 const headers = { ...r.headers() };375 if (req.body !== undefined && !headers["content-type"]) headers["content-type"] = typeof req.body === "string" ? "application/json" : "application/octet-stream";376 return route.fallback({ method: req.method, postData: req.body, headers });377 }378 } else if (r.url() !== initialUrl.toString()) {379 redirects++;380 if (req.onRedirect) {381 try {382 await req.onRedirect(r.url());383 } catch (e) {384 policyError = e instanceof FetchaError ? e : new FetchaError("URL_NOT_ALLOWED");385 return route.abort("blockedbyclient");386 }387 }388 }389 // fallback() (not continue()) lets Patchright's context-level route inject the init scripts.390 return route.fallback();391 }392 if (blockResources && !ANTIBOT_ASSET_RE.test(r.url())) {393 const type = r.resourceType();394 if (type === "image" || type === "media" || type === "font" || type === "manifest" || type === "texttrack") return route.abort("blockedbyclient");395 }396 return route.fallback();397 });398 page.on("requestfinished", (r) => {399 pendingSizes.push(400 r401 .sizes()402 .then((s) => {403 bytesOut += s.requestBodySize + s.requestHeadersSize;404 bytesIn += s.responseBodySize + s.responseHeadersSize;405 })406 .catch(() => {}),407 );408 });409 page.on("dialog", (d) => d.dismiss().catch(() => {}));410 const nav: { last: PwResponse | null } = { last: null }; // holder: assigned from an event callback411 page.on("response", (r) => {412 try {413 if (r.request().isNavigationRequest() && r.frame() === page.mainFrame() && r.status() !== 304) nav.last = r;414 } catch {415 /* ignore */416 }417 });418419 // Navigate420 const tNav0 = performance.now();421 let response: PwResponse | null = null;422 try {423 response = await page.goto(req.url, { waitUntil: req.waitUntil, timeout: Math.max(1000, Math.min(req.timeoutMs - 1500, deadline - performance.now())), referer: req.referer ?? undefined });424 } catch (e) {425 if (policyError) throw policyError;426 const msg = (e as Error).message ?? String(e);427 if (/Timeout/i.test(msg)) {428 // Partial content may still be useful: if the document exists, capture what we have.429 if (!(await page.content().catch(() => "")).includes("<body")) throw new FetchaError("BROWSER_TIMEOUT", "The page did not finish loading in time.");430 } else if (/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY|ERR_NO_SUPPORTED_PROXIES|407/i.test(msg)) {431 throw new FetchaError("PROVIDER_UNAVAILABLE", "The upstream network rejected the browser connection.");432 } else if (/ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_ADDRESS_UNREACHABLE|ERR_CONNECTION_TIMED_OUT|ERR_EMPTY_RESPONSE/i.test(msg)) {433 throw new FetchaError("TARGET_UNAVAILABLE", "The browser could not reach the target.");434 } else if (/ERR_BLOCKED_BY_CLIENT/i.test(msg)) {435 throw policyError ?? new FetchaError("URL_NOT_ALLOWED");436 } else if (/ERR_ABORTED/i.test(msg) && response) {437 /* download / non-HTML navigation: continue with what we have */438 } else if (/ERR_TOO_MANY_REDIRECTS/i.test(msg)) {439 throw new FetchaError("TOO_MANY_REDIRECTS");440 } else if (/ERR_CERT|ERR_SSL/i.test(msg)) {441 throw new FetchaError("TARGET_UNAVAILABLE", "TLS error while connecting to the target.");442 } else throw new FetchaError("INTERNAL_ERROR", "Browser navigation failed.", { cause: e });443 }444 if (policyError) throw policyError;445 const navigationMs = Math.round(performance.now() - tNav0);446447 // Challenge handling: give the page time to solve JS challenges (Cloudflare, DataDome, PX…),448 // and buy a Turnstile token from the captcha solver when the challenge is interactive.449 const tCh0 = performance.now();450 let challengeSolved = false;451 let captcha: CaptchaOutcome | null = null;452 let status = response?.status() ?? 200;453 let headers = lower(response?.headers() ?? {});454 let html = await page.content().catch(() => "");455 let verdict = looksBlocked(status, html, headers);456 if (verdict.blocked && verdict.challenge !== false && req.javascript) {457 blockResources = false;458 const challengeStatus = status;459 const solverAllowed = Boolean(this.solver) && req.solveCaptcha !== false;460 const loopStart = performance.now();461 let until = loopStart + Math.max(0, Math.min(this.opts.challengeWaitMs, deadline - performance.now() - 1500));462 let clickedTurnstile = false;463 let solveState: { started: boolean; done: boolean; sol: TurnstileSolution | null } = { started: false, done: false, sol: null };464 let injected = false;465 while (performance.now() < until) {466 await page.waitForTimeout(500).catch(() => {});467 // Natural resolution: the challenge script redirects / reloads the document. Fast reloads are468 // missed by waitForNavigation, so the main-frame response listener is the source of truth.469 await page.waitForNavigation({ timeout: 700, waitUntil: "domcontentloaded" }).catch(() => null);470 const navResp = nav.last && nav.last !== response ? nav.last : null;471 if (navResp) {472 response = navResp;473 status = navResp.status();474 headers = lower(navResp.headers());475 }476 html = await page.content().catch(() => "");477 // Interstitial markers only: the real page of a protected site also references478 // challenges.cloudflare.com (beacon), so vendor script URLs must not count here.479 const stillChallenge = /cf_chl_opt|__cf_chl_|id="challenge-(running|stage|form|error-text)"|<title>\s*Just a moment|id="px-captcha"|geo\.captcha-delivery\.com|id="datadome"|Verifying you are human|Performing security verification/i.test(html.slice(0, 60_000));480 // Headers of the interstitial (cf-mitigated…) must not veto a document that has moved on.481 const probeHeaders = stillChallenge ? headers : Object.fromEntries(Object.entries(headers).filter(([k]) => k !== "cf-mitigated" && k !== "x-amzn-waf-action" && k !== "x-datadome"));482 const v = looksBlocked(navResp ? status : 200, html, probeHeaders);483 if (!v.blocked && !stillChallenge && html.length > 800) {484 challengeSolved = true;485 if (!navResp) {486 status = 200; // the document was replaced in place487 headers = probeHeaders;488 }489 break;490 }491 // Captcha solver: once the widget parameters are visible, buy a token (in the background) and keep waiting.492 if (solverAllowed && !solveState.started && performance.now() - loopStart > 1200 && deadline - performance.now() > 15_000) {493 const params = await this.readTurnstileParams(page);494 if (params?.sitekey) {495 solveState.started = true;496 const websiteURL = page.url();497 this.solver!.solveTurnstile({ websiteURL, params, deadline: deadline - 4000 })498 .then((sol) => (solveState = { started: true, done: true, sol }))499 .catch(() => (solveState = { started: true, done: true, sol: null }));500 until = deadline - 2500; // wait for the solver instead of the short challenge budget501 }502 }503 if (solveState.done && !injected) {504 injected = true;505 const sol = solveState.sol;506 if (!sol) {507 until = Math.min(until, performance.now() + 2500);508 } else {509 captcha = { provider: "2captcha", solved: false, ms: sol.ms, costUsd: sol.costUsd };510 // Do NOT switch the page's User-Agent to the solver's: a mid-page UA change is exactly what511 // Cloudflare rejects (measured 2026-09-08: 0/2 with the CDP override, 2/2 without).512 const how = await this.injectTurnstileToken(page, sol.token);513 this.opts.log.info(`turnstile token injected via ${how} on ${new URL(page.url()).hostname} (solver ua ${sol.userAgent ? (sol.userAgent === userAgent ? "same" : "differs, kept ours") : "none"})`);514 until = Math.min(deadline - 1500, performance.now() + 15_000);515 }516 }517 // Best-effort click on a Turnstile / hCaptcha checkbox when no solver is available.518 if (!solverAllowed && !clickedTurnstile) clickedTurnstile = await this.tryClickChallenge(page);519 }520 verdict = looksBlocked(status, html, headers);521 if (challengeSolved) {522 this.stats.challengesSolved++;523 headers = { ...headers, "x-fetcha-challenge": `solved:${challengeStatus}${captcha ? ":2captcha" : ""}` };524 verdict = { blocked: false };525 if (captcha) captcha.solved = true;526 }527 }528 const challengeMs = Math.round(performance.now() - tCh0);529530 // Settle: wait for selector / extra time; light human-like interaction.531 const tSettle0 = performance.now();532 if (req.waitFor && !verdict.blocked) {533 const remaining = deadline - performance.now() - 800;534 if (remaining > 200) {535 try {536 await page.waitForSelector(req.waitFor, { timeout: Math.min(remaining, 45_000), state: "attached" });537 } catch {538 throw new FetchaError("BROWSER_TIMEOUT", `The selector "${req.waitFor}" did not appear in time.`, { details: { wait_for: req.waitFor } });539 }540 }541 }542 if (req.javascript && !verdict.blocked) {543 try {544 const vp = profile.viewport;545 await page.mouse.move(vp.width * (0.3 + Math.random() * 0.4), vp.height * (0.3 + Math.random() * 0.3), { steps: 5 });546 await page.mouse.wheel(0, 200 + Math.round(Math.random() * 400));547 } catch {548 /* ignore */549 }550 }551 if (req.waitMs) await page.waitForTimeout(Math.min(req.waitMs, Math.max(0, deadline - performance.now() - 500)));552 else if (req.javascript && req.waitUntil !== "networkidle") await page.waitForLoadState("networkidle", { timeout: Math.min(2500, Math.max(0, deadline - performance.now() - 500)) }).catch(() => {});553 const settleMs = Math.round(performance.now() - tSettle0);554555 // Capture556 const tCap0 = performance.now();557 html = await page.content().catch(() => html);558 const finalUrl = page.url();559 const body = Buffer.from(html, "utf8");560 if (body.length > req.maxResponseBytes) throw new FetchaError("RESPONSE_TOO_LARGE");561 let screenshot: Buffer | undefined;562 if (req.screenshot) screenshot = await page.screenshot({ type: "png", fullPage: false, timeout: 15_000 }).catch(() => undefined);563 const cookies = (await context.cookies().catch(() => [])).map((c) => ({ name: c.name, value: c.value, domain: c.domain, path: c.path, secure: c.secure, expires: c.expires && c.expires > 0 ? Math.round(c.expires * 1000) : null }));564 await Promise.race([Promise.allSettled(pendingSizes), new Promise((r) => setTimeout(r, 500))]);565 const captureMs = Math.round(performance.now() - tCap0);566 const totalMs = Math.round(performance.now() - started);567 const finalVerdict = challengeSolved ? { blocked: false } : looksBlocked(status, html, headers);568 this.stats.renders++;569 const outHeaders: Record<string, string> = { ...headers };570 delete outHeaders["content-encoding"];571 delete outHeaders["content-length"];572 if (!outHeaders["content-type"]) outHeaders["content-type"] = "text/html; charset=utf-8";573 return {574 status,575 headers: outHeaders,576 body,577 finalUrl,578 redirects,579 bytesIn: bytesIn || body.length,580 bytesOut,581 timing: { navigation_ms: navigationMs, challenge_ms: challengeMs, settle_ms: settleMs, capture_ms: captureMs, total_ms: totalMs },582 cookies,583 screenshot,584 challengeSolved,585 block: finalVerdict,586 captcha,587 };588 } catch (e) {589 this.stats.failures++;590 this.stats.lastError = (e as Error).message?.slice(0, 300) ?? String(e);591 throw e;592 } finally {593 await context?.close().catch(() => {});594 release();595 }596 }597598 /** Read the intercepted `turnstile.render` parameters, or fall back to a widget's data-* attributes. */599 private async readTurnstileParams(page: Page): Promise<TurnstileParams | null> {600 try {601 return await page.evaluate((key) => {602 const w = window as unknown as Record<string, { params?: TurnstileParams | null; callback?: unknown } | undefined>;603 const store = w[key];604 if (store?.params?.sitekey) return { ...store.params, hasCallback: typeof store.callback === "function" } as TurnstileParams;605 const el = document.querySelector(".cf-turnstile[data-sitekey], [data-sitekey][data-callback], #cf-chl-widget-container [data-sitekey]") as HTMLElement | null;606 if (el?.dataset.sitekey) return { sitekey: el.dataset.sitekey, action: el.dataset.action ?? null, cData: el.dataset.cdata ?? null, chlPageData: null, hasCallback: false } as TurnstileParams;607 return null;608 }, TURNSTILE_HOOK_KEY, undefined, false); // Patchright: 4th arg = isolatedContext → false = page's main world609 } catch {610 return null;611 }612 }613614 /** Hand the solved token to the widget's callback (challenge pages) or to the form field (standalone widgets). */615 private async injectTurnstileToken(page: Page, token: string): Promise<string> {616 try {617 return await page.evaluate(618 ({ key, token }) => {619 const w = window as unknown as Record<string, { callback?: (t: string) => void } | undefined>;620 const store = w[key];621 if (store && typeof store.callback === "function") {622 store.callback(token);623 return "callback";624 }625 const input = document.querySelector('input[name="cf-turnstile-response"]') as HTMLInputElement | null;626 if (input) {627 input.value = token;628 const form = input.closest("form");629 const cbName = (document.querySelector(".cf-turnstile[data-callback]") as HTMLElement | null)?.dataset.callback;630 const cb = cbName ? (window as unknown as Record<string, unknown>)[cbName] : null;631 if (typeof cb === "function") (cb as (t: string) => void)(token);632 else if (form) (form as HTMLFormElement & { requestSubmit?: () => void }).requestSubmit ? (form as HTMLFormElement).requestSubmit() : form.submit();633 return "form";634 }635 return "none";636 },637 { key: TURNSTILE_HOOK_KEY, token },638 undefined,639 false, // main world: the callback captured by the init script lives there640 );641 } catch (e) {642 return `error:${(e as Error).message.split("\n")[0]}`;643 }644 }645646 private async tryClickChallenge(page: Page): Promise<boolean> {647 try {648 for (const frame of page.frames()) {649 const u = frame.url();650 if (!/challenges\.cloudflare\.com|hcaptcha\.com|captcha-delivery\.com/.test(u)) continue;651 const el = await frame.frameElement().catch(() => null);652 const box = el ? await el.boundingBox().catch(() => null) : null;653 if (box && box.width > 20 && box.height > 20) {654 await page.mouse.move(box.x + 28 + Math.random() * 6, box.y + box.height / 2 + (Math.random() * 4 - 2), { steps: 8 });655 await page.waitForTimeout(150 + Math.random() * 200);656 await page.mouse.click(box.x + 30, box.y + box.height / 2, { delay: 40 + Math.random() * 60 });657 return true;658 }659 }660 const cb = page.locator("#challenge-stage input[type=checkbox], .ctp-checkbox-label, label.ctp-checkbox-label").first();661 if (await cb.isVisible({ timeout: 200 }).catch(() => false)) {662 await cb.click({ timeout: 1500 }).catch(() => {});663 return true;664 }665 } catch {666 /* ignore */667 }668 return false;669 }670}671672function lower(h: Record<string, string>): Record<string, string> {673 const out: Record<string, string> = {};674 for (const [k, v] of Object.entries(h)) out[k.toLowerCase()] = v;675 return out;676}677678let _pool: BrowserPool | null = null;679export function getBrowserPool(opts?: BrowserPoolOptions): BrowserPool {680 if (!_pool) _pool = new BrowserPool(opts);681 return _pool;682}683export async function closeBrowserPool(): Promise<void> {684 await _pool?.close();685 _pool = null;686}687