browser: keep the page's own UA when injecting the Turnstile token (CDP UA override made Cloudflare reject it)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +6 −43
modified
CLAUDE.md
+1 −1
@@ -26,7 +26,7 @@ and a full dashboard. **Private platform (v0.2, 2026-09-08)**: a single `unlimit | ||
| 26 | 26 | - API keys: shown once, SHA-256 stored. Request IDs `req_…` on every response (`X-Fetcha-Request-ID`). |
| 27 | 27 | - Not implemented yet (say so, don't fake): scripted browser actions (`POST /v1/browser`), `/v1/extract`, teams, webhook delivery (except the crawl `webhook_url` callback), OAuth, 2FA. Billing/Stripe is intentionally absent (private platform). |
| 28 | 28 | - Plans: always `normalizePlan(org.plan)`; never reintroduce tiers. Signup must stay allowlist-gated (Better Auth `user.create.before` hook in `apps/web/src/lib/auth.ts`). |
| 29 | −- Anti-bot: interactive Cloudflare Turnstile challenges are solved with 2captcha (`TWOCAPTCHA_API_KEY`, `packages/browser/src/captcha.ts`): the init script wraps the api.js `?onload=` callback to capture `turnstile.render` params (sitekey, action, cData, chlPageData, callback), a token is bought (~$0.0015, 5–10 s) and injected through the callback after switching the page UA (CDP `Emulation.setUserAgentOverride`) to the solver's UA. Patchright runs init scripts in the main world but `page.evaluate` in an isolated world — pass `undefined, false` (4th arg) to read main-world state. `looksBlocked()` on 2xx must only match interstitial markers (vendor beacons like `challenge-platform/scripts/jsd`, `tags.js`, `ips.js` are on every protected page). Prod browser = Google Chrome stable (brew cask on M3U96a), `FETCHA_BROWSER_HEADLESS=0` (real off-screen window in the node's GUI session). | |
| 29 | +- Anti-bot: interactive Cloudflare Turnstile challenges are solved with 2captcha (`TWOCAPTCHA_API_KEY`, `packages/browser/src/captcha.ts`): the init script wraps the api.js `?onload=` callback to capture `turnstile.render` params (sitekey, action, cData, chlPageData, callback), a token is bought (~$0.0015, 5–10 s) and injected through the callback — never switch the page UA to the solver's (Cloudflare then rejects the token). Patchright runs init scripts in the main world but `page.evaluate` in an isolated world — pass `undefined, false` (4th arg) to read main-world state. `looksBlocked()` on 2xx must only match interstitial markers (vendor beacons like `challenge-platform/scripts/jsd`, `tags.js`, `ips.js` are on every protected page). Prod browser = Google Chrome stable (brew cask on M3U96a), `FETCHA_BROWSER_HEADLESS=0` (real off-screen window in the node's GUI session). | |
| 30 | 30 | |
| 31 | 31 | ## Dev |
| 32 | 32 | `cp .env.example .env` (fill provider creds + RESEND_API_KEY), `createdb fetcha`, `pnpm db:migrate && pnpm db:seed`, |
modified
packages/browser/src/captcha.ts
+2 −2
@@ -4,8 +4,8 @@ | ||
| 4 | 4 | * Used for Cloudflare Turnstile — both the standalone widget (`data-sitekey`) and the managed |
| 5 | 5 | * challenge page ("Just a moment…"), where the `turnstile.render` call must be intercepted to grab |
| 6 | 6 | * `sitekey`, `action`, `cData` and `chlPageData`, and the returned token handed to the widget's |
| 7 | − * callback. The solver also returns the User-Agent the token was produced with; the page must adopt | |
| 8 | − * it before invoking the callback. | |
| 7 | + * callback. (The solver also returns the User-Agent it used; changing the page's UA to match makes | |
| 8 | + * Cloudflare reject the token — keep the browser's own UA.) | |
| 9 | 9 | */ |
| 10 | 10 | export interface TurnstileParams { |
| 11 | 11 | sitekey: string; |
modified
packages/browser/src/index.ts
+3 −40
@@ -114,8 +114,6 @@ export class BrowserPool { | ||
| 114 | 114 | /** User agent reported by the real Chromium build (with "HeadlessChrome" normalised to "Chrome"). */ |
| 115 | 115 | private nativeUa: string | null = null; |
| 116 | 116 | private readonly solver: TwoCaptchaSolver | null; |
| 117 | − /** User-Agent of the last solver token: reused up-front so challenge pages see one consistent UA. */ | |
| 118 | − private lastSolverUa: string | null = null; | |
| 119 | 117 | private running = 0; |
| 120 | 118 | private queue: Array<() => void> = []; |
| 121 | 119 | private idleTimer: NodeJS.Timeout | null = null; |
@@ -485,12 +483,10 @@ export class BrowserPool { | ||
| 485 | 483 | until = Math.min(until, performance.now() + 2500); |
| 486 | 484 | } else { |
| 487 | 485 | captcha = { provider: "2captcha", solved: false, ms: sol.ms, costUsd: sol.costUsd }; |
| 488 | − if (sol.userAgent && sol.userAgent !== userAgent) { | |
| 489 | − this.lastSolverUa = sol.userAgent; | |
| 490 | − await this.overrideUserAgent(context, page, sol.userAgent, extraHeaders["accept-language"]); | |
| 491 | − } | |
| 486 | + // Do NOT switch the page's User-Agent to the solver's: a mid-page UA change is exactly what | |
| 487 | + // Cloudflare rejects (measured 2026-09-08: 0/2 with the CDP override, 2/2 without). | |
| 492 | 488 | const how = await this.injectTurnstileToken(page, sol.token); |
| 493 | − this.opts.log.info(`turnstile token injected via ${how} on ${new URL(page.url()).hostname} (solver ua ${sol.userAgent ? (sol.userAgent === userAgent ? "same" : "override") : "none"})`); | |
| 489 | + this.opts.log.info(`turnstile token injected via ${how} on ${new URL(page.url()).hostname} (solver ua ${sol.userAgent ? (sol.userAgent === userAgent ? "same" : "differs, kept ours") : "none"})`); | |
| 494 | 490 | until = Math.min(deadline - 1500, performance.now() + 15_000); |
| 495 | 491 | } |
| 496 | 492 | } |
@@ -623,39 +619,6 @@ export class BrowserPool { | ||
| 623 | 619 | } |
| 624 | 620 | } |
| 625 | 621 | |
| 626 | − /** Adopt the User-Agent the token was produced with (Cloudflare binds tokens to the UA). */ | |
| 627 | − private async overrideUserAgent(context: BrowserContext, page: Page, userAgent: string, acceptLanguage?: string): Promise<void> { | |
| 628 | − try { | |
| 629 | − const cdp = await context.newCDPSession(page); | |
| 630 | − const major = /Chrome\/(\d+)/.exec(userAgent)?.[1] ?? "140"; | |
| 631 | − const platform = /Windows/.test(userAgent) ? "Windows" : /Macintosh/.test(userAgent) ? "macOS" : /Android/.test(userAgent) ? "Android" : "Linux"; | |
| 632 | − const brands = [ | |
| 633 | − { brand: "Chromium", version: major }, | |
| 634 | − { brand: /Edg\//.test(userAgent) ? "Microsoft Edge" : "Google Chrome", version: major }, | |
| 635 | − { brand: "Not_A Brand", version: "24" }, | |
| 636 | − ]; | |
| 637 | − await cdp.send("Emulation.setUserAgentOverride", { | |
| 638 | − userAgent, | |
| 639 | − acceptLanguage, | |
| 640 | − platform: platform === "Windows" ? "Win32" : platform === "macOS" ? "MacIntel" : platform === "Android" ? "Linux armv81" : "Linux x86_64", | |
| 641 | − userAgentMetadata: { | |
| 642 | − brands, | |
| 643 | − fullVersionList: brands.map((b) => ({ brand: b.brand, version: b.version === "24" ? "24.0.0.0" : `${b.version}.0.0.0` })), | |
| 644 | − platform, | |
| 645 | − platformVersion: platform === "Windows" ? "15.0.0" : platform === "macOS" ? "14.6.1" : platform === "Android" ? "15.0.0" : "6.8.0", | |
| 646 | − architecture: platform === "Android" ? "" : "x86", | |
| 647 | − model: "", | |
| 648 | − mobile: /Mobile/.test(userAgent), | |
| 649 | − bitness: "64", | |
| 650 | − wow64: false, | |
| 651 | − }, | |
| 652 | − }); | |
| 653 | − await cdp.detach().catch(() => {}); | |
| 654 | − } catch (e) { | |
| 655 | − this.opts.log.warn(`user-agent override failed: ${(e as Error).message.split("\n")[0]}`); | |
| 656 | − } | |
| 657 | − } | |
| 658 | − | |
| 659 | 622 | private async tryClickChallenge(page: Page): Promise<boolean> { |
| 660 | 623 | try { |
| 661 | 624 | for (const frame of page.frames()) { |
| 662 | 625 | |