browser: main-frame response listener + stale interstitial headers ignored (challenge completion missed after fast reloads)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +20 −5
modified
packages/browser/src/index.ts
+20 −5
@@ -407,6 +407,14 @@ export class BrowserPool { | ||
| 407 | 407 | ); |
| 408 | 408 | }); |
| 409 | 409 | page.on("dialog", (d) => d.dismiss().catch(() => {})); |
| 410 | + const nav: { last: PwResponse | null } = { last: null }; // holder: assigned from an event callback | |
| 411 | + page.on("response", (r) => { | |
| 412 | + try { | |
| 413 | + if (r.request().isNavigationRequest() && r.frame() === page.mainFrame() && r.status() !== 304) nav.last = r; | |
| 414 | + } catch { | |
| 415 | + /* ignore */ | |
| 416 | + } | |
| 417 | + }); | |
| 410 | 418 | |
| 411 | 419 | // Navigate |
| 412 | 420 | const tNav0 = performance.now(); |
@@ -456,8 +464,10 @@ export class BrowserPool { | ||
| 456 | 464 | let injected = false; |
| 457 | 465 | while (performance.now() < until) { |
| 458 | 466 | await page.waitForTimeout(500).catch(() => {}); |
| 459 | − // Natural resolution: the challenge script redirects / replaces the document. | |
| 460 | − const navResp = await page.waitForNavigation({ timeout: 900, waitUntil: "domcontentloaded" }).catch(() => null); | |
| 467 | + // Natural resolution: the challenge script redirects / reloads the document. Fast reloads are | |
| 468 | + // missed by waitForNavigation, so the main-frame response listener is the source of truth. | |
| 469 | + await page.waitForNavigation({ timeout: 700, waitUntil: "domcontentloaded" }).catch(() => null); | |
| 470 | + const navResp = nav.last && nav.last !== response ? nav.last : null; | |
| 461 | 471 | if (navResp) { |
| 462 | 472 | response = navResp; |
| 463 | 473 | status = navResp.status(); |
@@ -467,10 +477,15 @@ export class BrowserPool { | ||
| 467 | 477 | // Interstitial markers only: the real page of a protected site also references |
| 468 | 478 | // challenges.cloudflare.com (beacon), so vendor script URLs must not count here. |
| 469 | 479 | const stillChallenge = /cf_chl_opt|__cf_chl_|id="challenge-(running|stage|form|error-text)"|<title>\s*Just a moment|id="px-captcha"|geo\.captcha-delivery\.com|id="datadome"|Verifying you are human|Performing security verification/i.test(html.slice(0, 60_000)); |
| 470 | − const v = looksBlocked(navResp ? status : 200, html, headers); | |
| 471 | − if (!v.blocked && !stillChallenge && html.length > 1500) { | |
| 480 | + // Headers of the interstitial (cf-mitigated…) must not veto a document that has moved on. | |
| 481 | + const probeHeaders = stillChallenge ? headers : Object.fromEntries(Object.entries(headers).filter(([k]) => k !== "cf-mitigated" && k !== "x-amzn-waf-action" && k !== "x-datadome")); | |
| 482 | + const v = looksBlocked(navResp ? status : 200, html, probeHeaders); | |
| 483 | + if (!v.blocked && !stillChallenge && html.length > 800) { | |
| 472 | 484 | challengeSolved = true; |
| 473 | − if (!navResp) status = 200; // the document was replaced in place | |
| 485 | + if (!navResp) { | |
| 486 | + status = 200; // the document was replaced in place | |
| 487 | + headers = probeHeaders; | |
| 488 | + } | |
| 474 | 489 | break; |
| 475 | 490 | } |
| 476 | 491 | // Captcha solver: once the widget parameters are visible, buy a token (in the background) and keep waiting. |
| 477 | 492 | |