browser: main-frame response listener + stale interstitial headers ignored (challenge completion missed after fast reloads)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +20 −5
modified
packages/browser/src/index.ts
+20 −5
@@ -407,6 +407,14 @@ export class BrowserPool { | |||||
| 407 | ); | 407 | ); | ||
| 408 | }); | 408 | }); | ||
| 409 | page.on("dialog", (d) => d.dismiss().catch(() => {})); | 409 | page.on("dialog", (d) => d.dismiss().catch(() => {})); | ||
| 410 | + const nav: { last: PwResponse | null } = { last: null }; // holder: assigned from an event callback | ||||
| 411 | + page.on("response", (r) => { | ||||
| 412 | + try { | ||||
| 413 | + if (r.request().isNavigationRequest() && r.frame() === page.mainFrame() && r.status() !== 304) nav.last = r; | ||||
| 414 | + } catch { | ||||
| 415 | + /* ignore */ | ||||
| 416 | + } | ||||
| 417 | + }); | ||||
| 410 | 418 | ||||
| 411 | // Navigate | 419 | // Navigate | ||
| 412 | const tNav0 = performance.now(); | 420 | const tNav0 = performance.now(); | ||
@@ -456,8 +464,10 @@ export class BrowserPool { | |||||
| 456 | let injected = false; | 464 | let injected = false; | ||
| 457 | while (performance.now() < until) { | 465 | while (performance.now() < until) { | ||
| 458 | await page.waitForTimeout(500).catch(() => {}); | 466 | await page.waitForTimeout(500).catch(() => {}); | ||
| 459 | − // Natural resolution: the challenge script redirects / replaces the document. | 467 | + // Natural resolution: the challenge script redirects / reloads the document. Fast reloads are | ||
| 460 | − const navResp = await page.waitForNavigation({ timeout: 900, waitUntil: "domcontentloaded" }).catch(() => null); | 468 | + // missed by waitForNavigation, so the main-frame response listener is the source of truth. | ||
| 469 | + await page.waitForNavigation({ timeout: 700, waitUntil: "domcontentloaded" }).catch(() => null); | ||||
| 470 | + const navResp = nav.last && nav.last !== response ? nav.last : null; | ||||
| 461 | if (navResp) { | 471 | if (navResp) { | ||
| 462 | response = navResp; | 472 | response = navResp; | ||
| 463 | status = navResp.status(); | 473 | status = navResp.status(); | ||
@@ -467,10 +477,15 @@ export class BrowserPool { | |||||
| 467 | // Interstitial markers only: the real page of a protected site also references | 477 | // Interstitial markers only: the real page of a protected site also references | ||
| 468 | // challenges.cloudflare.com (beacon), so vendor script URLs must not count here. | 478 | // challenges.cloudflare.com (beacon), so vendor script URLs must not count here. | ||
| 469 | const stillChallenge = /cf_chl_opt|__cf_chl_|id="challenge-(running|stage|form|error-text)"|<title>\s*Just a moment|id="px-captcha"|geo\.captcha-delivery\.com|id="datadome"|Verifying you are human|Performing security verification/i.test(html.slice(0, 60_000)); | 479 | const stillChallenge = /cf_chl_opt|__cf_chl_|id="challenge-(running|stage|form|error-text)"|<title>\s*Just a moment|id="px-captcha"|geo\.captcha-delivery\.com|id="datadome"|Verifying you are human|Performing security verification/i.test(html.slice(0, 60_000)); | ||
| 470 | − const v = looksBlocked(navResp ? status : 200, html, headers); | 480 | + // Headers of the interstitial (cf-mitigated…) must not veto a document that has moved on. | ||
| 471 | − if (!v.blocked && !stillChallenge && html.length > 1500) { | 481 | + const probeHeaders = stillChallenge ? headers : Object.fromEntries(Object.entries(headers).filter(([k]) => k !== "cf-mitigated" && k !== "x-amzn-waf-action" && k !== "x-datadome")); | ||
| 482 | + const v = looksBlocked(navResp ? status : 200, html, probeHeaders); | ||||
| 483 | + if (!v.blocked && !stillChallenge && html.length > 800) { | ||||
| 472 | challengeSolved = true; | 484 | challengeSolved = true; | ||
| 473 | − if (!navResp) status = 200; // the document was replaced in place | 485 | + if (!navResp) { | ||
| 486 | + status = 200; // the document was replaced in place | ||||
| 487 | + headers = probeHeaders; | ||||
| 488 | + } | ||||
| 474 | break; | 489 | break; | ||
| 475 | } | 490 | } | ||
| 476 | // Captcha solver: once the widget parameters are visible, buy a token (in the background) and keep waiting. | 491 | // Captcha solver: once the widget parameters are visible, buy a token (in the background) and keep waiting. | ||
| 477 | 492 | ||||