# ============================================================================== # Author: Simon-Pierre Boucher # File: src/net.py (ka-apartments-com) # Desc: Fetcher Bright Data Web Unlocker — apartments.com est verrouillé par # Akamai (curl_cffi/Scrapfly/Oxylabs → 403) ; seul le Web Unlocker # passe. Simple POST https://api.brightdata.com/request qui renvoie le # HTML brut de la page débloquée. httpx async + sémaphore + retries à # backoff exponentiel. # ============================================================================== from __future__ import annotations import asyncio import random import httpx from apify import Actor API = "https://api.brightdata.com/request" def _backoff(attempt: int) -> float: return min(1.5 * (2 ** attempt), 15.0) + random.uniform(0.0, 1.0) class BrightData: """GET d'une page via le Web Unlocker Bright Data (déblocage géré).""" def __init__(self, token: str, zone: str, concurrency: int = 4, delay: float = 0.0) -> None: self._headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} self._zone = zone self._sem = asyncio.Semaphore(max(1, concurrency)) self._delay = max(0.0, delay) self._client = httpx.AsyncClient(timeout=120.0) async def close(self) -> None: await self._client.aclose() async def get(self, url: str, retries: int = 3) -> str | None: """HTML de la page, ou None si le déblocage échoue après retries.""" body = {"zone": self._zone, "url": url, "format": "raw", "country": "ca"} async with self._sem: for attempt in range(retries): try: resp = await self._client.post( API, headers=self._headers, json=body) if resp.status_code == 200 and resp.text.strip(): if self._delay: await asyncio.sleep(self._delay) return resp.text Actor.log.warning( f"Bright Data {resp.status_code} {url} — " f"{resp.text[:120]!r} (tentative {attempt + 1})") except httpx.HTTPError as exc: Actor.log.warning( f"Bright Data réseau {url} : {exc} " f"(tentative {attempt + 1})") if attempt < retries - 1: await asyncio.sleep(_backoff(attempt)) return None