spb/rent-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/net.py (ka-apartments-com)4# Desc:   Fetcher Bright Data Web Unlocker — apartments.com est verrouillé par5#         Akamai (curl_cffi/Scrapfly/Oxylabs → 403) ; seul le Web Unlocker6#         passe. Simple POST https://api.brightdata.com/request qui renvoie le7#         HTML brut de la page débloquée. httpx async + sémaphore + retries à8#         backoff exponentiel.9# ==============================================================================10from __future__ import annotations1112import asyncio13import random1415import httpx1617from apify import Actor1819API = "https://api.brightdata.com/request"202122def _backoff(attempt: int) -> float:23    return min(1.5 * (2 ** attempt), 15.0) + random.uniform(0.0, 1.0)242526class BrightData:27    """GET d'une page via le Web Unlocker Bright Data (déblocage géré)."""2829    def __init__(self, token: str, zone: str, concurrency: int = 4,30                 delay: float = 0.0) -> None:31        self._headers = {"Authorization": f"Bearer {token}",32                         "Content-Type": "application/json"}33        self._zone = zone34        self._sem = asyncio.Semaphore(max(1, concurrency))35        self._delay = max(0.0, delay)36        self._client = httpx.AsyncClient(timeout=120.0)3738    async def close(self) -> None:39        await self._client.aclose()4041    async def get(self, url: str, retries: int = 3) -> str | None:42        """HTML de la page, ou None si le déblocage échoue après retries."""43        body = {"zone": self._zone, "url": url,44                "format": "raw", "country": "ca"}45        async with self._sem:46            for attempt in range(retries):47                try:48                    resp = await self._client.post(49                        API, headers=self._headers, json=body)50                    if resp.status_code == 200 and resp.text.strip():51                        if self._delay:52                            await asyncio.sleep(self._delay)53                        return resp.text54                    Actor.log.warning(55                        f"Bright Data {resp.status_code} {url} — "56                        f"{resp.text[:120]!r} (tentative {attempt + 1})")57                except httpx.HTTPError as exc:58                    Actor.log.warning(59                        f"Bright Data réseau {url} : {exc} "60                        f"(tentative {attempt + 1})")61                if attempt < retries - 1:62                    await asyncio.sleep(_backoff(attempt))63        return None64