spb/lou-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/net.py (ka-fb-marketplace)4# Desc:   Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +5#         proxy Apify (résidentiel par défaut) OU gabarit de proxy externe6#         ({session} → id de session, nouvelle IP par tentative), throttling7#         poli, retries à backoff exponentiel + jitter sur 403/407/408/425/8#         429/500/502/503/504.9# ==============================================================================10from __future__ import annotations1112import asyncio13import random14import re1516from apify import Actor17from curl_cffi import requests as cffi1819UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "20      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")2122# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur23RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})242526def _backoff(attempt: int) -> float:27    """Backoff exponentiel plafonné + jitter (anti-troupeau)."""28    return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)293031class Fetcher:32    """HTTP poli : proxy Apify ou externe + impersonation Chrome + retries."""3334    def __init__(self, proxy_configuration, delay: float = 1.0,35                 proxy_template: str | None = None) -> None:36        self.proxy_configuration = proxy_configuration37        self.proxy_template = proxy_template or None38        self.delay = delay39        self._lock = asyncio.Lock()40        self._last = 0.04142    async def _throttle(self) -> None:43        async with self._lock:44            now = asyncio.get_event_loop().time()45            wait = self.delay - (now - self._last)46            if wait > 0:47                await asyncio.sleep(wait)48            self._last = asyncio.get_event_loop().time()4950    async def _proxy_url(self, session_id: str | None, attempt: int) -> str | None:51        sid = session_id or f"s{random.randint(1, 999_999)}"52        sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"53        # nouvelle session proxy (nouvelle IP) à chaque tentative54        sid = f"{sid}r{attempt}"55        if self.proxy_template:56            return self.proxy_template.replace("{session}", sid)57        if self.proxy_configuration:58            return await self.proxy_configuration.new_url(session_id=sid)59        return None6061    async def request(self, method: str, url: str,62                      headers: dict | None = None, data=None,63                      retries: int = 4, session_id: str | None = None,64                      impersonate: str | None = "chrome"):65        hdrs = {"User-Agent": UA,66                "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}67        if headers:68            hdrs.update(headers)69        last_exc: Exception | None = None70        for attempt in range(retries):71            await self._throttle()72            proxy = await self._proxy_url(session_id, attempt)73            try:74                resp = await asyncio.to_thread(75                    cffi.request, method, url, headers=hdrs, data=data,76                    impersonate=impersonate, timeout=60, allow_redirects=True,77                    proxies={"http": proxy, "https": proxy} if proxy else None)78                if resp.status_code in RETRY_STATUSES \79                        and attempt < retries - 1:80                    Actor.log.warning(81                        f"HTTP {resp.status_code} {url} — retry {attempt + 1}")82                    await asyncio.sleep(_backoff(attempt))83                    continue84                return resp85            except Exception as exc:  # réseau/proxy : on retente86                last_exc = exc87                await asyncio.sleep(_backoff(attempt))88        if last_exc:89            raise last_exc90        raise RuntimeError(f"échec après {retries} tentatives : {url}")9192    async def get(self, url: str, headers: dict | None = None,93                  retries: int = 4, session_id: str | None = None,94                  impersonate: str | None = "chrome"):95        return await self.request("GET", url, headers=headers,96                                  retries=retries, session_id=session_id,97                                  impersonate=impersonate)98