spb/crea-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/net.py4# Desc:   Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +5#         proxy Apify (résidentiel par défaut), throttling poli, retries à6#         backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/5047#         avec rotation de session proxy (nouvelle IP) à chaque tentative.8#         GET et POST partagent le même moteur de retries.9# ==============================================================================10from __future__ import annotations1112import asyncio13import random14import re1516from apify import Actor17from curl_cffi import requests as cffi1819UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "20      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")2122# statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur23RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504})242526def _backoff(attempt: int) -> float:27    """Backoff exponentiel plafonné + jitter (anti-troupeau)."""28    return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8)293031class Fetcher:32    """HTTP poli : proxy Apify + impersonation Chrome + retries robustes."""3334    def __init__(self, proxy_configuration, delay: float = 1.0) -> None:35        self.proxy_configuration = proxy_configuration36        self.delay = delay37        self._lock = asyncio.Lock()38        self._last = 0.03940    async def _throttle(self) -> None:41        async with self._lock:42            now = asyncio.get_event_loop().time()43            wait = self.delay - (now - self._last)44            if wait > 0:45                await asyncio.sleep(wait)46            self._last = asyncio.get_event_loop().time()4748    async def request(self, method: str, url: str,49                      headers: dict | None = None, data=None,50                      retries: int = 4, session_id: str | None = None,51                      impersonate: str | None = "chrome"):52        hdrs = {"User-Agent": UA,53                "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}54        if headers:55            hdrs.update(headers)56        last_exc: Exception | None = None57        for attempt in range(retries):58            await self._throttle()59            proxy = None60            if self.proxy_configuration:61                sid = session_id or f"s{random.randint(1, 999_999)}"62                sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"63                # nouvelle session proxy (nouvelle IP) à chaque tentative64                proxy = await self.proxy_configuration.new_url(65                    session_id=f"{sid}r{attempt}")66            try:67                resp = await asyncio.to_thread(68                    cffi.request, method, url, headers=hdrs, data=data,69                    impersonate=impersonate, timeout=60, allow_redirects=True,70                    proxies={"http": proxy, "https": proxy} if proxy else None)71                if resp.status_code in RETRY_STATUSES \72                        and attempt < retries - 1:73                    Actor.log.warning(74                        f"HTTP {resp.status_code} {url} — retry {attempt + 1}")75                    await asyncio.sleep(_backoff(attempt))76                    continue77                return resp78            except Exception as exc:  # réseau/proxy : on retente79                last_exc = exc80                await asyncio.sleep(_backoff(attempt))81        if last_exc:82            raise last_exc83        raise RuntimeError(f"échec après {retries} tentatives : {url}")8485    async def get(self, url: str, headers: dict | None = None,86                  retries: int = 4, session_id: str | None = None,87                  impersonate: str | None = "chrome"):88        return await self.request("GET", url, headers=headers,89                                  retries=retries, session_id=session_id,90                                  impersonate=impersonate)9192    async def post(self, url: str, data=None, headers: dict | None = None,93                   retries: int = 4, session_id: str | None = None,94                   impersonate: str | None = "chrome"):95        return await self.request("POST", url, data=data, headers=headers,96                                  retries=retries, session_id=session_id,97                                  impersonate=impersonate)98