# ============================================================================== # Author: Simon-Pierre Boucher # File: src/net.py # Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) + # proxy Apify (résidentiel par défaut), throttling poli, retries à # backoff EXPONENTIEL + jitter sur 403/407/408/425/429/500/502/503/504 # avec rotation de session proxy (nouvelle IP) à chaque tentative. # GET et POST partagent le même moteur de retries. # ============================================================================== from __future__ import annotations import asyncio import random import re from apify import Actor from curl_cffi import requests as cffi UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") # statuts transitoires : mur anti-bot, proxy, rate-limit, erreurs serveur RETRY_STATUSES = frozenset({403, 407, 408, 425, 429, 500, 502, 503, 504}) def _backoff(attempt: int) -> float: """Backoff exponentiel plafonné + jitter (anti-troupeau).""" return min(1.2 * (2 ** attempt), 12.0) + random.uniform(0.0, 0.8) class Fetcher: """HTTP poli : proxy Apify + impersonation Chrome + retries robustes.""" def __init__(self, proxy_configuration, delay: float = 1.0) -> None: self.proxy_configuration = proxy_configuration self.delay = delay self._lock = asyncio.Lock() self._last = 0.0 async def _throttle(self) -> None: async with self._lock: now = asyncio.get_event_loop().time() wait = self.delay - (now - self._last) if wait > 0: await asyncio.sleep(wait) self._last = asyncio.get_event_loop().time() async def request(self, method: str, url: str, headers: dict | None = None, data=None, retries: int = 4, session_id: str | None = None, impersonate: str | None = "chrome"): hdrs = {"User-Agent": UA, "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"} if headers: hdrs.update(headers) last_exc: Exception | None = None for attempt in range(retries): await self._throttle() proxy = None if self.proxy_configuration: sid = session_id or f"s{random.randint(1, 999_999)}" sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s" # nouvelle session proxy (nouvelle IP) à chaque tentative proxy = await self.proxy_configuration.new_url( session_id=f"{sid}r{attempt}") try: resp = await asyncio.to_thread( cffi.request, method, url, headers=hdrs, data=data, impersonate=impersonate, timeout=60, allow_redirects=True, proxies={"http": proxy, "https": proxy} if proxy else None) if resp.status_code in RETRY_STATUSES \ and attempt < retries - 1: Actor.log.warning( f"HTTP {resp.status_code} {url} — retry {attempt + 1}") await asyncio.sleep(_backoff(attempt)) continue return resp except Exception as exc: # réseau/proxy : on retente last_exc = exc await asyncio.sleep(_backoff(attempt)) if last_exc: raise last_exc raise RuntimeError(f"échec après {retries} tentatives : {url}") async def get(self, url: str, headers: dict | None = None, retries: int = 4, session_id: str | None = None, impersonate: str | None = "chrome"): return await self.request("GET", url, headers=headers, retries=retries, session_id=session_id, impersonate=impersonate) async def post(self, url: str, data=None, headers: dict | None = None, retries: int = 4, session_id: str | None = None, impersonate: str | None = "chrome"): return await self.request("POST", url, data=data, headers=headers, retries=retries, session_id=session_id, impersonate=impersonate)