Repository Public
1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File: src/net.py4# Desc: Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +5# proxy Apify (résidentiel par défaut), throttling poli + retries6# ==============================================================================7from __future__ import annotations89import asyncio10import random11import re1213from apify import Actor14from curl_cffi import requests as cffi1516UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "17 "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")181920class Fetcher:21 """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""2223 def __init__(self, proxy_configuration, delay: float = 1.0) -> None:24 self.proxy_configuration = proxy_configuration25 self.delay = delay26 self._lock = asyncio.Lock()27 self._last = 0.02829 async def _throttle(self) -> None:30 async with self._lock:31 now = asyncio.get_event_loop().time()32 wait = self.delay - (now - self._last)33 if wait > 0:34 await asyncio.sleep(wait)35 self._last = asyncio.get_event_loop().time()3637 async def get(self, url: str, headers: dict | None = None,38 retries: int = 3, session_id: str | None = None,39 impersonate: str | None = "chrome"):40 hdrs = {"User-Agent": UA,41 "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}42 if headers:43 hdrs.update(headers)44 last_exc: Exception | None = None45 for attempt in range(retries):46 await self._throttle()47 proxy = None48 if self.proxy_configuration:49 sid = session_id or f"s{random.randint(1, 999_999)}"50 sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"51 proxy = await self.proxy_configuration.new_url(52 session_id=f"{sid}r{attempt}")53 try:54 resp = await asyncio.to_thread(55 cffi.get, url, headers=hdrs, impersonate=impersonate,56 timeout=60, allow_redirects=True,57 proxies={"http": proxy, "https": proxy} if proxy else None)58 if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:59 Actor.log.warning(60 f"HTTP {resp.status_code} {url} — retry {attempt + 1}")61 await asyncio.sleep(1.5 * (attempt + 1))62 continue63 return resp64 except Exception as exc: # réseau/proxy : on retente65 last_exc = exc66 await asyncio.sleep(1.5 * (attempt + 1))67 if last_exc:68 raise last_exc69 raise RuntimeError(f"échec après {retries} tentatives : {url}")70