SPB Git

Repository Public

net.py 2.8 KB 70 lines python

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   src/net.py4# Desc:   Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +5#         proxy Apify (résidentiel par défaut), throttling poli + retries6# ==============================================================================7from __future__ import annotations89import asyncio10import random11import re1213from apify import Actor14from curl_cffi import requests as cffi1516UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "17      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")181920class Fetcher:21    """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""2223    def __init__(self, proxy_configuration, delay: float = 1.0) -> None:24        self.proxy_configuration = proxy_configuration25        self.delay = delay26        self._lock = asyncio.Lock()27        self._last = 0.02829    async def _throttle(self) -> None:30        async with self._lock:31            now = asyncio.get_event_loop().time()32            wait = self.delay - (now - self._last)33            if wait > 0:34                await asyncio.sleep(wait)35            self._last = asyncio.get_event_loop().time()3637    async def get(self, url: str, headers: dict | None = None,38                  retries: int = 3, session_id: str | None = None,39                  impersonate: str | None = "chrome"):40        hdrs = {"User-Agent": UA,41                "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}42        if headers:43            hdrs.update(headers)44        last_exc: Exception | None = None45        for attempt in range(retries):46            await self._throttle()47            proxy = None48            if self.proxy_configuration:49                sid = session_id or f"s{random.randint(1, 999_999)}"50                sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"51                proxy = await self.proxy_configuration.new_url(52                    session_id=f"{sid}r{attempt}")53            try:54                resp = await asyncio.to_thread(55                    cffi.get, url, headers=hdrs, impersonate=impersonate,56                    timeout=60, allow_redirects=True,57                    proxies={"http": proxy, "https": proxy} if proxy else None)58                if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:59                    Actor.log.warning(60                        f"HTTP {resp.status_code} {url} — retry {attempt + 1}")61                    await asyncio.sleep(1.5 * (attempt + 1))62                    continue63                return resp64            except Exception as exc:  # réseau/proxy : on retente65                last_exc = exc66                await asyncio.sleep(1.5 * (attempt + 1))67        if last_exc:68            raise last_exc69        raise RuntimeError(f"échec après {retries} tentatives : {url}")70
Files main
Menu

Theme