SPB Git

Repository Public

net.py 6.2 KB 156 lines python

1# =============================================================================2#                  SP BOUCHER | SOFTWARE & RESEARCH3#             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4# =============================================================================5#6# PROJECT METADATA7# -----------------------------------------------------------------------------8# Project          : KA Apify Actors9# File             : ka-twitch/src/net.py10# Module           : ka-twitch11# Repository       : https://git.spboucher.ai/ka-apify-actors12# Created          : 2026-08-1813# Last Modified    : 2026-08-1814#15# =============================================================================16# AUTHOR & PROFESSIONAL AFFILIATION17# =============================================================================18#19# Author           : Simon-Pierre Boucher20# Position         : Professor of Finance21# Institution      : Université du Québec en Outaouais (UQO)22# Department       : Département des sciences administratives23# Organization     : SP Boucher24#25# Website          : https://spboucher.ai26# Git              : https://git.spboucher.ai27#28# =============================================================================29# OFFICIAL CONTACT DIRECTORY30# =============================================================================31#32# General          : info@spboucher.ai33# Professional     : contact@spboucher.ai34# Direct           : simon@spboucher.ai35# Research         : research@spboucher.ai36# Development      : dev@spboucher.ai37# Support          : support@spboucher.ai38# Security/Admin   : admin@spboucher.ai39# Notifications    : notifications@spboucher.ai40#41# Preferred Contact: contact@spboucher.ai42#43# =============================================================================44# COPYRIGHT & INTELLECTUAL PROPERTY45# =============================================================================46#47# Copyright (c) 202648# Simon-Pierre Boucher49#50# All rights reserved where applicable.51#52# This file is governed by the applicable project license,53# copyright notices, institutional agreements and relevant law.54#55# Unless expressly authorized by the applicable license or56# rights holder, copying, redistribution, sublicensing,57# publication and commercial exploitation are not permitted.58#59# Existing third-party rights and notices remain unaffected.60#61# =============================================================================62# CONFIDENTIALITY & SECURITY63# =============================================================================64#65# Classification   : Public66# Distribution     : Public repository; use is governed by the project license67# Security Contact : admin@spboucher.ai68#69# Access to this source does not independently grant permission70# to redistribute or disclose confidential materials.71#72# Do not embed credentials, private keys, access tokens,73# certificates or confidential personal information.74#75# =============================================================================76# LICENSE & MAINTENANCE77# =============================================================================78#79# License          : None specified (no license file in this repository)80# Maintainer       : Simon-Pierre Boucher81# Technical Contact: dev@spboucher.ai82#83# =============================================================================84# END OF SP BOUCHER SOURCE HEADER85# =============================================================================8687# ==============================================================================88# Author: Simon-Pierre Boucher <contact@spboucher.ai>89# File:   src/net.py90# Desc:   Fetch commun des acteurs KA — curl_cffi (empreinte TLS Chrome) +91#         proxy Apify (résidentiel par défaut), throttling poli + retries92# ==============================================================================93from __future__ import annotations9495import asyncio96import random97import re9899from apify import Actor100from curl_cffi import requests as cffi101102UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "103      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")104105106class Fetcher:107    """GET poli avec proxy Apify + impersonation Chrome + retries 403/429."""108109    def __init__(self, proxy_configuration, delay: float = 1.0) -> None:110        self.proxy_configuration = proxy_configuration111        self.delay = delay112        self._lock = asyncio.Lock()113        self._last = 0.0114115    async def _throttle(self) -> None:116        async with self._lock:117            now = asyncio.get_event_loop().time()118            wait = self.delay - (now - self._last)119            if wait > 0:120                await asyncio.sleep(wait)121            self._last = asyncio.get_event_loop().time()122123    async def get(self, url: str, headers: dict | None = None,124                  retries: int = 3, session_id: str | None = None,125                  impersonate: str | None = "chrome"):126        hdrs = {"User-Agent": UA,127                "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}128        if headers:129            hdrs.update(headers)130        last_exc: Exception | None = None131        for attempt in range(retries):132            await self._throttle()133            proxy = None134            if self.proxy_configuration:135                sid = session_id or f"s{random.randint(1, 999_999)}"136                sid = re.sub(r"[^\w._~]", "", sid)[:40] or "s"137                proxy = await self.proxy_configuration.new_url(138                    session_id=f"{sid}r{attempt}")139            try:140                resp = await asyncio.to_thread(141                    cffi.get, url, headers=hdrs, impersonate=impersonate,142                    timeout=60, allow_redirects=True,143                    proxies={"http": proxy, "https": proxy} if proxy else None)144                if resp.status_code in (403, 407, 429, 502) and attempt < retries - 1:145                    Actor.log.warning(146                        f"HTTP {resp.status_code} {url} — retry {attempt + 1}")147                    await asyncio.sleep(1.5 * (attempt + 1))148                    continue149                return resp150            except Exception as exc:  # réseau/proxy : on retente151                last_exc = exc152                await asyncio.sleep(1.5 * (attempt + 1))153        if last_exc:154            raise last_exc155        raise RuntimeError(f"échec après {retries} tentatives : {url}")156
Files main
Menu

Theme