spb/ka2 Public

1# Author: Simon-Pierre Boucher2# Contact: contact@spboucher.ai3"""Client REST Scrapfly (https://scrapfly.io) — backend de scrape alternatif.45Atouts : rendu JavaScript, contournement anti-bot (ASP), proxies. Utile quand6Firecrawl échoue ou renvoie une page vide (SPA, protection anti-scraping).7"""89from __future__ import annotations1011import time12from typing import Any, Optional1314import requests1516from .util import RateLimiter171819class ScrapflyClient:20    def __init__(self, api_key: str, base_url: str = "https://api.scrapfly.io", delay: float = 1.5,21                 render_js: bool = False):22        self.api_key = api_key23        self.base = base_url.rstrip("/")24        self.render_js = render_js25        self.session = requests.Session()26        self.limiter = RateLimiter(delay)2728    def scrape(self, url: str, render_js: Optional[bool] = None, retries: int = 2) -> dict[str, Any]:29        params = {30            "key": self.api_key,31            "url": url,32            "format": "markdown",33            "asp": "true",  # Anti Scraping Protection bypass34            "render_js": "true" if (self.render_js if render_js is None else render_js) else "false",35        }36        last_err: Optional[Exception] = None37        for attempt in range(retries):38            self.limiter.wait()39            try:40                r = self.session.get(f"{self.base}/scrape", params=params, timeout=70)41                if r.status_code == 429:42                    time.sleep(1.5 * (attempt + 1))43                    continue44                r.raise_for_status()45                result = (r.json() or {}).get("result", {}) or {}46                content = result.get("content", "") or ""47                meta = {48                    "title": (result.get("metadata") or {}).get("title", "")49                    if isinstance(result.get("metadata"), dict)50                    else "",51                    "status_code": result.get("status_code"),52                    "url": result.get("url", url),53                }54                return {"markdown": content, "metadata": meta, "links": [], "backend": "scrapfly"}55            except Exception as e:  # noqa: BLE00156                last_err = e57                time.sleep(1.5 * (attempt + 1))58        raise RuntimeError(f"Scrapfly scrape a échoué: {last_err}")59