spb/ka4 Public

1# Author: Simon-Pierre Boucher2# Contact: contact@spboucher.ai3"""Dispatcher de scrape multi-backend + garde robots.txt.45- search / map : toujours via Firecrawl.6- scrape       : selon le backend choisi (auto | firecrawl | scrapfly).7  En mode `auto`, Firecrawl est tenté d'abord, puis Scrapfly en repli si le8  résultat est vide ou en erreur.9"""1011from __future__ import annotations1213from typing import Any, Optional1415from .config import Config16from .firecrawl_client import FirecrawlClient17from .robots import RobotsCache18from .scrapfly_client import ScrapflyClient192021class Scraper:22    def __init__(self, cfg: Config):23        self.cfg = cfg24        self.firecrawl = FirecrawlClient(cfg.firecrawl_api_key, cfg.firecrawl_base, cfg.request_delay)25        self.scrapfly = (26            ScrapflyClient(cfg.scrapfly_api_key, cfg.scrapfly_base, cfg.request_delay, cfg.scrapfly_render_js)27            if cfg.scrapfly_api_key28            else None29        )30        self.robots = RobotsCache(cfg.user_agent, enabled=cfg.respect_robots)31        self.backend = cfg.scraper_backend3233    # -- découverte (Firecrawl) -------------------------------------------34    def search(self, query: str, limit: int = 10) -> list[dict[str, Any]]:35        return self.firecrawl.search(query, limit)3637    def map(self, url: str, search: Optional[str] = None) -> list[str]:38        return self.firecrawl.map(url, search)3940    # -- scrape ------------------------------------------------------------41    def scrape(self, url: str) -> dict[str, Any]:42        if not self.robots.allowed(url):43            return {"blocked": True, "markdown": "", "metadata": {}, "links": [], "backend": None}4445        order = self._order()46        last_err: Optional[Exception] = None47        for backend in order:48            try:49                data = self._scrape_with(backend, url)50                if data.get("markdown", "").strip():51                    return data52            except Exception as e:  # noqa: BLE00153                last_err = e54        if last_err:55            return {"error": str(last_err), "markdown": "", "metadata": {}, "links": [], "backend": None}56        return {"markdown": "", "metadata": {}, "links": [], "backend": order[-1] if order else None}5758    def _order(self) -> list[str]:59        if self.backend == "firecrawl":60            return ["firecrawl"]61        if self.backend == "scrapfly":62            return ["scrapfly"] if self.scrapfly else ["firecrawl"]63        # auto : Firecrawl puis Scrapfly en repli64        return ["firecrawl", "scrapfly"] if self.scrapfly else ["firecrawl"]6566    def _scrape_with(self, backend: str, url: str) -> dict[str, Any]:67        if backend == "scrapfly":68            if not self.scrapfly:69                raise RuntimeError("Scrapfly non configuré (SCRAPFLY_API_KEY manquant)")70            return self.scrapfly.scrape(url)71        return self.firecrawl.scrape(url)72