spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/base.py : classe de base des connecteurs + backends de fetch5#                      (requests direct, Firecrawl ou Scrapfly pour les sites6#                       JavaScript / derrière anti-bot)7# -----------------------------------------------------------------------------8from __future__ import annotations910import json11import os12import time1314import requests1516from ..schema import Listing1718USER_AGENT = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "19              "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36 "20              "LouKaBot/1.0 (+https://www.lou-ka.com/bot; contact@spboucher.ai)")2122FIRECRAWL_API = "https://api.firecrawl.dev/v1/scrape"23SCRAPFLY_API = "https://api.scrapfly.io/scrape"242526class BaseConnector:27    """Un connecteur = un adaptateur propre à un site d'agence.2829    Sous-classes : définir `source_id` et implémenter `fetch()` qui retourne30    la liste complète des annonces actuellement affichées sur le site.31    Le pipeline (ingest.py) s'occupe du diff avec la base de données.32    """3334    source_id: str = ""35    request_delay: float = 0.6           # politesse entre requêtes36    timeout: int = 3037    use_detail_cache: bool = True        # cache BD des pages détail3839    def __init__(self) -> None:40        self.session = requests.Session()41        self.session.headers["User-Agent"] = USER_AGENT42        self._last_request = 0.043        self._detail_con = None4445    # -- backends -------------------------------------------------------------46    def get(self, url: str, **kw) -> requests.Response:47        """GET direct avec throttling poli."""48        wait = self.request_delay - (time.time() - self._last_request)49        if wait > 0:50            time.sleep(wait)51        resp = self.session.get(url, timeout=self.timeout, **kw)52        self._last_request = time.time()53        resp.raise_for_status()54        return resp5556    def post(self, url: str, **kw) -> requests.Response:57        """POST direct avec throttling poli (APIs de recherche internes)."""58        wait = self.request_delay - (time.time() - self._last_request)59        if wait > 0:60            time.sleep(wait)61        resp = self.session.post(url, timeout=self.timeout, **kw)62        self._last_request = time.time()63        resp.raise_for_status()64        return resp6566    def get_rendered(self, url: str) -> str:67        """Récupère le HTML rendu (JavaScript exécuté) via Firecrawl.6869        Nécessite FIRECRAWL_API_KEY dans l'environnement (.env).70        À utiliser pour les sites SPA (Logisco, Locago, etc.).71        """72        key = os.environ.get("FIRECRAWL_API_KEY")73        if not key:74            raise RuntimeError("FIRECRAWL_API_KEY manquant (voir .env)")75        resp = requests.post(76            FIRECRAWL_API,77            json={"url": url, "formats": ["html"]},78            headers={"Authorization": f"Bearer {key}"},79            timeout=90,80        )81        resp.raise_for_status()82        data = resp.json()83        return (data.get("data") or {}).get("html", "")8485    def scrapfly(self, url: str, render_js: bool = True, asp: bool = True,86                 rendering_wait: int = 0, country: str = "ca",87                 wait_for_selector: str | None = None,88                 js_scenario: list | str | None = None,89                 proxy_pool: str | None = None, headers: dict | None = None,90                 method: str = "GET", body: str | None = None) -> dict:91        """Appel Scrapfly complet — retourne le dict `result` (content, status_code…).9293        - `js_scenario` : liste d'étapes [{"scroll_y":…},{"wait":…}] (encodée base64)94          pour charger les listes virtualisées (BoldTrail/kvCORE, etc.).95        - `proxy_pool` : ex. "public_residential_pool" (WAF/anti-bot agressif).96        - `headers`/`method`/`body` : pour REJOUER une API JSON interne via ASP.97        """98        import base6499        key = os.environ.get("SCRAPFLY_KEY")100        if not key:101            raise RuntimeError("SCRAPFLY_KEY manquant (voir .env)")102        params: dict = {"key": key, "url": url, "country": country}103        if asp:104            params["asp"] = "true"105        if render_js:106            params["render_js"] = "true"107        if rendering_wait:108            params["rendering_wait"] = rendering_wait109        if wait_for_selector:110            params["wait_for_selector"] = wait_for_selector111        if proxy_pool:112            params["proxy_pool"] = proxy_pool113        if js_scenario is not None:114            js = js_scenario if isinstance(js_scenario, str) else json.dumps(js_scenario)115            params["js_scenario"] = base64.urlsafe_b64encode(js.encode()).decode()116        if headers:117            for k, v in headers.items():118                params[f"headers[{k}]"] = v119        wait = self.request_delay - (time.time() - self._last_request)120        if wait > 0:121            time.sleep(wait)122        if method.upper() == "POST":123            resp = requests.post(SCRAPFLY_API, params=params,124                                 data=(body or ""), timeout=180)125        else:126            resp = requests.get(SCRAPFLY_API, params=params, timeout=180)127        self._last_request = time.time()128        try:129            return resp.json().get("result") or {}130        except ValueError:131            return {}132133    def get_scrapfly(self, url: str, render_js: bool = True, asp: bool = True,134                     rendering_wait: int = 0, country: str = "ca",135                     wait_for_selector: str | None = None,136                     js_scenario: list | str | None = None,137                     proxy_pool: str | None = None) -> str:138        """HTML rendu via Scrapfly (ASP = bypass anti-bot + rendu JS). Retourne139        le HTML (result.content) ou "" en cas d'échec ASP."""140        return self.scrapfly(url, render_js=render_js, asp=asp,141                             rendering_wait=rendering_wait, country=country,142                             wait_for_selector=wait_for_selector,143                             js_scenario=js_scenario, proxy_pool=proxy_pool144                             ).get("content") or ""145146    def detail(self, external_id: str, key: str, fetch_fn) -> dict:147        """Payload « page détail » avec cache : `fetch_fn` n'est appelé que si148        l'annonce est nouvelle ou si sa clé (hash du contenu liste) a changé.149150        Permet d'extraire les champs riches (description, inclusions, contact…)151        sans revisiter chaque page détail à chaque synchronisation.152        `fetch_fn` doit retourner un dict JSON-sérialisable.153        """154        if not self.use_detail_cache:155            return fetch_fn() or {}156        from .. import db157        if self._detail_con is None:158            self._detail_con = db.connect()159        cached = db.get_cached_detail(self._detail_con, self.source_id,160                                      str(external_id), key)161        if cached is not None:162            return cached163        payload = fetch_fn() or {}164        db.put_cached_detail(self._detail_con, self.source_id,165                             str(external_id), key, payload)166        return payload167168    # -- contrat --------------------------------------------------------------169    def fetch(self) -> list[Listing]:170        raise NotImplementedError171