spb/house-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# mortgage/providers/base.py : classe de base des connecteurs de taux.5#   Contrat : fetch() retourne la liste des produits NORMALISÉS de6#   l'institution (dicts au format de make_product). Aucune écriture BD ici —7#   le scheduler valide puis enregistre. Jamais de taux inventé : un produit8#   non confirmé est simplement omis.9# -----------------------------------------------------------------------------10from __future__ import annotations1112import os13import re14import time1516import requests1718USER_AGENT = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "19              "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36 "20              "ImmoKaBot/1.0 (+https://www.immo-ka.com/bot; contact@spboucher.ai)")2122SCRAPFLY_API = "https://api.scrapfly.io/scrape"232425class RateProvider:26    provider_id: str = ""          # slug (rbc, td, desjardins…)27    institution: str = ""          # nom d'affichage28    source_url: str = ""           # page officielle des taux (lien public)29    request_delay: float = 1.0     # politesse30    timeout: int = 303132    def __init__(self) -> None:33        self.session = requests.Session()34        self.session.headers["User-Agent"] = USER_AGENT35        self._last_request = 0.03637    # -- backends -------------------------------------------------------------38    def get(self, url: str, **kw) -> requests.Response:39        wait = self.request_delay - (time.time() - self._last_request)40        if wait > 0:41            time.sleep(wait)42        resp = self.session.get(url, timeout=self.timeout, **kw)43        self._last_request = time.time()44        resp.raise_for_status()45        return resp4647    def get_scrapfly(self, url: str, render_js: bool = False,48                     asp: bool = True, country: str = "ca") -> str:49        """HTML via Scrapfly (anti-bot) — dernier recours seulement."""50        key = os.environ.get("SCRAPFLY_KEY") or os.environ.get("SCRAPFLY_API_KEY")51        if not key:52            raise RuntimeError("SCRAPFLY_KEY manquant (voir .env)")53        params: dict = {"key": key, "url": url, "country": country}54        if asp:55            params["asp"] = "true"56        if render_js:57            params["render_js"] = "true"58        wait = self.request_delay - (time.time() - self._last_request)59        if wait > 0:60            time.sleep(wait)61        resp = requests.get(SCRAPFLY_API, params=params, timeout=180)62        self._last_request = time.time()63        try:64            return (resp.json().get("result") or {}).get("content") or ""65        except ValueError:66            return ""6768    # -- normalisation --------------------------------------------------------69    def make_product(self, *, rate: float, rate_type: str, term_months: int,70                     kind: str, product_name: str | None = None,71                     apr: float | None = None,72                     insured_status: str = "unknown",73                     purpose: str = "purchase",74                     amortization_max_years: int | None = None,75                     conditions: str | None = None,76                     source_url: str | None = None,77                     confidence: float = 1.0,78                     raw=None) -> dict:79        """Observation normalisée commune à tous les providers."""80        return {81            "provider": self.provider_id,82            "institution": self.institution,83            "product_name": product_name,84            "rate_type": rate_type,85            "term_months": int(term_months),86            "kind": kind,87            "rate": round(float(rate), 4),88            "apr": round(float(apr), 4) if apr is not None else None,89            "insured_status": insured_status,90            "purpose": purpose,91            "amortization_max_years": amortization_max_years,92            "conditions": conditions,93            "source_url": source_url or self.source_url,94            "confidence": confidence,95            "raw": raw,96        }9798    # -- contrat --------------------------------------------------------------99    def fetch(self) -> list[dict]:100        raise NotImplementedError101102    # Point d'entrée pour les tests avec fixtures : si le provider parse un103    # payload texte, il expose parse(payload) et fetch() = parse(download()).104    def parse(self, payload: str) -> list[dict]:  # pragma: no cover105        raise NotImplementedError106107108def parse_rate(text: str) -> float | None:109    """Extrait un taux en % d'un texte : '4,19 %', '4.19%', ' 4.19 '.110    Retourne None si introuvable — jamais de valeur devinée."""111    if text is None:112        return None113    m = re.search(r"(\d{1,2}(?:[.,]\d{1,4})?)\s*%?", str(text).strip())114    if not m:115        return None116    try:117        return float(m.group(1).replace(",", "."))118    except ValueError:119        return None120121122def term_to_months(text: str) -> int | None:123    """'5 ans', '5 year', '5-year', '6 months', '6 mois' → mois."""124    if not text:125        return None126    t = str(text).lower()127    m = re.search(r"(\d{1,3})\s*(?:-|\s)?\s*(an|ans|année|year|yr)", t)128    if m:129        return int(m.group(1)) * 12130    m = re.search(r"(\d{1,3})\s*(?:-|\s)?\s*(mois|month)", t)131    if m:132        return int(m.group(1))133    return None134