spb/immo-ka Public

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/sothebys_quebec.py : Sotheby's International Realty Québec5#   Recherche rendue côté client : le grid public est plafonné (~40/page) et la6#   pagination passe par du JS (pas d'URL). On SHARD donc par région7#   administrative du Québec (chaque page « region-{slug}-real-estate » renvoie8#   son lot), on dédoublonne par id, puis on enrichit chaque fiche via son9#   JSON-LD `Product` (prix, galerie S3 complète, description, n° MLS = sku).10# -----------------------------------------------------------------------------11from __future__ import annotations1213import html as _html14import json15import os16import re1718from .base import BaseConnector19from . import _detailutil as du20from ..schema import PropertyListing2122SITE = "https://sothebysrealty.ca"23DETAIL_LIMIT = int(os.environ.get("IMMOKA_SOTHEBYS_DETAIL_LIMIT", "400"))2425# Régions administratives du Québec (slugs de recherche Sotheby's).26QC_REGIONS = [27    "region-montreal", "region-laval", "region-monteregie", "region-laurentides",28    "region-lanaudiere", "region-eastern-townships", "region-outaouais",29    "region-capitale-nationale", "region-mauricie", "region-centre-du-quebec",30    "region-chaudiere-appalaches", "region-bas-saint-laurent",31    "region-saguenay-lac-saint-jean", "region-cote-nord", "region-charlevoix",32    "region-gaspesie", "region-abitibi-temiscamingue",33]3435_PROP_RE = re.compile(36    r'/(?:fr|en)/property/quebec/region-([a-z-]+)/([a-z0-9-]+?)-real-estate/(\d{5,})/([a-z0-9-]+)/',37    re.I)383940class SothebysQuebecConnector(BaseConnector):41    source_id = "sothebys_quebec"42    request_delay = 1.04344    def _get_tolerant(self, url: str) -> str:45        """GET qui ignore le statut HTTP : les pages de résultats Sotheby's46        renvoient un 404 tout en servant le contenu complet."""47        import time48        wait = self.request_delay - (time.time() - self._last_request)49        if wait > 0:50            time.sleep(wait)51        resp = self.session.get(url, timeout=self.timeout)52        self._last_request = time.time()53        return resp.text5455    def fetch(self) -> list[PropertyListing]:56        by_id: dict[str, PropertyListing] = {}57        for region in QC_REGIONS:58            url = f"{SITE}/en/search-results/{region}-real-estate/"59            try:60                html = self._get_tolerant(url)61            except Exception:62                continue63            for m in _PROP_RE.finditer(html):64                lst = self._to_listing(m)65                if lst and lst.external_id not in by_id:66                    by_id[lst.external_id] = lst67        listings = list(by_id.values())68        # fiche détail : JSON-LD Product (prix, galerie complète, description, MLS)69        du.enrich(self, listings, DETAIL_LIMIT, parse_sothebys_detail, key="v1")70        return listings7172    def _to_listing(self, m: re.Match) -> PropertyListing | None:73        region_slug, city_slug, pid, addr_slug = m.group(1), m.group(2), m.group(3), m.group(4)74        url = f"{SITE}/fr/property/quebec/region-{region_slug}/{city_slug}-real-estate/{pid}/{addr_slug}/"75        return PropertyListing(76            source=self.source_id,77            external_id=pid,78            url=url,79            title=_deslug(addr_slug),80            address=_deslug(addr_slug),81            city=_deslug(city_slug),82            region=_deslug(region_slug),83            broker_name="Sotheby's International Realty Québec",84        )858687def parse_sothebys_detail(html: str) -> dict:88    """Extrait le JSON-LD Product : prix, galerie S3 complète, description, MLS."""89    out: dict = {}90    for node in du.ld_nodes(html):91        if node.get("@type") != "Product":92            continue93        offers = node.get("offers") or {}94        if isinstance(offers, list):95            offers = offers[0] if offers else {}96        price = offers.get("price")97        try:98            out["price"] = float(price) if price is not None else None99        except (TypeError, ValueError):100            pass101        if node.get("description"):102            out["description"] = _html.unescape(str(node["description"])).strip()103        if node.get("sku"):104            out["mls"] = str(node["sku"])105        cat = node.get("category")106        if cat:107            out["details"] = {"Type de propriété": str(cat)}108        break109110    # galerie complète : photos S3 de CETTE fiche (pleine résolution 'r'),111    # dédoublonnées par identifiant de base — plus complètes que le JSON-LD.112    mid = re.search(r'/live/images/listings/(\d+)/', html)113    if mid:114        lid = mid.group(1)115        seen, gallery = set(), []116        for u in re.findall(117                rf'https://sircmedia\.s3\.ca-central-1\.amazonaws\.com/live/images/listings/{lid}/'118                r'[^"\' ]+\.(?:jpg|jpeg|webp)', html):119            base = re.sub(r'(?:nr|m|r)?\.(?:jpg|jpeg|webp)$', '', u.rsplit("/", 1)[-1])120            full = re.sub(r'(?:nr|m)?\.jpg$', 'r.jpg', u)121            if base not in seen:122                seen.add(base)123                gallery.append(full)124        if gallery:125            out["images"] = gallery126    # repli sur les images du JSON-LD si aucune photo S3 trouvée127    if not out.get("images"):128        for node in du.ld_nodes(html):129            if node.get("@type") == "Product" and node.get("image"):130                imgs = node["image"]131                out["images"] = imgs if isinstance(imgs, list) else [imgs]132                break133134    # chambres / salles de bains (best-effort, valeurs plausibles seulement)135    text = re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", html)))136    mb = re.search(r"\b(\d{1,2})\s*(?:Bedroom|Chambre|bed\b|ch\.)", text, re.I)137    if mb and int(mb.group(1)) <= 20:138        out["bedrooms"] = int(mb.group(1))139    ms = re.search(r"\b(\d{1,2})\s*(?:Bathroom|Salle de bain|bath\b)", text, re.I)140    if ms and int(ms.group(1)) <= 20:141        out["bathrooms"] = int(ms.group(1))142    coords = du.gmaps_coords(html)143    if coords:144        out["lat"], out["lng"] = coords145    return out146147148def _deslug(s: str) -> str:149    return re.sub(r"\s+", " ", s.replace("-", " ")).strip().title()150