spb/immo-ka Public

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/sutton.py : Groupe Sutton Québec (suttonquebec.com)5#   Site rendu serveur : chaque page de résultats publie 10 propriétés en6#   JSON-LD schema.org (ItemList), avec prix, chambres, salles de bains,7#   superficie et adresse. Les photos suivent le motif immo.vrtx.co/pic450/{id}.8# -----------------------------------------------------------------------------9from __future__ import annotations1011import html as _html12import os13import re1415from .base import BaseConnector16from . import _detailutil as du17from .jsonld import item_list_elements, sqm_to_sqft18from ..schema import PropertyListing1920BASE = "https://www.suttonquebec.com"21DETAIL_LIMIT = int(os.environ.get("IMMOKA_SUTTON_DETAIL_LIMIT", "400"))22# toutes les photos de la fiche (la liste n'en montre que quelques-unes)23_DET_IMG_RE = re.compile(r'https://immo\.vrtx\.co/pic\d+/(\d+(?:-\d+)?)\.(?:jpg|jpeg|png|webp)', re.I)24_DESC_RE = re.compile(r'Description de la propri[ée]t[ée](?:\s+[àa]\s+vendre)?[^A-Za-z0-9]*(.*?)'25                      r'(?:Caract[ée]ristiques|Pi[èe]ce|Inclusions|Exclusions|Addenda|$)', re.I | re.S)26_ROOM_ROW_RE = re.compile(27    r'([A-ZÀ-Ÿ][^\d|]{2,38}?)\s+(\d[\d.,]*\s*[xX]\s*\d[\d.,]*)\s*P?\s+'28    r'([A-Za-zÀ-ÿ/ -]{3,30}?)\s+(RC|SS|AU|\d[eE]?r?e?)\b')29# Pages de résultats couvrant le résidentiel à vendre au Québec.30LISTING_PAGES = [31    "/fr/maisons-a-vendre.html",32    "/fr/condos-lofts-appartements-a-vendre.html",33    "/fr/chalets-a-vendre.html",34    "/fr/fermes-fermettes-a-vendre.html",35]36MAX_PAGES = 60                         # garde-fou (10 propriétés/page)37IMG_RE = re.compile(r'https://immo\.vrtx\.co/pic\d+/(\d+)(?:-\d+)?\.(?:jpg|jpeg|png|webp)',38                    re.I)39NOINS_RE = re.compile(r'noInscription=(\d+)')4041# Sitemap : couverture EXHAUSTIVE (l'index /sitemap.cfm pointe vers des42# sitemaps d'inscriptions paginés). typeInscription=1 = à vendre, =2 = à louer.43SITEMAP_INDEX = f"{BASE}/sitemap.cfm"44SITEMAP_MAX_PAGES = 40                  # garde-fou (≈12 pages réelles × ~1000)45_SM_SUB_RE = re.compile(r'sitemap_inscription\.cfm\?[^<"\s]+', re.I)46_SM_LOC_RE = re.compile(47    r'https://www\.suttonquebec\.com/fr/inscription/([^<"?]+)\.html\?'48    r'noInscription=(\d+)&(?:amp;)?typeInscription=(\d)', re.I)495051class SuttonConnector(BaseConnector):52    source_id = "sutton"53    request_delay = 0.55455    def fetch(self) -> list[PropertyListing]:56        by_id: dict[str, PropertyListing] = {}57        # 1) couverture exhaustive via le sitemap (adresse/type depuis le slug)58        for slug, centris, url in self._sitemap_listings():59            if centris in by_id:60                continue61            ptype, addr, city = _from_slug(slug)62            by_id[centris] = PropertyListing(63                source=self.source_id, external_id=centris, url=url,64                title=addr or _deslug(slug), address=addr, city=city,65                property_type=ptype, mls=centris,66                broker_name="Groupe Sutton Québec",67            )68        listings = list(by_id.values())69        # fiche détail : prix, chambres/sdb, galerie complète (~80 photos),70        # description, pièces, inclusions, caractéristiques — plafonné, cache71        # accumulé entre cycles (tout le parc finit enrichi au fil des syncs).72        du.enrich(self, listings, DETAIL_LIMIT, parse_sutton_detail, key="v4")73        for lst in listings:74            office = lst.details.pop("__agency", "")75            lst.agency = office or "Groupe Sutton Québec"76        return listings7778    def _sitemap_listings(self):79        """Itère (slug, centris, url) de toutes les inscriptions À VENDRE."""80        try:81            index = self.get(SITEMAP_INDEX).text82        except Exception:83            return84        subs = []85        for s in _SM_SUB_RE.findall(index):86            s = s.replace("&amp;", "&")87            if s not in subs:88                subs.append(s)89        if not subs:                       # repli : pagination directe90            subs = [f"sitemap_inscription.cfm?page={p}&platine=0"91                    for p in range(1, 14)] + ["sitemap_inscription.cfm?platine=1"]92        seen = set()93        for sub in subs[:SITEMAP_MAX_PAGES]:94            try:95                xml = self.get(f"{BASE}/{sub}").text96            except Exception:97                continue98            for m in _SM_LOC_RE.finditer(xml):99                slug, centris, typ = m.group(1), m.group(2), m.group(3)100                if typ != "1" or centris in seen:      # 1 = à vendre uniquement101                    continue102                seen.add(centris)103                url = (f"{BASE}/fr/inscription/{slug}.html"104                       f"?noInscription={centris}&typeInscription=1")105                yield slug, centris, url106107    def _fetch_section(self, path: str, out: dict) -> None:108        page = 1109        while page <= MAX_PAGES:110            url = f"{BASE}{path}?page={page}"111            try:112                html = self.get(url).text113            except Exception:114                break115            elems = item_list_elements(html)116            if not elems:117                break118            images_by_id = self._images_by_id(html)119            new = 0120            for el in elems:121                lst = self._to_listing(el, images_by_id)122                if lst and lst.uid not in out:123                    out[lst.uid] = lst124                    new += 1125            # dernière page : moins d'items que la pleine page, ou rien de neuf126            if len(elems) < 10 or new == 0:127                break128            page += 1129130    @staticmethod131    def _images_by_id(html: str) -> dict[str, list[str]]:132        groups: dict[str, list[str]] = {}133        for m in IMG_RE.finditer(html):134            groups.setdefault(m.group(1), [])135            if m.group(0) not in groups[m.group(1)]:136                groups[m.group(1)].append(m.group(0))137        return groups138139    def _to_listing(self, el: dict, images_by_id: dict) -> PropertyListing | None:140        item = el.get("item", el)141        url = item.get("url") or el.get("url") or ""142        if not url:143            return None144        if url.startswith("fr/") or url.startswith("/fr/"):145            url = f"{BASE}/{url.lstrip('/')}"146        m = NOINS_RE.search(url) or NOINS_RE.search(el.get("url", ""))147        external_id = m.group(1) if m else url.rsplit("/", 1)[-1].split(".")[0]148149        addr = el.get("address") or item.get("address") or {}150        offers = el.get("offers") or item.get("offers") or {}151        price = offers.get("price")152        try:153            price = float(price) if price is not None else None154        except (TypeError, ValueError):155            price = None156157        area = None158        fs = item.get("floorSize") or {}159        if isinstance(fs, dict) and fs.get("value"):160            val = fs["value"]161            # unitCode MTK = m² ; sinon on suppose des pi². Convertit si plausible.162            if fs.get("unitCode") == "MTK" and float(val) < 400:163                area = sqm_to_sqft(val)164            else:165                try:166                    area = round(float(val))167                except (TypeError, ValueError):168                    area = None169170        return PropertyListing(171            source=self.source_id,172            external_id=str(external_id),173            url=url,174            title=(item.get("name") or el.get("name") or "").strip(),175            address=addr.get("streetAddress", ""),176            sector=addr.get("addressLocality", ""),177            city=addr.get("addressLocality", ""),178            region=addr.get("addressRegion", ""),179            property_type=item.get("@type", ""),180            price=price,181            price_label=(f"{price:,.0f} $".replace(",", " ") if price else ""),182            bedrooms=_as_int(item.get("numberOfBedrooms")),183            bathrooms=_as_int(item.get("numberOfBathroomsTotal")),184            area_sqft=area,185            images=images_by_id.get(str(external_id), []),186            broker_name="Groupe Sutton Québec",187        )188189190def _as_int(v):191    try:192        return int(float(v))193    except (TypeError, ValueError):194        return None195196197def _deslug(s: str) -> str:198    return re.sub(r"\s+", " ", s.replace("-", " ")).strip().title()199200201# villes/arrondissements composés fréquents dans les slugs Sutton (fin du slug)202_CITY_HINTS = (203    "montreal-le-plateau-mont-royal", "montreal-ville-marie", "montreal-rosemont-la-petite-patrie",204    "montreal-le-sud-ouest", "montreal-cote-des-neiges-notre-dame-de-grace", "montreal-ahuntsic-cartierville",205    "montreal-villeray-saint-michel-parc-extension", "montreal-mercier-hochelaga-maisonneuve",206    "montreal-riviere-des-prairies-pointe-aux-trembles", "montreal-verdun-ile-des-soeurs",207    "montreal-saint-laurent", "montreal-lasalle", "montreal-outremont", "montreal-anjou",208    "montreal-pierrefonds-roxboro", "trois-rivieres", "saint-jean-sur-richelieu",209    "salaberry-de-valleyfield", "mont-saint-hilaire", "notre-dame-de-l-ile-perrot",210    "vaudreuil-dorion", "sainte-adele", "sainte-therese", "saint-jerome", "levis", "quebec",211    "laval", "gatineau", "longueuil", "brossard", "sherbrooke", "granby", "magog",212)213214215def _from_slug(slug: str) -> tuple[str, str, str]:216    """« maison-de-plain-pied-a-vendre-522-cedar-street-beaconsfield »217    -> (type, adresse, ville). Best-effort (l'adresse exacte vient de la fiche)."""218    m = re.search(r"^(.*?)-a-vendre-(.+)$", slug, re.I)219    if not m:220        return "", _deslug(slug), ""221    ptype = _deslug(m.group(1))222    rest = m.group(2).lower()223    city = ""224    for hint in _CITY_HINTS:225        if rest.endswith("-" + hint) or rest == hint:226            city = _deslug(hint)227            rest = rest[: -len(hint)].rstrip("-")228            break229    else:230        # sinon : dernier token comme ville (approx ; corrigé par la fiche détail)231        parts = rest.rsplit("-", 1)232        if len(parts) == 2 and not parts[1].isdigit():233            city, rest = _deslug(parts[1]), parts[0]234    return ptype, _deslug(rest), city235236237def _merge(dst: PropertyListing, src: PropertyListing) -> None:238    """Complète dst (issu du sitemap) avec les champs riches de src (JSON-LD)."""239    for f in ("price", "bedrooms", "bathrooms", "area_sqft"):240        if getattr(dst, f, None) in (None, 0) and getattr(src, f, None) not in (None, 0):241            setattr(dst, f, getattr(src, f))242    for f in ("address", "city", "sector", "region", "price_label", "property_type"):243        if not getattr(dst, f, "") and getattr(src, f, ""):244            setattr(dst, f, getattr(src, f))245    if len(src.images) > len(dst.images):246        dst.images = src.images247248249def parse_sutton_detail(html: str) -> dict:250    """Fiche Sutton : galerie complète, description, pièces (dimensions), inclusions."""251    out: dict = {}252    # photos : normaliser toute taille -> pic450, dédoublonner par id-index253    seen, imgs = {}, []254    for m in _DET_IMG_RE.finditer(html):255        key = m.group(1)256        if key not in seen:257            seen[key] = 1258            imgs.append(f"https://immo.vrtx.co/pic450/{key}.jpg")259    if imgs:260        out["images"] = imgs261262    text = re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", html)))263264    # prix : premier montant du texte (le prix demandé précède évaluation/taxes)265    for mnt in re.findall(r"([\d][\d  ]{3,})\s*\$", text):266        val = _as_int(mnt.replace(" ", "").replace("\xa0", ""))267        if val and val >= 20000:268            out["price"] = float(val)269            out["price_label"] = f"{val:,.0f} $".replace(",", " ")270            break271    mb = re.search(r"(\d+)\s*chambre", text, re.I)272    if mb:273        out["bedrooms"] = int(mb.group(1))274    ms = re.search(r"(\d+)\s*salle[s]?\s*de\s*bain", text, re.I)275    if ms:276        out["bathrooms"] = int(ms.group(1))277278    md = _DESC_RE.search(text)279    if md:280        desc = md.group(1).strip()281        if len(desc) > 30:282            out["description"] = desc[:4000]283284    # pièces : « Nom  DimxDim P  Revêtement  Niveau »285    rooms = []286    for m in _ROOM_ROW_RE.finditer(text):287        nom = m.group(1).strip(" -")288        # retirer l'en-tête de tableau qui peut coller au 1er nom289        nom = re.sub(r"^.*?(?:Niveau|Rev[êe]tement|Dimension)\s+", "", nom).strip(" -")290        if not nom or nom.lower() in ("pièce", "piece"):291            continue292        rooms.append({"nom": nom[:40], "dimensions": m.group(2).replace(" ", ""),293                      "revetement": m.group(3).strip(), "niveau": m.group(4)})294    if rooms:295        out.setdefault("details", {})["pieces"] = rooms[:20]296297    # inclusions298    mi = re.search(r"Inclusions?\s*:?(.*?)(?:Exclusions|Addenda|Financ|$)", text, re.I | re.S)299    if mi:300        inc = mi.group(1).strip()301        feats = [s.strip() for s in re.split(r"[,;•]", inc) if 3 <= len(s.strip()) <= 90]302        if feats:303            out["features"] = feats[:20]304    _det = du.centris_details(du.flatten(html))305    if _det:306        out.setdefault("details", {}).update(_det)307308    # sous-agence (bureau) : lien /agence-immobiliere/NNN-<slug>.html309    ma = re.search(r'/agence-immobiliere/\d+-([a-z0-9-]+)\.html', html, re.I)310    if ma:311        out.setdefault("details", {})["__agency"] = _deslug(ma.group(1))312    return out313