spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/duquesimms.py : Duque Simms (duquesimms.com) — agence indépendante5#   de Montréal (NDG, Côte-des-Neiges, Ouest-de-l'Île), ~25 inscriptions.6#7#   Site WordPress/Elementor (plateforme proprius/e-mmobilier, photos sur8#   cdn.proprius.e-mmobilier.ca). Découverte par property-sitemap.xml : URLs9#   fr /property/{n° Centris}/ (les /en/ sont des doublons de langue).10#11#   TOUT vient de la fiche détail (du.enrich) : JSON-LD RealEstateListing12#   (description, contentLocation adresse complète + GPS, offers[].price avec13#   businessFunction Sell, keywords = type, seller = courtier nom/téléphone)14#   + blocs HTML property-characteristic (libellé/valeurs), property-expense15#   (taxes, énergie), property-evaluation, compteurs chambres/salles de bain16#   (icônes bedroom.svg/bathroom.svg suivies du nombre), galerie swiper17#   cdn …-pi.jpg (les -c.jpg sont des portraits). Les locations (prix de18#   loyer) sont écartées par le seuil de prix vente.19#20#   Agence première main (pas d'infixe _ag_) : ses fiches battent les copies21#   portails au même n° Centris dans la dédup.22# -----------------------------------------------------------------------------23from __future__ import annotations2425import html as _html26import os27import re2829from .base import BaseConnector30from . import _detailutil as du31from ..schema import PropertyListing3233SITE = "https://duquesimms.com"34SITEMAP = SITE + "/property-sitemap.xml"35AGENCY = "Duque Simms"36DETAIL_LIMIT = int(os.environ.get("IMMOKA_DUQUESIMMS_DETAIL_LIMIT", "80"))37MIN_SALE_PRICE = 20000     # sous ce prix : loyer mensuel (location) → écarté3839_LOC_RE = re.compile(r"<loc>\s*(https://duquesimms\.com/property/(\d{6,10})/?)\s*</loc>")40_ADDR_RE = re.compile(r"^(.*?),\s*(.+?)\s*([A-Z]\d[A-Z]\s?\d[A-Z]\d)?$")41_CHAR_RE = re.compile(r'property-characteristic-type">(.*?)</span>(.*?)</div>', re.S)42_SUBVAL_RE = re.compile(r'property-subcharacteristic-value">(.*?)</span>', re.S)43_EXP_RE = re.compile(r'property-expense-label">(.*?)</span>.*?property-money-amount">'44                     r'([\d\s ,]+)</span>', re.S)45_EVAL_RE = re.compile(r'property-evaluation-label">(.*?)</span>.*?property-money-amount">'46                      r'([\d\s ,]+)</span>', re.S)47_BEDS_RE = re.compile(r'bedroom\.svg.{0,900}?elementor-widget-container">\s*(\d+)\s*<', re.S)48_BATHS_RE = re.compile(r'bathroom\.svg.{0,900}?elementor-widget-container">\s*(\d+)\s*<', re.S)49_IMG_RE = re.compile(r'https://cdn\.proprius\.e-mmobilier\.ca/([A-Z0-9]+-pi)\.jpe?g', re.I)505152class DuqueSimmsConnector(BaseConnector):53    source_id = "duquesimms"54    request_delay = 0.45556    def fetch(self) -> list[PropertyListing]:57        # SiteGround sert un challenge sgcaptcha en HTTP 202 (sans exception) :58        # get_resilient escalade (Oxylabs → Scrapfly ASP → Bright Data)59        xml = self.get_resilient(SITEMAP).text60        by_id: dict[str, PropertyListing] = {}61        for url, mls in _LOC_RE.findall(xml):62            by_id.setdefault(mls, PropertyListing(63                source=self.source_id, external_id=mls,64                url=url if url.endswith("/") else url + "/",65                title="", mls=mls, agency=AGENCY, broker_name=AGENCY,66            ))67        listings = list(by_id.values())68        du.enrich(self, listings, DETAIL_LIMIT, _parse_detail, key="v1",69                  fetch_html=lambda u: self.get_resilient(u).text)70        out = []71        for lst in listings:72            if not lst.price or lst.price < MIN_SALE_PRICE:73                continue               # location (loyer) ou fiche vide74            if not lst.title:75                lst.title = f"{lst.property_type or 'Propriété'} à vendre — " \76                            f"{lst.address or lst.city}".strip(" —")77            if lst.price and not lst.price_label:78                lst.price_label = f"{int(lst.price):,} $".replace(",", " ")79            out.append(lst)80        return out818283def _txt(s: str) -> str:84    return re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", s))).strip()858687def _parse_detail(html: str) -> dict:88    node = None89    for n in du.ld_nodes(html):90        if n.get("@type") == "RealEstateListing":91            node = n92            break93    if node is None:94        return {}95    out: dict = {}96    details: dict = {}97    features: list[str] = []9899    desc = _html.unescape(str(node.get("description") or "")).strip()100    if desc:101        out["description"] = desc102103    kw = node.get("keywords") or []104    if isinstance(kw, str):105        kw = [kw]106    if kw:107        out["property_type"] = str(kw[0]).strip()108        details["Type de propriété"] = out["property_type"]109        if len(kw) > 1:110            details["Catégorie"] = str(kw[1]).strip()111112    loc = node.get("contentLocation") or {}113    if isinstance(loc, dict):114        full = _html.unescape(str(loc.get("address") or "")).strip()115        m = _ADDR_RE.match(full)116        if m:117            out["address"] = m.group(1).strip()118            out["city"] = m.group(2).strip()119            if m.group(3):120                details["Code postal"] = m.group(3)121        elif full:122            out["address"] = full123        try:124            out["lat"], out["lng"] = float(loc["latitude"]), float(loc["longitude"])125        except (KeyError, TypeError, ValueError):126            pass127128    offers = node.get("offers") or []129    if isinstance(offers, dict):130        offers = [offers]131    sellers: list[str] = []132    for off in offers:133        if not isinstance(off, dict):134            continue135        if "Sell" not in str(off.get("businessFunction") or "Sell"):136            continue137        if out.get("price") is None and off.get("price"):138            try:139                out["price"] = float(str(off["price"]).replace(" ", ""))140            except ValueError:141                pass142        seller = off.get("seller") or {}143        if isinstance(seller, dict) and seller.get("name"):144            name = str(seller["name"]).strip()145            if name not in sellers:146                sellers.append(name)147            if "broker_phone" not in out:148                tel = str(seller.get("telephone") or "").strip()149                if re.fullmatch(r"\d{10}", tel):150                    out["broker_phone"] = f"{tel[:3]} {tel[3:6]}-{tel[6:]}"151    if sellers:152        out["broker_name"] = " et ".join(sellers[:2])153154    bm, tm = _BEDS_RE.search(html), _BATHS_RE.search(html)155    if bm:156        out["bedrooms"] = int(bm.group(1))157    if tm:158        out["bathrooms"] = int(tm.group(1))159160    for label, blob in _CHAR_RE.findall(html):161        label = _txt(label)162        vals = [_txt(v) for v in _SUBVAL_RE.findall(blob)]163        vals = [v for v in vals if v]164        if label and vals:165            details[label] = ", ".join(vals)166            features.append(f"{label} : {details[label]}"167                            if len(details[label]) < 40 else label)168169    for rx in (_EXP_RE, _EVAL_RE):170        for label, amount in rx.findall(html):171            label, amount = _txt(label), _txt(amount).replace(" ", " ")172            if label and amount:173                details[label] = f"{amount} $"174175    imgs, seen = [], set()176    for base in _IMG_RE.findall(html):177        if base not in seen:178            seen.add(base)179            imgs.append(f"https://cdn.proprius.e-mmobilier.ca/{base}.jpg")180    if imgs:181        out["images"] = imgs182183    if details:184        out["details"] = details185    if features:186        out["features"] = features187    return out188