spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/duproprio.py : DuProprio (FSBO — vente sans courtier)5#6#   DuProprio est la grande plateforme québécoise de vente « à vendre par le7#   propriétaire » (sans agence). Segment complémentaire absent de toutes les8#   bannières de courtage. Énumération EXHAUSTIVE via les sitemaps FR par région9#   du Québec (`/sitemaps/fr/<region>-listings.xml.gz`) : l'URL encode déjà10#   région, ville, type et l'id DuProprio (= external_id). La fiche détail11#   (JSON-LD Product) fournit prix, description et galerie ; enrichissement12#   plafonné + cache (comme les autres connecteurs). Scrapfly en secours si le13#   fetch direct est bloqué.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import gzip18import io19import os20import re2122from .base import BaseConnector23from . import _detailutil as du24from ..normalize import normalize_property_type, parse_price25from ..schema import PropertyListing2627SITEMAP_INDEX = "https://duproprio.com/sitemaps/fr/index.xml.gz"28DETAIL_LIMIT = int(os.environ.get("IMMOKA_DUPROPRIO_DETAIL_LIMIT", "500"))29AGENCY = "DuProprio (sans courtier)"3031_LOC_RE = re.compile(r"<loc>([^<]+)</loc>")32# /fr/<region>/<ville>/<type>-a-vendre/<slug>-<id>33_URL_RE = re.compile(34    r"https://duproprio\.com/fr/([a-z0-9-]+)/([a-z0-9-]+)/([a-z0-9-]+?)-a-vendre/([a-z0-9-]+?)-(\d{5,})/?$",35    re.I)36_PHOTO_RE = re.compile(37    r"https://photos\.duproprio\.com/photos/public/for_sale/\d+/\d+/[^\"'\\ ]+?\.jpg", re.I)38_LDPROD_RE = re.compile(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', re.S | re.I)394041class DuProprioConnector(BaseConnector):42    source_id = "duproprio"43    request_delay = 0.44445    def fetch(self) -> list[PropertyListing]:46        by_id: dict[str, PropertyListing] = {}47        for sm in self._region_listing_sitemaps():48            for url in self._sitemap_locs(sm):49                lst = self._to_listing(url)50                if lst and lst.external_id not in by_id:51                    by_id[lst.external_id] = lst52        listings = list(by_id.values())53        du.enrich(self, listings, DETAIL_LIMIT, _parse_dp_detail,54                  key="v2", fetch_html=self._fetch_detail)55        return listings5657    # -- sitemaps --------------------------------------------------------------58    def _region_listing_sitemaps(self) -> list[str]:59        xml = self._get_gz(SITEMAP_INDEX)60        # uniquement les sitemaps d'INSCRIPTIONS par région (Québec)61        return [u for u in _LOC_RE.findall(xml) if u.endswith("-listings.xml.gz")]6263    def _sitemap_locs(self, url: str) -> list[str]:64        return [u for u in _LOC_RE.findall(self._get_gz(url)) if "/fr/" in u]6566    def _get_gz(self, url: str) -> str:67        try:68            raw = self.get(url).content69        except Exception:70            return ""71        try:72            return gzip.GzipFile(fileobj=io.BytesIO(raw)).read().decode("utf-8", "ignore")73        except OSError:74            return raw.decode("utf-8", "ignore")   # déjà décompressé7576    # -- mapping ---------------------------------------------------------------77    def _to_listing(self, url: str) -> PropertyListing | None:78        m = _URL_RE.match(url.strip())79        if not m:80            return None81        region_s, city_s, type_s, addr_s, did = m.groups()82        return PropertyListing(83            source=self.source_id, external_id=did, url=url,84            title=_deslug(addr_s),85            address=_deslug(re.sub(r"^(hab|com|ter|multi|imm)-", "", addr_s)),86            city=_deslug(city_s),87            region=_region(region_s),88            property_type=normalize_property_type(type_s.replace("-", " ")),89            agency=AGENCY, broker_name="",90        )9192    def _fetch_detail(self, url: str) -> str:93        html = ""94        try:95            html = self.get(url).text96        except Exception:97            html = ""98        if "application/ld+json" not in html:      # bloqué -> Scrapfly (ASP)99            try:100                html = self.get_scrapfly(url, render_js=False, asp=True)101            except Exception:102                pass103        return html104105106# ---------------------------------------------------------------------------107def _parse_dp_detail(html: str) -> dict:108    import json, html as _h109    out: dict = {}110    for block in _LDPROD_RE.findall(html):111        try:112            d = json.loads(block)113        except ValueError:114            continue115        if isinstance(d, dict) and d.get("@type") == "Product":116            off = d.get("offers") or {}117            p = parse_price(str(off.get("price") or "")) if off.get("price") else None118            if p:119                out["price"] = p120                out["price_label"] = f"{p:,.0f} $".replace(",", " ")121            if d.get("description"):122                out["description"] = _h.unescape(str(d["description"])).strip()[:4000]123            break124    # coordonnées GPS du JSON-LD Residence (geo) — géocodage gratuit125    for block in _LDPROD_RE.findall(html):126        try:127            d = json.loads(block)128        except ValueError:129            continue130        geo = d.get("geo") if isinstance(d, dict) else None131        if isinstance(geo, dict):132            try:133                lat, lng = float(geo.get("latitude")), float(geo.get("longitude"))134            except (TypeError, ValueError):135                continue136            if 44.5 <= lat <= 63.0 and -80.0 <= lng <= -56.0:137                out["lat"], out["lng"] = lat, lng138            break139    # galerie COMPLÈTE : le JSON « "photos":[{...}] » de la page liste toutes les140    # photos (le JSON-LD/HTML n'en montre qu'une). On prend la plus haute résolution.141    imgs, seen = [], set()142    mgal = re.search(r'"photos"\s*:\s*(\[\{.*?\}\])', html, re.S)143    if mgal:144        try:145            for ph in json.loads(mgal.group(1)):146                fmt = ph.get("formats") or {}147                rel = fmt.get("1600") or fmt.get("1024") or fmt.get("600")148                if rel:149                    u = rel if rel.startswith("http") else f"https://photos.duproprio.com/{rel.lstrip('/')}"150                    if u not in seen:151                        seen.add(u); imgs.append(u)152        except ValueError:153            pass154    if not imgs:                       # repli : URLs photos dans le HTML brut155        for u in _PHOTO_RE.findall(html):156            if u not in seen:157                seen.add(u); imgs.append(u)158    if imgs:159        out["images"] = imgs160    text = re.sub(r"\s+", " ", _h_unescape(html))161    mb = re.search(r"(\d+)\s*chambre", text, re.I)162    if mb:163        out["bedrooms"] = int(mb.group(1))164    ms = re.search(r"(\d+)\s*salle[s]?\s*de\s*bain", text, re.I)165    if ms:166        out["bathrooms"] = int(ms.group(1))167168    details: dict = {}169    features: list[str] = []170171    # --- Dimensions des pièces (tableau des pièces) --------------------------172    rooms = []173    for blk in re.split(r'listing-rooms-details__table__item-container',174                        html)[1:41]:175        blk = blk.split("item-container")[0]176        def cell(marker):177            m = re.search(marker + r'[^>]*>(.*?)</div>', blk, re.S)178            return re.sub(r"\s+", " ", _h_unescape(m.group(1))).strip() if m else ""179        nom = cell(r'item--room"')180        niveau = cell(r'item--storey"').replace("Étage :", "").strip()181        dim = cell(r'item--dimensions"').replace("Dimensions :", "").strip()182        rev = cell(r'item--flooring"').replace("Plancher :", "").strip()183        if nom:184            rooms.append({"nom": nom, "niveau": niveau,185                          "dimensions": dim, "revetement": rev})186    if rooms:187        details["pieces"] = rooms[:30]188189    # --- Caractéristiques de la propriété (rangées pointillées) ---------------190    for lm, vm in re.findall(191            r'listing-box__dotted-row">\s*<div>(.*?)</div>\s*<div>\s*</div>\s*'192            r'<div>(.*?)</div>', html, re.S):193        label = re.sub(r"\s+", " ", _h_unescape(lm)).strip()194        value = re.sub(r"\s+", " ", _h_unescape(vm)).strip()195        if not label or not value or label == "Prix demandé":196            continue197        if label == "Année de construction":198            my = re.search(r"(19|20)\d{2}", value)199            if my:200                out["year_built"] = int(my.group(0))201        elif label in ("Superficie du terrain", "Superficie habitable",202                       "Aire habitable", "Superficie du bâtiment"):203            mp = re.search(r"([\d\s,.]+)\s*pi", value)204            if mp:205                try:206                    sqft = float(mp.group(1).replace(" ", "").replace(",", ""))207                    out["lot_sqft" if "terrain" in label else "area_sqft"] = sqft208                except ValueError:209                    pass210        features.append(f"{label} : {value}")211        details[label] = value212213    # --- Remarques du proprio (texte long, souvent plus riche que le JSON-LD) --214    mrq = re.search(r'listing-owners-comments__description[^>]*>(.*?)</div>',215                    html, re.S)216    if mrq:217        remarques = re.sub(r"\s+", " ", _h_unescape(mrq.group(1))).strip()218        if remarques:219            details["remarques_proprio"] = remarques[:6000]220            if len(remarques) > len(out.get("description") or ""):221                out["description"] = remarques[:6000]222223    if features:224        out["features"] = features225    if details:226        out["details"] = details227    return out228229230def _h_unescape(html: str) -> str:231    import html as _h232    return _h.unescape(re.sub(r"<[^>]+>", " ", html))233234235def _deslug(s: str) -> str:236    return re.sub(r"\s+", " ", s.replace("-", " ").replace("_", " ")).strip().title()237238239def _region(slug: str) -> str:240    r = _deslug(slug)241    return (r.replace("Quebec", "Québec").replace("Montreal", "Montréal")242             .replace("Monteregie", "Montérégie").replace("Laurentides", "Laurentides")243             .replace("Mauricie", "Mauricie").replace("Estrie", "Estrie"))244