spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/duproprio.py : DuProprio (FSBO — vente sans courtier)5#6#   DuProprio est la grande plateforme québécoise de vente « à vendre par le7#   propriétaire » (sans agence). Segment complémentaire absent de toutes les8#   bannières de courtage. Énumération EXHAUSTIVE via les sitemaps FR par région9#   du Québec (`/sitemaps/fr/<region>-listings.xml.gz`) : l'URL encode déjà10#   région, ville, type et l'id DuProprio (= external_id). La fiche détail11#   (JSON-LD Product) fournit prix, description et galerie ; enrichissement12#   plafonné + cache (comme les autres connecteurs). Scrapfly en secours si le13#   fetch direct est bloqué.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import gzip18import io19import os20import re2122from .base import BaseConnector23from . import _detailutil as du24from ..normalize import normalize_property_type, parse_price25from ..schema import PropertyListing2627SITEMAP_INDEX = "https://duproprio.com/sitemaps/fr/index.xml.gz"28DETAIL_LIMIT = int(os.environ.get("IMMOKA_DUPROPRIO_DETAIL_LIMIT", "500"))29AGENCY = "DuProprio (sans courtier)"3031_LOC_RE = re.compile(r"<loc>([^<]+)</loc>")32# /fr/<region>/<ville>/<type>-a-vendre/<slug>-<id>33_URL_RE = re.compile(34    r"https://duproprio\.com/fr/([a-z0-9-]+)/([a-z0-9-]+)/([a-z0-9-]+?)-a-vendre/([a-z0-9-]+?)-(\d{5,})/?$",35    re.I)36_PHOTO_RE = re.compile(37    r"https://photos\.duproprio\.com/photos/public/for_sale/\d+/\d+/[^\"'\\ ]+?\.jpg", re.I)38_LDPROD_RE = re.compile(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', re.S | re.I)394041class DuProprioConnector(BaseConnector):42    source_id = "duproprio"43    request_delay = 0.44445    def fetch(self) -> list[PropertyListing]:46        by_id: dict[str, PropertyListing] = {}47        for sm in self._region_listing_sitemaps():48            for url in self._sitemap_locs(sm):49                lst = self._to_listing(url)50                if lst and lst.external_id not in by_id:51                    by_id[lst.external_id] = lst52        listings = list(by_id.values())53        du.enrich(self, listings, DETAIL_LIMIT, _parse_dp_detail,54                  key="v2", fetch_html=self._fetch_detail)55        return listings5657    # -- sitemaps --------------------------------------------------------------58    def _region_listing_sitemaps(self) -> list[str]:59        xml = self._get_gz(SITEMAP_INDEX)60        # uniquement les sitemaps d'INSCRIPTIONS par région (Québec)61        return [u for u in _LOC_RE.findall(xml) if u.endswith("-listings.xml.gz")]6263    def _sitemap_locs(self, url: str) -> list[str]:64        return [u for u in _LOC_RE.findall(self._get_gz(url)) if "/fr/" in u]6566    def _get_gz(self, url: str) -> str:67        try:68            raw = self.get(url).content69        except Exception:70            return ""71        try:72            return gzip.GzipFile(fileobj=io.BytesIO(raw)).read().decode("utf-8", "ignore")73        except OSError:74            return raw.decode("utf-8", "ignore")   # déjà décompressé7576    # -- mapping ---------------------------------------------------------------77    def _to_listing(self, url: str) -> PropertyListing | None:78        m = _URL_RE.match(url.strip())79        if not m:80            return None81        region_s, city_s, type_s, addr_s, did = m.groups()82        return PropertyListing(83            source=self.source_id, external_id=did, url=url,84            title=_deslug(addr_s),85            address=_deslug(re.sub(r"^(hab|com|ter|multi|imm)-", "", addr_s)),86            city=_deslug(city_s),87            region=_region(region_s),88            property_type=normalize_property_type(type_s.replace("-", " ")),89            agency=AGENCY, broker_name="",90        )9192    def _fetch_detail(self, url: str) -> str:93        html = ""94        try:95            html = self.get(url).text96        except Exception:97            html = ""98        if "application/ld+json" not in html:      # bloqué -> Scrapfly (ASP)99            try:100                html = self.get_scrapfly(url, render_js=False, asp=True)101            except Exception:102                pass103        return html104105106# ---------------------------------------------------------------------------107def _parse_dp_detail(html: str) -> dict:108    import json, html as _h109    out: dict = {}110    for block in _LDPROD_RE.findall(html):111        try:112            d = json.loads(block)113        except ValueError:114            continue115        if isinstance(d, dict) and d.get("@type") == "Product":116            off = d.get("offers") or {}117            p = parse_price(str(off.get("price") or "")) if off.get("price") else None118            if p:119                out["price"] = p120                out["price_label"] = f"{p:,.0f} $".replace(",", " ")121            if d.get("description"):122                out["description"] = _h.unescape(str(d["description"])).strip()[:4000]123            break124    # galerie COMPLÈTE : le JSON « "photos":[{...}] » de la page liste toutes les125    # photos (le JSON-LD/HTML n'en montre qu'une). On prend la plus haute résolution.126    imgs, seen = [], set()127    mgal = re.search(r'"photos"\s*:\s*(\[\{.*?\}\])', html, re.S)128    if mgal:129        try:130            for ph in json.loads(mgal.group(1)):131                fmt = ph.get("formats") or {}132                rel = fmt.get("1600") or fmt.get("1024") or fmt.get("600")133                if rel:134                    u = rel if rel.startswith("http") else f"https://photos.duproprio.com/{rel.lstrip('/')}"135                    if u not in seen:136                        seen.add(u); imgs.append(u)137        except ValueError:138            pass139    if not imgs:                       # repli : URLs photos dans le HTML brut140        for u in _PHOTO_RE.findall(html):141            if u not in seen:142                seen.add(u); imgs.append(u)143    if imgs:144        out["images"] = imgs145    text = re.sub(r"\s+", " ", _h_unescape(html))146    mb = re.search(r"(\d+)\s*chambre", text, re.I)147    if mb:148        out["bedrooms"] = int(mb.group(1))149    ms = re.search(r"(\d+)\s*salle[s]?\s*de\s*bain", text, re.I)150    if ms:151        out["bathrooms"] = int(ms.group(1))152153    details: dict = {}154    features: list[str] = []155156    # --- Dimensions des pièces (tableau des pièces) --------------------------157    rooms = []158    for blk in re.split(r'listing-rooms-details__table__item-container',159                        html)[1:41]:160        blk = blk.split("item-container")[0]161        def cell(marker):162            m = re.search(marker + r'[^>]*>(.*?)</div>', blk, re.S)163            return re.sub(r"\s+", " ", _h_unescape(m.group(1))).strip() if m else ""164        nom = cell(r'item--room"')165        niveau = cell(r'item--storey"').replace("Étage :", "").strip()166        dim = cell(r'item--dimensions"').replace("Dimensions :", "").strip()167        rev = cell(r'item--flooring"').replace("Plancher :", "").strip()168        if nom:169            rooms.append({"nom": nom, "niveau": niveau,170                          "dimensions": dim, "revetement": rev})171    if rooms:172        details["pieces"] = rooms[:30]173174    # --- Caractéristiques de la propriété (rangées pointillées) ---------------175    for lm, vm in re.findall(176            r'listing-box__dotted-row">\s*<div>(.*?)</div>\s*<div>\s*</div>\s*'177            r'<div>(.*?)</div>', html, re.S):178        label = re.sub(r"\s+", " ", _h_unescape(lm)).strip()179        value = re.sub(r"\s+", " ", _h_unescape(vm)).strip()180        if not label or not value or label == "Prix demandé":181            continue182        if label == "Année de construction":183            my = re.search(r"(19|20)\d{2}", value)184            if my:185                out["year_built"] = int(my.group(0))186        elif label in ("Superficie du terrain", "Superficie habitable",187                       "Aire habitable", "Superficie du bâtiment"):188            mp = re.search(r"([\d\s,.]+)\s*pi", value)189            if mp:190                try:191                    sqft = float(mp.group(1).replace(" ", "").replace(",", ""))192                    out["lot_sqft" if "terrain" in label else "area_sqft"] = sqft193                except ValueError:194                    pass195        features.append(f"{label} : {value}")196        details[label] = value197198    # --- Remarques du proprio (texte long, souvent plus riche que le JSON-LD) --199    mrq = re.search(r'listing-owners-comments__description[^>]*>(.*?)</div>',200                    html, re.S)201    if mrq:202        remarques = re.sub(r"\s+", " ", _h_unescape(mrq.group(1))).strip()203        if remarques:204            details["remarques_proprio"] = remarques[:6000]205            if len(remarques) > len(out.get("description") or ""):206                out["description"] = remarques[:6000]207208    if features:209        out["features"] = features210    if details:211        out["details"] = details212    return out213214215def _h_unescape(html: str) -> str:216    import html as _h217    return _h.unescape(re.sub(r"<[^>]+>", " ", html))218219220def _deslug(s: str) -> str:221    return re.sub(r"\s+", " ", s.replace("-", " ").replace("_", " ")).strip().title()222223224def _region(slug: str) -> str:225    r = _deslug(slug)226    return (r.replace("Quebec", "Québec").replace("Montreal", "Montréal")227             .replace("Monteregie", "Montérégie").replace("Laurentides", "Laurentides")228             .replace("Mauricie", "Mauricie").replace("Estrie", "Estrie"))229