spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/duproprio.py : DuProprio (duproprio.com) — section LOCATION5#   ~4 200 annonces de particuliers (sans courtier : zéro chevauchement MLS).6#   Découverte par les 21 sitemaps régionaux `sitemaps/fr/<region>-listings.xml.gz`7#   filtrés sur `/fr/location/`. Fiches SSR (Laravel/CloudFront, aucun anti-bot) :8#   JSON-LD `Apartment` (adresse + code postal + chambres/SDB + amenityFeature)9#   et `RealEstateListing` (offers.price + availabilityStarts), lat/lng inline,10#   superficie « 970 pi² », galerie dans un JSON échappé ("photos":[…]).11#   Fiche retirée -> 302 vers la page parente (détectée par l'URL finale sans12#   `/hab-`). Cache détail avec TTL (_detailutil.TtlDetailCache).13#   Respect robots.txt : ne jamais toucher /webservice/, /fr-ca/api, /api-proxy.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import gzip18import html as H19import os20import re2122from ..schema import Listing23from .base import BaseConnector24from . import _detailutil as du2526BASE = "https://duproprio.com"27SITEMAP_INDEX = f"{BASE}/sitemaps/fr/index.xml.gz"2829_LOC_RE = re.compile(r"<loc>(.*?)</loc>")30# .../fr/location/<region>/<ville>/<type>-a-louer/hab-<slug>-<ID>31_FICHE_RE = re.compile(32    r"^https://duproprio\.com/fr/location/([a-z0-9-]+)/([a-z0-9-]+)/"33    r"([a-z0-9-]+)-a-louer/hab-[a-z0-9-]+-(\d+)$")34_GEO_RE = re.compile(r'"latitude":([\d.-]+),"longitude":([\d.-]+)')35_AREA_RE = re.compile(r"([\d\s,.]{1,10})\s*pi²")36_PHOTO_RE = re.compile(37    r"photos\\?/public\\?/for_rent\\?/[0-9]+\\?/[0-9]+\\?/[a-z0-9-]+-(\d+)\.jpg")38_PHOTO_FULL_RE = re.compile(39    r"photos[\\/]+public[\\/]+for_rent[\\/]+(\d+)[\\/]+(\d+)[\\/]+([a-z0-9-]+)-(\d+)\.jpg")40_TYPE_HALF_RE = re.compile(r"^(\d+)-1-2$")4142# types non résidentiels (premier mot du slug type) exclus du parc43_EXCLUDED_TYPES = {"commerce", "bureau", "local", "entrepot", "espace",44                   "terrain", "garage", "stationnement", "industriel"}4546DETAIL_LIMIT = int(os.environ.get("LOUKA_DUPROPRIO_DETAIL_LIMIT", "800"))47TTL_DAYS = float(os.environ.get("LOUKA_DUPROPRIO_TTL_DAYS", "7"))48MAX_FICHES = int(os.environ.get("LOUKA_DUPROPRIO_MAX", "0"))     # 0 = tout495051def _parse_fiche(html: str) -> dict:52    out: dict = {}53    for node in du.ld_nodes(html):54        t = node.get("@type") or ""55        if t in ("Apartment", "House", "SingleFamilyResidence", "Accommodation"):56            out["title"] = (node.get("name") or "").strip()57            addr = node.get("address") or {}58            out["address"] = (addr.get("streetAddress") or "").strip()59            out["city"] = (addr.get("addressLocality") or "").strip()60            out["postal_code"] = (addr.get("postalCode") or "").strip()61            for src, lab in (("numberOfBedrooms", "Chambres"),62                             ("numberOfFullBathrooms", "Salles de bain"),63                             ("floorLevel", "Étage")):64                if node.get(src):65                    out.setdefault("details", {})[lab] = str(node[src])66            if node.get("numberOfBedrooms"):67                out["bedrooms"] = str(node["numberOfBedrooms"])68            feats = node.get("amenityFeature") or []69            names = [f.get("name", "").strip() for f in feats70                     if isinstance(f, dict) and f.get("name")]71            if names:72                out["amenities"] = names73            if node.get("description"):74                out["description"] = H.unescape(str(node["description"])).strip()[:6000]75        elif t == "RealEstateListing":76            offers = node.get("offers") or {}77            try:78                out["price"] = float(str(offers.get("price")).replace(",", "."))79            except (TypeError, ValueError):80                pass81            if offers.get("availabilityStarts"):82                out["availability_date"] = str(offers["availabilityStarts"])[:10]8384    m = _GEO_RE.search(html)85    if m:86        out["lat"], out["lng"] = float(m.group(1)), float(m.group(2))87    m = _AREA_RE.search(html)88    if m:89        try:90            area = float(m.group(1).replace(" ", "").replace(",", ""))91            if 80 <= area <= 20000:92                out["area_sqft"] = area93        except ValueError:94            pass9596    # galerie : URLs (souvent échappées \/) dédupliquées par id photo, en 1600 px97    photos, seen = [], set()98    for m in _PHOTO_FULL_RE.finditer(html):99        ym, lid, slug, pid = m.groups()100        if pid in seen:101            continue102        seen.add(pid)103        # le slug embarque la taille en avant-dernier segment : la re-forcer à 1600104        slug1600 = re.sub(r"-\d+$", "-1600", slug)105        photos.append(f"https://photos.duproprio.com/photos/public/for_rent/"106                      f"{ym}/{lid}/{slug1600}-{pid}.jpg")107    if photos:108        out["images"] = photos109110    vt = du.virtual_tour(html)      # visite virtuelle (Matterport/iGUIDE…)111    if vt:112        out.setdefault("details", {})["virtual_tour"] = vt113    return out114115116class DuProprioConnector(BaseConnector):117    source_id = "duproprio"118    request_delay = 0.8119120    def _gz(self, url: str) -> str:121        resp = self.get(url)122        try:123            return gzip.decompress(resp.content).decode("utf-8", "replace")124        except (OSError, EOFError):125            return resp.text126127    def _fiche_urls(self) -> list[tuple[str, str, str, str]]:128        """(url, ville, type_slug, id) des fiches location, tous sitemaps."""129        index = self._gz(SITEMAP_INDEX)130        subs = [u for u in _LOC_RE.findall(index) if u.endswith("-listings.xml.gz")]131        out, seen = [], set()132        for sub in subs:133            try:134                xml = self._gz(sub)135            except Exception:136                continue137            for u in _LOC_RE.findall(xml):138                m = _FICHE_RE.match(u)139                if m and m.group(3).split("-")[0] in _EXCLUDED_TYPES:140                    continue141                if m and m.group(4) not in seen:142                    seen.add(m.group(4))143                    out.append((u, m.group(2), m.group(3), m.group(4)))144        return out145146    def _fetch_fiche(self, url: str) -> str:147        """GET d'une fiche ; une redirection vers la page parente = retirée."""148        resp = self.get(url, allow_redirects=True)149        if "/hab-" not in resp.url:150            return ""                     # 302 « annonce disparue »151        return resp.text152153    @staticmethod154    def _unit_type(type_slug: str, bedrooms: str | None) -> str:155        m = _TYPE_HALF_RE.match(type_slug)156        if m:157            n = int(m.group(1))158            return "6½+" if n >= 6 else f"{n}½"159        base = {"studio": "Studio", "loft": "Loft", "chambre": "Chambre",160                "maison": "Maison", "condo": "", "appartement": ""}161        label = base.get(type_slug.split("-")[0], "")162        if not label and bedrooms and bedrooms.isdigit():163            n = int(bedrooms) + 2164            return "6½+" if n >= 6 else f"{n}½"165        return label or type_slug.replace("-", " ").capitalize()166167    def fetch(self) -> list[Listing]:168        fiches = self._fiche_urls()169        if MAX_FICHES:170            fiches = fiches[:MAX_FICHES]171        cache = du.TtlDetailCache(self, budget=DETAIL_LIMIT, ttl_days=TTL_DAYS,172                                  fetch_html=self._fetch_fiche)173        out: list[Listing] = []174        try:175            for url, _ville, type_slug, eid in fiches:176                payload = cache.get(eid, url, _parse_fiche)177                if payload is None:       # jamais visitée + budget épuisé178                    continue179                if not payload or payload.get("gone"):180                    continue              # fiche retirée (302) ou page vide181                price = payload.get("price")182                # availabilityStarts dans le passé = déjà libre aujourd'hui183                avail = payload.get("availability_date")184                if avail:185                    import datetime as _dt186                    if avail <= _dt.date.today().isoformat():187                        avail = "now"188                # « Ahuntsic (Montréal) » -> secteur + ville ; « X (X) » -> X189                locality = payload.get("city") or ""190                sector = ""191                m = re.match(r"^(.*?)\s*\((.+)\)\s*$", locality)192                if m:193                    inner, outer = m.group(1).strip(), m.group(2).strip()194                    if inner.lower() == outer.lower():195                        locality, sector = inner, ""196                    else:197                        locality, sector = outer, inner198                lst = Listing(199                    source=self.source_id,200                    external_id=str(eid),201                    url=url,202                    title=payload.get("title") or "",203                    address=payload.get("address") or "",204                    sector=sector,205                    city=locality,206                    unit_type=self._unit_type(type_slug, payload.get("bedrooms")),207                    price=price,208                    price_label=(f"{price:,.0f} $/mois".replace(",", " ")209                                 if price else ""),210                    availability_date=avail,211                    availability=("Libre immédiatement" if avail == "now"212                                  else f"Libre le {avail}" if avail else ""),213                    area_sqft=payload.get("area_sqft"),214                    description=payload.get("description") or "",215                    amenities=payload.get("amenities") or [],216                    details=payload.get("details") or {},217                    images=payload.get("images") or [],218                    lat=payload.get("lat"),219                    lng=payload.get("lng"),220                )221                low = [a.lower() for a in lst.amenities]222                if any("animaux" in a and "permis" in a for a in low):223                    lst.pets = "oui"224                if any(a.startswith("meublé") for a in low):225                    lst.furnished = True226                out.append(lst)227        finally:228            cache.close()229        return out230