spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/gestion_cem.py : Gestion CEM (gestioncem.com — Montréal, Longueuil,5#   LaSalle/Sud-Ouest…) via son portail Eversa (gestioncem.eversa.app).6#   Techno : React Router v7 (Remix) « turbo-stream » — les routes exposent un7#   endpoint JSON `<route>.data` contenant un TABLEAU PLAT à références8#   indexées : {"_1":2} = clé à l'index 1, valeur à l'index 2 ; entier négatif9#   = null/undefined ; ["D", <millis>] = date epoch. On décode récursivement.10#   Granularité : UNITÉ (page détail par immeuble -> units[] avec listed=true,11#   prix, pi², chambres, sdb, photos, date de disponibilité). Repli sur la12#   carte de l'immeuble si la page détail est indisponible (404 transitoire).13#   Depuis ~2026-09-17 les routes exigent un préfixe de locale (/fr, /en) :14#   sans préfixe, `.data` renvoie un SingleFetchRedirect 301 (payload liste).15# -----------------------------------------------------------------------------16from __future__ import annotations1718import datetime19import json20import re2122import requests2324from .base import BaseConnector25from ..schema import Listing2627BASE = "https://gestioncem.eversa.app"2829# chunks turbo-stream inlinés dans le HTML (repli quand `.data` renvoie 404)30_ENQUEUE_RE = re.compile(r'streamController\.enqueue\((".*?")\);', re.S)313233def _decode(raw: str):34    """Décode un document turbo-stream (tableau plat à références indexées)."""35    # Depuis ~2026-08-23 le flux ajoute des lignes différées (`P14:[…]`, promesses36    # résolues — ex. walkscore) après le document principal : ne garder que la37    # 1re ligne, sinon json.loads échoue en « Extra data ».38    arr = json.loads(raw.split("\n", 1)[0])39    memo: dict = {}4041    def dec(i):42        if isinstance(i, int) and i < 0:        # codes spéciaux (null/undefined…)43            return None44        if not isinstance(i, int) or i >= len(arr):45            return i46        if i in memo:47            return memo[i]48        v = arr[i]49        if isinstance(v, dict):50            out: dict = {}51            memo[i] = out52            for k, val in v.items():53                key = dec(int(k[1:])) if k.startswith("_") else k54                out[key] = dec(val) if isinstance(val, int) else val55            return out56        if isinstance(v, list):57            # ["D", <epoch millis>] = date sérialisée (le millis est LITTÉRAL,58            # pas une référence — souvent hors du tableau)59            if len(v) == 2 and v[0] == "D" and isinstance(v[1], (int, float)):60                iso = datetime.datetime.fromtimestamp(61                    v[1] / 1000, tz=datetime.timezone.utc).date().isoformat()62                memo[i] = iso63                return iso64            out2: list = []65            memo[i] = out266            for e in v:67                out2.append(dec(e) if isinstance(e, int) else e)68            return out269        memo[i] = v70        return v7172    return dec(0)737475def _num(v) -> float | None:76    """'1695' / 1695 / '2 000' -> float (None si vide/0)."""77    if isinstance(v, (int, float)):78        return float(v) if v > 0 else None79    if isinstance(v, str):80        s = v.replace(" ", "").replace(" ", "").replace(",", ".")81        try:82            f = float(s)83            return f if f > 0 else None84        except ValueError:85            return None86    return None878889def _unit_type(bedrooms) -> str:90    """Chambres -> typologie QC (0 ch = studio, n ch = (n+2)½)."""91    try:92        bd = int(float(bedrooms))93    except (TypeError, ValueError):94        return ""95    if bd <= 0:96        return "Studio"97    if bd >= 5:98        return "6½+"99    return f"{bd + 2}½"100101102def _images(items, uid: str = "") -> list[str]:103    out: list[str] = []104    for i in items or []:105        if not isinstance(i, str) or not i:106            continue107        url = i if i.startswith("http") else BASE + i108        if url not in out:109            out.append(url)110    return out111112113class GestionCemConnector(BaseConnector):114    """Gestion CEM via Eversa — annonces à l'unité (repli : immeuble)."""115116    source_id = "gestion_cem"117    request_delay = 0.8118119    # -- helpers ---------------------------------------------------------------120    def _route_data(self, path: str):121        """Décode `<path>.data` ; repli sur les chunks inlinés du HTML."""122        try:123            resp = self.get(f"{BASE}{path}.data")124            resp.encoding = "utf-8"             # turbo-stream sans charset déclaré125            return _decode(resp.text)126        except requests.HTTPError:127            pass128        resp = self.get(f"{BASE}{path}")129        resp.encoding = "utf-8"130        chunks = _ENQUEUE_RE.findall(resp.text)131        if not chunks:132            return None133        raw = "".join(json.loads(c) for c in chunks)134        payload = _decode(raw)135        # le HTML enveloppe la même donnée sous loaderData136        if isinstance(payload, dict) and "loaderData" in payload:137            return payload["loaderData"]138        return payload139140    def _detail_listing(self, slug: str) -> dict | None:141        data = self._route_data(f"/fr/listings/{slug}/details")142        if not isinstance(data, dict):143            return None144        page = data.get("PublicListingDetailPage") or {}145        if "data" in page:146            page = page.get("data") or {}147        listing = page.get("listing")148        return listing if isinstance(listing, dict) else None149150    def _building_fallback(self, b: dict, page_url: str) -> Listing | None:151        """Carte de la liste -> une annonce « immeuble » (détail indisponible)."""152        price = _num(b.get("minPricePerMonth") or b.get("price"))153        bd = b.get("bedrooms")154        avail = str(b.get("availability") or "")[:10]155        return Listing(156            source=self.source_id,157            external_id=b.get("slug") or b.get("listingId") or "",158            url=page_url,159            title=(b.get("title") or b.get("street") or "").strip(),160            address=(b.get("street") or "").strip(),161            city=(b.get("city") or "").strip(),162            unit_type=_unit_type(bd),163            bedrooms=_num(bd),164            bathrooms=_num(b.get("bathrooms")),165            price=price,166            price_label=f"{price:.0f} $ /mois" if price else "",167            availability=f"Disponible le {avail}" if avail else "Disponible",168            availability_date=avail or None,169            images=_images(b.get("images") or b.get("buildingImages")),170            lat=b.get("latitude"),171            lng=b.get("longitude"),172        )173174    # -- contrat ---------------------------------------------------------------175    def fetch(self) -> list[Listing]:176        data = self._route_data("/fr/listings/list")177        page = (data or {}).get("PublicListingsLayoutPage") or {}178        if "data" in page:179            page = page.get("data") or {}180        buildings = page.get("listings") or []181182        out: list[Listing] = []183        seen: set[str] = set()184        for b in buildings:185            if not isinstance(b, dict):186                continue187            if b.get("isFullyBooked") or b.get("isAllCommercial"):188                continue189            if (b.get("spaceType") or "apartment") not in ("apartment", "condo",190                                                           "house", "loft"):191                continue                       # résidentiel au mois seulement192            slug = b.get("slug") or ""193            if not slug:194                continue195            page_url = f"{BASE}/fr/listings/{slug}/details"196197            # détail (cache BD : clé = contenu de la carte liste)198            key = json.dumps({k: b.get(k) for k in199                              ("price", "availability", "listedUnitCount",200                               "bedrooms", "isFullyBooked")},201                             sort_keys=True, default=str)202            try:203                detail = self.detail(slug, key,204                                     lambda s=slug: self._detail_listing(s) or {})205            except requests.RequestException:206                detail = {}207208            units = [u for u in (detail.get("units") or [])209                     if isinstance(u, dict) and u.get("listed")]210            if not units:                       # détail 404/vide -> carte liste211                lst = self._building_fallback(b, page_url)212                if lst and lst.external_id and lst.external_id not in seen:213                    seen.add(lst.external_id)214                    out.append(lst)215                continue216217            street = (detail.get("street") or b.get("street") or "").strip()218            city = (detail.get("city") or b.get("city") or "").strip()219            lat = detail.get("latitude") or b.get("latitude")220            lng = detail.get("longitude") or b.get("longitude")221            bldg_imgs = _images(b.get("images") or b.get("buildingImages"))222223            for u in units:224                uid = u.get("slug") or ""       # uuid stable de l'unité225                if not uid or uid in seen:226                    continue227                seen.add(uid)228                price = _num(u.get("price"))229                avail = str(u.get("availability") or "")[:10]230                alias = str(u.get("alias") or "").strip()231                imgs = _images(u.get("images")) or bldg_imgs232                details: dict = {}233                if u.get("floor") is not None:234                    details["floor"] = u["floor"]235                if alias:236                    details["unit_number"] = alias237                out.append(Listing(238                    source=self.source_id,239                    external_id=uid,240                    url=page_url,241                    title=f"{street} — app. {alias}" if alias else street,242                    address=street,243                    city=city,244                    unit_type=_unit_type(u.get("bedrooms")),245                    bedrooms=_num(u.get("bedrooms")),246                    bathrooms=_num(u.get("bathrooms")),247                    price=price,248                    price_label=f"{price:.0f} $ /mois" if price else "",249                    availability=(f"Disponible le {avail}" if avail250                                  else "Disponible"),251                    availability_date=avail or None,252                    area_sqft=_num(u.get("area")),253                    details=details,254                    images=imgs[:30],255                    lat=lat,256                    lng=lng,257                ))258        return out259