spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/summit.py : connecteur Summit Property Management5#   (summitmanagement.ca — 3 000+ unités à Montréal : Vue/Triangle, LIV,6#   Allegra, IVY, Le V, Le Duke, Skyla, Le 400 Sherbrooke Ouest, RIVA…).7#   Plateforme Rentsync/LiftSystem : la page /apartments expose client_id et8#   city_ids (div.search-data) et son /scripts/main.js embarque le jeton9#   public du flux officiel https://api.theliftsystem.com/v2/search — aucun10#   rendu JavaScript nécessaire. Comme hazelview.py, on interroge le flux par11#   nombre de chambres (min_bed/max_bed + only_available_suites) pour obtenir,12#   par immeuble ET par type d'unité, les loyers réels (stats min/max), la13#   superficie et le nombre d'unités disponibles -> une annonce par immeuble14#   et par type d'unité (uid stables « <id>-<n>bed »).15#   ⚠️ Gestionnaire pancanadien (Ottawa aussi servie par le même flux) :16#   garde-fou province_code == QC — seules les propriétés montréalaises17#   passent. La fiche immeuble du site (rendu serveur) fournit la galerie18#   assets.rentsync.com, via le cache BD self.detail() (revisitée seulement19#   quand la ligne du flux change).20# -----------------------------------------------------------------------------21from __future__ import annotations2223import hashlib24import re2526from bs4 import BeautifulSoup2728from ..schema import Listing, strip_accents29from .base import BaseConnector3031BASE = "https://www.summitmanagement.ca"32SEARCH_PAGE = f"{BASE}/apartments"33LIFT_API = "https://api.theliftsystem.com/v2/search"3435# Valeurs observées sur la page/main.js — repli si l'extraction dynamique casse36DEFAULT_CLIENT_ID = "162"37DEFAULT_AUTH_TOKEN = "sswpREkUtyeYjeoahA2i"    # jeton public (main.js, prod)38MONTREAL_CITY_ID = "1863"                      # Montréal dans la base Lift3940SEARCH_PARAMS = ("only_available_suites=true&show_all_properties=false"41                 "&min_bath=-1&max_bath=10&min_rate=0&max_rate=20000")4243# jeton de la branche production de main.js :44# a="https://api.theliftsystem.com/v2/search?locale="+s+"&",o="&auth_token=…"45_TOKEN_RE = re.compile(46    r'api\.theliftsystem\.com/v2/search\?locale="\+\w+\+"&",'47    r'\w+="&auth_token=([A-Za-z0-9]+)"')48_MAINJS_RE = re.compile(r'src="(/scripts/main\.js[^"]*)"')49# galerie de la fiche immeuble (assets Rentsync, rendu serveur)50_IMG_RE = re.compile(51    r"https://assets\.rentsync\.com/[^\"'\\)\s]+\.(?:jpg|jpeg|png|webp)", re.I)52_SKIP_IMG = re.compile(r"logo|icon|favicon|badge|theme-settings|/256/", re.I)53_TAG_RE = re.compile(r"<[^>]+>")5455# (min_bed, max_bed, type d'unité Lou-Ka)56_BED_QUERIES = [(0, 0, "Studio"), (1, 1, "3½"), (2, 2, "4½"),57                (3, 3, "5½"), (4, 5, "6½+")]5859# Villes QC admissibles (l'API renvoie aussi Ottawa pour ce client)60_QC_CITIES = {"montreal": "Montréal"}616263class SummitConnector(BaseConnector):64    source_id = "summit"65    request_delay = 0.766    max_details = 30         # garde-fou pages immeuble (vraies requêtes/sync)67    max_images = 206869    # -- paramètres du flux (page + main.js, avec replis) ----------------------70    def _feed_params(self) -> tuple[str, str, str]:71        """(client_id, city_ids, auth_token) lus sur le site, replis constants."""72        client_id, city_ids, token = (DEFAULT_CLIENT_ID, MONTREAL_CITY_ID,73                                      DEFAULT_AUTH_TOKEN)74        try:75            page = self.get(SEARCH_PAGE).text76            soup = BeautifulSoup(page, "html.parser")77            data = soup.find("div", class_="search-data")78            if data:79                client_id = (data.get("data-client-id") or client_id).strip()80                ids = (data.get("data-city-ids-string") or "").strip()81                if ids:            # le filtre province élimine Ottawa ensuite82                    city_ids = ids83            m = _MAINJS_RE.search(page)84            if m:85                js = self.get(BASE + m.group(1)).text86                mt = _TOKEN_RE.search(js)87                if mt:88                    token = mt.group(1)89        except Exception:90            pass    # replis : les constantes observées91        return client_id, city_ids, token9293    def fetch(self) -> list[Listing]:94        client_id, city_ids, token = self._feed_params()9596        listings: list[Listing] = []97        seen: set[str] = set()98        for min_bed, max_bed, unit_type in _BED_QUERIES:99            try:100                props = self._search(client_id, token, city_ids,101                                     min_bed, max_bed)102            except Exception:103                continue104            if not isinstance(props, list):105                continue106            for p in props:107                try:108                    lst = self._prop_listing(p, unit_type, min_bed)109                    if lst and lst.external_id not in seen:110                        seen.add(lst.external_id)111                        listings.append(lst)112                except Exception:113                    continue114115        # galerie de la fiche immeuble (cache BD, 1 requête par immeuble)116        self._fetched = 0117        memo: dict[str, dict] = {}118        for lst in listings:119            pid = lst.external_id.split("-")[0]120            if not lst.url:121                continue122            if pid not in memo:123                key = hashlib.sha1(124                    f"{lst.availability}|{lst.price}|{lst.images[:1]}"125                    .encode("utf-8")).hexdigest()126                try:127                    memo[pid] = self.detail(128                        pid, key, lambda u=lst.url: self._fetch_gallery(u))129                except Exception:130                    memo[pid] = {}131            extra = [u for u in (memo[pid].get("images") or [])132                     if u not in lst.images]133            lst.images = (lst.images + extra)[: self.max_images]134        return listings135136    def _search(self, client_id: str, token: str, city_ids: str,137                min_bed: int, max_bed: int) -> list:138        url = (f"{LIFT_API}?locale=en&client_id={client_id}"139               f"&auth_token={token}&city_ids={city_ids}"140               f"&min_bed={min_bed}&max_bed={max_bed}"141               f"&{SEARCH_PARAMS}&limit=100")142        return self.get(url, headers={"Accept": "application/json",143                                      "Referer": BASE + "/"}).json()144145    # -- une annonce par immeuble et par type d'unité ---------------------------146    def _prop_listing(self, p: dict, unit_type: str,147                      beds: int) -> Listing | None:148        if not p.get("availability_count"):149            return None150        addr = p.get("address") or {}151        if (addr.get("province_code") or "").upper() != "QC":152            return None         # gestionnaire pancanadien : Ottawa exclue153        city = _QC_CITIES.get(154            strip_accents((addr.get("city") or "").strip().lower()))155        if not city:156            return None         # ville QC inattendue : ne rien inventer157158        stats = ((p.get("statistics") or {}).get("suites") or {})159        rates = stats.get("rates") or {}160        sq = stats.get("square_feet") or {}161162        def _num(v):163            try:164                v = float(v)165            except (TypeError, ValueError):166                return None167            return v168169        rmin, rmax = _num(rates.get("min")), _num(rates.get("max"))170        # le flux publie parfois des sentinelles (0.01 $) — prix plausibles only171        price = rmin if rmin and 300 <= rmin <= 20000 else None172        if price and rmax and rmax != rmin:173            price_label = f"À partir de {int(price)} $ (max {int(rmax)} $)"174        elif price:175            price_label = f"{int(price)} $/mois"176        else:177            price_label = ""178179        sqmin = _num(sq.get("min"))180        area = sqmin if sqmin and 80 <= sqmin <= 20000 else None181182        details_src = p.get("details") or {}183        desc = _TAG_RE.sub(" ", details_src.get("overview") or "")184        desc = re.sub(r"\s+", " ", desc).strip()[:500]185        sbits = [f"{p['availability_count']} unité(s) disponible(s)"]186187        amenities: list[str] = []188        for a in p.get("amenities") or []:189            t = (a.get("name") if isinstance(a, dict) else str(a) or "").strip()190            if t and t not in amenities:191                amenities.append(t)192193        pid = p.get("id")194        name = (p.get("name") or "").strip()195        geo = p.get("geocode") or {}196        try:197            lat, lng = float(geo.get("latitude")), float(geo.get("longitude"))198        except (TypeError, ValueError):199            lat = lng = None200201        pets = None202        if isinstance(p.get("pet_friendly"), bool):203            pets = "oui" if p["pet_friendly"] else None   # false ≠ « interdit »204205        details: dict = {"building": name} if name else {}206        contact = p.get("contact") or {}207        cinfo: dict = {}208        if (contact.get("phone") or "").strip():209            cinfo["phone"] = contact["phone"].strip()210        for em in (contact.get("email") or "").split(","):211            em = em.strip()212            if em and "leadmanaging" not in em:213                cinfo["email"] = em214                break215        if cinfo:216            details["contact"] = cinfo217218        images = []219        if p.get("photo_path"):220            images.append(p["photo_path"])221222        # le site marque certains immeubles « furnished » dans le permalink223        furnished = True if "/furnished/" in (p.get("permalink") or "") else None224225        return Listing(226            source=self.source_id,227            external_id=f"{pid}-{beds}bed",228            url=(p.get("permalink") or SEARCH_PAGE).strip(),229            title=f"{name} — {unit_type}",230            address=", ".join(x for x in [231                (addr.get("address") or "").strip(), city,232                (addr.get("postal_code") or "").strip()] if x),233            sector=(addr.get("neighbourhood") or "").strip(),234            city=city,235            unit_type=unit_type,236            price=price,237            price_label=price_label,238            availability=(p.get("min_availability_date")239                          or p.get("availability_status_label") or ""),240            area_sqft=area,241            pets=pets,242            furnished=furnished,243            description=" — ".join([desc] + sbits if desc else sbits)[:600],244            amenities=amenities[:25],245            details=details,246            images=images,247            lat=lat,248            lng=lng,249        )250251    # -- galerie de la fiche immeuble (rendu serveur) ---------------------------252    def _fetch_gallery(self, url: str) -> dict:253        """Photos assets.rentsync.com de la fiche (variante /1152/ préférée)."""254        if self._fetched >= self.max_details:255            raise RuntimeError("budget de pages immeuble atteint")256        self._fetched += 1257        out: dict = {"images": []}258        try:259            page = self.get(url).text260        except Exception:261            return out262        urls = _IMG_RE.findall(page)263        images: list[str] = []264        seen: set[str] = set()265        for u in urls:266            if _SKIP_IMG.search(u):267                continue268            fname = u.rsplit("/", 1)[-1]269            if fname in seen:270                continue271            big = u.replace("/512/", "/1152/")272            seen.add(fname)273            images.append(big if big in urls else u)274        out["images"] = images[: self.max_images]275        return out276