# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/zumper.py : Zumper (zumper.com) — portail locatif nord-américain, # filtré PROVINCE DE QUÉBEC, en DIRECT (aucun proxy payant). # API JSON interne (celle du site) : # 1. GET /api/t/1/bundle → cookie `csrftoken` + jeton `csrf` # 2. POST /api/t/1/pages/listables → annonces par bbox (en-tête # `x-csrftoken`) ; paginé offset/limit, réponse = groupes « immeuble » # (fourchettes prix/chambres) et pads individuels, champ `state` filtrable # 3. GET /api/t/1/buildings/{id} → fiche immeuble : description, # `floorplan_listings` = TOUTES les unités (prix, chambres, sdb, pi², # disponibilité) — 1 requête par immeuble, mise en cache BD et invalidée # par `modified_on` (le champ bouge quand l'inventaire change). # Photos : https://img.zumpercdn.com/{media_id}/1280x960 (vérifié 2026-08-18). # ⚠️ Le HTML du site est derrière un « Client Challenge » JS, mais l'API # répond en direct avec des en-têtes navigateur + le rituel CSRF ci-dessus. # Un 429 ponctuel = throttling : une seule reprise après pause, sinon on # laisse le disjoncteur d'ingestion faire son travail. # ⚠️ PadMapper (padmapper.com) = MÊME compagnie et MÊME API (`/api/t/1/…`, # mêmes listing_id — vérifié 100/100 identiques sur Montréal 2026-08-18) : # un seul connecteur suffit, `padmapper` reste dans PORTALS (dédup) au cas où. # Recherche par bbox métropolitaines (le slug de ville est fragile, ex. # « quebec-city-qc » ne matche rien) + filtre state == QC (Ottawa exclu). # ----------------------------------------------------------------------------- from __future__ import annotations import json import os import time from ..schema import Listing from .base import BaseConnector BASE = "https://www.zumper.com" BUNDLE_URL = f"{BASE}/api/t/1/bundle" LISTABLES_URL = f"{BASE}/api/t/1/pages/listables" BUILDING_URL = f"{BASE}/api/t/1/buildings/{{}}" LISTING_URL = f"{BASE}/api/t/1/listings/{{}}" IMG_URL = "https://img.zumpercdn.com/{}/1280x960" # bbox des régions QC couvertes : nom -> (minLat, maxLat, minLng, maxLng) # surchargeable : LOUKA_ZUMPER_REGIONS="nom:minLat,maxLat,minLng,maxLng;…" _DEFAULT_REGIONS = { "montreal": (45.30, 45.80, -74.10, -73.20), # inclut Laval/Longueuil "quebec-levis": (46.65, 47.00, -71.55, -71.00), "gatineau": (45.35, 45.75, -76.10, -75.40), # state==QC exclut Ottawa "sherbrooke": (45.25, 45.55, -72.10, -71.70), "trois-rivieres": (46.25, 46.50, -72.75, -72.40), "saguenay": (48.30, 48.55, -71.35, -70.90), "drummondville": (45.80, 45.95, -72.60, -72.40), "granby-stjean": (45.20, 45.45, -73.35, -72.60), } def _regions() -> dict[str, tuple[float, float, float, float]]: raw = os.environ.get("LOUKA_ZUMPER_REGIONS", "") if not raw.strip(): return _DEFAULT_REGIONS out = {} for part in raw.split(";"): if ":" not in part: continue name, coords = part.split(":", 1) try: a, b, c, d = (float(x) for x in coords.split(",")) out[name.strip()] = (a, b, c, d) except ValueError: continue return out or _DEFAULT_REGIONS PAGE_SIZE = int(os.environ.get("LOUKA_ZUMPER_PAGE_SIZE", "100")) MAX_PER_REGION = int(os.environ.get("LOUKA_ZUMPER_MAX_PER_REGION", "1500")) # budget de fiches immeuble (vraies requêtes) par synchronisation — le cache BD # (clé modified_on) absorbe l'essentiel après le premier passage DETAIL_LIMIT = int(os.environ.get("LOUKA_ZUMPER_DETAIL_LIMIT", "150")) def _unit_type_from_bedrooms(bedrooms) -> str: if bedrooms is None: return "" try: n = int(float(bedrooms)) except (TypeError, ValueError): return "" if n <= 0: return "Studio" pieces = n + 2 # convention QC : n ch. -> (n+2)½ return "6½+" if pieces >= 6 else f"{pieces}½" class ZumperConnector(BaseConnector): source_id = "zumper" request_delay = 1.2 def __init__(self) -> None: super().__init__() self._csrf = "" self.session.headers.update({ "Accept": "application/json", "Origin": BASE, "Referer": f"{BASE}/apartments-for-rent/montreal-qc", }) # -- rituel CSRF ----------------------------------------------------------- def _bootstrap(self) -> None: data = self.get(BUNDLE_URL).json() self._csrf = data.get("csrf") or "" if not self._csrf: raise RuntimeError("zumper: jeton csrf absent du bundle") def _api(self, method: str, url: str, payload: dict | None = None) -> dict: """Appel API avec en-tête CSRF + une reprise sur 429/403 (rebootstrap).""" if not self._csrf: self._bootstrap() headers = {"x-csrftoken": self._csrf, "content-type": "application/json"} for attempt in (1, 2): try: if method == "POST": resp = self.post(url, data=json.dumps(payload or {}), headers=headers) else: resp = self.get(url, headers=headers) return resp.json() except Exception as exc: code = getattr(getattr(exc, "response", None), "status_code", None) if attempt == 1 and code in (403, 429): time.sleep(30 if code == 429 else 3) self._bootstrap() headers["x-csrftoken"] = self._csrf continue raise return {} # -- collecte -------------------------------------------------------------- def fetch(self) -> list[Listing]: rows: dict[int, dict] = {} for name, (min_lat, max_lat, min_lng, max_lng) in _regions().items(): offset = 0 while offset < MAX_PER_REGION: data = self._api("POST", LISTABLES_URL, { "external": True, "longTerm": True, "shortTerm": False, "minPrice": 0, "transits": {}, "minLat": min_lat, "maxLat": max_lat, "minLng": min_lng, "maxLng": max_lng, "limit": PAGE_SIZE, "offset": offset, "matching": True, "excludeGroupIds": [], "ignorePopular": True, "descriptionLength": 0, }) batch = data.get("listables") or [] for row in batch: lid = row.get("listing_id") if lid and row.get("state") == "QC": rows.setdefault(int(lid), row) total = data.get("matching") or 0 offset += PAGE_SIZE if not batch or offset >= total: break listings: dict[str, Listing] = {} budget = DETAIL_LIMIT for row in rows.values(): multi = (row.get("floorplan_count") or 1) > 1 or ( row.get("min_price") != row.get("max_price")) building_id = row.get("building_id") expanded = False payload = None # fiche immeuble même pour les rangées à unité unique : c'est la # SEULE source de description (listables ne renvoie jamais # short_description, peu importe descriptionLength — testé # 2026-08-18) ; cache BD (clé modified_on) + budget partagés if building_id: payload, budget = self._building_units(row, budget) if multi and payload: for unit in payload.get("floorplans", []): lst = self._unit_listing(row, unit, payload) if lst and lst.uid not in listings: listings[lst.uid] = lst expanded = True if not expanded: desc = (payload or {}).get("description") or "" if not desc and not row.get("short_description"): # pad sans immeuble (building_id null) : la description # vit sur /api/t/1/listings/{id} (fiche du pad) pad, budget = self._pad_detail(row, budget) desc = (pad or {}).get("description") or "" lst = self._row_listing(row, desc) if lst and lst.uid not in listings: listings[lst.uid] = lst return list(listings.values()) # -- fiche pad (cache BD, clé modified_on) ---------------------------------- def _pad_detail(self, row: dict, budget: int) -> tuple[dict | None, int]: """Fiche /api/t/1/listings/{id} d'un pad individuel : description longue (1 000-2 500 caractères). Même cache/budget que les immeubles.""" lid = row.get("listing_id") if not lid: return None, budget key = f"m{row.get('modified_on') or 0}" from .. import db if self._detail_con is None: self._detail_con = db.connect() cached = db.get_cached_detail(self._detail_con, self.source_id, f"p{lid}", key) if cached is not None: return cached, budget if budget <= 0: return None, budget budget -= 1 try: data = self._api("GET", LISTING_URL.format(lid)) except Exception: return None, budget payload = {"description": (data.get("description") or "")[:2500]} db.put_cached_detail(self._detail_con, self.source_id, f"p{lid}", key, payload) return payload, budget # -- fiche immeuble (cache BD, clé modified_on) ----------------------------- def _building_units(self, row: dict, budget: int) -> tuple[dict | None, int]: building_id = row["building_id"] key = f"m{row.get('modified_on') or 0}" from .. import db if self._detail_con is None: self._detail_con = db.connect() cached = db.get_cached_detail(self._detail_con, self.source_id, f"b{building_id}", key) if cached is not None: return cached, budget if budget <= 0: return None, budget budget -= 1 try: data = self._api("GET", BUILDING_URL.format(building_id)) except Exception: return None, budget fps = [] for fp in data.get("floorplan_listings") or []: if fp.get("listing_status") not in (None, 1): continue fps.append({ "listing_id": fp.get("listing_id"), "title": fp.get("title") or "", "price": fp.get("min_price") or fp.get("price"), "max_price": fp.get("max_price"), "bedrooms": fp.get("bedrooms"), "bathrooms": fp.get("bathrooms"), "half_bathrooms": fp.get("half_bathrooms"), "square_feet": fp.get("square_feet"), "date_available": fp.get("date_available"), "image_ids": [m.get("media_id") for m in fp.get("media") or [] if m.get("media_type") == 1][:8], }) payload = {"floorplans": fps, "description": (data.get("description") or "")[:2000], "year_built": data.get("year_built"), "homepage": data.get("homepage")} db.put_cached_detail(self._detail_con, self.source_id, f"b{building_id}", key, payload) return payload, budget # -- constructions ---------------------------------------------------------- def _common(self, row: dict) -> dict: amen = list(dict.fromkeys((row.get("amenity_tags") or []) + (row.get("building_amenity_tags") or []))) details = {k: row[k] for k in ("neighborhood_name", "brokerage_name", "building_name", "zipcode") if row.get(k)} return {"amenities": amen[:40], "details": details} def _url(self, row: dict) -> str: path = row.get("url") or "" if path.startswith("/"): return f"{BASE}{path}" if row.get("pb_url"): return f"{BASE}/apartment-buildings/p{row.get('pb_id')}/{row['pb_url']}" return f"{BASE}/apartments-for-rent/montreal-qc" def _unit_listing(self, row: dict, unit: dict, payload: dict) -> Listing | None: price = unit.get("price") if not price or price < 200: return None ext_id = str(unit.get("listing_id") or f"{row['building_id']}-{unit.get('title', '')}") beds = unit.get("bedrooms") baths = unit.get("bathrooms") if baths is not None and unit.get("half_bathrooms"): baths = float(baths) + 0.5 * float(unit["half_bathrooms"]) common = self._common(row) imgs = [IMG_URL.format(m) for m in unit.get("image_ids") or []] or \ [IMG_URL.format(m) for m in (row.get("image_ids") or [])[:8]] avail = unit.get("date_available") or "" return Listing( source=self.source_id, external_id=ext_id, url=self._url(row), title=(f"{row.get('building_name') or row.get('address', '')} — " f"{unit.get('title') or ''}").strip(" —"), address=row.get("address", ""), sector=row.get("neighborhood_name") or "", city=row.get("city", ""), unit_type=_unit_type_from_bedrooms(beds), bedrooms=float(beds) if beds is not None else None, bathrooms=float(baths) if baths is not None else None, price=float(price), price_label=f"{price:,.0f} $/mois".replace(",", " "), availability=str(avail), area_sqft=(float(unit["square_feet"]) if unit.get("square_feet") else None), description=payload.get("description") or "", amenities=common["amenities"], details=common["details"], images=imgs, lat=row.get("lat"), lng=row.get("lng"), ) def _row_listing(self, row: dict, description: str = "") -> Listing | None: price = row.get("min_price") if not price or price < 200: return None mn, mx = row.get("min_price"), row.get("max_price") from_price = bool(mx and mx != mn) beds = row.get("min_bedrooms") baths = row.get("min_bathrooms") common = self._common(row) label = (f"à partir de {price:,.0f} $/mois" if from_price else f"{price:,.0f} $/mois").replace(",", " ") return Listing( source=self.source_id, external_id=str(row["listing_id"]), url=self._url(row), title=(row.get("building_name") or row.get("title") or row.get("address", "")), address=row.get("address", ""), sector=row.get("neighborhood_name") or "", city=row.get("city", ""), unit_type=_unit_type_from_bedrooms(beds), bedrooms=float(beds) if beds is not None else None, bathrooms=float(baths) if baths is not None else None, price=float(price), price_label=label, availability=str(row.get("date_available") or ""), description=row.get("short_description") or description, amenities=common["amenities"], details=common["details"], images=[IMG_URL.format(m) for m in (row.get("image_ids") or [])[:8]], lat=row.get("lat"), lng=row.get("lng"), )