# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/louer_ca.py : Louer.ca — portail locatif 100 % Québec du réseau # Rentals.ca. Accès par l'API GraphQL interne (https://louer.ca/graphql), # rétro-conçue : clé publique `rentalsGqlKey` lue dans window.appconf de la # page d'accueil, mutation `acquireAuthInfo` -> JWT (scalaire JSON contenant # accessToken ~1 h + refreshToken), en-tête `Authorization: Bearer `. # La recherche passe par la ville : typeahead -> City.id, puis # `node(id){... on City{ rentalListings(first,after) }}` (curseur Relay ; # le champ rentalListings top-level renvoie 0). Le détail `node(id)` fournit # adresse, description, galerie (images.rentals.ca) et les floorPlans # (un type/prix par plan) -> une annonce Lou-Ka par plan d'étage. # ⚠️ Réseau Rentals.ca : CGU interdisant l'extraction sans accord écrit. # ----------------------------------------------------------------------------- from __future__ import annotations import base64 import datetime import json import os import re import time from ..schema import Listing from .base import BaseConnector from . import _detailutil as du HOME = "https://louer.ca/" GQL = "https://louer.ca/graphql" FALLBACK_KEY = "ME8N-J3IX-At86-2yIi" # villes québécoises visées (slug interne Louer.ca) ; résolues via typeahead CITIES = ["montreal", "quebec", "laval", "gatineau", "longueuil", "sherbrooke", "trois-rivieres", "levis", "terrebonne", "brossard", "saguenay", "drummondville", "granby", "saint-jean-sur-richelieu", "repentigny"] PAGE_SIZE = int(os.environ.get("LOUKA_LOUERCA_PAGE_SIZE", "50")) DETAIL_LIMIT = int(os.environ.get("LOUKA_LOUERCA_DETAIL_LIMIT", "600")) TTL_DAYS = float(os.environ.get("LOUKA_LOUERCA_TTL_DAYS", "5")) MAX_PER_CITY = int(os.environ.get("LOUKA_LOUERCA_MAX_PER_CITY", "0")) # 0 = tout # amenity (catégorie, valeur) -> libellé FR affichable _AMENITY_FR = { "laundry-facilities": "Buanderie", "in-suite-laundry": "Laveuse/sécheuse dans l'unité", "security-on-site": "Sécurité sur place", "storage-lockers": "Espace de rangement", "swimming-pool": "Piscine", "gym": "Salle d'entraînement", "elevator": "Ascenseur", "sauna": "Sauna", "on-site-staff": "Personnel sur place", "stove": "Cuisinière", "fridge": "Réfrigérateur", "balcony": "Balcon", "microwave": "Micro-ondes", "dishwasher": "Lave-vaisselle", "air-conditioning": "Air climatisé", "individual-thermostats": "Thermostats individuels", "heating": "Chauffage inclus", "water": "Eau incluse", "hydro-electricity": "Électricité incluse", "public-transit": "Transport en commun à proximité", "parking": "Stationnement", "no-smoking-allowed": "Non-fumeur", } class LouerCaConnector(BaseConnector): source_id = "louer_ca" request_delay = 0.5 # points d'entrée du réseau Rentals.ca — surchargés par rentals_ca.py home_url = HOME gql_url = GQL site = "https://louer.ca" fallback_key = FALLBACK_KEY page_size = PAGE_SIZE max_per_city = MAX_PER_CITY def __init__(self) -> None: super().__init__() self.session.headers.update({ "Content-Type": "application/json", "Origin": self.site, "Referer": self.home_url, }) self._token = "" self._token_time = 0.0 self._api_key = self.fallback_key # -- auth ------------------------------------------------------------------ def _ensure_token(self) -> None: if self._token and time.time() - self._token_time < 2700: # ~45 min return try: home = self.get(self.home_url).text m = re.search(r'"rentalsGqlKey":\s*"([^"]+)"', home) if m: self._api_key = m.group(1) except Exception: pass data = self._gql( "mutation($k:String!){acquireAuthInfo(credentials:{apiKey:$k}){jwt status}}", {"k": self._api_key}, auth=False) auth = (data or {}).get("acquireAuthInfo") or {} jwt = auth.get("jwt") if isinstance(jwt, str) and jwt.startswith("{"): jwt = json.loads(jwt) token = jwt.get("accessToken") if isinstance(jwt, dict) else jwt if not token: raise RuntimeError("Louer.ca: handshake JWT échoué") self._token = token self._token_time = time.time() def _gql(self, query: str, variables: dict, auth: bool = True) -> dict: headers = {} if auth: self._ensure_token() headers["Authorization"] = f"Bearer {self._token}" resp = self.post(self.gql_url, data=json.dumps({"query": query, "variables": variables}), headers=headers) payload = resp.json() if payload.get("errors"): msg = payload["errors"][0].get("message", "") if "AUTH" in msg.upper() and auth: # jeton expiré : on réessaie self._token = "" self._ensure_token() resp = self.post(self.gql_url, data=json.dumps({"query": query, "variables": variables}), headers={"Authorization": f"Bearer {self._token}"}) payload = resp.json() return payload.get("data") or {} # -- découverte ------------------------------------------------------------ def _city_id(self, slug: str) -> str | None: d = self._gql( "query($v:String!){typeahead(value:$v){nodeType node{id ... on City" "{name path regionCode listingCount}}}}", {"v": slug.replace("-", " ")}) for it in d.get("typeahead") or []: node = it.get("node") or {} if node.get("path") == slug and node.get("regionCode") == "QC": return node.get("id") # à défaut, première ville québécoise proposée for it in d.get("typeahead") or []: node = it.get("node") or {} if node.get("id") and node.get("regionCode") == "QC": return node.get("id") return None _LIST_FRAG = ("id name path location rentRange bedsRange bathsRange " "sizeRange type furnished petOptions amenities verified " "created modified") def _city_listings(self, city_id: str) -> list[dict]: query = ("query($id:ID!,$first:PositiveInt!,$after:String){node(id:$id)" "{... on City{rentalListings(first:$first,after:$after){" "meta{totalCount} pageInfo{hasNextPage endCursor} " "edges{node{" + self._LIST_FRAG + "}}}}}}") out, after = [], None while True: d = self._gql(query, {"id": city_id, "first": self.page_size, "after": after}) rl = ((d.get("node") or {}).get("rentalListings")) or {} edges = rl.get("edges") or [] out.extend(e["node"] for e in edges if e.get("node")) info = rl.get("pageInfo") or {} if not info.get("hasNextPage") or not edges: break if self.max_per_city and len(out) >= self.max_per_city: break after = info.get("endCursor") return out _DETAIL_FRAG = ("id name path location " "address{city{name regionCode} neighbourhood{name} " "postalCode street} description{plain} " "imagesCount images{scales} tours{name type refId} " "floorPlans{beds baths rent size availability furnished " "tours{name type refId}}") def _detail(self, gid: str) -> dict: d = self._gql("query($id:ID!){node(id:$id){... on RentalListing{" + self._DETAIL_FRAG + "}}}", {"id": gid}) return d.get("node") or {} # -- construction ---------------------------------------------------------- @staticmethod def _unit_type(beds) -> str: try: b = float(beds) except (TypeError, ValueError): return "" if b <= 0: return "Studio" n = int(b) + 2 # chambres -> pièces et demie return "6½+" if n >= 6 else f"{n}½" @staticmethod def _amenities(pairs) -> list[str]: out = [] for pair in pairs or []: val = pair[1] if isinstance(pair, list) and len(pair) > 1 else None label = _AMENITY_FR.get(val) if label and label not in out: out.append(label) return out def _images(self, node: dict) -> list[str]: imgs = [] for im in node.get("images") or []: scales = im.get("scales") if isinstance(scales, str): try: scales = json.loads(scales) except ValueError: scales = [] best = "" for sc in scales or []: if sc.get("name") in ("large", "medium") and sc.get("url"): best = sc["url"] if sc["name"] == "large": break if not best and scales: best = scales[0].get("url", "") if best and best not in imgs: imgs.append(best) return imgs @staticmethod def _count(v) -> float | None: """beds/baths GraphQL -> float (0 = studio) ; None si inconnu.""" try: f = float(v) except (TypeError, ValueError): return None return f if 0 <= f <= 20 else None @staticmethod def _tour_url(tours) -> str | None: """URL de visite virtuelle depuis les `tours` GraphQL du réseau Rentals.ca : refId Matterport/YouTube (ou URL complète). La visite interactive prime sur la simple vidéo.""" video = None for t in tours or []: ref = str(t.get("refId") or "").strip() typ = t.get("type") or "" if not ref: continue if ref.startswith("http"): url = ref elif "matterport" in typ: url = f"https://my.matterport.com/show/?m={ref}" elif "youtube" in typ: url = f"https://www.youtube.com/watch?v={ref}" else: continue if "interactive" in typ or "matterport" in typ: return url video = video or url return video @staticmethod def _numeric_id(gid: str) -> str: """« cmVudGFsbGlzdGluZzoxMTMyMTE2 » -> « 1132116 » (rentallisting:1132116).""" try: decoded = base64.b64decode(gid + "==").decode("utf-8", "ignore") m = re.search(r"(\d+)", decoded) if m: return m.group(1) except Exception: pass m = re.search(r"(\d+)", gid) return m.group(1) if m else gid def _card_listings(self, card: dict, node: dict, today: str, default_city: str = "") -> list[Listing]: """Annonces Lou-Ka d'une carte liste + son nœud détail GraphQL (une par plan d'étage). Partagé avec rentals_ca.py.""" out: list[Listing] = [] gid = card.get("id") addr = node.get("address") or {} city = (addr.get("city") or {}).get("name") or default_city sector = (addr.get("neighbourhood") or {}).get("name") or "" loc = card.get("location") or [] lng, lat = (loc + [None, None])[:2] base_desc = ((node.get("description") or {}).get("plain") or "")[:6000] images = self._images(node) amenities = self._amenities(card.get("amenities")) common = dict( source=self.source_id, url=f"{self.site}/{card.get('path','')}", address=addr.get("street") or "", sector=sector, city=city, description=base_desc, amenities=amenities, images=images, lat=lat, lng=lng, ) base_details: dict = {} if addr.get("postalCode"): base_details["Code postal"] = addr["postalCode"] tour = self._tour_url(node.get("tours")) if tour: base_details["virtual_tour"] = tour if base_details: common["details"] = base_details plans = node.get("floorPlans") or [] if not plans: # pas de plan détaillé : une annonce « à partir de » rng = card.get("rentRange") or [] price = rng[0] if rng else None beds = (card.get("bedsRange") or [None])[0] lst = self._mk(common, gid, "", price, self._unit_type(beds), None, today, price_from=bool(rng)) lst.bedrooms = self._count(beds) lst.bathrooms = self._count( (card.get("bathsRange") or [None])[0]) out.append(lst) return out for i, fp in enumerate(plans): avail = fp.get("availability") or {} adate = "now" if avail.get("now") else ( avail.get("date") or "")[:10] or None if adate and adate != "now" and adate <= today: adate = "now" lst = self._mk(common, gid, f"-{i}", fp.get("rent"), self._unit_type(fp.get("beds")), fp.get("size"), today, adate=adate) lst.bedrooms = self._count(fp.get("beds")) lst.bathrooms = self._count(fp.get("baths")) fp_tour = self._tour_url(fp.get("tours")) if fp_tour: # la visite du plan précis bat celle de l'immeuble lst.details["virtual_tour"] = fp_tour if fp.get("furnished") == "yes": lst.furnished = True out.append(lst) return out def fetch(self) -> list[Listing]: # le « détail » est une requête GraphQL, pas une page HTML : le # fetch_html du cache reçoit le gid et le renvoie tel quel, parse_fn # exécute la requête GraphQL. # clé v2 : force le re-téléchargement progressif des détails pour # capter tours{} (visites virtuelles) et floorPlans.baths ajoutés en # vague 2 — les payloads v1 périmés restent utilisés en attendant cache = du.TtlDetailCache(self, budget=DETAIL_LIMIT, ttl_days=TTL_DAYS, key="v2", fetch_html=lambda gid: gid) out: list[Listing] = [] today = datetime.date.today().isoformat() try: for slug in CITIES: cid = self._city_id(slug) if not cid: continue for card in self._city_listings(cid): gid = card.get("id") if not gid: continue node = cache.get(gid, gid, lambda g: self._detail(g)) or {} out.extend(self._card_listings(card, node, today, default_city=slug.title())) finally: cache.close() return out def _mk(self, common: dict, gid: str, suffix: str, price, unit_type, size, today, adate=None, price_from=False) -> Listing: num = self._numeric_id(gid) p = None try: p = float(price) if price is not None else None except (TypeError, ValueError): p = None details = dict(common.get("details") or {}) if price_from: details["price_from"] = True lst = Listing( **{k: v for k, v in common.items() if k != "details"}, external_id=f"{num}{suffix}", title=common.get("address") or "Logement à louer", unit_type=unit_type, price=p, price_label=(("À partir de " if price_from else "") + (f"{p:,.0f} $/mois".replace(",", " ") if p else "")), area_sqft=float(size) if size else None, availability_date=adate, availability=("Libre immédiatement" if adate == "now" else f"Libre le {adate}" if adate else ""), details=details, ) return lst