# ----------------------------------------------------------------------------- # Rent-Ka — Rental listings aggregator (Canada, outside Québec) # Author: Simon-Pierre Boucher — contact@spboucher.ai # connectors/rentals_ca.py : Rentals.ca — national rental portal, covering # CANADA OUTSIDE QUÉBEC (major metros in every province). Same GraphQL # platform as Louer.ca (same City.id, acquireAuthInfo mutation, public # `rentalsGqlKey` from window.appconf); cross-brand duplicates are folded # by the dedup pass (rentals_ca is in the PORTALS list). # ⚠️ rentals.ca sits behind Cloudflare («Just a moment…»): every GraphQL # call is REPLAYED through Scrapfly ASP WITHOUT JS rendering (POST JSON, # minimal cost vs a render_js). To contain the Scrapfly budget: # - bounded city list (RENTKA_RENTALSCA_CITIES) and per-city cap; # - 200-listing pages (1-4 list requests per city); # - details in BATCHES of 40 via nodes(ids:[…]) — 1 request per 40 — # with a TTL cache: only new/stale batches are re-downloaded. # ⚠️ Rentals.ca ToS restrict extraction without written agreement. # ----------------------------------------------------------------------------- from __future__ import annotations import datetime import json import os import re from ..schema import Listing from . import _detailutil as du from .louer_ca import LouerCaConnector # Cities queried (rentals.ca slugs) — deliberately bounded: each list page # costs one Scrapfly ASP call. Major metros in every province outside QC. CITIES = [c.strip() for c in os.environ.get( "RENTKA_RENTALSCA_CITIES", "toronto,ottawa,mississauga,hamilton,london,kitchener,windsor," "vancouver,victoria,surrey,burnaby,calgary,edmonton,winnipeg," "saskatoon,regina,halifax,moncton,fredericton,charlottetown,st-johns" ).split(",") if c.strip()] # default slug -> province (fallback when a detail node has no regionCode) _SLUG_PROV = { "toronto": "ON", "ottawa": "ON", "mississauga": "ON", "hamilton": "ON", "london": "ON", "kitchener": "ON", "windsor": "ON", "vancouver": "BC", "victoria": "BC", "surrey": "BC", "burnaby": "BC", "calgary": "AB", "edmonton": "AB", "winnipeg": "MB", "saskatoon": "SK", "regina": "SK", "halifax": "NS", "moncton": "NB", "fredericton": "NB", "charlottetown": "PE", "st-johns": "NL", } PAGE_SIZE = int(os.environ.get("RENTKA_RENTALSCA_PAGE_SIZE", "200")) MAX_PER_CITY = int(os.environ.get("RENTKA_RENTALSCA_MAX_PER_CITY", "800")) DETAIL_LIMIT = int(os.environ.get("RENTKA_RENTALSCA_DETAIL_LIMIT", "600")) DETAIL_BATCH = int(os.environ.get("RENTKA_RENTALSCA_DETAIL_BATCH", "40")) TTL_DAYS = float(os.environ.get("RENTKA_RENTALSCA_TTL_DAYS", "7")) class RentalsCaConnector(LouerCaConnector): source_id = "rentals_ca" request_delay = 0.3 # entre appels Scrapfly (API, pas le site) home_url = "https://rentals.ca/" gql_url = "https://rentals.ca/graphql" site = "https://rentals.ca" fallback_key = "kJFM-mm4c-xg6B-qiwy" # public rentalsGqlKey (2026-08) page_size = PAGE_SIZE max_per_city = MAX_PER_CITY cities = CITIES # -- transport : tout passe par Scrapfly ASP (POST JSON, sans rendu JS) ---- def _scrapfly_gql(self, query: str, variables: dict, token: str | None = None) -> dict: headers = {"Content-Type": "application/json", "Origin": self.site, "Referer": self.home_url} if token: headers["Authorization"] = f"Bearer {token}" res = self.scrapfly(self.gql_url, render_js=False, asp=True, method="POST", body=json.dumps({"query": query, "variables": variables}), headers=headers) try: return json.loads(res.get("content") or "{}") except ValueError: return {} def _ensure_token(self) -> None: import time as _time if self._token and _time.time() - self._token_time < 2700: return payload = self._scrapfly_gql( "mutation($k:String!){acquireAuthInfo(credentials:{apiKey:$k})" "{jwt status}}", {"k": self._api_key}) auth = ((payload.get("data") or {}).get("acquireAuthInfo")) or {} jwt = auth.get("jwt") if isinstance(jwt, str) and jwt.startswith("{"): jwt = json.loads(jwt) token = jwt.get("accessToken") if isinstance(jwt, dict) else jwt if not token: # clé périmée ? la relire dans window.appconf de la page d'accueil home = self.get_scrapfly(self.home_url, render_js=False, asp=True) m = re.search(r'"rentalsGqlKey":\s*"([^"]+)"', home or "") if m: self._api_key = m.group(1) payload = self._scrapfly_gql( "mutation($k:String!){acquireAuthInfo(credentials:" "{apiKey:$k}){jwt status}}", {"k": self._api_key}) auth = ((payload.get("data") or {}) .get("acquireAuthInfo")) or {} jwt = auth.get("jwt") if isinstance(jwt, str) and jwt.startswith("{"): jwt = json.loads(jwt) token = (jwt.get("accessToken") if isinstance(jwt, dict) else jwt) if not token: raise RuntimeError("Rentals.ca: JWT handshake failed") self._token = token self._token_time = _time.time() def _gql(self, query: str, variables: dict, auth: bool = True) -> dict: if auth: self._ensure_token() payload = self._scrapfly_gql(query, variables, token=self._token if auth else None) if payload.get("errors") and auth: msg = payload["errors"][0].get("message", "") if "AUTH" in msg.upper(): # jeton expiré : on réessaie self._token = "" self._ensure_token() payload = self._scrapfly_gql(query, variables, token=self._token) return payload.get("data") or {} # -- détail par LOTS (1 requête Scrapfly pour DETAIL_BATCH fiches) --------- def _details_batch(self, gids: list[str]) -> dict[str, dict]: q = ("query($ids:[ID!]!){nodes(ids:$ids){... on RentalListing{" + self._DETAIL_FRAG + "}}}") out: dict[str, dict] = {} for node in self._gql(q, {"ids": gids}).get("nodes") or []: if isinstance(node, dict) and node.get("id"): out[node["id"]] = node return out def fetch(self) -> list[Listing]: if not os.environ.get("SCRAPFLY_KEY"): raise RuntimeError("SCRAPFLY_KEY missing (see .env)") today = datetime.date.today().isoformat() cards: dict[str, tuple[dict, str]] = {} for slug in self.cities: cid = self._city_id(slug) if not cid: continue for card in self._city_listings(cid): gid = card.get("id") if gid: cards.setdefault(gid, (card, slug)) # cache TTL : détails frais réutilisés sans trafic, le reste par lots cache = du.TtlDetailCache(self, budget=DETAIL_LIMIT, ttl_days=TTL_DAYS, key="v1", fetch_html=lambda gid: gid) details: dict[str, dict] = {} todo: list[str] = [] out: list[Listing] = [] try: for gid in cards: payload, fresh = cache.peek(gid) if payload is not None: details[gid] = payload if not fresh and len(todo) < DETAIL_LIMIT: todo.append(gid) for i in range(0, len(todo), DETAIL_BATCH): got = self._details_batch(todo[i:i + DETAIL_BATCH]) for gid, node in got.items(): cache.put(gid, node) details[gid] = node for gid, (card, slug) in cards.items(): node = details.get(gid) or {} region = (((node.get("address") or {}).get("city") or {}) .get("regionCode")) or "" if region == "QC": continue # Québec is Rent-Ka's territory out.extend(self._card_listings( card, node, today, default_city=slug.replace("-", " ").title(), province=region or _SLUG_PROV.get(slug, "ON"))) finally: cache.close() return out