Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/louer_ca.py : ABSTRACT BASE for the Rentals.ca GraphQL network5# (kept under its historical name; the louer.ca QC connector itself is NOT6# registered — source_id is empty). Reverse-engineered internal GraphQL7# API: public `rentalsGqlKey` read from window.appconf on the home page,8# `acquireAuthInfo` mutation -> JWT (JSON scalar with ~1 h accessToken),9# `Authorization: Bearer <access>` header. Search goes through the city:10# typeahead -> City.id, then `node(id){... on City{ rentalListings(first,11# after) }}` (Relay cursor). The `node(id)` detail provides the address,12# description, gallery (images.rentals.ca) and floorPlans (one type/price13# per plan) -> one listing per floor plan.14# ⚠️ Rentals.ca network ToS restrict extraction without written agreement.15# -----------------------------------------------------------------------------16from __future__ import annotations1718import base6419import datetime20import json21import os22import re23import time2425from ..schema import Listing26from .base import BaseConnector27from . import _detailutil as du2829HOME = "https://louer.ca/"30GQL = "https://louer.ca/graphql"31FALLBACK_KEY = "ME8N-J3IX-At86-2yIi"3233PAGE_SIZE = int(os.environ.get("RENTKA_LOUERCA_PAGE_SIZE", "50"))34DETAIL_LIMIT = int(os.environ.get("RENTKA_LOUERCA_DETAIL_LIMIT", "600"))35TTL_DAYS = float(os.environ.get("RENTKA_LOUERCA_TTL_DAYS", "5"))36MAX_PER_CITY = int(os.environ.get("RENTKA_LOUERCA_MAX_PER_CITY", "0")) # 0=all3738# amenity (category, value) -> display label (normalized by finalize())39_AMENITY_FR = {40 "laundry-facilities": "Laundry facilities",41 "in-suite-laundry": "In-suite laundry",42 "security-on-site": "On-site security", "storage-lockers": "Storage locker",43 "swimming-pool": "Pool", "gym": "Gym", "elevator": "Elevator",44 "sauna": "Sauna", "on-site-staff": "On-site staff", "stove": "Stove",45 "fridge": "Fridge", "balcony": "Balcony", "microwave": "Microwave",46 "dishwasher": "Dishwasher", "air-conditioning": "Air conditioning",47 "individual-thermostats": "Individual thermostats",48 "heating": "Heat included",49 "water": "Water included", "hydro-electricity": "Electricity included",50 "public-transit": "Public transit nearby", "parking": "Parking",51 "no-smoking-allowed": "No smoking",52}5354# regionCode -> province code (identity for Canadian provinces)55_EXCLUDED_REGIONS = {"QC"} # Québec is Rent-Ka's territory565758class LouerCaConnector(BaseConnector):59 # Abstract base: empty source_id keeps it OUT of the connector registry.60 # Subclasses (rentals_ca) set their own source_id/transport/cities.61 source_id = ""62 request_delay = 0.563 home_url = HOME64 gql_url = GQL65 fallback_key = FALLBACK_KEY66 page_size = PAGE_SIZE67 max_per_city = MAX_PER_CITY68 cities: list[str] = []6970 def __init__(self) -> None:71 super().__init__()72 self.session.headers.update({73 "Content-Type": "application/json",74 "Origin": self.home_url.rstrip("/"),75 "Referer": self.home_url,76 })77 self._token = ""78 self._token_time = 0.079 self._api_key = self.fallback_key8081 # -- auth ------------------------------------------------------------------82 def _ensure_token(self) -> None:83 if self._token and time.time() - self._token_time < 2700: # ~45 min84 return85 try:86 home = self.get(self.home_url).text87 m = re.search(r'"rentalsGqlKey":\s*"([^"]+)"', home)88 if m:89 self._api_key = m.group(1)90 except Exception:91 pass92 data = self._gql(93 "mutation($k:String!){acquireAuthInfo(credentials:{apiKey:$k}){jwt status}}",94 {"k": self._api_key}, auth=False)95 auth = (data or {}).get("acquireAuthInfo") or {}96 jwt = auth.get("jwt")97 if isinstance(jwt, str) and jwt.startswith("{"):98 jwt = json.loads(jwt)99 token = jwt.get("accessToken") if isinstance(jwt, dict) else jwt100 if not token:101 raise RuntimeError("Rentals.ca network: JWT handshake failed")102 self._token = token103 self._token_time = time.time()104105 def _gql(self, query: str, variables: dict, auth: bool = True) -> dict:106 headers = {}107 if auth:108 self._ensure_token()109 headers["Authorization"] = f"Bearer {self._token}"110 resp = self.post(self.gql_url,111 data=json.dumps({"query": query,112 "variables": variables}),113 headers=headers)114 payload = resp.json()115 if payload.get("errors"):116 msg = payload["errors"][0].get("message", "")117 if "AUTH" in msg.upper() and auth: # expired token: retry once118 self._token = ""119 self._ensure_token()120 resp = self.post(self.gql_url,121 data=json.dumps({"query": query,122 "variables": variables}),123 headers={"Authorization": f"Bearer {self._token}"})124 payload = resp.json()125 return payload.get("data") or {}126127 # -- discovery ---------------------------------------------------------------128 def _city_id(self, slug: str) -> str | None:129 """Resolve a city slug to its City.id — any Canadian region except130 the excluded ones (Québec)."""131 d = self._gql(132 "query($v:String!){typeahead(value:$v){nodeType node{id ... on City"133 "{name path regionCode listingCount}}}}", {"v": slug.replace("-", " ")})134 for it in d.get("typeahead") or []:135 node = it.get("node") or {}136 if node.get("path") == slug \137 and node.get("regionCode") not in _EXCLUDED_REGIONS:138 return node.get("id")139 # otherwise the first proposed city outside the excluded regions140 for it in d.get("typeahead") or []:141 node = it.get("node") or {}142 if node.get("id") and node.get("regionCode") \143 and node.get("regionCode") not in _EXCLUDED_REGIONS:144 return node.get("id")145 return None146147 _LIST_FRAG = ("id name path location rentRange bedsRange bathsRange "148 "sizeRange type furnished petOptions amenities verified "149 "created modified")150151 def _city_listings(self, city_id: str) -> list[dict]:152 query = ("query($id:ID!,$first:PositiveInt!,$after:String){node(id:$id)"153 "{... on City{rentalListings(first:$first,after:$after){"154 "meta{totalCount} pageInfo{hasNextPage endCursor} "155 "edges{node{" + self._LIST_FRAG + "}}}}}}")156 out, after = [], None157 while True:158 d = self._gql(query, {"id": city_id, "first": self.page_size,159 "after": after})160 rl = ((d.get("node") or {}).get("rentalListings")) or {}161 edges = rl.get("edges") or []162 out.extend(e["node"] for e in edges if e.get("node"))163 info = rl.get("pageInfo") or {}164 if not info.get("hasNextPage") or not edges:165 break166 if self.max_per_city and len(out) >= self.max_per_city:167 break168 after = info.get("endCursor")169 return out170171 _DETAIL_FRAG = ("id name path location "172 "address{city{name regionCode} neighbourhood{name} "173 "postalCode street} description{plain} "174 "imagesCount images{scales} "175 "floorPlans{beds baths rent size availability furnished}")176177 def _detail(self, gid: str) -> dict:178 d = self._gql("query($id:ID!){node(id:$id){... on RentalListing{"179 + self._DETAIL_FRAG + "}}}", {"id": gid})180 return d.get("node") or {}181182 # -- construction ----------------------------------------------------------183 @staticmethod184 def _unit_type(beds) -> str:185 try:186 b = float(beds)187 except (TypeError, ValueError):188 return ""189 if b <= 0:190 return "Studio"191 n = int(b)192 if n >= 5:193 return "5+ bedrooms"194 return f"{n} bedroom" + ("s" if n > 1 else "")195196 @staticmethod197 def _amenities(pairs) -> list[str]:198 out = []199 for pair in pairs or []:200 val = pair[1] if isinstance(pair, list) and len(pair) > 1 else None201 label = _AMENITY_FR.get(val)202 if label and label not in out:203 out.append(label)204 return out205206 def _images(self, node: dict) -> list[str]:207 imgs = []208 for im in node.get("images") or []:209 scales = im.get("scales")210 if isinstance(scales, str):211 try:212 scales = json.loads(scales)213 except ValueError:214 scales = []215 best = ""216 for sc in scales or []:217 if sc.get("name") in ("large", "medium") and sc.get("url"):218 best = sc["url"]219 if sc["name"] == "large":220 break221 if not best and scales:222 best = scales[0].get("url", "")223 if best and best not in imgs:224 imgs.append(best)225 return imgs226227 @staticmethod228 def _numeric_id(gid: str) -> str:229 """« cmVudGFsbGlzdGluZzoxMTMyMTE2 » -> « 1132116 » (rentallisting:1132116)."""230 try:231 decoded = base64.b64decode(gid + "==").decode("utf-8", "ignore")232 m = re.search(r"(\d+)", decoded)233 if m:234 return m.group(1)235 except Exception:236 pass237 m = re.search(r"(\d+)", gid)238 return m.group(1) if m else gid239240 def _card_listings(self, card: dict, node: dict, today: str,241 default_city: str = "",242 province: str = "ON") -> list[Listing]:243 """One list card + its detail node -> one listing per floor plan244 (or a single «from $» listing when there are no plans)."""245 gid = card.get("id")246 if not gid:247 return []248 addr = node.get("address") or {}249 city = (addr.get("city") or {}).get("name") or default_city250 sector = (addr.get("neighbourhood") or {}).get("name") or ""251 loc = card.get("location") or []252 lng, lat = (loc + [None, None])[:2]253 base_desc = ((node.get("description") or {}).get("plain") or "")[:6000]254 images = self._images(node)255 amenities = self._amenities(card.get("amenities"))256 common = dict(257 source=self.source_id,258 url=f"{self.home_url.rstrip('/')}/{card.get('path', '')}",259 address=addr.get("street") or "", sector=sector, city=city,260 province=province,261 description=base_desc, amenities=amenities, images=images,262 lat=lat, lng=lng,263 )264 if addr.get("postalCode"):265 common["details"] = {"Postal code": addr["postalCode"]}266267 out: list[Listing] = []268 plans = node.get("floorPlans") or []269 if not plans:270 # no detailed plan: one «from $» listing271 rng = card.get("rentRange") or []272 price = rng[0] if rng else None273 beds = (card.get("bedsRange") or [None])[0]274 out.append(self._mk(common, gid, "", price,275 self._unit_type(beds), None, today,276 price_from=bool(rng)))277 return out278 for i, fp in enumerate(plans):279 avail = fp.get("availability") or {}280 adate = "now" if avail.get("now") else (281 avail.get("date") or "")[:10] or None282 if adate and adate != "now" and adate <= today:283 adate = "now"284 lst = self._mk(common, gid, f"-{i}", fp.get("rent"),285 self._unit_type(fp.get("beds")),286 fp.get("size"), today, adate=adate)287 if fp.get("furnished") == "yes":288 lst.furnished = True289 out.append(lst)290 return out291292 def fetch(self) -> list[Listing]:293 # the "detail" is a GraphQL request, not an HTML page: the cache's294 # fetch_html receives the gid and returns it as-is, parse_fn runs295 # the GraphQL query.296 cache = du.TtlDetailCache(self, budget=DETAIL_LIMIT, ttl_days=TTL_DAYS,297 key="v1", fetch_html=lambda gid: gid)298 out: list[Listing] = []299 today = datetime.date.today().isoformat()300 try:301 for slug in self.cities:302 cid = self._city_id(slug)303 if not cid:304 continue305 for card in self._city_listings(cid):306 gid = card.get("id")307 if not gid:308 continue309 node = cache.get(gid, gid,310 lambda g: self._detail(g)) or {}311 region = (((node.get("address") or {}).get("city") or {})312 .get("regionCode")) or ""313 if region in _EXCLUDED_REGIONS:314 continue315 out.extend(self._card_listings(316 card, node, today,317 default_city=slug.replace("-", " ").title(),318 province=region or "ON"))319 finally:320 cache.close()321 return out322323 def _mk(self, common: dict, gid: str, suffix: str, price, unit_type,324 size, today, adate=None, price_from=False) -> Listing:325 num = self._numeric_id(gid)326 p = None327 try:328 p = float(price) if price is not None else None329 except (TypeError, ValueError):330 p = None331 details = dict(common.get("details") or {})332 if price_from:333 details["price_from"] = True334 lst = Listing(335 **{k: v for k, v in common.items() if k != "details"},336 external_id=f"{num}{suffix}",337 title=common.get("address") or "Rental unit",338 unit_type=unit_type,339 price=p,340 price_label=(("From " if price_from else "")341 + (f"${p:,.0f}/month" if p else "")),342 area_sqft=float(size) if size else None,343 availability_date=adate,344 availability=("Available now" if adate == "now"345 else f"Available {adate}" if adate else ""),346 details=details,347 )348 return lst349