Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/centurion.py : Centurion / CP Living (cpliving.com)5# National REIT (Centurion Property Associates). The site runs on Rentsync:6# the search page references a legacy proxy (404 today) but its JS7# (scripts/main.js) actually calls the official JSON feed8# `https://api.theliftsystem.com/v2/search` with an embedded auth_token.9# client_id is read from the page, the token from main.js (constants as10# fallback), then the feed is queried directly WITHOUT city_ids — that11# returns the whole Centurion portfolio (~104 properties: Toronto/GTA,12# Ottawa, Kitchener-Waterloo, Barrie, Huntsville, plus BC/AB/NS/MB…).13# Rent-Ka keeps every province except QC. No JavaScript execution needed;14# Cloudflare accepts the base UA. The server-rendered property page15# provides the photo gallery through the self.detail() DB cache16# (revisited only when the feed row changes).17# -----------------------------------------------------------------------------18from __future__ import annotations1920import hashlib21import os22import re2324from bs4 import BeautifulSoup2526from ..schema import Listing, normalize_unit_type, strip_accents27from .base import BaseConnector2829BASE = "https://www.cpliving.com"30SEARCH_PAGE = f"{BASE}/apartments-for-rent/toronto"31LIFT_API = "https://api.theliftsystem.com/v2/search"3233# Values observed on the page/main.js — fallbacks if dynamic extraction breaks34DEFAULT_CLIENT_ID = "21"35DEFAULT_AUTH_TOKEN = "sswpREkUtyeYjeoahA2i"3637# jeton d'authentification dans main.js : `s="&client_id=21",e="&auth_token=…"`38_TOKEN_RE = r'client_id={cid}",\w+="&auth_token=([A-Za-z0-9]+)"'39_MAINJS_RE = re.compile(r'src="(/scripts/main\.js[^"]*)"')40# galerie de la fiche propriété (img + backgrounds CSS)41_IMG_RE = re.compile(42 r"https://assets\.rentsync\.com/[^\"'\\)\s]+\.(?:jpg|jpeg|png|webp)", re.I)43_SKIP_IMG = re.compile(r"logo|icon|favicon|badge|/thumb", re.I)444546def _city_key(city: str) -> str:47 key = strip_accents((city or "").strip().lower())48 return re.sub(r"\s+(?:qc|on)$", "", key) # le flux accole parfois la province495051class CenturionConnector(BaseConnector):52 source_id = "centurion"53 request_delay = 0.754 max_properties = 200 # safety cap (~104 properties Canada-wide)55 max_images = 205657 # -- feed parameters (page + main.js, with fallbacks) ----------------------58 def _feed_params(self) -> tuple[str, str]:59 """(client_id, auth_token) read from the site, constants as fallback."""60 client_id, token = DEFAULT_CLIENT_ID, DEFAULT_AUTH_TOKEN61 try:62 page = self.get(SEARCH_PAGE).text63 soup = BeautifulSoup(page, "html.parser")64 data = soup.find("div", class_="search-data")65 if data:66 client_id = (data.get("data-client-id") or client_id).strip()67 m = _MAINJS_RE.search(page)68 if m:69 js = self.get(BASE + m.group(1)).text70 mt = re.search(_TOKEN_RE.format(cid=re.escape(client_id)), js)71 if mt:72 token = mt.group(1)73 except Exception:74 pass # fallbacks: the observed constants75 return client_id, token7677 def fetch(self) -> list[Listing]:78 client_id, token = self._feed_params()79 params = {80 "client_id": client_id,81 "auth_token": token,82 # no city_ids: the feed returns the whole Centurion portfolio83 "show_all_properties": "true",84 "show_custom_fields": "true",85 "show_amenities": "true",86 "show_promotions": "true",87 "limit": "1000",88 }89 props = self.get(LIFT_API, params=params,90 headers={"Accept": "application/json",91 "Referer": BASE + "/"}).json()9293 listings: list[Listing] = []94 count = 095 for p in props:96 try:97 addr = p.get("address") or {}98 prov = (addr.get("province_code") or "").upper()99 if not prov or prov == "QC":100 continue # Québec is Rent-Ka's territory101 if count >= self.max_properties:102 break103 count += 1104 listings.append(self._listing(p, province=prov))105 except Exception:106 continue107 return listings108109 # -- one listing per property ----------------------------------------------110 def _listing(self, p: dict, province: str = "ON") -> Listing:111 pid = str(p.get("id"))112 addr = p.get("address") or {}113 url = p.get("permalink") or SEARCH_PAGE114 name = (p.get("name") or "").strip()115116 # full address: street + city + postal code (all provided by the feed);117 # strip the province suffix the feed sometimes appends («Barrie ON»)118 city = re.sub(rf"\s+{province}$", "",119 (addr.get("city") or "").strip(), flags=re.I)120 street = (addr.get("address") or "").strip()121 postal = (addr.get("postal_code") or "").strip()122 full_addr = ", ".join(x for x in (street, city) if x)123 if postal:124 full_addr += f", {province} {postal}"125 elif full_addr:126 full_addr += f", {province}"127 sector = (addr.get("neighbourhood") or "").strip()128129 # coordonnées GPS structurées du flux130 geo = p.get("geocode") or {}131 try:132 lat = float(geo["latitude"]) if geo.get("latitude") else None133 lng = float(geo["longitude"]) if geo.get("longitude") else None134 except (TypeError, ValueError):135 lat = lng = None136137 # sommaire des unités disponibles (rempli seulement s'il y a vacance)138 stats = ((p.get("statistics") or {}).get("suites") or {})139 rates = stats.get("rates") or {}140 beds = stats.get("bedrooms") or {}141 baths = stats.get("bathrooms") or {}142 sqft = stats.get("square_feet") or {}143 price = float(rates["min"]) if rates.get("min") else None144 price_label = ""145 if price is not None:146 price_label = (f"À partir de {price:.0f} $"147 if rates.get("max") and rates["max"] != rates["min"]148 else f"{price:.0f} $ /mois")149 # type d'unité : seulement si la gamme est sans ambiguïté150 unit_type = ""151 if beds.get("min") is not None and beds.get("min") == beds.get("max"):152 n = int(beds["min"])153 unit_type = "Studio" if n == 0 else normalize_unit_type(154 f"{n} chambres")155 # superficie : le flux publie parfois « 0.0 » (Gatineau) — ignorer156 area = None157 try:158 v = float(sqft.get("min") or 0)159 if 80 <= v <= 20000:160 area = v161 except (TypeError, ValueError):162 pass163164 # disponibilité : libellé du flux (« No Vacancy », « X Vacancies »…)165 availability = (p.get("availability_status_label") or "").strip()166 avail_date = None167 mad = str(p.get("min_availability_date") or "").strip()168 if re.fullmatch(r"20\d{2}-\d{2}-\d{2}", mad[:10]):169 avail_date = mad[:10]170171 # description : aperçu HTML du flux (rendu texte)172 details_src = p.get("details") or {}173 desc = BeautifulSoup(details_src.get("overview") or "",174 "html.parser").get_text(" ", strip=True)175 promo = p.get("promotion") or {}176 promo_txt = (promo.get("title") or promo.get("name") or "").strip() \177 if isinstance(promo, dict) else ""178 if promo_txt:179 desc = f"Promotion : {promo_txt}. {desc}".strip()180181 # commodités : liste du flux + champ personnalisé Rentsync (CSV)182 amenities: list[str] = []183 for a in p.get("amenities") or []:184 t = (a.get("name") if isinstance(a, dict) else str(a) or "").strip()185 if t and t not in amenities:186 amenities.append(t)187 cf = p.get("custom_fields") or {}188 for t in (cf.get("amenities") or "").split(","):189 t = t.strip()190 if t and t not in amenities:191 amenities.append(t)192193 # champs structurés du flux194 details: dict = {}195 contact = p.get("contact") or {}196 if contact.get("phone"):197 details["contact"] = {"phone": contact["phone"]}198 if contact.get("email"):199 details.setdefault("contact", {})["email"] = contact["email"]200 # pet_friendly=false ne distingue pas « interdit » de « non renseigné »201 pets = "oui" if p.get("pet_friendly") is True else None202203 # galerie photo de la fiche propriété — via le cache BD : revisitée204 # seulement quand la ligne du flux change205 feed_key = hashlib.sha1("|".join(str(x) for x in (206 p.get("availability_count"), p.get("availability_status"),207 rates.get("min"), rates.get("max"), mad, p.get("photo"),208 )).encode("utf-8")).hexdigest()209 d = self.detail(pid, feed_key, lambda: self._fetch_gallery(url))210 images = list(d.get("images") or [])211 photo = (p.get("photo_path") or "").strip()212 if photo and photo not in images:213 images.insert(0, photo)214215 return Listing(216 source=self.source_id,217 external_id=pid,218 url=url,219 title=name,220 address=full_addr,221 sector=sector,222 city=city or "Toronto",223 province=province,224 unit_type=unit_type,225 price=price,226 price_label=price_label,227 availability=availability,228 availability_date=avail_date,229 area_sqft=area,230 pets=pets,231 description=desc[:600] + (232 f" Salles de bain : {baths['min']:g}+."233 if baths.get("min") else ""),234 amenities=amenities[:25],235 details=details,236 images=images[: self.max_images],237 lat=lat,238 lng=lng,239 )240241 def _fetch_gallery(self, url: str) -> dict:242 """Scrape la galerie photo (assets.rentsync.com) de la fiche propriété."""243 out: dict = {"images": []}244 try:245 page = self.get(url).text246 except Exception:247 return out248 images: list[str] = []249 for u in _IMG_RE.findall(page):250 if _SKIP_IMG.search(u):251 continue252 # variante pleine résolution de la galerie (…/gallery/full/…)253 u = re.sub(r"/gallery/\d{3,4}/", "/gallery/full/", u)254 if u not in images:255 images.append(u)256 out["images"] = images[: self.max_images]257 return out258