Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/realstar.py : Realstar (realstar.ca)5# Cloudflare-protected (403 for robots) client-rendered site (RentCafe/6# Yardi engine): everything goes through Scrapfly rendering with a wait.7# 1) /searchlisting?province=<name> -> property cards (name, address,8# beds/baths/sqft, price range, phone, thumbnail) — one render per9# province, looped over Realstar's markets outside Québec (Ontario,10# Alberta, British Columbia, Nova Scotia, Newfoundland and Labrador);11# 2) each property page -> photo gallery, description, highlights;12# 3) /floorplans page -> structured plans (type, beds, sqft, price,13# number of available units) — real availability and prices.14# One listing per property (stable uids). Detail pages go through15# self.detail(...) (DB cache): the render is only re-done when the list16# card changed.17# -----------------------------------------------------------------------------18from __future__ import annotations1920import hashlib21import os22import re2324from bs4 import BeautifulSoup2526from ..schema import Listing, parse_price27from .base import BaseConnector2829# Realstar markets outside Québec: (search province name, URL path code,30# province code). One search render each.31_PROVINCES = [32 ("Ontario", "on", "ON"),33 ("Alberta", "ab", "AB"),34 ("British Columbia", "bc", "BC"),35 ("Nova Scotia", "ns", "NS"),36 ("Newfoundland and Labrador", "nl", "NL"),37]38SEARCH_URL = "https://www.realstar.ca/searchlisting?province={name}"3940# City slug in /apartments/<prov>/<city>/<slug> -> (display city, sector).41# Former Toronto boroughs fold into Toronto; unknown slugs pass through as42# Title Case (whole-province coverage).43_CITY_NORM = {44 "north-york": ("Toronto", "North York"),45 "etobicoke": ("Toronto", "Etobicoke"),46 "scarborough": ("Toronto", "Scarborough"),47 "east-york": ("Toronto", "East York"),48}4950_BED_TYPES = {"0": "Studio", "1": "1 bedroom", "2": "2 bedrooms",51 "3": "3 bedrooms", "4": "4 bedrooms"}52_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder", re.I)53_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?")545556class _BudgetReached(Exception):57 """Plafond de rendus atteint pour cette synchronisation."""585960class RealstarConnector(BaseConnector):61 source_id = "realstar"62 request_delay = 1.063 max_properties = 60 # global safety cap (2 renders per NEW property)64 max_images = 2565 max_renders = 60 # render cap per sync (cache hits are free)6667 # -- JS render with wait (SPA + Cloudflare) — Scrapfly ---------------------68 # (migrated from Firecrawl 2026-08-27; the short retry on 5xx/empty lives69 # in BaseConnector.get_rendered, which the fixtures monkeypatch)70 def _rendered(self, url: str, wait_ms: int = 9000) -> str:71 return self.get_rendered(url, wait_ms)7273 def fetch(self) -> list[Listing]:74 self._renders = 075 listings: list[Listing] = []76 seen: set[str] = set()77 count = 078 for prov_name, code, prov in _PROVINCES:79 try:80 html = self._rendered(81 SEARCH_URL.format(name=prov_name.replace(" ", "%20")),82 12000)83 soup = BeautifulSoup(html, "html.parser")84 # [class*=…]: the Scrapfly render captures the DOM before the85 # RentCafe JS reveals the cards (property-box-hidden) — the86 # hidden cards are complete87 cards = soup.select('li[class*="property-box"]')88 if not cards: # incomplete render: one more chance89 html = self._rendered(90 SEARCH_URL.format(name=prov_name.replace(" ", "%20")),91 15000)92 soup = BeautifulSoup(html, "html.parser")93 cards = soup.select('li[class*="property-box"]')94 except Exception:95 continue96 for card in cards:97 try:98 a = card.select_one(f"a[href*='/apartments/{code}/']")99 if not a:100 continue # another province's card101 url = (a.get("href") or "").split("?")[0]102 url = url.replace("http://", "https://")103 m = re.search(104 rf"/apartments/{code}/([a-z0-9\-.]+)/([a-z0-9\-]+)",105 url)106 if not m or url in seen:107 continue108 seen.add(url)109 city_slug, slug = m.group(1), m.group(2)110 if count >= self.max_properties:111 break112 count += 1113 listings.append(self._property_listing(114 card, url, city_slug, slug, province=prov))115 except Exception:116 continue117 return listings118119 def _property_listing(self, card, url: str, city_slug: str,120 slug: str, province: str = "ON") -> Listing:121 city, sector = _CITY_NORM.get(122 city_slug, (city_slug.replace("-", " ").title(), ""))123 name = ""124 fav = card.select_one("[data-property]")125 if fav:126 name = (fav.get("data-property") or "").strip()127 if not name:128 h = card.select_one(".property-name a")129 if h:130 name = h.get_text(" ", strip=True)131 name = re.sub(r"\s*opens in a new tab\s*", "", name).strip()132 name = name or slug.replace("-", " ").title()133134 addr_el = card.select_one(".card-prop-address")135 address = addr_el.get_text(" ", strip=True) if addr_el else ""136 if address and not re.search(rf",?\s+{province}\b", address):137 address = f"{address}, {province}"138139 meta = card.select_one(".card-bed-bath-rent")140 beds = baths = sqft = ""141 if meta:142 items = [li.get_text(" ", strip=True)143 for li in meta.select("li")]144 for it in items:145 if "Bed" in it:146 beds = it147 elif "Bath" in it:148 baths = it149 elif "Sq" in it:150 sqft = it151 unit_type = ""152 bm = re.match(r"^(\d)(?:\s|-)?.*Bed", beds or "")153 if bm and "-" not in beds.split("Bed")[0]:154 unit_type = _BED_TYPES.get(bm.group(1), "")155156 # Fourchette de prix « $1,645.00 - $2,630.00 »157 card_text = card.get_text(" ", strip=True)158 price = None159 price_label = ""160 pm = re.search(r"\$[\d,]+(?:\.\d{2})?(?:(?:\s*(?:-|to))+\s*"161 r"\$[\d,]+(?:\.\d{2})?)?", card_text)162 if pm:163 price_label = re.sub(r"\s*to\s*-\s*", " - ", pm.group(0))164 first = price_label.split("-")[0].replace("$", "").replace(165 ",", "").replace("to", "").strip()166 try:167 price = float(first)168 except ValueError:169 price = parse_price(price_label)170 if "-" in price_label:171 price_label = "From " + price_label172173 # Téléphone du bureau de location (lien tel: structuré de la carte)174 phone = ""175 tel = card.select_one("a[href^='tel:']")176 if tel:177 tm = re.search(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})",178 tel.get("href", ""))179 if tm:180 phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"181182 # Vignette de la carte183 images: list[str] = []184 img = card.select_one("img[src*='rentcafe']")185 if img and img.get("src"):186 images.append(img["src"])187188 # Pages détail (fiche + plans) via cache BD : rendu seulement si la189 # carte liste a changé (prix/dispo inclus dans le hash).190 key = hashlib.sha1(191 f"{name}|{address}|{beds}|{baths}|{sqft}|{price_label}"192 .encode("utf-8")).hexdigest()193 # uid: province-prefixed to avoid cross-province slug collisions.194 # Ontario keeps the historical "on-" prefix (uids in the seeded DB).195 ext = f"{province.lower()}-{slug}"196 try:197 payload = self.detail(ext, key, lambda: self._fetch_detail(url))198 except _BudgetReached:199 payload = {}200 except Exception:201 payload = {}202203 desc = payload.get("description", "")204 amenities = list(payload.get("amenities") or [])205 for im in (payload.get("images") or []):206 if im not in images:207 images.append(im)208209 # Plans structurés -> disponibilité, prix « à partir de », superficie210 availability = ""211 area_sqft = None212 plans = payload.get("floorplans") or []213 avail_plans = [p for p in plans if p.get("available", 0) > 0]214 if plans:215 total = sum(p.get("available", 0) for p in avail_plans)216 if total > 0:217 availability = (f"{total} unit(s) available — "218 + ", ".join(p["name"] for p in avail_plans[:6]))219 prices = [p["price"] for p in avail_plans220 if p.get("price") and 100 <= p["price"] <= 20000]221 if prices:222 price = min(prices)223 price_label = (f"From ${price:,.0f}/month"224 if len(avail_plans) > 1 or len(prices) > 1225 else f"${price:,.0f}/month")226 if len(avail_plans) == 1 and avail_plans[0].get("sqft"):227 # une seule unité type disponible : sa superficie est fiable228 area_sqft = avail_plans[0]["sqft"]229 if avail_plans[0].get("unit_type"):230 unit_type = avail_plans[0]["unit_type"]231232 # Summary of available plans in the description (faithful text)233 plan_bits = []234 for p in avail_plans[:8]:235 seg = p["name"]236 if p.get("sqft"):237 seg += f" ({p['sqft']:.0f} sq ft)"238 if p.get("price"):239 seg += f": ${p['price']:,.0f}/month"240 plan_bits.append(seg)241242 details: dict = {}243 if phone:244 details["contact"] = {"phone": phone}245246 bits = [b for b in [beds, baths, sqft] if b]247 desc_parts = ([desc] if desc else []) + bits248 if plan_bits:249 desc_parts.append("Available: " + "; ".join(plan_bits))250 return Listing(251 source=self.source_id,252 external_id=ext,253 url=url,254 title=name,255 address=address,256 sector=sector,257 city=city,258 province=province,259 unit_type=unit_type,260 price=price,261 price_label=price_label,262 availability=availability,263 area_sqft=area_sqft,264 description=" — ".join(desc_parts)[:900],265 amenities=amenities,266 details=details,267 images=images[: self.max_images],268 )269270 # -- pages détail (fiche propriété + plans) --------------------------------271 def _fetch_detail(self, url: str) -> dict:272 """2 rendus Scrapfly : fiche (photos, description, points forts) et273 /floorplans (plans structurés). Appelé seulement hors cache."""274 if self._renders + 2 > self.max_renders:275 raise _BudgetReached()276 self._renders += 2277278 payload: dict = {"description": "", "amenities": [], "images": [],279 "floorplans": []}280 try:281 ph = self._rendered(url, 8000)282 psoup = BeautifulSoup(ph, "html.parser")283 for im in psoup.select("img[src*='resource.rentcafe.com']"):284 src = im.get("src", "")285 if src and not _SKIP_IMG.search(src) \286 and src not in payload["images"]:287 payload["images"].append(src)288 # description : premiers paragraphes substantiels289 paras = [p.get_text(" ", strip=True)290 for p in psoup.find_all("p")]291 paras = [p for p in paras if len(p) > 80]292 if paras:293 payload["description"] = " ".join(paras[:2])[:600]294 # points forts de la propriété (courtes mentions après le titre)295 text = psoup.get_text("\n", strip=True)296 hm = re.search(r"Points forts de la propri[ée]t[ée]\n(.*?)\n"297 r"(?:Photos|Emplacement|Votre)", text, re.S)298 if not hm: # gabarit anglais (fiches Ontario)299 hm = re.search(r"Property Highlights\n(.*?)\n"300 r"(?:Photos|Location|Your)", text, re.S)301 if hm:302 amenities = []303 for t in hm.group(1).split("\n"):304 t = t.strip()305 if 2 < len(t) < 50 and t not in amenities:306 amenities.append(t)307 payload["amenities"] = amenities[:15]308 except Exception:309 pass310311 try:312 fh = self._rendered(url.rstrip("/") + "/floorplans", 10000)313 payload["floorplans"] = self._parse_floorplans(fh)314 except Exception:315 pass316 return payload317318 @staticmethod319 def _parse_floorplans(html: str) -> list[dict]:320 """Cartes de plans RentCafe : nom (« 4 ½ D »), chambres, pi², prix,321 nombre d'unités disponibles (structuré : .fp-availability)."""322 soup = BeautifulSoup(html, "html.parser")323 plans: list[dict] = []324 for cont in soup.select("div[id^='fp-container-']"):325 try:326 name_el = cont.select_one("span[data-selenium-id$='Name']")327 name = name_el.get_text(" ", strip=True) if name_el else ""328 if not name:329 continue330 avail = 0331 av_el = cont.select_one(".fp-availability")332 if av_el:333 am = re.search(r"(\d+)", av_el.get_text(" ", strip=True))334 if am:335 avail = int(am.group(1))336 sqft = None337 sq_el = cont.select_one("span[data-selenium-id$='SqFt']")338 if sq_el:339 # « Pi. Ca. » (gabarit FR) ou « Sq. Ft. » (gabarit EN/ON)340 sm = re.search(r"([\d,]{2,})\s*(?:Pi|Sq)",341 sq_el.get_text(" ", strip=True), re.I)342 if sm:343 v = float(sm.group(1).replace(",", ""))344 if 80 <= v <= 20000:345 sqft = v346 price = None347 pm = _PRICE_RE.search(cont.get_text(" ", strip=True))348 if pm:349 v = float(pm.group(0).replace("$", "").replace(",", ""))350 if 100 <= v <= 20000:351 price = v352 unit_type = ""353 um = re.match(r"^\s*(\d)\s*½", name)354 if um:355 # French template names («4 ½ D»): n½ -> n-2 bedrooms356 n = max(int(um.group(1)) - 2, 0)357 unit_type = _BED_TYPES.get(str(min(n, 4)), "")358 elif re.match(r"(?i)^\s*(?:studio|bachelor)", name):359 unit_type = "Studio"360 else:361 # English templates: «2 Bed 1 Bath A», «One Bedroom»…362 wm = re.match(r"(?i)^\s*(\d|one|two|three|four)\s*bed",363 name)364 if wm:365 n = {"one": 1, "two": 2, "three": 3, "four": 4}.get(366 wm.group(1).lower()) or int(wm.group(1))367 unit_type = _BED_TYPES.get(str(min(n, 4)), "")368 plans.append({"name": name, "available": avail,369 "sqft": sqft, "price": price,370 "unit_type": unit_type})371 except Exception:372 continue373 return plans374