Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/apartments_com.py : Apartments.com (CoStar) — US network with a5# real Canadian inventory (large managers, per-province search pages).6# The site is locked by Akamai (403 direct, via Scrapfly AND Oxylabs —7# probed 2026-08-18/22): scraping is delegated to the IN-HOUSE APIFY ACTOR8# gorgeous_thistle/ka-apartments-com (source: actors/ka-apartments-com in9# this repo) which goes through Bright Data's Web Unlocker —10# server-rendered HTML, no JS rendering.11# ONE listing = ONE PROPERTY (building): «from $» price (minimum of the12# unit plans), all plans in details["plans"]. Detail pages are read in13# English; 30-day detail cache + CADENCE: CoStar inventory moves slowly,14# the actor only runs every RENTKA_APTS_INTERVAL_H hours; between runs the15# active listings are re-emitted from the DB (zero cost).16# -----------------------------------------------------------------------------17from __future__ import annotations1819import json20import os21import re22import time2324import requests as _requests2526from ..schema import Listing27from .base import BaseConnector28from . import _detailutil as du2930APIFY_API = "https://api.apify.com/v2"31ACTOR = os.environ.get("RENTKA_APTS_ACTOR", "gorgeous_thistle~ka-apartments-com")3233# one search page per province outside Québec34SEARCH_URLS = [f"https://www.apartments.com/{p}/"35 for p in ("on", "bc", "ab", "sk", "mb", "ns", "nb", "pe", "nl")]36MAX_PAGES = int(os.environ.get("RENTKA_APTS_MAX_PAGES", "25"))37DETAIL_LIMIT = int(os.environ.get("RENTKA_APTS_DETAIL_LIMIT", "250"))38TTL_DAYS = float(os.environ.get("RENTKA_APTS_TTL_DAYS", "30"))39INTERVAL_H = float(os.environ.get("RENTKA_APTS_INTERVAL_H", "12"))40CONCURRENCY = int(os.environ.get("RENTKA_APTS_CONCURRENCY", "4"))41RUN_TIMEOUT = int(os.environ.get("RENTKA_APTS_RUN_TIMEOUT", "3600")) # s4243DETAIL_KEY = "v1"44PRICE_MIN, PRICE_MAX = 300, 12000 # garde-fous de loyer mensuel4546_MONEY_RX = re.compile(r"\$\s*([\d\s, ]+)")47_BEDS_LABEL_RX = re.compile(r"(\d+)\s*(?:bed|lit|c\.?\s*à\.?\s*c)", re.I)48_SQFT_RX = re.compile(r"([\d\s, ]+)\s*(?:pi²|sq\s*ft)", re.I)495051def _money(text: str | None) -> float | None:52 m = _MONEY_RX.search(text or "")53 if not m:54 return None55 try:56 return float(re.sub(r"[^\d]", "", m.group(1)))57 except ValueError:58 return None596061def _beds_count(label: str) -> float | None:62 low = (label or "").casefold()63 if "studio" in low or "bachelor" in low:64 return 0.065 m = _BEDS_LABEL_RX.search(low)66 return float(m.group(1)) if m else None676869def _unit_type(beds: float | None) -> str:70 if beds is None:71 return ""72 if beds <= 0:73 return "Studio"74 n = int(beds)75 if n >= 5:76 return "5+ bedrooms"77 return f"{n} bedroom" + ("s" if n > 1 else "")787980def _fmt_price(price: float) -> str:81 return f"${price:,.0f}/month"828384_PROV_RX = re.compile(r"\b(ON|BC|AB|SK|MB|NB|NS|PE|NL|YT|NT|NU|QC)\b")858687def _split_address(addr: str) -> tuple[str, str, str]:88 """«200 Bay St, Toronto, ON M5J 2J2» -> (street, city, province)."""89 parts = [p.strip() for p in (addr or "").split(",") if p.strip()]90 prov = ""91 if parts:92 m = _PROV_RX.search(parts[-1])93 if m:94 prov = m.group(1)95 if len(parts) >= 3:96 return ", ".join(parts[:-2]), parts[-2], prov97 return addr or "", "", prov9899100class ApartmentsComConnector(BaseConnector):101 source_id = "apartments_com"102 request_delay = 1.0103104 # -- orchestration de l'acteur Apify --------------------------------------105 def _run_actor(self, payload: dict, token: str) -> list[dict]:106 """Lance l'acteur, attend la fin, retourne les items du dataset."""107 r = _requests.post(108 f"{APIFY_API}/acts/{ACTOR}/runs?waitForFinish=120",109 json=payload, timeout=180,110 headers={"Authorization": f"Bearer {token}"})111 r.raise_for_status()112 run = r.json()["data"]113 deadline = time.time() + RUN_TIMEOUT114 while run["status"] in ("READY", "RUNNING") and time.time() < deadline:115 time.sleep(10)116 run = _requests.get(117 f"{APIFY_API}/actor-runs/{run['id']}", timeout=60,118 headers={"Authorization": f"Bearer {token}"}).json()["data"]119 if run["status"] != "SUCCEEDED":120 raise RuntimeError(f"acteur {ACTOR} : run {run['id']} "121 f"terminé en {run['status']}")122 items: list[dict] = []123 offset = 0124 while True:125 batch = _requests.get(126 f"{APIFY_API}/datasets/{run['defaultDatasetId']}/items"127 f"?limit=1000&offset={offset}", timeout=120,128 headers={"Authorization": f"Bearer {token}"}).json()129 items.extend(batch)130 if len(batch) < 1000:131 return items132 offset += 1000133134 # -- cadence / ré-émission -------------------------------------------------135 def _last_ok_sync(self, cache: du.TtlDetailCache) -> float:136 row = cache.con.execute(137 "SELECT MAX(ts) AS t FROM sync_log WHERE source=? AND ok=1"138 " AND found > 0", (self.source_id,)).fetchone()139 return row["t"] or 0.0140141 def _active_rows(self, cache: du.TtlDetailCache) -> list:142 return cache.con.execute(143 "SELECT external_id, url, title, address, sector, city, province,"144 " unit_type,"145 " bedrooms, bathrooms, price, price_label, availability,"146 " availability_date, area_sqft, pets, furnished, description,"147 " amenities, details, images, lat, lng"148 " FROM listings WHERE source=? AND active=1", (self.source_id,)149 ).fetchall()150151 def _listing_from_row(self, r) -> Listing:152 def js(s, default):153 try:154 return json.loads(s) if s else default155 except ValueError:156 return default157 return Listing(158 source=self.source_id, external_id=r["external_id"],159 url=r["url"], title=r["title"] or "", address=r["address"] or "",160 sector=r["sector"] or "", city=r["city"] or "",161 province=r["province"] or "ON",162 unit_type=r["unit_type"] or "", bedrooms=r["bedrooms"],163 bathrooms=r["bathrooms"], price=r["price"],164 price_label=r["price_label"] or "",165 availability=r["availability"] or "",166 availability_date=r["availability_date"],167 area_sqft=r["area_sqft"], pets=r["pets"],168 furnished=(None if r["furnished"] is None else bool(r["furnished"])),169 description=r["description"] or "",170 amenities=js(r["amenities"], []), details=js(r["details"], {}),171 images=js(r["images"], []), lat=r["lat"], lng=r["lng"],172 )173174 # -- payload détail -> champs Listing ---------------------------------------175 @staticmethod176 def _std_detail(d: dict) -> dict:177 std = {k: d[k] for k in ("description", "images", "lat", "lng",178 "amenities") if d.get(k)}179 if d.get("street"):180 std["address"] = d["street"]181 if d.get("city"):182 std["city"] = d["city"]183 plans, seen = [], set()184 for pl in d.get("plans") or []: # grille dupliquée desktop/mobile185 sig = (pl.get("key"), pl.get("name"), pl.get("rent"))186 if sig not in seen:187 seen.add(sig)188 plans.append(pl)189 rents = [p for p in (_money(pl.get("rent")) for pl in plans)190 if p is not None and PRICE_MIN <= p <= PRICE_MAX]191 if rents:192 std["price"] = min(rents)193 std["price_label"] = (f"From {_fmt_price(min(rents))}"194 if len(plans) > 1 else _fmt_price(min(rents)))195 # immeuble multi-plans : minimum, cohérent avec le prix « à partir196 # de » — et ça évite que finalize() devine n'importe quoi dans la197 # description marketing (« appartements de 1 à 4 chambres »)198 beds = {pl.get("beds") for pl in plans if pl.get("beds") is not None}199 baths = {pl.get("baths") for pl in plans if pl.get("baths") is not None}200 if beds:201 std["bedrooms"] = min(beds)202 if len(beds) == 1:203 std["unit_type"] = _unit_type(std["bedrooms"])204 if baths:205 std["bathrooms"] = min(baths)206 if len(plans) == 1:207 m = _SQFT_RX.search(" ".join(plans[0].get("details") or []))208 if m:209 try:210 std["area_sqft"] = float(re.sub(r"[^\d]", "", m.group(1)))211 except ValueError:212 pass213 avails = [pl.get("availability") or "" for pl in plans]214 now = next((a for a in avails215 if "maintenant" in a.casefold() or "now" in a.casefold()),216 "")217 if now or any(avails):218 std["availability"] = now or next(a for a in avails if a)219 if plans:220 std["details"] = {"plans": [221 {k: pl[k] for k in ("name", "rent", "beds", "baths",222 "details", "availability") if k in pl}223 for pl in plans]}224 return std225226 # -- pipeline principal ------------------------------------------------------227 def fetch(self) -> list[Listing]:228 token = os.environ.get("APIFY_TOKEN")229 bd_token = os.environ.get("BRIGHTDATA_API_KEY")230 if not token:231 raise RuntimeError("APIFY_TOKEN manquant (voir .env)")232 if not bd_token:233 raise RuntimeError("BRIGHTDATA_API_KEY manquant (voir .env)")234235 cache = du.TtlDetailCache(self, budget=0, ttl_days=TTL_DAYS,236 key=DETAIL_KEY, fetch_html=lambda _u: "")237 try:238 # inventaire lent + acteur payant : entre deux vrais runs on239 # ré-émet les actives telles quelles (aucune requête réseau)240 age_h = (time.time() - self._last_ok_sync(cache)) / 3600241 actives = self._active_rows(cache)242 if actives and age_h < INTERVAL_H:243 return [self._listing_from_row(r) for r in actives]244245 fresh = {r["external_id"] for r in cache.con.execute(246 "SELECT external_id FROM detail_cache"247 " WHERE source=? AND key=? AND fetched_at > ?",248 (self.source_id, DETAIL_KEY,249 time.time() - TTL_DAYS * 86400)).fetchall()}250 # rattrapage : actives jamais enrichies (sans GPS ou description)251 extra = [f'{r["external_id"]}|{r["url"]}' for r in actives252 if r["external_id"] not in fresh253 and (r["lat"] is None or not (r["description"] or "").strip())]254255 items = self._run_actor({256 "searchUrls": SEARCH_URLS,257 "maxPages": MAX_PAGES,258 "getDetails": True,259 "maxDetails": DETAIL_LIMIT,260 "skipDetailIds": sorted(fresh),261 "extraDetailIds": extra[:DETAIL_LIMIT],262 "brightdataToken": bd_token,263 "brightdataZone": os.environ.get("BRIGHTDATA_ZONE",264 "web_unlocker1"),265 "concurrency": CONCURRENCY,266 }, token)267268 for it in items:269 if it.get("kind") == "detail" and it.get("id"):270 payload = {k: v for k, v in it.items()271 if k not in ("kind",) and v is not None}272 cache.put(str(it["id"]), payload)273274 out: dict[str, Listing] = {}275 for it in items:276 if it.get("kind") != "listing":277 continue278 pid = str(it["id"])279 street, city, prov = _split_address(it.get("address") or "")280 if prov == "QC":281 continue # Québec is Rent-Ka's territory282 rents = [(r.get("beds") or "", _money(r.get("price")))283 for r in it.get("rents") or []]284 prices = [p for _b, p in rents285 if p is not None and PRICE_MIN <= p <= PRICE_MAX]286 from_price = any("+" in (r.get("price") or "")287 for r in it.get("rents") or [])288 beds = {b for b in (_beds_count(lbl) for lbl, _p in rents)289 if b is not None}290 lst = Listing(291 source=self.source_id,292 external_id=pid,293 url=it.get("url") or "",294 title=(it.get("title") or street)[:200],295 address=street,296 city=city,297 province=prov or "ON",298 price=min(prices) if prices else None,299 price_label=((f"From {_fmt_price(min(prices))}"300 if from_price or len(prices) > 1301 else _fmt_price(min(prices)))302 if prices else ""),303 unit_type=_unit_type(beds.pop() if len(beds) == 1304 else None),305 amenities=it.get("amenities") or [],306 images=[it["image"]] if it.get("image") else [],307 )308 detail, _f = cache.peek(pid)309 if detail:310 du.apply_detail(lst, self._std_detail(detail))311 if detail.get("name"):312 lst.title = detail["name"][:200]313 if lst.price is None or not (PRICE_MIN <= lst.price314 <= PRICE_MAX):315 continue # « Call for Rent » : pas affichable316 out[pid] = lst317 return list(out.values())318 finally:319 cache.close()320