Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/capreit.py : CAPREIT (capreit.ca)5# Official search-engine JSON feed (admin-ajax `property_json`) — a single6# request returns every Canadian property (~278, all provinces); filtering7# is purely client-side. Rent-Ka keeps EVERY province except QC (BC, AB,8# SK, MB, ON, NB, NS, PE, NL…). Server-rendered property pages provide9# unit types, prices, availability, amenities and the photo gallery10# (French /fr/ URLs — labels are normalized centrally by finalize()).11# One listing per available unit type.12# -----------------------------------------------------------------------------13from __future__ import annotations1415import hashlib16import os17import re1819from bs4 import BeautifulSoup2021from ..schema import (Listing, normalize_unit_type,22 parse_area_sqft, parse_availability_date, parse_price,23 strip_accents)24from .base import BaseConnector2526BASE = "https://www.capreit.ca"27FEED_URL = f"{BASE}/wp-admin/admin-ajax.php?action=property_json&language=fr"2829# Ontario display-name mapping: normalized feed key -> display name. The30# CAPREIT feed distinguishes Toronto's former boroughs (Scarborough, North31# York…): kept as cities, consistent with local usage. Cities absent from32# this map are NOT rejected — the feed city is used as-is (whole-province33# coverage); the map only normalizes names and regroups Ottawa's sectors.34_ON_CITIES = {35 # Toronto et arrondissements (ancienne métropole)36 "toronto": "Toronto",37 "scarborough": "Scarborough",38 "north york": "North York",39 "etobicoke": "Etobicoke",40 "york": "York",41 "east york": "East York",42 # GTA — York Region43 "thornhill": "Thornhill",44 "vaughan": "Vaughan",45 "markham": "Markham",46 "richmond hill": "Richmond Hill",47 # GTA — Peel / Halton48 "mississauga": "Mississauga",49 "brampton": "Brampton",50 "oakville": "Oakville",51 "burlington": "Burlington",52 "milton": "Milton",53 # GTA — Durham54 "pickering": "Pickering",55 "ajax": "Ajax",56 "whitby": "Whitby",57 "oshawa": "Oshawa",58 # Ottawa (Orléans/Nepean/Kanata/Gloucester = secteurs d'Ottawa)59 "ottawa": "Ottawa",60 "orleans": "Ottawa",61 "nepean": "Ottawa",62 "kanata": "Ottawa",63 "gloucester": "Ottawa",64 # London65 "london": "London",66 # Hamilton / Kitchener-Waterloo67 "hamilton": "Hamilton",68 "kitchener": "Kitchener",69 "waterloo": "Waterloo",70 "cambridge": "Cambridge",71}72_IMG_RE = re.compile(73 r'https://www\.capreit\.ca/wp-content/uploads/[^"\'\s\\]+'74 r'\.(?:jpg|jpeg|png|webp)', re.I)75_SKIP_IMG = re.compile(76 r"logo|icon|favicon|cropped|-\d{2,4}x\d{2,4}\.|BIL|Phone|badge", re.I)777879class CapreitConnector(BaseConnector):80 source_id = "capreit"81 request_delay = 0.682 max_properties = 260 # safety cap (whole Canada outside QC)83 max_images = 258485 @staticmethod86 def _city_key(city: str) -> str:87 return strip_accents((city or "").strip().lower())8889 def fetch(self) -> list[Listing]:90 props = self.get(FEED_URL).json()9192 listings: list[Listing] = []93 count = 094 for p in props:95 try:96 prov = (p.get("province") or "").strip().upper()97 if not prov or prov == "QC":98 continue # Québec is Rent-Ka's territory99 if not p.get("has_vacancies"):100 continue101 if count >= self.max_properties:102 break103 count += 1104 listings.extend(self._property_listings(p, province=prov))105 except Exception:106 continue107 return listings108109 def _property_listings(self, p: dict,110 province: str = "ON") -> list[Listing]:111 pid = str(p.get("id"))112 url = p.get("url") or ""113 title = (p.get("title") or "").strip()114 address = (p.get("address") or "").strip()115 feed_city = (p.get("city") or "").strip()116 # full address: street + city + province + postal code (from the feed)117 postal = (p.get("postal_code") or "").strip()118 if address and feed_city:119 address = f"{address}, {feed_city}" + (f", {province} {postal}"120 if postal else "")121 # coordonnées GPS du flux122 try:123 lat = float(p["latitude"]) if p.get("latitude") else None124 lng = float(p["longitude"]) if p.get("longitude") else None125 except (TypeError, ValueError):126 lat = lng = None127 incentive = (p.get("incentive") or "").strip()128 # city: feed city, normalized through the ON display map when it129 # regroups (e.g. Orléans -> Ottawa, old name becomes the sector);130 # sector defaults to the nearest intersection.131 city_key = self._city_key(feed_city)132 city = _ON_CITIES.get(city_key, feed_city) if province == "ON" \133 else feed_city134 if self._city_key(city) == city_key:135 sector = (p.get("nearest_intersection") or "").strip()136 else:137 sector = feed_city138139 # fiche propriété (rendu serveur) via le cache BD : revisitée140 # seulement quand la ligne du flux change141 feed_key = hashlib.sha1("|".join(142 str(p.get(k)) for k in143 ("id", "min_rent", "earliest_date", "vacancy_message",144 "price_range", "has_vacancies", "units_count", "incentive")145 ).encode("utf-8")).hexdigest()146 d = self.detail(pid, feed_key, lambda: self._fetch_property(url))147 desc = d.get("desc", "")148 amenities = d.get("amenities", [])149 images = d.get("images", [])150 rows = d.get("rows", [])151152 # promotion du flux (ex. « 1 mois de loyer gratuit »)153 if incentive:154 desc = f"Promotion : {incentive}. {desc}".strip()155156 # date de disponibilité structurée du flux (ex. 20260201)157 avail_date = None158 ed = str(p.get("earliest_date") or "")159 if re.fullmatch(r"20\d{6}", ed):160 avail_date = f"{ed[:4]}-{ed[4:6]}-{ed[6:]}"161162 out: list[Listing] = []163 if rows:164 for r in rows:165 ut = normalize_unit_type(r["unit_raw"])166 slug = re.sub(r"[^a-z0-9]+", "-",167 strip_accents(r["unit_raw"].lower())).strip("-")168 out.append(Listing(169 source=self.source_id,170 external_id=f"{pid}-{slug or 'u'}",171 url=url,172 title=f"{title} — {r['unit_raw']}" if r["unit_raw"]173 else title,174 address=address,175 sector=sector,176 city=city,177 province=province,178 unit_type=ut,179 price=parse_price(r["price"]),180 price_label=r["price"],181 availability=r["avail"],182 # date de la ligne (« Disponible 1 sept. ») sinon183 # earliest_date structuré du flux184 availability_date=(parse_availability_date(r["avail"])185 or avail_date),186 # superficie structurée de la ligne (ex. « 875 pi² »)187 area_sqft=parse_area_sqft(r["sqft"]),188 description=desc[:600],189 amenities=amenities,190 images=images,191 lat=lat,192 lng=lng,193 ))194 else:195 # repli : annonce par propriété avec le prix plancher du flux196 min_rent = p.get("min_rent")197 out.append(Listing(198 source=self.source_id,199 external_id=pid,200 url=url,201 title=title,202 address=address,203 sector=sector,204 city=city,205 province=province,206 unit_type=normalize_unit_type(207 (p.get("bedroom_range") or "").split("-")[0]),208 price=float(min_rent) if min_rent else None,209 price_label=p.get("price_range") or "",210 availability=p.get("vacancy_message") or "",211 availability_date=avail_date,212 description=desc,213 amenities=amenities,214 images=images,215 lat=lat,216 lng=lng,217 ))218 return out219220 def _fetch_property(self, url: str) -> dict:221 """Scrape la fiche propriété : galerie, commodités, description,222 et une ligne par type d'unité disponible (« Vos options »)."""223 out: dict = {"desc": "", "amenities": [], "images": [], "rows": []}224 try:225 page = self.get(url).text226 except Exception:227 return out228 soup = BeautifulSoup(page, "html.parser")229230 # galerie photos (héro + blocs JSON de la page)231 images: list[str] = []232 for u in _IMG_RE.findall(page):233 if _SKIP_IMG.search(u):234 continue235 if u not in images:236 images.append(u)237 out["images"] = images[: self.max_images]238239 # commodités (listes à icônes)240 amenities: list[str] = []241 seen = set()242 for li in soup.select("li"):243 if not li.find("div", class_="icon"):244 continue245 t = li.get_text(" ", strip=True)246 if t and len(t) < 60 and t not in seen:247 seen.add(t)248 amenities.append(t)249 out["amenities"] = amenities[:25]250251 # description (« Caractéristiques de l'immeuble »)252 h = soup.find(["h2", "h3"], string=re.compile(253 "Caractéristiques de l['’]immeuble"))254 if h:255 nxt = h.find_next(["p", "div"])256 if nxt:257 out["desc"] = nxt.get_text(" ", strip=True)[:600]258259 # types d'unités disponibles260 for li in soup.select("li.property-options-list-item"):261 avail_el = li.select_one(262 ".property-options-list-item-availability")263 price_el = li.select_one(264 ".property-options-list-item-price")265 details = [d.get_text(" ", strip=True)266 for d in li.select(".property-options-item")]267 unit_raw = details[0] if details else ""268 sqft = details[1] if len(details) > 1 else ""269 if li.get("data-available") == "false":270 continue271 out["rows"].append({272 "unit_raw": unit_raw,273 "sqft": sqft,274 "price": price_el.get_text(" ", strip=True)275 if price_el else "",276 "avail": avail_el.get_text(" ", strip=True)277 if avail_el else "",278 })279 return out280