Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/metcap.py : MetCap Living (metcap.com)5# Server-rendered WordPress site. Every page carries the full city menu:6# «province-search-results?province=<id>&city=<name>» links. Rent-Ka scans7# every province except Québec (ids verified live 2026-08-27: 113=ON8# 20 cities, 117=BC 8, 119=NS 7, 121=NB 1, 213=AB 4). Result pages list9# buildings (lat/lng in the onclick=centerMap attribute) and their unit10# types («Toronto 2 Bedrooms from $1,819»). /apartment/... pages give the11# structured detail: «Suite Details» table (status, beds, baths, sqft),12# «Building Amenities», «Rent Includes», «Pet Friendly» lists, leasing13# office contact, description and unit photos; /property/... pages give14# the building photo gallery. Detail pages go through self.detail() (DB15# cache).16# -----------------------------------------------------------------------------17from __future__ import annotations1819import hashlib20import os21import re22import urllib.parse2324from bs4 import BeautifulSoup2526from ..schema import Listing, parse_price, strip_accents27from .base import BaseConnector2829BASE = "https://www.metcap.com"30MENU_URL = f"{BASE}/province/ontario?lang=en" # any page carries the menu3132# MetCap province ids -> province code (Québec 115 excluded)33_PROVINCE_IDS = {"113": "ON", "117": "BC", "119": "NS", "121": "NB",34 "213": "AB"}3536# Former Toronto boroughs -> (display city, sector); other cities pass through37_CITY_NORM = {38 "north york": ("Toronto", "North York"),39 "scarborough": ("Toronto", "Scarborough"),40 "etobicoke": ("Toronto", "Etobicoke"),41 "east york": ("Toronto", "East York"),42}4344_TYPE_MAP = [45 (re.compile(r"bachelor|studio", re.I), "Studio"),46 (re.compile(r"1\s*bed", re.I), "1 bedroom"),47 (re.compile(r"2\s*bed", re.I), "2 bedrooms"),48 (re.compile(r"3\s*bed", re.I), "3 bedrooms"),49 (re.compile(r"4\s*bed", re.I), "4 bedrooms"),50]51_SKIP_IMG = re.compile(r"logo|icon|favicon|header|/map/|walk\.sc|sharethis",52 re.I)53_LATLNG_RE = re.compile(r"\{\s*lat:\s*(-?[\d.]+)\s*,\s*"54 r"lon:\s*(-?[\d.]+)\s*\}")55_PHONE_RE = re.compile(r"\(?\b([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-](\d{4})\b")56_EMAIL_RE = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.]+\b")5758# «Rent Includes» (structured English text) -> canonical inclusion keys59_INCLUDES_MAP = [60 (re.compile(r"heat", re.I), "heating"),61 (re.compile(r"hydro|electric", re.I), "electricity"),62 (re.compile(r"hot\s*water", re.I), "hot_water"),63 (re.compile(r"internet|wi-?fi", re.I), "internet"),64 (re.compile(r"cable", re.I), "cable"),65]666768class _CapAtteint(Exception):69 """Plafond de requêtes détail atteint pour cette synchronisation."""707172class MetcapConnector(BaseConnector):73 source_id = "metcap"74 request_delay = 0.675 max_units = 400 # safety cap on unit pages (all provinces)76 max_images = 2577 max_real_details = 200 # real detail requests per sync (cache hits free)7879 def fetch(self) -> list[Listing]:80 self._real_details = 081 self._unit_count = 082 listings: list[Listing] = []83 # One page carries the whole city menu: collect (province_id, city)84 html = self.get(MENU_URL).text85 targets: list[tuple[str, str, str]] = [] # (province_id, city, prov)86 seen = set()87 for href in re.findall(r'href="(/province-search-results\?[^"]+)"',88 html):89 q = urllib.parse.parse_qs(urllib.parse.urlparse(90 href.replace("&", "&")).query)91 pid = (q.get("province") or [""])[0]92 prov = _PROVINCE_IDS.get(pid)93 if not prov:94 continue # Québec (115) and unknown ids are skipped95 city = (q.get("city") or [""])[0]96 if city and (pid, city) not in seen:97 seen.add((pid, city))98 targets.append((pid, city, prov))99100 for pid, city_name, prov in targets:101 try:102 self._scan_city(pid, city_name, prov, listings)103 except Exception:104 continue105 return listings106107 def _scan_city(self, province_id: str, city_name: str, province: str,108 listings: list[Listing]) -> None:109 """Scan one province-search-results page (buildings + unit links)."""110 key = strip_accents(city_name.lower()).replace(".", "").strip()111 city, sector = _CITY_NORM.get(key, (city_name, ""))112 page = self.get(113 f"{BASE}/province-search-results?lang=en"114 f"&province={province_id}"115 f"&city={urllib.parse.quote(city_name)}").text116 soup = BeautifulSoup(page, "html.parser")117 for item in soup.select(".province-results__item"):118 try:119 block = item.select_one(".province-results__content")120 if not block:121 continue122 h2a = block.select_one("h2 a[href^='/property/']")123 if not h2a:124 continue125 address = h2a.get_text(" ", strip=True)126 prop_path = h2a.get("href", "").split("?")[0]127 # building lat/lng: onclick="centerMap(..., {lat, lon})"128 lat = lng = None129 lm = _LATLNG_RE.search(item.get("onclick", "") or "")130 if lm:131 lat, lng = float(lm.group(1)), float(lm.group(2))132 spans = block.select("p span.d-block")133 prop_name = ""134 if spans and not spans[0].find("a"):135 prop_name = spans[0].get_text(" ", strip=True)136 for a in block.select("a[href^='/apartment/']"):137 if self._unit_count >= self.max_units:138 break139 self._unit_count += 1140 text = a.get_text(" ", strip=True)141 lst = self._unit_listing(142 a.get("href", ""), text, address, prop_name,143 prop_path, city, sector, lat, lng,144 province=province)145 if lst:146 listings.append(lst)147 except Exception:148 continue149150 # -- pages détail (via cache BD self.detail) -------------------------------151 def _gallery(self, prop_path: str) -> list[str]:152 """Galerie photo de la fiche immeuble (partagée entre unités)."""153 def _fetch() -> dict:154 if self._real_details >= self.max_real_details:155 raise _CapAtteint()156 self._real_details += 1157 imgs: list[str] = []158 ph = self.get(f"{BASE}{prop_path}?lang=en").text159 for u in re.findall(160 r'https://www\.metcap\.com/wp-content/uploads/'161 r'[^"\'\s\)]+\.(?:jpg|jpeg|png|webp)', ph):162 if not _SKIP_IMG.search(u) and u not in imgs:163 imgs.append(u)164 return {"images": imgs[: self.max_images]}165166 try:167 return self.detail(f"property:{prop_path}", prop_path,168 _fetch).get("images") or []169 except Exception:170 return []171172 def _unit_detail(self, slug: str, url: str, card_key: str) -> dict:173 """Fiche unité : tableau Suite Details, listes sidebar, contact,174 description, intersection et photos d'unité."""175 def _fetch() -> dict:176 if self._real_details >= self.max_real_details:177 raise _CapAtteint()178 self._real_details += 1179 html = self.get(url).text180 soup = BeautifulSoup(html, "html.parser")181 out: dict = {}182183 # Tableau « Suite Details » : Price/Status/Beds/Baths/Sq. Ft184 suites = []185 table = soup.select_one("table.table-listing")186 if table:187 for tr in table.select("tbody tr"):188 row = {td.get("data-title", "").strip():189 td.get_text(" ", strip=True)190 for td in tr.select("td") if td.get("data-title")}191 if row:192 suites.append(row)193 out["suites"] = suites194195 # Listes structurées de la barre latérale196 def _ul(titre: str) -> list[str]:197 h = soup.find("h2", string=re.compile(198 rf"^\s*{titre}\s*$", re.I))199 ul = h.find_next_sibling("ul") if h else None200 return ([li.get_text(" ", strip=True) for li in201 ul.select("li")] if ul else [])202203 out["building_amenities"] = _ul("Building Amenities")204 out["rent_includes"] = _ul("Rent Includes")205 out["pet_friendly"] = _ul("Pet Friendly")206 out["local_amenities"] = _ul("Local Amenities")207208 # Contact du bureau de location209 contact = soup.select_one(".listing-contact")210 if contact:211 ctxt = contact.get_text(" ", strip=True)212 pm = _PHONE_RE.search(ctxt)213 if pm:214 out["phone"] = f"{pm.group(1)}-{pm.group(2)}-{pm.group(3)}"215 em = _EMAIL_RE.search(ctxt)216 if em:217 out["email"] = em.group(0)218219 # Description (partie anglaise, avant l'avis de non-responsabilité)220 dm = re.search(r"<h2>Description</h2>(.*?)(?:<h2|<hr)", html, re.S)221 if dm:222 dtxt = re.sub(r"<[^>]+>", " ", dm.group(1))223 dtxt = re.sub(r"\s+", " ", dtxt).strip()224 dtxt = re.split(r"The safest way|Disclaimer", dtxt)[0]225 out["description"] = dtxt.strip()[:600]226227 # Intersection (en-tête de fiche)228 txt = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))229 im = re.search(r"Intersection:\s*([^|]{3,60}?)\s{0,2}Suite", txt)230 if im:231 out["intersection"] = im.group(1).strip()232233 # Photos de l'unité (carrousel span data-bg)234 imgs = re.findall(235 r'data-bg="(https://www\.metcap\.com/wp-content/uploads/'236 r'[^"]+\.(?:jpg|jpeg|png|webp))"', html)237 out["images"] = [u for u in dict.fromkeys(imgs)238 if not _SKIP_IMG.search(u)][: self.max_images]239 return out240241 try:242 return self.detail(slug, card_key, _fetch)243 except Exception:244 return {}245246 # -- construction d'une annonce --------------------------------------------247 def _unit_listing(self, href: str, card_text: str, address: str,248 prop_name: str, prop_path: str, city: str, sector: str,249 lat: float | None, lng: float | None,250 province: str = "ON") -> Listing | None:251 path = href.split("?")[0]252 slug = path.rstrip("/").split("/")[-1]253 if not slug:254 return None255 url = f"{BASE}{path}?lang=en"256257 unit_type = ""258 for rx, ut in _TYPE_MAP:259 if rx.search(card_text):260 unit_type = ut261 break262 price = parse_price(263 card_text.replace("from $", "").replace(",", "") + " $")264 price_label = ""265 pm = re.search(r'from \$[\d,.]+', card_text)266 if pm:267 price_label = pm.group(0).replace("from", "From") + " /month"268269 # Fiche unité (cache BD, clé = contenu de la carte liste)270 card_key = hashlib.sha1(271 f"{card_text}|{address}".encode("utf-8")).hexdigest()[:16]272 det = self._unit_detail(slug, url, card_key)273274 # Tableau Suite Details : statut, superficie (structurés à la source)275 availability = ""276 area_sqft: float | None = None277 bits: list[str] = []278 suites = det.get("suites") or []279 row = next((r for r in suites280 if (r.get("Status") or "").lower() == "available"),281 suites[0] if suites else None)282 if row:283 availability = row.get("Status") or ""284 sq = re.sub(r"[^\d.]", "", row.get("Sq. Ft") or "")285 try:286 v = float(sq)287 if 80 <= v <= 20000:288 area_sqft = v289 except ValueError:290 pass291 beds, baths = row.get("Beds") or "", row.get("Baths") or ""292 if beds or baths:293 bits.append(" — ".join(x for x in [294 f"{beds} bed" if beds else "",295 f"{baths} bath" if baths else ""] if x))296 if det.get("intersection") and not sector:297 bits.append(f"Intersection: {det['intersection']}")298299 # Commodités brutes (immeuble + inclusions), fidèles à la source300 amenities = list(dict.fromkeys(301 (det.get("building_amenities") or []) +302 (det.get("rent_includes") or [])))[:25]303304 # Inclusions structurées (« Rent Includes ») et animaux (« Pet Friendly »)305 details: dict = {}306 inclusions: dict = {}307 for item in det.get("rent_includes") or []:308 for rx, cle in _INCLUDES_MAP:309 if rx.search(item):310 inclusions[cle] = True311 if inclusions:312 details["inclusions"] = inclusions313 pets = None314 pf = " ".join(det.get("pet_friendly") or []).strip().lower()315 if pf.startswith("yes"):316 pets = "oui"317 elif pf.startswith("no"):318 pets = "non"319 contact = {k: det[k] for k in ("phone", "email") if det.get(k)}320 if contact:321 details["contact"] = contact322323 # Photos : unité d'abord, sinon galerie de l'immeuble324 images = det.get("images") or []325 if not images:326 images = self._gallery(prop_path)327328 desc = det.get("description") or ""329 title_type = re.sub(r"\s*from \$[\d,.].*$", "", card_text).strip()330 title = (f"{prop_name} — {title_type}" if prop_name331 else f"{address} — {title_type}")332 if address and not re.search(rf",\s*{province}\b", address):333 address = f"{address}, {province}"334 return Listing(335 source=self.source_id,336 external_id=slug,337 url=url,338 title=title,339 address=address,340 province=province,341 sector=sector,342 city=city,343 unit_type=unit_type,344 price=price,345 price_label=price_label,346 availability=availability,347 area_sqft=area_sqft,348 pets=pets,349 description=" — ".join([desc] + bits if desc else bits)[:600],350 amenities=amenities,351 details=details,352 images=images,353 lat=lat,354 lng=lng,355 )356