Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Agrégateur de logements à louer (Québec + expansion Ontario)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/rentcafe.py : connecteur GÉNÉRIQUE Yardi RentCafe/SecureCafe5# (multi-clients Ontario — levier n° 2 de l'expansion, voir6# gestion-immobiliere-ontario.md §10)7#8# Une sous-classe est générée dynamiquement par client du registre9# data/rentcafe_clients.json (statut « ok ») : source_id = rc_<id>10# (vague 1 2026-08-26 : rc_effort, rc_osgoode, rc_gwlra, rc_oshanter ;11# vague 2 2026-08-27 : rc_claridge, rc_richcraft, rc_arnon, rc_concert,12# rc_caraco). Le registre auto-découvrant connectors/__init__.py les13# ramasse dans vars(module).14#15# Pattern « searchlisting » (validé sur les 9 clients actifs) :16# 1) <site>/searchlisting.aspx via Scrapfly (Cloudflare 403 en direct,17# contenu rendu côté serveur -> render_js inutile) :18# - cartes li.property-box-hidden : nom, lien fiche, adresse complète19# (« …, Kingston, ON K7P 1M8 »), lits/sdb/pi², fourchette de prix,20# téléphone, vignette resource.rentcafe.com ;21# - champ caché available_prop_map (JSON doublement encodé) :22# propertyid -> lat/lng + fourchette de prix des épingles de carte.23# Seules les cartes dont l'adresse est en Ontario sont conservées24# (Osgoode/GWLRA listent aussi AB/BC ; le QC reste aux connecteurs QC).25# 2) fiche propriété (+ /floorplans au besoin) via self.detail() (cache BD,26# budget Scrapfly par synchronisation) : galerie, description, plans27# structurés fp-container (2 gabarits : spans data-selenium-id ou cartes28# h2.card-title + nu-bed/nu-bathroom/nu-area + data-floorplan-*).29# Une annonce par propriété (uid stable = propertyid RentCafe) ; comme chez30# Osgoode, AUCUN décompte d'unités disponibles n'est publié -> availability31# reste vide (rien d'inventé).32#33# Pattern « securecafe » (<client>.securecafe.com/residentservices/34# apartmentsforrent/…) : vérifié fermé derrière login chez Old Oak,35# Paramount et Tricar — entrées « echec » du registre, aucune classe36# générée (voir les notes du registre avant de réessayer). Les autres37# impasses vérifiées (WordPress sans searchlisting, microsites par38# immeuble, Entrata, Rentsync, sites custom) sont aussi documentées39# en « echec » dans le registre.40#41# sans cette variable, disabled=True et le registre des connecteurs les42# ignore (zéro impact sur la prod Québec).43# -----------------------------------------------------------------------------44from __future__ import annotations4546import hashlib47import html as htmllib48import json49import os50import re5152from bs4 import BeautifulSoup5354from ..schema import Listing, parse_price55from .base import BaseConnector5657# Gate expansion Ontario (voir en-tête)58_ONTARIO = True # Rent-Ka: always on (ROC scope)5960_REGISTRY_PATH = os.path.join(os.path.dirname(__file__), "..", "..",61 "data", "rentcafe_clients.json")6263# « $1,449.00 - $1,899.00 » / « $1,499.00 » (format RentCafe, virgule = milliers)64_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?(?:\s*(?:-|to|à)+\s*"65 r"\$[\d,]+(?:\.\d{2})?)?")66_NUM_RE = re.compile(r"[\d,]+(?:\.\d+)?")67_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder|\.svg", re.I)68_PHONE_RE = re.compile(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})")6970# nombre de chambres -> type d'unité (normalize_unit_type fera la conversion71# canonique n½ à l'ingestion ; on passe le texte source, rien d'inventé)72_BED_TYPES = {0: "Studio", 1: "1 Bed", 2: "2 Beds", 3: "3 Beds", 4: "4 Beds"}737475def _to_float(txt: str) -> float | None:76 m = _NUM_RE.search(txt or "")77 if not m:78 return None79 try:80 return float(m.group(0).replace(",", ""))81 except ValueError:82 return None838485def _low_price(txt: str) -> float | None:86 """Borne basse d'une fourchette « $1,449.00 - $1,899.00 » (100-20000 $)."""87 vals = [_to_float(v) for v in re.findall(r"\$[\d,]+(?:\.\d{2})?", txt or "")]88 vals = [v for v in vals if v is not None and 100 <= v <= 20000]89 return min(vals) if vals else None909192class _DetailBudget(Exception):93 """Budget de nouvelles fiches Scrapfly épuisé pour cette synchronisation."""949596class RentCafeClientConnector(BaseConnector):97 province = "ON" # overridden per client ("province" key in the registry)9899 """Classe de base des clients RentCafe — ne PAS l'enregistrer telle quelle100 (source_id vide) : les sous-classes concrètes sont générées plus bas à101 partir du registre data/rentcafe_clients.json."""102103 source_id = "" # vide -> ignorée par connectors/__init__.py104 disabled = False105 request_delay = 1.5 # Scrapfly coûte : politesse renforcée106 client: dict = {} # entrée du registre (site, search_url…)107 max_properties = 160 # garde-fou (Effort Trust : 153 cartes ON)108 max_details = 8 # nouvelles fiches Scrapfly max par sync109 max_images = 20110111 # -- Scrapfly (Cloudflare -> ASP ; contenu rendu serveur, pas de JS) -------112 def _page(self, url: str) -> str:113 res = self.scrapfly(url, render_js=False, asp=True, country="ca")114 if (res.get("status_code") or 0) != 200:115 return ""116 return res.get("content") or ""117118 # -- fetch ------------------------------------------------------------------119 def fetch(self) -> list[Listing]:120 if (self.client.get("pattern") or "searchlisting") != "searchlisting":121 return [] # « securecafe » public : aucun client validé122 html = self._page(self.client["search_url"])123 if not html:124 raise RuntimeError(125 f"searchlisting inaccessible via Scrapfly ({self.source_id})")126 soup = BeautifulSoup(html, "html.parser")127 pins = self._map_pins(html)128129 self._detail_fetches = 0130 listings: list[Listing] = []131 seen: set[str] = set()132 cards = soup.select("li.property-box-hidden") \133 or soup.select("li.property-box, .property-box")134 for card in cards:135 if len(listings) >= self.max_properties:136 break137 try:138 lst = self._property_listing(card, pins)139 if lst and lst.external_id not in seen:140 seen.add(lst.external_id)141 listings.append(lst)142 except Exception:143 continue144 return listings145146 # -- épingles de carte (champ caché available_prop_map) ----------------------147 @staticmethod148 def _map_pins(html: str) -> dict[str, dict]:149 """propertyid -> {lat, lng, price, beds} (JSON doublement encodé)."""150 m = re.search(r"available_prop_map[^>]*value=(['\"])(.*?)\1", html, re.S)151 if not m:152 return {}153 try:154 data = json.loads(htmllib.unescape(m.group(2)))155 if isinstance(data, str):156 data = json.loads(data)157 pins = data.get("ListingsPins")158 if isinstance(pins, str):159 pins = json.loads(pins)160 except (ValueError, AttributeError):161 return {}162 out: dict[str, dict] = {}163 for grp in (pins or {}).get("groups") or []:164 for p in grp.get("points") or []:165 pid = str(p.get("propertyid") or p.get("id") or "")166 if not pid:167 continue168 hover = p.get("hover") or {}169 out[pid] = {"lat": p.get("y"), "lng": p.get("x"),170 "price": hover.get("Price") or "",171 "beds": hover.get("Beds") or ""}172 return out173174 # -- carte propriété ----------------------------------------------------------175 def _property_listing(self, card, pins: dict[str, dict]) -> Listing | None:176 a = card.select_one(".property-name a") or card.select_one("h3 a")177 if not a or not a.get("href"):178 return None179 url = (a.get("href") or "").strip()180 if url.startswith("/"):181 url = self.client["site"].rstrip("/") + url182 url = url.split("?")[0].rstrip("/")183184 name = re.sub(r"\s*opens in a new tab\s*", "",185 a.get_text(" ", strip=True)).strip()186187 addr_el = card.select_one(".card-prop-address")188 address = addr_el.get_text(" ", strip=True) if addr_el else ""189 # Ontario seulement (Osgoode/GWLRA listent aussi AB/BC ; QC = connecteurs QC)190 if not re.search(r",\s*ON(?:\s|,|$)", address):191 return None192 city = ""193 parts = [p.strip() for p in address.split(",")]194 for i, p in enumerate(parts):195 if re.match(r"^ON(\s|$)", p) and i > 0:196 city = parts[i - 1]197 break198199 # propertyid RentCafe stable (classe track-propertyurl-<id> ou épingle)200 pid = ""201 for el in card.select("[class*='track-propertyurl-']"):202 for cl in el.get("class") or []:203 if cl.startswith("track-propertyurl-"):204 pid = cl.rsplit("-", 1)[-1]205 break206 slug = re.sub(r"[^a-z0-9]+", "-",207 url.rstrip("/").rsplit("/", 1)[-1].lower()).strip("-")208 external_id = pid or slug209 if not external_id:210 return None211212 # lits / sdb / pi² de la carte (« 1.0Beds - 2.0Beds », « 799 - 1,018 Sq. Ft. »)213 beds_txt = baths_txt = sqft_txt = ""214 meta = card.select_one(".card-bed-bath-rent")215 if meta:216 for li in meta.select("li"):217 it = li.get_text(" ", strip=True)218 if "Bed" in it:219 beds_txt = it220 elif "Bath" in it:221 baths_txt = it222 elif "Sq" in it:223 sqft_txt = re.sub(r"\s*to\s*-\s*", " - ", it)224 unit_type = ""225 bm = re.match(r"^(\d+)(?:\.\d+)?\s*Beds?", beds_txt or "")226 if bm and "-" not in beds_txt.split("Bed")[0]:227 unit_type = _BED_TYPES.get(int(bm.group(1)), "")228229 # fourchette de prix : carte, sinon épingle de la carte interactive230 pin = pins.get(external_id) or {}231 pm = _PRICE_RE.search(card.get_text(" ", strip=True))232 price_label = pm.group(0) if pm else (pin.get("price") or "")233 price_label = re.sub(r"\s*(?:to|à)\s*", " - ", price_label).strip()234 price = _low_price(price_label)235 if price is not None and "-" in price_label:236 price_label = "À partir de " + price_label237 if price is None:238 price_label = "" # « Call for Details » : rien d'inventé239240 phone = ""241 tel = card.select_one("a[href^='tel:']")242 if tel:243 tm = _PHONE_RE.search(tel.get("href") or "")244 if tm:245 phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"246247 images: list[str] = []248 img = card.select_one("img[src*='rentcafe']") or card.select_one("img")249 if img and (img.get("src") or "").startswith("http") \250 and not _SKIP_IMG.search(img.get("src") or ""):251 images.append(img["src"])252253 # fiche + plans via cache BD (clé = contenu de la carte liste)254 key = hashlib.sha1(255 f"{name}|{address}|{beds_txt}|{baths_txt}|{sqft_txt}|{price_label}"256 .encode("utf-8")).hexdigest()257 try:258 payload = self.detail(external_id, key,259 lambda: self._fetch_detail(url))260 except _DetailBudget:261 payload = {}262 except Exception:263 payload = {}264265 for im in payload.get("images") or []:266 if im not in images:267 images.append(im)268269 # plans structurés : prix « à partir de » réel + résumé fidèle270 plans = payload.get("floorplans") or []271 prices = [p["price"] for p in plans272 if p.get("price") and 100 <= p["price"] <= 20000]273 if prices:274 price = min(prices)275 price_label = (f"À partir de {price:,.0f} $/mois".replace(",", " ")276 if len(plans) > 1 or "-" in price_label277 else f"{price:,.0f} $/mois".replace(",", " "))278 sqfts = [p["sqft"] for p in plans if p.get("sqft")]279 area_sqft = min(sqfts) if sqfts else None280 if len(plans) == 1 and plans[0].get("unit_type"):281 unit_type = plans[0]["unit_type"]282 plan_bits = []283 for p in plans[:8]:284 seg = p.get("name") or ""285 if p.get("sqft"):286 seg += f" ({p['sqft']:.0f} pi²)"287 if p.get("price"):288 seg += f" : {p['price']:,.0f} $/mois".replace(",", " ")289 if seg:290 plan_bits.append(seg)291292 bathrooms = None293 tb = re.match(r"^(\d+(?:\.\d+)?)\s*Bath", baths_txt or "")294 if tb and "-" not in baths_txt.split("Bath")[0]:295 bathrooms = float(tb.group(1))296297 details: dict = {}298 if phone:299 details["contact"] = {"phone": phone}300301 desc_parts = ([payload["description"]]302 if payload.get("description") else [])303 desc_parts += [b for b in [beds_txt, baths_txt, sqft_txt] if b]304 if plan_bits:305 desc_parts.append("Plans : " + " ; ".join(plan_bits))306307 lat = pin.get("lat")308 lng = pin.get("lng")309310 return Listing(311 source=self.source_id,312 external_id=str(external_id),313 url=url,314 title=name or slug.replace("-", " ").title(),315 address=address,316 sector="", # le gabarit RentCafe ne publie pas le quartier317 city=city,318 province=self.province,319 unit_type=unit_type,320 bathrooms=bathrooms,321 price=price,322 price_label=price_label,323 availability="", # aucun décompte d'unités publié (cf. en-tête)324 area_sqft=area_sqft,325 description=" — ".join(desc_parts)[:900],326 details=details,327 images=images[: self.max_images],328 lat=float(lat) if isinstance(lat, (int, float)) else None,329 lng=float(lng) if isinstance(lng, (int, float)) else None,330 )331332 # -- fiche propriété (galerie + description + plans) --------------------------333 def _fetch_detail(self, url: str) -> dict:334 if self._detail_fetches >= self.max_details:335 raise _DetailBudget()336 self._detail_fetches += 1337338 payload: dict = {"description": "", "images": [], "floorplans": []}339 html = self._page(url)340 if html:341 soup = BeautifulSoup(html, "html.parser")342 for im in soup.select("img[src*='resource.rentcafe.com']"):343 src = im.get("src") or ""344 if src and not _SKIP_IMG.search(src) \345 and src not in payload["images"]:346 payload["images"].append(src)347 paras = [p.get_text(" ", strip=True) for p in soup.find_all("p")]348 paras = [p for p in paras if len(p) > 80]349 if paras:350 payload["description"] = " ".join(paras[:2])[:600]351 payload["floorplans"] = self._parse_floorplans(soup)352353 # plans absents de la fiche (gabarit Osgoode/GWLRA) -> page /floorplans354 if not payload["floorplans"] and not url.endswith("default.aspx") \355 and self._detail_fetches < self.max_details:356 self._detail_fetches += 1357 fp_html = self._page(url + "/floorplans")358 if fp_html:359 payload["floorplans"] = self._parse_floorplans(360 BeautifulSoup(fp_html, "html.parser"))361 return payload362363 # -- plans fp-container (2 gabarits RentCafe) ----------------------------------364 @staticmethod365 def _parse_floorplans(soup) -> list[dict]:366 """Cartes de plans : nom, chambres, sdb, pi², prix (borne basse d'une367 fourchette). Gabarits : spans data-selenium-id (Osgoode) OU cartes368 h2.card-title + icônes nu-bed/nu-bathroom/nu-area + attributs369 data-floorplan-* (Effort, GWLRA). Dédoublonnés par id (carrousels)."""370 plans: list[dict] = []371 seen: set[str] = set()372 for cont in soup.select("div[id^='fp-container-']"):373 fpid = (cont.get("id") or "").rsplit("-", 1)[-1]374 if fpid in seen:375 continue376 seen.add(fpid)377 try:378 plan: dict = {}379 # gabarit 1 : spans data-selenium-id380 name_el = cont.select_one("span[data-selenium-id$='Name']")381 # gabarit 2 : cartes (titre + icônes)382 if name_el is None:383 name_el = cont.select_one("h2.card-title, .card-title")384 if name_el is not None:385 plan["name"] = name_el.get_text(" ", strip=True)386387 beds_el = cont.select_one("span[data-selenium-id$='Beds']")388 beds_txt = (beds_el.get_text(" ", strip=True) if beds_el389 else "")390 if not beds_txt:391 ic = cont.select_one(".nu-bed")392 if ic and ic.parent:393 beds_txt = ic.parent.get_text(" ", strip=True)394 bm = re.search(r"(\d+)\s*Bed", beds_txt)395 if bm:396 plan["bedrooms"] = float(bm.group(1))397 plan["unit_type"] = _BED_TYPES.get(int(bm.group(1)), "")398 elif re.search(r"studio", (plan.get("name") or "") + beds_txt,399 re.I):400 plan["bedrooms"] = 0.0401 plan["unit_type"] = "Studio"402403 baths_el = cont.select_one("span[data-selenium-id$='Baths']")404 baths_txt = (baths_el.get_text(" ", strip=True) if baths_el405 else "")406 if not baths_txt:407 ic = cont.select_one(".nu-bathroom")408 if ic and ic.parent:409 baths_txt = ic.parent.get_text(" ", strip=True)410 tm = re.search(r"(\d+(?:\.\d+)?)\s*Bath", baths_txt)411 if tm:412 plan["bathrooms"] = float(tm.group(1))413414 sq_el = cont.select_one("span[data-selenium-id$='SqFt']")415 sq_txt = sq_el.get_text(" ", strip=True) if sq_el else ""416 if not sq_txt:417 ic = cont.select_one(".nu-area")418 if ic and ic.parent:419 sq_txt = ic.parent.get_text(" ", strip=True)420 sv = _to_float(sq_txt)421 if sv and 80 <= sv <= 20000:422 plan["sqft"] = sv423424 # prix : attribut structuré data-floorplan-price (« 2113 -2163 »),425 # sinon encadré « Starting at $2,113.00 /Month », sinon span Rent426 pv = None427 btn = cont.select_one("[data-floorplan-price]")428 if btn:429 nums = [_to_float(x) for x in _NUM_RE.findall(430 btn.get("data-floorplan-price") or "")]431 nums = [n for n in nums if n and 100 <= n <= 20000]432 if nums:433 pv = min(nums)434 if pv is None:435 rent_el = cont.select_one(436 "span[data-selenium-id$='Rent']") \437 or cont.select_one(".fieldset .font-weight-bold, "438 ".fieldset span.font-weight-bold")439 if rent_el:440 pv = _low_price(rent_el.get_text(" ", strip=True))441 if pv is None:442 pv = _low_price(" ".join(443 _PRICE_RE.findall(cont.get_text(" ", strip=True))))444 if pv:445 plan["price"] = pv446447 if plan.get("name") or plan.get("price"):448 plans.append(plan)449 except Exception:450 continue451 return plans452453454# =============================================================================455# Génération des sous-classes concrètes à partir du registre456# data/rentcafe_clients.json — une classe par client « ok », déposée dans les457# globals du module pour que connectors/__init__.py la découvre. Un registre458# absent/corrompu ne doit JAMAIS casser l'import du paquet (prod QC).459# =============================================================================460def _load_clients() -> list[dict]:461 try:462 with open(_REGISTRY_PATH, encoding="utf-8") as f:463 return json.load(f).get("clients") or []464 except (OSError, ValueError):465 return []466467468for _c in _load_clients():469 if (_c.get("status") or "") != "ok" or not _c.get("id"):470 continue471 _cls_name = "RC" + "".join(472 w.capitalize() for w in re.split(r"[^a-z0-9]+", _c["id"]) if w) \473 + "Connector"474 globals()[_cls_name] = type(_cls_name, (RentCafeClientConnector,), {475 "source_id": f"rc_{_c['id']}",476 "client": _c,477 "province": (_c.get("province") or "ON").upper(),478 "disabled": False,479 "__doc__": f"Client RentCafe « {_c.get('name') or _c['id']} » "480 f"({_c.get('regions') or 'Ontario'}) — généré depuis "481 "data/rentcafe_clients.json.",482 })483del _c484