Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 98.9%
Python 0.6%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (Québec + expansion Ontario)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/rentcafe.py : connecteur GÉNÉRIQUE Yardi RentCafe/SecureCafe5# (multi-clients Ontario — levier n° 2 de l'expansion, voir6# gestion-immobiliere-ontario.md §10)7#8# Une sous-classe est générée dynamiquement par client du registre9# data/rentcafe_clients.json (statut « ok ») : source_id = rc_<id>10# (vague 1 2026-08-26 : rc_effort, rc_osgoode, rc_gwlra, rc_oshanter ;11# vague 2 2026-08-27 : rc_claridge, rc_richcraft, rc_arnon, rc_concert,12# rc_caraco). Le registre auto-découvrant connectors/__init__.py les13# ramasse dans vars(module).14#15# Pattern « searchlisting » (validé sur les 9 clients actifs) :16# 1) <site>/searchlisting.aspx via Scrapfly (Cloudflare 403 en direct,17# contenu rendu côté serveur -> render_js inutile) :18# - cartes li.property-box-hidden : nom, lien fiche, adresse complète19# (« …, Kingston, ON K7P 1M8 »), lits/sdb/pi², fourchette de prix,20# téléphone, vignette resource.rentcafe.com ;21# - champ caché available_prop_map (JSON doublement encodé) :22# propertyid -> lat/lng + fourchette de prix des épingles de carte.23# Seules les cartes dont l'adresse est en Ontario sont conservées24# (Osgoode/GWLRA listent aussi AB/BC ; le QC reste aux connecteurs QC).25# 2) fiche propriété (+ /floorplans au besoin) via self.detail() (cache BD,26# budget Scrapfly par synchronisation) : galerie, description, plans27# structurés fp-container (2 gabarits : spans data-selenium-id ou cartes28# h2.card-title + nu-bed/nu-bathroom/nu-area + data-floorplan-*).29# Une annonce par propriété (uid stable = propertyid RentCafe) ; comme chez30# Osgoode, AUCUN décompte d'unités disponibles n'est publié -> availability31# reste vide (rien d'inventé).32#33# Pattern « securecafe » (<client>.securecafe.com/residentservices/34# apartmentsforrent/…) : vérifié fermé derrière login chez Old Oak,35# Paramount et Tricar — entrées « echec » du registre, aucune classe36# générée (voir les notes du registre avant de réessayer). Les autres37# impasses vérifiées (WordPress sans searchlisting, microsites par38# immeuble, Entrata, Rentsync, sites custom) sont aussi documentées39# en « echec » dans le registre.40#41# ⚠ Gate Ontario : les classes ne sont actives que si LOUKA_ONTARIO=1 —42# sans cette variable, disabled=True et le registre des connecteurs les43# ignore (zéro impact sur la prod Québec).44# -----------------------------------------------------------------------------45from __future__ import annotations4647import hashlib48import html as htmllib49import json50import os51import re5253from bs4 import BeautifulSoup5455from ..schema import Listing, parse_price56from .base import BaseConnector5758# Gate expansion Ontario (voir en-tête)59_ONTARIO = os.environ.get("LOUKA_ONTARIO") == "1"6061_REGISTRY_PATH = os.path.join(os.path.dirname(__file__), "..", "..",62 "data", "rentcafe_clients.json")6364# « $1,449.00 - $1,899.00 » / « $1,499.00 » (format RentCafe, virgule = milliers)65_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?(?:\s*(?:-|to|à)+\s*"66 r"\$[\d,]+(?:\.\d{2})?)?")67_NUM_RE = re.compile(r"[\d,]+(?:\.\d+)?")68_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder|\.svg", re.I)69_PHONE_RE = re.compile(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})")7071# nombre de chambres -> type d'unité (normalize_unit_type fera la conversion72# canonique n½ à l'ingestion ; on passe le texte source, rien d'inventé)73_BED_TYPES = {0: "Studio", 1: "1 Bed", 2: "2 Beds", 3: "3 Beds", 4: "4 Beds"}747576def _to_float(txt: str) -> float | None:77 m = _NUM_RE.search(txt or "")78 if not m:79 return None80 try:81 return float(m.group(0).replace(",", ""))82 except ValueError:83 return None848586def _low_price(txt: str) -> float | None:87 """Borne basse d'une fourchette « $1,449.00 - $1,899.00 » (100-20000 $)."""88 vals = [_to_float(v) for v in re.findall(r"\$[\d,]+(?:\.\d{2})?", txt or "")]89 vals = [v for v in vals if v is not None and 100 <= v <= 20000]90 return min(vals) if vals else None919293class _DetailBudget(Exception):94 """Budget de nouvelles fiches Scrapfly épuisé pour cette synchronisation."""959697class RentCafeClientConnector(BaseConnector):98 """Classe de base des clients RentCafe — ne PAS l'enregistrer telle quelle99 (source_id vide) : les sous-classes concrètes sont générées plus bas à100 partir du registre data/rentcafe_clients.json."""101102 source_id = "" # vide -> ignorée par connectors/__init__.py103 disabled = not _ONTARIO # gate expansion Ontario104 request_delay = 1.5 # Scrapfly coûte : politesse renforcée105 client: dict = {} # entrée du registre (site, search_url…)106 max_properties = 160 # garde-fou (Effort Trust : 153 cartes ON)107 max_details = 8 # nouvelles fiches Scrapfly max par sync108 max_images = 20109110 # -- Scrapfly (Cloudflare -> ASP ; contenu rendu serveur, pas de JS) -------111 def _page(self, url: str) -> str:112 res = self.scrapfly(url, render_js=False, asp=True, country="ca")113 if (res.get("status_code") or 0) != 200:114 return ""115 return res.get("content") or ""116117 # -- fetch ------------------------------------------------------------------118 def fetch(self) -> list[Listing]:119 if (self.client.get("pattern") or "searchlisting") != "searchlisting":120 return [] # « securecafe » public : aucun client validé121 html = self._page(self.client["search_url"])122 if not html:123 raise RuntimeError(124 f"searchlisting inaccessible via Scrapfly ({self.source_id})")125 soup = BeautifulSoup(html, "html.parser")126 pins = self._map_pins(html)127128 self._detail_fetches = 0129 listings: list[Listing] = []130 seen: set[str] = set()131 cards = soup.select("li.property-box-hidden") \132 or soup.select("li.property-box, .property-box")133 for card in cards:134 if len(listings) >= self.max_properties:135 break136 try:137 lst = self._property_listing(card, pins)138 if lst and lst.external_id not in seen:139 seen.add(lst.external_id)140 listings.append(lst)141 except Exception:142 continue143 return listings144145 # -- épingles de carte (champ caché available_prop_map) ----------------------146 @staticmethod147 def _map_pins(html: str) -> dict[str, dict]:148 """propertyid -> {lat, lng, price, beds} (JSON doublement encodé)."""149 m = re.search(r"available_prop_map[^>]*value=(['\"])(.*?)\1", html, re.S)150 if not m:151 return {}152 try:153 data = json.loads(htmllib.unescape(m.group(2)))154 if isinstance(data, str):155 data = json.loads(data)156 pins = data.get("ListingsPins")157 if isinstance(pins, str):158 pins = json.loads(pins)159 except (ValueError, AttributeError):160 return {}161 out: dict[str, dict] = {}162 for grp in (pins or {}).get("groups") or []:163 for p in grp.get("points") or []:164 pid = str(p.get("propertyid") or p.get("id") or "")165 if not pid:166 continue167 hover = p.get("hover") or {}168 out[pid] = {"lat": p.get("y"), "lng": p.get("x"),169 "price": hover.get("Price") or "",170 "beds": hover.get("Beds") or ""}171 return out172173 # -- carte propriété ----------------------------------------------------------174 def _property_listing(self, card, pins: dict[str, dict]) -> Listing | None:175 a = card.select_one(".property-name a") or card.select_one("h3 a")176 if not a or not a.get("href"):177 return None178 url = (a.get("href") or "").strip()179 if url.startswith("/"):180 url = self.client["site"].rstrip("/") + url181 url = url.split("?")[0].rstrip("/")182183 name = re.sub(r"\s*opens in a new tab\s*", "",184 a.get_text(" ", strip=True)).strip()185186 addr_el = card.select_one(".card-prop-address")187 address = addr_el.get_text(" ", strip=True) if addr_el else ""188 # Ontario seulement (Osgoode/GWLRA listent aussi AB/BC ; QC = connecteurs QC)189 if not re.search(r",\s*ON(?:\s|,|$)", address):190 return None191 city = ""192 parts = [p.strip() for p in address.split(",")]193 for i, p in enumerate(parts):194 if re.match(r"^ON(\s|$)", p) and i > 0:195 city = parts[i - 1]196 break197198 # propertyid RentCafe stable (classe track-propertyurl-<id> ou épingle)199 pid = ""200 for el in card.select("[class*='track-propertyurl-']"):201 for cl in el.get("class") or []:202 if cl.startswith("track-propertyurl-"):203 pid = cl.rsplit("-", 1)[-1]204 break205 slug = re.sub(r"[^a-z0-9]+", "-",206 url.rstrip("/").rsplit("/", 1)[-1].lower()).strip("-")207 external_id = pid or slug208 if not external_id:209 return None210211 # lits / sdb / pi² de la carte (« 1.0Beds - 2.0Beds », « 799 - 1,018 Sq. Ft. »)212 beds_txt = baths_txt = sqft_txt = ""213 meta = card.select_one(".card-bed-bath-rent")214 if meta:215 for li in meta.select("li"):216 it = li.get_text(" ", strip=True)217 if "Bed" in it:218 beds_txt = it219 elif "Bath" in it:220 baths_txt = it221 elif "Sq" in it:222 sqft_txt = re.sub(r"\s*to\s*-\s*", " - ", it)223 unit_type = ""224 bm = re.match(r"^(\d+)(?:\.\d+)?\s*Beds?", beds_txt or "")225 if bm and "-" not in beds_txt.split("Bed")[0]:226 unit_type = _BED_TYPES.get(int(bm.group(1)), "")227228 # fourchette de prix : carte, sinon épingle de la carte interactive229 pin = pins.get(external_id) or {}230 pm = _PRICE_RE.search(card.get_text(" ", strip=True))231 price_label = pm.group(0) if pm else (pin.get("price") or "")232 price_label = re.sub(r"\s*(?:to|à)\s*", " - ", price_label).strip()233 price = _low_price(price_label)234 if price is not None and "-" in price_label:235 price_label = "À partir de " + price_label236 if price is None:237 price_label = "" # « Call for Details » : rien d'inventé238239 phone = ""240 tel = card.select_one("a[href^='tel:']")241 if tel:242 tm = _PHONE_RE.search(tel.get("href") or "")243 if tm:244 phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"245246 images: list[str] = []247 img = card.select_one("img[src*='rentcafe']") or card.select_one("img")248 if img and (img.get("src") or "").startswith("http") \249 and not _SKIP_IMG.search(img.get("src") or ""):250 images.append(img["src"])251252 # fiche + plans via cache BD (clé = contenu de la carte liste)253 key = hashlib.sha1(254 f"{name}|{address}|{beds_txt}|{baths_txt}|{sqft_txt}|{price_label}"255 .encode("utf-8")).hexdigest()256 try:257 payload = self.detail(external_id, key,258 lambda: self._fetch_detail(url))259 except _DetailBudget:260 payload = {}261 except Exception:262 payload = {}263264 for im in payload.get("images") or []:265 if im not in images:266 images.append(im)267268 # plans structurés : prix « à partir de » réel + résumé fidèle269 plans = payload.get("floorplans") or []270 prices = [p["price"] for p in plans271 if p.get("price") and 100 <= p["price"] <= 20000]272 if prices:273 price = min(prices)274 price_label = (f"À partir de {price:,.0f} $/mois".replace(",", " ")275 if len(plans) > 1 or "-" in price_label276 else f"{price:,.0f} $/mois".replace(",", " "))277 sqfts = [p["sqft"] for p in plans if p.get("sqft")]278 area_sqft = min(sqfts) if sqfts else None279 if len(plans) == 1 and plans[0].get("unit_type"):280 unit_type = plans[0]["unit_type"]281 plan_bits = []282 for p in plans[:8]:283 seg = p.get("name") or ""284 if p.get("sqft"):285 seg += f" ({p['sqft']:.0f} pi²)"286 if p.get("price"):287 seg += f" : {p['price']:,.0f} $/mois".replace(",", " ")288 if seg:289 plan_bits.append(seg)290291 bathrooms = None292 tb = re.match(r"^(\d+(?:\.\d+)?)\s*Bath", baths_txt or "")293 if tb and "-" not in baths_txt.split("Bath")[0]:294 bathrooms = float(tb.group(1))295296 details: dict = {}297 if phone:298 details["contact"] = {"phone": phone}299300 desc_parts = ([payload["description"]]301 if payload.get("description") else [])302 desc_parts += [b for b in [beds_txt, baths_txt, sqft_txt] if b]303 if plan_bits:304 desc_parts.append("Plans : " + " ; ".join(plan_bits))305306 lat = pin.get("lat")307 lng = pin.get("lng")308309 return Listing(310 source=self.source_id,311 external_id=str(external_id),312 url=url,313 title=name or slug.replace("-", " ").title(),314 address=address,315 sector="", # le gabarit RentCafe ne publie pas le quartier316 city=city,317 province="ON",318 unit_type=unit_type,319 bathrooms=bathrooms,320 price=price,321 price_label=price_label,322 availability="", # aucun décompte d'unités publié (cf. en-tête)323 area_sqft=area_sqft,324 description=" — ".join(desc_parts)[:900],325 details=details,326 images=images[: self.max_images],327 lat=float(lat) if isinstance(lat, (int, float)) else None,328 lng=float(lng) if isinstance(lng, (int, float)) else None,329 )330331 # -- fiche propriété (galerie + description + plans) --------------------------332 def _fetch_detail(self, url: str) -> dict:333 if self._detail_fetches >= self.max_details:334 raise _DetailBudget()335 self._detail_fetches += 1336337 payload: dict = {"description": "", "images": [], "floorplans": []}338 html = self._page(url)339 if html:340 soup = BeautifulSoup(html, "html.parser")341 for im in soup.select("img[src*='resource.rentcafe.com']"):342 src = im.get("src") or ""343 if src and not _SKIP_IMG.search(src) \344 and src not in payload["images"]:345 payload["images"].append(src)346 paras = [p.get_text(" ", strip=True) for p in soup.find_all("p")]347 paras = [p for p in paras if len(p) > 80]348 if paras:349 payload["description"] = " ".join(paras[:2])[:600]350 payload["floorplans"] = self._parse_floorplans(soup)351352 # plans absents de la fiche (gabarit Osgoode/GWLRA) -> page /floorplans353 if not payload["floorplans"] and not url.endswith("default.aspx") \354 and self._detail_fetches < self.max_details:355 self._detail_fetches += 1356 fp_html = self._page(url + "/floorplans")357 if fp_html:358 payload["floorplans"] = self._parse_floorplans(359 BeautifulSoup(fp_html, "html.parser"))360 return payload361362 # -- plans fp-container (2 gabarits RentCafe) ----------------------------------363 @staticmethod364 def _parse_floorplans(soup) -> list[dict]:365 """Cartes de plans : nom, chambres, sdb, pi², prix (borne basse d'une366 fourchette). Gabarits : spans data-selenium-id (Osgoode) OU cartes367 h2.card-title + icônes nu-bed/nu-bathroom/nu-area + attributs368 data-floorplan-* (Effort, GWLRA). Dédoublonnés par id (carrousels)."""369 plans: list[dict] = []370 seen: set[str] = set()371 for cont in soup.select("div[id^='fp-container-']"):372 fpid = (cont.get("id") or "").rsplit("-", 1)[-1]373 if fpid in seen:374 continue375 seen.add(fpid)376 try:377 plan: dict = {}378 # gabarit 1 : spans data-selenium-id379 name_el = cont.select_one("span[data-selenium-id$='Name']")380 # gabarit 2 : cartes (titre + icônes)381 if name_el is None:382 name_el = cont.select_one("h2.card-title, .card-title")383 if name_el is not None:384 plan["name"] = name_el.get_text(" ", strip=True)385386 beds_el = cont.select_one("span[data-selenium-id$='Beds']")387 beds_txt = (beds_el.get_text(" ", strip=True) if beds_el388 else "")389 if not beds_txt:390 ic = cont.select_one(".nu-bed")391 if ic and ic.parent:392 beds_txt = ic.parent.get_text(" ", strip=True)393 bm = re.search(r"(\d+)\s*Bed", beds_txt)394 if bm:395 plan["bedrooms"] = float(bm.group(1))396 plan["unit_type"] = _BED_TYPES.get(int(bm.group(1)), "")397 elif re.search(r"studio", (plan.get("name") or "") + beds_txt,398 re.I):399 plan["bedrooms"] = 0.0400 plan["unit_type"] = "Studio"401402 baths_el = cont.select_one("span[data-selenium-id$='Baths']")403 baths_txt = (baths_el.get_text(" ", strip=True) if baths_el404 else "")405 if not baths_txt:406 ic = cont.select_one(".nu-bathroom")407 if ic and ic.parent:408 baths_txt = ic.parent.get_text(" ", strip=True)409 tm = re.search(r"(\d+(?:\.\d+)?)\s*Bath", baths_txt)410 if tm:411 plan["bathrooms"] = float(tm.group(1))412413 sq_el = cont.select_one("span[data-selenium-id$='SqFt']")414 sq_txt = sq_el.get_text(" ", strip=True) if sq_el else ""415 if not sq_txt:416 ic = cont.select_one(".nu-area")417 if ic and ic.parent:418 sq_txt = ic.parent.get_text(" ", strip=True)419 sv = _to_float(sq_txt)420 if sv and 80 <= sv <= 20000:421 plan["sqft"] = sv422423 # prix : attribut structuré data-floorplan-price (« 2113 -2163 »),424 # sinon encadré « Starting at $2,113.00 /Month », sinon span Rent425 pv = None426 btn = cont.select_one("[data-floorplan-price]")427 if btn:428 nums = [_to_float(x) for x in _NUM_RE.findall(429 btn.get("data-floorplan-price") or "")]430 nums = [n for n in nums if n and 100 <= n <= 20000]431 if nums:432 pv = min(nums)433 if pv is None:434 rent_el = cont.select_one(435 "span[data-selenium-id$='Rent']") \436 or cont.select_one(".fieldset .font-weight-bold, "437 ".fieldset span.font-weight-bold")438 if rent_el:439 pv = _low_price(rent_el.get_text(" ", strip=True))440 if pv is None:441 pv = _low_price(" ".join(442 _PRICE_RE.findall(cont.get_text(" ", strip=True))))443 if pv:444 plan["price"] = pv445446 if plan.get("name") or plan.get("price"):447 plans.append(plan)448 except Exception:449 continue450 return plans451452453# =============================================================================454# Génération des sous-classes concrètes à partir du registre455# data/rentcafe_clients.json — une classe par client « ok », déposée dans les456# globals du module pour que connectors/__init__.py la découvre. Un registre457# absent/corrompu ne doit JAMAIS casser l'import du paquet (prod QC).458# =============================================================================459def _load_clients() -> list[dict]:460 try:461 with open(_REGISTRY_PATH, encoding="utf-8") as f:462 return json.load(f).get("clients") or []463 except (OSError, ValueError):464 return []465466467for _c in _load_clients():468 if (_c.get("status") or "") != "ok" or not _c.get("id"):469 continue470 _cls_name = "RC" + "".join(471 w.capitalize() for w in re.split(r"[^a-z0-9]+", _c["id"]) if w) \472 + "Connector"473 globals()[_cls_name] = type(_cls_name, (RentCafeClientConnector,), {474 "source_id": f"rc_{_c['id']}",475 "client": _c,476 "disabled": not _ONTARIO,477 "__doc__": f"Client RentCafe « {_c.get('name') or _c['id']} » "478 f"({_c.get('regions') or 'Ontario'}) — généré depuis "479 "data/rentcafe_clients.json.",480 })481del _c482