spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/realstar.py : connecteur Realstar (realstar.ca)5# Site protégé par Cloudflare (403 pour les robots) et rendu côté client6# (moteur RentCafe/Yardi) : tout passe par Firecrawl avec attente de rendu.7# 1) /searchlisting?province=Quebec -> cartes propriétés (nom, adresse,8# lits/sdb/pi², fourchette de prix, téléphone, vignette) ;9# 2) fiche de chaque propriété couverte (Grand Montréal, Gatineau,10# Sherbrooke) -> galerie photos, description, points forts ;11# 3) fiche /floorplans -> plans structurés (type, chambres, pi², prix,12# nombre d'unités disponibles) — disponibilité et prix réels.13# Une annonce par propriété (uid stables). Les pages détail passent par14# self.detail(...) (cache BD) : Firecrawl n'est rappelé que si la carte15# liste a changé.16# -----------------------------------------------------------------------------17from __future__ import annotations1819import hashlib20import os21import re2223from bs4 import BeautifulSoup2425from ..schema import Listing, parse_price26from .base import FIRECRAWL_API, BaseConnector2728SEARCH_URL = "https://www.realstar.ca/searchlisting?province=Quebec"2930# Ville du chemin /apartments/qc/<ville>/<slug> -> (ville affichée, secteur)31_GM_CITIES = {32 "montreal": ("Montréal", ""),33 "cote-saint-luc": ("Côte-Saint-Luc", ""),34 "brossard": ("Brossard", ""),35 "pointe-claire": ("Pointe-Claire", ""),36 "boisbriand": ("Boisbriand", ""), # Rive-Nord proche37 "sainte-therese": ("Sainte-Thérèse", ""), # Rive-Nord proche38 "laval": ("Laval", ""),39 "longueuil": ("Longueuil", ""),40 # Expansion provinciale (2026-08)41 "gatineau": ("Gatineau", ""),42 "hull": ("Gatineau", "Hull"),43 "sherbrooke": ("Sherbrooke", ""),44}4546_BED_TYPES = {"0": "Studio", "1": "3½", "2": "4½", "3": "5½", "4": "6½"}47_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder", re.I)48_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?")495051class _BudgetReached(Exception):52 """Plafond de requêtes Firecrawl atteint pour cette synchronisation."""535455class RealstarConnector(BaseConnector):56 source_id = "realstar"57 request_delay = 1.058 max_properties = 12 # garde-fou (2 appels Firecrawl par propriété)59 max_images = 2560 max_renders = 20 # plafond d'appels Firecrawl par sync (hors cache)6162 # -- Firecrawl avec attente de rendu (SPA + Cloudflare) -------------------63 def _rendered(self, url: str, wait_ms: int = 9000) -> str:64 # Clé absente : on tente quand même (le rejeu des fixtures intercepte65 # self.session ; en direct, Firecrawl répondra 401 -> erreur claire).66 key = os.environ.get("FIRECRAWL_API_KEY", "")67 # via self.session : l'enregistreur de fixtures capture la réponse68 resp = self.session.post(69 FIRECRAWL_API,70 json={"url": url, "formats": ["html"], "waitFor": wait_ms},71 headers={"Authorization": f"Bearer {key}"},72 timeout=150,73 )74 resp.raise_for_status()75 return (resp.json().get("data") or {}).get("html", "")7677 def fetch(self) -> list[Listing]:78 self._renders = 079 html = self._rendered(SEARCH_URL, 10000)80 soup = BeautifulSoup(html, "html.parser")81 cards = soup.select("li.property-box")82 if not cards: # rendu incomplet : une seconde chance83 html = self._rendered(SEARCH_URL, 15000)84 soup = BeautifulSoup(html, "html.parser")85 cards = soup.select("li.property-box")8687 listings: list[Listing] = []88 seen: set[str] = set()89 count = 090 for card in cards:91 try:92 a = card.select_one("a[href*='/apartments/qc/']")93 if not a:94 continue # autre province95 url = (a.get("href") or "").split("?")[0]96 url = url.replace("http://", "https://")97 m = re.search(r"/apartments/qc/([a-z0-9\-.]+)/([a-z0-9\-]+)",98 url)99 if not m or url in seen:100 continue101 seen.add(url)102 city_slug, slug = m.group(1), m.group(2)103 if city_slug not in _GM_CITIES:104 continue # ville QC non répertoriée dans _GM_CITIES105 if count >= self.max_properties:106 break107 count += 1108 listings.append(109 self._property_listing(card, url, city_slug, slug))110 except Exception:111 continue112 return listings113114 def _property_listing(self, card, url: str, city_slug: str,115 slug: str) -> Listing:116 city, sector = _GM_CITIES[city_slug]117 name = ""118 fav = card.select_one("[data-property]")119 if fav:120 name = (fav.get("data-property") or "").strip()121 if not name:122 h = card.select_one(".property-name a")123 if h:124 name = h.get_text(" ", strip=True)125 name = re.sub(r"\s*opens in a new tab\s*", "", name).strip()126 name = name or slug.replace("-", " ").title()127128 addr_el = card.select_one(".card-prop-address")129 address = addr_el.get_text(" ", strip=True) if addr_el else ""130131 meta = card.select_one(".card-bed-bath-rent")132 beds = baths = sqft = ""133 if meta:134 items = [li.get_text(" ", strip=True)135 for li in meta.select("li")]136 for it in items:137 if "Bed" in it:138 beds = it139 elif "Bath" in it:140 baths = it141 elif "Sq" in it:142 sqft = it143 unit_type = ""144 bm = re.match(r"^(\d)(?:\s|-)?.*Bed", beds or "")145 if bm and "-" not in beds.split("Bed")[0]:146 unit_type = _BED_TYPES.get(bm.group(1), "")147148 # Fourchette de prix « $1,645.00 - $2,630.00 »149 card_text = card.get_text(" ", strip=True)150 price = None151 price_label = ""152 pm = re.search(r"\$[\d,]+(?:\.\d{2})?(?:(?:\s*(?:-|to))+\s*"153 r"\$[\d,]+(?:\.\d{2})?)?", card_text)154 if pm:155 price_label = re.sub(r"\s*to\s*-\s*", " - ", pm.group(0))156 first = price_label.split("-")[0].replace("$", "").replace(157 ",", "").replace("to", "").strip()158 try:159 price = float(first)160 except ValueError:161 price = parse_price(price_label)162 if "-" in price_label:163 price_label = "À partir de " + price_label164165 # Téléphone du bureau de location (lien tel: structuré de la carte)166 phone = ""167 tel = card.select_one("a[href^='tel:']")168 if tel:169 tm = re.search(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})",170 tel.get("href", ""))171 if tm:172 phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"173174 # Vignette de la carte175 images: list[str] = []176 img = card.select_one("img[src*='rentcafe']")177 if img and img.get("src"):178 images.append(img["src"])179180 # Pages détail (fiche + plans) via cache BD : Firecrawl seulement si la181 # carte liste a changé (prix/dispo inclus dans le hash).182 key = hashlib.sha1(183 f"{name}|{address}|{beds}|{baths}|{sqft}|{price_label}"184 .encode("utf-8")).hexdigest()185 try:186 payload = self.detail(slug, key, lambda: self._fetch_detail(url))187 except _BudgetReached:188 payload = {}189 except Exception:190 payload = {}191192 desc = payload.get("description", "")193 amenities = list(payload.get("amenities") or [])194 for im in (payload.get("images") or []):195 if im not in images:196 images.append(im)197198 # Plans structurés -> disponibilité, prix « à partir de », superficie199 availability = ""200 area_sqft = None201 plans = payload.get("floorplans") or []202 avail_plans = [p for p in plans if p.get("available", 0) > 0]203 if plans:204 total = sum(p.get("available", 0) for p in avail_plans)205 if total > 0:206 availability = (f"{total} unité(s) disponible(s) — "207 + ", ".join(p["name"] for p in avail_plans[:6]))208 prices = [p["price"] for p in avail_plans209 if p.get("price") and 100 <= p["price"] <= 20000]210 if prices:211 price = min(prices)212 price_label = (f"À partir de {price:,.0f} $/mois"213 .replace(",", " ")214 if len(avail_plans) > 1 or len(prices) > 1215 else f"{price:,.0f} $/mois".replace(",", " "))216 if len(avail_plans) == 1 and avail_plans[0].get("sqft"):217 # une seule unité type disponible : sa superficie est fiable218 area_sqft = avail_plans[0]["sqft"]219 if avail_plans[0].get("unit_type"):220 unit_type = avail_plans[0]["unit_type"]221222 # Résumé des plans disponibles dans la description (texte fidèle)223 plan_bits = []224 for p in avail_plans[:8]:225 seg = p["name"]226 if p.get("sqft"):227 seg += f" ({p['sqft']:.0f} pi²)"228 if p.get("price"):229 seg += f" : {p['price']:,.0f} $/mois".replace(",", " ")230 plan_bits.append(seg)231232 details: dict = {}233 if phone:234 details["contact"] = {"phone": phone}235236 bits = [b for b in [beds, baths, sqft] if b]237 desc_parts = ([desc] if desc else []) + bits238 if plan_bits:239 desc_parts.append("Disponibles : " + " ; ".join(plan_bits))240 return Listing(241 source=self.source_id,242 external_id=slug,243 url=url,244 title=name,245 address=address,246 sector=sector,247 city=city,248 unit_type=unit_type,249 price=price,250 price_label=price_label,251 availability=availability,252 area_sqft=area_sqft,253 description=" — ".join(desc_parts)[:900],254 amenities=amenities,255 details=details,256 images=images[: self.max_images],257 )258259 # -- pages détail (fiche propriété + plans) --------------------------------260 def _fetch_detail(self, url: str) -> dict:261 """2 rendus Firecrawl : fiche (photos, description, points forts) et262 /floorplans (plans structurés). Appelé seulement hors cache."""263 if self._renders + 2 > self.max_renders:264 raise _BudgetReached()265 self._renders += 2266267 payload: dict = {"description": "", "amenities": [], "images": [],268 "floorplans": []}269 try:270 ph = self._rendered(url, 8000)271 psoup = BeautifulSoup(ph, "html.parser")272 for im in psoup.select("img[src*='resource.rentcafe.com']"):273 src = im.get("src", "")274 if src and not _SKIP_IMG.search(src) \275 and src not in payload["images"]:276 payload["images"].append(src)277 # description : premiers paragraphes substantiels278 paras = [p.get_text(" ", strip=True)279 for p in psoup.find_all("p")]280 paras = [p for p in paras if len(p) > 80]281 if paras:282 payload["description"] = " ".join(paras[:2])[:600]283 # points forts de la propriété (courtes mentions après le titre)284 text = psoup.get_text("\n", strip=True)285 hm = re.search(r"Points forts de la propri[ée]t[ée]\n(.*?)\n"286 r"(?:Photos|Emplacement|Votre)", text, re.S)287 if hm:288 amenities = []289 for t in hm.group(1).split("\n"):290 t = t.strip()291 if 2 < len(t) < 50 and t not in amenities:292 amenities.append(t)293 payload["amenities"] = amenities[:15]294 except Exception:295 pass296297 try:298 fh = self._rendered(url.rstrip("/") + "/floorplans", 10000)299 payload["floorplans"] = self._parse_floorplans(fh)300 except Exception:301 pass302 return payload303304 @staticmethod305 def _parse_floorplans(html: str) -> list[dict]:306 """Cartes de plans RentCafe : nom (« 4 ½ D »), chambres, pi², prix,307 nombre d'unités disponibles (structuré : .fp-availability)."""308 soup = BeautifulSoup(html, "html.parser")309 plans: list[dict] = []310 for cont in soup.select("div[id^='fp-container-']"):311 try:312 name_el = cont.select_one("span[data-selenium-id$='Name']")313 name = name_el.get_text(" ", strip=True) if name_el else ""314 if not name:315 continue316 avail = 0317 av_el = cont.select_one(".fp-availability")318 if av_el:319 am = re.search(r"(\d+)", av_el.get_text(" ", strip=True))320 if am:321 avail = int(am.group(1))322 sqft = None323 sq_el = cont.select_one("span[data-selenium-id$='SqFt']")324 if sq_el:325 sm = re.search(r"([\d,]{2,})\s*Pi",326 sq_el.get_text(" ", strip=True), re.I)327 if sm:328 v = float(sm.group(1).replace(",", ""))329 if 80 <= v <= 20000:330 sqft = v331 price = None332 pm = _PRICE_RE.search(cont.get_text(" ", strip=True))333 if pm:334 v = float(pm.group(0).replace("$", "").replace(",", ""))335 if 100 <= v <= 20000:336 price = v337 unit_type = ""338 um = re.match(r"^\s*(\d)\s*½", name)339 if um:340 n = int(um.group(1))341 unit_type = "6½+" if n >= 6 else f"{n}½"342 elif re.match(r"(?i)^\s*studio", name):343 unit_type = "Studio"344 plans.append({"name": name, "available": avail,345 "sqft": sqft, "price": price,346 "unit_type": unit_type})347 except Exception:348 continue349 return plans350