spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/osgoode.py : connecteur Osgoode Properties (osgoodeproperties.com)5# Grand gestionnaire Ottawa-Gatineau ; côté Québec : 4 immeubles à Gatineau6# (Le 700 St Joseph, Le Faubourg de l'Île, Village Cité-des-Jeunes,7# Le Salaberry). Site RentCafe/Yardi protégé par Cloudflare (403 direct) :8# tout passe par Firecrawl, comme realstar.py.9# 1) pages recherche /1-bedroom|2-bedroom/qc/gatineau/apartments -> cartes10# propriétés (li.property-box : nom, adresse, lits/sdb/pi², fourchette11# de prix, téléphone, vignette) — les cartes hors Québec (liens /on/)12# sont ignorées ;13# 2) fiche propriété -> galerie resource.rentcafe.com ;14# 3) /floorplans -> plans structurés (nom, chambres, sdb, pi², prix) ;15# ⚠ contrairement à Realstar, AUCUN décompte d'unités disponibles n'est16# publié -> availability reste vide (rien d'inventé).17# Une annonce par propriété (uid stables). Les fiches passent par18# self.detail(...) (cache BD) avec budget Firecrawl par sync.19# -----------------------------------------------------------------------------20from __future__ import annotations2122import hashlib23import os24import re2526from bs4 import BeautifulSoup2728from ..schema import Listing, parse_price29from .base import FIRECRAWL_API, BaseConnector3031BASE = "https://www.osgoodeproperties.com"32# pages recherche par typologie (chaque carte affiche la fourchette complète33# de l'immeuble : l'union 1-2 chambres couvre tout le parc résidentiel)34SEARCH_PATHS = ["/1-bedroom/qc/gatineau/apartments",35 "/2-bedroom/qc/gatineau/apartments"]3637_BED_TYPES = {"0": "Studio", "1": "3½", "2": "4½", "3": "5½", "4": "6½"}38_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder", re.I)394041class _BudgetReached(Exception):42 """Plafond de requêtes Firecrawl atteint pour cette synchronisation."""434445class OsgoodeConnector(BaseConnector):46 source_id = "osgoode"47 request_delay = 1.048 max_properties = 1049 max_images = 2050 max_renders = 14 # 2 recherches + 2 par propriété (hors cache)5152 # -- Firecrawl avec attente de rendu (Cloudflare + SPA RentCafe) -----------53 def _rendered(self, url: str, wait_ms: int = 9000) -> str:54 key = os.environ.get("FIRECRAWL_API_KEY", "")55 # via self.session : l'enregistreur de fixtures capture la réponse56 resp = self.session.post(57 FIRECRAWL_API,58 json={"url": url, "formats": ["html"], "waitFor": wait_ms},59 headers={"Authorization": f"Bearer {key}"},60 timeout=150,61 )62 resp.raise_for_status()63 return (resp.json().get("data") or {}).get("html", "")6465 # -- fetch -----------------------------------------------------------------66 def fetch(self) -> list[Listing]:67 self._renders = 068 listings: list[Listing] = []69 seen: set[str] = set()70 for path in SEARCH_PATHS:71 try:72 html = self._rendered(BASE + path, 10000)73 self._renders += 174 except Exception:75 continue76 soup = BeautifulSoup(html, "html.parser")77 for card in soup.select("li.property-box"):78 try:79 a = card.select_one("a[href*='/apartments/qc/']")80 if not a:81 continue # propriété ontarienne : exclue82 url = (a.get("href") or "").split("?")[0].rstrip("/")83 m = re.search(r"/apartments/qc/([a-z0-9\-.]+)/([a-z0-9\-]+)",84 url)85 if not m or m.group(2) in seen:86 continue87 if len(seen) >= self.max_properties:88 break89 seen.add(m.group(2))90 listings.append(91 self._property_listing(card, url, m.group(2)))92 except Exception:93 continue94 return listings9596 # -- carte propriété ---------------------------------------------------------97 def _property_listing(self, card, url: str, slug: str) -> Listing:98 name = ""99 fav = card.select_one("[data-property]")100 if fav:101 name = (fav.get("data-property") or "").strip()102 if not name:103 h = card.select_one(".property-name a")104 if h:105 name = h.get_text(" ", strip=True)106 name = re.sub(r"\s*opens in a new tab\s*", "", name).strip()107 name = name or slug.replace("-", " ").title()108109 addr_el = card.select_one(".card-prop-address")110 address = addr_el.get_text(" ", strip=True) if addr_el else ""111112 meta = card.select_one(".card-bed-bath-rent")113 beds = baths = sqft = ""114 if meta:115 for li in meta.select("li"):116 it = li.get_text(" ", strip=True)117 if "Bed" in it:118 beds = it119 elif "Bath" in it:120 baths = it121 elif "Sq" in it:122 sqft = re.sub(r"\s*to\s*-\s*", " - ", it)123 unit_type = ""124 bm = re.match(r"^(\d)(?:\s|-)?.*Bed", beds or "")125 if bm and "-" not in beds.split("Bed")[0]:126 unit_type = _BED_TYPES.get(bm.group(1), "")127128 # fourchette « $1,015.00 to - $1,544.00 » de la carte129 price = None130 price_label = ""131 pm = re.search(r"\$[\d,]+(?:\.\d{2})?(?:(?:\s*(?:-|to))+\s*"132 r"\$[\d,]+(?:\.\d{2})?)?",133 card.get_text(" ", strip=True))134 if pm:135 price_label = re.sub(r"\s*to\s*-\s*", " - ", pm.group(0))136 first = (price_label.split("-")[0]137 .replace("$", "").replace(",", "").strip())138 try:139 price = float(first)140 except ValueError:141 price = parse_price(price_label)142 if "-" in price_label:143 price_label = "À partir de " + price_label144145 phone = ""146 tel = card.select_one("a[href^='tel:']")147 if tel:148 tm = re.search(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})",149 tel.get("href", ""))150 if tm:151 phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"152153 images: list[str] = []154 img = card.select_one("img[src*='rentcafe']")155 if img and img.get("src"):156 images.append(img["src"])157158 # fiche + plans via cache BD (clé = contenu de la carte liste)159 key = hashlib.sha1(160 f"{name}|{address}|{beds}|{baths}|{sqft}|{price_label}"161 .encode("utf-8")).hexdigest()162 try:163 payload = self.detail(slug, key, lambda: self._fetch_detail(url))164 except Exception:165 payload = {}166167 for im in (payload.get("images") or []):168 if im not in images:169 images.append(im)170171 # plans structurés : prix « à partir de » réel + résumé fidèle172 plans = payload.get("floorplans") or []173 prices = [p["price"] for p in plans174 if p.get("price") and 100 <= p["price"] <= 20000]175 if prices:176 price = min(prices)177 price_label = (f"À partir de {price:,.0f} $/mois"178 .replace(",", " ") if len(plans) > 1179 else f"{price:,.0f} $/mois".replace(",", " "))180 plan_bits = []181 for p in plans[:8]:182 seg = p["name"]183 if p.get("sqft"):184 seg += f" ({p['sqft']:.0f} pi²)"185 if p.get("price"):186 seg += f" : {p['price']:,.0f} $/mois".replace(",", " ")187 plan_bits.append(seg)188 if len(plans) == 1 and plans[0].get("unit_type"):189 unit_type = plans[0]["unit_type"]190191 details: dict = {}192 if phone:193 details["contact"] = {"phone": phone}194195 desc_parts = ([payload["description"]]196 if payload.get("description") else [])197 desc_parts += [b for b in [beds, baths, sqft] if b]198 if plan_bits:199 desc_parts.append("Plans : " + " ; ".join(plan_bits))200201 return Listing(202 source=self.source_id,203 external_id=slug,204 url=url,205 title=name,206 address=address,207 sector="", # le site ne publie pas le secteur par immeuble208 city="Gatineau",209 unit_type=unit_type,210 price=price,211 price_label=price_label,212 availability="", # aucun décompte d'unités publié213 description=" — ".join(desc_parts)[:900],214 details=details,215 images=images[: self.max_images],216 )217218 # -- pages détail (fiche + plans) --------------------------------------------219 def _fetch_detail(self, url: str) -> dict:220 if self._renders + 2 > self.max_renders:221 raise _BudgetReached()222 self._renders += 2223224 payload: dict = {"description": "", "images": [], "floorplans": []}225 try:226 psoup = BeautifulSoup(self._rendered(url, 8000), "html.parser")227 for im in psoup.select("img[src*='resource.rentcafe.com']"):228 src = im.get("src", "")229 if src and not _SKIP_IMG.search(src) \230 and src not in payload["images"]:231 payload["images"].append(src)232 paras = [p.get_text(" ", strip=True) for p in psoup.find_all("p")]233 paras = [p for p in paras if len(p) > 80]234 if paras:235 payload["description"] = " ".join(paras[:2])[:600]236 except Exception:237 pass238239 try:240 fh = self._rendered(url + "/floorplans", 10000)241 payload["floorplans"] = self._parse_floorplans(fh)242 except Exception:243 pass244 return payload245246 @staticmethod247 def _parse_floorplans(html: str) -> list[dict]:248 """Cartes de plans RentCafe : nom, chambres, pi², prix (ou fourchette,249 borne basse retenue). Pas de décompte de disponibilité chez Osgoode."""250 soup = BeautifulSoup(html, "html.parser")251 plans: list[dict] = []252 for cont in soup.select("div[id^='fp-container-']"):253 try:254 name_el = cont.select_one("span[data-selenium-id$='Name']")255 name = name_el.get_text(" ", strip=True) if name_el else ""256 if not name:257 continue258 plan: dict = {"name": name}259 beds_el = cont.select_one("span[data-selenium-id$='Beds']")260 if beds_el:261 bm = re.match(r"^\s*(\d)\s*Bed",262 beds_el.get_text(" ", strip=True))263 if bm:264 plan["unit_type"] = _BED_TYPES.get(bm.group(1), "")265 sq_el = cont.select_one("span[data-selenium-id$='SqFt']")266 if sq_el:267 sm = re.search(r"([\d,]{2,})",268 sq_el.get_text(" ", strip=True))269 if sm:270 v = float(sm.group(1).replace(",", ""))271 if 80 <= v <= 20000:272 plan["sqft"] = v273 rent_el = cont.select_one("span[data-selenium-id$='Rent']")274 if rent_el:275 rm = re.search(r"\$([\d,]+)(?:\.\d{2})?",276 rent_el.get_text(" ", strip=True))277 if rm:278 plan["price"] = float(rm.group(1).replace(",", ""))279 plans.append(plan)280 except Exception:281 continue282 return plans283