spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/metcap.py : connecteur MetCap Living (metcap.com)5# Site WordPress rendu serveur. La page /province/quebec liste les villes QC6# (province=115) ; chaque page « province-search-results » liste les7# immeubles (avec lat/lng dans l'attribut onclick=centerMap) et leurs types8# d'unités (« Montreal 2 Bedrooms from $1,819 »). Les fiches /apartment/...9# donnent le détail structuré : tableau « Suite Details » (statut, lits,10# sdb, pi²), listes « Building Amenities », « Rent Includes »,11# « Pet Friendly », contact du bureau de location, description bilingue et12# photos d'unité ; les fiches /property/... la galerie photo de l'immeuble.13# Fiches visitées via self.detail(...) (cache BD). Gestionnaire pancanadien :14# seules les villes du Grand Montréal sont couvertes.15# -----------------------------------------------------------------------------16from __future__ import annotations1718import hashlib19import re20import urllib.parse2122from bs4 import BeautifulSoup2324from ..schema import Listing, parse_price, strip_accents25from .base import BaseConnector2627BASE = "https://www.metcap.com"28QC_PROVINCE_URL = f"{BASE}/province/quebec?lang=en"2930# Villes admissibles (Grand Montréal), clés sans accents/minuscules31_GM_CITIES = {32 "montreal": ("Montréal", ""),33 "saint laurent": ("Montréal", "Saint-Laurent"),34 "st laurent": ("Montréal", "Saint-Laurent"),35 "saint lambert": ("Saint-Lambert", ""),36 "st lambert": ("Saint-Lambert", ""),37 "verdun": ("Montréal", "Verdun"),38 "lasalle": ("Montréal", "LaSalle"),39 "laval": ("Laval", ""),40 "longueuil": ("Longueuil", ""),41 "brossard": ("Brossard", ""),42 "pointe-claire": ("Pointe-Claire", ""),43 "dorval": ("Dorval", ""),44}4546_TYPE_MAP = [47 (re.compile(r"bachelor|studio", re.I), "Studio"),48 (re.compile(r"1\s*bed", re.I), "3½"),49 (re.compile(r"2\s*bed", re.I), "4½"),50 (re.compile(r"3\s*bed", re.I), "5½"),51 (re.compile(r"4\s*bed", re.I), "6½"),52]53_SKIP_IMG = re.compile(r"logo|icon|favicon|header|/map/|walk\.sc|sharethis",54 re.I)55_LATLNG_RE = re.compile(r"\{\s*lat:\s*(-?[\d.]+)\s*,\s*"56 r"lon:\s*(-?[\d.]+)\s*\}")57_PHONE_RE = re.compile(r"\(?\b([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-](\d{4})\b")58_EMAIL_RE = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.]+\b")5960# « Rent Includes » (texte anglais structuré) -> clés inclusions Lou-Ka61_INCLUDES_MAP = [62 (re.compile(r"heat", re.I), "heating"),63 (re.compile(r"hydro|electric", re.I), "electricity"),64 (re.compile(r"hot\s*water", re.I), "hot_water"),65 (re.compile(r"internet|wi-?fi", re.I), "internet"),66 (re.compile(r"cable", re.I), "cable"),67]686970class _CapAtteint(Exception):71 """Plafond de requêtes détail atteint pour cette synchronisation."""727374class MetcapConnector(BaseConnector):75 source_id = "metcap"76 request_delay = 0.677 max_units = 60 # garde-fou fiches unités78 max_images = 2579 max_real_details = 150 # vraies requêtes détail par sync (hits cache exclus)8081 def fetch(self) -> list[Listing]:82 self._real_details = 083 html = self.get(QC_PROVINCE_URL).text84 # Liens de villes QC : /province-search-results?...province=115&city=X85 cities = []86 for href in re.findall(r'href="(/province-search-results\?[^"]+)"',87 html):88 q = urllib.parse.parse_qs(urllib.parse.urlparse(89 href.replace("&", "&")).query)90 if (q.get("province") or [""])[0] != "115":91 continue92 city = (q.get("city") or [""])[0]93 if city and city not in cities:94 cities.append(city)9596 listings: list[Listing] = []97 count = 098 for city_name in cities:99 key = strip_accents(city_name.lower()).replace(".", "").strip()100 if key not in _GM_CITIES:101 continue # hors Grand Montréal (garde REIT pancanadien)102 city, sector = _GM_CITIES[key]103 try:104 page = self.get(105 f"{BASE}/province-search-results?lang=en&province=115"106 f"&city={urllib.parse.quote(city_name)}").text107 except Exception:108 continue109 soup = BeautifulSoup(page, "html.parser")110 for item in soup.select(".province-results__item"):111 try:112 block = item.select_one(".province-results__content")113 if not block:114 continue115 h2a = block.select_one("h2 a[href^='/property/']")116 if not h2a:117 continue118 address = h2a.get_text(" ", strip=True)119 prop_path = h2a.get("href", "").split("?")[0]120 # lat/lng de l'immeuble : onclick="centerMap(..., {lat, lon})"121 lat = lng = None122 lm = _LATLNG_RE.search(item.get("onclick", "") or "")123 if lm:124 lat, lng = float(lm.group(1)), float(lm.group(2))125 spans = block.select("p span.d-block")126 prop_name = ""127 if spans and not spans[0].find("a"):128 prop_name = spans[0].get_text(" ", strip=True)129 for a in block.select("a[href^='/apartment/']"):130 if count >= self.max_units:131 break132 count += 1133 text = a.get_text(" ", strip=True)134 lst = self._unit_listing(135 a.get("href", ""), text, address, prop_name,136 prop_path, city, sector, lat, lng)137 if lst:138 listings.append(lst)139 except Exception:140 continue141 return listings142143 # -- pages détail (via cache BD self.detail) -------------------------------144 def _gallery(self, prop_path: str) -> list[str]:145 """Galerie photo de la fiche immeuble (partagée entre unités)."""146 def _fetch() -> dict:147 if self._real_details >= self.max_real_details:148 raise _CapAtteint()149 self._real_details += 1150 imgs: list[str] = []151 ph = self.get(f"{BASE}{prop_path}?lang=en").text152 for u in re.findall(153 r'https://www\.metcap\.com/wp-content/uploads/'154 r'[^"\'\s\)]+\.(?:jpg|jpeg|png|webp)', ph):155 if not _SKIP_IMG.search(u) and u not in imgs:156 imgs.append(u)157 return {"images": imgs[: self.max_images]}158159 try:160 return self.detail(f"property:{prop_path}", prop_path,161 _fetch).get("images") or []162 except Exception:163 return []164165 def _unit_detail(self, slug: str, url: str, card_key: str) -> dict:166 """Fiche unité : tableau Suite Details, listes sidebar, contact,167 description, intersection et photos d'unité."""168 def _fetch() -> dict:169 if self._real_details >= self.max_real_details:170 raise _CapAtteint()171 self._real_details += 1172 html = self.get(url).text173 soup = BeautifulSoup(html, "html.parser")174 out: dict = {}175176 # Tableau « Suite Details » : Price/Status/Beds/Baths/Sq. Ft177 suites = []178 table = soup.select_one("table.table-listing")179 if table:180 for tr in table.select("tbody tr"):181 row = {td.get("data-title", "").strip():182 td.get_text(" ", strip=True)183 for td in tr.select("td") if td.get("data-title")}184 if row:185 suites.append(row)186 out["suites"] = suites187188 # Listes structurées de la barre latérale189 def _ul(titre: str) -> list[str]:190 h = soup.find("h2", string=re.compile(191 rf"^\s*{titre}\s*$", re.I))192 ul = h.find_next_sibling("ul") if h else None193 return ([li.get_text(" ", strip=True) for li in194 ul.select("li")] if ul else [])195196 out["building_amenities"] = _ul("Building Amenities")197 out["rent_includes"] = _ul("Rent Includes")198 out["pet_friendly"] = _ul("Pet Friendly")199 out["local_amenities"] = _ul("Local Amenities")200201 # Contact du bureau de location202 contact = soup.select_one(".listing-contact")203 if contact:204 ctxt = contact.get_text(" ", strip=True)205 pm = _PHONE_RE.search(ctxt)206 if pm:207 out["phone"] = f"{pm.group(1)}-{pm.group(2)}-{pm.group(3)}"208 em = _EMAIL_RE.search(ctxt)209 if em:210 out["email"] = em.group(0)211212 # Description (partie anglaise, avant l'avis de non-responsabilité)213 dm = re.search(r"<h2>Description</h2>(.*?)(?:<h2|<hr)", html, re.S)214 if dm:215 dtxt = re.sub(r"<[^>]+>", " ", dm.group(1))216 dtxt = re.sub(r"\s+", " ", dtxt).strip()217 dtxt = re.split(r"The safest way|Disclaimer", dtxt)[0]218 out["description"] = dtxt.strip()[:600]219220 # Intersection (en-tête de fiche)221 txt = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))222 im = re.search(r"Intersection:\s*([^|]{3,60}?)\s{0,2}Suite", txt)223 if im:224 out["intersection"] = im.group(1).strip()225226 # Photos de l'unité (carrousel span data-bg)227 imgs = re.findall(228 r'data-bg="(https://www\.metcap\.com/wp-content/uploads/'229 r'[^"]+\.(?:jpg|jpeg|png|webp))"', html)230 out["images"] = [u for u in dict.fromkeys(imgs)231 if not _SKIP_IMG.search(u)][: self.max_images]232 return out233234 try:235 return self.detail(slug, card_key, _fetch)236 except Exception:237 return {}238239 # -- construction d'une annonce --------------------------------------------240 def _unit_listing(self, href: str, card_text: str, address: str,241 prop_name: str, prop_path: str, city: str, sector: str,242 lat: float | None, lng: float | None) -> Listing | None:243 path = href.split("?")[0]244 slug = path.rstrip("/").split("/")[-1]245 if not slug:246 return None247 url = f"{BASE}{path}?lang=en"248249 unit_type = ""250 for rx, ut in _TYPE_MAP:251 if rx.search(card_text):252 unit_type = ut253 break254 price = parse_price(255 card_text.replace("from $", "").replace(",", "") + " $")256 price_label = ""257 pm = re.search(r'from \$[\d,.]+', card_text)258 if pm:259 price_label = pm.group(0).replace("from", "À partir de") + " /mois"260261 # Fiche unité (cache BD, clé = contenu de la carte liste)262 card_key = hashlib.sha1(263 f"{card_text}|{address}".encode("utf-8")).hexdigest()[:16]264 det = self._unit_detail(slug, url, card_key)265266 # Tableau Suite Details : statut, superficie (structurés à la source)267 availability = ""268 area_sqft: float | None = None269 bits: list[str] = []270 suites = det.get("suites") or []271 row = next((r for r in suites272 if (r.get("Status") or "").lower() == "available"),273 suites[0] if suites else None)274 if row:275 status = row.get("Status") or ""276 availability = {"Available": "Disponible",277 "Waiting List": "Liste d'attente",278 "Rented": "Loué"}.get(status, status)279 sq = re.sub(r"[^\d.]", "", row.get("Sq. Ft") or "")280 try:281 v = float(sq)282 if 80 <= v <= 20000:283 area_sqft = v284 except ValueError:285 pass286 beds, baths = row.get("Beds") or "", row.get("Baths") or ""287 if beds or baths:288 bits.append(" — ".join(x for x in [289 f"{beds} ch." if beds else "",290 f"{baths} sdb" if baths else ""] if x))291 if det.get("intersection") and not sector:292 bits.append(f"Intersection : {det['intersection']}")293294 # Commodités brutes (immeuble + inclusions), fidèles à la source295 amenities = list(dict.fromkeys(296 (det.get("building_amenities") or []) +297 (det.get("rent_includes") or [])))[:25]298299 # Inclusions structurées (« Rent Includes ») et animaux (« Pet Friendly »)300 details: dict = {}301 inclusions: dict = {}302 for item in det.get("rent_includes") or []:303 for rx, cle in _INCLUDES_MAP:304 if rx.search(item):305 inclusions[cle] = True306 if inclusions:307 details["inclusions"] = inclusions308 pets = None309 pf = " ".join(det.get("pet_friendly") or []).strip().lower()310 if pf.startswith("yes"):311 pets = "oui"312 elif pf.startswith("no"):313 pets = "non"314 contact = {k: det[k] for k in ("phone", "email") if det.get(k)}315 if contact:316 details["contact"] = contact317318 # Photos : unité d'abord, sinon galerie de l'immeuble319 images = det.get("images") or []320 if not images:321 images = self._gallery(prop_path)322323 desc = det.get("description") or ""324 title_type = re.sub(r"\s*from \$[\d,.].*$", "", card_text).strip()325 title = (f"{prop_name} — {title_type}" if prop_name326 else f"{address} — {title_type}")327 return Listing(328 source=self.source_id,329 external_id=slug,330 url=url,331 title=title,332 address=address,333 sector=sector,334 city=city,335 unit_type=unit_type,336 price=price,337 price_label=price_label,338 availability=availability,339 area_sqft=area_sqft,340 pets=pets,341 description=" — ".join([desc] + bits if desc else bits)[:600],342 amenities=amenities,343 details=details,344 images=images,345 lat=lat,346 lng=lng,347 )348