# -----------------------------------------------------------------------------
# Lou-Ka — Agrégateur de logements à louer (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/metcap.py : connecteur MetCap Living (metcap.com)
# Site WordPress rendu serveur. La page /province/quebec liste les villes QC
# (province=115) ; chaque page « province-search-results » liste les
# immeubles (avec lat/lng dans l'attribut onclick=centerMap) et leurs types
# d'unités (« Montreal 2 Bedrooms from $1,819 »). Les fiches /apartment/...
# donnent le détail structuré : tableau « Suite Details » (statut, lits,
# sdb, pi²), listes « Building Amenities », « Rent Includes »,
# « Pet Friendly », contact du bureau de location, description bilingue et
# photos d'unité ; les fiches /property/... la galerie photo de l'immeuble.
# Fiches visitées via self.detail(...) (cache BD). Gestionnaire pancanadien :
# seules les villes du Grand Montréal sont couvertes.
# -----------------------------------------------------------------------------
from __future__ import annotations
import hashlib
import re
import urllib.parse
from bs4 import BeautifulSoup
from ..schema import Listing, parse_price, strip_accents
from .base import BaseConnector
BASE = "https://www.metcap.com"
QC_PROVINCE_URL = f"{BASE}/province/quebec?lang=en"
# Villes admissibles (Grand Montréal), clés sans accents/minuscules
_GM_CITIES = {
"montreal": ("Montréal", ""),
"saint laurent": ("Montréal", "Saint-Laurent"),
"st laurent": ("Montréal", "Saint-Laurent"),
"saint lambert": ("Saint-Lambert", ""),
"st lambert": ("Saint-Lambert", ""),
"verdun": ("Montréal", "Verdun"),
"lasalle": ("Montréal", "LaSalle"),
"laval": ("Laval", ""),
"longueuil": ("Longueuil", ""),
"brossard": ("Brossard", ""),
"pointe-claire": ("Pointe-Claire", ""),
"dorval": ("Dorval", ""),
}
_TYPE_MAP = [
(re.compile(r"bachelor|studio", re.I), "Studio"),
(re.compile(r"1\s*bed", re.I), "3½"),
(re.compile(r"2\s*bed", re.I), "4½"),
(re.compile(r"3\s*bed", re.I), "5½"),
(re.compile(r"4\s*bed", re.I), "6½"),
]
_SKIP_IMG = re.compile(r"logo|icon|favicon|header|/map/|walk\.sc|sharethis",
re.I)
_LATLNG_RE = re.compile(r"\{\s*lat:\s*(-?[\d.]+)\s*,\s*"
r"lon:\s*(-?[\d.]+)\s*\}")
_PHONE_RE = re.compile(r"\(?\b([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-](\d{4})\b")
_EMAIL_RE = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.]+\b")
# « Rent Includes » (texte anglais structuré) -> clés inclusions Lou-Ka
_INCLUDES_MAP = [
(re.compile(r"heat", re.I), "heating"),
(re.compile(r"hydro|electric", re.I), "electricity"),
(re.compile(r"hot\s*water", re.I), "hot_water"),
(re.compile(r"internet|wi-?fi", re.I), "internet"),
(re.compile(r"cable", re.I), "cable"),
]
class _CapAtteint(Exception):
"""Plafond de requêtes détail atteint pour cette synchronisation."""
class MetcapConnector(BaseConnector):
source_id = "metcap"
request_delay = 0.6
max_units = 60 # garde-fou fiches unités
max_images = 25
max_real_details = 150 # vraies requêtes détail par sync (hits cache exclus)
def fetch(self) -> list[Listing]:
self._real_details = 0
html = self.get(QC_PROVINCE_URL).text
# Liens de villes QC : /province-search-results?...province=115&city=X
cities = []
for href in re.findall(r'href="(/province-search-results\?[^"]+)"',
html):
q = urllib.parse.parse_qs(urllib.parse.urlparse(
href.replace("&", "&")).query)
if (q.get("province") or [""])[0] != "115":
continue
city = (q.get("city") or [""])[0]
if city and city not in cities:
cities.append(city)
listings: list[Listing] = []
count = 0
for city_name in cities:
key = strip_accents(city_name.lower()).replace(".", "").strip()
if key not in _GM_CITIES:
continue # hors Grand Montréal (garde REIT pancanadien)
city, sector = _GM_CITIES[key]
try:
page = self.get(
f"{BASE}/province-search-results?lang=en&province=115"
f"&city={urllib.parse.quote(city_name)}").text
except Exception:
continue
soup = BeautifulSoup(page, "html.parser")
for item in soup.select(".province-results__item"):
try:
block = item.select_one(".province-results__content")
if not block:
continue
h2a = block.select_one("h2 a[href^='/property/']")
if not h2a:
continue
address = h2a.get_text(" ", strip=True)
prop_path = h2a.get("href", "").split("?")[0]
# lat/lng de l'immeuble : onclick="centerMap(..., {lat, lon})"
lat = lng = None
lm = _LATLNG_RE.search(item.get("onclick", "") or "")
if lm:
lat, lng = float(lm.group(1)), float(lm.group(2))
spans = block.select("p span.d-block")
prop_name = ""
if spans and not spans[0].find("a"):
prop_name = spans[0].get_text(" ", strip=True)
for a in block.select("a[href^='/apartment/']"):
if count >= self.max_units:
break
count += 1
text = a.get_text(" ", strip=True)
lst = self._unit_listing(
a.get("href", ""), text, address, prop_name,
prop_path, city, sector, lat, lng)
if lst:
listings.append(lst)
except Exception:
continue
return listings
# -- pages détail (via cache BD self.detail) -------------------------------
def _gallery(self, prop_path: str) -> list[str]:
"""Galerie photo de la fiche immeuble (partagée entre unités)."""
def _fetch() -> dict:
if self._real_details >= self.max_real_details:
raise _CapAtteint()
self._real_details += 1
imgs: list[str] = []
ph = self.get(f"{BASE}{prop_path}?lang=en").text
for u in re.findall(
r'https://www\.metcap\.com/wp-content/uploads/'
r'[^"\'\s\)]+\.(?:jpg|jpeg|png|webp)', ph):
if not _SKIP_IMG.search(u) and u not in imgs:
imgs.append(u)
return {"images": imgs[: self.max_images]}
try:
return self.detail(f"property:{prop_path}", prop_path,
_fetch).get("images") or []
except Exception:
return []
def _unit_detail(self, slug: str, url: str, card_key: str) -> dict:
"""Fiche unité : tableau Suite Details, listes sidebar, contact,
description, intersection et photos d'unité."""
def _fetch() -> dict:
if self._real_details >= self.max_real_details:
raise _CapAtteint()
self._real_details += 1
html = self.get(url).text
soup = BeautifulSoup(html, "html.parser")
out: dict = {}
# Tableau « Suite Details » : Price/Status/Beds/Baths/Sq. Ft
suites = []
table = soup.select_one("table.table-listing")
if table:
for tr in table.select("tbody tr"):
row = {td.get("data-title", "").strip():
td.get_text(" ", strip=True)
for td in tr.select("td") if td.get("data-title")}
if row:
suites.append(row)
out["suites"] = suites
# Listes structurées de la barre latérale
def _ul(titre: str) -> list[str]:
h = soup.find("h2", string=re.compile(
rf"^\s*{titre}\s*$", re.I))
ul = h.find_next_sibling("ul") if h else None
return ([li.get_text(" ", strip=True) for li in
ul.select("li")] if ul else [])
out["building_amenities"] = _ul("Building Amenities")
out["rent_includes"] = _ul("Rent Includes")
out["pet_friendly"] = _ul("Pet Friendly")
out["local_amenities"] = _ul("Local Amenities")
# Contact du bureau de location
contact = soup.select_one(".listing-contact")
if contact:
ctxt = contact.get_text(" ", strip=True)
pm = _PHONE_RE.search(ctxt)
if pm:
out["phone"] = f"{pm.group(1)}-{pm.group(2)}-{pm.group(3)}"
em = _EMAIL_RE.search(ctxt)
if em:
out["email"] = em.group(0)
# Description (partie anglaise, avant l'avis de non-responsabilité)
dm = re.search(r"Description
(.*?)(?:]+>", " ", dm.group(1))
dtxt = re.sub(r"\s+", " ", dtxt).strip()
dtxt = re.split(r"The safest way|Disclaimer", dtxt)[0]
out["description"] = dtxt.strip()[:600]
# Intersection (en-tête de fiche)
txt = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))
im = re.search(r"Intersection:\s*([^|]{3,60}?)\s{0,2}Suite", txt)
if im:
out["intersection"] = im.group(1).strip()
# Photos de l'unité (carrousel span data-bg)
imgs = re.findall(
r'data-bg="(https://www\.metcap\.com/wp-content/uploads/'
r'[^"]+\.(?:jpg|jpeg|png|webp))"', html)
out["images"] = [u for u in dict.fromkeys(imgs)
if not _SKIP_IMG.search(u)][: self.max_images]
return out
try:
return self.detail(slug, card_key, _fetch)
except Exception:
return {}
# -- construction d'une annonce --------------------------------------------
def _unit_listing(self, href: str, card_text: str, address: str,
prop_name: str, prop_path: str, city: str, sector: str,
lat: float | None, lng: float | None) -> Listing | None:
path = href.split("?")[0]
slug = path.rstrip("/").split("/")[-1]
if not slug:
return None
url = f"{BASE}{path}?lang=en"
unit_type = ""
for rx, ut in _TYPE_MAP:
if rx.search(card_text):
unit_type = ut
break
price = parse_price(
card_text.replace("from $", "").replace(",", "") + " $")
price_label = ""
pm = re.search(r'from \$[\d,.]+', card_text)
if pm:
price_label = pm.group(0).replace("from", "À partir de") + " /mois"
# Fiche unité (cache BD, clé = contenu de la carte liste)
card_key = hashlib.sha1(
f"{card_text}|{address}".encode("utf-8")).hexdigest()[:16]
det = self._unit_detail(slug, url, card_key)
# Tableau Suite Details : statut, superficie (structurés à la source)
availability = ""
area_sqft: float | None = None
bits: list[str] = []
suites = det.get("suites") or []
row = next((r for r in suites
if (r.get("Status") or "").lower() == "available"),
suites[0] if suites else None)
if row:
status = row.get("Status") or ""
availability = {"Available": "Disponible",
"Waiting List": "Liste d'attente",
"Rented": "Loué"}.get(status, status)
sq = re.sub(r"[^\d.]", "", row.get("Sq. Ft") or "")
try:
v = float(sq)
if 80 <= v <= 20000:
area_sqft = v
except ValueError:
pass
beds, baths = row.get("Beds") or "", row.get("Baths") or ""
if beds or baths:
bits.append(" — ".join(x for x in [
f"{beds} ch." if beds else "",
f"{baths} sdb" if baths else ""] if x))
if det.get("intersection") and not sector:
bits.append(f"Intersection : {det['intersection']}")
# Commodités brutes (immeuble + inclusions), fidèles à la source
amenities = list(dict.fromkeys(
(det.get("building_amenities") or []) +
(det.get("rent_includes") or [])))[:25]
# Inclusions structurées (« Rent Includes ») et animaux (« Pet Friendly »)
details: dict = {}
inclusions: dict = {}
for item in det.get("rent_includes") or []:
for rx, cle in _INCLUDES_MAP:
if rx.search(item):
inclusions[cle] = True
if inclusions:
details["inclusions"] = inclusions
pets = None
pf = " ".join(det.get("pet_friendly") or []).strip().lower()
if pf.startswith("yes"):
pets = "oui"
elif pf.startswith("no"):
pets = "non"
contact = {k: det[k] for k in ("phone", "email") if det.get(k)}
if contact:
details["contact"] = contact
# Photos : unité d'abord, sinon galerie de l'immeuble
images = det.get("images") or []
if not images:
images = self._gallery(prop_path)
desc = det.get("description") or ""
title_type = re.sub(r"\s*from \$[\d,.].*$", "", card_text).strip()
title = (f"{prop_name} — {title_type}" if prop_name
else f"{address} — {title_type}")
return Listing(
source=self.source_id,
external_id=slug,
url=url,
title=title,
address=address,
sector=sector,
city=city,
unit_type=unit_type,
price=price,
price_label=price_label,
availability=availability,
area_sqft=area_sqft,
pets=pets,
description=" — ".join([desc] + bits if desc else bits)[:600],
amenities=amenities,
details=details,
images=images,
lat=lat,
lng=lng,
)