# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/cromwell.py : connecteur Cromwell Management (cromwellmgt.ca) # ~20 immeubles à Montréal (Plateau, Outremont, Westmount, centre-ville, # Côte-des-Neiges, Hampstead). Les unités individuelles avec prix sont # publiées sur le site jumeau cromwellmontreal.ca (WordPress + thème # Houzez, rendu serveur) : cartes .item-listing-wrap avec data-listid, # fiches /property// pour photos, statut et description. # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import re from bs4 import BeautifulSoup from ..schema import Listing, normalize_unit_type from .base import BaseConnector BASE = "https://cromwellmontreal.ca" LIST_URL = f"{BASE}/apartments-for-rent-montreal/" IMG_RE = re.compile( r"https://cromwellmontreal\.ca/wp-content/uploads/" r"[^\"\s\\]+?\.(?:jpg|jpeg|png|webp)", re.I) # Municipalités de l'île qui ne sont pas des arrondissements de Montréal _INDEPENDENT_CITIES = { "westmount": "Westmount", "hampstead": "Hampstead", "mont-royal": "Mont-Royal", "mount royal": "Mont-Royal", "côte-saint-luc": "Côte-Saint-Luc", "cote-saint-luc": "Côte-Saint-Luc", "montréal-ouest": "Montréal-Ouest", "montreal west": "Montréal-Ouest", } def _parse_price_us(raw: str) -> float | None: """'Starting at $1,995 /month' -> 1995.0 (symbole $ devant le nombre).""" m = re.search(r"\$\s*([\d,\s]+(?:\.\d{2})?)", raw or "") if not m: return None num = m.group(1).replace(",", "").replace(" ", "").replace(" ", "") try: val = float(num) except ValueError: return None return val if 100 <= val <= 20000 else None def _unit_type(ptype: str, beds: str) -> str: """'1 Bedroom (3 1/2)' -> 3½ ; sinon via nb de chambres (0.5 = studio).""" m = re.search(r"(\d)\s*1/2", ptype or "") if m: return f"{m.group(1)}½" if re.search(r"studio", ptype or "", re.I): return "Studio" m = re.search(r"[\d.]+", beds or "") if m: n = float(m.group(0)) if n < 1: return "Studio" return {1: "3½", 2: "4½", 3: "5½", 4: "6½"}.get(int(n), f"{int(n)} chambres") return normalize_unit_type(ptype) class CromwellConnector(BaseConnector): source_id = "cromwell" request_delay = 0.6 max_pages = 8 # garde-fou de pagination max_details = 60 # garde-fou de fetch des fiches def fetch(self) -> list[Listing]: listings: dict[str, Listing] = {} # 1) Liste paginée des unités disponibles url = LIST_URL for _ in range(self.max_pages): try: html = self.get(url).text except Exception: break soup = BeautifulSoup(html, "html.parser") for it in soup.select("div.item-listing-wrap"): try: lst = self._parse_card(it) except Exception: continue if lst and lst.external_id not in listings: listings[lst.external_id] = lst nxt = soup.select_one("a.page-link[rel=next], .pagination a.next," " a[rel=next]") if not nxt or not nxt.get("href"): break url = nxt["href"] # 2) Fiches détaillées (avec cache BD) : photos, description, statut, # type exact, caractéristiques complètes, secteur for i, lst in enumerate(listings.values()): if i >= self.max_details: break key = hashlib.sha1("|".join([ lst.title, lst.price_label, lst.address, ";".join(lst.amenities), ]).encode("utf-8")).hexdigest() try: payload = self.detail(lst.external_id, key, lambda u=lst.url: self._fetch_detail(u)) except Exception: continue if payload.get("images"): lst.images = payload["images"] if payload.get("description"): lst.description = payload["description"] if payload.get("availability"): lst.availability = payload["availability"] if payload.get("unit_type"): lst.unit_type = payload["unit_type"] if payload.get("amenities"): lst.amenities = list(dict.fromkeys( lst.amenities + payload["amenities"])) if payload.get("sector") and not lst.sector: lst.sector = payload["sector"] return list(listings.values()) # -- fiche détaillée (thème Houzez) ------------------------------------------ def _fetch_detail(self, url: str) -> dict: detail = self.get(url).text payload: dict = {} imgs = [u for u in dict.fromkeys(IMG_RE.findall(detail)) if not re.search(r"logo|favicon|icon|-\d+x\d+\.", u, re.I)] if imgs: payload["images"] = imgs[:30] dsoup = BeautifulSoup(detail, "html.parser") og = dsoup.find("meta", attrs={"property": "og:description"}) desc_el = dsoup.select_one("#property-description-wrap .block-content-wrap") if desc_el: payload["description"] = desc_el.get_text(" ", strip=True)[:600] elif og and og.get("content"): payload["description"] = og["content"].strip()[:600] labels = [a.get_text(" ", strip=True) for a in dsoup.select(".property-labels-wrap a")] labels = list(dict.fromkeys(l for l in labels if l)) if labels: payload["availability"] = ", ".join(labels)[:120].title() # Bloc Détails : type exact (« 1 Bedroom (3 1/2) »), salles de bain, # statut (repli si aucune étiquette) for li in dsoup.select(".detail-wrap li"): txt = li.get_text(" ", strip=True) low = txt.lower() if low.startswith("property type"): ut = _unit_type(txt, "") if ut: payload["unit_type"] = ut elif low.startswith("bathroom"): m = re.search(r"[\d.]+", txt) if m: payload.setdefault("amenities", []).append( f"{m.group(0)} Bathroom(s)") elif low.startswith("property status") and not labels: status = txt.split(None, 2)[-1] if len(txt.split()) > 2 else "" if status: payload["availability"] = status.title() # Caractéristiques complètes (Features : Elevator, Heating, Hot # Water, Laundry Room, Parking…) — plus riches que la carte liste feats = [li.get_text(" ", strip=True) for li in dsoup.select(".property-features-wrap li")] feats = [f for f in dict.fromkeys(feats) if f][:25] if feats: payload["amenities"] = payload.get("amenities", []) + feats # Bloc Adresse : « City/ Ville: Montreal, Plateau Mont-Royal » for li in dsoup.select(".property-address-wrap li"): txt = li.get_text(" ", strip=True) m = re.match(r"(?:City|Ville)[^:]*:\s*(.+)$", txt, re.I) if m: parts = [p.strip() for p in m.group(1).split(",")] if len(parts) >= 2 and parts[1]: payload["sector"] = parts[1] break return payload # -- parsing d'une carte --------------------------------------------------- def _parse_card(self, it) -> Listing | None: a = it.select_one('a[href*="/property/"]') if not a: return None url = a["href"].split("?")[0] m = re.search(r"/property/([a-z0-9\-]+)/?$", url) if not m: return None slug = m.group(1) lid = it.get("data-listid") or "" if not lid: el = it.select_one("[data-listid]") lid = el.get("data-listid") if el else "" title_el = it.select_one(".item-title") addr_el = it.select_one(".item-address") price_el = it.select_one(".item-price") title = title_el.get_text(" ", strip=True) if title_el else slug address = addr_el.get_text(" ", strip=True) if addr_el else "" price_label = price_el.get_text(" ", strip=True) if price_el else "" if re.search(r"parking|stationnement|commercial|garage", title, re.I): return None beds = baths = "" amenities: list[str] = [] for li in it.select(".item-amenities li"): txt = li.get_text(" ", strip=True) if re.match(r"bed", txt, re.I): beds = txt elif re.match(r"bath", txt, re.I): baths = txt elif txt: amenities.append(txt) # « 3605 Rue Saint-Urbain, Montréal, QC, Canada » -> secteur/ville. # NB : « Mont-Royal » dans un titre désigne l'avenue/le Plateau, pas VMR ; # on ne détecte les villes défusionnées que dans l'adresse (+ Westmount/ # Hampstead dans le titre, non ambigus). sector, city = "", "Montréal" parts = [p.strip() for p in address.split(",")] locality = parts[1] if len(parts) >= 2 else "" city = _INDEPENDENT_CITIES.get(locality.lower(), "") if not city: for key in ("westmount", "hampstead"): if key in f"{address} {title}".lower(): city = _INDEPENDENT_CITIES[key] break if not city: city = "Montréal" if (locality and not locality.lower().startswith(("montr", "qc")) and not re.search(r"\d", locality)): sector = locality # secteur depuis le titre si absent (quartiers connus de Cromwell) if city == "Montréal" and not sector: m2 = re.search(r"(Plateau(?:\s+Mont-Royal)?|Outremont|Downtown|" r"Golden Square Mile|Mile End|C[oô]te-des-Neiges|" r"Snowdon)", title, re.I) if m2: sector = m2.group(1) images: list[str] = [] img = it.select_one("img[data-src], img[src^='https']") if img: src = img.get("data-src") or img.get("src") or "" if src.startswith("https"): images.append(src) return Listing( source=self.source_id, external_id=lid or slug, url=url, title=title, address=address, sector=sector, city=city, unit_type=_unit_type(title, beds), price=_parse_price_us(price_label), price_label=price_label, availability="", amenities=amenities, images=images, )