spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/cromwell.py : connecteur Cromwell Management (cromwellmgt.ca)5#   ~20 immeubles à Montréal (Plateau, Outremont, Westmount, centre-ville,6#   Côte-des-Neiges, Hampstead). Les unités individuelles avec prix sont7#   publiées sur le site jumeau cromwellmontreal.ca (WordPress + thème8#   Houzez, rendu serveur) : cartes .item-listing-wrap avec data-listid,9#   fiches /property/<slug>/ pour photos, statut et description.10# -----------------------------------------------------------------------------11from __future__ import annotations1213import hashlib14import re1516from bs4 import BeautifulSoup1718from ..schema import Listing, normalize_unit_type19from .base import BaseConnector2021BASE = "https://cromwellmontreal.ca"22LIST_URL = f"{BASE}/apartments-for-rent-montreal/"2324IMG_RE = re.compile(25    r"https://cromwellmontreal\.ca/wp-content/uploads/"26    r"[^\"\s\\]+?\.(?:jpg|jpeg|png|webp)", re.I)2728# Municipalités de l'île qui ne sont pas des arrondissements de Montréal29_INDEPENDENT_CITIES = {30    "westmount": "Westmount", "hampstead": "Hampstead",31    "mont-royal": "Mont-Royal", "mount royal": "Mont-Royal",32    "côte-saint-luc": "Côte-Saint-Luc", "cote-saint-luc": "Côte-Saint-Luc",33    "montréal-ouest": "Montréal-Ouest", "montreal west": "Montréal-Ouest",34}353637def _parse_price_us(raw: str) -> float | None:38    """'Starting at $1,995 /month' -> 1995.0 (symbole $ devant le nombre)."""39    m = re.search(r"\$\s*([\d,\s]+(?:\.\d{2})?)", raw or "")40    if not m:41        return None42    num = m.group(1).replace(",", "").replace(" ", "").replace(" ", "")43    try:44        val = float(num)45    except ValueError:46        return None47    return val if 100 <= val <= 20000 else None484950def _unit_type(ptype: str, beds: str) -> str:51    """'1 Bedroom (3 1/2)' -> 3½ ; sinon via nb de chambres (0.5 = studio)."""52    m = re.search(r"(\d)\s*1/2", ptype or "")53    if m:54        return f"{m.group(1)}½"55    if re.search(r"studio", ptype or "", re.I):56        return "Studio"57    m = re.search(r"[\d.]+", beds or "")58    if m:59        n = float(m.group(0))60        if n < 1:61            return "Studio"62        return {1: "3½", 2: "4½", 3: "5½", 4: "6½"}.get(int(n), f"{int(n)} chambres")63    return normalize_unit_type(ptype)646566class CromwellConnector(BaseConnector):67    source_id = "cromwell"68    request_delay = 0.669    max_pages = 8            # garde-fou de pagination70    max_details = 60         # garde-fou de fetch des fiches7172    def fetch(self) -> list[Listing]:73        listings: dict[str, Listing] = {}7475        # 1) Liste paginée des unités disponibles76        url = LIST_URL77        for _ in range(self.max_pages):78            try:79                html = self.get(url).text80            except Exception:81                break82            soup = BeautifulSoup(html, "html.parser")83            for it in soup.select("div.item-listing-wrap"):84                try:85                    lst = self._parse_card(it)86                except Exception:87                    continue88                if lst and lst.external_id not in listings:89                    listings[lst.external_id] = lst90            nxt = soup.select_one("a.page-link[rel=next], .pagination a.next,"91                                  " a[rel=next]")92            if not nxt or not nxt.get("href"):93                break94            url = nxt["href"]9596        # 2) Fiches détaillées (avec cache BD) : photos, description, statut,97        #    type exact, caractéristiques complètes, secteur98        for i, lst in enumerate(listings.values()):99            if i >= self.max_details:100                break101            key = hashlib.sha1("|".join([102                lst.title, lst.price_label, lst.address,103                ";".join(lst.amenities),104            ]).encode("utf-8")).hexdigest()105            try:106                payload = self.detail(lst.external_id, key,107                                      lambda u=lst.url: self._fetch_detail(u))108            except Exception:109                continue110            if payload.get("images"):111                lst.images = payload["images"]112            if payload.get("description"):113                lst.description = payload["description"]114            if payload.get("availability"):115                lst.availability = payload["availability"]116            if payload.get("unit_type"):117                lst.unit_type = payload["unit_type"]118            if payload.get("amenities"):119                lst.amenities = list(dict.fromkeys(120                    lst.amenities + payload["amenities"]))121            if payload.get("sector") and not lst.sector:122                lst.sector = payload["sector"]123124        return list(listings.values())125126    # -- fiche détaillée (thème Houzez) ------------------------------------------127    def _fetch_detail(self, url: str) -> dict:128        detail = self.get(url).text129        payload: dict = {}130        imgs = [u for u in dict.fromkeys(IMG_RE.findall(detail))131                if not re.search(r"logo|favicon|icon|-\d+x\d+\.", u, re.I)]132        if imgs:133            payload["images"] = imgs[:30]134        dsoup = BeautifulSoup(detail, "html.parser")135        og = dsoup.find("meta", attrs={"property": "og:description"})136        desc_el = dsoup.select_one("#property-description-wrap .block-content-wrap")137        if desc_el:138            payload["description"] = desc_el.get_text(" ", strip=True)[:600]139        elif og and og.get("content"):140            payload["description"] = og["content"].strip()[:600]141        labels = [a.get_text(" ", strip=True)142                  for a in dsoup.select(".property-labels-wrap a")]143        labels = list(dict.fromkeys(l for l in labels if l))144        if labels:145            payload["availability"] = ", ".join(labels)[:120].title()146147        # Bloc Détails : type exact (« 1 Bedroom (3 1/2) »), salles de bain,148        # statut (repli si aucune étiquette)149        for li in dsoup.select(".detail-wrap li"):150            txt = li.get_text(" ", strip=True)151            low = txt.lower()152            if low.startswith("property type"):153                ut = _unit_type(txt, "")154                if ut:155                    payload["unit_type"] = ut156            elif low.startswith("bathroom"):157                m = re.search(r"[\d.]+", txt)158                if m:159                    payload.setdefault("amenities", []).append(160                        f"{m.group(0)} Bathroom(s)")161            elif low.startswith("property status") and not labels:162                status = txt.split(None, 2)[-1] if len(txt.split()) > 2 else ""163                if status:164                    payload["availability"] = status.title()165166        # Caractéristiques complètes (Features : Elevator, Heating, Hot167        # Water, Laundry Room, Parking…) — plus riches que la carte liste168        feats = [li.get_text(" ", strip=True)169                 for li in dsoup.select(".property-features-wrap li")]170        feats = [f for f in dict.fromkeys(feats) if f][:25]171        if feats:172            payload["amenities"] = payload.get("amenities", []) + feats173174        # Bloc Adresse : « City/ Ville: Montreal, Plateau Mont-Royal »175        for li in dsoup.select(".property-address-wrap li"):176            txt = li.get_text(" ", strip=True)177            m = re.match(r"(?:City|Ville)[^:]*:\s*(.+)$", txt, re.I)178            if m:179                parts = [p.strip() for p in m.group(1).split(",")]180                if len(parts) >= 2 and parts[1]:181                    payload["sector"] = parts[1]182                break183        return payload184185    # -- parsing d'une carte ---------------------------------------------------186    def _parse_card(self, it) -> Listing | None:187        a = it.select_one('a[href*="/property/"]')188        if not a:189            return None190        url = a["href"].split("?")[0]191        m = re.search(r"/property/([a-z0-9\-]+)/?$", url)192        if not m:193            return None194        slug = m.group(1)195        lid = it.get("data-listid") or ""196        if not lid:197            el = it.select_one("[data-listid]")198            lid = el.get("data-listid") if el else ""199200        title_el = it.select_one(".item-title")201        addr_el = it.select_one(".item-address")202        price_el = it.select_one(".item-price")203        title = title_el.get_text(" ", strip=True) if title_el else slug204        address = addr_el.get_text(" ", strip=True) if addr_el else ""205        price_label = price_el.get_text(" ", strip=True) if price_el else ""206207        if re.search(r"parking|stationnement|commercial|garage", title, re.I):208            return None209210        beds = baths = ""211        amenities: list[str] = []212        for li in it.select(".item-amenities li"):213            txt = li.get_text(" ", strip=True)214            if re.match(r"bed", txt, re.I):215                beds = txt216            elif re.match(r"bath", txt, re.I):217                baths = txt218            elif txt:219                amenities.append(txt)220221        # « 3605 Rue Saint-Urbain, Montréal, QC, Canada » -> secteur/ville.222        # NB : « Mont-Royal » dans un titre désigne l'avenue/le Plateau, pas VMR ;223        # on ne détecte les villes défusionnées que dans l'adresse (+ Westmount/224        # Hampstead dans le titre, non ambigus).225        sector, city = "", "Montréal"226        parts = [p.strip() for p in address.split(",")]227        locality = parts[1] if len(parts) >= 2 else ""228        city = _INDEPENDENT_CITIES.get(locality.lower(), "")229        if not city:230            for key in ("westmount", "hampstead"):231                if key in f"{address} {title}".lower():232                    city = _INDEPENDENT_CITIES[key]233                    break234        if not city:235            city = "Montréal"236            if (locality and not locality.lower().startswith(("montr", "qc"))237                    and not re.search(r"\d", locality)):238                sector = locality239240        # secteur depuis le titre si absent (quartiers connus de Cromwell)241        if city == "Montréal" and not sector:242            m2 = re.search(r"(Plateau(?:\s+Mont-Royal)?|Outremont|Downtown|"243                           r"Golden Square Mile|Mile End|C[oô]te-des-Neiges|"244                           r"Snowdon)", title, re.I)245            if m2:246                sector = m2.group(1)247248        images: list[str] = []249        img = it.select_one("img[data-src], img[src^='https']")250        if img:251            src = img.get("data-src") or img.get("src") or ""252            if src.startswith("https"):253                images.append(src)254255        return Listing(256            source=self.source_id,257            external_id=lid or slug,258            url=url,259            title=title,260            address=address,261            sector=sector,262            city=city,263            unit_type=_unit_type(title, beds),264            price=_parse_price_us(price_label),265            price_label=price_label,266            availability="",267            amenities=amenities,268            images=images,269        )270