spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/copley.py : connecteur Groupe Copley (groupecopley.com)5#   Locations haut de gamme — Westmount, Mont-Royal, Saint-Laurent,6#   centre-ville de Montréal, NDG (le site couvre aussi Toronto/Ottawa,7#   exclus ici). Webflow CMS rendu serveur : /properties paginé8#   (?15d7d54c_page=N), cartes avec champs fs-cmsfilter-*, fiches9#   détaillées pour les photos.10# -----------------------------------------------------------------------------11from __future__ import annotations1213import hashlib14import re1516from bs4 import BeautifulSoup1718from ..schema import Listing, normalize_unit_type19from .base import BaseConnector2021BASE = "https://www.groupecopley.com"22LIST_URL = f"{BASE}/properties"2324# Dossier d'assets Webflow des photos d'annonces (≠ dossier du thème)25IMG_RE = re.compile(26    r"https://cdn\.prod\.website-files\.com/6449860fc17b160d22960284/"27    r"[^\"\s]+?\.(?:jpg|jpeg|png|webp)", re.I)2829# Quartiers de Montréal qui sont en fait des villes distinctes30_CITY_FROM_NEIGHBOURHOOD = {31    "westmount": "Westmount",32    "mount royal": "Mont-Royal",33    "town of mount royal": "Mont-Royal",34}353637def _bedrooms_to_type(raw: str) -> str:38    """0 → Studio, 1 → 3½, 2 → 4½, 3 → 5½, 4 → 6½."""39    m = re.search(r"\d+", raw or "")40    if not m:41        return normalize_unit_type(raw)42    n = int(m.group(0))43    return {0: "Studio", 1: "3½", 2: "4½", 3: "5½", 4: "6½"}.get(44        n, f"{n} chambres")454647class CopleyConnector(BaseConnector):48    source_id = "copley"49    request_delay = 0.650    max_pages = 15           # garde-fou de pagination51    max_details = 60         # garde-fou de fetch des fiches5253    def fetch(self) -> list[Listing]:54        listings: dict[str, Listing] = {}5556        # 1) Pages de la liste (Webflow pagine avec ?15d7d54c_page=N)57        for page_no in range(1, self.max_pages + 1):58            url = LIST_URL if page_no == 1 else f"{LIST_URL}?15d7d54c_page={page_no}"59            try:60                html = self.get(url).text61            except Exception:62                break63            soup = BeautifulSoup(html, "html.parser")64            items = soup.select("div.property_item")65            if not items:66                break67            new = 068            for it in items:69                try:70                    lst = self._parse_card(it)71                except Exception:72                    continue73                if lst and lst.external_id not in listings:74                    listings[lst.external_id] = lst75                    new += 176            # plus de page suivante annoncée -> stop77            if f"?15d7d54c_page={page_no + 1}" not in html:78                break79            if new == 0 and page_no > 1:80                break8182        # 2) Fiches détaillées (avec cache BD) : photos, description,83        #    commodités, disponibilité, chauffage/climatisation/stationnement84        for i, lst in enumerate(listings.values()):85            if i >= self.max_details:86                break87            key = hashlib.sha1("|".join([88                lst.price_label, lst.availability, lst.unit_type,89                str(lst.area_sqft),90            ]).encode("utf-8")).hexdigest()91            try:92                payload = self.detail(lst.external_id, key,93                                      lambda u=lst.url: self._fetch_detail(u))94            except Exception:95                continue96            if payload.get("images"):97                lst.images = payload["images"]98            if payload.get("description"):99                lst.description = payload["description"]100            if payload.get("amenities"):101                lst.amenities = list(dict.fromkeys(102                    lst.amenities + payload["amenities"]))103            if payload.get("availability"):104                lst.availability = payload["availability"]105            if payload.get("details"):106                lst.details = payload["details"]107108        return list(listings.values())109110    # -- fiche détaillée ---------------------------------------------------------111    def _fetch_detail(self, url: str) -> dict:112        detail = self.get(url).text113        payload: dict = {}114        imgs = [u for u in dict.fromkeys(IMG_RE.findall(detail))115                if "-p-" not in u          # variantes responsive116                and not re.search(r"logo|icon|favicon|comingsoon", u, re.I)]117        payload["images"] = imgs[:30]118        dsoup = BeautifulSoup(detail, "html.parser")119        rich = dsoup.select_one(".property-header_description, .w-richtext")120        if rich:121            payload["description"] = rich.get_text(" ", strip=True)[:600]122123        # Commodités : liste d'icônes (Laundry, Balcony, Pool, Gym…) +124        # caractéristiques principales (Heating/Cooling/Parking/Backyard)125        amenities = [d.get_text(" ", strip=True)126                     for d in dsoup.select(".property-header_features-item")]127        details: dict = {}128        for item in dsoup.select(".main-features_item"):129            lab_el = item.select_one(".text-weight-medium")130            val_el = item.select_one(".text-color-grey50")131            lab = lab_el.get_text(" ", strip=True) if lab_el else ""132            val = val_el.get_text(" ", strip=True) if val_el else ""133            if not lab:134                continue135            amenities.append(f"{lab}: {val}" if val else lab)136            low = lab.lower()137            if low == "cooling" and val and val.lower() not in ("no", "none"):138                details["ac"] = True         # « Central Air » etc. (structuré)139            elif low == "parking":140                details["parking"] = {"available": True}141        payload["amenities"] = list(dict.fromkeys(a for a in amenities if a))[:25]142        if details:143            payload["details"] = details144145        # Disponibilité : fil d'Ariane — la variante avec date (« Available146        # Jun 2026 ») est prioritaire sur le simple « Available ».147        bc = dsoup.select_one(".property-header_breadcrumb")148        if bc:149            tags = [t.get_text(" ", strip=True)150                    for t in bc.select(".property_availability-tag")]151            tags = [t for t in tags if t]152            dated = next((t for t in tags153                          if re.search(r"available\s+\S", t, re.I)), "")154            payload["availability"] = dated or (tags[0] if tags else "")155        return payload156157    # -- parsing d'une carte ---------------------------------------------------158    def _parse_card(self, it) -> Listing | None:159        link = it.select_one("a.property_item-link")160        if not link:161            return None162        href = (link.get("href") or "").split("?")[0]163        m = re.match(r"/properties/([\w\-%.]+)$", href)164        if not m:165            return None166        slug = m.group(1)167168        fields: dict[str, list[str]] = {}169        for f in it.select("[fs-cmsfilter-field]"):170            key = f.get("fs-cmsfilter-field", "")171            fields.setdefault(key, []).append(f.get_text(" ", strip=True))172173        cities = fields.get("city", [])174        raw_city = cities[-1] if cities else ""175        if raw_city.lower() != "montreal":176            return None                       # Toronto / Ottawa : hors périmètre177        neighbourhood = (fields.get("neighbourhood") or [""])[0]178        ptype = (fields.get("type") or [""])[0]179        if re.search(r"parking|stationnement|commercial|office|storage",180                     ptype, re.I):181            return None182        title = (fields.get("title") or [""])[0]183        bedrooms = (fields.get("bedrooms") or [""])[0]184        bathrooms = (fields.get("bathrooms") or [""])[0]185        available = (fields.get("available") or [""])[0].strip().lower()186187        # Superficie : bloc « 1573 sqft » (structuré, sans fs-cmsfilter-field)188        area = None189        for md in it.select(".property_meta-details"):190            t = md.get_text(" ", strip=True)191            m2 = re.match(r"^([\d,]+)\s*sqft$", t, re.I)192            if m2:193                try:194                    val = float(m2.group(1).replace(",", ""))195                    if 80 <= val <= 20000:196                        area = val197                except ValueError:198                    pass199                break200201        # Coordonnées embarquées pour le JS de la carte202        lat = lng = None203        lat_el = it.select_one(".data---latitude")204        lng_el = it.select_one(".data---longitude")205        try:206            lat = float(lat_el.get_text(strip=True)) if lat_el else None207            lng = float(lng_el.get_text(strip=True)) if lng_el else None208        except (TypeError, ValueError):209            lat = lng = None210211        price = None212        price_label = ""213        price_el = it.select_one(".property_item-price-text")214        if price_el:215            num = re.sub(r"[^\d.]", "", price_el.get_text(strip=True))216            if num:217                try:218                    val = float(num)219                    if 100 <= val <= 20000:220                        price = val221                        price_label = f"${num} / month"222                except ValueError:223                    pass224225        city = _CITY_FROM_NEIGHBOURHOOD.get(neighbourhood.lower(), "Montréal")226        sector = "" if city != "Montréal" else neighbourhood227        if sector.lower() == "downtown montreal":228            sector = "Centre-ville"229        elif sector.lower() == "nuns' island":230            sector = "Île-des-Sœurs"231232        img = it.select_one("img.property_image")233        images = [img["src"]] if img and img.get("src") else []234235        # Bandeau de la carte (« Available ») — la fiche précisera la date236        avail_el = it.select_one("a > .text-block")237        availability = (avail_el.get_text(" ", strip=True) if avail_el238                        else ("Disponible" if available == "true" else ""))239240        amenities = []241        if bathrooms:242            amenities.append(f"{bathrooms} salle(s) de bain")243244        return Listing(245            source=self.source_id,246            external_id=slug,247            url=f"{BASE}/properties/{slug}",248            title=title or slug.replace("-", " ").title(),249            address=f"{title}, {city}, QC" if title else "",250            sector=sector,251            city=city,252            unit_type=_bedrooms_to_type(bedrooms),253            price=price,254            price_label=price_label,255            availability=availability,256            area_sqft=area,257            amenities=amenities,258            images=images,259            lat=lat,260            lng=lng,261        )262