# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/copley.py : connecteur Groupe Copley (groupecopley.com) # Locations haut de gamme — Westmount, Mont-Royal, Saint-Laurent, # centre-ville de Montréal, NDG (le site couvre aussi Toronto/Ottawa, # exclus ici). Webflow CMS rendu serveur : /properties paginé # (?15d7d54c_page=N), cartes avec champs fs-cmsfilter-*, fiches # détaillées pour les photos. # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import re from bs4 import BeautifulSoup from ..schema import Listing, normalize_unit_type from .base import BaseConnector BASE = "https://www.groupecopley.com" LIST_URL = f"{BASE}/properties" # Dossier d'assets Webflow des photos d'annonces (≠ dossier du thème) IMG_RE = re.compile( r"https://cdn\.prod\.website-files\.com/6449860fc17b160d22960284/" r"[^\"\s]+?\.(?:jpg|jpeg|png|webp)", re.I) # Quartiers de Montréal qui sont en fait des villes distinctes _CITY_FROM_NEIGHBOURHOOD = { "westmount": "Westmount", "mount royal": "Mont-Royal", "town of mount royal": "Mont-Royal", } def _bedrooms_to_type(raw: str) -> str: """0 → Studio, 1 → 3½, 2 → 4½, 3 → 5½, 4 → 6½.""" m = re.search(r"\d+", raw or "") if not m: return normalize_unit_type(raw) n = int(m.group(0)) return {0: "Studio", 1: "3½", 2: "4½", 3: "5½", 4: "6½"}.get( n, f"{n} chambres") class CopleyConnector(BaseConnector): source_id = "copley" request_delay = 0.6 max_pages = 15 # garde-fou de pagination max_details = 60 # garde-fou de fetch des fiches def fetch(self) -> list[Listing]: listings: dict[str, Listing] = {} # 1) Pages de la liste (Webflow pagine avec ?15d7d54c_page=N) for page_no in range(1, self.max_pages + 1): url = LIST_URL if page_no == 1 else f"{LIST_URL}?15d7d54c_page={page_no}" try: html = self.get(url).text except Exception: break soup = BeautifulSoup(html, "html.parser") items = soup.select("div.property_item") if not items: break new = 0 for it in items: try: lst = self._parse_card(it) except Exception: continue if lst and lst.external_id not in listings: listings[lst.external_id] = lst new += 1 # plus de page suivante annoncée -> stop if f"?15d7d54c_page={page_no + 1}" not in html: break if new == 0 and page_no > 1: break # 2) Fiches détaillées (avec cache BD) : photos, description, # commodités, disponibilité, chauffage/climatisation/stationnement for i, lst in enumerate(listings.values()): if i >= self.max_details: break key = hashlib.sha1("|".join([ lst.price_label, lst.availability, lst.unit_type, str(lst.area_sqft), ]).encode("utf-8")).hexdigest() try: payload = self.detail(lst.external_id, key, lambda u=lst.url: self._fetch_detail(u)) except Exception: continue if payload.get("images"): lst.images = payload["images"] if payload.get("description"): lst.description = payload["description"] if payload.get("amenities"): lst.amenities = list(dict.fromkeys( lst.amenities + payload["amenities"])) if payload.get("availability"): lst.availability = payload["availability"] if payload.get("details"): lst.details = payload["details"] return list(listings.values()) # -- fiche détaillée --------------------------------------------------------- def _fetch_detail(self, url: str) -> dict: detail = self.get(url).text payload: dict = {} imgs = [u for u in dict.fromkeys(IMG_RE.findall(detail)) if "-p-" not in u # variantes responsive and not re.search(r"logo|icon|favicon|comingsoon", u, re.I)] payload["images"] = imgs[:30] dsoup = BeautifulSoup(detail, "html.parser") rich = dsoup.select_one(".property-header_description, .w-richtext") if rich: payload["description"] = rich.get_text(" ", strip=True)[:600] # Commodités : liste d'icônes (Laundry, Balcony, Pool, Gym…) + # caractéristiques principales (Heating/Cooling/Parking/Backyard) amenities = [d.get_text(" ", strip=True) for d in dsoup.select(".property-header_features-item")] details: dict = {} for item in dsoup.select(".main-features_item"): lab_el = item.select_one(".text-weight-medium") val_el = item.select_one(".text-color-grey50") lab = lab_el.get_text(" ", strip=True) if lab_el else "" val = val_el.get_text(" ", strip=True) if val_el else "" if not lab: continue amenities.append(f"{lab}: {val}" if val else lab) low = lab.lower() if low == "cooling" and val and val.lower() not in ("no", "none"): details["ac"] = True # « Central Air » etc. (structuré) elif low == "parking": details["parking"] = {"available": True} payload["amenities"] = list(dict.fromkeys(a for a in amenities if a))[:25] if details: payload["details"] = details # Disponibilité : fil d'Ariane — la variante avec date (« Available # Jun 2026 ») est prioritaire sur le simple « Available ». bc = dsoup.select_one(".property-header_breadcrumb") if bc: tags = [t.get_text(" ", strip=True) for t in bc.select(".property_availability-tag")] tags = [t for t in tags if t] dated = next((t for t in tags if re.search(r"available\s+\S", t, re.I)), "") payload["availability"] = dated or (tags[0] if tags else "") return payload # -- parsing d'une carte --------------------------------------------------- def _parse_card(self, it) -> Listing | None: link = it.select_one("a.property_item-link") if not link: return None href = (link.get("href") or "").split("?")[0] m = re.match(r"/properties/([\w\-%.]+)$", href) if not m: return None slug = m.group(1) fields: dict[str, list[str]] = {} for f in it.select("[fs-cmsfilter-field]"): key = f.get("fs-cmsfilter-field", "") fields.setdefault(key, []).append(f.get_text(" ", strip=True)) cities = fields.get("city", []) raw_city = cities[-1] if cities else "" if raw_city.lower() != "montreal": return None # Toronto / Ottawa : hors périmètre neighbourhood = (fields.get("neighbourhood") or [""])[0] ptype = (fields.get("type") or [""])[0] if re.search(r"parking|stationnement|commercial|office|storage", ptype, re.I): return None title = (fields.get("title") or [""])[0] bedrooms = (fields.get("bedrooms") or [""])[0] bathrooms = (fields.get("bathrooms") or [""])[0] available = (fields.get("available") or [""])[0].strip().lower() # Superficie : bloc « 1573 sqft » (structuré, sans fs-cmsfilter-field) area = None for md in it.select(".property_meta-details"): t = md.get_text(" ", strip=True) m2 = re.match(r"^([\d,]+)\s*sqft$", t, re.I) if m2: try: val = float(m2.group(1).replace(",", "")) if 80 <= val <= 20000: area = val except ValueError: pass break # Coordonnées embarquées pour le JS de la carte lat = lng = None lat_el = it.select_one(".data---latitude") lng_el = it.select_one(".data---longitude") try: lat = float(lat_el.get_text(strip=True)) if lat_el else None lng = float(lng_el.get_text(strip=True)) if lng_el else None except (TypeError, ValueError): lat = lng = None price = None price_label = "" price_el = it.select_one(".property_item-price-text") if price_el: num = re.sub(r"[^\d.]", "", price_el.get_text(strip=True)) if num: try: val = float(num) if 100 <= val <= 20000: price = val price_label = f"${num} / month" except ValueError: pass city = _CITY_FROM_NEIGHBOURHOOD.get(neighbourhood.lower(), "Montréal") sector = "" if city != "Montréal" else neighbourhood if sector.lower() == "downtown montreal": sector = "Centre-ville" elif sector.lower() == "nuns' island": sector = "Île-des-Sœurs" img = it.select_one("img.property_image") images = [img["src"]] if img and img.get("src") else [] # Bandeau de la carte (« Available ») — la fiche précisera la date avail_el = it.select_one("a > .text-block") availability = (avail_el.get_text(" ", strip=True) if avail_el else ("Disponible" if available == "true" else "")) amenities = [] if bathrooms: amenities.append(f"{bathrooms} salle(s) de bain") return Listing( source=self.source_id, external_id=slug, url=f"{BASE}/properties/{slug}", title=title or slug.replace("-", " ").title(), address=f"{title}, {city}, QC" if title else "", sector=sector, city=city, unit_type=_bedrooms_to_type(bedrooms), price=price, price_label=price_label, availability=availability, area_sqft=area, amenities=amenities, images=images, lat=lat, lng=lng, )