spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/rentalys.py : connecteur Rentalys (location.rentalys.ca)5#   Agence de location du Grand Montréal (Verdun, Villeray, Rosemont,6#   Saint-Michel, Longueuil...). Site WordPress (thème Houzez) rendu7#   serveur : archives /property/page/N/ = cartes complètes (prix, adresse,8#   étiquettes de disponibilité), pages /property/<slug>/ = photos, détails,9#   description bilingue, superficie structurée (.h-area) et coordonnées10#   lat/lng (variable JS houzez_single_property_map). Les fiches passent par11#   self.detail(...) (cache BD) : re-téléchargées seulement si la carte change.12# -----------------------------------------------------------------------------13from __future__ import annotations1415import hashlib16import json17import re1819from bs4 import BeautifulSoup2021from ..schema import Listing, normalize_unit_type, parse_price, strip_accents22from .base import BaseConnector2324BASE = "https://location.rentalys.ca"2526# Arrondissements/quartiers de l'île de Montréal (partie avant ", QC" de27# l'adresse Houzez) -> ville Montréal. Les autres villes du Grand Montréal28# (Longueuil, Laval, Brossard...) restent des villes à part entière.29MTL_BOROUGHS = {30    "montreal", "verdun", "villeray", "rosemont", "la petite-patrie",31    "petite-patrie", "rosemont/la petite patrie", "saint-michel", "st-michel",32    "ville-marie", "le plateau-mont-royal", "plateau-mont-royal", "plateau",33    "hochelaga-maisonneuve", "mercier", "ahuntsic", "ahuntsic-cartierville",34    "cote-des-neiges", "notre-dame-de-grace", "ndg", "outremont", "lachine",35    "lasalle", "saint-laurent", "st-laurent", "anjou", "saint-leonard",36    "st-leonard", "montreal-nord", "riviere-des-prairies", "pointe-aux-trembles",37    "griffintown", "le sud-ouest", "sud-ouest", "pointe-saint-charles",38    "saint-henri", "st-henri", "viau", "westmount", "mont-royal",39}4041GRAND_MTL_CITIES = {42    "longueuil", "laval", "brossard", "saint-lambert", "boucherville",43    "candiac", "chateauguay", "repentigny", "terrebonne", "dorval",44    "pointe-claire", "dollard-des-ormeaux", "kirkland", "beaconsfield",45    "sainte-julie", "varennes", "chambly", "la prairie", "saint-constant",46    "sainte-catherine", "delson", "blainville", "mirabel", "saint-eustache",47    "deux-montagnes", "rosemere", "boisbriand", "sainte-therese",48    "vaudreuil-dorion", "charlemagne", "mascouche", "l'assomption",49}505152def _city_sector(address: str, title: str) -> tuple[str, str]:53    """Déduit (ville, secteur) de l'adresse Houzez '..., Secteur, QC, Canada'."""54    sector = ""55    parts = [p.strip() for p in (address or "").split(",")]56    # partie juste avant "QC"57    for i, p in enumerate(parts):58        if p.upper().startswith("QC") and i > 0:59            sector = parts[i - 1]60            break61    if not sector and len(parts) >= 2:62        sector = parts[1]63    key = strip_accents(sector.lower())64    if key in GRAND_MTL_CITIES:65        return sector, ""                      # ville de banlieue, pas de secteur66    if key in MTL_BOROUGHS or key.replace("st-", "saint-") in MTL_BOROUGHS:67        return "Montréal", "" if key == "montreal" else sector68    # repli : préfixe du titre ("VERDUN – ...", "LONGUEUIL maison ...")69    t = strip_accents((title or "").lower())70    for c in GRAND_MTL_CITIES:71        if t.startswith(c):72            return sector or c.title(), ""73    return "Montréal", sector747576def _unit_type(title: str, beds: str) -> str:77    t = normalize_unit_type(title)78    if re.match(r"^\d½$", t) or t in ("Studio", "Loft", "Maison", "Chambre"):79        return t80    m = re.search(r"(\d)\s*(?:1/2|½|%c2%bd)", (title or "").lower())81    if m:82        return f"{m.group(1)}½"83    if "maison" in (title or "").lower():84        return "Maison"85    if "studio" in (title or "").lower():86        return "Studio"87    try:88        n = int(str(beds).strip())89        return "Studio" if n == 0 else f"{n + 2}½"90    except (TypeError, ValueError):91        return ""929394def _parse_houzez_price(label: str) -> float | None:95    """'$2,177/mois' -> 2177.0 (format nord-américain, $ devant)."""96    m = re.search(r"\$\s*([\d][\d,\.\s]*)", label or "")97    if not m:98        return parse_price(label)99    try:100        val = float(m.group(1).replace(",", "").replace(" ", ""))101    except ValueError:102        return None103    return val if 100 <= val <= 20000 else None104105106class _BudgetReached(Exception):107    """Plafond de requêtes « fiche » atteint pour cette synchronisation."""108109110class RentalysConnector(BaseConnector):111    source_id = "rentalys"112    request_delay = 0.6113    max_pages = 10           # garde-fou d'archives114    max_details = 60         # plafond de vraies requêtes fiche par sync115116    def fetch(self) -> list[Listing]:117        listings: dict[str, Listing] = {}118119        for page in range(1, self.max_pages + 1):120            url = f"{BASE}/property/" if page == 1 \121                else f"{BASE}/property/page/{page}/"122            try:123                html = self.get(url).text124            except Exception:125                break126            found = self._parse_archive(html, listings)127            if not found:128                break129130        # Fiches détaillées (cache BD) : photos, description, lat/lng, pi²131        self._fetches = 0132        for lst in listings.values():133            key = hashlib.sha1(134                f"{lst.title}|{lst.price_label}|{lst.availability}|"135                f"{'|'.join(lst.amenities)}".encode("utf-8")).hexdigest()136            try:137                payload = self.detail(138                    lst.external_id, key,139                    lambda u=lst.url: self._fetch_detail(u))140            except Exception:      # _BudgetReached inclus : rien de caché141                continue142            self._apply_detail(lst, payload)143144        return list(listings.values())145146    # -- archive ------------------------------------------------------------------147    def _parse_archive(self, html: str, listings: dict[str, Listing]) -> int:148        soup = BeautifulSoup(html, "html.parser")149        found = 0150        for card in soup.select(".item-listing-wrap"):151            try:152                a = card.select_one('a[href*="/property/"]')153                if not a:154                    continue155                href = (a.get("href") or "").split("?")[0]156                m = re.search(r"/property/([^/]+)/?$", href)157                if not m:158                    continue159                slug = m.group(1)160                found += 1161                if slug in listings:162                    continue163164                title_el = card.select_one(".item-title")165                title = title_el.get_text(" ", strip=True) if title_el else slug166                # exclure stationnements / commercial167                if re.search(r"parking|stationnement|commercial|garage",168                             title, re.I):169                    continue170171                addr_el = card.select_one(".item-address")172                address = addr_el.get_text(" ", strip=True) if addr_el else ""173                price_el = card.select_one(".item-price")174                price_label = price_el.get_text(" ", strip=True) \175                    if price_el else ""176177                labels = [el.get_text(" ", strip=True)178                          for el in card.select(".hz-label, .label-status,"179                                                " .labels-wrap a")]180                labels = [l for l in dict.fromkeys(labels) if l]181                availability = next(182                    (l for l in labels if re.search(r"disponible", l, re.I)), "")183                amenities = [l for l in labels184                             if not re.search(r"disponible", l, re.I)]185186                beds = baths = area = ""187                for amen in card.select(".item-amenities li"):188                    txt = amen.get_text(" ", strip=True)189                    if re.search(r"Lits?|Bed", txt, re.I):190                        beds = re.sub(r"\D", "", txt)191                    elif re.search(r"Bains?|Bath", txt, re.I):192                        baths = re.sub(r"\D", "", txt)193                    elif re.search(r"\d{3,}", txt):194                        area = re.search(r"(\d[\d,\s]*)", txt).group(1)195                if baths:196                    amenities.append(f"{baths} salle(s) de bain")197                if area:198                    amenities.append(f"{area.strip()} pi²")199200                city, sector = _city_sector(address, title)201                # repli secteur : préfixe du titre ("SAINT-MICHEL – ...")202                if not sector and city == "Montréal":203                    mt = re.match(r"^([A-ZÉÈÀÂÎÔÛÇ/\-\s\.]{3,30})[–\-—]",204                                  title)205                    if mt:206                        sector = mt.group(1).strip().title()207                img = card.select_one("img")208                cover = (img.get("src") or img.get("data-src") or "") \209                    if img else ""210211                listings[slug] = Listing(212                    source=self.source_id,213                    external_id=slug,214                    url=href,215                    title=title,216                    address=address.replace(", Canada", ""),217                    sector=sector,218                    city=city,219                    unit_type=_unit_type(title, beds),220                    price=_parse_houzez_price(price_label),221                    price_label=price_label,222                    availability=availability,223                    amenities=amenities,224                    images=[cover] if cover.startswith("http") else [],225                )226            except Exception:227                continue228        return found229230    # -- fiche --------------------------------------------------------------------231    def _fetch_detail(self, url: str) -> dict:232        """Télécharge une fiche /property/<slug>/ (appelé seulement hors cache)."""233        if self._fetches >= self.max_details:234            raise _BudgetReached()235        self._fetches += 1236        html = self.get(url).text237        payload: dict = {}238239        imgs = re.findall(240            r'https://location\.rentalys\.ca/wp-content/uploads/'241            r'[^"\s\\\)]+\.(?:jpg|jpeg|png|webp)', html)242        imgs = [u for u in dict.fromkeys(imgs)243                if not re.search(r"logo|icon|favicon|avatar|-\d{2,3}x\d{2,3}\.",244                                 u, re.I)]245        if imgs:246            payload["images"] = imgs[:40]247248        soup = BeautifulSoup(html, "html.parser")249        desc = soup.select_one("#property-description-wrap .block-content-wrap")250        if desc:251            payload["description"] = desc.get_text(" ", strip=True)[:900]252        else:253            og = soup.find("meta", attrs={"property": "og:description"})254            if og and og.get("content"):255                payload["description"] = og["content"].strip()[:900]256257        # Superficie structurée (bloc « Caractéristiques clés », libellé .h-area)258        area_ul = soup.select_one(".property-overview-data .h-area")259        if area_ul:260            strong = area_ul.find_parent("ul")261            strong = strong.find("strong") if strong else None262            if strong:263                m = re.search(r"([\d\s,]+)", strong.get_text(strip=True))264                if m:265                    try:266                        v = float(m.group(1).replace(",", "").replace(" ", ""))267                        if 80 <= v <= 20000:268                            payload["area_sqft"] = v269                    except ValueError:270                        pass271272        # Coordonnées lat/lng (variable JS de la carte Houzez)273        m = re.search(r"houzez_single_property_map\s*=\s*({.*?});", html, re.S)274        if m:275            try:276                data = json.loads(m.group(1))277                lat, lng = float(data.get("lat")), float(data.get("lng"))278                if 44 <= lat <= 63 and -80 <= lng <= -57:   # Québec plausible279                    payload["lat"], payload["lng"] = lat, lng280            except (ValueError, TypeError):281                pass282        return payload283284    @staticmethod285    def _apply_detail(lst: Listing, payload: dict) -> None:286        if payload.get("images"):287            lst.images = payload["images"]288        if payload.get("description"):289            lst.description = payload["description"]290        if payload.get("area_sqft") and lst.area_sqft is None:291            lst.area_sqft = payload["area_sqft"]292        if payload.get("lat") is not None:293            lst.lat, lst.lng = payload["lat"], payload["lng"]294