spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/denux.py : connecteur Groupe Denux (groupedenux.com)5#   Portefeuille pancanadien (plateforme Rentsync / The Lift System).6#   Seules les villes québécoises sont interrogées (Montréal, Saint-Lambert,7#   Mascouche) — la Colombie-Britannique, l'Alberta et la France sont exclues8#   d'office. Découverte des immeubles via l'API JSON du site9#   (api.theliftsystem.com/v2/search, jeton public embarqué dans le JS du10#   site), puis parsing des fiches /residential/<slug> rendues serveur :11#   suites disponibles (div.suite[data-suite-id] avec type, prix, chambres,12#   date de disponibilité), commodités et galerie photos.13# -----------------------------------------------------------------------------14from __future__ import annotations1516import html as htmllib17import re1819from bs4 import BeautifulSoup2021from ..schema import Listing22from .base import BaseConnector2324SITE = "https://www.groupedenux.com"25API = "https://api.theliftsystem.com/v2/search"26AUTH_TOKEN = "sswpREkUtyeYjeoahA2i"      # jeton public (scripts/main.js du site)27CLIENT_ID = "654"2829# Villes québécoises desservies (id Lift System -> nom normalisé)30QC_CITIES = {31    "1863": "Montréal",32    "2789": "Saint-Lambert",33    "1741": "Mascouche",34}3536_GALLERY_RE = re.compile(37    r'https://assets\.rentsync\.com/groupe_denux/images/gallery/'38    r'[0-9]+/[^"\'\s\\)]+\.(?:jpg|jpeg|png|webp)', re.I)39_HALF_RE = re.compile(r"(\d)\s*(?:½|1/2|[.,]5)")4041# Nombre de chambres -> type d'unité (à défaut d'un « X.5 » dans le libellé)42_BED_TYPE = {0: "Studio", 1: "3½", 2: "4½", 3: "5½", 4: "6½"}434445def _clean(txt: str) -> str:46    txt = htmllib.unescape(htmllib.unescape(txt or ""))47    txt = re.sub(r"<[^>]+>", " ", txt)48    return re.sub(r"\s+", " ", txt).strip()495051class DenuxConnector(BaseConnector):52    source_id = "denux"53    request_delay = 0.654    max_buildings = 40           # garde-fou de crawl5556    # -- API de recherche (backend officiel du site) ---------------------------57    def _search_city(self, city_id: str) -> list[dict]:58        params = {59            "locale": "fr",60            "client_id": CLIENT_ID,61            "auth_token": AUTH_TOKEN,62            "city_id": city_id,63            "geocode": "",64            "min_bed": "-1", "max_bed": "100",65            "min_bath": "-1", "max_bath": "10",66            "min_rate": "0", "max_rate": "100000",67            "property_types": "apartments, houses",68            "order": "min_rate ASC",69            "limit": "66", "offset": "0",70            "count": "false",71            "show_all_properties": "true",72        }73        data = self.get(API, params=params).json()74        return data if isinstance(data, list) else []7576    def fetch(self) -> list[Listing]:77        listings: list[Listing] = []78        buildings: list[dict] = []79        for city_id in QC_CITIES:80            try:81                buildings.extend(self._search_city(city_id))82            except Exception:83                continue8485        seen: set = set()86        for i, b in enumerate(buildings):87            if i >= self.max_buildings:88                break89            try:90                bid = b.get("id")91                if bid in seen:92                    continue93                seen.add(bid)94                addr = b.get("address") or {}95                # Garde-fou : Québec seulement (exclut C.-B., Alberta, France)96                if (addr.get("province_code") or "").upper() != "QC":97                    continue98                if int(b.get("availability_count") or 0) <= 0:99                    continue             # aucune unité disponible100                listings.extend(self._parse_building(b))101            except Exception:102                continue103        return listings104105    # -- fiche immeuble : suites rendues serveur --------------------------------106    def _parse_building(self, b: dict) -> list[Listing]:107        slug = (b.get("permalink") or "").rstrip("/").split("/")[-1]108        if not slug:109            return []110        url = f"{SITE}/residential/{slug}"111112        addr = b.get("address") or {}113        name = _clean(b.get("name") or slug)114        address = _clean(addr.get("address") or "")115        city = _clean(addr.get("city") or "")116        sector = _clean(addr.get("neighbourhood") or "")117        if sector.isupper():118            sector = sector.title()119        if sector.lower() in ("", city.lower(), "montreal", "montréal"):120            sector = ""121122        bdetails = b.get("details") or {}123        description = _clean(bdetails.get("overview") or "")[:600]124125        # Champs structurés de l'API Lift System (jamais devinés du texte)126        pets = None127        if isinstance(b.get("pet_friendly"), bool):128            pets = "oui" if b["pet_friendly"] else "non"129        details: dict = {}130        contact = b.get("contact") or {}131        cinfo = {}132        if _clean(contact.get("phone") or ""):133            cinfo["phone"] = _clean(contact["phone"])134        if _clean(contact.get("email") or ""):135            cinfo["email"] = _clean(contact["email"])136        if cinfo:137            details["contact"] = cinfo138        parking = b.get("parking") or {}139        if parking.get("indoor") or parking.get("outdoor"):140            details["parking"] = {141                "available": True,142                "type": "intérieur" if parking.get("indoor") else "extérieur",143            }144145        geo = b.get("geocode") or {}146        try:147            lat, lng = float(geo.get("latitude")), float(geo.get("longitude"))148        except (TypeError, ValueError):149            lat = lng = None150151        html = self.get(url).text152        soup = BeautifulSoup(html, "html.parser")153154        # Commodités (suite + immeuble)155        amenities = [el.get_text(" ", strip=True)156                     for el in soup.select(".amenities .amenity-holder")]157        amenities = [a for a in dict.fromkeys(amenities) if a][:25]158159        # Galerie photos (dédupliquée par nom de fichier, tailles multiples)160        images: list[str] = []161        seen_files: set[str] = set()162        for u in _GALLERY_RE.findall(html):163            fname = u.rsplit("/", 1)[-1]164            if fname not in seen_files:165                seen_files.add(fname)166                images.append(u)167        images = images[:25]168169        # Détails par suite (ul.suite-info) indexés par data-suite-id170        info_by_id: dict[str, dict[str, str]] = {}171        photos_by_id: dict[str, list[str]] = {}172        for ul in soup.select("ul.suite-info[data-suite-id]"):173            sid = ul.get("data-suite-id") or ""174            fields: dict[str, str] = {}175            for li in ul.select("li.info-block"):176                lab = li.select_one(".label")177                val = li.select_one(".info")178                if not (lab and val):179                    continue180                label = lab.get_text(strip=True).lower()181                # Photos propres à la suite (liens « View » de la galerie)182                if "photo" in label:183                    urls = [a.get("href") or "" for a in val.select("a")]184                    urls = [u for u in dict.fromkeys(urls)185                            if u.startswith("http")]186                    if urls:187                        photos_by_id[sid] = urls[:25]188                    continue189                # Le champ « Availability » contient un lien + un modal de190                # formulaire : ne garder que le libellé du lien.191                link = val.select_one("a.open-suite-modal")192                text = (link.get_text(" ", strip=True) if link193                        else val.get_text(" ", strip=True))194                fields[label] = text[:80].strip()195            info_by_id[sid] = fields196197        results: list[Listing] = []198        for div in soup.select("div.suite[data-suite-id]"):199            sid = div.get("data-suite-id") or ""200            if not sid:201                continue202            type_el = div.select_one(".suite-type")203            rate_el = div.select_one(".suite-rate")204            suite_label = _clean(type_el.get_text(" ", strip=True)205                                 if type_el else "")206            # Nettoyage du libellé (certains contiennent dispo + prix)207            suite_label = re.sub(r"\s*[-–]?\s*Starting at\s*\$[\d,]+", "",208                                 suite_label, flags=re.I)209            suite_label = re.sub(r"\s*[-–]?\s*Available\s+(now|immediately)\b",210                                 "", suite_label, flags=re.I).strip(" -–,")211            info = info_by_id.get(sid, {})212213            # Type d'unité : « Grand 5.5, balcon... » -> 5½, sinon nb chambres214            unit_type = ""215            hm = _HALF_RE.search(suite_label)216            if hm:217                unit_type = f"{hm.group(1)}½"218            else:219                beds_txt = (info.get("bedrooms") or220                            (div.get("class") and221                             next((c.replace("beds_", "")222                                   for c in div.get("class")223                                   if c.startswith("beds_")), "")) or "")224                try:225                    unit_type = _BED_TYPE.get(int(beds_txt), "")226                except (ValueError, TypeError):227                    unit_type = ""228229            # Prix : « $1,320 » (à partir de)230            price = None231            price_label = ""232            if rate_el:233                raw = rate_el.get_text(" ", strip=True)234                digits = re.sub(r"[^\d.]", "", raw)235                if digits:236                    try:237                        price = float(digits)238                    except ValueError:239                        price = None240                    price_label = f"À partir de {raw}/mois"241            if price is not None and not (100 <= price <= 20000):242                price = None243244            availability = info.get("availability", "") or \245                _clean(b.get("availability_status_label") or "")246247            # Commodités de l'immeuble + salles de bain de la suite248            suite_amenities = list(amenities)249            baths = (info.get("bathrooms") or "").strip()250            if baths and re.match(r"^[\d.]+$", baths):251                suite_amenities.append(f"{baths} salle(s) de bain")252253            results.append(Listing(254                source=self.source_id,255                external_id=str(sid),256                url=url,257                title=f"{name} — {suite_label}" if suite_label else name,258                address=address,259                sector=sector,260                city=city or QC_CITIES.get(str(addr.get("city_id")), ""),261                unit_type=unit_type,262                price=price,263                price_label=price_label,264                availability=availability,265                pets=pets,266                description=description,267                amenities=suite_amenities,268                details={k: dict(v) if isinstance(v, dict) else v269                         for k, v in details.items()},270                images=photos_by_id.get(sid) or images,271                lat=lat,272                lng=lng,273            ))274        return results275