spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/urban_services.py : connecteur Urban Services (urbanservices.ca)5#   Gestionnaire de Gatineau (Urban Tenant Solutions) ; portail de location6#   dédié rent.urbanservices.ca sur la plateforme Building Stack — même7#   famille que le portail edifialocation.com du connecteur edifia.py :8#     - /Listing/Listings embarque `var units = [...]` (une entrée par9#       appartement annoncé : prix, pi², chambres, sdb, adresse complète,10#       GPS, contact de l'immeuble, photo) ;11#     - les pages immeuble /b/<id> listent TOUTES les unités disponibles12#       (onglets : numéro, prix, pi², chambres, sdb, date de disponibilité)13#       + les commodités de l'immeuble — le JSON de la liste est parfois14#       tronqué (immeubles à fort inventaire), la page /b/<id> fait donc foi15#       pour l'inventaire et les dates ; le JSON enrichit (GPS, contact…).16#   Une annonce Lou-Ka = une unité disponible. Le parc ontarien (Ottawa)17#   est exclu (province != QC). ~17 requêtes par sync (1 + 16 immeubles).18# -----------------------------------------------------------------------------19from __future__ import annotations2021import html as _html22import json23import re2425from bs4 import BeautifulSoup2627from ..schema import Listing, parse_price28from .base import BaseConnector2930BASE = "https://rent.urbanservices.ca"31LIST_URL = f"{BASE}/Listing/Listings"323334def _extract_json(html: str, marker: str):35    """Décode la structure JSON qui suit `marker` dans un script inline."""36    i = html.find(marker)37    if i < 0:38        return None39    try:40        data, _ = json.JSONDecoder().raw_decode(html[i + len(marker):])41    except Exception:42        return None43    return data444546def _sqft(raw) -> float | None:47    try:48        v = float(str(raw).strip().replace(" ", "").replace(" ", "")49                  .replace(",", "."))50    except (TypeError, ValueError):51        return None52    return v if 80 <= v <= 20000 else None535455class UrbanServicesConnector(BaseConnector):56    source_id = "urban_services"57    request_delay = 0.658    max_buildings = 40          # garde-fou pages immeuble (16 au 2026-08)59    max_images = 126061    def fetch(self) -> list[Listing]:62        html = self.get(LIST_URL).text63        bs_units = _extract_json(html, "var units = ") or []6465        # regrouper le JSON par immeuble (parc québécois seulement)66        by_pub: dict[str, list[dict]] = {}67        for u in bs_units:68            addr = u.get("Address") or {}69            prov = ((addr.get("Province") or {}).get("ProvinceCode") or "")70            if prov.upper() != "QC":71                continue                     # Ottawa (Ontario) : exclu72            pub = str((u.get("Building") or {})73                      .get("PublicListBuildingName") or "")74            if pub:75                by_pub.setdefault(pub, []).append(u)7677        listings: list[Listing] = []78        for pub, units in sorted(by_pub.items())[: self.max_buildings]:79            try:80                listings.extend(self._building(pub, units))81            except Exception:82                continue                     # un immeuble ne bloque pas le reste83        # dédup par external_id (sécurité)84        uniq: dict[str, Listing] = {}85        for lst in listings:86            uniq.setdefault(lst.external_id, lst)87        return list(uniq.values())8889    # -- page immeuble /b/<id> : inventaire complet + commodités -------------------90    def _building_page(self, pub: str) -> tuple[list[dict], list[str]]:91        """(lignes d'unités des onglets, commodités de l'immeuble)."""92        html = self.get(f"{BASE}/b/{pub}").text93        soup = BeautifulSoup(html, "html.parser")94        rows: list[dict] = []95        for a in soup.select("ul.wpb-tabs-menu a.apartment-view-potential"):96            spans = [s.get_text(" ", strip=True) for s in a.find_all("span")]97            if len(spans) >= 6 and spans[0]:98                rows.append({           # Unité, Prix, pi², chambres, sdb, dispo99                    "unit": spans[0], "price": spans[1], "sqft": spans[2],100                    "beds": spans[3], "baths": spans[4], "dispo": spans[5]})101        amenities: list[str] = []102        for li in soup.select(".facilities ul li label"):103            t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))104            if 3 <= len(t) <= 90 and t not in amenities:105                amenities.append(t)106        return rows, amenities107108    def _building(self, pub: str, units: list[dict]) -> list[Listing]:109        rows, amenities = self._building_page(pub)110111        # infos communes de l'immeuble depuis le JSON de la liste112        ref = units[0]113        addr = ref.get("Address") or {}114        # le JSON Razor du portail encode les apostrophes (&#39;) : décoder115        bname = _html.unescape(str(ref.get("BuildingName") or "")).strip()116        try:117            lat = float(addr.get("Latitude"))118            lng = float(addr.get("Longitude"))119        except (TypeError, ValueError):120            lat = lng = None121        contact: dict = {}122        contacts = ((ref.get("Building") or {})123                    .get("ListingEmployeesContacts") or [])124        if contacts:125            c = contacts[0]126            contact = {k: v for k, v in [127                ("name", c.get("FullName")),128                ("phone", c.get("FormattedPhoneNumber")),129                ("email", c.get("Email"))] if v}130        b_img = str(ref.get("BuildingPreviewUrl") or "")131        by_name = {str((u.get("Apartment") or {}).get("UnitName") or ""): u132                   for u in units}133134        out: list[Listing] = []135        seen: set[str] = set()136        for row in rows:137            num = row["unit"].strip()138            if not num or num in seen:139                continue140            seen.add(num)141            js = by_name.get(num) or {}142            out.append(self._listing(pub, num, row, js, bname, addr,143                                     amenities, contact, b_img, lat, lng))144        # unités du JSON absentes des onglets (prudence : ne rien perdre)145        for num, js in by_name.items():146            if num and num not in seen:147                seen.add(num)148                out.append(self._listing(pub, num, None, js, bname, addr,149                                         amenities, contact, b_img, lat, lng))150        return out151152    # -- une annonce par unité -------------------------------------------------------153    def _listing(self, pub: str, num: str, row: dict | None, js: dict,154                 bname: str, addr: dict, amenities: list[str], contact: dict,155                 b_img: str, lat, lng) -> Listing:156        apt = js.get("Apartment") or {}157158        # chambres / sdb : onglet de la page immeuble, sinon JSON de la liste159        def _int(v) -> int | None:160            try:161                return int(str(v).strip())162            except (TypeError, ValueError):163                return None164165        beds = _int(row["beds"]) if row else None166        if beds is None:167            beds = _int(apt.get("NumberOfBedrooms"))168        baths = _int(row["baths"]) if row else None169        if baths is None:170            baths = _int(apt.get("NumberOfBathrooms"))171        # convention Building Stack (comme edifia) : 0 chambre = studio172        unit_type = "" if beds is None else ("Studio" if beds == 0173                                             else f"{beds + 2}½")174175        # prix : JSON structuré prioritaire, sinon texte de l'onglet176        price = float(apt.get("Price") or 0) or None177        price_label = str(apt.get("PriceFormatted") or "")178        if row and not price_label:179            price_label = row["price"]180        if price is None and row:181            price = parse_price(row["price"])182183        area = _sqft(apt.get("Area"))184        if area is None and row:185            area = _sqft(row["sqft"])186187        # disponibilité : texte de l'onglet (« Disponible dès maintenant! »,188        # « oct. 01, 2027 »…) — préfixé quand c'est une date nue189        availability = ""190        if row:191            dispo = row["dispo"].strip()192            if dispo and re.search(r"\d", dispo) and "maintenant" not in dispo.lower():193                availability = f"Libre {dispo}"194            else:195                availability = dispo196197        desc = " — ".join(x for x in [198            f"{area:g} pi²" if area else "",199            f"{beds} chambre(s)" if beds else "",200            f"{baths} salle(s) de bain" if baths else ""] if x)201202        details: dict = {}203        if bname:204            details["building"] = bname205        if contact:206            details["contact"] = dict(contact)207        if beds is not None:208            details["bedrooms"] = beds209        if baths is not None:210            details["bathrooms"] = baths211212        images: list[str] = []213        prev = str(js.get("PreviewUrl") or "")214        if prev.startswith("http"):215            images.append(prev)216        if b_img.startswith("http") and b_img not in images:217            images.append(b_img)218219        apt_id = js.get("ApartmentId")220        url = (f"{BASE}/b/{pub}/{apt_id}" if apt_id else f"{BASE}/b/{pub}")221        title = (f"{bname} — unité {num}" if bname222                 else f"Urban Services — unité {num}")223        return Listing(224            source=self.source_id,225            external_id=f"{pub}-{num}",226            url=url,227            title=title,228            address=_html.unescape(str(addr.get("Full") or "")),229            sector="",230            city=_html.unescape(str(addr.get("City") or "")),231            unit_type=unit_type,232            price=price,233            price_label=price_label,234            availability=availability,235            area_sqft=area,236            description=desc,237            amenities=list(amenities),238            details=details,239            images=images[: self.max_images],240            lat=lat,241            lng=lng,242        )243