spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/org_dupuis.py : connecteur Organisation Dupuis5#   (organisationdupuis.ca — promoteur/gestionnaire de Granby ; portail de6#    location locationorganisationdupuis.bstk.io sur la plateforme Building7#    Stack, même famille que les portails des connecteurs summum.py /8#    urban_services.py / edifia.py) :9#     - /Listing/Listings embarque `var units = [...]` (une entrée par10#       appartement annoncé : ApartmentId stable, prix, pi², chambres, sdb,11#       adresse complète avec GPS, contact de location, photo) ;12#     - les pages immeuble /b/<id> donnent la date de disponibilité par unité13#       et les commodités de l'immeuble (une requête par immeuble, via le14#       cache self.detail, plafonnée).15#   Parc étendu au-delà de Granby/Bromont/Waterloo : Sherbrooke, Terrebonne,16#   L'Assomption, Bécancour, Sorel-Tracy, Trois-Rivières, Victoriaville…17#   ⚠️ Le portail liste aussi Edmundston (Nouveau-Brunswick) : tout ce qui18#   n'est pas Province == QC est exclu.19# -----------------------------------------------------------------------------20from __future__ import annotations2122import hashlib23import html as _html24import json25import re2627from bs4 import BeautifulSoup2829from ..schema import Listing30from .base import BaseConnector3132BASE = "https://locationorganisationdupuis.bstk.io"33LIST_URL = f"{BASE}/Listing/Listings"3435_WS_RE = re.compile(r"\s+")363738def _extract_json(html: str, marker: str):39    """Décode la structure JSON qui suit `marker` dans un script inline."""40    i = html.find(marker)41    if i < 0:42        return None43    try:44        data, _ = json.JSONDecoder().raw_decode(html[i + len(marker):].lstrip())45    except Exception:46        return None47    return data484950def _sqft(raw) -> float | None:51    try:52        v = float(str(raw).strip().replace(" ", "").replace(",", "."))53    except (TypeError, ValueError):54        return None55    return v if 80 <= v <= 20000 else None565758class OrgDupuisConnector(BaseConnector):59    source_id = "org_dupuis"60    request_delay = 0.661    max_buildings = 60           # garde-fou pages immeuble (26 au 2026-08)62    max_images = 126364    # -- page immeuble /b/<id> : dates de disponibilité + commodités -----------65    def _building_page(self, pub: str, key: str) -> dict:66        """{ 'dates': {unité: dispo}, 'amenities': [...], 'images': [...] }."""67        def _fetch() -> dict:68            html = self.get(f"{BASE}/b/{pub}").text69            soup = BeautifulSoup(html, "html.parser")70            dates: dict[str, str] = {}71            for a in soup.select("ul.wpb-tabs-menu a.apartment-view-potential"):72                spans = [s.get_text(" ", strip=True) for s in a.find_all("span")]73                if len(spans) >= 6 and spans[0]:74                    dates[spans[0]] = spans[5]        # Unité -> Disponible75            amenities: list[str] = []76            for li in soup.select(".facilities ul li label"):77                t = _WS_RE.sub(" ", li.get_text(" ", strip=True))78                if 3 <= len(t) <= 90 and t not in amenities:79                    amenities.append(t)80            imgs = list(dict.fromkeys(re.findall(81                r"https://wfiles\.buildingstack\.com/resources/image/\w+",82                html)))83            return {"dates": dates, "amenities": amenities[:20],84                    "images": imgs[:10]}8586        try:87            return self.detail(pub, key, _fetch)88        except Exception:89            return {}9091    # -- fetch ------------------------------------------------------------------92    def fetch(self) -> list[Listing]:93        html = self.get(LIST_URL).text94        units = _extract_json(html, "var units =") or []9596        # regrouper par immeuble (parc québécois, résidentiel seulement)97        by_pub: dict[str, list[dict]] = {}98        for u in units:99            addr = u.get("Address") or {}100            prov = ((addr.get("Province") or {}).get("ProvinceCode") or "")101            if prov.upper() != "QC":102                continue                  # Edmundston (N.-B.) : exclu103            if not (u.get("Apartment") or {}).get("IsResidential", True):104                continue                  # commercial / stationnement105            pub = str((u.get("Building") or {})106                      .get("PublicListBuildingName") or "")107            if pub:108                by_pub.setdefault(pub, []).append(u)109110        # une page immeuble par immeuble (clé de cache = contenu des unités)111        bldg_info: dict[str, dict] = {}112        for pub, us in sorted(by_pub.items())[: self.max_buildings]:113            key = hashlib.sha1(json.dumps(114                sorted((str(x.get("ApartmentId")),115                        str((x.get("Apartment") or {}).get("Price")))116                       for x in us)).encode("utf-8")).hexdigest()[:16]117            bldg_info[pub] = self._building_page(pub, key)118119        listings: list[Listing] = []120        for pub, us in by_pub.items():121            info = bldg_info.get(pub) or {}122            for u in us:123                try:124                    lst = self._listing(u, pub, info)125                    if lst:126                        listings.append(lst)127                except Exception:128                    continue              # une unité ne bloque pas le reste129130        # dédup par external_id (sécurité)131        uniq: dict[str, Listing] = {}132        for lst in listings:133            uniq.setdefault(lst.external_id, lst)134        return list(uniq.values())135136    # -- une annonce par unité ----------------------------------------------------137    def _listing(self, u: dict, pub: str, info: dict) -> Listing | None:138        apt = u.get("Apartment") or {}139        adr = u.get("Address") or {}140        bld = u.get("Building") or {}141        ext_id = str(u.get("ApartmentId") or "")142        if not ext_id:143            return None144        num = str(apt.get("UnitName") or "").strip()145        bname = _WS_RE.sub(" ", _html.unescape(146            str(u.get("BuildingName") or ""))).strip()147148        # convention Building Stack (comme edifia) : 0 chambre = studio149        beds = apt.get("NumberOfBedrooms") or 0150        baths = apt.get("NumberOfBathrooms") or 0151        unit_type = "Studio" if not beds else f"{int(beds) + 2}½"152153        dispo = (info.get("dates") or {}).get(num, "")154        if not dispo:155            availability = "Disponible"156        elif re.search(r"\d", dispo):     # « sept. 01, 2026 »157            availability = f"Libre {dispo}"158        else:                             # « Disponible dès maintenant! »159            availability = dispo160161        area = _sqft(apt.get("Area"))162        desc = " — ".join(x for x in [163            f"{area:g} pi²" if area else "",164            f"{beds} chambre(s)" if beds else "",165            f"{baths} salle(s) de bain" if baths else ""] if x)166167        details: dict = {"bedrooms": int(beds), "bathrooms": int(baths)}168        if isinstance(u.get("ParkingsIsAvailable"), bool):169            details["parking"] = {"available": u["ParkingsIsAvailable"]}170        if isinstance(u.get("StoragesIsAvailable"), bool):171            details["storage"] = u["StoragesIsAvailable"]172        contacts = bld.get("ListingEmployeesContacts") or []173        if contacts:174            c = contacts[0]175            contact = {k: v for k, v in [176                ("name", c.get("FullName")),177                ("phone", c.get("FormattedPhoneNumber")),178                ("email", c.get("Email"))] if v}179            if contact:180                details["contact"] = contact181182        # photo de l'unité (aperçu liste), puis photos de l'immeuble183        imgs: list[str] = []184        prev = str(u.get("PreviewUrl") or "")185        if prev.startswith("http"):186            imgs.append(prev)187        imgs += [x for x in (info.get("images") or []) if x not in imgs]188189        try:190            lat, lng = float(adr.get("Latitude")), float(adr.get("Longitude"))191        except (TypeError, ValueError):192            lat = lng = None193194        burl = str(u.get("BuildingUrl") or "")195        url = BASE + burl if burl.startswith("/") else f"{BASE}/b/{pub}"196        return Listing(197            source=self.source_id,198            external_id=ext_id,199            url=url,200            title=(f"{bname} — Unité {num}" if num else bname),201            address=_html.unescape(str(adr.get("Full") or "")),202            sector="",203            city=_html.unescape(str(adr.get("City") or "")),204            unit_type=unit_type,205            price=float(apt.get("Price") or 0) or None,206            price_label=str(apt.get("PriceFormatted") or ""),207            availability=availability,208            area_sqft=area,209            description=desc[:600],210            amenities=list(info.get("amenities") or []),211            details=details,212            images=imgs[: self.max_images],213            lat=lat,214            lng=lng,215        )216