spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/homestead.py : connecteur Homestead Land Holdings (homestead.ca)5#   Un des 5 plus gros gestionnaires de l'Ontario (~27 000 unités, siège à6#   Kingston) : Toronto/GTA, Ottawa, Hamilton, London, Kitchener-Waterloo,7#   Kingston, Guelph, Sarnia… Le site est une SPA Rentsync « nouvelle8#   génération » (bundle cdn.rentsync.com/site/homestead_rebuild) qui parle à9#   la passerelle JSON PUBLIQUE (aucune auth, aucun anti-bot) :10#     https://website-gateway.rentsync.com/v1/homestead_rebuild/11#       properties?limit=500                        → 172 immeubles (adresse,12#           GPS, description, animaux, permaLink, cityId, modified)13#       cities/property-summary?limit=100           → cityId → nom + province14#       units?where=buildingId~in:a|b|…,status~in:enabled15#           → types d'unités avec bed/bath/pi²/prix/dispo/date (séparateur16#             multi-valeurs : « | » ; paginer via meta.totalPages)17#       properties/{id}/photos + /utilities         → galerie + services inclus18#   Photos : https://s3.amazonaws.com/lws_lift/homestead/images/gallery/full/…19#   (clé S3 « homestead », PAS « homestead_rebuild » qui sert au contenu CMS).20#   On émet UNE annonce par type d'unité DISPONIBLE (available=1), regroupées21#   par (immeuble, type, cc, sdb) — certains immeubles listent chaque logement.22#   Fiche (photos+services) via le cache BD self.detail(), clé = modified.23# -----------------------------------------------------------------------------24# Expansion Ontario — activer via LOUKA_ONTARIO=125from __future__ import annotations2627import hashlib28import os29import re3031from bs4 import BeautifulSoup3233from ..schema import Listing, normalize_unit_type34from .base import BaseConnector3536_ONTARIO = os.environ.get("LOUKA_ONTARIO") == "1"3738BASE = "https://www.homestead.ca"39GATEWAY = "https://website-gateway.rentsync.com/v1/homestead_rebuild"40# galerie S3 : tailles 512/768/1152/full — « full » validé live41IMG_BASE = "https://s3.amazonaws.com/lws_lift/homestead/images/gallery/full"4243_ISO_DATE = re.compile(r"^20\d{2}-\d{2}-\d{2}")444546def _slug(s: str) -> str:47    return re.sub(r"[^a-z0-9]+", "-", (s or "").lower()).strip("-")484950def _txt(html: str) -> str:51    """HTML de la passerelle → texte plat."""52    if not html:53        return ""54    return BeautifulSoup(html, "html.parser").get_text(" ", strip=True)555657class HomesteadConnector(BaseConnector):58    source_id = "homestead"59    request_delay = 0.860    disabled = not _ONTARIO       # Expansion Ontario — activer via LOUKA_ONTARIO=161    max_properties = 300          # garde-fou (172 immeubles au 2026-08)62    max_images = 1563    chunk_size = 25               # immeubles par requête « units »6465    # -- passerelle JSON --------------------------------------------------------66    def _api(self, path: str, **params) -> dict:67        resp = self.get(f"{GATEWAY}/{path}", params=params,68                        headers={"Accept": "application/json",69                                 "Origin": BASE, "Referer": BASE + "/"})70        return resp.json()7172    def _api_all(self, path: str, **params) -> list[dict]:73        """Toutes les pages d'une collection (meta.totalPages)."""74        params.setdefault("limit", 500)75        out: list[dict] = []76        page = 177        while True:78            d = self._api(path, page=page, **params)79            out.extend(d.get("data") or [])80            meta = d.get("meta") or {}81            total = meta.get("totalPages") or 182            if page >= total:83                return out84            page += 18586    # -- villes : cityId → (nom, code province) ---------------------------------87    def _cities(self) -> dict[int, tuple[str, str]]:88        cities: dict[int, tuple[str, str]] = {}89        try:90            for c in self._api_all("cities/property-summary", limit=100):91                cid = c.get("cityId")92                if cid is not None:93                    cities[int(cid)] = ((c.get("cityName") or "").strip(),94                                        (c.get("provinceCode") or "").strip().upper())95        except Exception:96            pass    # repli : ville dérivée du permaLink dans _listing97        return cities9899    def fetch(self) -> list[Listing]:100        cities = self._cities()101        props = self._api_all("properties")[: self.max_properties]102        by_id = {int(p["id"]): p for p in props if p.get("id") is not None}103104        # types d'unités actifs, par lots d'immeubles (séparateur « | »)105        units: list[dict] = []106        ids = list(by_id)107        for i in range(0, len(ids), self.chunk_size):108            chunk = "|".join(str(x) for x in ids[i:i + self.chunk_size])109            try:110                units.extend(self._api_all(111                    "units", where=f"buildingId~in:{chunk},status~in:enabled"))112            except Exception:113                continue114115        # regrouper les unités DISPONIBLES par (immeuble, type, cc, sdb) —116        # certains immeubles publient une ligne par logement individuel117        groups: dict[tuple, list[dict]] = {}118        for u in units:119            if not u.get("available") or u.get("hideSuiteTypeWebsite"):120                continue121            bid = int(u.get("buildingId") or 0)122            if bid not in by_id:123                continue124            key = (bid, _slug(u.get("typeName") or ""),125                   u.get("bed"), u.get("bath"))126            groups.setdefault(key, []).append(u)127128        listings: list[Listing] = []129        for key, grp in groups.items():130            try:131                listings.append(self._listing(by_id[key[0]], grp, cities))132            except Exception:133                continue134        return listings135136    # -- une annonce par type d'unité disponible dans un immeuble ---------------137    def _listing(self, p: dict, grp: list[dict],138                 cities: dict[int, tuple[str, str]]) -> Listing:139        pid = int(p["id"])140        u0 = grp[0]141        type_name = (u0.get("typeName") or "").strip()142        name = (p.get("buildingName") or "").strip()143        perma = (p.get("permaLink") or "").strip()144        url = f"{BASE}/residential/{perma}" if perma else BASE145146        # ville : mapping cityId → nom officiel ; repli = dernier segment du slug147        city, prov = cities.get(int(p.get("cityId") or 0), ("", "ON"))148        if not city and perma:149            city = perma.rsplit("-", 1)[-1].replace("-", " ").title()150151        # adresse complète : rue + ville + ON + code postal152        street = " ".join(x for x in ((p.get("streetNumber") or "").strip(),153                                      (p.get("streetName") or "").strip()) if x)154        postal = (p.get("postal") or "").strip()155        full_addr = ", ".join(x for x in (street, city) if x)156        if full_addr:157            full_addr += f", ON {postal}".rstrip()158159        # coordonnées GPS structurées de la passerelle160        try:161            lat = float(p["latitude"]) if p.get("latitude") else None162            lng = float(p["longitude"]) if p.get("longitude") else None163        except (TypeError, ValueError):164            lat = lng = None165166        # prix : plus bas tarif affichable du groupe (0 = prix masqué)167        rates = []168        for u in grp:169            if u.get("hideRateWebsites"):170                continue171            try:172                r = float(u.get("rate") or 0)173            except (TypeError, ValueError):174                r = 0.0175            if r > 0:176                rates.append(r)177        price = min(rates) if rates else None178        price_label = ""179        if price is not None:180            price_label = (f"À partir de {price:.0f} $"181                           if len(grp) > 1 or (rates and max(rates) != price)182                           else f"{price:.0f} $ /mois")183184        # disponibilité : plus proche date du groupe (None = maintenant)185        dates = sorted(str(u.get("availabilityDate") or "")[:10]186                       for u in grp if _ISO_DATE.match(187                           str(u.get("availabilityDate") or "")))188        avail_date = dates[0] if dates else None189        availability = (f"Disponible le {avail_date}" if avail_date190                        else "Disponible maintenant")191192        # superficie : plus petite valeur plausible du groupe193        area = None194        for u in grp:195            for k in ("sqFt", "sqFtMin"):196                try:197                    v = float(u.get(k) or 0)198                except (TypeError, ValueError):199                    continue200                if 80 <= v <= 20000 and (area is None or v < area):201                    area = v202203        # chambres / salles de bain : champs structurés de l'unité204        try:205            bedrooms = float(u0["bed"]) if u0.get("bed") is not None else None206            bathrooms = float(u0["bath"]) if u0.get("bath") is not None else None207        except (TypeError, ValueError):208            bedrooms = bathrooms = None209        unit_type = ("Studio" if bedrooms == 0210                     else normalize_unit_type(type_name))211212        # animaux : indicateurs structurés de l'immeuble213        if p.get("petsNotAllowed"):214            pets = "non"215        elif p.get("petFriendly"):216            pets = "oui"217        else:218            pets = None219220        # description : aperçu de l'immeuble + détails de suites (HTML → texte)221        desc = " ".join(x for x in (222            _txt(p.get("buildingOverview") or ""),223            _txt(p.get("suiteDetails") or ""),224        ) if x)[:800]225226        # commodités : caractéristiques de l'immeuble + services inclus (fiche)227        amenities: list[str] = []228        for t in re.split(r"[\n;•]|</li>|<li>",229                          p.get("buildingFeatures") or ""):230            t = _txt(t)231            if t and t not in amenities:232                amenities.append(t)233234        # champs structurés235        details: dict = {}236        contact: dict = {}237        if (p.get("phone") or "").strip():238            contact["phone"] = p["phone"].strip()239        if (p.get("email") or "").strip():240            contact["email"] = p["email"].strip()241        if contact:242            details["contact"] = contact243        if (p.get("neighbourhood") or "").strip():244            details["Quartier"] = p["neighbourhood"].strip()245246        # fiche (galerie photo + services inclus) via le cache BD : revisitée247        # seulement quand l'immeuble change (horodatage « modified »)248        det_key = hashlib.sha1(str(p.get("modified") or "").encode()).hexdigest()249        d = self.detail(f"b{pid}", det_key, lambda: self._fetch_detail(pid))250        for t in d.get("utilities") or []:251            t = f"{t} incluse" if t in ("Eau", "Électricité") else t252            if t and t not in amenities:253                amenities.append(t)254        images = list(d.get("images") or [])255256        return Listing(257            source=self.source_id,258            external_id=f"{pid}-{_slug(type_name) or 'unite'}-"259                        f"{u0.get('bed')}cc-{u0.get('bath')}sdb",260            url=url,261            title=f"{name} — {type_name}" if type_name else name,262            address=full_addr,263            sector=(p.get("neighbourhood") or "").strip(),264            city=city,265            province=prov or "ON",266            unit_type=unit_type,267            bedrooms=bedrooms,268            bathrooms=bathrooms,269            price=price,270            price_label=price_label,271            availability=availability,272            availability_date=avail_date,273            area_sqft=area,274            pets=pets,275            description=desc,276            amenities=amenities[:25],277            details=details,278            images=images[: self.max_images],279            lat=lat,280            lng=lng,281        )282283    # -- fiche immeuble : galerie photo + services inclus (2 appels, cachés) ----284    def _fetch_detail(self, pid: int) -> dict:285        out: dict = {"images": [], "utilities": []}286        # traductions FR des services inclus les plus fréquents287        fr = {"Water": "Eau", "Heat": "Chauffage", "Hydro": "Électricité",288              "Electricity": "Électricité", "Internet": "Internet",289              "Cable": "Câble"}290        try:291            photos = self._api_all(f"properties/{pid}/photos", limit=100)292        except Exception:293            photos = []294        photos = [ph for ph in photos295                  if ph.get("active") and (ph.get("image") or "").strip()]296        photos.sort(key=lambda ph: (0 if ph.get("mainGallery") else 1,297                                    ph.get("orderBy") or 0))298        for ph in photos:299            u = f"{IMG_BASE}/{ph['image'].strip()}"300            if u not in out["images"]:301                out["images"].append(u)302        out["images"] = out["images"][: self.max_images]303        try:304            for ut in self._api_all(f"properties/{pid}/utilities", limit=50):305                t = (ut.get("name") or "").strip()306                if t:307                    out["utilities"].append(fr.get(t, t))308        except Exception:309            pass310        return out311