spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/hazelview.py : connecteur Hazelview Properties5#   (hazelviewproperties.com — ex-Timbercreek). Le site est rendu côté client6#   via l'API RentSync/LiftSystem (lift-api.rentsync.com/v2, client_id 497,7#   jeton public embarqué dans le JS du site). On interroge /v2/cities pour8#   les villes QC (toutes dans le Grand Montréal : Montréal, Verdun,9#   Côte-Saint-Luc, Pointe-Claire, Longueuil...), puis /v2/search par ville et10#   par nombre de chambres pour obtenir les types d'unités disponibles et leur11#   loyer. Une annonce par immeuble et par type d'unité. La page immeuble du12#   site (via cache BD) embarque un JSON complet (attribut data-locations de13#   la carte) : commodités, galerie photos, animaux détaillés, suites.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import html as htmllib18import json19import re2021from bs4 import BeautifulSoup2223from ..schema import Listing, strip_accents24from .base import BaseConnector2526GALLERY = "https://assets.rentsync.com/timbercreek_communities/images/gallery/full/"2728API = "https://lift-api.rentsync.com/v2"29CLIENT_ID = "497"30AUTH_TOKEN = "sswpREkUtyeYjeoahA2i"    # jeton public (présent dans main.js)3132SEARCH_PARAMS = ("only_available_suites=true&show_all_properties=false"33                 "&min_bath=-1&max_bath=10&min_rate=0&max_rate=10000")3435# Villes QC admissibles (Grand Montréal) -> (ville, secteur imposé)36_GM_CITIES = {37    "montreal": ("Montréal", None),38    "verdun": ("Montréal", "Verdun"),39    "cote-saint-luc": ("Côte-Saint-Luc", None),40    "dollard-des-ormeaux": ("Dollard-des-Ormeaux", None),41    "pointe-claire": ("Pointe-Claire", None),42    "longueuil": ("Longueuil", None),43    "lasalle": ("Montréal", "LaSalle"),44}4546# (min_bed, max_bed, type d'unité)47_BED_QUERIES = [(0, 0, "Studio"), (1, 1, "3½"), (2, 2, "4½"),48                (3, 3, "5½"), (4, 5, "6½")]49_TAG_RE = re.compile(r"<[^>]+>")505152def _pets_from_flags(d: dict) -> str | None:53    """Drapeaux animaux du JSON immeuble ('1'/'0'/None) -> oui/non/conditions."""54    def flag(k):55        v = d.get(k)56        return None if v in (None, "") else str(v) == "1"57    if flag("pets_not_allowed"):58        return "non"59    small, cats, large = (flag("pets_small_dogs"), flag("pets_cats"),60                          flag("pets_large_dogs"))61    if any(v for v in (small, cats, large)):62        # certains types refusés explicitement -> sous conditions63        if False in (small, cats, large):64            return "conditions"65        return "oui"66    if flag("pet_friendly"):67        return "oui"68    if flag("pet_friendly") is False:69        return "non"70    return None717273class HazelviewConnector(BaseConnector):74    source_id = "hazelview"75    request_delay = 0.676    max_cities = 10          # garde-fou77    max_details = 150        # garde-fou pages immeuble (vraies requêtes)7879    def _api(self, path: str, extra: str = "") -> list | dict:80        url = (f"{API}/{path}?client_id={CLIENT_ID}&auth_token={AUTH_TOKEN}"81               f"&locale=en{('&' + extra) if extra else ''}")82        return self.get(url).json()8384    def fetch(self) -> list[Listing]:85        cities = self._api("cities")86        qc = []87        for c in cities if isinstance(cities, list) else []:88            if (c.get("province_code") or "").upper() != "QC":89                continue90            key = strip_accents((c.get("city_name") or "").strip().lower())91            if key in _GM_CITIES:92                qc.append((c.get("id"), key))9394        listings: list[Listing] = []95        bed_range: dict[str, tuple[int, int]] = {}96        for city_id, key in qc[: self.max_cities]:97            city, forced_sector = _GM_CITIES[key]98            for min_bed, max_bed, unit_type in _BED_QUERIES:99                try:100                    props = self._api(101                        "search",102                        f"city_ids={city_id}&min_bed={min_bed}"103                        f"&max_bed={max_bed}&{SEARCH_PARAMS}&limit=50")104                except Exception:105                    continue106                if not isinstance(props, list):107                    continue108                for p in props:109                    try:110                        lst = self._prop_listing(p, unit_type, min_bed,111                                                 city, forced_sector)112                        if lst:113                            listings.append(lst)114                            bed_range[lst.external_id] = (min_bed, max_bed)115                    except Exception:116                        continue117118        # Page immeuble du site (cache BD, 1 requête par immeuble) : le JSON119        # embarqué (data-locations) fournit commodités, galerie, animaux120        # détaillés et suites (superficie, date de disponibilité par unité)121        self._fetched = 0122        memo: dict[str, dict] = {}123        for lst in listings:124            pid = lst.external_id.split("-")[0]125            if not lst.url:126                continue127            if pid not in memo:128                key = f"{pid}|{lst.availability}|{lst.address}"129                try:130                    memo[pid] = self.detail(131                        pid, key, lambda u=lst.url: self._fetch_building(u))132                except Exception:133                    memo[pid] = {}134            mn, mx = bed_range.get(lst.external_id, (None, None))135            self._apply_building(lst, memo[pid], mn, mx)136        return listings137138    def _prop_listing(self, p: dict, unit_type: str, beds: int,139                      city: str, forced_sector: str | None) -> Listing | None:140        if not p.get("availability_count"):141            return None142        addr = p.get("address") or {}143        stats = ((p.get("statistics") or {}).get("suites") or {})144        rates = stats.get("rates") or {}145        rmin, rmax = rates.get("min"), rates.get("max")146        sq = stats.get("square_feet") or {}147148        def _num(v):149            try:150                return float(v)151            except (TypeError, ValueError):152                return None153154        rmin, rmax = _num(rmin), _num(rmax)155        price = rmin156        if rmin and rmax and rmax != rmin:157            price_label = f"À partir de {int(rmin)} $ (max {int(rmax)} $)"158        elif rmin:159            price_label = f"{int(rmin)} $/mois"160        else:161            price_label = ""162163        sector = forced_sector or (addr.get("neighbourhood") or "").strip()164        details = p.get("details") or {}165        desc = _TAG_RE.sub(" ", details.get("overview") or "")166        desc = re.sub(r"\s+", " ", desc).strip()[:500]167        sbits = []168        sqmin, sqmax = _num(sq.get("min")), _num(sq.get("max"))169        if sqmin:170            sqtxt = (f"{int(sqmin)}-{int(sqmax)}"171                     if sqmax and sqmax != sqmin else f"{int(sqmin)}")172            sbits.append(f"{sqtxt} pi²")173        sbits.append(f"{p['availability_count']} unité(s) disponible(s)")174175        amenities = []176        feats = _TAG_RE.sub("|", details.get("features") or "")177        for f in feats.split("|"):178            f = f.strip()179            if 2 < len(f) < 60 and f not in amenities:180                amenities.append(f)181        amenities = amenities[:20]182183        images = []184        if p.get("photo_path"):185            images.append(p["photo_path"])186187        pid = p.get("id")188        name = (p.get("name") or "").strip()189        geo = p.get("geocode") or {}190        try:191            lat = float(geo.get("latitude"))192            lng = float(geo.get("longitude"))193        except (TypeError, ValueError):194            lat = lng = None195196        # champs structurés de l'API : animaux (bool), contact de location197        pets = None198        if isinstance(p.get("pet_friendly"), bool):199            pets = "oui" if p["pet_friendly"] else "non"200        details: dict = {}201        contact = p.get("contact") or {}202        cinfo: dict = {}203        if (contact.get("phone") or "").strip():204            cinfo["phone"] = contact["phone"].strip()205        for em in (contact.get("email") or "").split(","):206            em = em.strip()207            if em and "leadmanaging" not in em:      # relais de tracking exclu208                cinfo["email"] = em209                break210        if cinfo:211            details["contact"] = cinfo212213        return Listing(214            source=self.source_id,215            external_id=f"{pid}-{beds}bed",216            url=p.get("permalink") or "",217            title=f"{name} — {unit_type}",218            address=", ".join(x for x in [219                (addr.get("address") or "").strip(), city,220                (addr.get("postal_code") or "").strip()] if x),221            sector=sector,222            city=city,223            unit_type=unit_type,224            price=price,225            price_label=price_label,226            availability=(p.get("min_availability_date")227                          or p.get("availability_status_label") or ""),228            area_sqft=sqmin if sqmin else None,      # stats API (min du type)229            pets=pets,230            description=" — ".join([desc] + sbits if desc else sbits)[:600],231            amenities=amenities,232            details=details,233            images=images,234            lat=lat,235            lng=lng,236        )237238    # -- page immeuble du site (JSON embarqué data-locations) ------------------239    def _fetch_building(self, url: str) -> dict:240        """Extrait le JSON immeuble embarqué dans la page (widget carte) :241        commodités, services inclus, galerie, animaux détaillés, suites."""242        if self._fetched >= self.max_details:243            raise RuntimeError("budget de pages immeuble atteint")244        self._fetched += 1245        html = self.get(url).text246        soup = BeautifulSoup(html, "html.parser")247        el = soup.select_one("[data-locations]")248        if not el:249            return {}250        raw = el.get("data-locations") or ""251        try:252            data = json.loads(raw)253        except Exception:254            data = json.loads(htmllib.unescape(raw))255        node = (data[0] if isinstance(data, list) and data else {}) or {}256        d = node.get("data") or {}257        out: dict = {}258259        out["amenities"] = [a.get("name", "").strip()260                            for a in (d.get("Amenities") or [])261                            if a.get("name", "").strip()][:30]262        out["utilities"] = [u.get("name", "").strip()263                            for u in (d.get("Utilities") or [])264                            if isinstance(u, dict) and u.get("name", "").strip()]265        out["photos"] = [GALLERY + ph["image"]266                         for ph in (d.get("photos") or [])267                         if ph.get("image")][:15]268        out["pets_flags"] = {k: d.get(k) for k in269                             ("pet_friendly", "pets_small_dogs",270                              "pets_large_dogs", "pets_cats",271                              "pets_not_allowed") if d.get(k) is not None}272        out["pets_details"] = (d.get("pets_details") or "").strip()273        out["suites"] = [{274            "bed": s.get("bed"), "available": s.get("available"),275            "availability_date": s.get("availability_date"),276            "sq_ft": s.get("sq_ft"), "furnished": s.get("furnished"),277        } for s in (d.get("suites") or [])]278        return out279280    def _apply_building(self, lst: Listing, d: dict,281                        min_bed: int | None, max_bed: int | None) -> None:282        """Reporte le JSON immeuble (frais/cache) sur l'annonce."""283        if not d:284            return285        merged = list(dict.fromkeys(286            lst.amenities + (d.get("amenities") or []) +287            (d.get("utilities") or [])))288        if merged:289            lst.amenities = merged[:30]290        if d.get("photos"):291            lst.images = list(dict.fromkeys(lst.images + d["photos"]))[:15]292        pets = _pets_from_flags(d.get("pets_flags") or {})293        if pets:294            lst.pets = pets295296        # suites du type demandé : superficie et date précise si publiées297        suites = []298        for s in d.get("suites") or []:299            try:300                bed = int(s.get("bed"))301            except (TypeError, ValueError):302                continue303            if min_bed is None or not (min_bed <= bed <= (max_bed or min_bed)):304                continue305            if str(s.get("available")) == "1":306                suites.append(s)307        if suites:308            if lst.area_sqft is None:309                sqs = []310                for s in suites:311                    try:312                        v = float(s.get("sq_ft") or 0)313                    except (TypeError, ValueError):314                        v = 0315                    if v >= 80:316                        sqs.append(v)317                if sqs:318                    lst.area_sqft = min(sqs)319            dates = [s.get("availability_date") for s in suites320                     if s.get("availability_date")321                     and not str(s["availability_date"]).startswith("0000")]322            if dates and len(dates) == len(suites):323                # toutes les unités du type ont une date précise publiée324                lst.availability = min(dates)325            if all(str(s.get("furnished")) == "1" for s in suites):326                lst.furnished = True327