spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/apartments_com.py : Apartments.com (CoStar) — réseau US avec un5#   vrai parc québécois (~500 propriétés / 8 400 unités via /qc/, surtout de6#   grands gestionnaires). Le site est verrouillé par Akamai (403 en direct,7#   via Scrapfly ET via Oxylabs — sondé 2026-08-18/22) : le scraping est8#   délégué à l'ACTEUR APIFY MAISON gorgeous_thistle/ka-apartments-com9#   (source : actors/ka-apartments-com de ce repo) qui passe par le Web10#   Unlocker Bright Data — HTML server-rendered, pas de rendu JS.11#   UNE annonce = UNE PROPRIÉTÉ (immeuble) : prix « à partir de » (minimum12#   des plans d'unités), tous les plans dans details["plans"]. Les fiches13#   détail sont lues en FRANÇAIS (/fr/) : description, GPS, commodités,14#   galerie. Cache détail 30 j + CADENCE : l'inventaire CoStar bouge15#   lentement, l'acteur ne tourne qu'aux LOUKA_APTS_INTERVAL_H heures ;16#   entre deux runs on ré-émet les annonces actives depuis la BD (coût 0).17# -----------------------------------------------------------------------------18from __future__ import annotations1920import json21import os22import re23import time2425import requests as _requests2627from ..schema import Listing28from .base import BaseConnector29from . import _detailutil as du3031APIFY_API = "https://api.apify.com/v2"32ACTOR = os.environ.get("LOUKA_APTS_ACTOR", "gorgeous_thistle~ka-apartments-com")3334SEARCH_URLS = ["https://www.apartments.com/qc/"]     # toute la province35MAX_PAGES = int(os.environ.get("LOUKA_APTS_MAX_PAGES", "25"))36DETAIL_LIMIT = int(os.environ.get("LOUKA_APTS_DETAIL_LIMIT", "250"))37TTL_DAYS = float(os.environ.get("LOUKA_APTS_TTL_DAYS", "30"))38INTERVAL_H = float(os.environ.get("LOUKA_APTS_INTERVAL_H", "12"))39CONCURRENCY = int(os.environ.get("LOUKA_APTS_CONCURRENCY", "4"))40RUN_TIMEOUT = int(os.environ.get("LOUKA_APTS_RUN_TIMEOUT", "3600"))  # s4142DETAIL_KEY = "v1"43PRICE_MIN, PRICE_MAX = 300, 12000     # garde-fous de loyer mensuel4445_MONEY_RX = re.compile(r"\$\s*([\d\s,  ]+)")46_BEDS_LABEL_RX = re.compile(r"(\d+)\s*(?:bed|lit|c\.?\s*à\.?\s*c)", re.I)47_SQFT_RX = re.compile(r"([\d\s,  ]+)\s*(?:pi²|sq\s*ft)", re.I)484950def _money(text: str | None) -> float | None:51    m = _MONEY_RX.search(text or "")52    if not m:53        return None54    try:55        return float(re.sub(r"[^\d]", "", m.group(1)))56    except ValueError:57        return None585960def _beds_count(label: str) -> float | None:61    low = (label or "").casefold()62    if "studio" in low or "bachelor" in low:63        return 0.064    m = _BEDS_LABEL_RX.search(low)65    return float(m.group(1)) if m else None666768def _unit_type(beds: float | None) -> str:69    if beds is None:70        return ""71    if beds <= 0:72        return "Studio"73    return "6½+" if beds >= 4 else f"{int(beds) + 2}½"747576def _fmt_price(price: float) -> str:77    return f"{price:,.0f} $/mois".replace(",", " ")787980def _split_address(addr: str) -> tuple[str, str]:81    """« 1411 St Du Fort, Montréal, QC H3H 2N7 » -> (rue, ville)."""82    parts = [p.strip() for p in (addr or "").split(",") if p.strip()]83    if len(parts) >= 3:84        return ", ".join(parts[:-2]), parts[-2]85    return addr or "", ""868788class ApartmentsComConnector(BaseConnector):89    source_id = "apartments_com"90    request_delay = 1.09192    # -- orchestration de l'acteur Apify --------------------------------------93    def _run_actor(self, payload: dict, token: str) -> list[dict]:94        """Lance l'acteur, attend la fin, retourne les items du dataset."""95        r = _requests.post(96            f"{APIFY_API}/acts/{ACTOR}/runs?waitForFinish=120",97            json=payload, timeout=180,98            headers={"Authorization": f"Bearer {token}"})99        r.raise_for_status()100        run = r.json()["data"]101        deadline = time.time() + RUN_TIMEOUT102        while run["status"] in ("READY", "RUNNING") and time.time() < deadline:103            time.sleep(10)104            run = _requests.get(105                f"{APIFY_API}/actor-runs/{run['id']}", timeout=60,106                headers={"Authorization": f"Bearer {token}"}).json()["data"]107        if run["status"] != "SUCCEEDED":108            raise RuntimeError(f"acteur {ACTOR} : run {run['id']} "109                               f"terminé en {run['status']}")110        items: list[dict] = []111        offset = 0112        while True:113            batch = _requests.get(114                f"{APIFY_API}/datasets/{run['defaultDatasetId']}/items"115                f"?limit=1000&offset={offset}", timeout=120,116                headers={"Authorization": f"Bearer {token}"}).json()117            items.extend(batch)118            if len(batch) < 1000:119                return items120            offset += 1000121122    # -- cadence / ré-émission -------------------------------------------------123    def _last_ok_sync(self, cache: du.TtlDetailCache) -> float:124        row = cache.con.execute(125            "SELECT MAX(ts) AS t FROM sync_log WHERE source=? AND ok=1"126            " AND found > 0", (self.source_id,)).fetchone()127        return row["t"] or 0.0128129    def _active_rows(self, cache: du.TtlDetailCache) -> list:130        return cache.con.execute(131            "SELECT external_id, url, title, address, sector, city, unit_type,"132            " bedrooms, bathrooms, price, price_label, availability,"133            " availability_date, area_sqft, pets, furnished, description,"134            " amenities, details, images, lat, lng"135            " FROM listings WHERE source=? AND active=1", (self.source_id,)136        ).fetchall()137138    def _listing_from_row(self, r) -> Listing:139        def js(s, default):140            try:141                return json.loads(s) if s else default142            except ValueError:143                return default144        return Listing(145            source=self.source_id, external_id=r["external_id"],146            url=r["url"], title=r["title"] or "", address=r["address"] or "",147            sector=r["sector"] or "", city=r["city"] or "",148            unit_type=r["unit_type"] or "", bedrooms=r["bedrooms"],149            bathrooms=r["bathrooms"], price=r["price"],150            price_label=r["price_label"] or "",151            availability=r["availability"] or "",152            availability_date=r["availability_date"],153            area_sqft=r["area_sqft"], pets=r["pets"],154            furnished=(None if r["furnished"] is None else bool(r["furnished"])),155            description=r["description"] or "",156            amenities=js(r["amenities"], []), details=js(r["details"], {}),157            images=js(r["images"], []), lat=r["lat"], lng=r["lng"],158        )159160    # -- payload détail -> champs Listing ---------------------------------------161    @staticmethod162    def _std_detail(d: dict) -> dict:163        std = {k: d[k] for k in ("description", "images", "lat", "lng",164                                 "amenities") if d.get(k)}165        if d.get("street"):166            std["address"] = d["street"]167        if d.get("city"):168            std["city"] = d["city"]169        plans, seen = [], set()170        for pl in d.get("plans") or []:      # grille dupliquée desktop/mobile171            sig = (pl.get("key"), pl.get("name"), pl.get("rent"))172            if sig not in seen:173                seen.add(sig)174                plans.append(pl)175        rents = [p for p in (_money(pl.get("rent")) for pl in plans)176                 if p is not None and PRICE_MIN <= p <= PRICE_MAX]177        if rents:178            std["price"] = min(rents)179            std["price_label"] = (f"à partir de {_fmt_price(min(rents))}"180                                  if len(plans) > 1 else _fmt_price(min(rents)))181        # immeuble multi-plans : minimum, cohérent avec le prix « à partir182        # de » — et ça évite que finalize() devine n'importe quoi dans la183        # description marketing (« appartements de 1 à 4 chambres »)184        beds = {pl.get("beds") for pl in plans if pl.get("beds") is not None}185        baths = {pl.get("baths") for pl in plans if pl.get("baths") is not None}186        if beds:187            std["bedrooms"] = min(beds)188            if len(beds) == 1:189                std["unit_type"] = _unit_type(std["bedrooms"])190        if baths:191            std["bathrooms"] = min(baths)192        if len(plans) == 1:193            m = _SQFT_RX.search(" ".join(plans[0].get("details") or []))194            if m:195                try:196                    std["area_sqft"] = float(re.sub(r"[^\d]", "", m.group(1)))197                except ValueError:198                    pass199        avails = [pl.get("availability") or "" for pl in plans]200        now = next((a for a in avails201                    if "maintenant" in a.casefold() or "now" in a.casefold()),202                   "")203        if now or any(avails):204            std["availability"] = now or next(a for a in avails if a)205        if plans:206            std["details"] = {"plans": [207                {k: pl[k] for k in ("name", "rent", "beds", "baths",208                                    "details", "availability") if k in pl}209                for pl in plans]}210        return std211212    # -- pipeline principal ------------------------------------------------------213    def fetch(self) -> list[Listing]:214        token = os.environ.get("APIFY_TOKEN")215        bd_token = os.environ.get("BRIGHTDATA_API_KEY")216        if not token:217            raise RuntimeError("APIFY_TOKEN manquant (voir .env)")218        if not bd_token:219            raise RuntimeError("BRIGHTDATA_API_KEY manquant (voir .env)")220221        cache = du.TtlDetailCache(self, budget=0, ttl_days=TTL_DAYS,222                                  key=DETAIL_KEY, fetch_html=lambda _u: "")223        try:224            # inventaire lent + acteur payant : entre deux vrais runs on225            # ré-émet les actives telles quelles (aucune requête réseau)226            age_h = (time.time() - self._last_ok_sync(cache)) / 3600227            actives = self._active_rows(cache)228            if actives and age_h < INTERVAL_H:229                return [self._listing_from_row(r) for r in actives]230231            fresh = {r["external_id"] for r in cache.con.execute(232                "SELECT external_id FROM detail_cache"233                " WHERE source=? AND key=? AND fetched_at > ?",234                (self.source_id, DETAIL_KEY,235                 time.time() - TTL_DAYS * 86400)).fetchall()}236            # rattrapage : actives jamais enrichies (sans GPS ou description)237            extra = [f'{r["external_id"]}|{r["url"]}' for r in actives238                     if r["external_id"] not in fresh239                     and (r["lat"] is None or not (r["description"] or "").strip())]240241            items = self._run_actor({242                "searchUrls": SEARCH_URLS,243                "maxPages": MAX_PAGES,244                "getDetails": True,245                "maxDetails": DETAIL_LIMIT,246                "skipDetailIds": sorted(fresh),247                "extraDetailIds": extra[:DETAIL_LIMIT],248                "brightdataToken": bd_token,249                "brightdataZone": os.environ.get("BRIGHTDATA_ZONE",250                                                 "web_unlocker1"),251                "concurrency": CONCURRENCY,252            }, token)253254            for it in items:255                if it.get("kind") == "detail" and it.get("id"):256                    payload = {k: v for k, v in it.items()257                               if k not in ("kind",) and v is not None}258                    cache.put(str(it["id"]), payload)259260            out: dict[str, Listing] = {}261            for it in items:262                if it.get("kind") != "listing":263                    continue264                pid = str(it["id"])265                street, city = _split_address(it.get("address") or "")266                rents = [(r.get("beds") or "", _money(r.get("price")))267                         for r in it.get("rents") or []]268                prices = [p for _b, p in rents269                          if p is not None and PRICE_MIN <= p <= PRICE_MAX]270                from_price = any("+" in (r.get("price") or "")271                                 for r in it.get("rents") or [])272                beds = {b for b in (_beds_count(lbl) for lbl, _p in rents)273                        if b is not None}274                lst = Listing(275                    source=self.source_id,276                    external_id=pid,277                    url=(it.get("url") or "").replace(278                        "www.apartments.com/", "www.apartments.com/fr/", 1),279                    title=(it.get("title") or street)[:200],280                    address=street,281                    city=city,282                    price=min(prices) if prices else None,283                    price_label=((f"à partir de {_fmt_price(min(prices))}"284                                  if from_price or len(prices) > 1285                                  else _fmt_price(min(prices)))286                                 if prices else ""),287                    unit_type=_unit_type(beds.pop() if len(beds) == 1288                                         else None),289                    amenities=it.get("amenities") or [],290                    images=[it["image"]] if it.get("image") else [],291                )292                detail, _f = cache.peek(pid)293                if detail:294                    du.apply_detail(lst, self._std_detail(detail))295                    if detail.get("name"):296                        lst.title = detail["name"][:200]297                if lst.price is None or not (PRICE_MIN <= lst.price298                                             <= PRICE_MAX):299                    continue          # « Call for Rent » : pas affichable300                out[pid] = lst301            return list(out.values())302        finally:303            cache.close()304