spb/rent-ka

1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/realtor_ca.py : REALTOR.ca (CREA) — annonces de location publiées5#   par les COURTIERS immobiliers (MLS). C'est LA source pancanadienne des6#   logements mis en location via courtier (condos en particulier), absente des7#   portails de gestionnaires.8#9#   API interne : POST https://api2.realtor.ca/Listing.svc/PropertySearch_Post10#   (form-urlencoded, TransactionTypeId=3 = à louer, PropertySearchTypeId=1 =11#   résidentiel). Protégée par Incapsula/Imperva → chaque requête passe par12#   Scrapfly ASP + pool résidentiel (validé en live le 2026-08-28 : 200 OK,13#   7 779 locations dans le seul Grand Toronto).14#15#   L'API plafonne à MaxRecords=600 par zone (3 pages × 200) : le fetch part de16#   boîtes provinciales (ROC) et les DIVISE récursivement en quadrants tant que17#   TotalRecords > 600 — couverture complète garantie, dédup par Id (les boîtes18#   se chevauchent aux frontières). Les requêtes sont parallélisées (pool de19#   threads, compteur verrouillé, garde-fou max_requests).20#21#   Une annonce = un résultat de recherche : prix (LeaseRent), adresse complète22#   avec GPS, chambres/SDB, photo, remarques publiques, et surtout le COURTIER23#   (Individual + Organization = agence) exposé dans details.contact. Pas de24#   visite des pages détail (40 k fiches × Scrapfly = hors budget) — la photo25#   de couverture et les champs du résultat suffisent à publier.26#   Échec HONNÊTE : si une part significative des requêtes échoue, on lève27#   plutôt que de laisser l'ingestion archiver l'inventaire manquant.28# -----------------------------------------------------------------------------29from __future__ import annotations3031import json32import os33import re34import threading35import time36from concurrent.futures import ThreadPoolExecutor37from urllib.parse import urlencode3839import requests4041from ..schema import Listing, normalize_unit_type42from .base import SCRAPFLY_API, BaseConnector4344_API = "https://api2.realtor.ca/Listing.svc/PropertySearch_Post"45_MAX_RECORDS = 600          # plafond serveur (Paging.MaxRecords)4647# Boîtes de départ — Canada hors Québec, généreuses (le Québec limitrophe est48# filtré au parse via ProvinceName). Chevauchements sans conséquence : dédup49# par Id.50_SEED_BOXES: list[tuple[float, float, float, float]] = [51    # (lat_min, lat_max, lng_min, lng_max)52    (41.6, 47.6, -83.7, -74.3),     # Ontario sud (Windsor→Ottawa)53    (46.0, 57.0, -95.4, -79.0),     # Ontario nord54    (48.2, 60.0, -139.1, -114.0),   # Colombie-Britannique55    (48.9, 60.0, -120.0, -110.0),   # Alberta56    (48.9, 60.0, -110.0, -101.4),   # Saskatchewan57    (48.9, 60.0, -102.2, -95.0),    # Manitoba58    (43.3, 48.3, -69.1, -59.6),     # NB + NS + IPE59    (46.5, 60.5, -67.9, -52.5),     # Terre-Neuve-et-Labrador60    (60.0, 70.5, -141.0, -61.0),    # YT + TNO + NU61]6263_PROVINCE_CODES = {64    "ontario": "ON", "british columbia": "BC", "alberta": "AB",65    "saskatchewan": "SK", "manitoba": "MB", "new brunswick": "NB",66    "nova scotia": "NS", "prince edward island": "PE",67    "newfoundland & labrador": "NL", "newfoundland and labrador": "NL",68    "yukon": "YT", "northwest territories": "NT", "nunavut": "NU",69}7071# loyers non mensuels (rarissimes en résidentiel) : hors sujet72_NON_MONTHLY_RE = re.compile(r"/(week|night|dai|day|year|annum)", re.I)7374# « 600 sqft » ou « 500-599 sqft » — borne basse plausible seulement75_AREA_RE = re.compile(r"(\d[\d,]*)(?:\s*-\s*(\d[\d,]*))?\s*sqft", re.I)7677# types de bâtiment non résidentiels (filet — PropertySearchTypeId=1 est déjà78# résidentiel, mais quelques stationnements/locaux passent la maille)79_NON_RESIDENTIAL_RE = re.compile(80    r"\b(parking|locker|storage|office|retail|commercial|warehouse|"81    r"vacant land|land|farm|business|agriculture)\b", re.I)828384class RealtorCaConnector(BaseConnector):85    """REALTOR.ca — locations MLS des courtiers, Canada hors Québec."""8687    source_id = "realtor_ca"88    use_detail_cache = False    # tout vient des résultats de recherche89    workers = 6                 # requêtes Scrapfly simultanées90    max_requests = 900          # garde-fou budget Scrapfly par sync91    max_depth = 10              # profondeur de subdivision des quadrants92    max_images = 1              # les résultats n'exposent que la couverture9394    def fetch(self) -> list[Listing]:95        self._lock = threading.Lock()96        self._nreq = 097        self._failures = 098        seen: dict[str, Listing] = {}99        queue: list[tuple[tuple[float, float, float, float], int]] = \100            [(b, 0) for b in _SEED_BOXES]101        with ThreadPoolExecutor(max_workers=self.workers) as ex:102            while queue:103                wave, queue = queue[:24], queue[24:]104                probes = list(ex.map(105                    lambda bd: self._search(bd[0], 1), wave))106                page_jobs: list[tuple[tuple, int]] = []107                for (box, depth), data in zip(wave, probes):108                    paging = (data or {}).get("Paging") or {}109                    total = int(paging.get("TotalRecords") or 0)110                    if not total:111                        continue112                    if total > _MAX_RECORDS and depth < self.max_depth:113                        queue.extend((q, depth + 1)114                                     for q in _quadrants(box))115                        continue116                    self._collect(data, seen)117                    pages = min(int(paging.get("TotalPages") or 1), 3)118                    page_jobs += [(box, p) for p in range(2, pages + 1)]119                for d in ex.map(lambda bp: self._search(bp[0], bp[1]),120                                page_jobs):121                    self._collect(d, seen)122        # échec honnête : trop d'échecs = inventaire incomplet, on n'archive pas123        if self._failures > max(3, self._nreq // 20):124            raise RuntimeError(125                f"realtor_ca : {self._failures} échecs sur {self._nreq} "126                f"requêtes — sync abandonné (inventaire incomplet)")127        if self._nreq >= self.max_requests:128            raise RuntimeError(129                f"realtor_ca : garde-fou max_requests ({self.max_requests}) "130                f"atteint — couverture incomplète, sync abandonné")131        return list(seen.values())132133    # -- une requête de recherche (Scrapfly ASP, thread-safe) ------------------134    def _search(self, box: tuple, page: int) -> dict:135        with self._lock:136            if self._nreq >= self.max_requests:137                return {}138            self._nreq += 1139        key = os.environ.get("SCRAPFLY_KEY")140        if not key:141            raise RuntimeError("SCRAPFLY_KEY manquant (voir .env)")142        lat_min, lat_max, lng_min, lng_max = box143        body = urlencode({144            "ZoomLevel": "11",145            "LatitudeMin": f"{lat_min:.5f}", "LatitudeMax": f"{lat_max:.5f}",146            "LongitudeMin": f"{lng_min:.5f}", "LongitudeMax": f"{lng_max:.5f}",147            "Sort": "6-D", "PropertyTypeGroupID": "1",148            "TransactionTypeId": "3", "PropertySearchTypeId": "1",149            "Currency": "CAD", "IncludeHiddenListings": "false",150            "RecordsPerPage": "200", "ApplicationId": "1",151            "CultureId": "1", "Version": "7.0", "CurrentPage": str(page),152        })153        params = {154            "key": key, "url": _API, "asp": "true", "country": "ca",155            "proxy_pool": "public_residential_pool",156            "headers[Content-Type]":157                "application/x-www-form-urlencoded; charset=UTF-8",158            "headers[Referer]": "https://www.realtor.ca/",159            "headers[Origin]": "https://www.realtor.ca",160        }161        for attempt in range(3):162            try:163                resp = requests.post(SCRAPFLY_API, params=params, data=body,164                                     timeout=180)165                res = resp.json().get("result") or {}166                if (res.get("status_code") or 0) == 200:167                    return json.loads(res.get("content") or "{}")168            except (requests.RequestException, ValueError):169                pass170            if attempt < 2:171                time.sleep(4 * (attempt + 1))172        with self._lock:173            self._failures += 1174        return {}175176    # -- accumulation dédupliquée (les boîtes se chevauchent) ------------------177    def _collect(self, data: dict, seen: dict[str, Listing]) -> None:178        for r in (data or {}).get("Results") or []:179            try:180                lst = self._listing(r)181            except Exception:182                continue183            if lst is not None and lst.external_id not in seen:184                seen[lst.external_id] = lst185186    # -- un résultat de recherche -> Listing -----------------------------------187    def _listing(self, r: dict) -> Listing | None:188        prov = _PROVINCE_CODES.get(189            (r.get("ProvinceName") or "").strip().lower())190        if not prov:191            return None                       # Québec (ou inconnu) : hors champ192        prop = r.get("Property") or {}193        bld = r.get("Building") or {}194        btype = (bld.get("Type") or prop.get("Type") or "").strip()195        if _NON_RESIDENTIAL_RE.search(btype):196            return None197        rent = (prop.get("LeaseRent") or "").strip()198        if _NON_MONTHLY_RE.search(rent):199            return None                       # loyer non mensuel : hors sujet200201        pid = str(r.get("Id") or "").strip()202        mls = (r.get("MlsNumber") or "").strip()203        if not pid:204            return None205206        # adresse « unité - rue|Ville (Secteur), Province CodePostal »207        addr = prop.get("Address") or {}208        text = (addr.get("AddressText") or "").strip()209        street, _, tail = text.partition("|")210        city, sector = "", ""211        m = re.match(r"^([^(,]+?)\s*(?:\(([^)]+)\))?\s*,", tail)212        if m:213            city = m.group(1).strip()214            sector = (m.group(2) or "").strip()215        postal = (r.get("PostalCode") or "").strip()216        full_addr = ", ".join(x for x in (street.strip(), city) if x)217        if full_addr:218            full_addr += f", {prov} {postal}".rstrip()219220        try:221            lat = float(addr.get("Latitude") or "")222            lng = float(addr.get("Longitude") or "")223        except (TypeError, ValueError):224            lat = lng = None225226        # prix : valeur brute du flux (mensuelle)227        price = None228        try:229            v = float(str(prop.get("LeaseRentUnformattedValue") or "")230                      .replace(",", ""))231            if 200 <= v <= 100000:232                price = v233        except (TypeError, ValueError):234            pass235        price_label = rent.replace("/Monthly", "/mo") if rent else ""236237        # chambres « 2 + 1 » (TRREB : + den) -> 2 ; « 0 » -> studio238        bedrooms = None239        mb = re.match(r"\s*(\d+)", str(bld.get("Bedrooms") or ""))240        if mb:241            bedrooms = float(mb.group(1))242        bathrooms = None243        try:244            bathrooms = float(bld.get("BathroomTotal") or "")245        except (TypeError, ValueError):246            pass247        if bedrooms is not None:248            unit_type = "Studio" if bedrooms == 0 else \249                normalize_unit_type(f"{int(bedrooms)} bedrooms")250        else:251            unit_type = normalize_unit_type(btype)252253        # superficie : « 600 sqft » ou plage « 500-599 sqft » (borne basse)254        area = None255        for src in (bld.get("SizeInterior") or "",256                    " ".join((fm or {}).get("Area") or ""257                             for fm in bld.get("FloorAreaMeasurements")258                             or [])):259            ma = _AREA_RE.search(src)260            if ma:261                try:262                    v = float(ma.group(1).replace(",", ""))263                    if 80 <= v <= 20000:264                        area = v265                        break266                except ValueError:267                    pass268269        amenities = [a.strip() for a in270                     (bld.get("Ammenities") or "").split(",") if a.strip()]271272        # courtier + agence — la valeur ajoutée « annonce par courtier »273        details: dict = {"mls": mls} if mls else {}274        agents = []275        for ind in r.get("Individual") or []:276            nm = (ind.get("Name") or "").strip()277            if nm and nm not in agents:278                agents.append(nm)279            org = ind.get("Organization") or {}280            if org.get("Name") and "organization" not in details:281                details["organization"] = org["Name"].strip()282                for ph in org.get("Phones") or []:283                    if ph.get("PhoneNumber"):284                        details.setdefault("contact", {})["phone"] = \285                            f"{ph.get('AreaCode', '')}-{ph['PhoneNumber']}" \286                            .strip("-")287                        break288        if agents:289            details.setdefault("contact", {})["name"] = ", ".join(agents[:3])290        if prop.get("ParkingType"):291            details["parking"] = prop["ParkingType"]292        if prop.get("OwnershipType"):293            details["ownership"] = prop["OwnershipType"]294295        images = [p.get("HighResPath") or p.get("MedResPath") or ""296                  for p in prop.get("Photo") or []]297        images = [u for u in images if u][: self.max_images]298299        rel = r.get("RelativeDetailsURL") or r.get("RelativeURLEn") or ""300        title = street.strip() or text301        if unit_type and city:302            title = f"{title} — {city}"303304        return Listing(305            source=self.source_id,306            external_id=pid,307            url=f"https://www.realtor.ca{rel}" if rel else308                "https://www.realtor.ca",309            title=title,310            address=full_addr,311            sector=sector,312            city=city,313            province=prov,314            unit_type=unit_type,315            bedrooms=bedrooms,316            bathrooms=bathrooms,317            price=price,318            price_label=price_label,319            area_sqft=area,320            description=(r.get("PublicRemarks") or "").strip()[:900],321            amenities=amenities[:25],322            details=details,323            images=images,324            lat=lat,325            lng=lng,326        )327328329def _quadrants(box: tuple) -> list[tuple]:330    lat_min, lat_max, lng_min, lng_max = box331    lat_mid = (lat_min + lat_max) / 2332    lng_mid = (lng_min + lng_max) / 2333    return [334        (lat_min, lat_mid, lng_min, lng_mid),335        (lat_min, lat_mid, lng_mid, lng_max),336        (lat_mid, lat_max, lng_min, lng_mid),337        (lat_mid, lat_max, lng_mid, lng_max),338    ]339