spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (Québec + Ontario)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/realtypress.py : connecteur GÉNÉRIQUE RealtyPress (Ontario)5#   RealtyPress = plugin WordPress branché sur le flux CREA DDF ; ~35 sites6#   d'agences/équipes ontariennes confirmés (recensement 2026-08-27, voir7#   docs/ontario-agences-connecteurs.md). Chaque site expose l'IDX/DDF complet8#   de son board (OREB, ITSO, KAREA…) en HTML server-rendered, sans anti-bot :9#   un seul parseur couvre quasi toute la province.10#11#   - liste : archive /listing/page/N/?posts_per_page=100 (100 cartes/page ;12#     ⚠ ?posts_per_page directement sur /listing = 301/vide) ; cartes13#     class="rps-property-result" (ruban For sale/For rent, prix, adresse,14#     ville, caractéristiques) ;15#   - fiche : mur CREA « I Accept The Terms » contourné par le cookie16#     `disclaimer=accepted` ; tableaux <strong>Label</strong>/valeur (MLS®17#     Number, Property Type, Bedrooms…), description « … (id:NNNNN) »,18#     lat/lng JSON-LD, photos ddfcdn.realtor.ca ;19#   - external_id = ddf<id> (préfixe : jamais de collision avec les n° Centris20#     QC) ; sources avec infixe _ag_ → la dédup par external_id masque les21#     doublons inter-sites (le même bien DDF publié sur plusieurs sites).22#   Sites générés depuis data/ontario_agencies.json (un source_id par site).23# -----------------------------------------------------------------------------24from __future__ import annotations2526import html as _html27import json28import os29import re30import urllib.parse31from pathlib import Path3233from .base import BaseConnector34from . import _detailutil as du35from ..schema import PropertyListing3637REGISTRY = Path(__file__).resolve().parent.parent.parent / "data" / "ontario_agencies.json"38DETAIL_LIMIT = int(os.environ.get("IMMOKA_RP_DETAIL_LIMIT",39                                  os.environ.get("IMMOKA_DETAIL_LIMIT", "150")))4041_CARD_RE = re.compile(r'<div class="rps-property-result">')42# fiche = 1er lien de la carte finissant par -<id DDF>/ ; le chemin varie selon43# le site (/listing/, /listings/, /all-regional-listings/…)44_LINK_RE = re.compile(r'href="(https?://[^"]+?-(\d{6,10})/?)"')45_RIBBON_RE = re.compile(r'rps-ribbon[^>]*>\s*([^<]+?)\s*<')46_PRICE_RE = re.compile(r'rps-price[^>]*>\s*\$\s*([\d,]+)')47_H4_RE = re.compile(r"<h4>\s*(.*?)\s*</h4>", re.S)48# avec ou sans <strong> selon le thème du site49_CITY_RE = re.compile(r'city-province-postalcode[^>]*>\s*(?:<strong>\s*)?([^<]+?)\s*<', re.S)50_FEAT_RE = re.compile(r'rps-result-feature-label[^>]*>\s*([^<]+?)\s*<')51_CARD_BROKER_RE = re.compile(r'text-muted[^>]*>\s*<small>\s*([^<]+?)\s*(?:<br|</small>)', re.S)52_DDFIMG_RE = re.compile(r'https://ddfcdn\.realtor\.ca/[^")\'\s\\]+')53_ROW_RE = re.compile(r"<td[^>]*>\s*<strong>([^<]{2,45})</strong>\s*</td>\s*"54                     r"<td[^>]*>(.*?)</td>", re.S)55_DESC_RE = re.compile(r'<!--\s*Description\s*-->\s*<p[^>]*>(.*?)</p>', re.S)56_DESC_RE2 = re.compile(r'<p itemprop="description"[^>]*>(.*?)</p>', re.S)57# ville depuis <title> « 3383 Romeo Street, Greater Sudbury (Valley East), Ontario … »58_TITLE_CITY_RE = re.compile(r",\s*([^,<>|]{2,60}),\s*Ontario\b")59_PRICING_RE = re.compile(r'rps-pricing[^>]*>\s*\$\s*([\d,]+)')60_ID_TAIL_RE = re.compile(r"\s*\(id:\d{4,9}\)\s*$")61_TAG_RE = re.compile(r"<[^>]+>")62_NUM_RE = re.compile(r"[\d,]+(?:\.\d+)?")63_YEAR_RE = re.compile(r"\b(1[6-9]\d{2}|20\d{2})\b")64# territoire couvert (Québec + Ontario) — même boîte que schema.finalize()65_BBOX = (41.6, 63.0, -95.5, -56.0)666768def _num(s: str) -> float | None:69    m = _NUM_RE.search(s or "")70    if not m:71        return None72    try:73        return float(m.group(0).replace(",", ""))74    except ValueError:75        return None767778class _RealtyPress(BaseConnector):79    """Connecteur générique de site RealtyPress (voir data/ontario_agencies.json)."""8081    agency_name = ""82    site_url = ""83    archive = "listing"      # chemin de l'archive (revelrealty: "listings",84                             # codygroup: "all-regional-listings")85    max_pages = 150          # 100 cartes/page → jusqu'à 15 000 fiches par site86    request_delay = 0.68788    def fetch(self) -> list[PropertyListing]:89        # mur CREA des fiches détail : le cookie suffit (posé pour tout domaine,90        # les redirections www/apex restent couvertes)91        self.session.cookies.set("disclaimer", "accepted")92        by_id: dict[str, PropertyListing] = {}93        base = self.site_url.rstrip("/")94        dry = 095        for page in range(1, self.max_pages + 1):96            url = f"{base}/{self.archive}/page/{page}/?posts_per_page=100"97            try:98                body = self.get(url).text99            except Exception:100                break101            cards = self._cards(body)102            if not cards:103                break104            before = len(by_id)105            for card in cards:106                self._parse_card(card, by_id)107            dry = dry + 1 if len(by_id) == before else 0108            if dry >= 2:109                break110        listings = list(by_id.values())111        du.enrich(self, listings, DETAIL_LIMIT, parse_rp_detail, key="v1")112        for lst in listings:113            # n° MLS du board (fiche détail) — utile à la dédup inter-plateformes114            if not lst.mls and lst.details.get("MLS® Number"):115                lst.mls = str(lst.details["MLS® Number"])116            if not lst.title:117                lst.title = ", ".join(filter(None, (lst.address, lst.city))) \118                    or "Propriété à vendre"119        return listings120121    def _cards(self, body: str) -> list[str]:122        marks = list(_CARD_RE.finditer(body))123        return [body[m.start():(marks[i + 1].start() if i + 1 < len(marks)124                                else m.start() + 6000)]125                for i, m in enumerate(marks)]126127    def _parse_card(self, card: str, by_id: dict) -> None:128        ml = _LINK_RE.search(card)129        if not ml:130            return131        url, ddf = ml.group(1), ml.group(2)132        eid = f"ddf{ddf}"133        if eid in by_id:134            return135        mr = _RIBBON_RE.search(card)136        ribbon = (mr.group(1) if mr else "").strip().lower()137        if "rent" in ribbon or "lease" in ribbon:138            return                      # locations : hors périmètre139        lst = PropertyListing(source=self.source_id, external_id=eid, url=url,140                              region="Ontario", agency=self.agency_name,141                              broker_name=self.agency_name)142        ma = _H4_RE.search(card)143        if ma:144            lst.address = _html.unescape(_TAG_RE.sub(" ", ma.group(1))).strip()145        mc = _CITY_RE.search(card)146        if mc:147            city = _html.unescape(mc.group(1)).strip().rstrip(",")148            city = re.sub(r",?\s*Ontario\b.*$", "", city, flags=re.I)149            lst.city = city.split("(")[0].strip()150        mp = _PRICE_RE.search(card)151        if mp:152            lst.price = _num(mp.group(1))153            lst.price_label = f"{mp.group(1)} $"154        for feat in _FEAT_RE.findall(card):155            f = _html.unescape(feat).strip()156            low = f.lower()157            n = _num(f)158            if not n:159                continue160            if "bedroom" in low:161                lst.bedrooms = int(n)162            elif "bathroom" in low:163                lst.bathrooms = int(n)164            elif "sqft" in low or "sq ft" in low or "ft" in low:165                lst.area_sqft = n       # plage « 1,100 - 1,500 ft² » : borne basse166        mbk = _CARD_BROKER_RE.search(card)167        if mbk:168            lst.broker_name = _html.unescape(mbk.group(1)).strip()[:120]169        mi = _DDFIMG_RE.search(card)170        if mi:171            lst.images = [mi.group(0)]172        by_id[lst.external_id] = lst173174175def parse_rp_detail(html: str) -> dict:176    """Fiche RealtyPress : tableaux DDF, description, GPS, galerie, courtier."""177    out: dict = {}178    details: dict = {}179180    for lab, val in _ROW_RE.findall(html):181        label = _html.unescape(lab).strip().rstrip(":")182        value = re.sub(r"\s+", " ", _html.unescape(_TAG_RE.sub(" ", val))).strip()183        if label and value and len(value) <= 300:184            details.setdefault(label, value)185186    def dv(*labels: str) -> str:187        for lb in labels:188            if details.get(lb):189                return details[lb]190        return ""191192    b = _num(dv("Bedrooms Total", "Bedrooms", "Bedrooms Above Ground"))193    if b is not None and 0 < b <= 30:194        out["bedrooms"] = int(b)195    b = _num(dv("Bathroom Total", "Bathrooms"))196    if b is not None and 0 < b <= 30:197        out["bathrooms"] = int(b)198    b = _num(dv("Half Bath Total"))199    if b is not None and 0 < b <= 10:200        out["powder_rooms"] = int(b)201    my = _YEAR_RE.search(dv("Constructed Date", "Construction Year", "Age"))202    if my:203        out["year_built"] = int(my.group(1))204    si = dv("Size Interior")205    if si and "sqft" in si.lower().replace(" ", ""):206        a = _num(si)                    # « 7,901 Sqft » / « 1200 - 1399 sqft »207        if a and a >= 100:208            out["area_sqft"] = a209    pt = dv("Property Type", "Building Type", "Type")210    if pt:211        out["property_type"] = pt       # anglais DDF — normalisé par finalize()212    sec = dv("Neigbourhood", "Neighbourhood", "Community Name")213    if sec:214        out["sector"] = sec215216    mp = _PRICING_RE.search(html)217    if mp:218        out["price"] = _num(mp.group(1))219        out["price_label"] = f"{mp.group(1)} $"220221    md = _DESC_RE.search(html) or _DESC_RE2.search(html)222    if md:223        desc = _html.unescape(_TAG_RE.sub(" ", md.group(1)))224        desc = re.sub(r"\s+", " ", desc).strip()225        out["description"] = _ID_TAIL_RE.sub("", desc)[:6000]226227    mt = re.search(r"<title>(.*?)</title>", html, re.S)228    if mt:229        mc = _TITLE_CITY_RE.search(_html.unescape(mt.group(1)))230        if mc:231            # « Greater Sudbury (Valley East) » : le secteur part dans sector232            city = mc.group(1).split("(")[0].strip()233            if city and not any(c.isdigit() for c in city):234                out["city"] = city235                msec = re.search(r"\(([^)]{2,45})\)", mc.group(1))236                if msec and "sector" not in out:237                    out["sector"] = msec.group(1).strip()238239    for n in du.ld_nodes(html):240        t = n.get("@type")241        types = set(t if isinstance(t, list) else [t])242        geo = n.get("geo") or {}243        if isinstance(geo, dict) and "lat" not in out:244            try:245                lat, lng = float(geo["latitude"]), float(geo["longitude"])246                if _BBOX[0] <= lat <= _BBOX[1] and _BBOX[2] <= lng <= _BBOX[3]:247                    out["lat"], out["lng"] = lat, lng248            except (KeyError, TypeError, ValueError):249                pass250        if types & {"RealEstateAgent", "Organization"}:251            name = str(n.get("name") or "").strip()252            if name and "broker_name" not in out:253                out["broker_name"] = name[:120]254            tel = str(n.get("telephone") or "").strip()255            if tel and "broker_phone" not in out:256                out["broker_phone"] = tel[:40]257    if "lat" not in out:258        m = re.search(r'"latitude"\s*:\s*"?(-?\d{1,2}\.\d{3,})"?\s*,\s*'259                      r'"longitude"\s*:\s*"?(-?\d{2,3}\.\d{3,})"?', html)260        if m:261            lat, lng = float(m.group(1)), float(m.group(2))262            if _BBOX[0] <= lat <= _BBOX[1] and _BBOX[2] <= lng <= _BBOX[3]:263                out["lat"], out["lng"] = lat, lng264265    gal = [u for u in dict.fromkeys(_DDFIMG_RE.findall(html))266           if "/listings/" in u.lower()]267    if gal:268        out["images"] = gal[:60]269270    if details:271        out["details"] = details272    return out273274275def _load() -> list[dict]:276    try:277        return json.loads(REGISTRY.read_text(encoding="utf-8"))278    except Exception:279        return []280281282# PAUSE ONTARIO (2026-08-27) : l'expansion ON est suspendue — les classes ne283# s'enregistrent dans le registre CONNECTORS que si IMMOKA_ONTARIO=1 est posé284# (.env). Les annonces déjà en base sont conservées mais dépubliées par285# quality.refresh (raison « pause-ontario »). Rien n'est effacé.286_ONTARIO = os.environ.get("IMMOKA_ONTARIO") == "1"287288# Génère une classe par site du registre.289for _ag in (_load() if _ONTARIO else []):290    if not all(_ag.get(k) for k in ("id", "site")):291        continue292    _sid = _ag["id"]293    globals()[f"REALTYPRESS_{_sid.upper()}"] = type(294        "RealtyPress" + "".join(p.title() for p in _sid.split("_")),295        (_RealtyPress,),296        {297            "source_id": _sid,298            "site_url": _ag["site"],299            "agency_name": _ag.get("name", _sid),300            "archive": _ag.get("archive", "listing"),301            "max_pages": int(_ag.get("max_pages", 150)),302        },303    )304