1# -----------------------------------------------------------------------------2# House-Ka — Agrégateur de maisons à vendre (Canada hors Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/realtypress.py : connecteur GÉNÉRIQUE RealtyPress (Canada)5# RealtyPress = plugin WordPress branché sur le flux CREA DDF ; ~35 sites6# d'agences/équipes ontariennes confirmés (recensement 2026-08-27, voir7# docs/ontario-agences-connecteurs.md). Chaque site expose l'IDX/DDF complet8# de son board (OREB, ITSO, KAREA…) en HTML server-rendered, sans anti-bot :9# un seul parseur couvre quasi toute la province.10#11# - liste : archive /listing/page/N/?posts_per_page=100 (100 cartes/page ;12# ⚠ ?posts_per_page directement sur /listing = 301/vide) ; cartes13# class="rps-property-result" (ruban For sale/For rent, prix, adresse,14# ville, caractéristiques) ;15# - fiche : mur CREA « I Accept The Terms » contourné par le cookie16# `disclaimer=accepted` ; tableaux <strong>Label</strong>/valeur (MLS®17# Number, Property Type, Bedrooms…), description « … (id:NNNNN) »,18# lat/lng JSON-LD, photos ddfcdn.realtor.ca ;19# - external_id = ddf<id> (préfixe : jamais de collision avec les n° Centris20# QC) ; sources avec infixe _ag_ → la dédup par external_id masque les21# doublons inter-sites (le même bien DDF publié sur plusieurs sites).22# Sites générés depuis data/canada_agencies.json (un source_id par site).23# -----------------------------------------------------------------------------24from __future__ import annotations2526import html as _html27import json28import os29import re30import urllib.parse31from pathlib import Path3233from .base import BaseConnector34from . import _detailutil as du35from ..schema import PropertyListing3637REGISTRY = Path(__file__).resolve().parent.parent.parent / "data" / "canada_agencies.json"38DETAIL_LIMIT = int(os.environ.get("IMMOKA_RP_DETAIL_LIMIT",39 os.environ.get("IMMOKA_DETAIL_LIMIT", "150")))4041_CARD_RE = re.compile(r'<div class="rps-property-result">')42# fiche = 1er lien de la carte finissant par -<id DDF>/ ; le chemin varie selon43# le site (/listing/, /listings/, /all-regional-listings/…)44_LINK_RE = re.compile(r'href="(https?://[^"]+?-(\d{6,10})/?)"')45_RIBBON_RE = re.compile(r'rps-ribbon[^>]*>\s*([^<]+?)\s*<')46_PRICE_RE = re.compile(r'rps-price[^>]*>\s*\$\s*([\d,]+)')47_H4_RE = re.compile(r"<h4>\s*(.*?)\s*</h4>", re.S)48# avec ou sans <strong> selon le thème du site49_CITY_RE = re.compile(r'city-province-postalcode[^>]*>\s*(?:<strong>\s*)?([^<]+?)\s*<', re.S)50_FEAT_RE = re.compile(r'rps-result-feature-label[^>]*>\s*([^<]+?)\s*<')51_CARD_BROKER_RE = re.compile(r'text-muted[^>]*>\s*<small>\s*([^<]+?)\s*(?:<br|</small>)', re.S)52_DDFIMG_RE = re.compile(r'https://ddfcdn\.realtor\.ca/[^")\'\s\\]+')53_ROW_RE = re.compile(r"<td[^>]*>\s*<strong>([^<]{2,45})</strong>\s*</td>\s*"54 r"<td[^>]*>(.*?)</td>", re.S)55_DESC_RE = re.compile(r'<!--\s*Description\s*-->\s*<p[^>]*>(.*?)</p>', re.S)56_DESC_RE2 = re.compile(r'<p itemprop="description"[^>]*>(.*?)</p>', re.S)57# provinces couvertes (House-Ka = Canada HORS Québec — le Québec vit sur immo-ka)58_PROVINCES = ("Ontario", "British Columbia", "Alberta", "Saskatchewan",59 "Manitoba", "New Brunswick", "Nova Scotia",60 "Prince Edward Island", "Newfoundland and Labrador",61 "Newfoundland & Labrador", "Yukon", "Northwest Territories",62 "Nunavut")63_PROV_ALT = "|".join(_PROVINCES)64_QC_RE = re.compile(r"\bQu[ée]bec\b", re.I)65# ville + province depuis <title> « 3383 Romeo Street, Greater Sudbury (Valley East), Ontario … »66_TITLE_CITY_RE = re.compile(r",\s*([^,<>|]{2,60}),\s*(" + _PROV_ALT + r")\b")67_PRICING_RE = re.compile(r'rps-pricing[^>]*>\s*\$\s*([\d,]+)')68_ID_TAIL_RE = re.compile(r"\s*\(id:\d{4,9}\)\s*$")69_TAG_RE = re.compile(r"<[^>]+>")70_NUM_RE = re.compile(r"[\d,]+(?:\.\d+)?")71_YEAR_RE = re.compile(r"\b(1[6-9]\d{2}|20\d{2})\b")72# territoire couvert (Canada au complet) — même boîte que schema.finalize()73_BBOX = (41.6, 83.2, -141.1, -52.5)747576def _num(s: str) -> float | None:77 m = _NUM_RE.search(s or "")78 if not m:79 return None80 try:81 return float(m.group(0).replace(",", ""))82 except ValueError:83 return None848586class _RealtyPress(BaseConnector):87 """Connecteur générique de site RealtyPress (voir data/canada_agencies.json)."""8889 agency_name = ""90 site_url = ""91 province = "Ontario" # région par défaut des fiches (registre : "province")92 archive = "listing" # chemin de l'archive (revelrealty: "listings",93 # codygroup: "all-regional-listings")94 max_pages = 150 # 100 cartes/page → jusqu'à 15 000 fiches par site95 use_pp = True # False (registre "page_size": 10) : pagination96 # NATIVE 10/page — pour les sites qui plafonnent97 # l'offset de posts_per_page (ex. hanlonrealty98 # ~3 500 items) mais paginent à fond en natif99 request_delay = 0.6100101 def fetch(self) -> list[PropertyListing]:102 # mur CREA des fiches détail : le cookie suffit (posé pour tout domaine,103 # les redirections www/apex restent couvertes)104 self.session.cookies.set("disclaimer", "accepted")105 by_id: dict[str, PropertyListing] = {}106 base = self.site_url.rstrip("/")107 dry = 0108 pp = "?posts_per_page=100" if self.use_pp else ""109 for page in range(1, self.max_pages + 1):110 # page 1 : archive nue — sur certains sites (denisedunnrealtor…)111 # /page/1/ répond 200 SANS cartes au lieu de rediriger112 url = (f"{base}/{self.archive}/{pp}" if page == 1113 else f"{base}/{self.archive}/page/{page}/{pp}")114 try:115 body = self.get(url).text116 except Exception:117 break118 cards = self._cards(body)119 if not cards:120 break121 before = len(by_id)122 for card in cards:123 self._parse_card(card, by_id)124 dry = dry + 1 if len(by_id) == before else 0125 if dry >= 2:126 break127 listings = list(by_id.values())128 du.enrich(self, listings, DETAIL_LIMIT, parse_rp_detail, key="v1")129 for lst in listings:130 # n° MLS du board (fiche détail) — utile à la dédup inter-plateformes131 if not lst.mls and lst.details.get("MLS® Number"):132 lst.mls = str(lst.details["MLS® Number"])133 if not lst.title:134 lst.title = ", ".join(filter(None, (lst.address, lst.city))) \135 or "Propriété à vendre"136 return listings137138 def _cards(self, body: str) -> list[str]:139 marks = list(_CARD_RE.finditer(body))140 return [body[m.start():(marks[i + 1].start() if i + 1 < len(marks)141 else m.start() + 6000)]142 for i, m in enumerate(marks)]143144 def _parse_card(self, card: str, by_id: dict) -> None:145 ml = _LINK_RE.search(card)146 if not ml:147 return148 url, ddf = ml.group(1), ml.group(2)149 eid = f"ddf{ddf}"150 if eid in by_id:151 return152 mr = _RIBBON_RE.search(card)153 ribbon = (mr.group(1) if mr else "").strip().lower()154 if "rent" in ribbon or "lease" in ribbon:155 return # locations : hors périmètre156 lst = PropertyListing(source=self.source_id, external_id=eid, url=url,157 region=self.province, agency=self.agency_name,158 broker_name=self.agency_name)159 ma = _H4_RE.search(card)160 if ma:161 lst.address = _html.unescape(_TAG_RE.sub(" ", ma.group(1))).strip()162 mc = _CITY_RE.search(card)163 if mc:164 raw = _html.unescape(mc.group(1)).strip().rstrip(",")165 # fiches québécoises (Gatineau… dans les pools DDF frontaliers) :166 # hors périmètre House-Ka — elles vivent sur immo-ka167 if _QC_RE.search(raw):168 return169 for prov in _PROVINCES:170 if prov.lower() in raw.lower():171 # forme canonique unique (« & » → « and »)172 lst.region = prov.replace(" & ", " and ")173 raw = re.sub(r",?\s*" + re.escape(prov) + r"\b.*$", "",174 raw, flags=re.I)175 break176 lst.city = raw.split("(")[0].strip()177 mp = _PRICE_RE.search(card)178 if mp:179 lst.price = _num(mp.group(1))180 lst.price_label = f"{mp.group(1)} $"181 for feat in _FEAT_RE.findall(card):182 f = _html.unescape(feat).strip()183 low = f.lower()184 n = _num(f)185 if not n:186 continue187 if "bedroom" in low:188 lst.bedrooms = int(n)189 elif "bathroom" in low:190 lst.bathrooms = int(n)191 elif "sqft" in low or "sq ft" in low or "ft" in low:192 lst.area_sqft = n # plage « 1,100 - 1,500 ft² » : borne basse193 mbk = _CARD_BROKER_RE.search(card)194 if mbk:195 lst.broker_name = _html.unescape(mbk.group(1)).strip()[:120]196 mi = _DDFIMG_RE.search(card)197 if mi:198 lst.images = [mi.group(0)]199 by_id[lst.external_id] = lst200201202def parse_rp_detail(html: str) -> dict:203 """Fiche RealtyPress : tableaux DDF, description, GPS, galerie, courtier."""204 out: dict = {}205 details: dict = {}206207 for lab, val in _ROW_RE.findall(html):208 label = _html.unescape(lab).strip().rstrip(":")209 value = re.sub(r"\s+", " ", _html.unescape(_TAG_RE.sub(" ", val))).strip()210 if label and value and len(value) <= 300:211 details.setdefault(label, value)212213 def dv(*labels: str) -> str:214 for lb in labels:215 if details.get(lb):216 return details[lb]217 return ""218219 b = _num(dv("Bedrooms Total", "Bedrooms", "Bedrooms Above Ground"))220 if b is not None and 0 < b <= 30:221 out["bedrooms"] = int(b)222 b = _num(dv("Bathroom Total", "Bathrooms"))223 if b is not None and 0 < b <= 30:224 out["bathrooms"] = int(b)225 b = _num(dv("Half Bath Total"))226 if b is not None and 0 < b <= 10:227 out["powder_rooms"] = int(b)228 my = _YEAR_RE.search(dv("Constructed Date", "Construction Year", "Age"))229 if my:230 out["year_built"] = int(my.group(1))231 si = dv("Size Interior")232 if si and "sqft" in si.lower().replace(" ", ""):233 a = _num(si) # « 7,901 Sqft » / « 1200 - 1399 sqft »234 if a and a >= 100:235 out["area_sqft"] = a236 pt = dv("Property Type", "Building Type", "Type")237 if pt:238 out["property_type"] = pt # anglais DDF — normalisé par finalize()239 sec = dv("Neigbourhood", "Neighbourhood", "Community Name")240 if sec:241 out["sector"] = sec242243 mp = _PRICING_RE.search(html)244 if mp:245 out["price"] = _num(mp.group(1))246 out["price_label"] = f"{mp.group(1)} $"247248 md = _DESC_RE.search(html) or _DESC_RE2.search(html)249 if md:250 desc = _html.unescape(_TAG_RE.sub(" ", md.group(1)))251 desc = re.sub(r"\s+", " ", desc).strip()252 out["description"] = _ID_TAIL_RE.sub("", desc)[:6000]253254 mt = re.search(r"<title>(.*?)</title>", html, re.S)255 if mt:256 title_txt = _html.unescape(mt.group(1))257 # (fiches québécoises : déjà filtrées au niveau des cartes de liste)258 mc = _TITLE_CITY_RE.search(title_txt)259 if mc and not _QC_RE.search(title_txt):260 # « Greater Sudbury (Valley East) » : le secteur part dans sector261 city = mc.group(1).split("(")[0].strip()262 if city and not any(c.isdigit() for c in city):263 out["city"] = city264 msec = re.search(r"\(([^)]{2,45})\)", mc.group(1))265 if msec and "sector" not in out:266 out["sector"] = msec.group(1).strip()267268 for n in du.ld_nodes(html):269 t = n.get("@type")270 types = set(t if isinstance(t, list) else [t])271 geo = n.get("geo") or {}272 if isinstance(geo, dict) and "lat" not in out:273 try:274 lat, lng = float(geo["latitude"]), float(geo["longitude"])275 if _BBOX[0] <= lat <= _BBOX[1] and _BBOX[2] <= lng <= _BBOX[3]:276 out["lat"], out["lng"] = lat, lng277 except (KeyError, TypeError, ValueError):278 pass279 if types & {"RealEstateAgent", "Organization"}:280 name = str(n.get("name") or "").strip()281 if name and "broker_name" not in out:282 out["broker_name"] = name[:120]283 tel = str(n.get("telephone") or "").strip()284 if tel and "broker_phone" not in out:285 out["broker_phone"] = tel[:40]286 if "lat" not in out:287 m = re.search(r'"latitude"\s*:\s*"?(-?\d{1,2}\.\d{3,})"?\s*,\s*'288 r'"longitude"\s*:\s*"?(-?\d{2,3}\.\d{3,})"?', html)289 if m:290 lat, lng = float(m.group(1)), float(m.group(2))291 if _BBOX[0] <= lat <= _BBOX[1] and _BBOX[2] <= lng <= _BBOX[3]:292 out["lat"], out["lng"] = lat, lng293294 gal = [u for u in dict.fromkeys(_DDFIMG_RE.findall(html))295 if "/listings/" in u.lower()]296 if gal:297 out["images"] = gal[:60]298299 if details:300 out["details"] = details301 return out302303304def _load() -> list[dict]:305 try:306 return json.loads(REGISTRY.read_text(encoding="utf-8"))307 except Exception:308 return []309310311# House-Ka : les connecteurs RealtyPress sont le cœur du site — toujours312# enregistrés (pas de gate IMMOKA_ONTARIO, contrairement à Immo-Ka).313314# Génère une classe par site du registre.315for _ag in _load():316 if not all(_ag.get(k) for k in ("id", "site")):317 continue318 _sid = _ag["id"]319 globals()[f"REALTYPRESS_{_sid.upper()}"] = type(320 "RealtyPress" + "".join(p.title() for p in _sid.split("_")),321 (_RealtyPress,),322 {323 "source_id": _sid,324 "site_url": _ag["site"],325 "agency_name": _ag.get("name", _sid),326 "province": _ag.get("province", "Ontario"),327 "archive": _ag.get("archive", "listing"),328 "use_pp": int(_ag.get("page_size", 100)) >= 100,329 "max_pages": int(_ag.get("max_pages", 150)),330 },331 )332