')
# fiche = 1er lien de la carte finissant par -
/ ; le chemin varie selon
# le site (/listing/, /listings/, /all-regional-listings/…)
_LINK_RE = re.compile(r'href="(https?://[^"]+?-(\d{6,10})/?)"')
_RIBBON_RE = re.compile(r'rps-ribbon[^>]*>\s*([^<]+?)\s*<')
_PRICE_RE = re.compile(r'rps-price[^>]*>\s*\$\s*([\d,]+)')
_H4_RE = re.compile(r"\s*(.*?)\s*
", re.S)
# avec ou sans selon le thème du site
_CITY_RE = re.compile(r'city-province-postalcode[^>]*>\s*(?:\s*)?([^<]+?)\s*<', re.S)
_FEAT_RE = re.compile(r'rps-result-feature-label[^>]*>\s*([^<]+?)\s*<')
_CARD_BROKER_RE = re.compile(r'text-muted[^>]*>\s*\s*([^<]+?)\s*(?:)', re.S)
_DDFIMG_RE = re.compile(r'https://ddfcdn\.realtor\.ca/[^")\'\s\\]+')
_ROW_RE = re.compile(r"]*>\s*([^<]{2,45})\s*\s*"
r"]*>(.*?)", re.S)
_DESC_RE = re.compile(r'\s*]*>(.*?)', re.S)
_DESC_RE2 = re.compile(r']*>(.*?)
', re.S)
# provinces couvertes (House-Ka = Canada HORS Québec — le Québec vit sur immo-ka)
_PROVINCES = ("Ontario", "British Columbia", "Alberta", "Saskatchewan",
"Manitoba", "New Brunswick", "Nova Scotia",
"Prince Edward Island", "Newfoundland and Labrador",
"Newfoundland & Labrador", "Yukon", "Northwest Territories",
"Nunavut")
_PROV_ALT = "|".join(_PROVINCES)
_QC_RE = re.compile(r"\bQu[ée]bec\b", re.I)
# ville + province depuis « 3383 Romeo Street, Greater Sudbury (Valley East), Ontario … »
_TITLE_CITY_RE = re.compile(r",\s*([^,<>|]{2,60}),\s*(" + _PROV_ALT + r")\b")
_PRICING_RE = re.compile(r'rps-pricing[^>]*>\s*\$\s*([\d,]+)')
_ID_TAIL_RE = re.compile(r"\s*\(id:\d{4,9}\)\s*$")
_TAG_RE = re.compile(r"<[^>]+>")
_NUM_RE = re.compile(r"[\d,]+(?:\.\d+)?")
_YEAR_RE = re.compile(r"\b(1[6-9]\d{2}|20\d{2})\b")
# territoire couvert (Canada au complet) — même boîte que schema.finalize()
_BBOX = (41.6, 83.2, -141.1, -52.5)
def _num(s: str) -> float | None:
m = _NUM_RE.search(s or "")
if not m:
return None
try:
return float(m.group(0).replace(",", ""))
except ValueError:
return None
class _RealtyPress(BaseConnector):
"""Connecteur générique de site RealtyPress (voir data/canada_agencies.json)."""
agency_name = ""
site_url = ""
province = "Ontario" # région par défaut des fiches (registre : "province")
archive = "listing" # chemin de l'archive (revelrealty: "listings",
# codygroup: "all-regional-listings")
max_pages = 150 # 100 cartes/page → jusqu'à 15 000 fiches par site
use_pp = True # False (registre "page_size": 10) : pagination
# NATIVE 10/page — pour les sites qui plafonnent
# l'offset de posts_per_page (ex. hanlonrealty
# ~3 500 items) mais paginent à fond en natif
request_delay = 0.6
def fetch(self) -> list[PropertyListing]:
# mur CREA des fiches détail : le cookie suffit (posé pour tout domaine,
# les redirections www/apex restent couvertes)
self.session.cookies.set("disclaimer", "accepted")
by_id: dict[str, PropertyListing] = {}
base = self.site_url.rstrip("/")
dry = 0
pp = "?posts_per_page=100" if self.use_pp else ""
for page in range(1, self.max_pages + 1):
# page 1 : archive nue — sur certains sites (denisedunnrealtor…)
# /page/1/ répond 200 SANS cartes au lieu de rediriger
url = (f"{base}/{self.archive}/{pp}" if page == 1
else f"{base}/{self.archive}/page/{page}/{pp}")
try:
body = self.get(url).text
except Exception:
if page > 1:
break
# page 1 : une erreur réseau transitoire ne doit pas devenir un
# sync « réussi » à 0 fiche (rlpheartland/grapevine 2026-08-29 :
# fetch en échec → found=0 en 0,2 s, ok=1) — passer par les
# reprises ci-dessous, qui lèvent si ça persiste
body = ""
if page == 1 and not _CARD_RE.search(body):
# page 1 vide par intermittence (hanlonrealty sert parfois un
# gabarit sans cartes) : réessayer, puis variantes d'URL — un
# échec ici ferait retirer TOUTES les fiches de la source
import time as _t
alts = [(self.archive, url),
(self.archive, f"{base}/{self.archive}/?posts_per_page=100"),
(self.archive, f"{base}/{self.archive}/page/2/{pp}")]
# refonte de site : l'archive change parfois de chemin SANS
# redirection (greybruce 2026-08 : /listing/ → 404 avec widget
# « 12 vedettes », archive réelle déplacée sur /listings/) —
# essayer les autres chemins connus du parc RealtyPress et
# basculer self.archive pour les pages suivantes. Seuil de
# cartes pour ne pas confondre un widget vedette (~12 cartes)
# avec une vraie page d'archive (100/page, 10 en natif).
min_cards = 20 if self.use_pp else 5
alts += [(a, f"{base}/{a}/{pp}")
for a in ("listings", "listing", "all-regional-listings")
if a != self.archive]
orig_archive, accepted = self.archive, False
for arch, alt in alts:
_t.sleep(2.0)
try:
body = self.get(alt).text
except Exception:
continue
n = len(_CARD_RE.findall(body))
if n and (arch == orig_archive or n >= min_cards):
self.archive = arch
accepted = True
break
if not accepted:
raise RuntimeError(
f"{self.source_id}: archive sans cartes après reprises "
"(page vide intermittente ?) — sync abandonné pour "
"protéger l'inventaire")
cards = self._cards(body)
if not cards and page > 1:
# page vide INTERMITTENTE en cours de pagination (greybruce
# 2026-09-01 : sync tronqué à 2 071/7 600 — le site sert
# parfois un gabarit 200 sans cartes, que l'escalade anti-bot
# de get() ne voit pas) : retenter avant de conclure à la fin
# d'archive. Une vraie fin d'archive ne coûte que 2 requêtes
# de plus.
import time as _t
for _ in range(2):
_t.sleep(2.0)
try:
body = self.get(url).text
except Exception:
continue
cards = self._cards(body)
if cards:
break
if not cards:
break
before = len(by_id)
for card in cards:
self._parse_card(card, by_id)
dry = dry + 1 if len(by_id) == before else 0
if dry >= 2:
break
listings = list(by_id.values())
du.enrich(self, listings, DETAIL_LIMIT, parse_rp_detail, key="v1")
for lst in listings:
# n° MLS du board (fiche détail) — utile à la dédup inter-plateformes
if not lst.mls and lst.details.get("MLS® Number"):
lst.mls = str(lst.details["MLS® Number"])
if not lst.title:
lst.title = ", ".join(filter(None, (lst.address, lst.city))) \
or "Propriété à vendre"
return listings
def _cards(self, body: str) -> list[str]:
marks = list(_CARD_RE.finditer(body))
return [body[m.start():(marks[i + 1].start() if i + 1 < len(marks)
else m.start() + 6000)]
for i, m in enumerate(marks)]
def _parse_card(self, card: str, by_id: dict) -> None:
ml = _LINK_RE.search(card)
if not ml:
return
url, ddf = ml.group(1), ml.group(2)
eid = f"ddf{ddf}"
if eid in by_id:
return
mr = _RIBBON_RE.search(card)
ribbon = (mr.group(1) if mr else "").strip().lower()
if "rent" in ribbon or "lease" in ribbon:
return # locations : hors périmètre
lst = PropertyListing(source=self.source_id, external_id=eid, url=url,
region=self.province, agency=self.agency_name,
broker_name=self.agency_name)
ma = _H4_RE.search(card)
if ma:
lst.address = _html.unescape(_TAG_RE.sub(" ", ma.group(1))).strip()
mc = _CITY_RE.search(card)
if mc:
raw = _html.unescape(mc.group(1)).strip().rstrip(",")
# fiches québécoises (Gatineau… dans les pools DDF frontaliers) :
# hors périmètre House-Ka — elles vivent sur immo-ka
if _QC_RE.search(raw):
return
for prov in _PROVINCES:
if prov.lower() in raw.lower():
# forme canonique unique (« & » → « and »)
lst.region = prov.replace(" & ", " and ")
raw = re.sub(r",?\s*" + re.escape(prov) + r"\b.*$", "",
raw, flags=re.I)
break
lst.city = raw.split("(")[0].strip()
mp = _PRICE_RE.search(card)
if mp:
lst.price = _num(mp.group(1))
lst.price_label = f"{mp.group(1)} $"
for feat in _FEAT_RE.findall(card):
f = _html.unescape(feat).strip()
low = f.lower()
n = _num(f)
if not n:
continue
if "bedroom" in low:
lst.bedrooms = int(n)
elif "bathroom" in low:
lst.bathrooms = int(n)
elif "sqft" in low or "sq ft" in low or "ft" in low:
lst.area_sqft = n # plage « 1,100 - 1,500 ft² » : borne basse
mbk = _CARD_BROKER_RE.search(card)
if mbk:
lst.broker_name = _html.unescape(mbk.group(1)).strip()[:120]
mi = _DDFIMG_RE.search(card)
if mi:
lst.images = [mi.group(0)]
by_id[lst.external_id] = lst
def parse_rp_detail(html: str) -> dict:
"""Fiche RealtyPress : tableaux DDF, description, GPS, galerie, courtier."""
out: dict = {}
details: dict = {}
for lab, val in _ROW_RE.findall(html):
label = _html.unescape(lab).strip().rstrip(":")
value = re.sub(r"\s+", " ", _html.unescape(_TAG_RE.sub(" ", val))).strip()
if label and value and len(value) <= 300:
details.setdefault(label, value)
def dv(*labels: str) -> str:
for lb in labels:
if details.get(lb):
return details[lb]
return ""
b = _num(dv("Bedrooms Total", "Bedrooms", "Bedrooms Above Ground"))
if b is not None and 0 < b <= 30:
out["bedrooms"] = int(b)
b = _num(dv("Bathroom Total", "Bathrooms"))
if b is not None and 0 < b <= 30:
out["bathrooms"] = int(b)
b = _num(dv("Half Bath Total"))
if b is not None and 0 < b <= 10:
out["powder_rooms"] = int(b)
my = _YEAR_RE.search(dv("Constructed Date", "Construction Year", "Age"))
if my:
out["year_built"] = int(my.group(1))
si = dv("Size Interior")
if si and "sqft" in si.lower().replace(" ", ""):
a = _num(si) # « 7,901 Sqft » / « 1200 - 1399 sqft »
if a and a >= 100:
out["area_sqft"] = a
pt = dv("Property Type", "Building Type", "Type")
if pt:
out["property_type"] = pt # anglais DDF — normalisé par finalize()
sec = dv("Neigbourhood", "Neighbourhood", "Community Name")
if sec:
out["sector"] = sec
mp = _PRICING_RE.search(html)
if mp:
out["price"] = _num(mp.group(1))
out["price_label"] = f"{mp.group(1)} $"
md = _DESC_RE.search(html) or _DESC_RE2.search(html)
if md:
desc = _html.unescape(_TAG_RE.sub(" ", md.group(1)))
desc = re.sub(r"\s+", " ", desc).strip()
out["description"] = _ID_TAIL_RE.sub("", desc)[:6000]
mt = re.search(r"<title>(.*?)", html, re.S)
if mt:
title_txt = _html.unescape(mt.group(1))
# (fiches québécoises : déjà filtrées au niveau des cartes de liste)
mc = _TITLE_CITY_RE.search(title_txt)
if mc and not _QC_RE.search(title_txt):
# « Greater Sudbury (Valley East) » : le secteur part dans sector
city = mc.group(1).split("(")[0].strip()
if city and not any(c.isdigit() for c in city):
out["city"] = city
msec = re.search(r"\(([^)]{2,45})\)", mc.group(1))
if msec and "sector" not in out:
out["sector"] = msec.group(1).strip()
for n in du.ld_nodes(html):
t = n.get("@type")
types = set(t if isinstance(t, list) else [t])
geo = n.get("geo") or {}
if isinstance(geo, dict) and "lat" not in out:
try:
lat, lng = float(geo["latitude"]), float(geo["longitude"])
if _BBOX[0] <= lat <= _BBOX[1] and _BBOX[2] <= lng <= _BBOX[3]:
out["lat"], out["lng"] = lat, lng
except (KeyError, TypeError, ValueError):
pass
if types & {"RealEstateAgent", "Organization"}:
name = str(n.get("name") or "").strip()
if name and "broker_name" not in out:
out["broker_name"] = name[:120]
tel = str(n.get("telephone") or "").strip()
if tel and "broker_phone" not in out:
out["broker_phone"] = tel[:40]
if "lat" not in out:
m = re.search(r'"latitude"\s*:\s*"?(-?\d{1,2}\.\d{3,})"?\s*,\s*'
r'"longitude"\s*:\s*"?(-?\d{2,3}\.\d{3,})"?', html)
if m:
lat, lng = float(m.group(1)), float(m.group(2))
if _BBOX[0] <= lat <= _BBOX[1] and _BBOX[2] <= lng <= _BBOX[3]:
out["lat"], out["lng"] = lat, lng
gal = [u for u in dict.fromkeys(_DDFIMG_RE.findall(html))
if "/listings/" in u.lower()]
if gal:
out["images"] = gal[:60]
if details:
out["details"] = details
return out
def _load() -> list[dict]:
try:
return json.loads(REGISTRY.read_text(encoding="utf-8"))
except Exception:
return []
# House-Ka : les connecteurs RealtyPress sont le cœur du site — toujours
# enregistrés (pas de gate IMMOKA_ONTARIO, contrairement à Immo-Ka).
# Génère une classe par site du registre.
for _ag in _load():
if not all(_ag.get(k) for k in ("id", "site")):
continue
_sid = _ag["id"]
globals()[f"REALTYPRESS_{_sid.upper()}"] = type(
"RealtyPress" + "".join(p.title() for p in _sid.split("_")),
(_RealtyPress,),
{
"source_id": _sid,
"site_url": _ag["site"],
"agency_name": _ag.get("name", _sid),
"province": _ag.get("province", "Ontario"),
"archive": _ag.get("archive", "listing"),
"use_pp": int(_ag.get("page_size", 100)) >= 100,
"max_pages": int(_ag.get("max_pages", 150)),
},
)