# ----------------------------------------------------------------------------- # Rent-Ka — Rental listings aggregator (Canada, outside Québec) # Author: Simon-Pierre Boucher — contact@spboucher.ai # connectors/homestead.py : connecteur Homestead Land Holdings (homestead.ca) # Un des 5 plus gros gestionnaires de l'Ontario (~27 000 unités, siège à # Kingston) : Toronto/GTA, Ottawa, Hamilton, London, Kitchener-Waterloo, # Kingston, Guelph, Sarnia… Le site est une SPA Rentsync « nouvelle # génération » (bundle cdn.rentsync.com/site/homestead_rebuild) qui parle à # la passerelle JSON PUBLIQUE (aucune auth, aucun anti-bot) : # https://website-gateway.rentsync.com/v1/homestead_rebuild/ # properties?limit=500 → 172 immeubles (adresse, # GPS, description, animaux, permaLink, cityId, modified) # cities/property-summary?limit=100 → cityId → nom + province # units?where=buildingId~in:a|b|…,status~in:enabled # → types d'unités avec bed/bath/pi²/prix/dispo/date (séparateur # multi-valeurs : « | » ; paginer via meta.totalPages) # properties/{id}/photos + /utilities → galerie + services inclus # Photos : https://s3.amazonaws.com/lws_lift/homestead/images/gallery/full/… # (clé S3 « homestead », PAS « homestead_rebuild » qui sert au contenu CMS). # On émet UNE annonce par type d'unité DISPONIBLE (available=1), regroupées # par (immeuble, type, cc, sdb) — certains immeubles listent chaque logement. # Fiche (photos+services) via le cache BD self.detail(), clé = modified. # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import os import re from bs4 import BeautifulSoup from ..schema import Listing, normalize_unit_type from .base import BaseConnector _ONTARIO = True # Rent-Ka: always on (ROC scope) BASE = "https://www.homestead.ca" GATEWAY = "https://website-gateway.rentsync.com/v1/homestead_rebuild" # galerie S3 : tailles 512/768/1152/full — « full » validé live IMG_BASE = "https://s3.amazonaws.com/lws_lift/homestead/images/gallery/full" _ISO_DATE = re.compile(r"^20\d{2}-\d{2}-\d{2}") def _slug(s: str) -> str: return re.sub(r"[^a-z0-9]+", "-", (s or "").lower()).strip("-") def _txt(html: str) -> str: """HTML de la passerelle → texte plat.""" if not html: return "" return BeautifulSoup(html, "html.parser").get_text(" ", strip=True) class HomesteadConnector(BaseConnector): source_id = "homestead" request_delay = 0.8 disabled = False max_properties = 300 # garde-fou (172 immeubles au 2026-08) max_images = 15 chunk_size = 25 # immeubles par requête « units » # -- passerelle JSON -------------------------------------------------------- def _api(self, path: str, **params) -> dict: resp = self.get(f"{GATEWAY}/{path}", params=params, headers={"Accept": "application/json", "Origin": BASE, "Referer": BASE + "/"}) return resp.json() def _api_all(self, path: str, **params) -> list[dict]: """Toutes les pages d'une collection (meta.totalPages).""" params.setdefault("limit", 500) out: list[dict] = [] page = 1 while True: d = self._api(path, page=page, **params) out.extend(d.get("data") or []) meta = d.get("meta") or {} total = meta.get("totalPages") or 1 if page >= total: return out page += 1 # -- villes : cityId → (nom, code province) --------------------------------- def _cities(self) -> dict[int, tuple[str, str]]: cities: dict[int, tuple[str, str]] = {} try: for c in self._api_all("cities/property-summary", limit=100): cid = c.get("cityId") if cid is not None: cities[int(cid)] = ((c.get("cityName") or "").strip(), (c.get("provinceCode") or "").strip().upper()) except Exception: pass # repli : ville dérivée du permaLink dans _listing return cities def fetch(self) -> list[Listing]: cities = self._cities() props = self._api_all("properties")[: self.max_properties] by_id = {int(p["id"]): p for p in props if p.get("id") is not None} # types d'unités actifs, par lots d'immeubles (séparateur « | ») units: list[dict] = [] ids = list(by_id) for i in range(0, len(ids), self.chunk_size): chunk = "|".join(str(x) for x in ids[i:i + self.chunk_size]) try: units.extend(self._api_all( "units", where=f"buildingId~in:{chunk},status~in:enabled")) except Exception: continue # regrouper les unités DISPONIBLES par (immeuble, type, cc, sdb) — # certains immeubles publient une ligne par logement individuel groups: dict[tuple, list[dict]] = {} for u in units: if not u.get("available") or u.get("hideSuiteTypeWebsite"): continue bid = int(u.get("buildingId") or 0) if bid not in by_id: continue key = (bid, _slug(u.get("typeName") or ""), u.get("bed"), u.get("bath")) groups.setdefault(key, []).append(u) listings: list[Listing] = [] for key, grp in groups.items(): try: listings.append(self._listing(by_id[key[0]], grp, cities)) except Exception: continue return listings # -- une annonce par type d'unité disponible dans un immeuble --------------- def _listing(self, p: dict, grp: list[dict], cities: dict[int, tuple[str, str]]) -> Listing: pid = int(p["id"]) u0 = grp[0] type_name = (u0.get("typeName") or "").strip() name = (p.get("buildingName") or "").strip() perma = (p.get("permaLink") or "").strip() url = f"{BASE}/residential/{perma}" if perma else BASE # ville : mapping cityId → nom officiel ; repli = dernier segment du slug city, prov = cities.get(int(p.get("cityId") or 0), ("", "ON")) if not city and perma: city = perma.rsplit("-", 1)[-1].replace("-", " ").title() # adresse complète : rue + ville + ON + code postal street = " ".join(x for x in ((p.get("streetNumber") or "").strip(), (p.get("streetName") or "").strip()) if x) postal = (p.get("postal") or "").strip() full_addr = ", ".join(x for x in (street, city) if x) if full_addr: full_addr += f", ON {postal}".rstrip() # coordonnées GPS structurées de la passerelle try: lat = float(p["latitude"]) if p.get("latitude") else None lng = float(p["longitude"]) if p.get("longitude") else None except (TypeError, ValueError): lat = lng = None # prix : plus bas tarif affichable du groupe (0 = prix masqué) rates = [] for u in grp: if u.get("hideRateWebsites"): continue try: r = float(u.get("rate") or 0) except (TypeError, ValueError): r = 0.0 if r > 0: rates.append(r) price = min(rates) if rates else None price_label = "" if price is not None: price_label = (f"À partir de {price:.0f} $" if len(grp) > 1 or (rates and max(rates) != price) else f"{price:.0f} $ /mois") # disponibilité : plus proche date du groupe (None = maintenant) dates = sorted(str(u.get("availabilityDate") or "")[:10] for u in grp if _ISO_DATE.match( str(u.get("availabilityDate") or ""))) avail_date = dates[0] if dates else None availability = (f"Disponible le {avail_date}" if avail_date else "Disponible maintenant") # superficie : plus petite valeur plausible du groupe area = None for u in grp: for k in ("sqFt", "sqFtMin"): try: v = float(u.get(k) or 0) except (TypeError, ValueError): continue if 80 <= v <= 20000 and (area is None or v < area): area = v # chambres / salles de bain : champs structurés de l'unité try: bedrooms = float(u0["bed"]) if u0.get("bed") is not None else None bathrooms = float(u0["bath"]) if u0.get("bath") is not None else None except (TypeError, ValueError): bedrooms = bathrooms = None unit_type = ("Studio" if bedrooms == 0 else normalize_unit_type(type_name)) # animaux : indicateurs structurés de l'immeuble if p.get("petsNotAllowed"): pets = "non" elif p.get("petFriendly"): pets = "oui" else: pets = None # description : aperçu de l'immeuble + détails de suites (HTML → texte) desc = " ".join(x for x in ( _txt(p.get("buildingOverview") or ""), _txt(p.get("suiteDetails") or ""), ) if x)[:800] # commodités : caractéristiques de l'immeuble + services inclus (fiche) amenities: list[str] = [] for t in re.split(r"[\n;•]||
  • ", p.get("buildingFeatures") or ""): t = _txt(t) if t and t not in amenities: amenities.append(t) # champs structurés details: dict = {} contact: dict = {} if (p.get("phone") or "").strip(): contact["phone"] = p["phone"].strip() if (p.get("email") or "").strip(): contact["email"] = p["email"].strip() if contact: details["contact"] = contact if (p.get("neighbourhood") or "").strip(): details["Quartier"] = p["neighbourhood"].strip() # fiche (galerie photo + services inclus) via le cache BD : revisitée # seulement quand l'immeuble change (horodatage « modified ») det_key = hashlib.sha1(str(p.get("modified") or "").encode()).hexdigest() d = self.detail(f"b{pid}", det_key, lambda: self._fetch_detail(pid)) for t in d.get("utilities") or []: t = f"{t} incluse" if t in ("Eau", "Électricité") else t if t and t not in amenities: amenities.append(t) images = list(d.get("images") or []) return Listing( source=self.source_id, external_id=f"{pid}-{_slug(type_name) or 'unite'}-" f"{u0.get('bed')}cc-{u0.get('bath')}sdb", url=url, title=f"{name} — {type_name}" if type_name else name, address=full_addr, sector=(p.get("neighbourhood") or "").strip(), city=city, province=prov or "ON", unit_type=unit_type, bedrooms=bedrooms, bathrooms=bathrooms, price=price, price_label=price_label, availability=availability, availability_date=avail_date, area_sqft=area, pets=pets, description=desc, amenities=amenities[:25], details=details, images=images[: self.max_images], lat=lat, lng=lng, ) # -- fiche immeuble : galerie photo + services inclus (2 appels, cachés) ---- def _fetch_detail(self, pid: int) -> dict: out: dict = {"images": [], "utilities": []} # traductions FR des services inclus les plus fréquents fr = {"Water": "Eau", "Heat": "Chauffage", "Hydro": "Électricité", "Electricity": "Électricité", "Internet": "Internet", "Cable": "Câble"} try: photos = self._api_all(f"properties/{pid}/photos", limit=100) except Exception: photos = [] photos = [ph for ph in photos if ph.get("active") and (ph.get("image") or "").strip()] photos.sort(key=lambda ph: (0 if ph.get("mainGallery") else 1, ph.get("orderBy") or 0)) for ph in photos: u = f"{IMG_BASE}/{ph['image'].strip()}" if u not in out["images"]: out["images"].append(u) out["images"] = out["images"][: self.max_images] try: for ut in self._api_all(f"properties/{pid}/utilities", limit=50): t = (ut.get("name") or "").strip() if t: out["utilities"].append(fr.get(t, t)) except Exception: pass return out