spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/interrent.py : connecteur InterRent REIT (irent.com)5# Site Next.js (app router) : la page « communities/city/montreal » embarque6# dans son flux RSC (self.__next_f.push) le JSON complet des communautés7# (adresse, quartier, photos, commodités Amenities/Utilities, animaux8# PetFriendly*, stationnement, contact, description via référence $NN)9# avec leurs suites disponibles (type, chambres, sdb, pi², loyer, date).10# Une seule requête suffit.11# REIT pancanadien : filtre strict sur les villes du Grand Montréal.12# -----------------------------------------------------------------------------13from __future__ import annotations1415import html as htmllib16import json17import re1819from ..schema import Listing, strip_accents20from .base import BaseConnector2122BASE = "https://www.irent.com"23CITY_URL = f"{BASE}/communities/city/montreal"2425CHUNK_RE = re.compile(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)')26OBJ_START_RE = re.compile(r'\{"Id":\d+,"ImportId"')2728# Villes admissibles (Grand Montréal), clés sans accents/minuscules29_GM_CITIES = {30 "montreal": "Montréal",31 "cote-saint-luc": "Côte-Saint-Luc",32 "cote saint-luc": "Côte-Saint-Luc",33 "brossard": "Brossard",34 "laval": "Laval",35 "longueuil": "Longueuil",36 "verdun": "Montréal",37 "lasalle": "Montréal",38 "pointe-claire": "Pointe-Claire",39 "dollard-des-ormeaux": "Dollard-des-Ormeaux",40}4142_BED_TYPES = {0: "Studio", 1: "3½", 2: "4½", 3: "5½", 4: "6½"}434445def _unescape_js(s: str) -> str:46 """Déséchappe une chaîne JS du flux RSC (\\" \\n \\uXXXX...)."""47 try:48 return json.loads(f'"{s}"')49 except ValueError:50 try:51 return (s.encode("latin-1", "backslashreplace")52 .decode("unicode_escape"))53 except Exception:54 return s555657def _clean(s: str) -> str:58 """Déséchappe les entités HTML répétées (&amp;amp;...)."""59 s = s or ""60 for _ in range(4):61 t = htmllib.unescape(s)62 if t == s:63 break64 s = t65 return s.strip()666768def _as_list(node) -> list:69 """Les nœuds XML->JSON du flux : dict simple ou liste."""70 if isinstance(node, list):71 return node72 if isinstance(node, dict):73 return [node]74 return []757677def _strip_html(s: str) -> str:78 """HTML -> texte plat (entités déjà gérées par _clean)."""79 return _clean(re.sub(r"<[^>]+>", " ", s or ""))808182# Références texte du flux RSC : « \nNN:Txxx, » suivi de xxx (hex) caractères.83_TEXT_REF_RE = re.compile(r"\n(\d+):T([0-9a-f]+),")848586def _text_refs(blob: str) -> dict[str, str]:87 """Table des chaînes référencées « $NN » (descriptions d'immeubles…)."""88 refs: dict[str, str] = {}89 for m in _TEXT_REF_RE.finditer(blob):90 n = int(m.group(2), 16)91 refs[m.group(1)] = blob[m.end():m.end() + n]92 return refs939495# Utilities (services inclus au bail) -> clés canoniques d'inclusions96def _utility_key(u: str) -> str | None:97 s = u.lower()98 if "hot water" in s:99 return "hot_water"100 if "heat" in s:101 return "heating"102 if "hydro" in s or "electric" in s:103 return "electricity"104 if "internet" in s:105 return "internet"106 if "cable" in s:107 return "cable"108 return None # « Water » (eau froide) : pas de clé canonique109110111class InterrentConnector(BaseConnector):112 source_id = "interrent"113 request_delay = 0.6114 max_images = 20115116 def fetch(self) -> list[Listing]:117 page = self.get(CITY_URL).text118 blob = "".join(_unescape_js(c) for c in CHUNK_RE.findall(page))119120 # Objets communauté : {"Id":N,"ImportId":...,"PermaLink":...}121 communities: dict[int, dict] = {}122 pos = 0123 while True:124 m = OBJ_START_RE.search(blob, pos)125 if not m:126 break127 obj, end = self._read_object(blob, m.start())128 pos = end if end > m.start() else m.start() + 1129 if not obj or "PermaLink" not in obj or "Location" not in obj:130 continue131 cid = obj.get("Id")132 if isinstance(cid, int) and cid not in communities:133 communities[cid] = obj134135 refs = _text_refs(blob)136 listings: list[Listing] = []137 for c in communities.values():138 try:139 listings.extend(self._community_listings(c, refs))140 except Exception:141 continue142 return listings143144 @staticmethod145 def _read_object(blob: str, start: int) -> tuple[dict | None, int]:146 """Extrait un objet JSON par appariement d'accolades."""147 depth = 0148 in_str = False149 esc = False150 for i in range(start, min(len(blob), start + 400_000)):151 ch = blob[i]152 if in_str:153 if esc:154 esc = False155 elif ch == "\\":156 esc = True157 elif ch == '"':158 in_str = False159 continue160 if ch == '"':161 in_str = True162 elif ch == "{":163 depth += 1164 elif ch == "}":165 depth -= 1166 if depth == 0:167 try:168 return json.loads(blob[start:i + 1]), i + 1169 except ValueError:170 return None, i + 1171 return None, start + 1172173 def _community_listings(self, c: dict,174 refs: dict[str, str] | None = None) -> list[Listing]:175 refs = refs or {}176 loc = c.get("Location") or {}177 if (loc.get("ProvinceCode") or "").upper() != "QC":178 return []179 raw_city = _clean(loc.get("City") or "")180 key = strip_accents(raw_city.lower()).strip()181 city = _GM_CITIES.get(key)182 if not city:183 return [] # hors Grand Montréal184 sector = _clean(loc.get("Neighbourhood") or c.get("TagLine") or "")185 if strip_accents(sector.lower()) == strip_accents(city.lower()):186 sector = "" if city != "Montréal" else sector187 if key in ("cote-saint-luc", "verdun", "lasalle") and not sector:188 sector = raw_city if city == "Montréal" else ""189190 name = _clean(c.get("Name") or "")191 url = c.get("Url") or f"{BASE}/communities/{c.get('PermaLink', '')}"192 address = _clean(loc.get("Address") or "")193 postal = _clean(loc.get("PostalCode") or "")194195 # Photos de la communauté196 images: list[str] = []197 photos = (c.get("Photos") or {})198 for p in _as_list(photos.get("Photo") if isinstance(photos, dict)199 else photos):200 u = (p or {}).get("Url") or ""201 if u.startswith("http") and u not in images:202 images.append(u)203 images = images[: self.max_images]204205 # Commodités : nœud Amenities.Amenity (liste) + services inclus206 am_node = (c.get("Amenities") or {})207 amenities = [_clean(str(a)) for a in208 _as_list(am_node.get("Amenity") if isinstance(am_node, dict)209 else am_node) if a]210 if not amenities: # ancien champ, gardé en repli211 amenities = [a.strip() for a in212 (c.get("amenities_TextField") or "").split(",")213 if a.strip()]214 ut_node = (c.get("Utilities") or {})215 utilities = [_clean(str(u)) for u in216 _as_list(ut_node.get("Utility") if isinstance(ut_node, dict)217 else ut_node) if u]218 if utilities:219 amenities.append("Utilities included: " + ", ".join(utilities))220 amenities = amenities[:25]221222 # Détails structurés communs à la communauté223 details: dict = {}224 inclusions = {}225 for u in utilities:226 k = _utility_key(u)227 if k:228 inclusions[k] = True229 if inclusions:230 details["inclusions"] = inclusions231232 # Animaux : indicateurs PetFriendly* (structurés à la source)233 pets = None234 if c.get("PetFriendlyNotAllowed"):235 pets = "non"236 elif c.get("PetFriendly"):237 sub = [c.get("PetFriendlyCats"), c.get("PetFriendlySmallDogs"),238 c.get("PetFriendlyLargeDogs")]239 pets = "conditions" if any(sub) and not all(sub) else "oui"240241 # Contact du bureau de location242 ci = c.get("ContactInformation") or {}243 contact = {}244 if ci.get("Phone"):245 contact["phone"] = _clean(ci["Phone"])246 if ci.get("Email"):247 contact["email"] = _clean(ci["Email"])248 if contact:249 details["contact"] = contact250251 # Stationnement : champ ParkingDetails (« Indoor Parking: $150 / month »)252 parking_txt = _strip_html(c.get("ParkingDetails") or "")253 if parking_txt:254 if re.search(r"no parking|not available", parking_txt, re.I):255 details["parking"] = {"available": False}256 else:257 parking: dict = {"available": True}258 indoor = re.search(r"indoor|interior|underground",259 parking_txt, re.I)260 outdoor = re.search(r"outdoor|exterior|surface",261 parking_txt, re.I)262 if indoor and not outdoor:263 parking["type"] = "intérieur"264 elif outdoor and not indoor:265 parking["type"] = "extérieur"266 prices = [float(p) for p in267 re.findall(r"\$\s*(\d{2,4})", parking_txt)]268 if re.search(r"included|free", parking_txt, re.I):269 parking["included"] = True270 elif prices:271 parking["included"] = False272 parking["price"] = min(prices)273 details["parking"] = parking274275 # Description de l'immeuble (référence $NN du flux RSC) + promo + animaux276 desc_ref = str(c.get("BuildingDescription") or "")277 building_desc = ""278 if desc_ref.startswith("$"):279 building_desc = _strip_html(refs.get(desc_ref[1:], ""))280 elif desc_ref:281 building_desc = _strip_html(desc_ref)282 promo = (c.get("Promotions") or {})283 promo_title = _clean(promo.get("Title") or "") \284 if isinstance(promo, dict) else ""285 pet_details = _strip_html(c.get("PetDetails") or "")286 extra_desc = " — ".join(x for x in [287 f"Promotion : {promo_title}" if promo_title else "",288 building_desc,289 f"Stationnement : {parking_txt}" if parking_txt else "",290 f"Animaux : {pet_details}" if pet_details else ""] if x)291292 suites = (c.get("Suites") or {})293 out: list[Listing] = []294 for s in _as_list(suites.get("Suite") if isinstance(suites, dict)295 else suites):296 try:297 if (s.get("Available") or "").lower() != "yes":298 continue299 type_name = _clean(s.get("TypeName") or "")300 # pseudo-unités « Promotional Price » sans numéro : ignorées301 if "promotional" in type_name.lower() and not s.get("Number"):302 continue303 rate = s.get("Rate")304 price = float(rate) if isinstance(rate, (int, float)) \305 and rate else None306 beds = s.get("Bedrooms")307 unit_type = _BED_TYPES.get(beds, "") \308 if isinstance(beds, int) else ""309 sqft = s.get("SquareFeet")310 baths = s.get("Bathrooms")311 bits = []312 if sqft:313 bits.append(f"{sqft} pi²")314 if baths:315 bits.append(f"{baths} sdb")316 if type_name:317 bits.append(f"plan {type_name}")318 suite_desc = _strip_html(s.get("Description") or "")319 if suite_desc:320 bits.append(suite_desc)321 if extra_desc:322 bits.append(extra_desc)323324 # plan d'étage en tête de galerie s'il existe325 imgs = list(images)326 fps = (s.get("Floorplans") or {})327 for fp in _as_list(fps.get("Floorplan")328 if isinstance(fps, dict) else fps):329 fu = (fp or {}).get("Image") or ""330 if fu.startswith("http") and fu not in imgs:331 imgs.insert(0, fu)332333 num = s.get("Number") or ""334 label = f"{name} — {type_name}" if type_name else name335 if num:336 label += f" (app. {num})"337 out.append(Listing(338 source=self.source_id,339 external_id=str(s.get("Id")),340 url=url,341 title=label,342 address=", ".join(x for x in [address, city, postal]343 if x),344 sector=sector,345 city=city,346 unit_type=unit_type,347 price=price,348 price_label=f"{int(rate)} $/mois" if price else "",349 area_sqft=float(sqft) if isinstance(sqft, (int, float))350 and 80 <= sqft <= 20000 else None,351 availability=_clean(s.get("AvailabilityDate") or ""),352 pets=pets,353 description=" — ".join(bits)[:900],354 amenities=list(amenities),355 details=dict(details),356 images=imgs[: self.max_images + 1],357 lat=loc.get("Latitude"),358 lng=loc.get("Longitude"),359 ))360 except Exception:361 continue362 return out363