spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/progim.py : connecteur Gestion Immobilière Progim5# (progimannonces.bstk.io — plateforme Building Stack, Grand Montréal :6# Montréal, Dorval, Longueuil, Sainte-Julie, Châteauguay, Charlemagne...)7# La page /Listing/Listings embarque `var units = [...]` (JSON complet des8# unités). Le détail d'une unité (photos, équipements, description) vient9# de POST /Listing/ApartmentView avec `id=<ApartmentId>`.10# -----------------------------------------------------------------------------11from __future__ import annotations1213import hashlib14import json15import re16import time1718from bs4 import BeautifulSoup1920from ..schema import Listing, normalize_unit_type, strip_accents21from .base import BaseConnector2223BASE = "https://progimannonces.bstk.io"24LIST_URL = f"{BASE}/Listing/Listings"25VIEW_URL = f"{BASE}/Listing/ApartmentView"2627# Villes de la Communauté métropolitaine de Montréal (Grand Montréal)28# desservies par Progim. Tout le reste (ex. Bromont, Saint-Césaire) est exclu.29GRAND_MTL = {30 "montreal", "laval", "longueuil", "dorval", "sainte-julie", "charlemagne",31 "chateauguay", "brossard", "boucherville", "repentigny", "terrebonne",32 "saint-lambert", "pointe-claire", "kirkland", "beaconsfield", "lachine",33 "verdun", "lasalle", "mont-royal", "westmount", "cote-saint-luc",34 "dollard-des-ormeaux", "pierrefonds", "anjou", "saint-leonard",35 "montreal-nord", "montreal-est", "saint-laurent", "candiac", "la prairie",36 "chambly", "varennes", "sainte-catherine", "delson", "saint-constant",37 "mascouche", "blainville", "mirabel", "saint-eustache", "deux-montagnes",38 "rosemere", "boisbriand", "sainte-therese", "vaudreuil-dorion",39 "l'ile-perrot", "pincourt", "beauharnois", "mercier", "saint-bruno",40 "saint-basile-le-grand", "mcmasterville", "beloeil", "otterburn park",41 "mont-saint-hilaire", "carignan", "richelieu", "l'assomption",42 "saint-sulpice",43}444546def _in_grand_mtl(city: str) -> bool:47 key = strip_accents((city or "").strip().lower())48 return any(key == c or key.startswith(c + "-") for c in GRAND_MTL)495051_CITY_CANON = {"montreal": "Montréal", "chateauguay": "Châteauguay",52 "levis": "Lévis", "quebec": "Québec"}535455def _canon_city(city: str) -> str:56 """Uniformise la graphie ('Montreal' -> 'Montréal')."""57 return _CITY_CANON.get(strip_accents(city.strip().lower()), city.strip())585960# Clés structurées des fragments ApartmentView (attributs `label for=`)61# de la plateforme Building Stack -> champs canoniques Lou-Ka.62_EQUIP_INCLUSIONS = {63 "HeatingIncluded": "heating",64 "HotWaterIncluded": "hot_water",65 "ElectricityIncluded": "electricity",66 "InternetIncluded": "internet",67 "CableIncluded": "cable",68}69_EQUIP_APPLIANCES = {70 "Fridge": "fridge",71 "Stove": "stove",72 "Dishwasher": "dishwasher",73 "DishWasher": "dishwasher",74 "Washer": "washer_dryer",75 "Dryer": "washer_dryer",76}77def _unit_type_from_bedrooms(bedrooms, unit_name: str = "") -> str:78 """Building Stack donne le nb de chambres ; 0 ch -> Studio, n ch -> (n+2)½."""79 try:80 n = int(bedrooms)81 except (TypeError, ValueError):82 return normalize_unit_type(unit_name)83 if n <= 0:84 return "Studio"85 return f"{n + 2}½"868788class ProgimConnector(BaseConnector):89 source_id = "progim"90 request_delay = 0.591 max_details = 120 # garde-fou (une requête ApartmentView par unité)9293 def fetch(self) -> list[Listing]:94 listings: list[Listing] = []95 try:96 html = self.get(LIST_URL).text97 except Exception:98 return listings99100 m = re.search(r"var units = (\[.*?\]);", html, re.S)101 if not m:102 return listings103 try:104 units = json.loads(m.group(1))105 except ValueError:106 return listings107108 blobs: dict[str, dict] = {} # external_id -> unité JSON brute109 for u in units:110 try:111 apt = u.get("Apartment") or {}112 addr = u.get("Address") or {}113 city = (addr.get("City") or u.get("City") or "").strip()114 if not _in_grand_mtl(city):115 continue # hors Grand Montréal (ex. Bromont)116 if not apt.get("IsResidential", True):117 continue # commercial / stationnement118 ext_id = str(apt.get("ApartmentId") or u.get("ApartmentId") or "")119 if not ext_id:120 continue121 price = apt.get("Price")122 price_label = apt.get("PriceFormatted") or ""123 address = addr.get("AddressLine1") or ""124 building = u.get("BuildingName") or address125 unit_name = apt.get("UnitName") or ""126 # placeholders relatifs (/Content/…/residential.svg) exclus127 images = [img for img in128 (u.get("PreviewUrl"), u.get("BuildingPreviewUrl"))129 if img and img.startswith("http")]130 building_url = u.get("BuildingUrl") or ""131 url = BASE + building_url if building_url.startswith("/") \132 else (building_url or LIST_URL)133 lat = lng = None134 try:135 lat = float(addr.get("Latitude"))136 lng = float(addr.get("Longitude"))137 except (TypeError, ValueError):138 pass139140 # Superficie : champ structuré `Apartment.Area` (pi²)141 area = apt.get("Area")142 area_sqft = float(area) if isinstance(area, (int, float)) \143 and 80 <= area <= 20000 else None144145 # Salles de bains (structuré) -> commodité d'affichage146 amenities: list[str] = []147 nb_bath = apt.get("NumberOfBathrooms")148 if isinstance(nb_bath, (int, float)) and nb_bath > 0:149 n = int(nb_bath)150 amenities.append(f"{n} salles de bains" if n > 1151 else "1 salle de bain")152153 # Détails structurés : stationnement/rangement (booléens de la154 # plateforme) + contact de location (nom, téléphone, courriel)155 details: dict = {}156 if isinstance(u.get("ParkingsIsAvailable"), bool):157 details["parking"] = {"available": u["ParkingsIsAvailable"]}158 if isinstance(u.get("StoragesIsAvailable"), bool):159 details["storage"] = u["StoragesIsAvailable"]160 contact: dict = {}161 contacts = (u.get("Building") or {}).get(162 "ListingEmployeesContacts") or []163 if contacts:164 c = contacts[0]165 phone = c.get("FormattedPhoneNumber") or c.get("PhoneNumber")166 if phone:167 digits = re.sub(r"\D", "", phone)[-10:]168 if len(digits) == 10:169 contact["phone"] = (f"{digits[:3]}-{digits[3:6]}"170 f"-{digits[6:]}")171 if c.get("Email"):172 contact["email"] = c["Email"]173 elif (u.get("Building") or {}).get("Phone"):174 digits = re.sub(r"\D", "", u["Building"]["Phone"])[-10:]175 if len(digits) == 10:176 contact["phone"] = (f"{digits[:3]}-{digits[3:6]}"177 f"-{digits[6:]}")178 if contact:179 details["contact"] = contact180181 blobs[ext_id] = u182 listings.append(Listing(183 source=self.source_id,184 external_id=ext_id,185 url=url,186 title=f"{building} — unité {unit_name}" if unit_name187 else building,188 address=address,189 sector="",190 city=_canon_city(city),191 unit_type=_unit_type_from_bedrooms(192 apt.get("NumberOfBedrooms"), unit_name),193 price=float(price) if isinstance(price, (int, float))194 and 100 <= price <= 20000 else None,195 price_label=price_label,196 availability="",197 area_sqft=area_sqft,198 amenities=amenities,199 details=details,200 images=images,201 lat=lat,202 lng=lng,203 ))204 except Exception:205 continue206207 # Détail de chaque unité (fragment ApartmentView via cache self.detail :208 # un vrai POST seulement si l'unité JSON de la liste a changé)209 budget = {"n": 0}210 for lst in listings:211 key = hashlib.sha1(json.dumps(212 blobs.get(lst.external_id, {}), sort_keys=True,213 ensure_ascii=False).encode()).hexdigest()214215 def _fetch(ext_id=lst.external_id):216 if budget["n"] >= self.max_details:217 raise RuntimeError("plafond de fiches atteint")218 budget["n"] += 1219 return self._fetch_view(ext_id)220221 try:222 payload = self.detail(lst.external_id, key, _fetch)223 except Exception:224 payload = {}225 self._apply_view(lst, payload)226227 return listings228229 # -- détail (fragment HTML ApartmentView) ---------------------------------230 def _fetch_view(self, ext_id: str) -> dict:231 """POST /Listing/ApartmentView (throttlé) -> payload structuré."""232 wait = self.request_delay - (time.time() - self._last_request)233 if wait > 0:234 time.sleep(wait)235 resp = self.session.post(VIEW_URL, data={"id": ext_id},236 timeout=self.timeout)237 self._last_request = time.time()238 resp.raise_for_status()239 return self._parse_view(resp.text)240241 @staticmethod242 def _parse_view(frag: str) -> dict:243 payload: dict = {}244 soup = BeautifulSoup(frag, "html.parser")245246 # Photos de l'unité247 imgs = re.findall(248 r'https://wfiles\.buildingstack\.com/resources/image/[A-Za-z0-9]+'249 r'(?:/[a-z]+)?', frag)250 if imgs:251 payload["images"] = list(dict.fromkeys(imgs))252253 # Disponibilité : la plateforme n'affiche que des unités disponibles ;254 # le fragment précise parfois une date ("Disponible dès maintenant!").255 avail = soup.find(string=re.compile(256 r"^\s*Disponible (dès|le|à partir|maintenant|immédiatement)", re.I))257 if avail:258 payload["availability"] = avail.strip()259260 # Général : paires titre/valeur (superficie, étage, chambres, sdb)261 general: dict[str, str] = {}262 for li in soup.select("ul.main-items li"):263 t = li.select_one("p.title")264 v = li.select_one("h4")265 if t and v:266 k = strip_accents(t.get_text(" ", strip=True).lower())267 general.setdefault(k, v.get_text(" ", strip=True))268 m = re.match(r"(\d[\d\s]*(?:[.,]\d+)?)",269 general.get("superficie (pi.ca)", ""))270 if m:271 try:272 area = float(m.group(1).replace(" ", "").replace(",", "."))273 if 80 <= area <= 20000:274 payload["area_sqft"] = area275 except ValueError:276 pass277 if general.get("etage", "").isdigit():278 floor = int(general["etage"])279 if 0 < floor <= 60:280 payload["floor"] = floor281282 # Équipements booléens : `label for=<Clé>` + valeur Oui/Non (clés283 # structurées Building Stack : HeatingIncluded, Furnished, Fridge…)284 inclusions: dict[str, bool] = {}285 appliances: dict[str, bool] = {}286 amenities: list[str] = []287 for li in soup.select("ul.main-items.amentities li"):288 lab = li.find("label")289 v = li.select_one("h4")290 if not lab or not v:291 continue292 key = lab.get("for") or ""293 val = v.get_text(strip=True).strip().lower()294 flag = val in ("oui", "yes")295 if key in _EQUIP_INCLUSIONS:296 inclusions[_EQUIP_INCLUSIONS[key]] = flag297 elif key == "Furnished":298 payload["furnished"] = flag299 if flag:300 amenities.append(lab.get_text(" ", strip=True))301 for lab in soup.select("ul.amentity-links label"):302 key = lab.get("for") or ""303 if key in _EQUIP_APPLIANCES:304 appliances[_EQUIP_APPLIANCES[key]] = True305 txt = lab.get_text(" ", strip=True)306 if txt and txt not in amenities:307 amenities.append(txt)308 if inclusions:309 payload["inclusions"] = inclusions310 if appliances:311 payload["appliances"] = appliances312 if amenities:313 payload["amenities"] = amenities314315 # Description (section Commentaires)316 notes = soup.select_one("div.unit-notes")317 if notes:318 desc = re.sub(r"\s+", " ", notes.get_text(" ", strip=True))319 desc = re.sub(r"^Commentaires\s*", "", desc).strip(" •")320 if desc:321 payload["description"] = desc[:600]322323 return payload324325 @staticmethod326 def _apply_view(lst: Listing, payload: dict) -> None:327 """Applique le payload (frais ou en cache) sur l'annonce."""328 if payload.get("images"):329 lst.images = list(dict.fromkeys(lst.images + payload["images"]))330 lst.availability = payload.get("availability", "Disponible")331 if lst.area_sqft is None and payload.get("area_sqft") is not None:332 lst.area_sqft = payload["area_sqft"]333 if payload.get("floor") is not None:334 lst.details["floor"] = payload["floor"]335 if payload.get("furnished") is not None:336 lst.furnished = payload["furnished"]337 if payload.get("inclusions"):338 lst.details["inclusions"] = {**payload["inclusions"],339 **lst.details.get("inclusions", {})}340 if payload.get("appliances"):341 lst.details["appliances"] = {**payload["appliances"],342 **lst.details.get("appliances", {})}343 for a in payload.get("amenities") or []:344 if a not in lst.amenities:345 lst.amenities.append(a)346 if payload.get("description"):347 lst.description = payload["description"]348