spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/immostar.py : connecteur Immostar (immostar.ca / immostaralouer.ca)5# Le résidentiel d'Immostar vit sur un micro-site par projet :6# - MU (habitermu.ca) et Le Huppé (lehuppe.ca) -> JSON `var fiches`7# - Kwayaweh (kwayaweh.ca) -> SVG + REST do-selecteur8# - Le 1070 Cap-Rouge (le1070.ca) -> SVG + REST do-selecteur9# - Le Maguire (lemaguire.ca) -> cartes .fiche_unite10# - Loges Saint-Nicolas (loges.ca, Lévis) -> cartes .fiche_unite11# Tous les micro-sites partagent la même plateforme : les pages12# /aires-de-vie/ et /votre-espace/ listent les commodités du projet.13# Exclus : Alo Ste-Foy (livraison 2028, rien à louer), Le Florent14# (Trois-Rivières, hors région Québec/Lévis), immostaralouer.ca (commercial).15# -----------------------------------------------------------------------------16from __future__ import annotations1718import hashlib19import json20import re2122from bs4 import BeautifulSoup2324from ..schema import Listing, infer_city, parse_area_sqft, parse_price25from .base import BaseConnector2627_IMG_RE = re.compile(28 r'https?://[^"\'\s\)]+/wp-content/uploads/[^"\'\s\)]+\.(?:jpg|jpeg|webp)',29 re.I)30_SKIP_IMG = re.compile(31 r"logo|icon|favicon|plan|selecteur|niveau-|-\d{2,3}x\d{2,3}\.", re.I)323334def _type_to_demi(raw: str) -> str:35 """'2 chambres (+ bureau)' -> '4½', 'Studio' -> 'Studio'."""36 s = (raw or "").strip().lower()37 if not s:38 return ""39 if "studio" in s:40 return "Studio"41 if "maison" in s or s.startswith("mdv"):42 return "Maison"43 m = re.search(r"(\d)\s*ch", s)44 if m:45 return f"{int(m.group(1)) + 2}½"46 m = re.search(r"(\d)\s*(?:½|1/2)", s)47 if m:48 return f"{m.group(1)}½"49 return raw.strip()505152class ImmostarConnector(BaseConnector):53 source_id = "immostar"54 request_delay = 0.655 max_images = 1556 max_rest_units = 60 # garde-fou REST (le1070)5758 # -- helpers ---------------------------------------------------------------59 def _site_images(self, url: str) -> list[str]:60 try:61 html = self.get(url).text62 except Exception:63 return []64 imgs = [u for u in dict.fromkeys(_IMG_RE.findall(html))65 if not _SKIP_IMG.search(u)]66 return imgs[: self.max_images]6768 def _site_amenities(self, site: str) -> list[str]:69 """Commodités du projet : listes à puces des pages /aires-de-vie/ et70 /votre-espace/ (plateforme commune aux micro-sites Immostar), sinon71 /le-projet/ (Le Maguire). Les <li> contenant des liens (menus,72 pieds de page) sont ignorés."""73 items: list[str] = []74 for page in ("aires-de-vie/", "votre-espace/", "le-projet/"):75 if items and page == "le-projet/":76 break # page de repli seulement77 try:78 html = self.get(site + page).text79 except Exception:80 continue81 soup = BeautifulSoup(html, "html.parser")82 for ul in soup.find_all("ul"):83 if ul.find_parent(("nav", "header", "footer")):84 continue85 cls = " ".join(ul.get("class") or []) + " " + (ul.get("id") or "")86 if re.search(r"menu|nav|footer|cookie", cls, re.I):87 continue88 for li in ul.find_all("li"):89 if li.find("a"):90 continue91 t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))92 t = t.lstrip("—-– ").strip()93 if 3 <= len(t) <= 90 and t not in items:94 items.append(t)95 # variante Loges : info-boxes WPBakery (h3.aio-icon-title)96 for h3 in soup.select("h3.aio-icon-title"):97 t = re.sub(r"\s+", " ", h3.get_text(" ", strip=True))98 if 3 <= len(t) <= 90 and t not in items:99 items.append(t)100 return items[:30]101102 @staticmethod103 def _fiches(html: str) -> list[dict]:104 """Extrait `var fiches = {"data": {...}}` (plateforme MU / Le Huppé)."""105 i = html.find("var fiches = ")106 if i < 0:107 return []108 try:109 data, _ = json.JSONDecoder().raw_decode(html[i + len("var fiches = "):])110 except Exception:111 return []112 return list((data.get("data") or {}).values())113114 def _mk(self, proj: dict, num: str, unit_type_raw: str, price_label: str,115 availability: str, desc: str, images: list[str],116 area_sqft: float | None = None, title_num: str = "") -> Listing:117 sector = proj["sector"]118 return Listing(119 source=self.source_id,120 external_id=f"{proj['id']}-{num}",121 url=f"{proj['plans'][0]}#unite-{num}",122 title=f"{proj['name']} — Unité {title_num or num}",123 address=proj.get("address", ""),124 sector=sector,125 city=infer_city(sector),126 unit_type=_type_to_demi(unit_type_raw),127 price=parse_price(price_label),128 price_label=price_label,129 availability=availability,130 area_sqft=area_sqft,131 description=desc[:600],132 amenities=list(proj.get("amenities", [])),133 images=images,134 )135136 @staticmethod137 def _sqft(raw) -> float | None:138 """'630' / '1 102' -> pi² (None si absent ou implausible)."""139 s = str(raw or "").strip().replace(" ", "").replace(",", ".")140 try:141 v = float(s)142 except ValueError:143 return None144 return v if 80 <= v <= 20000 else None145146 # -- parseurs par plateforme ------------------------------------------------147 def _parse_fiches(self, proj: dict, html: str,148 images: list[str]) -> list[Listing]:149 out = []150 for f in self._fiches(html):151 dispo = f.get("disponibilite")152 if isinstance(dispo, dict): # variante MU153 dispo = dispo.get("value")154 if dispo != "available":155 continue156 num = str(f.get("numero") or "").strip()157 if not num:158 continue159 t = f.get("type") or ""160 if isinstance(t, dict): # variante MU161 t = t.get("label", "")162 if str(f.get("bureau")) == "1":163 t += " + bureau"164 prix = str(f.get("prix") or "").strip()165 sup = str(f.get("superficie") or "").strip()166 terrasse = str(f.get("superficie_terrasse") or "").strip()167 chambres = str(f.get("chambres") or "").strip()168 sdb = str(f.get("salles_de_bain") or "").strip()169 caract = str(f.get("caracteristiques") or "").strip()170 desc = " — ".join(x for x in [171 t, f"{sup} pi²" if sup else "",172 f"terrasse {terrasse} pi²" if terrasse else "",173 f"{chambres} chambre(s)" if chambres else "",174 f"{sdb} salle(s) de bain" if sdb else "",175 caract if caract and caract.lower() != "regulier" else "",176 f"étage {f.get('etage')}" if f.get("etage") else ""] if x)177 # plan de l'unité en tête de galerie (si image, pas PDF)178 imgs = list(images)179 plan = str(f.get("plan") or "")180 if re.search(r"\.(?:jpe?g|png|webp)$", plan, re.I):181 imgs.insert(0, plan)182 out.append(self._mk(183 proj, num, t,184 f"{prix}$ / mois" if prix else "",185 f"Libre {f['date']}" if f.get("date") else "Disponible",186 desc, imgs, area_sqft=self._sqft(sup)))187 return out188189 def _parse_do_rest(self, proj: dict, html: str,190 images: list[str]) -> list[Listing]:191 """Kwayaweh / Le 1070 : éléments SVG `disponible` data-numero= + REST192 /wp-json/do-selecteur-plans/v1/unite?id=N (prix, superficie, type,193 date de libération, plan). Les éléments Kwayaweh portent194 data-prix/type/promotion : cette carte sert de clé de cache BD195 (self.detail) pour ne re-frapper le REST que si l'unité a changé."""196 base = re.match(r"https?://[^/]+", proj["plans"][0]).group(0)197 soup = BeautifulSoup(html, "html.parser")198 cards: list[tuple[str, dict]] = []199 seen: set[str] = set()200 for el in soup.select("[data-numero]"):201 cls = " ".join(el.get("class") or [])202 if not re.search(r"\b(disponible|available)\b", cls) \203 or "indisponible" in cls:204 continue205 num = (el.get("data-numero") or "").strip()206 if not num or num in seen:207 continue208 seen.add(num)209 attrs = {k: str(v) for k, v in el.attrs.items()210 if k.startswith("data-")}211 attrs["class"] = cls212 cards.append((num, attrs))213214 uid_from_numero = proj.get("uid_field") == "numero"215 out = []216 for num, attrs in cards[: self.max_rest_units]:217 def _fetch(n=num):218 if self._rest_fetched >= self.max_rest_units * 2:219 raise RuntimeError("budget de requêtes REST atteint")220 self._rest_fetched += 1221 return self.get(222 f"{base}/wp-json/do-selecteur-plans/v1/unite",223 params={"id": n}).json()224 try:225 if uid_from_numero and "data-prix" in attrs:226 # uid connu d'avance -> cache BD (clé = attributs SVG)227 key = hashlib.sha1(json.dumps(228 attrs, sort_keys=True).encode()).hexdigest()[:20]229 data = self.detail(f"{proj['id']}-{num}", key, _fetch)230 else:231 data = _fetch()232 except Exception:233 continue234 if not data or data.get("etat") != "disponible":235 continue236 unite = str(data.get("unite") or num)237 t = str(data.get("type") or "")238 sup = str(data.get("superficie_habitable") or "").strip()239 terrasse = str(data.get("superficie_terrasse") or "").strip()240 chambres = str(data.get("nb_chambres") or "").strip()241 style = str(data.get("style") or "").strip()242 parking = str(data.get("stationnement") or "").strip()243 desc = " — ".join(x for x in [244 t, f"{sup} pi²" if sup else "",245 f"terrasse {terrasse} pi²" if terrasse else "",246 f"{chambres} chambre(s)" if chambres else "",247 style,248 f"stationnement {parking}" if parking else "",249 f"étage {attrs.get('data-etage')}"250 if attrs.get("data-etage") else ""] if x)251 date_libre = str(data.get("date_libre") or "").strip()252 a_partir = str(data.get("a_partir") or "").strip()253 avail = (f"Libre {date_libre}" if date_libre254 else a_partir or "Disponible")255 imgs = list(images)256 for g in (data.get("gallery") or []):257 gu = g.get("url") if isinstance(g, dict) else str(g or "")258 if gu and gu.startswith("http") and gu not in imgs:259 imgs.insert(0, gu)260 montant = str(data.get("montant") or "").strip()261 out.append(self._mk(262 proj, num if uid_from_numero else unite,263 t or str(data.get("type_demi") or ""),264 f"À partir de {montant}" if montant else "",265 avail, desc, imgs,266 area_sqft=self._sqft(sup), title_num=unite))267 return out268269 def _parse_maguire(self, proj: dict, html: str,270 images: list[str]) -> list[Listing]:271 """Le Maguire : cartes `.fiche_unite.available` rendues serveur272 (prix, superficies, chambres, plan JPG de l'unité)."""273 soup = BeautifulSoup(html, "html.parser")274 out = []275 for card in soup.select(".fiche_unite"):276 cls = card.get("class") or []277 if "available" not in cls or "rented" in cls:278 continue279 num = (card.get("data-unite") or "").strip()280 if not num:281 continue282 prix_el = card.select_one(".prix")283 price_label = prix_el.get_text(" ", strip=True) if prix_el else ""284 sup_el = card.select_one("p.superficie:not(.terrasse)")285 sup_txt = sup_el.get_text(" ", strip=True) if sup_el else ""286 terr_el = card.select_one("p.superficie.terrasse")287 terr_txt = terr_el.get_text(" ", strip=True) if terr_el else ""288 notes_el = card.select_one(".notes")289 t = notes_el.get_text(" ", strip=True) if notes_el else ""290 dispo = card.select_one(".disponibilite")291 etage = (card.get("data-etage") or "").strip()292 desc = " — ".join(x for x in [293 t, sup_txt, terr_txt,294 f"étage {etage}" if etage else ""] if x)295 imgs = list(images)296 plan = card.select_one("a.pdf[href]")297 if plan and re.search(r"\.(?:jpe?g|png|webp)$",298 plan.get("href") or "", re.I):299 imgs.insert(0, plan["href"])300 out.append(self._mk(301 proj, num, t, price_label,302 dispo.get_text(strip=True) if dispo else "Disponible",303 desc, imgs, area_sqft=parse_area_sqft(sup_txt)))304 return out305306 def _parse_loges(self, proj: dict, html: str,307 images: list[str]) -> list[Listing]:308 """Loges : cartes `.affichage.liste .fiche_unite` rendues serveur."""309 soup = BeautifulSoup(html, "html.parser")310 out = []311 for card in soup.select(".affichage.liste .fiche_unite"):312 num = (card.get("data-unite") or "").strip()313 h4 = card.select_one("h4")314 head = h4.get_text(" ", strip=True) if h4 else ""315 if not num or "$" not in head:316 continue # unité louée ou sans prix317 info = card.select_one("header p")318 info_txt = info.get_text(" ", strip=True) if info else ""319 m = re.search(r"(\d)\s*chambre", info_txt)320 t = f"{m.group(1)} chambres" if m else ""321 dispo = card.select_one(".disponibilite")322 desc_el = card.select_one("main .description")323 desc_txt = (re.sub(r"\s+", " ", desc_el.get_text(" ", strip=True))324 if desc_el else "")325 desc = " — ".join(x for x in [info_txt, desc_txt] if x)326 price_m = re.search(r"À partir de\s*([\d\s,.]+\$)", head)327 price_label = (f"À partir de {price_m.group(1)}/mois"328 if price_m else head)329 out.append(self._mk(330 proj, num, t, price_label,331 dispo.get_text(strip=True) if dispo else "Disponible",332 desc, images, area_sqft=parse_area_sqft(info_txt)))333 return out334335 # -- fetch -----------------------------------------------------------------336 def fetch(self) -> list[Listing]:337 projects = [338 {"id": "mu", "name": "MU", "sector": "Sainte-Foy",339 "address": "2605, boulevard Laurier, Québec",340 "site": "https://habitermu.ca/",341 "plans": ["https://habitermu.ca/plans/"], "mode": "fiches"},342 {"id": "huppe", "name": "Le Huppé", "sector": "Lebourgneuf",343 "address": "5055, boulevard des Gradins, Québec",344 "site": "https://lehuppe.ca/",345 "plans": ["https://lehuppe.ca/plans/"], "mode": "fiches"},346 {"id": "1070", "name": "Le 1070 Cap-Rouge", "sector": "Cap-Rouge",347 "address": "1070, boulevard de la Chaudière, Québec",348 "site": "https://le1070.ca/",349 "plans": ["https://le1070.ca/plans/"], "mode": "do_rest"},350 {"id": "kwayaweh", "name": "Kwayaweh", "sector": "Wendake",351 "address": "Wendake",352 "site": "https://kwayaweh.ca/",353 "plans": ["https://kwayaweh.ca/selecteur/"], "mode": "do_rest",354 "uid_field": "numero"}, # uid historique = data-numero (post id)355 {"id": "maguire", "name": "Le Maguire sur l'avenue",356 "sector": "Sillery", "address": "Avenue Maguire, Québec",357 "site": "https://lemaguire.ca/",358 "plans": ["https://lemaguire.ca/les-plans/"], "mode": "maguire"},359 {"id": "loges-555", "name": "Loges Saint-Nicolas (Le 555)",360 "sector": "Saint-Nicolas", "address": "555, rue Marie-Victorin, Lévis",361 "site": "https://loges.ca/",362 "plans": ["https://loges.ca/plans/le-555/"], "mode": "loges"},363 {"id": "loges-550", "name": "Loges Saint-Nicolas (Le 550)",364 "sector": "Saint-Nicolas", "address": "550, rue Jérôme-Demers, Lévis",365 "site": "https://loges.ca/",366 "plans": ["https://loges.ca/plans/le-550/"], "mode": "loges"},367 {"id": "loges-600", "name": "Loges Saint-Nicolas (Le 600)",368 "sector": "Saint-Nicolas", "address": "600, rue Pierre-Perrault, Lévis",369 "site": "https://loges.ca/",370 "plans": ["https://loges.ca/plans/le-600/"], "mode": "loges"},371 ]372 parsers = {373 "fiches": self._parse_fiches,374 "do_rest": self._parse_do_rest,375 "maguire": self._parse_maguire,376 "loges": self._parse_loges,377 }378379 self._rest_fetched = 0380 listings: list[Listing] = []381 site_imgs: dict[str, list[str]] = {}382 site_amen: dict[str, list[str]] = {}383 for proj in projects:384 try:385 site = proj["site"]386 if site not in site_imgs:387 site_imgs[site] = self._site_images(site)388 if site not in site_amen:389 site_amen[site] = self._site_amenities(site)390 images = site_imgs[site]391 proj["amenities"] = site_amen[site]392 for plans_url in proj["plans"]:393 html = self.get(plans_url).text394 listings.extend(parsers[proj["mode"]](proj, html, images))395 except Exception:396 continue397398 # dédup par external_id (sécurité)399 uniq: dict[str, Listing] = {}400 for lst in listings:401 uniq.setdefault(lst.external_id, lst)402 return list(uniq.values())403