# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/sibelanger.py : connecteur Société immobilière Bélanger # (sibelanger.com) — page /appartements-a-louer/ : cartes d'unités avec # prix, « Disponible dès… », secteur, commodités et carrousel de photos. # Pages détail (cache BD self.detail) : adresse complète avec code postal, # superficie (« Plan de l'unité … pi2 »), description riche, commodités # de l'appartement et de l'immeuble, galerie complète. # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import re from bs4 import BeautifulSoup from ..schema import Listing, infer_city, normalize_unit_type, parse_price from .base import BaseConnector BASE = "https://sibelanger.com" LIST_URL = f"{BASE}/appartements-a-louer/" IMG_RE = re.compile( r"https://sibelanger\.com/app/uploads/[^\"'\\\s\)]+" r"\.(?:jpg|jpeg|png|webp)", re.I) IMG_NOISE_RE = re.compile(r"logo|favicon|icon|sib_ico", re.I) class SibelangerConnector(BaseConnector): source_id = "sibelanger" request_delay = 0.6 max_details = 60 # garde-fou def fetch(self) -> list[Listing]: html = self.get(LIST_URL).text soup = BeautifulSoup(html, "html.parser") listings: dict[str, Listing] = {} for card in soup.select("div.listing__thumbnail"): try: lst = self._parse_card(card) except Exception: continue if lst and lst.external_id not in listings: listings[lst.external_id] = lst # Pages détail (cache BD) : adresse complète, superficie, description, # commodités, galerie complète fetched = 0 for i, lst in enumerate(listings.values()): if i >= self.max_details: break key = hashlib.sha1( f"{lst.title}|{lst.price_label}|{lst.availability}" .encode("utf-8")).hexdigest() def _fetch(url=lst.url): nonlocal fetched if fetched >= self.max_details: raise RuntimeError("plafond de requêtes détail atteint") fetched += 1 return self._fetch_detail(url) try: payload = self.detail(lst.external_id, key, _fetch) except Exception: payload = {} if payload: self._apply_detail(lst, payload) return list(listings.values()) def _parse_card(self, card) -> Listing | None: link = card.select_one("a.listing__thumbnail__content__title-wrapper") \ or card.select_one("a[href*='/appartements-a-louer/']") if not link: return None url = link.get("href", "").split("?")[0] m = re.search(r"/appartements-a-louer/([a-z0-9\-]+)/?$", url) if not m: return None slug = m.group(1) fav = card.select_one("[data-unit-id]") ext_id = (fav.get("data-unit-id", "").strip() if fav else "") or slug h3 = card.select_one("h3") title = h3.get_text(" ", strip=True) if h3 else slug sector_el = card.select_one( ".listing__thumbnail__content__title-wrapper p") sector = sector_el.get_text(strip=True) if sector_el else "" price_el = card.select_one(".listing__thumbnail__price") price_label = price_el.get_text(" ", strip=True) if price_el else "" avail_el = card.select_one(".listing__thumbnail__availability") avail = avail_el.get_text(" ", strip=True) if avail_el else "" size_el = card.select_one(".listing__thumbnail__size") unit_raw = size_el.get_text(" ", strip=True) if size_el else "" # Exclusions (prudence : le site est résidentiel) if re.search(r"stationnement|commercial|rangement|entrepos", f"{title} {unit_raw}", re.I): return None # Adresse dérivée du slug : « 350-101-chemin-ste-foy-… » -> # « 350, Chemin Ste-Foy » (n° d'immeuble, n° d'unité, rue) address = "" s = re.sub(r"^copie-de-", "", slug) ma = re.match(r"^(\d+)-\d+[a-z]?-([a-z\-]+?)" r"(?:-selection|-modele|-app|$)", s) if ma: street = " ".join(w.capitalize() for w in ma.group(2).split("-")) address = f"{ma.group(1)}, {street}" amenities = [img.get("title") or img.get("alt", "") for img in card.select( ".listing__thumbnail__content__features img")] amenities = [a.strip() for a in amenities if a and a.strip()] images = [] for img in card.select(".swiper-slide img"): src = img.get("src") or img.get("data-src") or "" if src.startswith("http") and not IMG_NOISE_RE.search(src): images.append(src) return Listing( source=self.source_id, external_id=ext_id, url=url, title=title, address=address, sector=sector, city=infer_city(sector), unit_type=normalize_unit_type(unit_raw), price=parse_price(price_label), price_label=price_label, availability=avail, amenities=amenities, images=list(dict.fromkeys(images)), ) def _fetch_detail(self, url: str) -> dict: """Télécharge une page détail et en extrait le payload brut (cacheable).""" html = self.get(url).text soup = BeautifulSoup(html, "html.parser") # entête : adresse complète (code postal) + disponibilité address = availability = "" info = soup.select_one("div.single-unit__info") if info: paras = [p.get_text(" ", strip=True) for p in info.select("p")] for t in paras: if not address and re.search(r"\d.+(?:Québec|Lévis|G\d[A-Z]\s?\d[A-Z]\d)", t): address = t elif re.search(r"Disponible|Libre|Loué", t, re.I): availability = t.strip(" .") # description riche (sans les invites « Demander une visite ») desc = "" sec = soup.select_one("section.single-unit__description") if sec: desc = re.sub(r"\s+", " ", sec.get_text(" ", strip=True)) desc = re.sub(r"Cliquer sur le lien ici pour\s*(Demander une visite)?\s*", "", desc).strip() if not desc: og = soup.find("meta", attrs={"property": "og:description"}) or \ soup.find("meta", attrs={"name": "description"}) if og and og.get("content"): desc = og["content"].strip() # superficie affichée sous le plan (« 1360 pi2 ») — texte brut, # normalisé ensuite par finalize() surf_el = soup.select_one("span.single-unit__plan-surface") surface = surf_el.get_text(" ", strip=True) if surf_el else "" # caractéristiques de l'appartement + commodités de l'immeuble # (les « Services de proximité » — épicerie, parc… — sont exclus) amenities: list[str] = [] for section in soup.select("section.single-unit__features"): title_el = section.select_one("h2") title = title_el.get_text(" ", strip=True) if title_el else "" if re.search(r"proximit", title, re.I): continue for li in section.select("li"): t = li.get_text(" ", strip=True) if t and len(t) < 60 and t not in amenities: amenities.append(t) images = [u for u in dict.fromkeys(IMG_RE.findall(html)) if not IMG_NOISE_RE.search(u) and not re.search(r"-\d+x\d+\.", u)] return {"address": address, "availability": availability, "description": desc[:800], "surface": surface, "amenities": amenities, "images": images} def _apply_detail(self, lst: Listing, payload: dict) -> None: """Applique le payload d'une page détail (frais ou depuis le cache).""" if payload.get("address"): lst.address = payload["address"] if payload.get("availability") and not lst.availability: lst.availability = payload["availability"] if payload.get("description"): lst.description = payload["description"] # superficie : texte source (« 1360 pi2 ») ajouté aux commodités pour # affichage + parsing par finalize() extra = list(payload.get("amenities") or []) if payload.get("surface"): extra.append(payload["surface"]) lst.amenities = list(dict.fromkeys(lst.amenities + extra)) merged = list(dict.fromkeys((payload.get("images") or []) + lst.images)) if merged: lst.images = merged[:25]