# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/sdg.py : connecteur SDG Immobilier (sdgimmobilier.ca) # Site WordPress (thème Houzez) : /appartements-a-louer/ liste des # immeubles (fiches « immeuble »). Une annonce par immeuble (types # d'unités affichés, pas de prix ni de disponibilités publiés). # Galerie via data-images ; fiche : description, caractéristiques, # bloc adresse structuré (adresse, arrondissement, code postal, ville), # téléphone (lien tel:). Fiches derrière self.detail(...) (cache BD). # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import json import re from bs4 import BeautifulSoup from ..schema import Listing, infer_city, normalize_unit_type from .base import BaseConnector BASE = "https://www.sdgimmobilier.ca" LIST_URL = f"{BASE}/appartements-a-louer/" IMG_RE = re.compile( r"https://www\.sdgimmobilier\.ca/wp-content/uploads/[^\"'\\\s\)]+" r"\.(?:jpg|jpeg|png|webp)", re.I) IMG_NOISE_RE = re.compile(r"logo|favicon|icon|-\d+x\d+\.", re.I) EXCLUDE_RE = re.compile(r"stationnement|commercial|rangement|entrepos|bureau", re.I) class _BudgetReached(Exception): """Plafond de requêtes « fiche » atteint pour cette synchronisation.""" class SDGConnector(BaseConnector): source_id = "sdg" request_delay = 0.6 max_details = 40 # plafond de vraies requêtes fiche par sync def fetch(self) -> list[Listing]: html = self.get(LIST_URL).text soup = BeautifulSoup(html, "html.parser") listings: dict[str, Listing] = {} cards: dict[str, str] = {} # ext_id -> texte de carte (clé de cache) for card in soup.select("div.item-listing-wrap[data-hz-id]"): try: lst = self._parse_card(card) except Exception: continue if lst and lst.external_id not in listings: listings[lst.external_id] = lst cards[lst.external_id] = card.get_text(" ", strip=True) # Fiches immeuble (cache BD) : description, types, commodités, adresse self._fetches = 0 for lst in listings.values(): key = hashlib.sha1( f"{lst.title}|{cards.get(lst.external_id, '')}" .encode("utf-8")).hexdigest() try: payload = self.detail( lst.external_id, key, lambda u=lst.url: self._fetch_detail(u)) except Exception: # _BudgetReached inclus : rien de caché continue self._apply_detail(lst, payload) return list(listings.values()) def _parse_card(self, card) -> Listing | None: ext_id = card.get("data-hz-id", "").strip() title_a = card.select_one(".item-title a") if not ext_id or not title_a: return None url = title_a.get("href", "") if "/immeuble/" not in url: return None title = title_a.get_text(" ", strip=True) if EXCLUDE_RE.search(f"{title} {url}"): return None addr_el = card.select_one(".item-address span") or \ card.select_one(".item-address") sector = addr_el.get_text(" ", strip=True) if addr_el else "" # Galerie complète fournie dans l'attribut data-images (JSON) images: list[str] = [] raw = card.get("data-images", "") if raw: try: images = [d.get("image", "") for d in json.loads(raw) if d.get("image")] except (ValueError, TypeError): images = [] if not images: img_el = card.select_one(".listing-thumb img") if img_el and (img_el.get("src") or "").startswith("http"): images = [img_el["src"]] return Listing( source=self.source_id, external_id=ext_id, url=url, title=title, address=title if re.match(r"^\d", title) else "", sector=sector, city=infer_city(sector), images=list(dict.fromkeys(images))[:25], ) def _fetch_detail(self, url: str) -> dict: """Télécharge une fiche /immeuble// (appelé seulement hors cache).""" if self._fetches >= self.max_details: raise _BudgetReached() self._fetches += 1 html = self.get(url).text soup = BeautifulSoup(html, "html.parser") payload: dict = {} # Description desc_el = soup.select_one(".property-description-content") or \ soup.select_one("#property-description-wrap") if desc_el: payload["description"] = desc_el.get_text(" ", strip=True)[:600] # Caractéristiques : « Type d'unités: 3 ½ » + commodités payload["features"] = [a.get_text(" ", strip=True) for a in soup.select("#property-features-wrap li a")] # Bloc « Adresse » Houzez (structuré : adresse, arrondissement, # code postal, ville) addr: dict = {} for row in soup.select("#property-address-wrap .list-lined-item"): label = row.find("strong") value = row.find("span") if not label or not value: continue key = label.get_text(strip=True).lower().rstrip(" :") val = value.get_text(" ", strip=True) if val: addr[key] = val payload["address_block"] = addr # Téléphone du gestionnaire (lien tel: structuré) tel = soup.select_one('a[href^="tel:"]') if tel: m = re.search(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})", tel.get("href", "")) if m: payload["phone"] = f"{m.group(1)}-{m.group(2)}-{m.group(3)}" # Photos de la fiche payload["images"] = [u for u in dict.fromkeys(IMG_RE.findall(html)) if not IMG_NOISE_RE.search(u)] return payload @staticmethod def _apply_detail(lst: Listing, payload: dict) -> None: if payload.get("description"): lst.description = payload["description"] amenities = [] for f in (payload.get("features") or []): m = re.search(r"types? d.unités?\s*:?\s*(.+)", f, re.I) if m and not lst.unit_type: # « 2 ½, 3 ½, 4 ½ et 5 ½ » -> plus petit type (standard Lou-Ka) lst.unit_type = normalize_unit_type(m.group(1)) if f and not m: amenities.append(f) elif m and "," in m.group(1): amenities.append(f) # garder le détail multi-types if amenities: lst.amenities = amenities[:20] # Adresse structurée : civique + code postal ; arrondissement ; ville addr = payload.get("address_block") or {} for key, val in addr.items(): if key.startswith("adresse"): lst.address = val elif key.startswith("arrondissement") and not lst.sector: lst.sector = val lst.city = infer_city(val) if addr.get("ville"): lst.city = addr["ville"] # ville structurée if addr.get("code postal") and lst.address \ and addr["code postal"] not in lst.address: lst.address = f"{lst.address}, {addr['code postal']}" if payload.get("phone"): lst.details = {**lst.details, "contact": {"phone": payload["phone"]}} # Compléter la galerie avec les photos de la fiche lst.images = list(dict.fromkeys( lst.images + (payload.get("images") or [])))[:25]