# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/immeubles_bc.py : connecteur Les Immeubles Beaulieu et Collin # (immeublesbc.com — Rimouski, 900+ appartements, plus gros gestionnaire du # Bas-Saint-Laurent). WordPress + thème immobilier Houzez, tout rendu serveur. # Page /appartements/ : toutes les unités actuellement disponibles (cartes # Houzez : prix, adresse complète avec code postal, statut, chambres/sdb, # galerie dans l'attribut data-images). Fiche détail /property// (via # cache BD) : description, commodités et coordonnées GPS (carte Houzez). # robots.txt ouvert (Disallow /wp-admin/ seulement), sitemap XML. # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import html as htmllib import json import re from bs4 import BeautifulSoup from ..schema import Listing, normalize_unit_type, parse_price from .base import BaseConnector BASE = "https://immeublesbc.com" LIST_URL = f"{BASE}/appartements/" # suffixe de redimensionnement WordPress (« -592x444.jpg » -> pleine taille) _SIZE_SUFFIX = re.compile(r"-\d{2,4}x\d{2,4}(?=\.(?:jpg|jpeg|png|webp)$)", re.I) _MAP_LATLNG_RE = re.compile( r'"lat"\s*:\s*"(-?\d+\.\d+)"\s*,\s*"lng"\s*:\s*"(-?\d+\.\d+)"') def _clean_price_label(label: str) -> str: """'$1,410/mois' -> '$1410/mois' compatible parse_price (virgule = milliers).""" return re.sub(r"(\d),(\d{3})", r"\1\2", label) class ImmeublesBCConnector(BaseConnector): source_id = "immeubles_bc" request_delay = 0.6 max_details = 60 # garde-fou fiches détail (vraies requêtes par sync) def fetch(self) -> list[Listing]: html = self.get(LIST_URL).text soup = BeautifulSoup(html, "html.parser") listings: dict[str, Listing] = {} for card in soup.select("div.item-listing-wrap"): try: self._parse_card(card, listings) except Exception: continue # fiches détail (cache BD) : description, commodités, GPS self._fetched = 0 for lst in listings.values(): key = hashlib.sha1( f"{lst.title}|{lst.price_label}|{lst.availability}|{lst.url}" .encode("utf-8")).hexdigest() try: payload = self.detail(lst.external_id, key, lambda u=lst.url: self._fetch_detail(u)) except Exception: continue self._apply_detail(lst, payload) return list(listings.values()) # -- carte Houzez ------------------------------------------------------------- def _parse_card(self, card, listings: dict[str, Listing]) -> None: link = card.select_one("h2.item-title a[href]") if not link: return url = link["href"] title = link.get_text(strip=True) m = re.search(r"/property/([^/]+)/?", url) slug = m.group(1) if m else "" listid_el = card.select_one("[data-listid]") ext_id = (listid_el.get("data-listid") if listid_el else "") or slug if not ext_id or str(ext_id) in listings: return # exclusions : commercial / stationnement / rangement if re.search(r"commercial|bureau|local|stationnement|garage|entrep[oô]t", title, re.I): return # statut (« Disponible », « Loué ») — on saute les logements loués status_el = card.select_one(".label-status") availability = status_el.get_text(strip=True) if status_el else "" if re.search(r"lou[ée]", availability, re.I): return # adresse complète : « 20 Rue St Laurent E, Rimouski, QC G5L 2C4 » addr_el = card.select_one("address.item-address") address = addr_el.get_text(" ", strip=True) if addr_el else "" city = "Rimouski" # tout le parc est à Rimouski parts = [p.strip() for p in address.split(",") if p.strip()] for p in parts[1:]: if not re.match(r"^(QC|Québec|Quebec|G\d[A-Z])", p, re.I): city = re.sub(r"\s+(QC|Québec|Quebec).*$", "", p, flags=re.I).strip() or city break price_el = card.select_one("li.item-price") price_label = price_el.get_text(strip=True) if price_el else "" # chambres / salles de bain : colonnes structurées de la carte amenities: list[str] = [] beds = "" beds_el = card.select_one("li.h-beds .hz-figure") if beds_el: beds = beds_el.get_text(strip=True) if beds: amenities.append(f"{beds} chambre(s)") baths_el = card.select_one("li.h-baths .hz-figure") if baths_el and baths_el.get_text(strip=True): amenities.append(f"{baths_el.get_text(strip=True)} salle(s) de bain") # type d'unité dérivé des chambres structurées (0 = Studio, n = (n+2)½) unit_type = "" if beds.isdigit(): unit_type = "Studio" if beds == "0" else normalize_unit_type(f"{beds} chambres") # galerie complète : attribut data-images (JSON, URLs redimensionnées) images: list[str] = [] raw = card.get("data-images") or "" if raw: try: entries = json.loads(htmllib.unescape(raw)) urls = [e.get("image", "") for e in entries if isinstance(e, dict)] except Exception: urls = re.findall(r"https?:[^\"',\\]+", htmllib.unescape(raw)) for u in urls: u = u.replace("\\/", "/").strip() if not u.startswith("http"): continue u = _SIZE_SUFFIX.sub("", u) # version pleine taille (WordPress) if u not in images: images.append(u) if not images: thumb = card.select_one("img.wp-post-image[src]") if thumb: images = [_SIZE_SUFFIX.sub("", thumb["src"])] listings[str(ext_id)] = Listing( source=self.source_id, external_id=str(ext_id), url=url, title=title, address=address, city=city, unit_type=unit_type, price=parse_price(_clean_price_label(price_label)), price_label=price_label, availability=availability, amenities=amenities, images=images[:30], ) # -- fiche détail (Houzez) ------------------------------------------------------ def _fetch_detail(self, url: str) -> dict: """Description, commodités (#property-features-wrap) et GPS (carte).""" if self._fetched >= self.max_details: raise RuntimeError("budget de fiches détail atteint") self._fetched += 1 html = self.get(url).text soup = BeautifulSoup(html, "html.parser") out: dict = {} desc_el = soup.select_one("#property-description-wrap") if desc_el: txt = desc_el.get_text("\n", strip=True) txt = re.sub(r"^Description\n", "", txt) out["description"] = re.sub(r"[ \t]+", " ", txt).strip()[:1200] out["amenities"] = [a.get_text(" ", strip=True) for a in soup.select("#property-features-wrap li") if a.get_text(strip=True)][:20] m = _MAP_LATLNG_RE.search(html) if m: out["lat"], out["lng"] = float(m.group(1)), float(m.group(2)) return out def _apply_detail(self, lst: Listing, d: dict) -> None: """Reporte le payload (frais ou en cache) sur l'annonce.""" if not d: return desc = d.get("description") or "" # la « description » Houzez du site répète parfois l'adresse : ignorer if desc and desc.strip() != lst.address.strip(): lst.description = desc if d.get("amenities"): lst.amenities = list(dict.fromkeys(lst.amenities + d["amenities"])) if d.get("lat") is not None and d.get("lng") is not None: lst.lat, lst.lng = d["lat"], d["lng"]