spb/lou-ka Public
Lou·Ka — tous les logements à louer du Québec, un seul endroit.
HTML 99.7%
1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/immeubles_bc.py : connecteur Les Immeubles Beaulieu et Collin5# (immeublesbc.com — Rimouski, 900+ appartements, plus gros gestionnaire du6# Bas-Saint-Laurent). WordPress + thème immobilier Houzez, tout rendu serveur.7# Page /appartements/ : toutes les unités actuellement disponibles (cartes8# Houzez : prix, adresse complète avec code postal, statut, chambres/sdb,9# galerie dans l'attribut data-images). Fiche détail /property/<slug>/ (via10# cache BD) : description, commodités et coordonnées GPS (carte Houzez).11# robots.txt ouvert (Disallow /wp-admin/ seulement), sitemap XML.12# -----------------------------------------------------------------------------13from __future__ import annotations1415import hashlib16import html as htmllib17import json18import re1920from bs4 import BeautifulSoup2122from ..schema import Listing, normalize_unit_type, parse_price23from .base import BaseConnector2425BASE = "https://immeublesbc.com"26LIST_URL = f"{BASE}/appartements/"2728# suffixe de redimensionnement WordPress (« -592x444.jpg » -> pleine taille)29_SIZE_SUFFIX = re.compile(r"-\d{2,4}x\d{2,4}(?=\.(?:jpg|jpeg|png|webp)$)", re.I)30_MAP_LATLNG_RE = re.compile(31 r'"lat"\s*:\s*"(-?\d+\.\d+)"\s*,\s*"lng"\s*:\s*"(-?\d+\.\d+)"')323334def _clean_price_label(label: str) -> str:35 """'$1,410/mois' -> '$1410/mois' compatible parse_price (virgule = milliers)."""36 return re.sub(r"(\d),(\d{3})", r"\1\2", label)373839class ImmeublesBCConnector(BaseConnector):40 source_id = "immeubles_bc"41 request_delay = 0.642 max_details = 60 # garde-fou fiches détail (vraies requêtes par sync)4344 def fetch(self) -> list[Listing]:45 html = self.get(LIST_URL).text46 soup = BeautifulSoup(html, "html.parser")4748 listings: dict[str, Listing] = {}49 for card in soup.select("div.item-listing-wrap"):50 try:51 self._parse_card(card, listings)52 except Exception:53 continue5455 # fiches détail (cache BD) : description, commodités, GPS56 self._fetched = 057 for lst in listings.values():58 key = hashlib.sha1(59 f"{lst.title}|{lst.price_label}|{lst.availability}|{lst.url}"60 .encode("utf-8")).hexdigest()61 try:62 payload = self.detail(lst.external_id, key,63 lambda u=lst.url: self._fetch_detail(u))64 except Exception:65 continue66 self._apply_detail(lst, payload)67 return list(listings.values())6869 # -- carte Houzez -------------------------------------------------------------70 def _parse_card(self, card, listings: dict[str, Listing]) -> None:71 link = card.select_one("h2.item-title a[href]")72 if not link:73 return74 url = link["href"]75 title = link.get_text(strip=True)76 m = re.search(r"/property/([^/]+)/?", url)77 slug = m.group(1) if m else ""78 listid_el = card.select_one("[data-listid]")79 ext_id = (listid_el.get("data-listid") if listid_el else "") or slug80 if not ext_id or str(ext_id) in listings:81 return8283 # exclusions : commercial / stationnement / rangement84 if re.search(r"commercial|bureau|local|stationnement|garage|entrep[oô]t",85 title, re.I):86 return8788 # statut (« Disponible », « Loué ») — on saute les logements loués89 status_el = card.select_one(".label-status")90 availability = status_el.get_text(strip=True) if status_el else ""91 if re.search(r"lou[ée]", availability, re.I):92 return9394 # adresse complète : « 20 Rue St Laurent E, Rimouski, QC G5L 2C4 »95 addr_el = card.select_one("address.item-address")96 address = addr_el.get_text(" ", strip=True) if addr_el else ""97 city = "Rimouski" # tout le parc est à Rimouski98 parts = [p.strip() for p in address.split(",") if p.strip()]99 for p in parts[1:]:100 if not re.match(r"^(QC|Québec|Quebec|G\d[A-Z])", p, re.I):101 city = re.sub(r"\s+(QC|Québec|Quebec).*$", "", p, flags=re.I).strip() or city102 break103104 price_el = card.select_one("li.item-price")105 price_label = price_el.get_text(strip=True) if price_el else ""106107 # chambres / salles de bain : colonnes structurées de la carte108 amenities: list[str] = []109 beds = ""110 beds_el = card.select_one("li.h-beds .hz-figure")111 if beds_el:112 beds = beds_el.get_text(strip=True)113 if beds:114 amenities.append(f"{beds} chambre(s)")115 baths_el = card.select_one("li.h-baths .hz-figure")116 if baths_el and baths_el.get_text(strip=True):117 amenities.append(f"{baths_el.get_text(strip=True)} salle(s) de bain")118119 # type d'unité dérivé des chambres structurées (0 = Studio, n = (n+2)½)120 unit_type = ""121 if beds.isdigit():122 unit_type = "Studio" if beds == "0" else normalize_unit_type(f"{beds} chambres")123124 # galerie complète : attribut data-images (JSON, URLs redimensionnées)125 images: list[str] = []126 raw = card.get("data-images") or ""127 if raw:128 try:129 entries = json.loads(htmllib.unescape(raw))130 urls = [e.get("image", "") for e in entries if isinstance(e, dict)]131 except Exception:132 urls = re.findall(r"https?:[^\"',\\]+", htmllib.unescape(raw))133 for u in urls:134 u = u.replace("\\/", "/").strip()135 if not u.startswith("http"):136 continue137 u = _SIZE_SUFFIX.sub("", u) # version pleine taille (WordPress)138 if u not in images:139 images.append(u)140 if not images:141 thumb = card.select_one("img.wp-post-image[src]")142 if thumb:143 images = [_SIZE_SUFFIX.sub("", thumb["src"])]144145 listings[str(ext_id)] = Listing(146 source=self.source_id,147 external_id=str(ext_id),148 url=url,149 title=title,150 address=address,151 city=city,152 unit_type=unit_type,153 price=parse_price(_clean_price_label(price_label)),154 price_label=price_label,155 availability=availability,156 amenities=amenities,157 images=images[:30],158 )159160 # -- fiche détail (Houzez) ------------------------------------------------------161 def _fetch_detail(self, url: str) -> dict:162 """Description, commodités (#property-features-wrap) et GPS (carte)."""163 if self._fetched >= self.max_details:164 raise RuntimeError("budget de fiches détail atteint")165 self._fetched += 1166 html = self.get(url).text167 soup = BeautifulSoup(html, "html.parser")168 out: dict = {}169170 desc_el = soup.select_one("#property-description-wrap")171 if desc_el:172 txt = desc_el.get_text("\n", strip=True)173 txt = re.sub(r"^Description\n", "", txt)174 out["description"] = re.sub(r"[ \t]+", " ", txt).strip()[:1200]175176 out["amenities"] = [a.get_text(" ", strip=True)177 for a in soup.select("#property-features-wrap li")178 if a.get_text(strip=True)][:20]179180 m = _MAP_LATLNG_RE.search(html)181 if m:182 out["lat"], out["lng"] = float(m.group(1)), float(m.group(2))183 return out184185 def _apply_detail(self, lst: Listing, d: dict) -> None:186 """Reporte le payload (frais ou en cache) sur l'annonce."""187 if not d:188 return189 desc = d.get("description") or ""190 # la « description » Houzez du site répète parfois l'adresse : ignorer191 if desc and desc.strip() != lst.address.strip():192 lst.description = desc193 if d.get("amenities"):194 lst.amenities = list(dict.fromkeys(lst.amenities + d["amenities"]))195 if d.get("lat") is not None and d.get("lng") is not None:196 lst.lat, lst.lng = d["lat"], d["lng"]197