# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/brochu.py : connecteur Groupe Immobilier Brochu # (groupeimmobilierbrochu.com — Lévis : St-Romuald, Charny, St-Nicolas, # centre-ville + Québec : Les Saules). Une annonce par projet/immeuble # (pas d'unités individuelles listées sur le site). # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import re from bs4 import BeautifulSoup from ..schema import Listing, infer_city, normalize_unit_type, parse_price from .base import BaseConnector BASE = "https://groupeimmobilierbrochu.com" LIST_URL = f"{BASE}/projets/" # Hors agglomération Québec/Lévis EXCLUDED_SECTORS = {"saint-lambert-de-lauzon", "st-lambert-de-lauzon"} ADDR_RE = re.compile( r"\d{1,5},?\s+(?:rue|avenue|av\.|boulevard|boul\.?|chemin|route|rang|" r"place|mont[ée]e)\s+[^,<>]{2,50},?\s*(?:L[ée]vis|Qu[ée]bec)" r"(?:\s*\(Qu[ée]bec\))?(?:\s*,?\s*[A-Z]\d[A-Z]\s?\d[A-Z]\d)?", re.I) IMG_RE = re.compile(r"https://groupeimmobilierbrochu\.com/wp-content/uploads/" r'[^"\s\\)]+\.(?:jpe?g|png|webp|avif)', re.I) PHONE_RE = re.compile(r"\b(\d{3})[\s.\-](\d{3})[\s.\-](\d{4})\b") def _decode_cfemail(hexstr: str) -> str: """Décode un courriel protégé Cloudflare (attribut data-cfemail).""" try: raw = bytes.fromhex(hexstr) key = raw[0] return bytes(b ^ key for b in raw[1:]).decode("utf-8") except Exception: return "" class BrochuConnector(BaseConnector): source_id = "brochu" request_delay = 0.6 max_projects = 20 # garde-fou de crawl def fetch(self) -> list[Listing]: listings: list[Listing] = [] try: html = self.get(LIST_URL).text except Exception: return listings soup = BeautifulSoup(html, "html.parser") seen: set[str] = set() for card in soup.select("div.project"): if len(seen) >= self.max_projects: break try: a = card.select_one('a[href*="/projets/"]') if not a: continue url = a.get("href", "").split("?")[0] slug = url.rstrip("/").split("/")[-1] if not slug or slug in seen: continue seen.add(slug) label_el = card.select_one(".uk-label") label = label_el.get_text(" ", strip=True) if label_el else "" meta_el = card.select_one(".el-meta") sector = meta_el.get_text(" ", strip=True) if meta_el else "" title_el = card.select_one(".el-title") name = title_el.get_text(" ", strip=True) if title_el else slug bold_el = card.select_one(".uk-text-bold") bold = (re.sub(r"\s+", " ", bold_el.get_text(" ", strip=True)) if bold_el else "") ps = [p.get_text(" ", strip=True) for p in card.select("p") if p.get_text(strip=True)] # Projets complets ou hors Québec/Lévis : exclure full_text = f"{label} {' '.join(ps)}" if re.search(r"\bComplet\b", full_text, re.I) and \ not re.search(r"disponible|libre", full_text, re.I): continue key = sector.lower().replace(" ", "-").replace("--", "-") if key in EXCLUDED_SECTORS or "lauzon" in key: continue availability = label or (ps[0] if ps else "") price_label = bold if "$" in bold else "" unit_type = normalize_unit_type(bold) # ne garder qu'un vrai type d'unité (ex. "4½"), pas le texte brut if not re.match(r"^\d½$|^Studio$|^Loft$|^Maison$", unit_type): unit_type = "" city = infer_city(sector, default="Québec" if "saules" in key else "Québec") # Page projet : adresse, dispo, description, commodités, # contact, photos — via le cache BD des pages détail # (revisitée seulement si la carte liste a changé) card_key = hashlib.sha1( f"{label}|{sector}|{name}|{bold}|{'|'.join(ps)}" .encode("utf-8")).hexdigest() d = self.detail(slug, card_key, lambda url=url: self._fetch_detail(url)) if d.get("availability"): availability = d["availability"] details: dict = {} contact = {k: d[k] for k in ("phone", "email") if d.get(k)} if contact: details["contact"] = contact listings.append(Listing( source=self.source_id, external_id=slug, url=url, title=name, address=d.get("address", ""), sector=sector, city=city, unit_type=unit_type, price=parse_price(price_label), price_label=price_label, availability=availability, description=d.get("description") or bold, amenities=d.get("amenities") or [], details=details, images=d.get("images") or [], )) except Exception: continue return listings def _fetch_detail(self, url: str) -> dict: """Extrait les champs riches d'une page projet (thème UIkit). Adresse (lien Google Maps), disponibilité (encadré Statut), description + commodités (colonne principale), contact (téléphone affiché + courriel protégé Cloudflare décodé), photos. """ out: dict = {} try: detail = self.get(url).text except Exception: return out dsoup = BeautifulSoup(detail, "html.parser") # adresse : lien Google Maps du projet (hors pied de page, qui porte # l'adresse du bureau du Groupe Brochu), sinon regex globale for maps in dsoup.select('a[href*="maps"]'): if maps.find_parent("footer"): continue addr = re.sub(r"\s+", " ", maps.get_text(" ", strip=True)).strip() if re.search(r"\d", addr): out["address"] = addr break if not out.get("address"): dtext = re.sub(r"\s+", " ", dsoup.get_text(" ", strip=True)) m = ADDR_RE.search(dtext) if m: out["address"] = m.group(0).strip().rstrip(",") # disponibilité : encadré « Statut » (uk-alert-primary) alert = dsoup.select_one("div.uk-alert-primary p") if alert: out["availability"] = alert.get_text(" ", strip=True) # colonne principale : description (h3 + p) et commodités (li) main = dsoup.select_one('div[class*="uk-width-3-5"]') if main: paras = [p.get_text(" ", strip=True) for p in main.find_all("p") if p.get_text(strip=True)] desc = " ".join(paras) out["description"] = re.sub(r"\s+", " ", desc).strip()[:900] out["amenities"] = [li.get_text(" ", strip=True) for li in main.select("li") if li.get_text(strip=True)][:25] # contact : téléphone affiché dans l'encadré Statut + courriel # protégé Cloudflare (data-cfemail, décodable) panel = dsoup.select_one("div.uk-panel.uk-background-muted") m = PHONE_RE.search(panel.get_text(" ", strip=True) if panel else "") if m: out["phone"] = f"{m.group(1)}-{m.group(2)}-{m.group(3)}" cf = (panel.select_one("[data-cfemail]") if panel else None) or \ dsoup.select_one("[data-cfemail]") if cf: email = _decode_cfemail(cf.get("data-cfemail", "")) if email: out["email"] = email out["images"] = [u for u in dict.fromkeys(IMG_RE.findall(detail)) if not re.search(r"logo|icon|favicon|cropped-|" r"-\d{2,3}x\d{2,3}\.", u, re.I)][:25] return out