spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/brochu.py : connecteur Groupe Immobilier Brochu5#   (groupeimmobilierbrochu.com — Lévis : St-Romuald, Charny, St-Nicolas,6#    centre-ville + Québec : Les Saules). Une annonce par projet/immeuble7#   (pas d'unités individuelles listées sur le site).8# -----------------------------------------------------------------------------9from __future__ import annotations1011import hashlib12import re1314from bs4 import BeautifulSoup1516from ..schema import Listing, infer_city, normalize_unit_type, parse_price17from .base import BaseConnector1819BASE = "https://groupeimmobilierbrochu.com"20LIST_URL = f"{BASE}/projets/"2122# Hors agglomération Québec/Lévis23EXCLUDED_SECTORS = {"saint-lambert-de-lauzon", "st-lambert-de-lauzon"}2425ADDR_RE = re.compile(26    r"\d{1,5},?\s+(?:rue|avenue|av\.|boulevard|boul\.?|chemin|route|rang|"27    r"place|mont[ée]e)\s+[^,<>]{2,50},?\s*(?:L[ée]vis|Qu[ée]bec)"28    r"(?:\s*\(Qu[ée]bec\))?(?:\s*,?\s*[A-Z]\d[A-Z]\s?\d[A-Z]\d)?", re.I)29IMG_RE = re.compile(r"https://groupeimmobilierbrochu\.com/wp-content/uploads/"30                    r'[^"\s\\)]+\.(?:jpe?g|png|webp|avif)', re.I)31PHONE_RE = re.compile(r"\b(\d{3})[\s.\-](\d{3})[\s.\-](\d{4})\b")323334def _decode_cfemail(hexstr: str) -> str:35    """Décode un courriel protégé Cloudflare (attribut data-cfemail)."""36    try:37        raw = bytes.fromhex(hexstr)38        key = raw[0]39        return bytes(b ^ key for b in raw[1:]).decode("utf-8")40    except Exception:41        return ""424344class BrochuConnector(BaseConnector):45    source_id = "brochu"46    request_delay = 0.647    max_projects = 20        # garde-fou de crawl4849    def fetch(self) -> list[Listing]:50        listings: list[Listing] = []51        try:52            html = self.get(LIST_URL).text53        except Exception:54            return listings55        soup = BeautifulSoup(html, "html.parser")5657        seen: set[str] = set()58        for card in soup.select("div.project"):59            if len(seen) >= self.max_projects:60                break61            try:62                a = card.select_one('a[href*="/projets/"]')63                if not a:64                    continue65                url = a.get("href", "").split("?")[0]66                slug = url.rstrip("/").split("/")[-1]67                if not slug or slug in seen:68                    continue69                seen.add(slug)7071                label_el = card.select_one(".uk-label")72                label = label_el.get_text(" ", strip=True) if label_el else ""73                meta_el = card.select_one(".el-meta")74                sector = meta_el.get_text(" ", strip=True) if meta_el else ""75                title_el = card.select_one(".el-title")76                name = title_el.get_text(" ", strip=True) if title_el else slug77                bold_el = card.select_one(".uk-text-bold")78                bold = (re.sub(r"\s+", " ", bold_el.get_text(" ", strip=True))79                        if bold_el else "")80                ps = [p.get_text(" ", strip=True) for p in card.select("p")81                      if p.get_text(strip=True)]8283                # Projets complets ou hors Québec/Lévis : exclure84                full_text = f"{label} {' '.join(ps)}"85                if re.search(r"\bComplet\b", full_text, re.I) and \86                        not re.search(r"disponible|libre", full_text, re.I):87                    continue88                key = sector.lower().replace(" ", "-").replace("--", "-")89                if key in EXCLUDED_SECTORS or "lauzon" in key:90                    continue9192                availability = label or (ps[0] if ps else "")93                price_label = bold if "$" in bold else ""94                unit_type = normalize_unit_type(bold)95                # ne garder qu'un vrai type d'unité (ex. "4½"), pas le texte brut96                if not re.match(r"^\d½$|^Studio$|^Loft$|^Maison$", unit_type):97                    unit_type = ""98                city = infer_city(sector,99                                  default="Québec" if "saules" in key else "Québec")100101                # Page projet : adresse, dispo, description, commodités,102                # contact, photos — via le cache BD des pages détail103                # (revisitée seulement si la carte liste a changé)104                card_key = hashlib.sha1(105                    f"{label}|{sector}|{name}|{bold}|{'|'.join(ps)}"106                    .encode("utf-8")).hexdigest()107                d = self.detail(slug, card_key,108                                lambda url=url: self._fetch_detail(url))109110                if d.get("availability"):111                    availability = d["availability"]112113                details: dict = {}114                contact = {k: d[k] for k in ("phone", "email") if d.get(k)}115                if contact:116                    details["contact"] = contact117118                listings.append(Listing(119                    source=self.source_id,120                    external_id=slug,121                    url=url,122                    title=name,123                    address=d.get("address", ""),124                    sector=sector,125                    city=city,126                    unit_type=unit_type,127                    price=parse_price(price_label),128                    price_label=price_label,129                    availability=availability,130                    description=d.get("description") or bold,131                    amenities=d.get("amenities") or [],132                    details=details,133                    images=d.get("images") or [],134                ))135            except Exception:136                continue137138        return listings139140    def _fetch_detail(self, url: str) -> dict:141        """Extrait les champs riches d'une page projet (thème UIkit).142143        Adresse (lien Google Maps), disponibilité (encadré Statut),144        description + commodités (colonne principale), contact (téléphone145        affiché + courriel protégé Cloudflare décodé), photos.146        """147        out: dict = {}148        try:149            detail = self.get(url).text150        except Exception:151            return out152        dsoup = BeautifulSoup(detail, "html.parser")153154        # adresse : lien Google Maps du projet (hors pied de page, qui porte155        # l'adresse du bureau du Groupe Brochu), sinon regex globale156        for maps in dsoup.select('a[href*="maps"]'):157            if maps.find_parent("footer"):158                continue159            addr = re.sub(r"\s+", " ", maps.get_text(" ", strip=True)).strip()160            if re.search(r"\d", addr):161                out["address"] = addr162                break163        if not out.get("address"):164            dtext = re.sub(r"\s+", " ", dsoup.get_text(" ", strip=True))165            m = ADDR_RE.search(dtext)166            if m:167                out["address"] = m.group(0).strip().rstrip(",")168169        # disponibilité : encadré « Statut » (uk-alert-primary)170        alert = dsoup.select_one("div.uk-alert-primary p")171        if alert:172            out["availability"] = alert.get_text(" ", strip=True)173174        # colonne principale : description (h3 + p) et commodités (li)175        main = dsoup.select_one('div[class*="uk-width-3-5"]')176        if main:177            paras = [p.get_text(" ", strip=True) for p in main.find_all("p")178                     if p.get_text(strip=True)]179            desc = " ".join(paras)180            out["description"] = re.sub(r"\s+", " ", desc).strip()[:900]181            out["amenities"] = [li.get_text(" ", strip=True)182                                for li in main.select("li")183                                if li.get_text(strip=True)][:25]184185        # contact : téléphone affiché dans l'encadré Statut + courriel186        # protégé Cloudflare (data-cfemail, décodable)187        panel = dsoup.select_one("div.uk-panel.uk-background-muted")188        m = PHONE_RE.search(panel.get_text(" ", strip=True) if panel else "")189        if m:190            out["phone"] = f"{m.group(1)}-{m.group(2)}-{m.group(3)}"191        cf = (panel.select_one("[data-cfemail]") if panel else None) or \192            dsoup.select_one("[data-cfemail]")193        if cf:194            email = _decode_cfemail(cf.get("data-cfemail", ""))195            if email:196                out["email"] = email197198        out["images"] = [u for u in dict.fromkeys(IMG_RE.findall(detail))199                         if not re.search(r"logo|icon|favicon|cropped-|"200                                          r"-\d{2,3}x\d{2,3}\.", u, re.I)][:25]201        return out202