spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/brigil.py : connecteur Brigil (brigil.com — Gatineau/Outaouais)5#   Promoteur-gestionnaire dominant de Gatineau. Site = portail Yardi RentCafe6#   protégé par Cloudflare : curl/requests reçoivent 403 (« Just a moment »),7#   TOUT passe par Firecrawl avec attente de rendu — même patron que8#   realstar.py (RentCafe + Cloudflare) :9#     1) /searchlisting?province=Quebec -> cartes propriétés (nom, gamme10#        Aura/Apogee/Comfort, lits, fourchette de prix, vignette) ; le portail11#        ne filtre PAS par ville côté serveur et les cartes ne portent pas la12#        ville : elle est lue sur la fiche de chaque propriété ;13#     2) fiche propriété -> adresse structurée (itemprop=address), photos,14#        description, plus les plans quand ils y sont intégrés ;15#     3) /floorplans si la fiche n'a pas de plans -> plans structurés16#        (nom, chambres, sdb, pi², « Starting at »).17#   Les propriétés d'Ottawa/Ontario sont écartées : d'abord par l'URL18#   (/apartments/on/…), ensuite par la ville lue sur la fiche.19#   Une annonce par propriété. Les rendus passent par self.detail(...) (cache20#   BD) : Firecrawl n'est rappelé que si la carte a changé, et un plafond21#   (max_renders) borne le coût de chaque synchronisation.22# -----------------------------------------------------------------------------23from __future__ import annotations2425import hashlib26import os27import re2829from bs4 import BeautifulSoup3031from ..schema import Listing, normalize_unit_type, parse_price, strip_accents32from .base import FIRECRAWL_API, BaseConnector3334SEARCH_URL = "https://www.brigil.com/searchlisting?province=Quebec"3536# Villes desservies côté québécois (Gatineau + couronne outaouaise). Les37# anciennes villes fusionnées deviennent des secteurs de Gatineau.38_GATINEAU_SECTORS = {39    "hull": "Hull", "aylmer": "Aylmer", "buckingham": "Buckingham",40    "masson-angers": "Masson-Angers", "templeton": "Templeton",41    "gatineau": "",42}43_QC_CITIES = {44    "gatineau": "Gatineau", "hull": "Gatineau", "aylmer": "Gatineau",45    "buckingham": "Gatineau", "chelsea": "Chelsea", "cantley": "Cantley",46    "val-des-monts": "Val-des-Monts", "thurso": "Thurso",47    "la peche": "La Pêche",48}4950_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder", re.I)51_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?")52_BED_LOW_RE = re.compile(r"(?i)^\s*(studio|\d+)")535455class _BudgetReached(Exception):56    """Plafond de rendus Firecrawl atteint pour cette synchronisation."""575859class BrigilConnector(BaseConnector):60    source_id = "brigil"61    request_delay = 1.062    max_images = 2063    max_renders = 26         # plafond d'appels Firecrawl par sync (hors cache)6465    # -- Firecrawl avec attente de rendu (RentCafe + Cloudflare) ---------------66    def _rendered(self, url: str, wait_ms: int = 9000) -> str:67        key = os.environ.get("FIRECRAWL_API_KEY", "")68        # via self.session : l'enregistreur de fixtures capture la réponse69        resp = self.session.post(70            FIRECRAWL_API,71            json={"url": url, "formats": ["html"], "waitFor": wait_ms},72            headers={"Authorization": f"Bearer {key}"},73            timeout=150,74        )75        resp.raise_for_status()76        return (resp.json().get("data") or {}).get("html", "")7778    # -- helpers ---------------------------------------------------------------79    @staticmethod80    def _slug(url: str) -> str:81        """URL de propriété -> identifiant stable (1er segment du chemin)."""82        m = re.search(r"brigil\.com/(?:apartments/[a-z]{2}/[^/]+/)?([^/?#]+)",83                      url)84        if not m:85            return ""86        slug = m.group(1)87        return "" if slug in ("searchlisting", "default.aspx") else slug8889    @staticmethod90    def _city_sector(address: str) -> tuple[str, str]:91        """Ville et secteur lus DANS l'adresse publiée par la fiche.9293        Retourne ('', '') si l'adresse ne nomme aucune ville québécoise94        desservie (propriété ontarienne ou adresse illisible).95        """96        key = strip_accents(address or "").lower()97        if re.search(r"\bon\b|ontario", key) and not re.search(r"\bqc\b|quebec",98                                                               key):99            return "", ""100        city, sector = "", ""101        for token, canon in _QC_CITIES.items():102            if re.search(rf"\b{re.escape(token)}\b", key):103                city = canon104                break105        if city == "Gatineau":106            for token, canon in _GATINEAU_SECTORS.items():107                if canon and re.search(rf"\b{re.escape(token)}\b", key):108                    sector = canon109                    break110        return city, sector111112    @staticmethod113    def _unit_type(beds: str) -> str:114        """« Studio-2 Beds » -> Studio ; « 2-3 Beds » -> 4½ (borne basse, en115        cohérence avec le prix « à partir de » de la carte)."""116        m = _BED_LOW_RE.match(beds or "")117        if not m:118            return ""119        low = m.group(1).lower()120        if low == "studio":121            return "Studio"122        n = int(low)123        return "Studio" if n == 0 else normalize_unit_type(f"{n} chambres")124125    # -- fetch -----------------------------------------------------------------126    def fetch(self) -> list[Listing]:127        self._renders = 0128        html = self._rendered(SEARCH_URL, 10000)129        soup = BeautifulSoup(html, "html.parser")130        cards = soup.select("li.property-box")131        if not cards:                      # rendu incomplet : seconde chance132            html = self._rendered(SEARCH_URL, 15000)133            soup = BeautifulSoup(html, "html.parser")134            cards = soup.select("li.property-box")135136        # candidats : tout sauf les fiches explicitement ontariennes137        candidates: list[tuple[str, str, object]] = []138        seen: set[str] = set()139        for card in cards:140            a = card.select_one("a[href*='brigil.com']")141            if not a:142                continue143            url = (a.get("href") or "").split("?")[0].replace("http://",144                                                              "https://")145            if "/apartments/on/" in url:146                continue                   # Ottawa / Ontario147            slug = self._slug(url)148            if not slug or slug in seen:149                continue150            seen.add(slug)151            candidates.append((slug, url, card))152        # les URLs qui annoncent déjà /qc/ passent en premier (rendu utile sûr)153        candidates.sort(key=lambda c: 0 if "/apartments/qc/" in c[1] else 1)154155        listings: list[Listing] = []156        for slug, url, card in candidates:157            try:158                lst = self._property_listing(slug, url, card)159            except _BudgetReached:160                break                      # budget épuisé : au prochain sync161            except Exception:162                continue163            if lst is not None:164                listings.append(lst)165        return listings166167    def _property_listing(self, slug: str, url: str, card) -> Listing | None:168        name = ""169        fav = card.select_one("[data-property]")170        if fav:171            name = (fav.get("data-property") or "").strip()172        if not name:173            h = card.select_one(".property-name a")174            if h:175                name = h.get_text(" ", strip=True)176        name = re.sub(r"\s*opens in a new tab\s*", "", name).strip()177        name = name or slug.replace("-", " ").title()178179        types_el = card.select_one(".unit-types")180        unit_types = types_el.get_text(" ", strip=True) if types_el else ""181        beds = ""182        for li in card.select("ul.list-inline li"):183            txt = li.get_text(" ", strip=True)184            if re.search(r"(?i)bed|studio", txt):185                beds = txt186                break187188        card_text = card.get_text(" ", strip=True)189        price = None190        price_label = ""191        pm = re.search(r"\$[\d,]+(?:\.\d{2})?(?:(?:\s*(?:-|to))+\s*"192                       r"\$[\d,]+(?:\.\d{2})?)?", card_text)193        if pm:194            price_label = re.sub(r"\s*to\s*-\s*", " - ", pm.group(0))195            price = parse_price(price_label.split("-")[0])196            if "-" in price_label:197                price_label = "À partir de " + price_label198199        images: list[str] = []200        img = card.select_one("img[src*='rentcafe']")201        if img and img.get("src"):202            images.append(img["src"])203204        # fiche propriété (+ plans) via cache BD : Firecrawl n'est rappelé que205        # si la carte a changé (nom, lits, prix)206        key = hashlib.sha1(207            f"{name}|{unit_types}|{beds}|{price_label}".encode("utf-8")208        ).hexdigest()[:20]209        payload = self.detail(slug, key, lambda: self._fetch_detail(url))210211        city = payload.get("city", "")212        if not city:213            return None            # ville non québécoise ou fiche illisible214        sector = payload.get("sector", "")215        address = payload.get("address", "")216217        for im in (payload.get("images") or []):218            if im not in images:219                images.append(im)220221        area_sqft = None222        unit_type = self._unit_type(beds)223        plans = payload.get("floorplans") or []224        priced = [p for p in plans if p.get("price")]225        if priced:226            low = min(p["price"] for p in priced)227            if 100 <= low <= 20000:228                price = low229                price_label = (f"À partir de {low:,.0f} $/mois"230                               .replace(",", " ") if len(priced) > 1231                               else f"{low:,.0f} $/mois".replace(",", " "))232        if len(plans) == 1 and plans[0].get("sqft"):233            area_sqft = plans[0]["sqft"]234            if plans[0].get("unit_type"):235                unit_type = plans[0]["unit_type"]236237        plan_bits = []238        for p in plans[:8]:239            seg = p["name"]240            if p.get("sqft"):241                seg += f" ({p['sqft']:.0f} pi²)"242            if p.get("price"):243                seg += f" : {p['price']:,.0f} $/mois".replace(",", " ")244            plan_bits.append(seg)245246        desc_parts = [x for x in [payload.get("description", ""),247                                  unit_types, beds] if x]248        if plan_bits:249            desc_parts.append("Plans : " + " ; ".join(plan_bits))250251        details: dict = {}252        if payload.get("phone"):253            details["contact"] = {"phone": payload["phone"]}254255        return Listing(256            source=self.source_id,257            external_id=slug,258            url=url,259            title=name,260            address=address,261            sector=sector,262            city=city,263            unit_type=unit_type,264            price=price,265            price_label=price_label,266            availability="",       # RentCafe n'affiche pas de date par plan267            area_sqft=area_sqft,268            description=" — ".join(desc_parts)[:900],269            amenities=list(payload.get("amenities") or []),270            details=details,271            images=images[: self.max_images],272        )273274    # -- fiche propriété + plans (hors cache seulement) ------------------------275    def _fetch_detail(self, url: str) -> dict:276        """1 à 2 rendus Firecrawl : fiche (adresse, photos, description,277        commodités, plans intégrés) puis /floorplans si nécessaire.278279        La fiche est rendue AVANT tout : sans elle, la ville de la propriété280        est inconnue et l'annonce est écartée.281        """282        if self._renders + 1 > self.max_renders:283            raise _BudgetReached()284        self._renders += 1285286        payload: dict = {"city": "", "sector": "", "address": "",287                         "description": "", "amenities": [], "images": [],288                         "floorplans": [], "phone": ""}289        html = self._rendered(url, 8000)290        soup = BeautifulSoup(html, "html.parser")291292        addr_el = soup.select_one("[itemprop=address]")293        address = (addr_el.get_text(" ", strip=True) if addr_el else "")294        address = re.sub(r"\s+,", ",", re.sub(r"\s{2,}", " ", address)).strip()295        payload["address"] = address296        payload["city"], payload["sector"] = self._city_sector(address)297        if not payload["city"]:298            return payload            # hors Québec : on n'ira pas plus loin299300        for im in soup.select("img[src*='resource.rentcafe.com']"):301            src = im.get("src", "")302            if src and not _SKIP_IMG.search(src) \303                    and src not in payload["images"]:304                payload["images"].append(src)305306        paras = [p.get_text(" ", strip=True) for p in soup.find_all("p")]307        paras = [p for p in paras if len(p) > 80]308        if paras:309            payload["description"] = " ".join(paras[:2])[:600]310311        tel = soup.select_one("a[href^='tel:']")312        if tel:313            tm = re.search(r"(\d{3})\D*(\d{3})\D*(\d{4})", tel.get("href", ""))314            if tm:315                payload["phone"] = \316                    f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"317318        payload["floorplans"] = self._parse_floorplans(soup)319        if not payload["floorplans"]:320            fp_url = re.sub(r"/default(?:\.aspx)?$", "", url).rstrip("/") \321                + "/floorplans"322            if self._renders + 1 <= self.max_renders:323                self._renders += 1324                try:325                    payload["floorplans"] = self._parse_floorplans(326                        BeautifulSoup(self._rendered(fp_url, 10000),327                                      "html.parser"))328                except Exception:329                    pass330        return payload331332    @staticmethod333    def _parse_floorplans(soup) -> list[dict]:334        """Cartes de plans RentCafe : nom (« 2 Bed 2 Bath »), chambres, pi²335        (borne basse de la fourchette), prix « Starting at » — « Call for336        details » = prix non publié (None, jamais deviné)."""337        plans: list[dict] = []338        for cont in soup.select("div[id^='fp-container-'], div.fp-container"):339            try:340                name_el = cont.select_one(".card-title, h2, h3")341                name = name_el.get_text(" ", strip=True) if name_el else ""342                if not name:343                    continue344                text = cont.get_text(" ", strip=True)345                sqft = None346                sm = re.search(r"([\d,]{3,6})\s*(?:-\s*)?(?:[\d,]{3,6}\s*)?"347                               r"Sq\.?\s*Ft", text, re.I)348                if sm:349                    v = float(sm.group(1).replace(",", ""))350                    if 80 <= v <= 20000:351                        sqft = v352                price = None353                if re.search(r"(?i)starting at", text):354                    pm = _PRICE_RE.search(text)355                    if pm:356                        v = float(pm.group(0).replace("$", "").replace(",", ""))357                        if 100 <= v <= 20000:358                            price = v359                unit_type = ""360                bm = re.search(r"(\d)\s*Bed", name, re.I)361                if bm:362                    unit_type = normalize_unit_type(f"{bm.group(1)} chambres")363                elif re.search(r"(?i)studio", name):364                    unit_type = "Studio"365                if any(p["name"] == name for p in plans):366                    continue367                plans.append({"name": name, "sqft": sqft, "price": price,368                              "unit_type": unit_type})369            except Exception:370                continue371        return plans372