spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/lbm.py : connecteur Groupe Rive-Nord / LBM Gestion Locative5#   (grouperivenord.ca — info@lbmgestionlocative.com). Condos locatifs neufs6#   « tout inclus » sur le chemin d'Oka : Le 2922 et Le 3215 à7#   Sainte-Marthe-sur-le-Lac, Le 3533 (page /le-3537/) à Saint-Joseph-du-Lac.8#   WordPress/Divi rendu serveur : les pages projet (liens « le-* » du menu)9#   sont des vitrines ; certaines publient des cartes « Modèle N » (type,10#   « À partir de X$/m », superficie, chambres, inclusions), les autres11#   seulement les typologies offertes (« 4 1/2 et 5 1/2 », 1100 pi²).12#   Granularité MODÈLE/TYPOLOGIE par immeuble, prix seulement où publié.13#   2026-09-13 : SiteGround sert son anti-bot sgcaptcha (202 + challenge JS,14#   en-tête sg-captcha) aux IP datacenter — invisible du wrapper résilient15#   (2xx) ; on bascule alors la session sur un proxy résidentiel Oxylabs CA16#   (même pattern que boreal_abitibi/citiluxx/cromwell/deschenes_pepin/17#   gestion_habitation/gimcote/appartements_rimouski, hébergeur identique).18# -----------------------------------------------------------------------------19from __future__ import annotations2021import os22import re23import time24from urllib.parse import quote2526from ..schema import Listing, normalize_unit_type27from .base import BaseConnector2829BASE = "https://grouperivenord.ca"3031# page interstitielle sgcaptcha SiteGround (meta refresh vers /.well-known/…)32_SG_MARKER = "/.well-known/sgcaptcha/"333435def _sg_challenge(resp) -> bool:36    """True si la réponse est le challenge anti-bot SiteGround (202 + JS)."""37    if "challenge" in str(resp.headers.get("sg-captcha", "")).lower():38        return True39    return _SG_MARKER in (resp.text or "")[:600]4041PROJECT_RE = re.compile(r'href="(https://grouperivenord\.ca/(le-[\w-]+)/)"')42HERO_RE = re.compile(r"Bienvenue au\s*\|?\s*(\d{2,5}[^|]{0,40}?)\|", re.I)43CITY_RE = re.compile(r"\|\s*((?:Saint|Sainte)e?-[A-Za-zÀ-ü’'-]+(?:-[A-Za-zÀ-ü’'-]+)*)\s*\|")44OCCUP_RE = re.compile(r"Occupation d[èe]s[^|]{0,50}", re.I)45SQFT_RE = re.compile(r"(\d{3,4})\s*pieds?\s*carr[ée]s", re.I)46TYPES_RE = re.compile(r"\d\s*1/2")47PRICE_HERO_RE = re.compile(r"À partir de\s*([\d\s ]{3,7})\$", re.I)4849MODEL_RE = re.compile(50    r"Modèle\s*(\d+)\|\s*\|?Type de logement\s*:\s*([\d\s/½]+)\|"51    r".{0,80}?À partir(?:\s*de)?\|([\d\s ]{3,7})\$\|/\|m"52    r"\|Superficie:\s*([\d\s ]{2,6})\s*pi2"53    r"(?:\|(\d)\s*chambres?)?"54    r"((?:\|[^|]{3,70}){0,10})", re.I | re.S)5556IMG_RE = re.compile(57    r"https://grouperivenord\.ca/wp-content/uploads/[^\"'\s\\)]+?"58    r"\.(?:jpg|jpeg|png|webp)", re.I)596061def _flat(html: str) -> str:62    txt = re.sub(r"<script.*?</script>", "", html, flags=re.S)63    txt = re.sub(r"<style.*?</style>", "", txt, flags=re.S)64    txt = re.sub(r"<[^>]+>", "|", txt).replace("&nbsp;", " ")65    txt = txt.replace("&#039;", "'").replace("&rsquo;", "'")66    return re.sub(r"[ \t\n]+", " ", re.sub(r"\|+", "|", txt))676869def _num(txt: str) -> float | None:70    digits = re.sub(r"[^\d]", "", txt or "")71    return float(digits) if digits else None727374class LbmConnector(BaseConnector):75    source_id = "lbm"76    request_delay = 0.67778    def _enable_residential_proxy(self) -> bool:79        """Route toute la session via Oxylabs résidentiel CA (session collante)."""80        endpoint = os.environ.get("OXYLABS_PROXY")81        user = os.environ.get("OXYLABS_PROXY_USER")82        pwd = os.environ.get("OXYLABS_PROXY_PASS")83        if not (endpoint and user and pwd):84            return False85        puser = f"{user}-cc-CA-sessid-lbm{int(time.time())}-sesstime-10"86        proxy = f"http://{quote(puser, safe='')}:{quote(pwd, safe='')}@{endpoint}"87        self.session.proxies = {"http": proxy, "https": proxy}88        self.session.verify = False  # CA MITM du proxy Oxylabs89        return True9091    def _get_html(self, url: str) -> str:92        """GET avec contournement du challenge sgcaptcha (proxy résidentiel)."""93        resp = self.get(url)94        if not _sg_challenge(resp):95            return resp.text96        if not self.session.proxies and self._enable_residential_proxy():97            resp = self.get(url)98            if not _sg_challenge(resp):99                return resp.text100        raise RuntimeError(f"challenge sgcaptcha non contourné — {url}")101102    def fetch(self) -> list[Listing]:103        listings: list[Listing] = []104        # échec franc sur challenge/panne (fini le « 0 trouvé ok » silencieux)105        home = self._get_html(BASE + "/")106        pages = list(dict.fromkeys(PROJECT_RE.findall(home)))107        for page_url, slug in pages:108            try:109                html = self._get_html(page_url)110            except Exception:111                continue112            flat = _flat(html)113114            m = HERO_RE.search(flat)115            address = (m.group(1).strip() if m else "").replace("ch. Oka", "chemin d'Oka")116            address = address.replace("Chemin D'Oka", "chemin d'Oka")117            cm = CITY_RE.search(flat)118            city = cm.group(1).strip() if cm else ""119            om = OCCUP_RE.search(flat)120            availability = om.group(0).strip() if om else ""121            images = [u for u in dict.fromkeys(IMG_RE.findall(html))122                      if not re.search(r"logo|favicon|icon|-\d+x\d+\.", u,123                                       re.I)][:20]124125            models = list(MODEL_RE.finditer(flat))126            seen: set[str] = set()127            if models:128                for mm in models:129                    n, raw_type, price, sqft, beds, tail = mm.groups()130                    ext_id = f"{slug}-modele-{n}"131                    if ext_id in seen:      # blocs répétés desktop/mobile132                        continue133                    seen.add(ext_id)134                    unit_type = normalize_unit_type(raw_type.strip())135                    amenities = [t.strip(" .") for t in tail.split("|")136                                 if 3 < len(t.strip()) < 70137                                 and "Modèle" not in t]138                    listings.append(Listing(139                        source=self.source_id,140                        external_id=ext_id,141                        url=page_url,142                        title=(f"{unit_type} (modèle {n}) — "143                               f"{address}, {city}"),144                        address=address,145                        city=city,146                        unit_type=unit_type,147                        price=_num(price),148                        price_label=f"À partir de {_num(price):.0f} $/mois",149                        availability=availability,150                        area_sqft=_num(sqft),151                        bedrooms=float(beds) if beds else None,152                        description=("Condo locatif neuf en formule tout "153                                     f"inclus au {address}, {city} "154                                     "(Groupe Rive-Nord / LBM Gestion "155                                     "Locative)."),156                        amenities=amenities[:12],157                        details={"model": f"Modèle {n}"},158                        images=images,159                    ))160            else:161                # vitrine sans modèles : une annonce par typologie citée162                sq = SQFT_RE.search(flat)163                hero_zone = flat[:2500]164                for raw_type in dict.fromkeys(TYPES_RE.findall(hero_zone)):165                    unit_type = normalize_unit_type(raw_type)166                    ext_id = f"{slug}-{unit_type.replace('½', '.5')}"167                    if ext_id in seen:168                        continue169                    seen.add(ext_id)170                    pm = PRICE_HERO_RE.search(hero_zone)171                    price = _num(pm.group(1)) if pm else None172                    listings.append(Listing(173                        source=self.source_id,174                        external_id=ext_id,175                        url=page_url,176                        title=f"{unit_type} — {address}, {city}",177                        address=address,178                        city=city,179                        unit_type=unit_type,180                        price=price,181                        price_label=(f"À partir de {price:.0f} $/mois"182                                     if price else ""),183                        availability=availability,184                        area_sqft=_num(sq.group(1)) if sq else None,185                        description=("Condo locatif clé en main, formule "186                                     f"tout inclus, au {address}, {city} "187                                     "(Groupe Rive-Nord / LBM Gestion "188                                     "Locative)."),189                        amenities=["Formule tout inclus"],190                        images=images,191                    ))192        return listings193