spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/localys.py : connecteur Gestion Localys (gestionlocalys.com,5#   240+ unités). Site Next.js rendu serveur : /fr/logements liste les6#   logements disponibles (cartes avec adresse, quartier, « 2 ch. · 1 sdb. »7#   et prix « 1 500 $ / mois »), chaque fiche /fr/logements/<slug> fournit la8#   description (avec **DISPONIBLE …**, sections #### Inclusions/Exclusions),9#   les caractéristiques (« 2 ch. 1 sdb. 750 pi² ») et la galerie (photos10#   ws.jumptools.com encodées dans /_next/image?url=…). Une annonce par11#   logement ; external_id = slug ; fiches via le cache détail (clé = hash du12#   texte de la carte).13# -----------------------------------------------------------------------------14from __future__ import annotations1516import hashlib17import re18import urllib.parse1920from bs4 import BeautifulSoup2122from ..schema import Listing, parse_price23from .base import BaseConnector2425BASE = "https://www.gestionlocalys.com"26LIST_URL = f"{BASE}/fr/logements"2728# « 2558 Rue De Cadillac, Montréal (Mercier/Hochelaga-Maisonneuve), QC, … »29H1_RE = re.compile(30    r"^(?P<street>.+?),\s*(?P<city>[^,(]+?)\s*(?:\((?P<sector>[^)]+)\))?"31    r",\s*QC", re.S)32PRICE_RE = re.compile(r"([\d\s ]+)\$\s*/\s*mois")33CH_RE = re.compile(r"(\d+)\s*ch\.")34SDB_RE = re.compile(r"([\d.]+)\s*sdb\.")35SQFT_RE = re.compile(r"([\d\s ]+)\s*pi²")36NEXT_IMG_RE = re.compile(r'/_next/image\?url=([^&"\']+)')37DISPO_RE = re.compile(r"DISPONIBLE[^*\n]{0,40}", re.I)383940def _clean_md(t: str) -> str:41    """Retire les marqueurs markdown (** et ####) du texte rendu."""42    return re.sub(r"\s+", " ", t.replace("**", " ").replace("####", " ")43                  ).strip()444546class LocalysConnector(BaseConnector):47    source_id = "localys"48    request_delay = 0.74950    def fetch(self) -> list[Listing]:51        html = self.get(LIST_URL).text52        soup = BeautifulSoup(html, "html.parser")5354        # cartes : ancêtre du lien qui contient le prix55        cards: dict[str, str] = {}       # slug -> texte de la carte56        for a in soup.find_all("a", href=True):57            if "/fr/logements/" not in a["href"]:58                continue59            slug = a["href"].rstrip("/").rsplit("/", 1)[-1]60            if slug in cards and cards[slug]:61                continue62            node, text = a, ""63            for _ in range(6):64                node = node.parent65                if node is None:66                    break67                t = re.sub(r"\s+", " ", node.get_text(" ", strip=True))68                if "$" in t:69                    text = t70                    break71            cards[slug] = text7273        listings: list[Listing] = []74        for slug, card_text in cards.items():75            try:76                key = hashlib.sha1(card_text.encode("utf-8")).hexdigest()77                d = self.detail(slug, key,78                                lambda s=slug: self._logement(s))79                lst = self._listing(slug, card_text, d)80                if lst:81                    listings.append(lst)82            except Exception:83                continue84        return listings8586    def _logement(self, slug: str) -> dict:87        """Scrape la fiche : h1, description, caractéristiques, photos."""88        out: dict = {"h1": "", "desc": "", "carac": "", "images": []}89        html = self.get(f"{LIST_URL}/{slug}").text90        soup = BeautifulSoup(html, "html.parser")91        for s in soup.find_all("script"):92            s.decompose()                # écarter le payload RSC de Next.js93        if soup.h1:94            out["h1"] = soup.h1.get_text(" ", strip=True)95        text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))96        i = text.find("/ mois")97        j = text.find("Caractéristiques")98        if i >= 0 and j > i:99            out["desc"] = text[i + len("/ mois"):j].strip()[:1500]100        if j >= 0:101            out["carac"] = text[j:j + 120]102        images = []103        for enc in NEXT_IMG_RE.findall(html):104            u = urllib.parse.unquote(enc)105            if u.startswith("http") and "jumptools.com" in u \106                    and u not in images:107                images.append(u)108        out["images"] = images[:15]109        return out110111    def _listing(self, slug: str, card_text: str, d: dict) -> Listing | None:112        # fiche morte : la source amont (jumptools) renvoie sa page 404113        if "cannot be found" in d.get("desc", ""):114            return None115        h1 = d.get("h1") or ""116        street = city = sector = ""117        m = H1_RE.match(h1)118        if m:119            street = m.group("street").strip()120            city = m.group("city").strip()121            sector = (m.group("sector") or "").strip()122        address = ", ".join(x for x in (street, city) if x)123124        price = None125        pm = PRICE_RE.search(card_text) or PRICE_RE.search(d.get("carac", ""))126        if pm:127            price = parse_price(pm.group(1).strip() + " $")128        bedrooms = bathrooms = None129        blob = f"{card_text} {d.get('carac', '')}"130        bm = CH_RE.search(blob)131        if bm:132            bedrooms = float(bm.group(1))133        sm = SDB_RE.search(blob)134        if sm:135            try:136                bathrooms = float(sm.group(1))137            except ValueError:138                pass139        area = None140        am = SQFT_RE.search(d.get("carac", "")) or SQFT_RE.search(card_text)141        if am:142            try:143                area = float(re.sub(r"[\s ]", "", am.group(1)))144            except ValueError:145                pass146147        desc_raw = d.get("desc", "")148        availability = ""149        dm = DISPO_RE.search(desc_raw)150        if dm:151            availability = _clean_md(dm.group(0)).capitalize()152153        return Listing(154            source=self.source_id,155            external_id=slug,156            url=f"{LIST_URL}/{slug}",157            title=h1.replace(", QC, Canada", "") or slug,158            address=address,159            sector=sector,160            city=city or "Montréal",161            unit_type="",162            bedrooms=bedrooms,163            bathrooms=bathrooms,164            price=price,165            price_label=f"{pm.group(1).strip()} $ / mois" if pm else "",166            availability=availability,167            area_sqft=area,168            description=_clean_md(desc_raw),169            images=d.get("images", []),170        )171