spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/jcl.py : connecteur petites annonces Groupe JCL5#   (annonces.groupejcl.com) — journal local des Basses-Laurentides / Rive-Nord6#   (Sainte-Thérèse, Saint-Jérôme, Saint-Eustache et environs). WordPress +7#   plugin « Classified Listing » (rtcl) rendu serveur : on ne moissonne QUE la8#   catégorie immobilier/logements-a-louer (cartes .rtcl-listing-item : titre,9#   prix, localité, lien). La page détail (.rtin-des / .rtin-location) fournit10#   description, adresse et photos. Granularité : annonce (unité).11#   ⚠️ Portail de petites annonces de particuliers : certaines annonces restent12#   affichées longtemps — Lou-Ka les reprend telles qu'affichées par la source.13# -----------------------------------------------------------------------------14from __future__ import annotations1516import re1718from bs4 import BeautifulSoup1920from ..schema import Listing21from .base import BaseConnector2223BASE = "https://annonces.groupejcl.com"24CAT_URL = f"{BASE}/categories/immobilier/logements-a-louer/"25MAX_PAGES = 102627IMG_RE = re.compile(28    r"https://annonces\.groupejcl\.com/wp-content/uploads/classified-listing/"29    r"[^\"'\s\\]+\.(?:jpe?g|png|webp)", re.I)30THUMB_RE = re.compile(r"-\d{2,4}x\d{2,4}(?=\.(?:jpe?g|png|webp)$)", re.I)31PRICE_RE = re.compile(r"(\d[\d\s,.]*)\s*\$?")32REGION_RE = re.compile(r"\s+et\s+environs\s*$", re.I)333435class JclConnector(BaseConnector):36    source_id = "jcl"37    request_delay = 0.838    max_images = 103940    # -- page détail (cachée en BD via self.detail) ---------------------------41    def _detail(self, url: str) -> dict:42        try:43            html = self.get(url).text44        except Exception:45            return {}46        soup = BeautifulSoup(html, "html.parser")47        payload: dict = {}4849        el = soup.select_one(".rtin-content-area .rtin-content")50        if el is not None:51            payload["description"] = re.sub(52                r"\s+", " ", el.get_text(" ", strip=True))[:1500]5354        # « 367 rue leonard | Saint-Jérôme et environs, Québec »55        for el in soup.select(".rtin-location"):56            parts = [p.strip() for p in57                     el.get_text("|", strip=True).split("|") if p.strip()]58            street = [p for p in parts59                      if not re.search(r"et environs|Québec$", p, re.I)]60            if street:61                payload["address"] = street[0]62                break6364        imgs = []65        for u in dict.fromkeys(IMG_RE.findall(html)):66            full = THUMB_RE.sub("", u)67            if full not in imgs:68                imgs.append(full)69        payload["images"] = imgs[: self.max_images]70        return payload7172    # -- fetch -----------------------------------------------------------------73    def fetch(self) -> list[Listing]:74        listings: list[Listing] = []75        seen: set[str] = set()7677        for page in range(1, MAX_PAGES + 1):78            url = CAT_URL if page == 1 else f"{CAT_URL}page/{page}/"79            try:80                html = self.get(url).text81            except Exception:82                break83            soup = BeautifulSoup(html, "html.parser")84            items = soup.select(".rtcl-listing-item")85            if not items:86                break8788            new_on_page = 089            for it in items:90                a = it.select_one(".rtin-title a[href]")91                if a is None:92                    continue93                href = a["href"].split("?")[0]94                slug = href.rstrip("/").rsplit("/", 1)[-1]95                if not slug or slug in seen:96                    continue97                seen.add(slug)98                new_on_page += 199100                title = re.sub(r"\s+", " ", a.get_text(" ", strip=True))101102                price = None103                price_label = ""104                el = it.select_one(".rtcl-price-amount")105                if el is not None:106                    price_label = re.sub(r"\s+", " ",107                                         el.get_text(" ", strip=True))108                    m = PRICE_RE.search(price_label.replace(",", ""))109                    if m:110                        try:111                            val = float(m.group(1).replace(" ", "")112                                        .replace(" ", ""))113                            if 300 <= val <= 20000:114                                price = val115                        except ValueError:116                            pass117118                # localité : lien /listing-location/quebec/<ville>/119                city = ""120                loc = it.select_one('a[href*="/listing-location/"]')121                if loc is not None:122                    city = REGION_RE.sub(123                        "", loc.get_text(" ", strip=True)).strip()124125                key = f"{title}|{price_label}|{city}"126                det = self.detail(slug, key, lambda u=href: self._detail(u))127128                listings.append(Listing(129                    source=self.source_id,130                    external_id=slug,131                    url=href,132                    title=title,133                    address=det.get("address", ""),134                    city=city,135                    price=price,136                    price_label=price_label,137                    description=det.get("description", ""),138                    images=list(det.get("images") or []),139                ))140            if new_on_page == 0:141                break142        return listings143