spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/info_logement.py : connecteur Info-Logement (info-logement.com)5#   Gestionnaire locatif de Lanaudière (Joliette, St-Charles-Borromée,6#   Notre-Dame-des-Prairies, Berthierville…). Site custom : la liste7#   /logements/tous (paginée ?page=N, filtre gardé en session) expose des8#   cartes <a class="result"> avec data-logid stable, adresse (h2), tableau9#   Dimensions/Ville/Disponibilité, loyer et badge « En rénovation ». Les10#   fiches détail (via cache BD) ajoutent description, commodités,11#   proximités, adresse complète, galerie et GPS (LatLng de la carte).12#   robots.txt : « Disallow: » vide (tout permis).13# -----------------------------------------------------------------------------14from __future__ import annotations1516import hashlib17import re1819from bs4 import BeautifulSoup2021from ..schema import Listing, normalize_unit_type, parse_price22from .base import BaseConnector2324BASE = "https://www.info-logement.com"25LIST_URL = f"{BASE}/logements/tous"26PAGE_URL = f"{BASE}/logements?page={{n}}"2728_LATLNG_RE = re.compile(r"LatLng\((-?\d+\.\d+),\s*(-?\d+\.\d+)\)")29_THUMB_SUFFIX = re.compile(r"-\d{2,4}x\d{2,4}(?=\.(?:jpg|jpeg|png|webp)$)", re.I)303132class InfoLogementConnector(BaseConnector):33    source_id = "info_logement"34    request_delay = 0.735    max_pages = 10      # garde-fou de pagination (3 pages actuellement)36    max_details = 60    # garde-fou fiches détail (vraies requêtes)3738    def fetch(self) -> list[Listing]:39        listings: dict[str, Listing] = {}40        for page in range(1, self.max_pages + 1):41            # /logements/tous fixe le filtre « tous » en session ; les pages42            # suivantes se parcourent via /logements?page=N (mêmes cookies)43            url = LIST_URL if page == 1 else PAGE_URL.format(n=page)44            try:45                html = self.get(url).text46            except Exception:47                break48            soup = BeautifulSoup(html, "html.parser")49            cards = soup.select("a.result")50            if not cards:51                break52            before = len(listings)53            for card in cards:54                try:55                    self._parse_card(card, listings)56                except Exception:57                    continue58            if len(listings) == before:      # page sans nouvelle annonce59                break6061        # fiches détail (cache BD) : description, commodités, GPS, galerie62        self._fetched = 063        for lst in listings.values():64            card_key = hashlib.sha1(65                f"{lst.title}|{lst.price_label}|{lst.availability}|{lst.url}"66                .encode("utf-8")).hexdigest()67            try:68                payload = self.detail(lst.external_id, card_key,69                                      lambda u=lst.url: self._fetch_detail(u))70            except Exception:71                continue72            self._apply_detail(lst, payload)7374        return list(listings.values())7576    # -- carte de la liste --------------------------------------------------------77    def _parse_card(self, card, listings: dict[str, Listing]) -> None:78        url = card.get("href", "")79        # type dans l'URL : /logements/<ville>/<type>/<dim>/<id> —80        # on exclut garages/commercial (le site liste aussi des garages)81        m = re.search(r"/logements/([^/]+)/([^/]+)/([^/]+)/(\d+)$", url)82        if not m:83            return84        type_slug, ext_id = m.group(2), m.group(4)85        if re.search(r"garage|commercial|stationnement|rangement", type_slug):86            return87        if ext_id in listings:88            return8990        h2 = card.select_one("h2")91        address = h2.get_text(" ", strip=True) if h2 else ""92        rows: dict[str, str] = {}93        for tr in card.select("table.resultData tr"):94            tds = tr.find_all("td")95            if len(tds) == 2:96                rows[tds[0].get_text(strip=True).lower()] = \97                    tds[1].get_text(" ", strip=True)98        city = rows.get("ville", "")99        dim = rows.get("dimensions", "")100        availability = rows.get("disponibilité", "")101        price_el = card.select_one("p.left")102        price_label = price_el.get_text(" ", strip=True) if price_el else ""103104        # badge « En rénovation » : conservé (texte source) dans la description105        reno = card.select_one("span.reno")106        reno_txt = reno.get_text(" ", strip=True) if reno else ""107108        images: list[str] = []109        img = card.select_one(".resultPic img[src]")110        if img and img["src"].startswith("http"):111            images.append(_THUMB_SUFFIX.sub("", img["src"]))112113        listings[ext_id] = Listing(114            source=self.source_id,115            external_id=ext_id,          # data-logid / id numérique de l'URL116            url=url,117            title=address,118            address=address,119            sector="",120            city=city,                   # ville affichée sur la carte121            unit_type=normalize_unit_type(dim),122            price=parse_price(price_label),123            price_label=price_label,124            availability=availability,125            description=reno_txt,126            images=images,127        )128129    # -- fiche détail ------------------------------------------------------------130    def _fetch_detail(self, url: str) -> dict:131        """Description, commodités/proximités, adresse complète, GPS, galerie."""132        if self._fetched >= self.max_details:133            raise RuntimeError("budget de fiches détail atteint")134        self._fetched += 1135        html = self.get(url).text136        soup = BeautifulSoup(html, "html.parser")137        out: dict = {}138139        # sections titrées h1 : Commodités / À proximité de / Description140        for h in soup.find_all("h1"):141            name = h.get_text(strip=True)142            if name not in ("Commodités", "À proximité de", "Description"):143                continue144            lines: list[str] = []145            sib = h.find_next_sibling()146            while sib is not None and sib.name != "h1":147                for t in sib.stripped_strings:148                    t = re.sub(r"\s+", " ", t).strip()149                    if t and t not in lines:150                        lines.append(t)151                sib = sib.find_next_sibling()152            if name == "Description":153                out["description"] = "\n".join(lines)[:1200]154            else:155                out.setdefault("amenities", []).extend(lines[:15])156157        # adresse complète (« 1400, Line-Rainville, app. 201, Joliette QC J6E »)158        h1 = soup.find("h1")159        if h1:160            nxt = h1.find_next(string=re.compile(r"QC"))161            if nxt:162                out["address"] = re.sub(r"\s+", " ", str(nxt)).strip()163164        m = _LATLNG_RE.search(html)165        if m:166            out["lat"], out["lng"] = float(m.group(1)), float(m.group(2))167168        images: list[str] = []169        for img in soup.select('img[src*="/medias/"]'):170            src = _THUMB_SUFFIX.sub("", img["src"])171            if src.startswith("http") and src not in images:172                images.append(src)173        if images:174            out["images"] = images[:25]175        return out176177    def _apply_detail(self, lst: Listing, d: dict) -> None:178        if not d:179            return180        if d.get("description"):181            lst.description = (lst.description + "\n" + d["description"]).strip()182        if d.get("amenities"):183            lst.amenities = list(dict.fromkeys(d["amenities"]))184        if d.get("address"):185            lst.address = d["address"]186        if d.get("images"):187            lst.images = d["images"]188        if d.get("lat") is not None and d.get("lng") is not None:189            lst.lat, lst.lng = d["lat"], d["lng"]190