spb/lou-ka

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/immo_3r.py : connecteur IMMO 3R (immo3r.com)5#   Gestion immobilière locative en Mauricie / Centre-du-Québec :6#   Trois-Rivières (Cap-de-la-Madeleine), Shawinigan, Bécancour, Nicolet,7#   Saint-Maurice. WordPress rendu serveur : pages ville /location/<ville>/8#   avec cartes (type, rue, ville, prix, badge dispo-oui/dispo-non) ; pages9#   détail /apartments/<slug>/ avec JSON-LD schema.org (Offer → Apartment :10#   adresse postale complète, description, commodités, photos).11#   Granularité : une fiche par modèle d'appartement (adresse + typologie),12#   pas par unité individuelle.13# -----------------------------------------------------------------------------14from __future__ import annotations1516import hashlib17import json18import re1920from bs4 import BeautifulSoup2122from ..schema import Listing, normalize_unit_type23from .base import BaseConnector2425BASE = "https://immo3r.com"2627# Pages ville (rendues serveur, toutes les cartes présentes dans le HTML)28CITY_PAGES = [29    (f"{BASE}/location/trois-rivieres/", "Trois-Rivières"),30    (f"{BASE}/location/shawinigan/", "Shawinigan"),31    (f"{BASE}/location/becancour/", "Bécancour"),32    (f"{BASE}/location/nicolet/", "Nicolet"),33    (f"{BASE}/location/saint-maurice/", "Saint-Maurice"),34]3536# Badges de disponibilité conservés (cartes) — « En construction » et37# « Non disponible » = pas louable maintenant, on exclut.38KEEP_STATUS = {"disponible", "bientôt disponible", "bientot disponible"}3940PRICE_RE = re.compile(r"([\d\s ,]+)\s*\$")41IMG_JUNK_RE = re.compile(r"logo|icon|favicon|-\d{2,3}x\d{2,3}\.", re.I)424344class Immo3RConnector(BaseConnector):45    source_id = "immo_3r"46    request_delay = 0.74748    def fetch(self) -> list[Listing]:49        listings: list[Listing] = []50        seen: set[str] = set()51        for page_url, city in CITY_PAGES:52            try:53                html = self.get(page_url).text54            except Exception:55                continue56            soup = BeautifulSoup(html, "html.parser")57            for a in soup.select('a[href*="/apartments/"]'):58                try:59                    card = a.select_one("article.apartment-single")60                    if card is None:61                        continue62                    url = a.get("href", "").split("?")[0]63                    slug = url.rstrip("/").split("/")[-1]64                    if not slug or slug in seen:65                        continue6667                    status_el = card.select_one(".dispo-oui, .dispo-non")68                    status = (status_el.get_text(" ", strip=True)69                              if status_el else "")70                    if status.strip().lower() not in KEEP_STATUS:71                        continue72                    seen.add(slug)7374                    # type d'unité : badge « 5 1/2 » (ou le titre en secours)75                    rooms_el = card.select_one(".apartment-single__rooms")76                    sector_el = card.select_one(".apartment-single__sector")77                    title_txt = (sector_el.get_text(" ", strip=True)78                                 if sector_el else "")79                    unit_type = normalize_unit_type(80                        rooms_el.get_text(" ", strip=True) if rooms_el81                        else title_txt)8283                    # « Rue des Prairies, Trois-Rivières » (2e span du titre)84                    street = ""85                    h2 = card.select_one(".apartment-single__title")86                    if h2 is not None:87                        spans = [s for s in h2.find_all("span", recursive=False)88                                 if "apartment-single__sector"89                                 not in (s.get("class") or [])]90                        if spans:91                            street = spans[0].get_text(" ", strip=True)92                            street = re.sub(r"\s*,\s*" + re.escape(city) + r"$",93                                            "", street).strip(" ,")9495                    price = None96                    price_label = ""97                    price_el = card.select_one(".apartment-single__price")98                    if price_el is not None:99                        price_label = re.sub(r"\s+", " ",100                                             price_el.get_text(" ", strip=True))101                        m = PRICE_RE.search(price_label)102                        if m:103                            try:104                                price = float(m.group(1)105                                              .replace(" ", "")106                                              .replace(" ", "")107                                              .replace(",", ""))108                            except ValueError:109                                pass110111                    # page détail (JSON-LD) via cache BD — revisitée112                    # seulement si la carte liste a changé113                    card_key = hashlib.sha1(114                        f"{status}|{price_label}|{title_txt}|{street}"115                        .encode("utf-8")).hexdigest()116                    d = self.detail(slug, card_key,117                                    lambda url=url: self._fetch_detail(url))118119                    title = d.get("name") or title_txt or slug120                    address = d.get("address") or street121                    listings.append(Listing(122                        source=self.source_id,123                        external_id=slug,124                        url=url,125                        title=title,126                        address=address,127                        sector=d.get("sector", ""),128                        city=d.get("city") or city,129                        unit_type=unit_type,130                        price=price if price is not None else d.get("price"),131                        price_label=price_label,132                        availability=status,133                        description=d.get("description", ""),134                        amenities=d.get("amenities") or [],135                        images=d.get("images") or [],136                    ))137                except Exception:138                    continue139        return listings140141    def _fetch_detail(self, url: str) -> dict:142        """Champs riches depuis le JSON-LD `Offer` de la page détail :143        adresse postale, secteur, description, prix, commodités, photos."""144        out: dict = {}145        try:146            html = self.get(url).text147        except Exception:148            return out149        offer = None150        for m in re.finditer(151                r'<script type="application/ld\+json"[^>]*>([\s\S]*?)</script>',152                html):153            try:154                data = json.loads(m.group(1))155            except ValueError:156                continue157            if isinstance(data, dict) and data.get("@type") == "Offer" \158                    and isinstance(data.get("itemOffered"), dict):159                offer = data160                break161        if offer is None:162            return out163        apt = offer.get("itemOffered") or {}164        out["name"] = (apt.get("name") or "").strip()165        addr = apt.get("address") or {}166        street = (addr.get("streetAddress") or "").strip()167        if street:168            out["address"] = street169        if addr.get("addressLocality"):170            out["city"] = addr["addressLocality"].strip()171        out["description"] = (offer.get("description") or "").strip()[:900]172        out["amenities"] = [173            f["name"].strip() for f in (apt.get("amenityFeature") or [])174            if isinstance(f, dict) and f.get("name") and f.get("value")][:25]175        spec = offer.get("priceSpecification") or {}176        try:177            out["price"] = float(spec.get("price"))178        except (TypeError, ValueError):179            pass180        for prop in offer.get("additionalProperty") or []:181            if isinstance(prop, dict) and prop.get("name") == "Secteur":182                out["sector"] = (prop.get("value") or "").strip()183        imgs = offer.get("image") or []184        if isinstance(imgs, str):185            imgs = [imgs]186        out["images"] = [u for u in dict.fromkeys(imgs)187                         if isinstance(u, str) and not IMG_JUNK_RE.search(u)][:25]188        return out189