spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/mkar.py : connecteur Gestion Immobilière MKAR5#   (gestionimmobilieremkar.ca — Saguenay : Jonquière/Chicoutimi/La Baie, Alma,6#   Chambord ; gère aussi pour des tiers). WordPress + thème maison blitzWP7#   (CPT property), tout rendu serveur. Archive /proprietes-gestion-immobiliere-8#   mkar/ : cartes `article.archive__item--property` — badge de disponibilité9#   (« Disponible »/« Complet »), type (« 4½ », « Chambre », « Loft »),10#   catégorie (Duplex, Colocation…), adresse + municipalité, photos. Fiche11#   /propriete/<slug>/ (via cache BD) : inclusions (chambres, salles de bain,12#   chauffage, wifi…), conditions de location (animaux, fumeur) et galerie.13#   Aucun prix publié nulle part -> price = None (rien d'inventé).14#   robots.txt Yoast standard (admin seulement), sitemap.15# -----------------------------------------------------------------------------16from __future__ import annotations1718import hashlib19import re2021from bs4 import BeautifulSoup2223from ..schema import Listing, normalize_unit_type, strip_accents24from .base import BaseConnector2526BASE = "https://gestionimmobilieremkar.ca"27LIST_URL = f"{BASE}/proprietes-gestion-immobiliere-mkar/"2829# municipalités du filtre du site ; celles de gauche = secteurs de Saguenay30_SAGUENAY = {"jonquiere", "chicoutimi", "la baie", "lac kenogami"}31_MUNICIPALITIES = _SAGUENAY | {"alma", "chambord"}323334def _split_locality(raw: str) -> tuple[str, str]:35    """« 1745 Rue Haziel - 1745 Chicoutimi » -> (adresse, municipalité)."""36    txt = re.sub(r"\s+", " ", raw).strip()37    for name in sorted(_MUNICIPALITIES, key=len, reverse=True):38        m = re.search(rf"\s({name})\s*$", strip_accents(txt.lower()))39        if m:40            return txt[: m.start()].strip(" -,"), txt[m.start():].strip()41    return txt, ""424344class MkarConnector(BaseConnector):45    source_id = "mkar"46    request_delay = 0.647    max_details = 25     # garde-fou fiches détail (vraies requêtes par sync)4849    def fetch(self) -> list[Listing]:50        html = self.get(LIST_URL).text51        soup = BeautifulSoup(html, "html.parser")5253        self._fetched = 054        listings: dict[str, Listing] = {}55        for card in soup.select("article.archive__item--property"):56            try:57                self._parse_card(card, listings)58            except Exception:59                continue60        return list(listings.values())6162    # -- carte (article.archive__item--property) --------------------------------------63    def _parse_card(self, card, listings: dict[str, Listing]) -> None:64        link = card.select_one('a.item__link[href*="/propriete/"]') \65            or card.select_one('a[href*="/propriete/"]')66        if not link:67            return68        url = link["href"]69        m = re.search(r"/propriete/([^/?#]+)", url)70        if not m:71            return72        ext_id = m.group(1)73        if not ext_id or ext_id in listings:74            return7576        # badge de disponibilité : on ne garde que les unités « Disponible »77        avail_el = card.select_one(".item__availability")78        availability = avail_el.get_text(strip=True) if avail_el else ""79        if availability and not re.search(r"disponible", availability, re.I):80            return8182        type_el = card.select_one(".item__room")83        unit_label = type_el.get_text(strip=True) if type_el else ""84        cat_el = card.select_one(".item__category")85        category = cat_el.get_text(strip=True) if cat_el else ""8687        # garages et espaces non résidentiels : exclus88        if re.search(r"garage|entrep[oô]t|commercial|bureau|local",89                     f"{unit_label} {category}", re.I):90            return9192        addr_el = card.select_one(".item__addresse")93        address, locality = _split_locality(94            addr_el.get_text(" ", strip=True) if addr_el else "")9596        loc_key = strip_accents(locality.lower())97        if loc_key in _SAGUENAY:98            city, sector = "Saguenay", locality99        else:100            city, sector = locality, ""101102        images: list[str] = []103        for img in card.select(".item__image img[src]"):104            u = img["src"]105            if u.startswith("http") and u not in images:106                images.append(u)107108        title = " — ".join(p for p in (category, address) if p) or unit_label109110        lst = Listing(111            source=self.source_id,112            external_id=ext_id,113            url=url,114            title=title,115            address=address,116            sector=sector,117            city=city,118            unit_type=normalize_unit_type(unit_label),119            price=None,                       # aucun prix publié par MKAR120            price_label="",121            availability=availability,122            amenities=[category] if category else [],123            images=images[:15],124        )125126        key = hashlib.sha1(127            f"{unit_label}|{category}|{address}|{availability}"128            .encode("utf-8")).hexdigest()129        try:130            payload = self.detail(ext_id, key,131                                  lambda u=url: self._fetch_detail(u))132            self._apply_detail(lst, payload)133        except Exception:134            pass135        listings[ext_id] = lst136137    # -- fiche détail (/propriete/<slug>/) ---------------------------------------------138    def _fetch_detail(self, url: str) -> dict:139        """Inclusions, conditions de location (animaux) et galerie complète."""140        if self._fetched >= self.max_details:141            raise RuntimeError("budget de fiches détail atteint")142        self._fetched += 1143        html = self.get(url).text144        soup = BeautifulSoup(html, "html.parser")145        out: dict = {}146147        # « Inclusions » : items structurés (1 Chambre, 1 Salle de bain, Wifi…)148        inclusions = [re.sub(r"\s+", " ", li.get_text(" ", strip=True))149                      for li in soup.select(".property__inclusions .inclusions__item h3")]150        out["amenities"] = [t for t in inclusions if t][:20]151152        # « Conditions de location » : <strong> successifs (bail, animaux, fumeur)153        conditions = [re.sub(r"\s+", " ", s.get_text(" ", strip=True))154                      for s in soup.select(".property__conditions strong")]155        conditions = [t for t in conditions if t]156        if conditions:157            out["description"] = ("Conditions de location : "158                                  + " ; ".join(conditions))[:1200]159            joined = " ".join(conditions).lower()160            if re.search(r"animaux ne sont pas permis", joined):161                out["pets"] = "non"162163        # GPS : carte du thème (attributs data-lat / data-lng)164        map_el = soup.select_one(".property__map .map[data-lat][data-lng]")165        if map_el:166            try:167                out["lat"] = float(map_el["data-lat"])168                out["lng"] = float(map_el["data-lng"])169            except (TypeError, ValueError):170                pass171172        # galerie (swiper) : photos pleine taille, icônes SVG exclues173        images: list[str] = []174        for img in soup.select(".property img[src*='/wp-content/uploads/']"):175            u = img["src"]176            if u.startswith("http") and not u.endswith(".svg") and u not in images:177                images.append(u)178        out["images"] = images[:25]179        return out180181    def _apply_detail(self, lst: Listing, d: dict) -> None:182        """Reporte le payload (frais ou en cache) sur l'annonce."""183        if not d:184            return185        if d.get("description"):186            lst.description = d["description"]187        if d.get("amenities"):188            lst.amenities = list(dict.fromkeys(lst.amenities + d["amenities"]))189        if d.get("pets"):190            lst.pets = d["pets"]191        if d.get("lat") is not None and d.get("lng") is not None:192            lst.lat, lst.lng = d["lat"], d["lng"]193        if d.get("images") and len(d["images"]) > len(lst.images):194            lst.images = d["images"]195