spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/gimcote.py : connecteur GIM Côté inc. (gimcote.com)5#   WordPress + thème immobilier Houzez. Archive /property-type/appartement6#   paginée : chaque carte contient prix, adresse, statut, type et la galerie7#   complète d'images (attribut data-images). Les fiches détail (via cache BD)8#   ajoutent description, caractéristiques, bloc « Détails » structuré9#   (animaux, meublé, fumeur, stationnement) et coordonnées GPS.10# -----------------------------------------------------------------------------11from __future__ import annotations1213import hashlib14import html as htmllib15import json16import re1718from bs4 import BeautifulSoup1920from ..schema import (Listing, infer_city, normalize_unit_type, parse_price,21                      strip_accents)22from .base import BaseConnector2324BASE = "https://gimcote.com"25LIST_URL = f"{BASE}/property-type/appartement/"2627_SIZE_SUFFIX = re.compile(r"-\d{2,4}x\d{2,4}(?=\.(?:jpg|jpeg|png|webp)$)", re.I)282930def _clean_price_label(label: str) -> str:31    """'1,450$/Par mois' -> '1450$' compatible parse_price (virgule = milliers)."""32    return re.sub(r"(\d),(\d{3})", r"\1\2", label)333435_MAP_LATLNG_RE = re.compile(r'"lat"\s*:\s*"(-?\d+\.\d+)"\s*,\s*"lng"\s*:\s*"(-?\d+\.\d+)"')363738def _oui_non(raw: str) -> bool | None:39    """'Oui'/'Non' (et variantes) -> bool, sinon None (inconnu)."""40    k = strip_accents((raw or "").strip().lower())41    if k in ("oui", "yes") or k.startswith("oui"):42        return True43    if k in ("non", "no") or k.startswith("non"):44        return False45    return None464748def _pets_value(raw: str) -> str | None:49    """Valeur « Animaux » de la fiche -> oui/non/conditions (jamais deviné)."""50    k = strip_accents((raw or "").strip().lower())51    if not k:52        return None53    if k.startswith("non") or "refus" in k or "aucun" in k:54        return "non"55    if k.startswith("oui") or "accepte" in k:56        return "oui"57    # « Chat opéré », « Chat seulement », « Petit chien »… = sous conditions58    if re.search(r"chat|chien|opere|seulement|condition|approbation", k):59        return "conditions"60    return None616263class GimCoteConnector(BaseConnector):64    source_id = "gimcote"65    request_delay = 0.666    max_pages = 20      # garde-fou de pagination67    max_details = 150   # garde-fou fiches détail (vraies requêtes)6869    def fetch(self) -> list[Listing]:70        listings: dict[str, Listing] = {}71        for page in range(1, self.max_pages + 1):72            url = LIST_URL if page == 1 else f"{LIST_URL}page/{page}/"73            try:74                html = self.get(url).text75            except Exception:76                break77            soup = BeautifulSoup(html, "html.parser")78            cards = soup.select("div.item-listing-wrap")79            if not cards:80                break81            for card in cards:82                try:83                    self._parse_card(card, listings)84                except Exception:85                    continue8687        # Fiches détail (cache BD) : description, caractéristiques, bloc88        # « Détails » structuré (animaux/meublé/fumeur/stationnement), GPS89        self._fetched = 090        for lst in listings.values():91            card_key = hashlib.sha1(92                f"{lst.title}|{lst.price_label}|{lst.availability}|{lst.url}"93                .encode("utf-8")).hexdigest()94            try:95                payload = self.detail(lst.external_id, card_key,96                                      lambda u=lst.url: self._fetch_detail(u))97            except Exception:98                continue99            self._apply_detail(lst, payload)100101        return list(listings.values())102103    # -- carte Houzez -----------------------------------------------------------104    def _parse_card(self, card, listings: dict[str, Listing]) -> None:105        link = card.select_one("h2.item-title a[href]")106        if not link:107            return108        url = link["href"]109        title = link.get_text(strip=True)110        m = re.search(r"/property/([^/]+)/?", url)111        slug = m.group(1) if m else ""112        listid_el = card.select_one("[data-listid]")113        ext_id = (listid_el.get("data-listid") if listid_el else "") or slug114        if not ext_id or ext_id in listings:115            return116117        # exclusions : stationnement / commercial / rangement118        if re.search(r"stationnement|commercial|rangement|garage|entrep[oô]t",119                     title, re.I):120            return121122        # adresse complète (Nominatim) : "3345, Avenue du Colisée, Lairet,123        # La Cité-Limoilou, Quebec, Urban agglomeration of Québec, ..."124        addr_el = card.select_one("address.item-address")125        full_addr = addr_el.get_text(" ", strip=True) if addr_el else ""126        parts = [p.strip() for p in full_addr.split(",") if p.strip()]127        address = ", ".join(parts[:2]) if len(parts) >= 2 else full_addr128        if "lévis" in full_addr.lower() or "levis" in full_addr.lower():129            city = "Lévis"130        elif "québec" in full_addr.lower() or "quebec" in full_addr.lower() or not full_addr:131            city = "Québec"132        else:133            return  # hors Québec / Lévis134        # secteur = micro-quartier + arrondissement (avant les mentions génériques)135        sector_parts = [p for p in parts[2:]136                        if not re.search(r"^(quebec|québec|urban agglomeration|"137                                         r"capitale-nationale|chaudière-appalaches|"138                                         r"canada|g\d[a-z]\s?\d[a-z]\d)", p, re.I)]139        sector = ", ".join(sector_parts[:2])140        city = infer_city(sector, default=city)141142        # statut / disponibilité — on saute les logements déjà loués143        status_el = card.select_one("a[href*='/status/']")144        availability = status_el.get_text(strip=True) if status_el else ""145        if re.search(r"lou[ée]", availability, re.I):146            return147148        # type d'unité : le titre (rédigé par l'agence) prime sur l'étiquette,149        # parfois erronée ; repli sur l'étiquette /label/ de la carte150        type_el = card.select_one("a[href*='/label/']")151        unit_type = normalize_unit_type(title)152        if not re.fullmatch(r"\d½|Studio|Loft|Chambre|Maison", unit_type or ""):153            unit_type = normalize_unit_type(type_el.get_text(strip=True) if type_el else "")154        price_el = card.select_one("li.item-price")155        price_label = price_el.get_text(strip=True) if price_el else ""156157        amenities = []158        for li in card.select("ul.item-amenities li"):159            t = re.sub(r"\s+", " ", li.get_text(" ", strip=True))160            if t and t not in amenities:161                amenities.append(t)162163        # galerie complète : attribut data-images (JSON, URLs redimensionnées)164        images: list[str] = []165        raw = card.get("data-images") or ""166        if raw:167            try:168                urls = json.loads(htmllib.unescape(raw))169            except Exception:170                urls = re.findall(r"https?:[^\"',\\]+", htmllib.unescape(raw))171            for u in urls:172                u = u.replace("\\/", "/").strip()173                if not u.startswith("http"):174                    continue175                u = _SIZE_SUFFIX.sub("", u)   # version pleine taille (WordPress)176                if u not in images:177                    images.append(u)178        if not images:179            thumb = card.select_one("img.wp-post-image[src]")180            if thumb:181                images = [_SIZE_SUFFIX.sub("", thumb["src"])]182183        listings[str(ext_id)] = Listing(184            source=self.source_id,185            external_id=str(ext_id),186            url=url,187            title=title,188            address=address,189            sector=sector,190            city=city,191            unit_type=unit_type,192            price=parse_price(_clean_price_label(price_label)),193            price_label=price_label,194            availability=availability,195            amenities=amenities,196            images=images[:30],197        )198199    # -- fiche détail (Houzez) ----------------------------------------------------200    def _fetch_detail(self, url: str) -> dict:201        """Description complète, caractéristiques, bloc « Détails » (paires202        libellé/valeur) et coordonnées GPS (JSON de la carte Houzez)."""203        if self._fetched >= self.max_details:204            raise RuntimeError("budget de fiches détail atteint")205        self._fetched += 1206        html = self.get(url).text207        soup = BeautifulSoup(html, "html.parser")208        out: dict = {}209210        desc_el = soup.select_one("#property-description-wrap")211        if desc_el:212            txt = desc_el.get_text("\n", strip=True)213            txt = re.sub(r"^Description\n", "", txt)214            out["description"] = re.sub(r"[ \t]+", " ", txt).strip()[:1200]215216        out["amenities"] = [a.get_text(" ", strip=True)217                            for a in soup.select("#property-features-wrap li")218                            if a.get_text(strip=True)][:20]219220        # Paires « Stationnement: Non », « Animaux: Chat opéré », etc.221        for li in soup.select("#property-detail-wrap li"):222            st, sp = li.find("strong"), li.find("span")223            if not (st and sp):224                continue225            lab = strip_accents(st.get_text(" ", strip=True).lower())226            val = sp.get_text(" ", strip=True)227            if "animaux" in lab:228                out["pets_raw"] = val229            elif "meuble" in lab:230                out["furnished_raw"] = val231            elif "fumeur" in lab:232                out["smoking_raw"] = val233            elif "stationnement" in lab:234                out["parking_raw"] = val235236        m = _MAP_LATLNG_RE.search(html)237        if m:238            out["lat"], out["lng"] = float(m.group(1)), float(m.group(2))239        return out240241    def _apply_detail(self, lst: Listing, d: dict) -> None:242        """Reporte le payload (frais/cache) sur l'annonce."""243        if not d:244            return245        if d.get("description"):246            lst.description = d["description"]247        if d.get("amenities"):248            lst.amenities = list(dict.fromkeys(lst.amenities + d["amenities"]))249        pets = _pets_value(d.get("pets_raw", ""))250        if pets:251            lst.pets = pets252        furn = _oui_non(d.get("furnished_raw", ""))253        if furn is not None:254            lst.furnished = furn255        details: dict = {}256        smoking = _oui_non(d.get("smoking_raw", ""))257        if smoking is not None:258            details["smoking"] = smoking259        parking = _oui_non(d.get("parking_raw", ""))260        if parking is not None:261            details["parking"] = {"available": parking}262        if details:263            lst.details = details264        if d.get("lat") is not None and d.get("lng") is not None:265            lst.lat, lst.lng = d["lat"], d["lng"]266