spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/gestipro.py : connecteur Gestipro (gestipro.info)5#   Site WordPress (thème Houzez) : liste paginée /a-louer/ avec fiches6#   « propriete ». Une annonce par unité; pages détail pour la galerie photos.7#   Stationnements, locaux commerciaux et rangements exclus.8# -----------------------------------------------------------------------------9from __future__ import annotations1011import hashlib12import json13import re1415from bs4 import BeautifulSoup1617from ..schema import (Listing, infer_city, normalize_unit_type,18                      parse_area_sqft, strip_accents)19from .base import BaseConnector2021BASE = "https://gestipro.info"22LIST_URL = f"{BASE}/a-louer/"2324IMG_RE = re.compile(25    r"https://gestipro\.info/wp-content/uploads/[^\"'\\\s\)]+"26    r"\.(?:jpg|jpeg|png|webp)", re.I)27IMG_NOISE_RE = re.compile(r"logo|favicon|icon|cluster|-\d+x\d+\.", re.I)28EXCLUDE_RE = re.compile(29    r"stationnement|parking|garage|rangement|entrepos|commercial|bureau|local",30    re.I)313233def _parse_price(text: str) -> float | None:34    """Gère « 2,427$/mois » (virgule = séparateur de milliers)."""35    if not text:36        return None37    s = text.replace(" ", " ").replace(" ", " ")38    s = re.sub(r"(\d),(\d{3})", r"\1\2", s)39    m = re.search(r"(\d[\d\s]*(?:[.,]\d{2})?)\s*\$", s)40    if not m:41        return None42    try:43        val = float(m.group(1).replace(" ", "").replace(",", "."))44    except ValueError:45        return None46    return val if 100 <= val <= 20000 else None474849def _pets_value(raw: str) -> str | None:50    """Mappe la valeur « Animaux toléré » de la fiche vers oui/non/conditions."""51    k = strip_accents((raw or "").lower())52    if not k:53        return None54    if re.search(r"refus|interdit|non admis|non accepte|aucun|pas d", k):55        return "non"56    if re.search(r"seulement|condition|petit|approbation|restriction", k):57        return "conditions"58    if re.search(r"oui|accepte|admis|autorise|tolere|bienvenu", k):59        return "oui"60    return None616263class GestiproConnector(BaseConnector):64    source_id = "gestipro"65    request_delay = 0.566    max_list_pages = 15          # garde-fou pagination67    max_details = 150            # garde-fou fiches détail6869    def fetch(self) -> list[Listing]:70        # 1) Pagination : /a-louer/ puis /a-louer/page/N/71        first = self.get(LIST_URL).text72        pages = [first]73        nums = [int(n) for n in re.findall(r"/a-louer/page/(\d+)/", first)]74        last = min(max(nums) if nums else 1, self.max_list_pages)75        for n in range(2, last + 1):76            try:77                pages.append(self.get(f"{LIST_URL}page/{n}/").text)78            except Exception:79                continue8081        # 2) Cartes Houzez82        listings: dict[str, Listing] = {}83        for page in pages:84            soup = BeautifulSoup(page, "html.parser")85            for card in soup.select("div.item-listing-wrap[data-hz-id]"):86                try:87                    lst = self._parse_card(card)88                except Exception:89                    continue90                if lst and lst.external_id not in listings:91                    listings[lst.external_id] = lst9293        # 3) Fiches détail (cache BD) : galerie complète, description longue,94        #    superficie, animaux, frais inclus, coordonnées GPS95        self._fetched = 096        for lst in listings.values():97            card_key = hashlib.sha1(98                f"{lst.title}|{lst.price_label}|{lst.availability}|{lst.url}"99                .encode("utf-8")).hexdigest()100            try:101                payload = self.detail(lst.external_id, card_key,102                                      lambda u=lst.url: self._fetch_detail(u))103            except Exception:104                continue105            self._apply_detail(lst, payload)106107        return list(listings.values())108109    def _parse_card(self, card) -> Listing | None:110        ext_id = card.get("data-hz-id", "").strip()111        title_a = card.select_one(".item-title a")112        if not ext_id or not title_a:113            return None114        url = title_a.get("href", "")115        title = title_a.get_text(" ", strip=True)116117        type_el = card.select_one(".h-type span")118        unit_raw = type_el.get_text(" ", strip=True) if type_el else ""119120        # Exclusions : stationnement, commercial, rangement...121        if EXCLUDE_RE.search(f"{title} {unit_raw} {url}"):122            return None123124        addr_el = card.select_one(".item-address span") or \125            card.select_one(".item-address")126        addr_raw = addr_el.get_text(" ", strip=True) if addr_el else ""127        # « 7170 Boulevard Cloutier, Québec, QC, Canada, Charlesbourg, Québec »128        parts = [p.strip() for p in addr_raw.split(",") if p.strip()]129        address = parts[0] if parts else ""130        sector = ""131        for p in reversed(parts[1:]):132            if p not in ("Québec", "QC", "Canada", "Quebec", "Lévis", "Levis"):133                sector = p134                break135        city = "Lévis" if re.search(r"l[ée]vis", addr_raw, re.I) else "Québec"136137        price_el = card.select_one(".item-price")138        price_label = price_el.get_text(" ", strip=True) if price_el else ""139140        avail = ", ".join(a.get_text(" ", strip=True)141                          for a in card.select(".label-status")[:2])142143        img_el = card.select_one(".listing-thumb img")144        img = ""145        if img_el:146            img = img_el.get("data-src") or img_el.get("src") or ""147            if img.startswith("data:"):148                img = img_el.get("data-src") or ""149150        return Listing(151            source=self.source_id,152            external_id=ext_id,153            url=url,154            title=title,155            address=address,156            sector=sector,157            city=infer_city(sector, default=city),158            unit_type=normalize_unit_type(unit_raw),159            price=_parse_price(price_label),160            price_label=price_label,161            availability=avail,162            images=[img] if img else [],163        )164165    def _fetch_detail(self, url: str) -> dict:166        """Fiche Houzez : description complète, caractéristiques, bloc167        « Détails » (superficie, animaux, frais, disponibilité) et géoloc168        (JSON-LD schema.org Place). Retour JSON-sérialisable (cache BD)."""169        if self._fetched >= self.max_details:170            raise RuntimeError("budget de fiches détail atteint")171        self._fetched += 1172        html = self.get(url).text173        soup = BeautifulSoup(html, "html.parser")174        out: dict = {}175176        # description complète (repli : og:description)177        desc_el = soup.select_one("#property-description-wrap")178        if desc_el:179            txt = desc_el.get_text("\n", strip=True)180            txt = re.sub(r"^Description\n", "", txt)181            txt = txt.replace("Read More", " ").replace("Read Less", " ")182            out["description"] = re.sub(r"[ \t]+", " ", txt).strip()[:1200]183        else:184            og = soup.find("meta", attrs={"property": "og:description"}) or \185                soup.find("meta", attrs={"name": "description"})186            if og and og.get("content"):187                out["description"] = og["content"].strip()[:600]188189        # Commodités (bloc « Caractéristiques » Houzez)190        amen = [a.get_text(" ", strip=True)191                for a in soup.select("#property-features-wrap li a")]192        out["amenities"] = [a for a in amen if a][:20]193194        # Bloc « Détails » : paires libellé/valeur structurées195        pairs: dict[str, str] = {}196        for li in soup.select("#property-detail-wrap .list-lined-item"):197            st, sp = li.find("strong"), li.find("span")198            if st and sp:199                lab = strip_accents(st.get_text(" ", strip=True).lower())200                pairs[lab] = sp.get_text(" ", strip=True)201        for lab, val in pairs.items():202            if "dimension" in lab or "superficie" in lab:203                out["area_label"] = val                 # ex. « 456 pi² »204            elif "animaux" in lab:205                out["pets_raw"] = val                   # ex. « Chat seulement »206            elif "frais" in lab:207                out["frais"] = val                      # ex. « Eau chaude et wifi inclus »208            elif "disponibilit" in lab:209                out["availability"] = val210211        # Géolocalisation : JSON-LD schema.org (Place -> geo)212        for sc in soup.find_all("script", type="application/ld+json"):213            try:214                data = json.loads(sc.string or "")215            except Exception:216                continue217            geo = data.get("geo") if isinstance(data, dict) else None218            if isinstance(geo, dict):219                try:220                    out["lat"] = float(geo.get("latitude"))221                    out["lng"] = float(geo.get("longitude"))222                except (TypeError, ValueError):223                    pass224                break225226        imgs = [u for u in dict.fromkeys(IMG_RE.findall(html))227                if not IMG_NOISE_RE.search(u)]228        out["images"] = imgs[:25]229        return out230231    def _apply_detail(self, lst: Listing, d: dict) -> None:232        """Reporte le payload (frais/cache) sur l'annonce."""233        if not d:234            return235        if d.get("description"):236            lst.description = d["description"]237        amenities = list(d.get("amenities") or [])238        if d.get("frais"):239            amenities.append(f"Frais et charge : {d['frais']}")240        if amenities:241            lst.amenities = amenities242        if d.get("availability"):243            lst.availability = d["availability"]244        if d.get("area_label"):245            lst.area_sqft = parse_area_sqft(d["area_label"])246        pets = _pets_value(d.get("pets_raw", ""))247        if pets:248            lst.pets = pets249        if d.get("lat") is not None and d.get("lng") is not None:250            lst.lat, lst.lng = d["lat"], d["lng"]251        if d.get("images"):252            lst.images = d["images"]253