# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/gestipro.py : connecteur Gestipro (gestipro.info) # Site WordPress (thème Houzez) : liste paginée /a-louer/ avec fiches # « propriete ». Une annonce par unité; pages détail pour la galerie photos. # Stationnements, locaux commerciaux et rangements exclus. # ----------------------------------------------------------------------------- from __future__ import annotations import hashlib import json import re from bs4 import BeautifulSoup from ..schema import (Listing, infer_city, normalize_unit_type, parse_area_sqft, strip_accents) from .base import BaseConnector BASE = "https://gestipro.info" LIST_URL = f"{BASE}/a-louer/" IMG_RE = re.compile( r"https://gestipro\.info/wp-content/uploads/[^\"'\\\s\)]+" r"\.(?:jpg|jpeg|png|webp)", re.I) IMG_NOISE_RE = re.compile(r"logo|favicon|icon|cluster|-\d+x\d+\.", re.I) EXCLUDE_RE = re.compile( r"stationnement|parking|garage|rangement|entrepos|commercial|bureau|local", re.I) def _parse_price(text: str) -> float | None: """Gère « 2,427$/mois » (virgule = séparateur de milliers).""" if not text: return None s = text.replace(" ", " ").replace(" ", " ") s = re.sub(r"(\d),(\d{3})", r"\1\2", s) m = re.search(r"(\d[\d\s]*(?:[.,]\d{2})?)\s*\$", s) if not m: return None try: val = float(m.group(1).replace(" ", "").replace(",", ".")) except ValueError: return None return val if 100 <= val <= 20000 else None def _pets_value(raw: str) -> str | None: """Mappe la valeur « Animaux toléré » de la fiche vers oui/non/conditions.""" k = strip_accents((raw or "").lower()) if not k: return None if re.search(r"refus|interdit|non admis|non accepte|aucun|pas d", k): return "non" if re.search(r"seulement|condition|petit|approbation|restriction", k): return "conditions" if re.search(r"oui|accepte|admis|autorise|tolere|bienvenu", k): return "oui" return None class GestiproConnector(BaseConnector): source_id = "gestipro" request_delay = 0.5 max_list_pages = 15 # garde-fou pagination max_details = 150 # garde-fou fiches détail def fetch(self) -> list[Listing]: # 1) Pagination : /a-louer/ puis /a-louer/page/N/ first = self.get(LIST_URL).text pages = [first] nums = [int(n) for n in re.findall(r"/a-louer/page/(\d+)/", first)] last = min(max(nums) if nums else 1, self.max_list_pages) for n in range(2, last + 1): try: pages.append(self.get(f"{LIST_URL}page/{n}/").text) except Exception: continue # 2) Cartes Houzez listings: dict[str, Listing] = {} for page in pages: soup = BeautifulSoup(page, "html.parser") for card in soup.select("div.item-listing-wrap[data-hz-id]"): try: lst = self._parse_card(card) except Exception: continue if lst and lst.external_id not in listings: listings[lst.external_id] = lst # 3) Fiches détail (cache BD) : galerie complète, description longue, # superficie, animaux, frais inclus, coordonnées GPS self._fetched = 0 for lst in listings.values(): card_key = hashlib.sha1( f"{lst.title}|{lst.price_label}|{lst.availability}|{lst.url}" .encode("utf-8")).hexdigest() try: payload = self.detail(lst.external_id, card_key, lambda u=lst.url: self._fetch_detail(u)) except Exception: continue self._apply_detail(lst, payload) return list(listings.values()) def _parse_card(self, card) -> Listing | None: ext_id = card.get("data-hz-id", "").strip() title_a = card.select_one(".item-title a") if not ext_id or not title_a: return None url = title_a.get("href", "") title = title_a.get_text(" ", strip=True) type_el = card.select_one(".h-type span") unit_raw = type_el.get_text(" ", strip=True) if type_el else "" # Exclusions : stationnement, commercial, rangement... if EXCLUDE_RE.search(f"{title} {unit_raw} {url}"): return None addr_el = card.select_one(".item-address span") or \ card.select_one(".item-address") addr_raw = addr_el.get_text(" ", strip=True) if addr_el else "" # « 7170 Boulevard Cloutier, Québec, QC, Canada, Charlesbourg, Québec » parts = [p.strip() for p in addr_raw.split(",") if p.strip()] address = parts[0] if parts else "" sector = "" for p in reversed(parts[1:]): if p not in ("Québec", "QC", "Canada", "Quebec", "Lévis", "Levis"): sector = p break city = "Lévis" if re.search(r"l[ée]vis", addr_raw, re.I) else "Québec" price_el = card.select_one(".item-price") price_label = price_el.get_text(" ", strip=True) if price_el else "" avail = ", ".join(a.get_text(" ", strip=True) for a in card.select(".label-status")[:2]) img_el = card.select_one(".listing-thumb img") img = "" if img_el: img = img_el.get("data-src") or img_el.get("src") or "" if img.startswith("data:"): img = img_el.get("data-src") or "" return Listing( source=self.source_id, external_id=ext_id, url=url, title=title, address=address, sector=sector, city=infer_city(sector, default=city), unit_type=normalize_unit_type(unit_raw), price=_parse_price(price_label), price_label=price_label, availability=avail, images=[img] if img else [], ) def _fetch_detail(self, url: str) -> dict: """Fiche Houzez : description complète, caractéristiques, bloc « Détails » (superficie, animaux, frais, disponibilité) et géoloc (JSON-LD schema.org Place). Retour JSON-sérialisable (cache BD).""" if self._fetched >= self.max_details: raise RuntimeError("budget de fiches détail atteint") self._fetched += 1 html = self.get(url).text soup = BeautifulSoup(html, "html.parser") out: dict = {} # description complète (repli : og:description) desc_el = soup.select_one("#property-description-wrap") if desc_el: txt = desc_el.get_text("\n", strip=True) txt = re.sub(r"^Description\n", "", txt) txt = txt.replace("Read More", " ").replace("Read Less", " ") out["description"] = re.sub(r"[ \t]+", " ", txt).strip()[:1200] else: og = soup.find("meta", attrs={"property": "og:description"}) or \ soup.find("meta", attrs={"name": "description"}) if og and og.get("content"): out["description"] = og["content"].strip()[:600] # Commodités (bloc « Caractéristiques » Houzez) amen = [a.get_text(" ", strip=True) for a in soup.select("#property-features-wrap li a")] out["amenities"] = [a for a in amen if a][:20] # Bloc « Détails » : paires libellé/valeur structurées pairs: dict[str, str] = {} for li in soup.select("#property-detail-wrap .list-lined-item"): st, sp = li.find("strong"), li.find("span") if st and sp: lab = strip_accents(st.get_text(" ", strip=True).lower()) pairs[lab] = sp.get_text(" ", strip=True) for lab, val in pairs.items(): if "dimension" in lab or "superficie" in lab: out["area_label"] = val # ex. « 456 pi² » elif "animaux" in lab: out["pets_raw"] = val # ex. « Chat seulement » elif "frais" in lab: out["frais"] = val # ex. « Eau chaude et wifi inclus » elif "disponibilit" in lab: out["availability"] = val # Géolocalisation : JSON-LD schema.org (Place -> geo) for sc in soup.find_all("script", type="application/ld+json"): try: data = json.loads(sc.string or "") except Exception: continue geo = data.get("geo") if isinstance(data, dict) else None if isinstance(geo, dict): try: out["lat"] = float(geo.get("latitude")) out["lng"] = float(geo.get("longitude")) except (TypeError, ValueError): pass break imgs = [u for u in dict.fromkeys(IMG_RE.findall(html)) if not IMG_NOISE_RE.search(u)] out["images"] = imgs[:25] return out def _apply_detail(self, lst: Listing, d: dict) -> None: """Reporte le payload (frais/cache) sur l'annonce.""" if not d: return if d.get("description"): lst.description = d["description"] amenities = list(d.get("amenities") or []) if d.get("frais"): amenities.append(f"Frais et charge : {d['frais']}") if amenities: lst.amenities = amenities if d.get("availability"): lst.availability = d["availability"] if d.get("area_label"): lst.area_sqft = parse_area_sqft(d["area_label"]) pets = _pets_value(d.get("pets_raw", "")) if pets: lst.pets = pets if d.get("lat") is not None and d.get("lng") is not None: lst.lat, lst.lng = d["lat"], d["lng"] if d.get("images"): lst.images = d["images"]