spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/picard.py : connecteur Picard Immobilier (picardimmobilier.net)5#   La page /recherche/ liste tous les logements disponibles (rendu serveur),6#   cartes .property-box-2 avec lien /logement/<secteur>/ref=<id>/.7#   Fiches détaillées : photos S3, disponibilité, description, commodités.8# -----------------------------------------------------------------------------9from __future__ import annotations1011import hashlib12import re1314from bs4 import BeautifulSoup1516from ..schema import Listing, infer_city, normalize_unit_type, parse_price17from .base import BaseConnector1819BASE = "https://picardimmobilier.net"20SEARCH_URL = f"{BASE}/recherche/"2122SECTOR_LABELS = {23    "charny": "Charny",24    "les-saules": "Les Saules",25    "sainte-foy": "Sainte-Foy",26    "beauport": "Beauport",27    "limoilou": "Limoilou",28    "loretteville": "Loretteville",29    "vieux-quebec": "Vieux-Québec",30    "montcalm": "Montcalm",31}32IMG_RE = re.compile(r"https://picardimmobilier\.s3\.amazonaws\.com/media/"33                    r'[^"\s\\)]+\.(?:jpe?g|png|webp)', re.I)34LATLNG_RE = re.compile(r'data-lat="(-?[\d.]+)"\s+data-lng="(-?[\d.]+)"')353637class _DetailBudget(Exception):38    """Plafond de requêtes de fiches atteint pour cette synchronisation."""394041class PicardConnector(BaseConnector):42    source_id = "picard"43    request_delay = 0.644    max_details = 80        # garde-fou de fetch des fiches4546    def fetch(self) -> list[Listing]:47        listings: dict[str, Listing] = {}48        try:49            html = self.get(SEARCH_URL).text50        except Exception:51            return []52        soup = BeautifulSoup(html, "html.parser")5354        for card in soup.select("div.property-box-2"):55            try:56                a = card.select_one('a[href*="ref="]')57                if not a:58                    continue59                m = re.search(r"/logement/([a-z\-]+)/ref=(\d+)/", a.get("href", ""))60                if not m:61                    continue62                sector_slug, ext_id = m.group(1), m.group(2)63                if ext_id in listings:64                    continue65                sector = SECTOR_LABELS.get(sector_slug,66                                           sector_slug.replace("-", " ").title())67                title_el = card.select_one("h3.title a")68                title = title_el.get_text(" ", strip=True) if title_el else ""69                # Exclure commercial / stationnement / rangement70                if re.search(r"Commercial|Stationnement|Rangement|Parking",71                             title, re.I):72                    continue73                loc_el = card.select_one("h5.location a")74                address = loc_el.get_text(" ", strip=True) if loc_el else ""75                price_el = card.select_one(".price-box")76                price_label = (re.sub(r"\s+", " ", price_el.get_text(" ", strip=True))77                               if price_el else "")78                # ex. "$1055.00 Par mois" -> normaliser pour parse_price79                pm = re.search(r"\$\s*([\d\s,\.]+)", price_label)80                price = None81                if pm:82                    try:83                        price = float(pm.group(1).replace(",", "").replace(" ", ""))84                        if not (100 <= price <= 20000):85                            price = None86                    except ValueError:87                        price = None88                unit_type = ""89                for li in card.select("ul.facilities-list li"):90                    t = li.get_text(" ", strip=True)91                    if "Chambre" in t:92                        unit_type = t.replace("Chambres", "").strip()93                        break94                img = card.select_one(".property-photo img")95                images = [img["src"]] if img and img.get("src") else []96                listings[ext_id] = Listing(97                    source=self.source_id,98                    external_id=ext_id,99                    url=f"{BASE}/logement/{sector_slug}/ref={ext_id}/",100                    title=title or (address or f"Logement {ext_id}"),101                    address=address,102                    sector=sector,103                    city=infer_city(sector),104                    unit_type=normalize_unit_type(unit_type),105                    price=price,106                    price_label=price_label,107                    images=images,108                )109            except Exception:110                continue111112        # Fiches détaillées (via cache self.detail : une vraie requête113        # seulement si la carte liste a changé) : photos, disponibilité,114        # adresse complète, description, commodités, lat/lng.115        budget = {"n": 0}116        for lst in listings.values():117            key = hashlib.sha1("|".join(118                (lst.title, lst.price_label, lst.address, lst.unit_type,119                 lst.images[0] if lst.images else "")).encode()).hexdigest()120121            def _fetch(url=lst.url):122                if budget["n"] >= self.max_details:123                    raise _DetailBudget(url)124                budget["n"] += 1125                return self._parse_detail(self.get(url).text)126127            try:128                payload = self.detail(lst.external_id, key, _fetch)129            except Exception:130                continue131            if payload.get("images"):132                lst.images = payload["images"][:25]133            if payload.get("availability"):134                lst.availability = payload["availability"]135            if payload.get("address"):136                lst.address = payload["address"]137            if payload.get("description"):138                lst.description = payload["description"]139            if payload.get("amenities"):140                lst.amenities = payload["amenities"]141            if payload.get("lat") is not None:142                lst.lat, lst.lng = payload["lat"], payload["lng"]143144        return list(listings.values())145146    # -- fiche /logement/<secteur>/ref=<id>/ ----------------------------------147    @staticmethod148    def _parse_detail(detail: str) -> dict:149        payload: dict = {}150        imgs = [u for u in dict.fromkeys(IMG_RE.findall(detail))151                if not re.search(r"logo|icon|favicon|static", u, re.I)]152        if imgs:153            payload["images"] = imgs154        dsoup = BeautifulSoup(detail, "html.parser")155        text = re.sub(r"\s+", " ", dsoup.get_text(" ", strip=True))156        # Adresse complète + disponibilité en tête de fiche157        am = re.search(r"(Disponible[^#]{0,40}?)\s*#?\s*Référence", text)158        if am:159            payload["availability"] = am.group(1).strip()160        adm = re.search(161            r"par mois\s+(.{5,90}?)\s+(?:Disponible|#\s*Référence)", text)162        if adm:163            payload["address"] = adm.group(1).strip()164        dm = re.search(r"Description\s+(.{10,600}?)\s+Caractéristiques", text)165        if dm:166            payload["description"] = dm.group(1).strip()167        amenities = [re.sub(r"\s+", " ", d.get_text(" ", strip=True))168                     for d in dsoup.select("div.col-6.text-dark")169                     if d.get_text(strip=True)][:15]170        if amenities:171            payload["amenities"] = amenities172        # Coordonnées : carte Google `#map-custom` (attributs data-lat/lng)173        mm = LATLNG_RE.search(detail)174        if mm:175            try:176                payload["lat"] = float(mm.group(1))177                payload["lng"] = float(mm.group(2))178            except ValueError:179                pass180        return payload181