spb/lou-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/capreit.py : connecteur CAPREIT (capreit.ca)5#   Flux JSON officiel du moteur de recherche (admin-ajax `property_json`)6#   filtré sur les villes de la région de Québec ET du Grand Montréal (île de7#   Montréal, Laval, Rive-Sud, Rive-Nord proche); les fiches propriétés (rendu8#   serveur) fournissent les types d'unités, prix, disponibilités, commodités9#   et la galerie photo. Une annonce par type d'unité disponible.10#   Exclus : hors-province (province != QC) et villes hors des deux régions.11# -----------------------------------------------------------------------------12from __future__ import annotations1314import hashlib15import re1617from bs4 import BeautifulSoup1819from ..schema import (Listing, infer_city, normalize_unit_type, parse_price,20                      strip_accents)21from .base import BaseConnector2223BASE = "https://www.capreit.ca"24FEED_URL = f"{BASE}/wp-admin/admin-ajax.php?action=property_json&language=fr"2526# Villes du flux correspondant à la région Québec/Lévis27_QC_CITIES = {28    "ville de quebec", "quebec", "beauport", "levis", "sainte-foy",29    "charlesbourg", "loretteville", "sillery", "cap-rouge", "val-belair",30    "l-ancienne-lorette", "saint-augustin-de-desmaures", "wendake",31    "saint-romuald", "saint-nicolas", "charny",32}3334# Villes du Grand Montréal : clé normalisée du flux -> nom d'affichage35_GM_CITIES = {36    # Île de Montréal37    "montreal": "Montréal",38    "cote saint-luc": "Côte Saint-Luc",39    "cote-saint-luc": "Côte Saint-Luc",40    "westmount": "Westmount",41    "dorval": "Dorval",42    "pointe-claire": "Pointe-Claire",43    "mont-royal": "Mont-Royal",44    "dollard-des-ormeaux": "Dollard-des-Ormeaux",45    # Laval46    "laval": "Laval",47    # Longueuil / Rive-Sud48    "longueuil": "Longueuil",49    "brossard": "Brossard",50    "boucherville": "Boucherville",51    "saint-lambert": "Saint-Lambert",52    "saint-hubert": "Longueuil",53    "candiac": "Candiac",54    "chateauguay": "Châteauguay",55    # Rive-Nord proche56    "boisbriand": "Boisbriand",57    "repentigny": "Repentigny",58    "terrebonne": "Terrebonne",59    "mascouche": "Mascouche",60    "rosemere": "Rosemère",61    "sainte-therese": "Sainte-Thérèse",62    "blainville": "Blainville",63}64_IMG_RE = re.compile(65    r'https://www\.capreit\.ca/wp-content/uploads/[^"\'\s\\]+'66    r'\.(?:jpg|jpeg|png|webp)', re.I)67_SKIP_IMG = re.compile(68    r"logo|icon|favicon|cropped|-\d{2,4}x\d{2,4}\.|BIL|Phone|badge", re.I)697071class CapreitConnector(BaseConnector):72    source_id = "capreit"73    request_delay = 0.674    max_properties = 60      # garde-fou (région de Québec + Grand Montréal)75    max_images = 257677    @staticmethod78    def _city_key(city: str) -> str:79        return strip_accents((city or "").strip().lower())8081    def fetch(self) -> list[Listing]:82        props = self.get(FEED_URL).json()8384        listings: list[Listing] = []85        count = 086        for p in props:87            try:88                if (p.get("province") or "").strip().upper() != "QC":89                    continue90                ck = self._city_key(p.get("city", ""))91                if ck not in _QC_CITIES and ck not in _GM_CITIES:92                    continue93                if not p.get("has_vacancies"):94                    continue95                if count >= self.max_properties:96                    break97                count += 198                listings.extend(self._property_listings(p))99            except Exception:100                continue101        return listings102103    def _property_listings(self, p: dict) -> list[Listing]:104        pid = str(p.get("id"))105        url = p.get("url") or ""106        title = (p.get("title") or "").strip()107        address = (p.get("address") or "").strip()108        feed_city = (p.get("city") or "").strip()109        # adresse complète : rue + ville + code postal (tous fournis au flux)110        postal = (p.get("postal_code") or "").strip()111        if address and feed_city:112            address = f"{address}, {feed_city}" + (f", QC {postal}"113                                                   if postal else "")114        # coordonnées GPS du flux115        try:116            lat = float(p["latitude"]) if p.get("latitude") else None117            lng = float(p["longitude"]) if p.get("longitude") else None118        except (TypeError, ValueError):119            lat = lng = None120        incentive = (p.get("incentive") or "").strip()121        # secteur : ville précise du flux (ex. Beauport) sinon intersection122        city_key = self._city_key(feed_city)123        if city_key in _GM_CITIES:124            # Grand Montréal : la ville du flux est la vraie ville125            sector = (p.get("nearest_intersection") or "").strip()126            city = _GM_CITIES[city_key]127        elif city_key in ("ville de quebec", "quebec"):128            sector = (p.get("nearest_intersection") or "").strip()129            city = infer_city(sector, default="Québec")130        else:131            sector = feed_city132            city = infer_city(sector, default="Québec")133134        # fiche propriété (rendu serveur) via le cache BD : revisitée135        # seulement quand la ligne du flux change136        feed_key = hashlib.sha1("|".join(137            str(p.get(k)) for k in138            ("id", "min_rent", "earliest_date", "vacancy_message",139             "price_range", "has_vacancies", "units_count", "incentive")140        ).encode("utf-8")).hexdigest()141        d = self.detail(pid, feed_key, lambda: self._fetch_property(url))142        desc = d.get("desc", "")143        amenities = d.get("amenities", [])144        images = d.get("images", [])145        rows = d.get("rows", [])146147        # promotion du flux (ex. « 1 mois de loyer gratuit »)148        if incentive:149            desc = f"Promotion : {incentive}. {desc}".strip()150151        out: list[Listing] = []152        if rows:153            for r in rows:154                ut = normalize_unit_type(r["unit_raw"])155                slug = re.sub(r"[^a-z0-9]+", "-",156                              strip_accents(r["unit_raw"].lower())).strip("-")157                out.append(Listing(158                    source=self.source_id,159                    external_id=f"{pid}-{slug or 'u'}",160                    url=url,161                    title=f"{title} — {r['unit_raw']}" if r["unit_raw"]162                    else title,163                    address=address,164                    sector=sector,165                    city=city,166                    unit_type=ut,167                    price=parse_price(r["price"]),168                    price_label=r["price"],169                    availability=r["avail"],170                    description=" — ".join(x for x in [desc, r["sqft"]] if x)[:600],171                    amenities=amenities,172                    images=images,173                    lat=lat,174                    lng=lng,175                ))176        else:177            # repli : annonce par propriété avec le prix plancher du flux178            min_rent = p.get("min_rent")179            # date de disponibilité structurée du flux (ex. 20260201)180            avail_date = None181            ed = str(p.get("earliest_date") or "")182            if re.fullmatch(r"20\d{6}", ed):183                avail_date = f"{ed[:4]}-{ed[4:6]}-{ed[6:]}"184            out.append(Listing(185                source=self.source_id,186                external_id=pid,187                url=url,188                title=title,189                address=address,190                sector=sector,191                city=city,192                unit_type=normalize_unit_type(193                    (p.get("bedroom_range") or "").split("-")[0]),194                price=float(min_rent) if min_rent else None,195                price_label=p.get("price_range") or "",196                availability=p.get("vacancy_message") or "",197                availability_date=avail_date,198                description=desc,199                amenities=amenities,200                images=images,201                lat=lat,202                lng=lng,203            ))204        return out205206    def _fetch_property(self, url: str) -> dict:207        """Scrape la fiche propriété : galerie, commodités, description,208        et une ligne par type d'unité disponible (« Vos options »)."""209        out: dict = {"desc": "", "amenities": [], "images": [], "rows": []}210        try:211            page = self.get(url).text212        except Exception:213            return out214        soup = BeautifulSoup(page, "html.parser")215216        # galerie photos (héro + blocs JSON de la page)217        images: list[str] = []218        for u in _IMG_RE.findall(page):219            if _SKIP_IMG.search(u):220                continue221            if u not in images:222                images.append(u)223        out["images"] = images[: self.max_images]224225        # commodités (listes à icônes)226        amenities: list[str] = []227        seen = set()228        for li in soup.select("li"):229            if not li.find("div", class_="icon"):230                continue231            t = li.get_text(" ", strip=True)232            if t and len(t) < 60 and t not in seen:233                seen.add(t)234                amenities.append(t)235        out["amenities"] = amenities[:25]236237        # description (« Caractéristiques de l'immeuble »)238        h = soup.find(["h2", "h3"], string=re.compile(239            "Caractéristiques de l['’]immeuble"))240        if h:241            nxt = h.find_next(["p", "div"])242            if nxt:243                out["desc"] = nxt.get_text(" ", strip=True)[:600]244245        # types d'unités disponibles246        for li in soup.select("li.property-options-list-item"):247            avail_el = li.select_one(248                ".property-options-list-item-availability")249            price_el = li.select_one(250                ".property-options-list-item-price")251            details = [d.get_text(" ", strip=True)252                       for d in li.select(".property-options-item")]253            unit_raw = details[0] if details else ""254            sqft = details[1] if len(details) > 1 else ""255            if li.get("data-available") == "false":256                continue257            out["rows"].append({258                "unit_raw": unit_raw,259                "sqft": sqft,260                "price": price_el.get_text(" ", strip=True)261                if price_el else "",262                "avail": avail_el.get_text(" ", strip=True)263                if avail_el else "",264            })265        return out266