spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Lou-Ka — Agrégateur de logements à louer (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/kijiji.py : Kijiji (kijiji.ca) — petites annonces de LOCATION5#   UNIQUEMENT les catégories logement À LOUER, UNIQUEMENT le Québec (l9001) :6#     c37 appartements & condos à louer · c36 chambres à louer & colocation7#   Les pages listent 40+ annonces dans __NEXT_DATA__ (Apollo state) avec titre,8#   prix, GPS, adresse, date de disponibilité et attributs (meublé, animaux,9#   inclusions…) — aucune API privée nécessaire. Adapté du connecteur « à10#   vendre » d'Immo-Ka (agent-courtage/immoka).11# -----------------------------------------------------------------------------12from __future__ import annotations1314import json15import os16import re1718from ..schema import Listing19from .base import BaseConnector2021from . import _detailutil as du2223BASE = "https://www.kijiji.ca"24# (code catégorie, segment d'URL, type d'unité par défaut)25CATEGORIES = [26    (37, "b-appartement-condo", ""),                 # unité déduite des attributs27    (36, "b-chambre-a-louer-colocataire", "Chambre"),28]29MAX_PAGES = int(os.environ.get("LOUKA_KIJIJI_MAX_PAGES", "100"))30DETAIL_LIMIT = int(os.environ.get("LOUKA_KIJIJI_DETAIL_LIMIT", "400"))3132# les annonces vivent sous des clés Apollo « RealEstateListing:123 » (c37)33# ou « StandardListing:123 » (c36)34_LISTING_KEY_RE = re.compile(r"^(?:RealEstate|Standard)Listing:\d+$")35_NEXT_RE = re.compile(36    r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', re.S)3738# attributs binaires -> commodité affichable (uniquement si la valeur est vraie)39_AMENITY_LABELS = {40    "heat": "Chauffage inclus", "hydro": "Électricité incluse",41    "water": "Eau incluse", "internet": "Internet inclus",42    "cabletv": "Câble/télé inclus", "laundryinunit": "Laveuse/sécheuse dans l'unité",43    "laundryinbuilding": "Buanderie dans l'immeuble", "dishwasher": "Lave-vaisselle",44    "fridgefreezer": "Réfrigérateur/congélateur", "airconditioning": "Air climatisé",45    "balcony": "Balcon", "elevator": "Ascenseur", "gym": "Salle d'entraînement",46    "pool": "Piscine", "concierge": "Concierge",47    "twentyfourhoursecurity": "Sécurité 24 h", "storagelocker": "Espace de rangement",48    "bicycleparking": "Stationnement pour vélo", "yard": "Cour",49    "wheelchairaccessible": "Accessible en fauteuil roulant",50}51_UNIT_TYPES = {52    "apartment": "Appartement", "condo": "Condo",53    "basement-apartment": "Appartement au sous-sol", "house": "Maison",54    "townhouse": "Maison de ville", "duplex-triplex": "Duplex/Triplex",55}56_AGREEMENTS = {"one-year": "Bail de 1 an", "month-to-month": "Au mois",57               "not-available": ""}58# villes fréquentes sans accents dans les adresses Kijiji59_CITY_FIX = {60    "montreal": "Montréal", "quebec": "Québec", "levis": "Lévis",61    "trois-rivieres": "Trois-Rivières", "riviere-des-prairies": "Montréal",62    "ville de montreal": "Montréal", "ville de quebec": "Québec",63}646566def _fix_city(raw: str) -> str:67    key = (raw or "").strip().lower()68    if key in _CITY_FIX:69        return _CITY_FIX[key]70    return " ".join(w.capitalize() for w in key.replace("-", " ").split())717273def _attr_value(a: dict) -> str:74    """Première valeur d'un attribut Apollo (canonique, sinon affichée)."""75    for k in ("canonicalValues", "values"):76        vals = a.get(k) or []77        if vals:78            return str(vals[0])79    return ""808182def _apply_attrs(attrs: list[dict], out: dict) -> None:83    """Interprète les attributs Kijiji (mêmes clés en liste et en fiche)."""84    amenities = out.setdefault("amenities", [])85    details = out.setdefault("details", {})86    for a in attrs or []:87        cn = a.get("canonicalName") or ""88        val = _attr_value(a)89        if not val:90            continue91        if cn in _AMENITY_LABELS:92            if val == "1":93                amenities.append(_AMENITY_LABELS[cn])94        elif cn == "furnished":95            out["furnished"] = val == "1"96        elif cn == "petsallowed":97            out["pets"] = "oui" if val == "1" else "non"98        elif cn == "numberbedrooms":99            out["bedrooms"] = val          # '0' = studio, sinon nb de chambres100        elif cn == "numberbathrooms":101            try:                            # canonique en dixièmes : '15' = 1.5102                n = int(val) / 10103                details["Salles de bain"] = f"{n:g}"104            except ValueError:105                pass106        elif cn in ("areainfeet", "sizesqft"):107            m = re.search(r"[\d.]+", val.replace(",", ""))108            if m and float(m.group(0)) > 0:109                out["area_sqft"] = float(m.group(0))110        elif cn == "dateavailable":111            m = re.match(r"(\d{4}-\d{2}-\d{2})", val)112            if m:113                out["availability_date"] = m.group(1)114        elif cn == "unittype":115            details["Type d'unité"] = _UNIT_TYPES.get(val, val)116        elif cn == "agreementtype":117            bail = _AGREEMENTS.get(val, val)118            if bail:119                details["Bail"] = bail120        elif cn == "numberparkingspots" and val.isdigit() and int(val) > 0:121            amenities.append(f"Stationnement ({val})")122123124def _parse_kijiji_detail(html: str) -> dict:125    """Fiche Kijiji : description complète, attributs, galerie haute résolution."""126    m = _NEXT_RE.search(html)127    if not m:128        return {}129    try:130        data = json.loads(m.group(1))131    except ValueError:132        return {}133    apollo = data.get("props", {}).get("pageProps", {}).get("__APOLLO_STATE__", {})134    it = next((v for k, v in apollo.items()135               if _LISTING_KEY_RE.match(k) and isinstance(v, dict)136               and v.get("description")), None)137    if not it:138        return {}139    out: dict = {}140    if it.get("description"):141        out["description"] = str(it["description"]).strip()[:6000]142    imgs = [re.sub(r"rule=kijijica-\d+-\w+", "rule=kijijica-1600-jpg", u)143            for u in it.get("imageUrls") or []]144    if imgs:145        out["images"] = imgs146    _apply_attrs((it.get("attributes") or {}).get("all") or [], out)147    out.pop("bedrooms", None)   # le type d'unité est déjà fixé au niveau liste148    loc = it.get("location") or {}149    addr = (loc.get("address") or "").replace(", Canada", "")150    if re.match(r"\s*\d", addr):151        out["address"] = addr.split(",")[0]152    return out153154155class KijijiConnector(BaseConnector):156    source_id = "kijiji"157    request_delay = 1.2158159    def _page(self, seg: str, cat: int, page: int) -> list[dict]:160        """Annonces (Apollo state) d'une page de catégorie."""161        path = (f"{seg}/quebec/c{cat}l9001" if page == 1162                else f"{seg}/quebec/page-{page}/c{cat}l9001")163        html = self.get(f"{BASE}/{path}").text164        m = _NEXT_RE.search(html)165        data = json.loads(m.group(1)) if m else {}166        apollo = (data.get("props", {}).get("pageProps", {})167                  .get("__APOLLO_STATE__", {}))168        return [v for k, v in apollo.items()169                if _LISTING_KEY_RE.match(k) and isinstance(v, dict)]170171    def _to_listing(self, it: dict, unit_default: str) -> Listing | None:172        lid = str(it.get("id") or "")173        url = it.get("url") or ""174        if not lid or not url:175            return None176        price = None177        pr = it.get("price") or {}178        if isinstance(pr, dict) and pr.get("amount"):179            price = round(pr["amount"] / 100.0, 0)   # cents → $/mois180        loc = it.get("location") or {}181        coords = loc.get("coordinates") or {}182        address = (loc.get("address") or "").replace(", Canada", "")183        parts = [p.strip() for p in address.split(",") if p.strip()]184        street = parts[0] if parts and re.match(r"\s*\d", parts[0]) else ""185        city = _fix_city(parts[1] if street and len(parts) > 1186                         else (loc.get("name") or (parts[0] if parts else "")))187        images = [re.sub(r"rule=kijijica-\d+-", "rule=kijijica-640-", u)188                  for u in it.get("imageUrls") or []]189        extra: dict = {}190        _apply_attrs((it.get("attributes") or {}).get("all") or [], extra)191        unit_type = unit_default192        beds = extra.pop("bedrooms", None)193        if not unit_type and beds:194            try:                     # Kijiji code parfois « 2.5 » (2 ch. + den)195                n = int(float(beds))196            except ValueError:197                n = 0198            unit_type = "Studio" if n == 0 else f"{n} chambres"  # → n+2 ½199        lst = Listing(200            source=self.source_id,201            external_id=lid,202            url=url,203            title=it.get("title") or "",204            address=street,205            city=city,206            unit_type=unit_type,207            price=price,208            price_label=(f"{price:,.0f} $/mois".replace(",", " ")209                         if price else ""),210            description=(it.get("description") or "")[:2000],211            amenities=extra.get("amenities") or [],212            details=extra.get("details") or {},213            images=images,214            lat=coords.get("latitude"),215            lng=coords.get("longitude"),216        )217        if extra.get("availability_date"):218            lst.availability_date = extra["availability_date"]219            lst.availability = f"Libre le {extra['availability_date']}"220        if extra.get("furnished") is not None:221            lst.furnished = extra["furnished"]222        if extra.get("pets"):223            lst.pets = extra["pets"]224        if extra.get("area_sqft"):225            lst.area_sqft = extra["area_sqft"]226        return lst227228    def fetch(self) -> list[Listing]:229        out: dict[str, Listing] = {}230        for cat, seg, unit_default in CATEGORIES:231            for page in range(1, MAX_PAGES + 1):232                try:233                    items = self._page(seg, cat, page)234                except Exception:235                    break236                fresh = 0237                for it in items:238                    lst = self._to_listing(it, unit_default)239                    if lst is not None and lst.uid not in out:240                        out[lst.uid] = lst241                        fresh += 1242                # plus rien de neuf (page de fin remplie de topAds répétés)243                if fresh == 0 or len(items) < 10:244                    break245        listings = list(out.values())246        du.enrich(self, listings, DETAIL_LIMIT, _parse_kijiji_detail, key="v1")247        return listings248