spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/kijiji.py : Kijiji (kijiji.ca) — petites annonces immobilières5#   UNIQUEMENT les catégories immobilier À VENDRE, UNIQUEMENT le Québec (l9001) :6#     c35  maisons à vendre · c643 condos à vendre · c641 terrains à vendre7#   Les pages listent 40+ annonces dans __NEXT_DATA__ (Apollo state) avec titre,8#   prix, GPS, adresse et vignette — aucune API privée nécessaire.9# -----------------------------------------------------------------------------10from __future__ import annotations1112import json13import os14import re1516from ..schema import PropertyListing17from .base import BaseConnector1819from . import _detailutil as du2021BASE = "https://www.kijiji.ca"22# (code catégorie, segment d'URL, type canonique)23CATEGORIES = [24    (35, "b-maison-a-vendre", "Maison"),25    (643, "b-condo-a-vendre", "Condo"),26    (641, "b-terrain-a-vendre", "Terrain"),27]28MAX_PAGES = int(os.environ.get("IMMOKA_KIJIJI_MAX_PAGES", "100"))29DETAIL_LIMIT = int(os.environ.get("IMMOKA_KIJIJI_DETAIL_LIMIT", "400"))3031_ATTR_LABELS = {32    "numberbedrooms": "Chambres", "numberbathrooms": "Salles de bain",33    "areainfeet": "Superficie (pi²)", "forsalebyhousing": "À vendre par",34    "yearbuilt": "Année de construction", "sizesqft": "Superficie (pi²)",35}363738def _parse_kijiji_detail(html: str) -> dict:39    """Fiche Kijiji : description complète, attributs, galerie haute résolution."""40    m = re.search(41        r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',42        html, re.S)43    if not m:44        return {}45    try:46        data = json.loads(m.group(1))47    except ValueError:48        return {}49    apollo = data.get("props", {}).get("pageProps", {}).get("__APOLLO_STATE__", {})50    it = next((v for k, v in apollo.items()51               if k.startswith("StandardListing:") and isinstance(v, dict)52               and v.get("description")), None)53    if not it:54        return {}55    out: dict = {}56    if it.get("description"):57        out["description"] = str(it["description"]).strip()[:6000]58    imgs = [re.sub(r"rule=kijijica-\d+-\w+", "rule=kijijica-1600-jpg", u)59            for u in it.get("imageUrls") or []]60    if imgs:61        out["images"] = imgs62    features, details = [], {}63    for a in (it.get("attributes") or {}).get("all") or []:64        cn = a.get("canonicalName") or ""65        val = ", ".join(str(v) for v in a.get("values") or [])66        if not val:67            continue68        label = _ATTR_LABELS.get(cn, a.get("name") or cn)69        features.append(f"{label} : {val}")70        details[label] = val71        if cn == "numberbedrooms" and val.isdigit():72            out["bedrooms"] = int(val)73        elif cn == "numberbathrooms" and val.isdigit():74            out["bathrooms"] = int(val)75        elif cn in ("areainfeet", "sizesqft"):76            mn = re.search(r"[\d.]+", val.replace(",", ""))77            if mn:78                out["area_sqft"] = float(mn.group(0))79        elif cn == "yearbuilt" and val.isdigit():80            out["year_built"] = int(val)81    if features:82        out["features"] = features83    if details:84        out["details"] = details85    loc = it.get("location") or {}86    addr = (loc.get("address") or "").replace(", Canada", "")87    if re.match(r"\s*\d", addr):88        out["address"] = addr.split(",")[0]89    return out909192class KijijiConnector(BaseConnector):93    source_id = "kijiji"94    request_delay = 1.29596    def _next_data(self, html: str) -> dict:97        m = re.search(98            r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',99            html, re.S)100        return json.loads(m.group(1)) if m else {}101102    def _page(self, seg: str, cat: int, page: int) -> list[dict]:103        """Annonces (Apollo state) d'une page de catégorie."""104        path = (f"{seg}/quebec/c{cat}l9001" if page == 1105                else f"{seg}/quebec/page-{page}/c{cat}l9001")106        html = self.get(f"{BASE}/{path}").text107        data = self._next_data(html)108        apollo = (data.get("props", {}).get("pageProps", {})109                  .get("__APOLLO_STATE__", {}))110        return [v for k, v in apollo.items()111                if k.startswith("StandardListing:") and isinstance(v, dict)]112113    def _to_listing(self, it: dict, ptype: str) -> PropertyListing | None:114        lid = str(it.get("id") or "")115        url = it.get("url") or ""116        if not lid or not url:117            return None118        price = None119        pr = it.get("price") or {}120        if isinstance(pr, dict) and pr.get("amount"):121            price = round(pr["amount"] / 100.0, 0)   # cents → $122        loc = it.get("location") or {}123        coords = loc.get("coordinates") or {}124        address = (loc.get("address") or "").replace(", Canada", "")125        # « Saint-Hubert, QC J3Y 6Y3 » → ville avant la 1re virgule126        city = loc.get("name") or (address.split(",")[0] if address else "")127        images = []128        for u in it.get("imageUrls") or []:129            images.append(re.sub(r"rule=kijijica-\d+-", "rule=kijijica-640-", u))130        return PropertyListing(131            source=self.source_id,132            external_id=lid,133            url=url,134            title=it.get("title") or "",135            address=address.split(",")[0] if re.match(r"\s*\d", address) else "",136            city=city,137            property_type=ptype,138            price=price,139            price_label=f"{price:,.0f} $".replace(",", " ") if price else "",140            description=(it.get("description") or "")[:2000],141            images=images,142            lat=coords.get("latitude"),143            lng=coords.get("longitude"),144            broker_name="Kijiji (particuliers)",145            agency="Kijiji Québec",146        )147148    def fetch(self) -> list[PropertyListing]:149        out: dict[str, PropertyListing] = {}150        for cat, seg, ptype in CATEGORIES:151            for page in range(1, MAX_PAGES + 1):152                try:153                    items = self._page(seg, cat, page)154                except Exception:155                    break156                fresh = 0157                for it in items:158                    lst = self._to_listing(it, ptype)159                    if lst is not None and lst.uid not in out:160                        out[lst.uid] = lst161                        fresh += 1162                # plus rien de neuf (page de fin remplie de topAds répétés)163                if fresh == 0 or len(items) < 10:164                    break165        listings = list(out.values())166        du.enrich(self, listings, DETAIL_LIMIT, _parse_kijiji_detail, key="v1")167        return listings168