spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/kijiji.py : Kijiji (kijiji.ca) — petites annonces immobilières5#   UNIQUEMENT les catégories immobilier À VENDRE, UNIQUEMENT le Québec (l9001) :6#     c35  maisons à vendre · c643 condos à vendre · c641 terrains à vendre7#   Les pages listent 40+ annonces dans __NEXT_DATA__ (Apollo state) avec titre,8#   prix, GPS, adresse et vignette — aucune API privée nécessaire.9# -----------------------------------------------------------------------------10from __future__ import annotations1112import json13import os14import re1516from ..schema import PropertyListing17from .base import BaseConnector1819from . import _detailutil as du2021BASE = "https://www.kijiji.ca"22# (code catégorie, segment d'URL, type canonique)23CATEGORIES = [24    (35, "b-maison-a-vendre", "Maison"),25    (643, "b-condo-a-vendre", "Condo"),26    (641, "b-terrain-a-vendre", "Terrain"),27]28MAX_PAGES = int(os.environ.get("IMMOKA_KIJIJI_MAX_PAGES", "100"))29DETAIL_LIMIT = int(os.environ.get("IMMOKA_KIJIJI_DETAIL_LIMIT", "400"))3031# attributs Apollo (canonicalName) -> libellé FR. Les attributs absents de la32# table gardent leur `name` Kijiji d'origine dans details.33_ATTR_LABELS = {34    "numberbedrooms": "Chambres", "numberbathrooms": "Salles de bain",35    "areainfeet": "Superficie (pi²)", "sizesqft": "Superficie (pi²)",36    "areainacres": "Superficie du terrain",   # texte libre : « 26 acres », « 35 000 pi2 »…37    "yearbuilt": "Année de construction", "forsalebyhousing": "À vendre par",38    "numberparkingspots": "Stationnements", "parkingincluded": "Stationnement inclus",39    "unittype": "Type d'unité", "furnished": "Meublé",40    "virtualtour": "Visite virtuelle", "videochat": "Visite par vidéo",41}4243_ACRES_RE = re.compile(r"([\d\s,.]+)\s*(?:acres?\b|ac\.?$)", re.I)444546def _lot_sqft_from_free_text(val: str) -> float | None:47    """Superficie de terrain depuis le champ libre « Size (acres) » de Kijiji :48    « 26 acres », « 35 000 pi2 », « 1586 mètre carré », ou un nombre nu (acres)."""49    from ..normalize import parse_area_sqft, parse_float50    t = (val or "").replace("pieds carres", "pi²").replace("pieds carrés", "pi²") \51        .replace("pied carré", "pi²").replace("mètres carrés", "m²") \52        .replace("mètre carré", "m²").replace("metre carre", "m²")53    t = re.sub(r"(\d),(\d{3})\b", r"\1\2", t)     # « 18,000 » = milliers, pas décimale54    v = parse_area_sqft(t)                    # unités pi²/m² explicites55    if v:56        return v57    m = _ACRES_RE.search(t)58    n = parse_float(m.group(1)) if m else None59    if n is None and re.fullmatch(r"[\d\s,.]+", t.strip()):60        n = parse_float(t)                    # nombre nu = acres (nom du champ)61    if n and 0 < n < 1000:62        return round(n * 43560)               # acres -> pi²63    return None646566def _parse_kijiji_detail(html: str) -> dict:67    """Fiche Kijiji : description complète, attributs, galerie haute résolution."""68    m = re.search(69        r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',70        html, re.S)71    if not m:72        return {}73    try:74        data = json.loads(m.group(1))75    except ValueError:76        return {}77    apollo = data.get("props", {}).get("pageProps", {}).get("__APOLLO_STATE__", {})78    it = next((v for k, v in apollo.items()79               if k.startswith("StandardListing:") and isinstance(v, dict)80               and v.get("description")), None)81    if not it:82        return {}83    out: dict = {}84    if it.get("description"):85        out["description"] = str(it["description"]).strip()[:6000]86    imgs = [re.sub(r"rule=kijijica-\d+-\w+", "rule=kijijica-1600-jpg", u)87            for u in it.get("imageUrls") or []]88    if imgs:89        out["images"] = imgs90    features, details = [], {}91    for a in (it.get("attributes") or {}).get("all") or []:92        cn = a.get("canonicalName") or ""93        val = ", ".join(str(v) for v in a.get("values") or [])94        if not val:95            continue96        label = _ATTR_LABELS.get(cn, a.get("name") or cn)97        features.append(f"{label} : {val}")98        details[label] = val99        if cn == "numberbedrooms" and val.isdigit():100            out["bedrooms"] = int(val)101        elif cn == "numberbathrooms":102            mn = re.search(r"\d+", val)       # « 1.5 » -> 1103            if mn:104                out["bathrooms"] = int(mn.group(0))105        elif cn in ("areainfeet", "sizesqft"):106            mn = re.search(r"[\d.]+", val.replace(",", ""))107            if mn:108                out["area_sqft"] = float(mn.group(0))109        elif cn == "areainacres":110            lot = _lot_sqft_from_free_text(val)111            if lot:112                out["lot_sqft"] = lot113        elif cn == "yearbuilt" and val.isdigit():114            out["year_built"] = int(val)115    if features:116        out["features"] = features117    if details:118        out["details"] = details119    loc = it.get("location") or {}120    addr = (loc.get("address") or "").replace(", Canada", "")121    if re.match(r"\s*\d", addr):122        out["address"] = addr.split(",")[0]123    return out124125126class KijijiConnector(BaseConnector):127    source_id = "kijiji"128    request_delay = 1.2129130    def _next_data(self, html: str) -> dict:131        m = re.search(132            r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',133            html, re.S)134        return json.loads(m.group(1)) if m else {}135136    def _page(self, seg: str, cat: int, page: int) -> list[dict]:137        """Annonces (Apollo state) d'une page de catégorie."""138        path = (f"{seg}/quebec/c{cat}l9001" if page == 1139                else f"{seg}/quebec/page-{page}/c{cat}l9001")140        html = self.get(f"{BASE}/{path}").text141        data = self._next_data(html)142        apollo = (data.get("props", {}).get("pageProps", {})143                  .get("__APOLLO_STATE__", {}))144        return [v for k, v in apollo.items()145                if k.startswith("StandardListing:") and isinstance(v, dict)]146147    def _to_listing(self, it: dict, ptype: str) -> PropertyListing | None:148        lid = str(it.get("id") or "")149        url = it.get("url") or ""150        if not lid or not url:151            return None152        price = None153        pr = it.get("price") or {}154        if isinstance(pr, dict) and pr.get("amount"):155            price = round(pr["amount"] / 100.0, 0)   # cents → $156            if price < 5000:      # prix bidon fréquent sur Kijiji (1 $, 123 $…)157                price = None158        loc = it.get("location") or {}159        coords = loc.get("coordinates") or {}160        address = (loc.get("address") or "").replace(", Canada", "")161        # « Saint-Hubert, QC J3Y 6Y3 » → ville avant la 1re virgule162        city = loc.get("name") or (address.split(",")[0] if address else "")163        images = []164        for u in it.get("imageUrls") or []:165            images.append(re.sub(r"rule=kijijica-\d+-", "rule=kijijica-640-", u))166        details = {}167        # date de mise en ligne : activationDate = 1re publication (sortingDate168        # est re-bumpée par les remontées/TOP AD)169        posted = str(it.get("activationDate") or it.get("sortingDate") or "")170        if re.match(r"\d{4}-\d{2}-\d{2}", posted):171            details["listed_at"] = posted[:10]172        return PropertyListing(173            source=self.source_id,174            external_id=lid,175            url=url,176            title=it.get("title") or "",177            address=address.split(",")[0] if re.match(r"\s*\d", address) else "",178            city=city,179            property_type=ptype,180            price=price,181            price_label=f"{price:,.0f} $".replace(",", " ") if price else "",182            description=(it.get("description") or "")[:2000],183            details=details,184            images=images,185            lat=coords.get("latitude"),186            lng=coords.get("longitude"),187            broker_name="Kijiji (particuliers)",188            agency="Kijiji Québec",189        )190191    def fetch(self) -> list[PropertyListing]:192        out: dict[str, PropertyListing] = {}193        for cat, seg, ptype in CATEGORIES:194            for page in range(1, MAX_PAGES + 1):195                try:196                    items = self._page(seg, cat, page)197                except Exception:198                    break199                fresh = 0200                for it in items:201                    lst = self._to_listing(it, ptype)202                    if lst is not None and lst.uid not in out:203                        out[lst.uid] = lst204                        fresh += 1205                # plus rien de neuf (page de fin remplie de topAds répétés)206                if fresh == 0 or len(items) < 10:207                    break208        listings = list(out.values())209        # v2 = _ATTR_LABELS étendus (terrain/stationnement/année…) + lot_sqft210        du.enrich(self, listings, DETAIL_LIMIT, _parse_kijiji_detail, key="v2")211        return listings212