spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/lespac.py : LesPAC (lespac.com) — petites annonces du Québec5#   UNIQUEMENT l'immobilier ACHAT-VENTE (pas la location, pas les entreprises) :6#     b37 résidentiel · b38 terrains · b39 commercial-industriel ·7#     b40 chalets · b41 fermes · b42 immeubles à revenus8#   Les pages « /quebec/… _b{cat}k{page}R2.jsa » (toute la province) embarquent9#   `var searchResponse = {…}` côté serveur : 20 annonces/page + totalPages.10# -----------------------------------------------------------------------------11from __future__ import annotations1213import html as _html14import json15import os16import re1718from ..schema import PropertyListing19from .base import BaseConnector2021from . import _detailutil as du2223BASE = "https://www.lespac.com"24DETAIL_LIMIT = int(os.environ.get("IMMOKA_LESPAC_DETAIL_LIMIT", "400"))25CATEGORIES = [26    (37, "immobilier-achat-vente-residentiel", "Maison"),27    (38, "immobilier-achat-vente-terrains", "Terrain"),28    (39, "immobilier-achat-vente-commercial-industriel", "Commercial"),29    (40, "immobilier-achat-vente-chalets", "Chalet"),30    (41, "immobilier-achat-vente-fermes", "Fermette"),31    (42, "immobilier-achat-vente-immeubles-a-revenus", "Immeuble à revenus"),32]33_RE_RESP = re.compile(r"var searchResponse = (\{.*?\});\s*[\r\n]", re.S)343536class LesPacConnector(BaseConnector):37    source_id = "lespac"38    request_delay = 0.93940    def _search_page(self, slug: str, cat: int, page: int) -> dict | None:41        url = f"{BASE}/quebec/{slug}_b{cat}k{page}R2.jsa"42        m = _RE_RESP.search(self.get(url).text)43        return json.loads(m.group(1)) if m else None4445    def _to_listing(self, r: dict, ptype: str) -> PropertyListing | None:46        lid = str(r.get("listingPublicId") or "")47        url = (r.get("listingDisplayUrl") or "").split("?")[0]48        if not lid or not url:49            return None50        # caractéristiques structurées (« Type de propriété », « Chambres »…)51        chars = {c.get("label", ""): str(c.get("value", ""))52                 for c in r.get("characteristics") or [] if c.get("label")}53        def as_int(label):54            m = re.search(r"\d+", chars.get(label, ""))55            return int(m.group(0)) if m else None56        # ville : cityLabel (accents corrects), sinon 1er segment du chemin57        seg = url.replace(BASE + "/", "").split("/")58        city = r.get("cityLabel") or (seg[0].replace("-", " ").title() if seg else "")59        images = [i["formattableImageUrl"].replace("%FORMAT%", "zoomedGallery")60                  for i in r.get("images") or [] if i.get("formattableImageUrl")]61        details = {}62        ts = r.get("publicReleaseTimestamp")   # date de mise en ligne (ms epoch)63        if ts:64            import datetime65            try:66                details["listed_at"] = datetime.datetime.fromtimestamp(67                    ts / 1000.0, datetime.timezone.utc).strftime("%Y-%m-%d")68            except (ValueError, OSError, OverflowError):69                pass70        return PropertyListing(71            source=self.source_id,72            external_id=lid,73            url=url,74            title=r.get("title") or "",75            city=city,76            property_type=chars.get("Type de propriété") or ptype,77            price=r.get("price"),78            price_label=r.get("priceLabel") or "",79            bedrooms=as_int("Chambres"),80            bathrooms=as_int("Salles de bain") or as_int("Salle de bain"),81            year_built=as_int("Année de construction"),82            description=(r.get("description") or "")[:2000],83            features=[f"{k} : {v}" for k, v in chars.items()],84            details=details,85            images=images,86            broker_name=r.get("advertiserName") or "LesPAC (particuliers)",87            agency="LesPAC Québec",88        )8990    def fetch(self) -> list[PropertyListing]:91        out: dict[str, PropertyListing] = {}92        for cat, slug, ptype in CATEGORIES:93            page, total_pages = 1, 194            while page <= total_pages:95                try:96                    d = self._search_page(slug, cat, page)97                except Exception:98                    break99                if not d:100                    break101                total_pages = min(int(d.get("totalPages") or 1), 400)102                for r in d.get("searchResults") or []:103                    lst = self._to_listing(r, ptype)104                    if lst is not None:105                        out.setdefault(lst.uid, lst)106                page += 1107        listings = list(out.values())108        # v2 = TOUTES les boîtes de caractéristiques (l'ancien parseur ne lisait109        # que la 1re) + salles de bain/d'eau + superficies (dimensions du terrain)110        du.enrich(self, listings, DETAIL_LIMIT, _parse_lespac_detail, key="v2")111        return listings112113114def _clean(s: str) -> str:115    return re.sub(r"\s+", " ", _html.unescape(re.sub(r"<[^>]+>", " ", s))).strip()116117118def _area_pi2(value: str) -> float | None:119    """« 1 200 pi2 » / « 111 m2 » -> pi² (via la normalisation commune)."""120    from ..normalize import parse_area_sqft121    return parse_area_sqft(value)122123124_DIMS_RE = re.compile(r"([\d\s]+(?:,\d+)?)\s*x\s*([\d\s]+(?:,\d+)?)\s*(pieds|m[èe]tres)",125                      re.I)126127128def _dims_pi2(value: str) -> float | None:129    """« 15,24 x 30,48 mètres » / « 50 x 100 pieds » -> superficie en pi²."""130    m = _DIMS_RE.search(value or "")131    if not m:132        return None133    try:134        a = float(m.group(1).replace(" ", "").replace(",", "."))135        b = float(m.group(2).replace(" ", "").replace(",", "."))136    except ValueError:137        return None138    if a <= 0 or b <= 0 or a > 100000 or b > 100000:139        return None140    s = a * b141    if m.group(3).lower().startswith("m"):142        s *= 10.7639143    return round(s) if s >= 300 else None    # < 300 pi² : dimensions suspectes144145146def _parse_lespac_detail(html: str) -> dict:147    """Fiche LesPAC : description complète, adresse civique, caractéristiques148    (boîte « Caractéristiques » : <p><span>Label</span><span>Valeur</span></p>)149    et galerie pleine taille (binary/basephoto)."""150    out: dict = {}151    md = re.search(r'class="description"[^>]*>(.*?)</(?:p|div)>', html, re.S | re.I)152    if md:153        desc = _clean(md.group(1))154        if desc:155            out["description"] = desc[:6000]156    features, details = [], {}157    # section « Caractéristiques » -> fin des boîtes : PLUSIEURS <div class="box">158    # (types, équipements, description des pièces…) — on lit toutes les rangées159    # <p><span>Label</span> <span>Valeur</span> jusqu'à la bannière suivante.160    i = html.find(">Caractéristiques</p>")161    if i >= 0:162        j = html.find("pub-middle-listing-detail", i)163        seg = html[i:j if j > i else i + 30000]164        for lm, vm in re.findall(r"<p><span>(.*?)</span>\s*<span>(.*?)</span>",165                                 seg, re.S):166            label, value = _clean(lm), _clean(vm)167            if not label or not value or len(label) > 45 or len(value) > 120:168                continue169            if label not in details:170                features.append(f"{label} : {value}")171                details[label] = value172            low = label.lower()173            if label == "Adresse" and re.match(r"\s*\d", value):174                out["address"] = value175            elif label == "Année":176                my = re.search(r"(18|19|20)\d{2}", value)177                if my:178                    out["year_built"] = int(my.group(0))179            elif label == "Type de propriété":180                out["property_type"] = value181            elif "chambre" in low:182                mn = re.search(r"\d+", value)183                if mn:184                    out["bedrooms"] = int(mn.group(0))185            elif "salle" in low and "eau" in low and "bain" not in low:186                mn = re.search(r"\d+", value)187                if mn and int(mn.group(0)) > 0:188                    out["powder_rooms"] = int(mn.group(0))189            elif "salle" in low and "bain" in low:190                mn = re.search(r"\d+", value)191                if mn:192                    out["bathrooms"] = int(mn.group(0))193            elif "superficie" in low and "terrain" in low:194                v = _area_pi2(value)195                if v:196                    out["lot_sqft"] = v197            elif "superficie" in low:198                v = _area_pi2(value)199                if v:200                    out["area_sqft"] = v201            elif label == "Dimension du terrain":202                v = _dims_pi2(value)203                if v:204                    out["lot_sqft"] = v205    if features:206        out["features"] = features207    if details:208        out["details"] = details209    imgs, seen = [], set()210    for u in re.findall(r'https://cdn\.lespac\.com/binary/basephoto/\d+\.jpg', html):211        if u not in seen:212            seen.add(u)213            imgs.append(u)214    if imgs:215        out["images"] = imgs216    return out217