spb/immo-ka Public

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/kw_urbain.py : Keller Williams Urbain (kwurbain.ca)5#   Page /inscriptions rendue serveur : toutes les inscriptions de l'agence en6#   cartes HTML (plateforme marketingwebsites.ca, commune aux bureaux KW Canada).7# -----------------------------------------------------------------------------8from __future__ import annotations910import html as _html11import os12import re1314from .base import BaseConnector15from . import _detailutil as du16from ..normalize import parse_price17from ..schema import PropertyListing1819BASE = "https://www.kwurbain.ca"20LISTINGS = f"{BASE}/inscriptions/"21IMG_TMPL = "https://realestate.marketingwebsites.ca/property-images/{id}/{id}-{n:02d}.jpg"22DETAIL_LIMIT = int(os.environ.get("IMMOKA_KW_DETAIL_LIMIT", "400"))23_KW_IMG_RE = re.compile(24    r'https://realestate\.marketingwebsites\.ca/property-images/\d+/[^"\'\\ ]+\.(?:jpg|jpeg|png|webp)',25    re.I)26_META_DESC_RE = re.compile(r'<meta\s+name="description"\s+content="([^"]{40,})"', re.I)2728_CARD_RE = re.compile(r'property-card mix property-(\d+)"(.*?)(?=property-card mix property-\d+"|</div>\s*</div>\s*</div>\s*</section>|$)', re.S)29_H5_RE = re.compile(r'<h5>(.*?)</h5>', re.S)30_PRICE_RE = re.compile(r'<span>(.*?)</span>', re.S)31_SUB_RE = re.compile(r'card-subtitle[^>]*>(.*?)</h6>', re.S)32_BED_RE = re.compile(r'fa-bed"></i>\s*(\d+)')33_BATH_RE = re.compile(r'fa-bath"></i>\s*(\d+)')34_STATUS_RE = re.compile(r'card-status[^>]*>(.*?)</div>', re.S)353637def _txt(s: str) -> str:38    return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()394041class KwUrbainConnector(BaseConnector):42    source_id = "kw_urbain"43    request_delay = 0.64445    def fetch(self) -> list[PropertyListing]:46        html = self.get(LISTINGS).text47        listings: list[PropertyListing] = []48        for pid, block in _CARD_RE.findall(html):49            lst = self._to_listing(pid, block)50            if lst is not None:51                listings.append(lst)52        # fiche détail : galerie complète + description53        du.enrich(self, listings, DETAIL_LIMIT, parse_kw_detail, key="v2")54        return listings5556    def _to_listing(self, pid: str, block: str) -> PropertyListing | None:57        addr = _txt((_H5_RE.search(block) or [None, ""])[1] if _H5_RE.search(block) else "")58        m_addr = _H5_RE.search(block)59        addr = _txt(m_addr.group(1)) if m_addr else ""60        m_price = _PRICE_RE.search(block)61        price_label = _txt(m_price.group(1)) if m_price else ""62        m_sub = _SUB_RE.search(block)63        sub = _txt(m_sub.group(1)) if m_sub else ""     # "Blainville, Laurentides J7B1M1"64        city = region = ""65        if sub:66            parts = [p.strip() for p in sub.split(",")]67            city = parts[0]68            if len(parts) > 1:69                # « Laurentides J7B1M1 » -> retirer le code postal70                region = re.sub(r"\s*[A-Z]\d[A-Z]\s?\d[A-Z]\d\s*$", "", parts[1]).strip()71        beds = _BED_RE.search(block)72        baths = _BATH_RE.search(block)73        return PropertyListing(74            source=self.source_id,75            external_id=pid,76            url=f"{BASE}/inscriptions/inscription/{pid}",77            title=addr,78            address=addr,79            city=city,80            region=region,81            price=parse_price(price_label),82            price_label=price_label,83            bedrooms=int(beds.group(1)) if beds else None,84            bathrooms=int(baths.group(1)) if baths else None,85            images=[IMG_TMPL.format(id=pid, n=1)],86            broker_name="Keller Williams Urbain",87        )888990def parse_kw_detail(html: str) -> dict:91    """Galerie complète + description (balise meta) de la fiche KW."""92    out: dict = {}93    seen, imgs = set(), []94    for u in _KW_IMG_RE.findall(html):95        if u not in seen:96            seen.add(u)97            imgs.append(u)98    if imgs:99        out["images"] = imgs100    m = _META_DESC_RE.search(html)101    if m:102        d = _html.unescape(m.group(1)).strip()103        if d and not d.lower().startswith(("keller williams", "trouvez")):104            out["description"] = d105    _det = du.centris_details(du.flatten(html))106    if _det:107        out.setdefault("details", {}).update(_det)108    return out109