spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/bhhs_quebec.py : Berkshire Hathaway HomeServices Québec5#   (bhhsquebec.ca) — Grand Montréal, Laurentides, Montérégie, Outaouais.6#   Site WordPress (plugin « MW Properties » / realestate.marketingwebsites.ca),7#   liste rendue SERVEUR sur /proprietes/, paginée par ?pages=N (~9/page,8#   ~11 pages). Chaque carte porte : n° Centris (URL + image + property-meta),9#   région, adresse, ville, prix ($US), chambres/sdb et photo.10#11#   Page détail : description + addenda (<div class="description block">,12#   collapse #showAddendum), bloc « features » (Année de construction, pièces,13#   chambres, sdb… en h5/p), tableau « Caractéristiques » (#charateristics —14#   sic — th/td), GPS inline (google.maps.LatLng(lat,lng)), visite virtuelle15#   (iframe #virtualCollapse), courtier (.agents : nom + tel:) et galerie16#   complète property-images/{Centris}/{Centris}-NN.jpg (pleine résolution,17#   ordre d'origine ; les miniatures /thumbs-320/ sont ignorées).18#   Enrichissement plafonné + cache (du.enrich).19#20#   source_id « bhhs_ag_qc » : infixe _ag_ = dédup Centris (db.refresh_dedup).21# -----------------------------------------------------------------------------22from __future__ import annotations2324import html as _html25import os26import re27import urllib.parse2829from .base import BaseConnector30from . import _detailutil as du31from ..normalize import parse_price32from ..schema import PropertyListing3334SITE = "https://bhhsquebec.ca"35LISTING = SITE + "/proprietes/"36DETAIL_LIMIT = int(os.environ.get("IMMOKA_BHHS_DETAIL_LIMIT", "150"))37AGENCY = "Berkshire Hathaway HomeServices Québec"3839_CARD_SPLIT = re.compile(r'<a href="(https://bhhsquebec\.ca/properties/[^"]*?/(\d{6,9}))"\s+class="list"', re.I)40_IMG_RE = re.compile(r'(https://realestate\.marketingwebsites\.ca/property-images/\d+/[^"\']+\.(?:jpg|jpeg|png|webp))', re.I)41_REGION_RE = re.compile(r'<h3>.*?</i>\s*([^<]+)</h3>', re.S | re.I)42_ADDR_RE = re.compile(r'<div class="top-ctn">.*?<p>([^<]+)</p>', re.S | re.I)43_PRICE_RE = re.compile(r'class="price">\s*(\$[\d,]+|[\d\s ]+\$)', re.I)44_AMEN_RE = re.compile(r'(?:bed|hotel)[^>]*></i>\s*(\d+).*?(?:bath|shower)[^>]*></i>\s*(\d+)', re.S | re.I)4546# --- page détail ---47_GALLERY_RE = re.compile(48    r'https://realestate\.marketingwebsites\.ca/property-images/\d+/\d+-\d+'49    r'\.(?:jpg|jpeg|png|webp)', re.I)50_P_RE = re.compile(r'<p[^>]*>(.*?)</p>', re.S | re.I)51_CHAR_ROW_RE = re.compile(r'<th[^>]*>(.*?)</th>\s*<td[^>]*>(.*?)</td>', re.S | re.I)52_FEAT_RE = re.compile(r'<h5 class="title">([^<]+)</h5>\s*<p>([^<]*)</p>', re.I)53_LATLNG_RE = re.compile(r'google\.maps\.LatLng\(\s*(-?\d{1,2}\.\d+)\s*,\s*(-?\d{2,3}\.\d+)\s*\)')54_AGENT_RE = re.compile(r'class="agents">.*?<h3 class="name">([^<]+)</h3>(.*?)</ul>', re.S | re.I)55_TEL_RE = re.compile(r'href="tel:(\+?\d{7,11})"')56_IFRAME_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.I)5758# libellés du bloc « features » promus en colonnes ; le reste va dans details59_FEAT_FIELDS = {60    "Année de construction": "year_built",61    "Chambre(s) à coucher": "bedrooms",62    "Salle(s) de bain": "bathrooms",63    "Salle(s) d'eau": "powder_rooms",64}656667class BhhsQuebecConnector(BaseConnector):68    source_id = "bhhs_ag_qc"69    request_delay = 0.470    max_pages = 407172    def fetch(self) -> list[PropertyListing]:73        by_id: dict[str, PropertyListing] = {}74        dry = 075        for page in range(1, self.max_pages + 1):76            url = LISTING if page == 1 else f"{LISTING}?pages={page}"77            try:78                html = self.get(url).text79            except Exception:80                break81            before = len(by_id)82            parts = _CARD_SPLIT.split(html)83            # parts = [pre, url1, mls1, blk1, url2, mls2, blk2, …]84            for i in range(1, len(parts) - 2, 3):85                url_p, mls, blk = parts[i], parts[i + 1], parts[i + 2][:1800]86                lst = self._card(url_p, mls, blk)87                if lst:88                    by_id.setdefault(lst.external_id, lst)89            dry = dry + 1 if len(by_id) == before else 090            if dry >= 2:91                break92        listings = list(by_id.values())93        du.enrich(self, listings, DETAIL_LIMIT, _parse_detail, key="v1")94        return listings9596    def _card(self, url_p: str, mls: str, blk: str) -> PropertyListing | None:97        addr, city = "", ""98        am = _ADDR_RE.search(blk)99        if am:100            raw = _html.unescape(am.group(1)).strip()101            if "," in raw:102                addr, city = [x.strip() for x in raw.rsplit(",", 1)]103            else:104                addr = raw105        if not addr:106            um = re.search(r'/properties/([^/]+)/\d{6,9}', url_p)107            if um:108                addr = _html.unescape(urllib.parse.unquote_plus(um.group(1))).strip()109110        rm = _REGION_RE.search(blk)111        region = _html.unescape(rm.group(1)).strip() if rm else ""112        pm = _PRICE_RE.search(blk)113        price_label = pm.group(1).strip() if pm else ""114        price = parse_price(price_label.replace("$", "").replace(",", "")) \115            if price_label.startswith("$") else parse_price(price_label)116        amn = _AMEN_RE.search(blk)117        im = _IMG_RE.search(blk)118119        return PropertyListing(120            source=self.source_id,121            external_id=mls,122            url=url_p,123            title=addr or "Propriété à vendre",124            address=addr,125            city=city,126            region=region,127            price=price,128            price_label=price_label,129            bedrooms=int(amn.group(1)) if amn else None,130            bathrooms=int(amn.group(2)) if amn else None,131            mls=mls,132            images=[im.group(1)] if im else [],133            agency=AGENCY,134            broker_name=AGENCY,135        )136137138def _txt(fragment: str) -> str:139    t = _html.unescape(re.sub(r"<br\s*/?>", "\n", fragment, flags=re.I))140    t = re.sub(r"<[^>]+>", " ", t).replace("\xa0", " ")141    t = re.sub(r"[ \t]+", " ", t)142    return re.sub(r"\n{3,}", "\n\n", t).strip()143144145def _parse_detail(html: str) -> dict:146    """Fiche détail BHHS (MW Properties) : description+addenda, features,147    caractéristiques, GPS, visite virtuelle, courtier, galerie complète."""148    out: dict = {}149150    # description : <p> du bloc description + addenda (collapse #showAddendum)151    i = html.find('class="description block')152    if i >= 0:153        j = html.find('id="addendum-btn"', i)154        if j < 0:155            j = html.find('class="char block"', i)156        seg = html[i:j if j > 0 else i + 30000]157        paras = [_txt(p) for p in _P_RE.findall(seg)]158        desc = "\n\n".join(p for p in paras if p)159        if desc:160            out["description"] = desc161162    details: dict = {}163    # bloc « features » (année, pièces, chambres, sdb…)164    for lab, val in _FEAT_RE.findall(html):165        lab, val = _txt(lab), _txt(val)166        if not val:167            continue168        field = _FEAT_FIELDS.get(lab)169        if field:170            m = re.search(r"\d+", val)171            if m:172                out[field] = int(m.group())173        elif len(val) <= 120:174            details[lab] = val175176    # tableau « Caractéristiques » (th/td)177    i = html.find('id="charateristics"')178    if i >= 0:179        seg = html[i:html.find("</table>", i)]180        for th, td in _CHAR_ROW_RE.findall(seg):181            lab, val = _txt(th), _txt(td)182            if lab and val and len(lab) <= 60 and len(val) <= 250:183                details[lab] = val184185    # GPS inline186    m = _LATLNG_RE.search(html)187    if m:188        lat, lng = float(m.group(1)), float(m.group(2))189        if 44.5 <= lat <= 63.0 and -80.0 <= lng <= -56.0:190            out["lat"], out["lng"] = lat, lng191192    # visite virtuelle (iframe du collapse dédié)193    i = html.find('id="virtualCollapse"')194    if i >= 0:195        m = _IFRAME_RE.search(html[i:i + 3000])196        if m and m.group(1).startswith("http"):197            details["Visite virtuelle"] = m.group(1)198199    # courtier inscripteur (carte .agents)200    m = _AGENT_RE.search(html)201    if m:202        out["broker_name"] = _txt(m.group(1))203        t = _TEL_RE.search(m.group(2))204        if t:205            tel = t.group(1).lstrip("+")206            if len(tel) == 10:207                tel = f"({tel[:3]}) {tel[3:6]}-{tel[6:]}"208            out["broker_phone"] = tel209210    if details:211        out["details"] = details212213    # galerie pleine résolution (sans /thumbs-320/), ordre d'origine214    seen, uniq = set(), []215    for u in _GALLERY_RE.findall(html):216        if u not in seen:217            seen.add(u)218            uniq.append(u)219    if not uniq:220        # toutes les fiches BHHS ont une galerie (flux Centris) ; une page sans221        # galerie = rendu transitoirement dégradé (l'API images du plugin MW a222        # flanché) → échec de parse : on ne met PAS en cache un payload sans223        # photos, la fiche sera réessayée au prochain cycle (du.enrich garde224        # l'éventuel payload périmé en attendant)225        return {}226    out["images"] = uniq227    return out228