spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/guidehabitation.py : Guide Habitation (guidehabitation.ca)5#   Répertoire de référence des PROJETS NEUFS au Québec (condos et maisons6#   neuves des promoteurs). Pages région /fr/projets-immobiliers/{region}/7#   rendues serveur avec microdonnées schema.org complètes par carte projet :8#   adresse (streetAddress), ville, GPS, prix « À partir de » (itemprop=price),9#   type (SingleFamilyResidence → maison, ApartmentComplex → condo),10#   description, chambres offertes, photo. Aucune clé, aucun rendu JS.11#   On écarte les projets LOCATIFS (prix mensuel / « locatif » dans le texte) :12#   Immo-Ka couvre le neuf À VENDRE. external_id = data-id du projet.13#14#   Page détail projet : UN gros JSON-LD RealEstateListing très riche —15#   about.articleBody (présentation longue, pseudo-markdown), mainEntity.image16#   (galerie complète pleine résolution, ordre d'origine), mainEntity.geo,17#   amenityFeature, offers[] (promoteur offeredBy : nom/téléphone/site,18#   chambres, garantie, date de livraison), additionalProperty (statut du19#   projet, site web externe). Enrichissement plafonné + cache (du.enrich).20# -----------------------------------------------------------------------------21from __future__ import annotations2223import html as _html24import os25import re2627from .base import BaseConnector28from . import _detailutil as du29from ..schema import PropertyListing3031SITE = "https://www.guidehabitation.ca"32DETAIL_LIMIT = int(os.environ.get("IMMOKA_GH_DETAIL_LIMIT", "150"))3334# Régions du répertoire (repli si le sitemap est indisponible).35REGIONS = ["montreal", "monteregie", "laval", "laurentides", "lanaudiere",36           "quebec", "estrie", "outaouais", "mauricie"]3738_SITEMAP_REGION_RE = re.compile(r'/fr/projets-immobiliers/([a-z-]+)/')39_CARD_SPLIT_RE = re.compile(r'(?=<article class="project-card )')40_ID_RE = re.compile(r'data-id="(\d+)"')41_URL_RE = re.compile(r'href="(https://www\.guidehabitation\.ca/fr/\d+/[^"]+)"')42_IMG_RE = re.compile(r'<img[^>]+src="([^"]+)"')43_META_RE = re.compile(r'itemprop="(name|streetAddress|addressLocality|latitude|'44                      r'longitude|price)"[^>]*content="([^"]*)"')45_LOCALITY_RE = re.compile(r'itemprop="addressLocality">([^<]*)<')46_NAME_H3_RE = re.compile(r'<h3 itemprop="name">([^<]*)</h3>')47_DESC_RE = re.compile(r'itemprop="description">([^<]*)<')48_TYPE_RE = re.compile(r'itemtype="https://schema\.org/(SingleFamilyResidence|'49                      r'ApartmentComplex|Residence)"')50_BEDROOMS_RE = re.compile(r'class="card-bedrooms">([^<]*)<')51_PRICE_TXT_RE = re.compile(r'À partir de\s*([\d\s ,.]+)\s*\$')52_RENTAL_RE = re.compile(r'locati|à louer|for rent', re.I)53_SQFT_RE = re.compile(r'[Ss]uperficie habitable[\s ]*:?[\s ]*'54                      r'([\d][\d\s  ,]*)\s*pi')5556# En deçà de ce prix « à partir de », c'est un loyer mensuel (projet locatif).57MIN_SALE_PRICE = 50_000585960class GuideHabitationConnector(BaseConnector):61    source_id = "guidehabitation"62    request_delay = 1.06364    def _regions(self) -> list[str]:65        try:66            xml = self.get(f"{SITE}/sitemap.xml").text67            found = sorted({m.group(1) for m in _SITEMAP_REGION_RE.finditer(xml)})68            if found:69                return found70        except Exception:71            pass72        return REGIONS7374    def fetch(self) -> list[PropertyListing]:75        by_id: dict[str, PropertyListing] = {}76        for region in self._regions():77            try:78                html = self.get(f"{SITE}/fr/projets-immobiliers/{region}/").text79            except Exception:80                continue81            for card in _CARD_SPLIT_RE.split(html):82                if 'data-id="' not in card:83                    continue84                lst = self._to_listing(card, region)85                if lst and lst.external_id not in by_id:86                    by_id[lst.external_id] = lst87        listings = list(by_id.values())88        du.enrich(self, listings, DETAIL_LIMIT, _parse_gh_detail, key="v1")89        return listings9091    def _to_listing(self, card: str, region: str) -> PropertyListing | None:92        mid = _ID_RE.search(card)93        murl = _URL_RE.search(card)94        if not mid or not murl:95            return None96        meta = {k: _html.unescape(v).strip() for k, v in _META_RE.findall(card)}97        name = meta.get("name") or ""98        mh3 = _NAME_H3_RE.search(card)99        if mh3:100            name = _html.unescape(mh3.group(1)).strip() or name101        desc = ""102        mdesc = _DESC_RE.search(card)103        if mdesc:104            desc = _html.unescape(mdesc.group(1)).strip()105        # prix « À partir de » — absent ou mensuel (locatif) → hors périmètre106        price = None107        try:108            price = float(meta.get("price"))109        except (TypeError, ValueError):110            mtxt = _PRICE_TXT_RE.search(card)111            if mtxt:112                try:113                    price = float(re.sub(r"[\s ,]", "", mtxt.group(1)))114                except ValueError:115                    price = None116        if price is None or price < MIN_SALE_PRICE:117            return None118        if _RENTAL_RE.search(name) or _RENTAL_RE.search(desc):119            return None120        mtype = _TYPE_RE.search(card)121        ptype = {"SingleFamilyResidence": "Maison",122                 "ApartmentComplex": "Condo"}.get(123                     mtype.group(1) if mtype else "", "")124        city = meta.get("addressLocality") or ""125        if not city:126            mloc = _LOCALITY_RE.search(card)127            if mloc:128                city = _html.unescape(mloc.group(1)).strip()129        address = meta.get("streetAddress") or ""130        # « Chemin du Golf, Sainte-Julie, QC, Canada » → rue seulement131        address = re.sub(r",\s*(QC|Québec|Quebec)\b.*$", "", address).strip()132        if city and address.lower().endswith(", " + city.lower()):133            address = address[: -(len(city) + 2)].strip().rstrip(",")134        lst = PropertyListing(135            source=self.source_id,136            external_id=mid.group(1),137            url=murl.group(1),138            title=name,139            address=address or name,140            city=city,141            property_type=ptype,142            price=price,143            price_label=f"À partir de {price:,.0f} $".replace(",", " "),144            description=desc,145            broker_name="Guide Habitation — projet neuf",146            agency="Guide Habitation (projets neufs)",147        )148        mimg = _IMG_RE.search(card)149        if mimg:150            lst.images = [mimg.group(1)]151        try:152            lst.lat = float(meta.get("latitude"))153            lst.lng = float(meta.get("longitude"))154        except (TypeError, ValueError):155            pass156        mbeds = _BEDROOMS_RE.search(card)157        if mbeds:158            beds = re.findall(r"(\d+)\s*ch", mbeds.group(1))159            if beds:160                lst.bedrooms = int(beds[0])           # minimum offert161                lst.details["Chambres offertes"] = _html.unescape(162                    mbeds.group(1)).strip()163        lst.details["Projet neuf"] = True164        return lst165166167def _clean_md(text: str) -> str:168    """articleBody pseudo-markdown -> texte propre (garde les paragraphes)."""169    t = _html.unescape(text).replace("\xa0", " ")170    t = re.sub(r"\*\*+", "", t)                        # **gras**171    t = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", t)     # [lien](url)172    t = re.sub(r"^[\s]*[-•#]+\s*", "", t, flags=re.M)  # puces/titres173    t = re.sub(r"[ \t]+", " ", t)174    return re.sub(r"\n{3,}", "\n\n", t).strip()175176177def _parse_gh_detail(html: str) -> dict:178    """Fiche projet Guide Habitation : tout est dans le JSON-LD RealEstateListing."""179    out: dict = {}180    node = None181    for n in du.ld_nodes(html):182        t = n.get("@type")183        if "RealEstateListing" in (t if isinstance(t, list) else [t]):184            node = n185            break186    if not node:187        return out188189    about = node.get("about") or {}190    body = str(about.get("articleBody") or "")191    if body:192        desc = _clean_md(body)193        if desc:194            out["description"] = desc195196    main = node.get("mainEntity") or {}197    imgs = main.get("image") or []198    if isinstance(imgs, str):199        imgs = [imgs]200    imgs = [u for u in imgs if isinstance(u, str) and u.startswith("http")]201    if imgs:202        out["images"] = imgs203    geo = main.get("geo") or {}204    try:205        lat, lng = float(geo.get("latitude")), float(geo.get("longitude"))206        if 44.5 <= lat <= 63.0 and -80.0 <= lng <= -56.0:207            out["lat"], out["lng"] = lat, lng208    except (TypeError, ValueError):209        pass210211    details: dict = {}212    features: list[str] = []213    for af in main.get("amenityFeature") or []:214        v = str(af.get("value") or "").strip()215        if v and v.lower() not in ("true", "false"):216            features.append(v)217        elif v.lower() == "true" and af.get("name"):218            features.append(str(af["name"]))219    if features:220        out["features"] = features221222    for ap in node.get("additionalProperty") or []:223        pid, val = ap.get("propertyID") or "", str(ap.get("value") or "").strip()224        if not val:225            continue226        if pid == "gh:status":227            details["Statut du projet"] = val228        elif pid == "gh:deliveryDate":229            details["Livraison"] = val.split(" ")[0]230        elif ap.get("name") == "externalUrl":231            details["Site du projet"] = val232233    # offres (par modèle) : promoteur, chambres min, garantie234    offers = node.get("offers") or []235    if isinstance(offers, dict):236        offers = [offers]237    beds = []238    for of in offers:239        item = of.get("itemOffered") or {}240        try:241            beds.append(int(item.get("numberOfBedrooms")))242        except (TypeError, ValueError):243            pass244        if of.get("warranty") and "Garantie" not in details:245            details["Garantie"] = str(of["warranty"])246        by = of.get("offeredBy") or {}247        if by.get("name") and "broker_name" not in out:248            out["broker_name"] = str(by["name"])249            if by.get("telephone"):250                out["broker_phone"] = str(by["telephone"])251    if beds:252        out["bedrooms"] = min(beds)253254    # superficies habitables des modèles (texte de présentation) : minimum255    # offert (cohérent avec le prix « à partir de »)256    areas = []257    for m in _SQFT_RE.finditer(body):258        try:259            areas.append(float(re.sub(r"[^\d]", "", m.group(1))))260        except ValueError:261            pass262    areas = [a for a in areas if 300 <= a <= 20000]263    if areas:264        out["area_sqft"] = min(areas)265        if len(areas) > 1:266            details["Superficies offertes"] = (267                f"{min(areas):,.0f} à {max(areas):,.0f} pi²".replace(",", " "))268269    if details:270        out["details"] = details271    return out272