spb/immo-ka

1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/agences_web.py : agences INDÉPENDANTES à site web « classique »5#   (WordPress ou CMS immobilier server-rendered). Issu du recensement OACIQ6#   2026-08 : chaque bannière non couverte dont la page des inscriptions expose7#   des liens de fiches portant le n° Centris (7-9 chiffres) reçoit SON8#   connecteur, généré depuis data/web_agencies.json (un source_id par bannière).9#10#   Fonctionnement générique :11#   - liste : chaque list_url est lue (pagination via «{page}» au besoin) ; les12#     liens de fiches sont extraits par regex (href même domaine contenant un13#     n° 7-9 chiffres, ou <loc> si l'URL est un sitemap XML) ;14#   - détail : chaque fiche est enrichie (cache BD, _detailutil.enrich) par un15#     parseur générique — JSON-LD (Place/Offer/RealEstateListing), tableaux16#     Centris aplatis, prix/chambres/sdb par regex, galerie (og:image, CDN17#     Centris, fancybox/lightbox), GPS Google Maps.18#19#   Ces agences ne sont couvertes par AUCUNE bannière déjà agrégée → fiches20#   ADDITIVES, source_id simple (pas d'infixe _ag_). Le n° Centris devient21#   l'external_id : vendre_ag_ca (portail, _ag_) se fait masquer ses doublons.22# -----------------------------------------------------------------------------23from __future__ import annotations2425import html as _html26import json27import os28import re29import urllib.parse30from pathlib import Path3132from .base import BaseConnector33from . import _detailutil as du34from ..normalize import normalize_property_type35from ..schema import PropertyListing3637REGISTRY = Path(__file__).resolve().parent.parent.parent / "data" / "web_agencies.json"38DETAIL_LIMIT = int(os.environ.get("IMMOKA_WEB_DETAIL_LIMIT",39                                  os.environ.get("IMMOKA_DETAIL_LIMIT", "150")))4041_HREF_RE = re.compile(r'href="([^"#]+)"', re.I)42_LOC_RE = re.compile(r"<loc>([^<]+)</loc>")43_ID_RE = re.compile(r"(?<!\d)(\d{7,9})(?!\d)")44# jamais des fiches : assets, réseaux sociaux, flux, plugins45_JUNK = ("wp-content", "wp-json", "wp-includes", "facebook.", "instagram.",46         "linkedin.", "twitter.", "youtube.", "fonts.", "squarespace",47         "iubenda", "gravatar", "cdn-cgi", "mailto:", "tel:", "/feed",48         "javascript:", "google.", "centris.ca", "realtor.ca", ".css", ".js",49         ".jpg", ".jpeg", ".png", ".webp", ".svg", ".gif", ".pdf", ".ico",50         ".xml", "?share=", "replytocom")51_SOLD = ("vendu", "sold", "-loue", "/loue", "a-louer", "for-rent", "rented")5253_PRICE_RE = re.compile(r"(\d{1,3}(?:[ \u00a0\u202f\u2009,]\d{3})+(?:[.,]\d{2})?)\s*\$"54                       r"|\$\s?(\d{1,3}(?:,\d{3})+)")55_BEDS_RE = re.compile(r"(\d{1,2})\s*(?:chambre|cac\b|bedroom|beds?\b)", re.I)56_BATHS_RE = re.compile(r"(\d{1,2})\s*(?:salle[s]?\s*de\s*bain|sdb\b|bathroom|baths?\b)", re.I)57_OG_RE = re.compile(r'property="og:image"\s+content="([^"]+)"|'58                    r'content="([^"]+)"\s+property="og:image"', re.I)59_CENTRIS_IMG_RE = re.compile(r'https?://mspublic\.centris\.ca/media\.ashx\?[^"\'\\ ]+', re.I)60_GALLERY_RE = re.compile(r'(?:data-(?:fancybox|lightbox|src)|data-full)="'61                         r'(https?://[^"]+\.(?:jpe?g|png|webp)[^"]*)"', re.I)62_LD_IMG_TYPES = {"RealEstateListing", "Product", "Residence", "House",63                 "SingleFamilyResidence", "Apartment", "Accommodation", "Place"}64_META_DESC_RE = re.compile(r'<meta\s+(?:name="description"|property="og:description")\s+'65                           r'content="([^"]{40,})"', re.I)66_H1_RE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.I | re.S)67_TAG_RE = re.compile(r"<[^>]+>")68_REL_IMG_RE = re.compile(r'<img[^>]+src="(/[^"]+\.(?:jpe?g|png|webp)[^"]*)"', re.I)69_POSTAL_RE = re.compile(r"^[A-Z]\d[A-Z]\s?\d[A-Z]\d$")70_TITLE_RE = re.compile(r"<title>(.*?)</title>", re.I | re.S)71# « Candiac, QC », « Laval (Chomedey), Québec »72_CITY_QC_RE = re.compile(r"([A-ZÀ-Ü][\w'’.() \-]{2,40}),\s*(?:QC\b|Québec)")73# <title> « À vendre — Montréal (Mercier) | Agence »74_VENDRE_CITY_RE = re.compile(r"[àa] vendre\s*[—–:-]\s*([A-ZÀ-Ü][^|<>{}]{2,50}?)"75                             r"\s*(?:\||$)", re.I)76# texte aplati : « Description | {paragraphe} »77_DESC_FLAT_RE = re.compile(r"(?:Description(?: de la propriété)?|"78                           r"À propos de cette propriété)\s*\|\s*([^|]{60,})", re.I)79# jamais une ville : libellés de voie (l'adresse déborde parfois dans le titre)80_STREET_RE = re.compile(r"(?:rue|route|rte|ch\.?|chemin|boul\.?|boulevard|"81                        r"av\.?|avenue|rang|montée|croissant|impasse|place)\s",82                        re.I)83# seuls les types canoniques d'Immo-Ka sont retenus (normalize_property_type84# renvoie le texte capitalisé quand rien ne matche — on l'écarte)85_CANON_TYPES = {"Maison", "Maison mobile", "Jumelé", "Maison de ville", "Condo",86                "Duplex", "Triplex", "Multiplex", "Chalet", "Terrain",87                "Fermette/Agricole", "Commercial"}888990class _AgenceWeb(BaseConnector):91    """Connecteur générique d'agence à site server-rendered (voir registre)."""9293    agency_name = ""94    site_url = ""95    list_urls: list[str] = []96    require = ""            # sous-chaîne obligatoire dans l'URL de fiche97    render = False          # fiches JS (Next.js…) : détail via Scrapfly98    max_pages = 3099    request_delay = 0.5100101    def fetch(self) -> list[PropertyListing]:102        by_id: dict[str, PropertyListing] = {}103        for tpl in self.list_urls:104            if "{page}" in tpl:105                dry = 0106                for page in range(1, self.max_pages + 1):107                    before = len(by_id)108                    if not self._collect(tpl.format(page=page), by_id):109                        break110                    dry = dry + 1 if len(by_id) == before else 0111                    if dry >= 2:112                        break113            else:114                self._collect(tpl, by_id)115        listings = list(by_id.values())116117        def fetch_html(u: str) -> str:118            h = self.get_scrapfly(u, rendering_wait=3000) if self.render \119                else self.get(u).text120            # marqueur pour que le parseur puisse absolutiser les URLs relatives121            return h + f"\n<!--immoka-url {u}-->"122123        du.enrich(self, listings, DETAIL_LIMIT, parse_web_detail, key="v3",124                  fetch_html=fetch_html)125        # les loyers (« … $ par mois ») ne sont pas des maisons à vendre126        listings = [l for l in listings127                    if not (l.price is None and "par mois" in (l.price_label or ""))]128        for lst in listings:129            if not lst.title:130                lst.title = lst.address or _title_from_slug(lst.url) \131                    or "Propriété à vendre"132        return listings133134    def _collect(self, url: str, by_id: dict) -> bool:135        try:136            body = self.get(url).text137        except Exception:138            return False139        host = urllib.parse.urlparse(self.site_url).netloc.lower().removeprefix("www.")140        # certains CMS déclarent <base href> : les liens relatifs partent de là141        mb = re.search(r'<base\s+href="([^"]+)"', body[:6000], re.I)142        base = mb.group(1) if mb else url143        raw = _LOC_RE.findall(body) if "<loc>" in body[:4000] or url.endswith(".xml") \144            else _HREF_RE.findall(body)145        for href in raw:146            href = _html.unescape(href)147            low = href.lower()148            if any(j in low for j in _JUNK) or any(s in low for s in _SOLD):149                continue150            m = _ID_RE.search(href)151            if not m:152                continue153            absu = urllib.parse.urljoin(base, href)154            h = urllib.parse.urlparse(absu).netloc.lower().removeprefix("www.")155            if h != host:156                continue157            if self.require and self.require not in absu.lower():158                continue159            eid = m.group(1)160            if eid not in by_id:161                by_id[eid] = PropertyListing(162                    source=self.source_id, external_id=eid, url=absu,163                    mls=eid, agency=self.agency_name, broker_name=self.agency_name)164        return True165166167def _title_from_slug(url: str) -> str:168    """« …/triplex-a-vendre-longueuil-st-hubert-15180332-detail-Fr » → titre."""169    stop = {"details", "detail", "proprietes", "propriete", "properties",170            "property", "listings", "listing", "inscriptions", "fiche",171            "fr", "en", "index"}172    segs = [s for s in urllib.parse.urlparse(url).path.split("/") if s]173    words: list[str] = []174    for seg in reversed(segs):175        seg = urllib.parse.unquote_plus(seg)176        seg = re.sub(r"\.(?:html?|php|aspx?)$", "", seg, flags=re.I)177        seg = re.sub(r"\d{7,9}", " ", seg)178        seg = re.sub(r"detail-?fr", " ", seg, flags=re.I)179        words = [w for w in seg.replace("_", "-").replace(" ", "-").split("-") if w]180        if words and not all(w.lower() in stop for w in words):181            break182        words = []183    if not words:184        return ""185    txt = " ".join(words)186    txt = re.sub(r"\ba vendre\b", "à vendre", txt, flags=re.I)187    txt = re.sub(r"\ba louer\b", "à louer", txt, flags=re.I)188    return (txt[:1].upper() + txt[1:])[:120]189190191# --- parseur détail générique -------------------------------------------------192_RENT_TAIL_RE = re.compile(r"\s*(?:\+\s*tx\s*)?(?:par mois|/\s*mois|/\s*month|"193                           r"month|mensuel|par semaine|par nuit)", re.I)194# queues qui trahissent un filtre de recherche, une plage ou une évaluation/taxe195_JUNK_TAIL_RE = re.compile(r"\s*(?:et (?:moins|plus)|-\s*\d|à\s*\d|\(20\d\d\))", re.I)196# contextes boilerplate (calculatrice hypothécaire, plages de filtres)197_JUNK_CTX = ("de plus de", "supérieure à", "inférieure à", "mise de fond",198             "more than", "down payment", "valeur de")199_ASKED_RE = re.compile(r"prix demandé", re.I)200201202def _pick_price(flat: str, out: dict) -> None:203    """Choisit le prix de vente dans le texte aplati ; marque les locations204    (« … $ par mois ») via price_label sans prix — la fiche sera écartée."""205    candidates = []206    rent_label = ""207    for m in _PRICE_RE.finditer(flat):208        raw = (m.group(1) or m.group(2))209        try:210            val = float(re.sub(r"[^\d.]", "", raw.replace(",", ""))[:12])211        except ValueError:212            continue213        tail = flat[m.end():m.end() + 16]214        if _RENT_TAIL_RE.match(tail):215            # loyer — la fiche peut aussi être à vendre : on continue à chercher216            rent_label = rent_label or (m.group(0).strip() + " par mois")217            continue218        if _JUNK_TAIL_RE.match(tail):219            continue220        ctx = flat[max(0, m.start() - 70):m.start()]221        if ctx.rstrip().endswith("-") or any(w in ctx.lower() for w in _JUNK_CTX):222            continue223        if 25_000 <= val <= 100_000_000:224            labeled = bool(_ASKED_RE.search(ctx[-30:]))225            candidates.append((labeled, val, m.group(0).strip()))226            if labeled:227                break228    if candidates:229        # « Prix demandé » bat le premier montant venu (évaluations, comparables…)230        candidates.sort(key=lambda c: not c[0])231        _, val, label = candidates[0]232        out["price"], out["price_label"] = val, label233    elif rent_label:234        # uniquement un loyer : fiche en location (écartée par le connecteur)235        out["price_label"] = rent_label236237238def parse_web_detail(html: str) -> dict:239    """Fiche server-rendered générique : JSON-LD, tableaux Centris, regex."""240    out: dict = {}241    details: dict = {}242243    # JSON-LD (adresse, géo, prix, description, images)244    imgs: list[str] = []245    for n in du.ld_nodes(html):246        t = n.get("@type")247        types = set(t if isinstance(t, list) else [t])248        if not (types & _LD_IMG_TYPES) and "Offer" not in types:249            continue250        addr = n.get("address")251        if isinstance(addr, dict):252            if addr.get("streetAddress"):253                st = str(addr["streetAddress"]).strip().lstrip(", ").strip()254                if st:255                    out.setdefault("address", st)256            if addr.get("addressLocality"):257                loc = str(addr["addressLocality"]).strip()258                # certains sites mettent le code postal dans la localité259                if _POSTAL_RE.match(loc.upper()):260                    details.setdefault("postal_code", loc)261                elif loc:262                    out.setdefault("city", loc)263            if addr.get("postalCode"):264                details.setdefault("postal_code", str(addr["postalCode"]))265        geo = n.get("geo") or {}266        if isinstance(geo, dict):267            try:268                lat, lng = float(geo["latitude"]), float(geo["longitude"])269                if 44.5 <= lat <= 63.0 and -80.0 <= lng <= -56.0:270                    out.setdefault("lat", lat), out.setdefault("lng", lng)271            except (KeyError, TypeError, ValueError):272                pass273        offers = n.get("offers") or (n if "Offer" in types else {})274        if isinstance(offers, dict) and offers.get("price"):275            try:276                out.setdefault("price", float(str(offers["price"]).replace(",", "")))277            except ValueError:278                pass279        img = n.get("image")280        for u in (img if isinstance(img, list) else [img]):281            if isinstance(u, str) and u.startswith("http"):282                imgs.append(u)283            elif isinstance(u, dict) and u.get("url"):284                imgs.append(u["url"])285        y = n.get("yearBuilt")286        if y and str(y).isdigit():287            out.setdefault("year_built", int(y))288289    desc = du.ld_description(html)290    if not desc:291        md = _META_DESC_RE.search(html)292        if md:293            desc = _html.unescape(md.group(1)).strip()294    if desc:295        out["description"] = desc[:6000]296297    # adresse depuis le <h1> (« 72Z Ch. Valley, Brome ») si le JSON-LD est muet298    if "address" not in out:299        mh = _H1_RE.search(html)300        if mh:301            h1 = _html.unescape(_TAG_RE.sub(" ", mh.group(1)))302            h1 = re.sub(r"\s+", " ", h1).strip().strip(",-– ").strip()303            if 5 <= len(h1) <= 120 and any(c.isdigit() for c in h1) \304                    and "$" not in h1:305                parts = [p.strip() for p in h1.split(",") if p.strip()]306                out["address"] = parts[0]307                for p in parts[1:]:308                    if _POSTAL_RE.match(p.upper()):309                        details.setdefault("postal_code", p)310                    else:311                        out.setdefault("city", p)312                        break313314    # texte aplati : prix, chambres, sdb + tableaux Centris315    flat = du.flatten(html)316    if "price" not in out:317        _pick_price(flat, out)318    mb = _BEDS_RE.search(flat)319    if mb and int(mb.group(1)) <= 19:320        out["bedrooms"] = int(mb.group(1))321    ms = _BATHS_RE.search(flat)322    if ms and int(ms.group(1)) <= 19:323        out["bathrooms"] = int(ms.group(1))324    details.update(du.centris_details(flat))325326    # description encore vide : paragraphe « Description | … » du texte aplati,327    # sinon le plus long segment de prose (≥ 150 caractères, pas de code)328    if len(out.get("description") or "") < 50:329        md = _DESC_FLAT_RE.search(flat)330        if md:331            out["description"] = md.group(1).strip()[:6000]332        else:333            segs = [s.strip() for s in flat.split(" | ")334                    if len(s.strip()) >= 150 and "{" not in s335                    and s.count(";") < 5 and not any(336                        j in s for j in ("function", "var ", "=>",337                                         "fbq(", "gtag(", "img:", "http"))]338            if segs:339                out["description"] = max(segs, key=len)[:6000]340341    mt = _TITLE_RE.search(html)342    title_txt = _html.unescape(mt.group(1)).strip() if mt else ""343344    # ville : « Candiac, QC » dans le texte > « À vendre — Ville » du <title>345    # > dernier segment « adresse, Ville » du <title>346    if "city" not in out:347        cand = ""348        mc = _CITY_QC_RE.search(flat)349        if mc:350            cand = mc.group(1).strip()351        elif title_txt:352            mv = _VENDRE_CITY_RE.search(title_txt)353            if mv:354                cand = mv.group(1).strip()355            else:356                base = re.split(r"\s*[|–—-]\s*(?=[A-ZÀ-Ü][\w' ]+$)", title_txt)[0]357                if "," in base and "$" not in base:358                    cand = base.rsplit(",", 1)[1].strip()359        if (2 <= len(cand) <= 45 and not any(c.isdigit() for c in cand)360                and not _STREET_RE.match(cand)):361            out["city"] = cand362363    # <title> « 179 Prom. St-Louis, Notre-Dame-de-l'Île-Perrot, Montérégie,364    # J7W3J6 » : premier segment sans chiffres après l'adresse = la ville365    if "city" not in out and "," in title_txt:366        seg0 = title_txt.split("|")[0]367        for p in [p.strip() for p in seg0.split(",")][1:]:368            if (2 <= len(p) <= 45 and not any(c.isdigit() for c in p)369                    and p[:1].isupper() and not _STREET_RE.match(p)370                    and normalize_property_type(p) not in _CANON_TYPES):371                out["city"] = p372                break373374    # type de bien : libellé « Type de propriété | X », slug d'URL375    # (« maison-a-etages/… »), <title>, description376    mu0 = re.search(r"<!--immoka-url (\S+)-->", html[-3000:])377    path = urllib.parse.unquote_plus(378        urllib.parse.urlparse(mu0.group(1)).path) if mu0 else ""379    mtype = re.search(r"Type(?: de propriété| de bien)?\s*\|\s*([^|]{3,40})\s*\|",380                      flat, re.I)381    for cand in ((mtype.group(1) if mtype else ""),382                 path.replace("-", " ").replace("/", " | "), title_txt,383                 (out.get("description") or "")[:300], flat[:300]):384        if not cand:385            continue386        pt = normalize_property_type(cand)387        if pt in _CANON_TYPES:388            out["property_type"] = pt389            break390391    # galerie : fancybox/lightbox > CDN Centris > <img> relatifs > og:image392    gal = list(dict.fromkeys(_GALLERY_RE.findall(html)))393    if not gal:394        gal = list(dict.fromkeys(_CENTRIS_IMG_RE.findall(html)))395    if not gal:396        mu = re.search(r"<!--immoka-url (\S+)-->", html[-3000:])397        if mu:398            rel = [s for s in _REL_IMG_RE.findall(html)399                   if not any(j in s.lower() for j in ("logo", "icon", "favicon"))]400            gal = [urllib.parse.urljoin(mu.group(1), s)401                   for s in dict.fromkeys(rel)]402    if not gal:403        for a, b in _OG_RE.findall(html):404            u = _html.unescape(a or b)405            if u.startswith("http"):406                gal.append(u)407    if not gal and imgs:408        gal = list(dict.fromkeys(imgs))409    if gal:410        out["images"] = [_html.unescape(u) for u in gal[:80]]411412    if "lat" not in out:413        coords = du.gmaps_coords(html)414        if coords:415            out["lat"], out["lng"] = coords416417    if details:418        out["details"] = details419    return out420421422def _load() -> list[dict]:423    try:424        return json.loads(REGISTRY.read_text(encoding="utf-8"))425    except Exception:426        return []427428429# Génère une classe par agence du registre.430for _ag in _load():431    if not all(_ag.get(k) for k in ("id", "site", "list_urls")):432        continue433    _sid = _ag["id"]434    globals()[f"AGENCE_WEB_{_sid.upper()}"] = type(435        "AgenceWeb" + "".join(p.title() for p in _sid.split("_")),436        (_AgenceWeb,),437        {438            "source_id": _sid,439            "site_url": _ag["site"],440            "list_urls": list(_ag["list_urls"]),441            "agency_name": _ag.get("name", _sid.title()),442            "require": _ag.get("require", ""),443            "render": bool(_ag.get("render")),444            "max_pages": int(_ag.get("max_pages", 30)),445        },446    )447