# -----------------------------------------------------------------------------
# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/agences_web.py : agences INDÉPENDANTES à site web « classique »
#   (WordPress ou CMS immobilier server-rendered). Issu du recensement OACIQ
#   2026-08 : chaque bannière non couverte dont la page des inscriptions expose
#   des liens de fiches portant le n° Centris (7-9 chiffres) reçoit SON
#   connecteur, généré depuis data/web_agencies.json (un source_id par bannière).
#
#   Fonctionnement générique :
#   - liste : chaque list_url est lue (pagination via «{page}» au besoin) ; les
#     liens de fiches sont extraits par regex (href même domaine contenant un
#     n° 7-9 chiffres, ou <loc> si l'URL est un sitemap XML) ;
#   - détail : chaque fiche est enrichie (cache BD, _detailutil.enrich) par un
#     parseur générique — JSON-LD (Place/Offer/RealEstateListing), tableaux
#     Centris aplatis, prix/chambres/sdb par regex, galerie (og:image, CDN
#     Centris, fancybox/lightbox), GPS Google Maps.
#
#   Ces agences ne sont couvertes par AUCUNE bannière déjà agrégée → fiches
#   ADDITIVES, source_id simple (pas d'infixe _ag_). Le n° Centris devient
#   l'external_id : vendre_ag_ca (portail, _ag_) se fait masquer ses doublons.
# -----------------------------------------------------------------------------
from __future__ import annotations

import html as _html
import json
import os
import re
import urllib.parse
from pathlib import Path

from .base import BaseConnector
from . import _detailutil as du
from ..normalize import normalize_property_type
from ..schema import PropertyListing

REGISTRY = Path(__file__).resolve().parent.parent.parent / "data" / "web_agencies.json"
DETAIL_LIMIT = int(os.environ.get("IMMOKA_WEB_DETAIL_LIMIT",
                                  os.environ.get("IMMOKA_DETAIL_LIMIT", "150")))

_HREF_RE = re.compile(r'href="([^"#]+)"', re.I)
_LOC_RE = re.compile(r"<loc>([^<]+)</loc>")
_ID_RE = re.compile(r"(?<!\d)(\d{7,9})(?!\d)")
# jamais des fiches : assets, réseaux sociaux, flux, plugins
_JUNK = ("wp-content", "wp-json", "wp-includes", "facebook.", "instagram.",
         "linkedin.", "twitter.", "youtube.", "fonts.", "squarespace",
         "iubenda", "gravatar", "cdn-cgi", "mailto:", "tel:", "/feed",
         "javascript:", "google.", "centris.ca", "realtor.ca", ".css", ".js",
         ".jpg", ".jpeg", ".png", ".webp", ".svg", ".gif", ".pdf", ".ico",
         ".xml", "?share=", "replytocom")
_SOLD = ("vendu", "sold", "-loue", "/loue", "a-louer", "for-rent", "rented")

_PRICE_RE = re.compile(r"(\d{1,3}(?:[ \u00a0\u202f\u2009,]\d{3})+(?:[.,]\d{2})?)\s*\$"
                       r"|\$\s?(\d{1,3}(?:,\d{3})+)")
_BEDS_RE = re.compile(r"(\d{1,2})\s*(?:chambre|cac\b|bedroom|beds?\b)", re.I)
_BATHS_RE = re.compile(r"(\d{1,2})\s*(?:salle[s]?\s*de\s*bain|sdb\b|bathroom|baths?\b)", re.I)
_OG_RE = re.compile(r'property="og:image"\s+content="([^"]+)"|'
                    r'content="([^"]+)"\s+property="og:image"', re.I)
_CENTRIS_IMG_RE = re.compile(r'https?://mspublic\.centris\.ca/media\.ashx\?[^"\'\\ ]+', re.I)
_GALLERY_RE = re.compile(r'(?:data-(?:fancybox|lightbox|src)|data-full)="'
                         r'(https?://[^"]+\.(?:jpe?g|png|webp)[^"]*)"', re.I)
_LD_IMG_TYPES = {"RealEstateListing", "Product", "Residence", "House",
                 "SingleFamilyResidence", "Apartment", "Accommodation", "Place"}
_META_DESC_RE = re.compile(r'<meta\s+(?:name="description"|property="og:description")\s+'
                           r'content="([^"]{40,})"', re.I)
_H1_RE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.I | re.S)
_TAG_RE = re.compile(r"<[^>]+>")
_REL_IMG_RE = re.compile(r'<img[^>]+src="(/[^"]+\.(?:jpe?g|png|webp)[^"]*)"', re.I)
_POSTAL_RE = re.compile(r"^[A-Z]\d[A-Z]\s?\d[A-Z]\d$")
_TITLE_RE = re.compile(r"<title>(.*?)</title>", re.I | re.S)
# « Candiac, QC », « Laval (Chomedey), Québec »
_CITY_QC_RE = re.compile(r"([A-ZÀ-Ü][\w'’.() \-]{2,40}),\s*(?:QC\b|Québec)")
# <title> « À vendre — Montréal (Mercier) | Agence »
_VENDRE_CITY_RE = re.compile(r"[àa] vendre\s*[—–:-]\s*([A-ZÀ-Ü][^|<>{}]{2,50}?)"
                             r"\s*(?:\||$)", re.I)
# texte aplati : « Description | {paragraphe} »
_DESC_FLAT_RE = re.compile(r"(?:Description(?: de la propriété)?|"
                           r"À propos de cette propriété)\s*\|\s*([^|]{60,})", re.I)
# jamais une ville : libellés de voie (l'adresse déborde parfois dans le titre)
_STREET_RE = re.compile(r"(?:rue|route|rte|ch\.?|chemin|boul\.?|boulevard|"
                        r"av\.?|avenue|rang|montée|croissant|impasse|place)\s",
                        re.I)
# seuls les types canoniques d'Immo-Ka sont retenus (normalize_property_type
# renvoie le texte capitalisé quand rien ne matche — on l'écarte)
_CANON_TYPES = {"Maison", "Maison mobile", "Jumelé", "Maison de ville", "Condo",
                "Duplex", "Triplex", "Multiplex", "Chalet", "Terrain",
                "Fermette/Agricole", "Commercial"}


class _AgenceWeb(BaseConnector):
    """Connecteur générique d'agence à site server-rendered (voir registre)."""

    agency_name = ""
    site_url = ""
    list_urls: list[str] = []
    require = ""            # sous-chaîne obligatoire dans l'URL de fiche
    render = False          # fiches JS (Next.js…) : détail via Scrapfly
    max_pages = 30
    request_delay = 0.5

    def fetch(self) -> list[PropertyListing]:
        by_id: dict[str, PropertyListing] = {}
        for tpl in self.list_urls:
            if "{page}" in tpl:
                dry = 0
                for page in range(1, self.max_pages + 1):
                    before = len(by_id)
                    if not self._collect(tpl.format(page=page), by_id):
                        break
                    dry = dry + 1 if len(by_id) == before else 0
                    if dry >= 2:
                        break
            else:
                self._collect(tpl, by_id)
        listings = list(by_id.values())

        def fetch_html(u: str) -> str:
            h = self.get_scrapfly(u, rendering_wait=3000) if self.render \
                else self.get(u).text
            # marqueur pour que le parseur puisse absolutiser les URLs relatives
            return h + f"\n<!--immoka-url {u}-->"

        du.enrich(self, listings, DETAIL_LIMIT, parse_web_detail, key="v3",
                  fetch_html=fetch_html)
        # les loyers (« … $ par mois ») ne sont pas des maisons à vendre
        listings = [l for l in listings
                    if not (l.price is None and "par mois" in (l.price_label or ""))]
        for lst in listings:
            if not lst.title:
                lst.title = lst.address or _title_from_slug(lst.url) \
                    or "Propriété à vendre"
        return listings

    def _collect(self, url: str, by_id: dict) -> bool:
        try:
            body = self.get(url).text
        except Exception:
            return False
        host = urllib.parse.urlparse(self.site_url).netloc.lower().removeprefix("www.")
        # certains CMS déclarent <base href> : les liens relatifs partent de là
        mb = re.search(r'<base\s+href="([^"]+)"', body[:6000], re.I)
        base = mb.group(1) if mb else url
        raw = _LOC_RE.findall(body) if "<loc>" in body[:4000] or url.endswith(".xml") \
            else _HREF_RE.findall(body)
        for href in raw:
            href = _html.unescape(href)
            low = href.lower()
            if any(j in low for j in _JUNK) or any(s in low for s in _SOLD):
                continue
            m = _ID_RE.search(href)
            if not m:
                continue
            absu = urllib.parse.urljoin(base, href)
            h = urllib.parse.urlparse(absu).netloc.lower().removeprefix("www.")
            if h != host:
                continue
            if self.require and self.require not in absu.lower():
                continue
            eid = m.group(1)
            if eid not in by_id:
                by_id[eid] = PropertyListing(
                    source=self.source_id, external_id=eid, url=absu,
                    mls=eid, agency=self.agency_name, broker_name=self.agency_name)
        return True


def _title_from_slug(url: str) -> str:
    """« …/triplex-a-vendre-longueuil-st-hubert-15180332-detail-Fr » → titre."""
    stop = {"details", "detail", "proprietes", "propriete", "properties",
            "property", "listings", "listing", "inscriptions", "fiche",
            "fr", "en", "index"}
    segs = [s for s in urllib.parse.urlparse(url).path.split("/") if s]
    words: list[str] = []
    for seg in reversed(segs):
        seg = urllib.parse.unquote_plus(seg)
        seg = re.sub(r"\.(?:html?|php|aspx?)$", "", seg, flags=re.I)
        seg = re.sub(r"\d{7,9}", " ", seg)
        seg = re.sub(r"detail-?fr", " ", seg, flags=re.I)
        words = [w for w in seg.replace("_", "-").replace(" ", "-").split("-") if w]
        if words and not all(w.lower() in stop for w in words):
            break
        words = []
    if not words:
        return ""
    txt = " ".join(words)
    txt = re.sub(r"\ba vendre\b", "à vendre", txt, flags=re.I)
    txt = re.sub(r"\ba louer\b", "à louer", txt, flags=re.I)
    return (txt[:1].upper() + txt[1:])[:120]


# --- parseur détail générique -------------------------------------------------
_RENT_TAIL_RE = re.compile(r"\s*(?:\+\s*tx\s*)?(?:par mois|/\s*mois|/\s*month|"
                           r"month|mensuel|par semaine|par nuit)", re.I)
# queues qui trahissent un filtre de recherche, une plage ou une évaluation/taxe
_JUNK_TAIL_RE = re.compile(r"\s*(?:et (?:moins|plus)|-\s*\d|à\s*\d|\(20\d\d\))", re.I)
# contextes boilerplate (calculatrice hypothécaire, plages de filtres)
_JUNK_CTX = ("de plus de", "supérieure à", "inférieure à", "mise de fond",
             "more than", "down payment", "valeur de")
_ASKED_RE = re.compile(r"prix demandé", re.I)


def _pick_price(flat: str, out: dict) -> None:
    """Choisit le prix de vente dans le texte aplati ; marque les locations
    (« … $ par mois ») via price_label sans prix — la fiche sera écartée."""
    candidates = []
    rent_label = ""
    for m in _PRICE_RE.finditer(flat):
        raw = (m.group(1) or m.group(2))
        try:
            val = float(re.sub(r"[^\d.]", "", raw.replace(",", ""))[:12])
        except ValueError:
            continue
        tail = flat[m.end():m.end() + 16]
        if _RENT_TAIL_RE.match(tail):
            # loyer — la fiche peut aussi être à vendre : on continue à chercher
            rent_label = rent_label or (m.group(0).strip() + " par mois")
            continue
        if _JUNK_TAIL_RE.match(tail):
            continue
        ctx = flat[max(0, m.start() - 70):m.start()]
        if ctx.rstrip().endswith("-") or any(w in ctx.lower() for w in _JUNK_CTX):
            continue
        if 25_000 <= val <= 100_000_000:
            labeled = bool(_ASKED_RE.search(ctx[-30:]))
            candidates.append((labeled, val, m.group(0).strip()))
            if labeled:
                break
    if candidates:
        # « Prix demandé » bat le premier montant venu (évaluations, comparables…)
        candidates.sort(key=lambda c: not c[0])
        _, val, label = candidates[0]
        out["price"], out["price_label"] = val, label
    elif rent_label:
        # uniquement un loyer : fiche en location (écartée par le connecteur)
        out["price_label"] = rent_label


def parse_web_detail(html: str) -> dict:
    """Fiche server-rendered générique : JSON-LD, tableaux Centris, regex."""
    out: dict = {}
    details: dict = {}

    # JSON-LD (adresse, géo, prix, description, images)
    imgs: list[str] = []
    for n in du.ld_nodes(html):
        t = n.get("@type")
        types = set(t if isinstance(t, list) else [t])
        if not (types & _LD_IMG_TYPES) and "Offer" not in types:
            continue
        addr = n.get("address")
        if isinstance(addr, dict):
            if addr.get("streetAddress"):
                st = str(addr["streetAddress"]).strip().lstrip(", ").strip()
                if st:
                    out.setdefault("address", st)
            if addr.get("addressLocality"):
                loc = str(addr["addressLocality"]).strip()
                # certains sites mettent le code postal dans la localité
                if _POSTAL_RE.match(loc.upper()):
                    details.setdefault("postal_code", loc)
                elif loc:
                    out.setdefault("city", loc)
            if addr.get("postalCode"):
                details.setdefault("postal_code", str(addr["postalCode"]))
        geo = n.get("geo") or {}
        if isinstance(geo, dict):
            try:
                lat, lng = float(geo["latitude"]), float(geo["longitude"])
                if 44.5 <= lat <= 63.0 and -80.0 <= lng <= -56.0:
                    out.setdefault("lat", lat), out.setdefault("lng", lng)
            except (KeyError, TypeError, ValueError):
                pass
        offers = n.get("offers") or (n if "Offer" in types else {})
        if isinstance(offers, dict) and offers.get("price"):
            try:
                out.setdefault("price", float(str(offers["price"]).replace(",", "")))
            except ValueError:
                pass
        img = n.get("image")
        for u in (img if isinstance(img, list) else [img]):
            if isinstance(u, str) and u.startswith("http"):
                imgs.append(u)
            elif isinstance(u, dict) and u.get("url"):
                imgs.append(u["url"])
        y = n.get("yearBuilt")
        if y and str(y).isdigit():
            out.setdefault("year_built", int(y))

    desc = du.ld_description(html)
    if not desc:
        md = _META_DESC_RE.search(html)
        if md:
            desc = _html.unescape(md.group(1)).strip()
    if desc:
        out["description"] = desc[:6000]

    # adresse depuis le <h1> (« 72Z Ch. Valley, Brome ») si le JSON-LD est muet
    if "address" not in out:
        mh = _H1_RE.search(html)
        if mh:
            h1 = _html.unescape(_TAG_RE.sub(" ", mh.group(1)))
            h1 = re.sub(r"\s+", " ", h1).strip().strip(",-– ").strip()
            if 5 <= len(h1) <= 120 and any(c.isdigit() for c in h1) \
                    and "$" not in h1:
                parts = [p.strip() for p in h1.split(",") if p.strip()]
                out["address"] = parts[0]
                for p in parts[1:]:
                    if _POSTAL_RE.match(p.upper()):
                        details.setdefault("postal_code", p)
                    else:
                        out.setdefault("city", p)
                        break

    # texte aplati : prix, chambres, sdb + tableaux Centris
    flat = du.flatten(html)
    if "price" not in out:
        _pick_price(flat, out)
    mb = _BEDS_RE.search(flat)
    if mb and int(mb.group(1)) <= 19:
        out["bedrooms"] = int(mb.group(1))
    ms = _BATHS_RE.search(flat)
    if ms and int(ms.group(1)) <= 19:
        out["bathrooms"] = int(ms.group(1))
    details.update(du.centris_details(flat))

    # description encore vide : paragraphe « Description | … » du texte aplati,
    # sinon le plus long segment de prose (≥ 150 caractères, pas de code)
    if len(out.get("description") or "") < 50:
        md = _DESC_FLAT_RE.search(flat)
        if md:
            out["description"] = md.group(1).strip()[:6000]
        else:
            segs = [s.strip() for s in flat.split(" | ")
                    if len(s.strip()) >= 150 and "{" not in s
                    and s.count(";") < 5 and not any(
                        j in s for j in ("function", "var ", "=>",
                                         "fbq(", "gtag(", "img:", "http"))]
            if segs:
                out["description"] = max(segs, key=len)[:6000]

    mt = _TITLE_RE.search(html)
    title_txt = _html.unescape(mt.group(1)).strip() if mt else ""

    # ville : « Candiac, QC » dans le texte > « À vendre — Ville » du <title>
    # > dernier segment « adresse, Ville » du <title>
    if "city" not in out:
        cand = ""
        mc = _CITY_QC_RE.search(flat)
        if mc:
            cand = mc.group(1).strip()
        elif title_txt:
            mv = _VENDRE_CITY_RE.search(title_txt)
            if mv:
                cand = mv.group(1).strip()
            else:
                base = re.split(r"\s*[|–—-]\s*(?=[A-ZÀ-Ü][\w' ]+$)", title_txt)[0]
                if "," in base and "$" not in base:
                    cand = base.rsplit(",", 1)[1].strip()
        if (2 <= len(cand) <= 45 and not any(c.isdigit() for c in cand)
                and not _STREET_RE.match(cand)):
            out["city"] = cand

    # <title> « 179 Prom. St-Louis, Notre-Dame-de-l'Île-Perrot, Montérégie,
    # J7W3J6 » : premier segment sans chiffres après l'adresse = la ville
    if "city" not in out and "," in title_txt:
        seg0 = title_txt.split("|")[0]
        for p in [p.strip() for p in seg0.split(",")][1:]:
            if (2 <= len(p) <= 45 and not any(c.isdigit() for c in p)
                    and p[:1].isupper() and not _STREET_RE.match(p)
                    and normalize_property_type(p) not in _CANON_TYPES):
                out["city"] = p
                break

    # type de bien : libellé « Type de propriété | X », slug d'URL
    # (« maison-a-etages/… »), <title>, description
    mu0 = re.search(r"<!--immoka-url (\S+)-->", html[-3000:])
    path = urllib.parse.unquote_plus(
        urllib.parse.urlparse(mu0.group(1)).path) if mu0 else ""
    mtype = re.search(r"Type(?: de propriété| de bien)?\s*\|\s*([^|]{3,40})\s*\|",
                      flat, re.I)
    for cand in ((mtype.group(1) if mtype else ""),
                 path.replace("-", " ").replace("/", " | "), title_txt,
                 (out.get("description") or "")[:300], flat[:300]):
        if not cand:
            continue
        pt = normalize_property_type(cand)
        if pt in _CANON_TYPES:
            out["property_type"] = pt
            break

    # galerie : fancybox/lightbox > CDN Centris > <img> relatifs > og:image
    gal = list(dict.fromkeys(_GALLERY_RE.findall(html)))
    if not gal:
        gal = list(dict.fromkeys(_CENTRIS_IMG_RE.findall(html)))
    if not gal:
        mu = re.search(r"<!--immoka-url (\S+)-->", html[-3000:])
        if mu:
            rel = [s for s in _REL_IMG_RE.findall(html)
                   if not any(j in s.lower() for j in ("logo", "icon", "favicon"))]
            gal = [urllib.parse.urljoin(mu.group(1), s)
                   for s in dict.fromkeys(rel)]
    if not gal:
        for a, b in _OG_RE.findall(html):
            u = _html.unescape(a or b)
            if u.startswith("http"):
                gal.append(u)
    if not gal and imgs:
        gal = list(dict.fromkeys(imgs))
    if gal:
        out["images"] = [_html.unescape(u) for u in gal[:80]]

    if "lat" not in out:
        coords = du.gmaps_coords(html)
        if coords:
            out["lat"], out["lng"] = coords

    if details:
        out["details"] = details
    return out


def _load() -> list[dict]:
    try:
        return json.loads(REGISTRY.read_text(encoding="utf-8"))
    except Exception:
        return []


# Génère une classe par agence du registre.
for _ag in _load():
    if not all(_ag.get(k) for k in ("id", "site", "list_urls")):
        continue
    _sid = _ag["id"]
    globals()[f"AGENCE_WEB_{_sid.upper()}"] = type(
        "AgenceWeb" + "".join(p.title() for p in _sid.split("_")),
        (_AgenceWeb,),
        {
            "source_id": _sid,
            "site_url": _ag["site"],
            "list_urls": list(_ag["list_urls"]),
            "agency_name": _ag.get("name", _sid.title()),
            "require": _ag.get("require", ""),
            "render": bool(_ag.get("render")),
            "max_pages": int(_ag.get("max_pages", 30)),
        },
    )