# -----------------------------------------------------------------------------
# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/duproprio.py : DuProprio (FSBO — vente sans courtier)
#
#   DuProprio est la grande plateforme québécoise de vente « à vendre par le
#   propriétaire » (sans agence). Segment complémentaire absent de toutes les
#   bannières de courtage. Énumération EXHAUSTIVE via les sitemaps FR par région
#   du Québec (`/sitemaps/fr/<region>-listings.xml.gz`) : l'URL encode déjà
#   région, ville, type et l'id DuProprio (= external_id). La fiche détail
#   (JSON-LD Product) fournit prix, description et galerie ; enrichissement
#   plafonné + cache (comme les autres connecteurs). Scrapfly en secours si le
#   fetch direct est bloqué.
# -----------------------------------------------------------------------------
from __future__ import annotations

import gzip
import io
import os
import re

from .base import BaseConnector
from . import _detailutil as du
from ..normalize import normalize_property_type, parse_price
from ..schema import PropertyListing

SITEMAP_INDEX = "https://duproprio.com/sitemaps/fr/index.xml.gz"
DETAIL_LIMIT = int(os.environ.get("IMMOKA_DUPROPRIO_DETAIL_LIMIT", "500"))
AGENCY = "DuProprio (sans courtier)"

_LOC_RE = re.compile(r"<loc>([^<]+)</loc>")
# /fr/<region>/<ville>/<type>-a-vendre/<slug>-<id>
_URL_RE = re.compile(
    r"https://duproprio\.com/fr/([a-z0-9-]+)/([a-z0-9-]+)/([a-z0-9-]+?)-a-vendre/([a-z0-9-]+?)-(\d{5,})/?$",
    re.I)
_PHOTO_RE = re.compile(
    r"https://photos\.duproprio\.com/photos/public/for_sale/\d+/\d+/[^\"'\\ ]+?\.jpg", re.I)
_LDPROD_RE = re.compile(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', re.S | re.I)


class DuProprioConnector(BaseConnector):
    source_id = "duproprio"
    request_delay = 0.4

    def fetch(self) -> list[PropertyListing]:
        by_id: dict[str, PropertyListing] = {}
        for sm in self._region_listing_sitemaps():
            for url in self._sitemap_locs(sm):
                lst = self._to_listing(url)
                if lst and lst.external_id not in by_id:
                    by_id[lst.external_id] = lst
        listings = list(by_id.values())
        du.enrich(self, listings, DETAIL_LIMIT, _parse_dp_detail,
                  key="v2", fetch_html=self._fetch_detail)
        return listings

    # -- sitemaps --------------------------------------------------------------
    def _region_listing_sitemaps(self) -> list[str]:
        xml = self._get_gz(SITEMAP_INDEX)
        # uniquement les sitemaps d'INSCRIPTIONS par région (Québec)
        return [u for u in _LOC_RE.findall(xml) if u.endswith("-listings.xml.gz")]

    def _sitemap_locs(self, url: str) -> list[str]:
        return [u for u in _LOC_RE.findall(self._get_gz(url)) if "/fr/" in u]

    def _get_gz(self, url: str) -> str:
        try:
            raw = self.get(url).content
        except Exception:
            return ""
        try:
            return gzip.GzipFile(fileobj=io.BytesIO(raw)).read().decode("utf-8", "ignore")
        except OSError:
            return raw.decode("utf-8", "ignore")   # déjà décompressé

    # -- mapping ---------------------------------------------------------------
    def _to_listing(self, url: str) -> PropertyListing | None:
        m = _URL_RE.match(url.strip())
        if not m:
            return None
        region_s, city_s, type_s, addr_s, did = m.groups()
        return PropertyListing(
            source=self.source_id, external_id=did, url=url,
            title=_deslug(addr_s),
            address=_deslug(re.sub(r"^(hab|com|ter|multi|imm)-", "", addr_s)),
            city=_deslug(city_s),
            region=_region(region_s),
            property_type=normalize_property_type(type_s.replace("-", " ")),
            agency=AGENCY, broker_name="",
        )

    def _fetch_detail(self, url: str) -> str:
        html = ""
        try:
            html = self.get(url).text
        except Exception:
            html = ""
        if "application/ld+json" not in html:      # bloqué -> Scrapfly (ASP)
            try:
                html = self.get_scrapfly(url, render_js=False, asp=True)
            except Exception:
                pass
        return html


# ---------------------------------------------------------------------------
def _parse_dp_detail(html: str) -> dict:
    import json, html as _h
    out: dict = {}
    for block in _LDPROD_RE.findall(html):
        try:
            d = json.loads(block)
        except ValueError:
            continue
        if isinstance(d, dict) and d.get("@type") == "Product":
            off = d.get("offers") or {}
            p = parse_price(str(off.get("price") or "")) if off.get("price") else None
            if p:
                out["price"] = p
                out["price_label"] = f"{p:,.0f} $".replace(",", " ")
            if d.get("description"):
                out["description"] = _h.unescape(str(d["description"])).strip()[:4000]
            break
    # galerie COMPLÈTE : le JSON « "photos":[{...}] » de la page liste toutes les
    # photos (le JSON-LD/HTML n'en montre qu'une). On prend la plus haute résolution.
    imgs, seen = [], set()
    mgal = re.search(r'"photos"\s*:\s*(\[\{.*?\}\])', html, re.S)
    if mgal:
        try:
            for ph in json.loads(mgal.group(1)):
                fmt = ph.get("formats") or {}
                rel = fmt.get("1600") or fmt.get("1024") or fmt.get("600")
                if rel:
                    u = rel if rel.startswith("http") else f"https://photos.duproprio.com/{rel.lstrip('/')}"
                    if u not in seen:
                        seen.add(u); imgs.append(u)
        except ValueError:
            pass
    if not imgs:                       # repli : URLs photos dans le HTML brut
        for u in _PHOTO_RE.findall(html):
            if u not in seen:
                seen.add(u); imgs.append(u)
    if imgs:
        out["images"] = imgs
    text = re.sub(r"\s+", " ", _h_unescape(html))
    mb = re.search(r"(\d+)\s*chambre", text, re.I)
    if mb:
        out["bedrooms"] = int(mb.group(1))
    ms = re.search(r"(\d+)\s*salle[s]?\s*de\s*bain", text, re.I)
    if ms:
        out["bathrooms"] = int(ms.group(1))

    details: dict = {}
    features: list[str] = []

    # --- Dimensions des pièces (tableau des pièces) --------------------------
    rooms = []
    for blk in re.split(r'listing-rooms-details__table__item-container',
                        html)[1:41]:
        blk = blk.split("item-container")[0]
        def cell(marker):
            m = re.search(marker + r'[^>]*>(.*?)</div>', blk, re.S)
            return re.sub(r"\s+", " ", _h_unescape(m.group(1))).strip() if m else ""
        nom = cell(r'item--room"')
        niveau = cell(r'item--storey"').replace("Étage :", "").strip()
        dim = cell(r'item--dimensions"').replace("Dimensions :", "").strip()
        rev = cell(r'item--flooring"').replace("Plancher :", "").strip()
        if nom:
            rooms.append({"nom": nom, "niveau": niveau,
                          "dimensions": dim, "revetement": rev})
    if rooms:
        details["pieces"] = rooms[:30]

    # --- Caractéristiques de la propriété (rangées pointillées) ---------------
    for lm, vm in re.findall(
            r'listing-box__dotted-row">\s*<div>(.*?)</div>\s*<div>\s*</div>\s*'
            r'<div>(.*?)</div>', html, re.S):
        label = re.sub(r"\s+", " ", _h_unescape(lm)).strip()
        value = re.sub(r"\s+", " ", _h_unescape(vm)).strip()
        if not label or not value or label == "Prix demandé":
            continue
        if label == "Année de construction":
            my = re.search(r"(19|20)\d{2}", value)
            if my:
                out["year_built"] = int(my.group(0))
        elif label in ("Superficie du terrain", "Superficie habitable",
                       "Aire habitable", "Superficie du bâtiment"):
            mp = re.search(r"([\d\s,.]+)\s*pi", value)
            if mp:
                try:
                    sqft = float(mp.group(1).replace(" ", "").replace(",", ""))
                    out["lot_sqft" if "terrain" in label else "area_sqft"] = sqft
                except ValueError:
                    pass
        features.append(f"{label} : {value}")
        details[label] = value

    # --- Remarques du proprio (texte long, souvent plus riche que le JSON-LD) --
    mrq = re.search(r'listing-owners-comments__description[^>]*>(.*?)</div>',
                    html, re.S)
    if mrq:
        remarques = re.sub(r"\s+", " ", _h_unescape(mrq.group(1))).strip()
        if remarques:
            details["remarques_proprio"] = remarques[:6000]
            if len(remarques) > len(out.get("description") or ""):
                out["description"] = remarques[:6000]

    if features:
        out["features"] = features
    if details:
        out["details"] = details
    return out


def _h_unescape(html: str) -> str:
    import html as _h
    return _h.unescape(re.sub(r"<[^>]+>", " ", html))


def _deslug(s: str) -> str:
    return re.sub(r"\s+", " ", s.replace("-", " ").replace("_", " ")).strip().title()


def _region(slug: str) -> str:
    r = _deslug(slug)
    return (r.replace("Quebec", "Québec").replace("Montreal", "Montréal")
             .replace("Monteregie", "Montérégie").replace("Laurentides", "Laurentides")
             .replace("Mauricie", "Mauricie").replace("Estrie", "Estrie"))