# -----------------------------------------------------------------------------
# Lou-Ka — Agrégateur de logements à louer (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/duproprio.py : DuProprio (duproprio.com) — section LOCATION
#   ~4 200 annonces de particuliers (sans courtier : zéro chevauchement MLS).
#   Découverte par les 21 sitemaps régionaux `sitemaps/fr/<region>-listings.xml.gz`
#   filtrés sur `/fr/location/`. Fiches SSR (Laravel/CloudFront, aucun anti-bot) :
#   JSON-LD `Apartment` (adresse + code postal + chambres/SDB + amenityFeature)
#   et `RealEstateListing` (offers.price + availabilityStarts), lat/lng inline,
#   superficie « 970 pi² », galerie dans un JSON échappé ("photos":[…]).
#   Fiche retirée -> 302 vers la page parente (détectée par l'URL finale sans
#   `/hab-`). Cache détail avec TTL (_detailutil.TtlDetailCache).
#   Respect robots.txt : ne jamais toucher /webservice/, /fr-ca/api, /api-proxy.
# -----------------------------------------------------------------------------
from __future__ import annotations

import gzip
import html as H
import os
import re

from ..schema import Listing
from .base import BaseConnector
from . import _detailutil as du

BASE = "https://duproprio.com"
SITEMAP_INDEX = f"{BASE}/sitemaps/fr/index.xml.gz"

_LOC_RE = re.compile(r"<loc>(.*?)</loc>")
# .../fr/location/<region>/<ville>/<type>-a-louer/hab-<slug>-<ID>
_FICHE_RE = re.compile(
    r"^https://duproprio\.com/fr/location/([a-z0-9-]+)/([a-z0-9-]+)/"
    r"([a-z0-9-]+)-a-louer/hab-[a-z0-9-]+-(\d+)$")
_GEO_RE = re.compile(r'"latitude":([\d.-]+),"longitude":([\d.-]+)')
_AREA_RE = re.compile(r"([\d\s,.]{1,10})\s*pi²")
_PHOTO_RE = re.compile(
    r"photos\\?/public\\?/for_rent\\?/[0-9]+\\?/[0-9]+\\?/[a-z0-9-]+-(\d+)\.jpg")
_PHOTO_FULL_RE = re.compile(
    r"photos[\\/]+public[\\/]+for_rent[\\/]+(\d+)[\\/]+(\d+)[\\/]+([a-z0-9-]+)-(\d+)\.jpg")
_TYPE_HALF_RE = re.compile(r"^(\d+)-1-2$")

# types non résidentiels (premier mot du slug type) exclus du parc
_EXCLUDED_TYPES = {"commerce", "bureau", "local", "entrepot", "espace",
                   "terrain", "garage", "stationnement", "industriel"}

DETAIL_LIMIT = int(os.environ.get("LOUKA_DUPROPRIO_DETAIL_LIMIT", "800"))
TTL_DAYS = float(os.environ.get("LOUKA_DUPROPRIO_TTL_DAYS", "7"))
MAX_FICHES = int(os.environ.get("LOUKA_DUPROPRIO_MAX", "0"))     # 0 = tout


def _parse_fiche(html: str) -> dict:
    out: dict = {}
    for node in du.ld_nodes(html):
        t = node.get("@type") or ""
        if t in ("Apartment", "House", "SingleFamilyResidence", "Accommodation"):
            out["title"] = (node.get("name") or "").strip()
            addr = node.get("address") or {}
            out["address"] = (addr.get("streetAddress") or "").strip()
            out["city"] = (addr.get("addressLocality") or "").strip()
            out["postal_code"] = (addr.get("postalCode") or "").strip()
            for src, lab in (("numberOfBedrooms", "Chambres"),
                             ("numberOfFullBathrooms", "Salles de bain"),
                             ("floorLevel", "Étage")):
                if node.get(src):
                    out.setdefault("details", {})[lab] = str(node[src])
            if node.get("numberOfBedrooms"):
                out["bedrooms"] = str(node["numberOfBedrooms"])
            feats = node.get("amenityFeature") or []
            names = [f.get("name", "").strip() for f in feats
                     if isinstance(f, dict) and f.get("name")]
            if names:
                out["amenities"] = names
            if node.get("description"):
                out["description"] = H.unescape(str(node["description"])).strip()[:6000]
        elif t == "RealEstateListing":
            offers = node.get("offers") or {}
            try:
                out["price"] = float(str(offers.get("price")).replace(",", "."))
            except (TypeError, ValueError):
                pass
            if offers.get("availabilityStarts"):
                out["availability_date"] = str(offers["availabilityStarts"])[:10]

    m = _GEO_RE.search(html)
    if m:
        out["lat"], out["lng"] = float(m.group(1)), float(m.group(2))
    m = _AREA_RE.search(html)
    if m:
        try:
            area = float(m.group(1).replace(" ", "").replace(",", ""))
            if 80 <= area <= 20000:
                out["area_sqft"] = area
        except ValueError:
            pass

    # galerie : URLs (souvent échappées \/) dédupliquées par id photo, en 1600 px
    photos, seen = [], set()
    for m in _PHOTO_FULL_RE.finditer(html):
        ym, lid, slug, pid = m.groups()
        if pid in seen:
            continue
        seen.add(pid)
        # le slug embarque la taille en avant-dernier segment : la re-forcer à 1600
        slug1600 = re.sub(r"-\d+$", "-1600", slug)
        photos.append(f"https://photos.duproprio.com/photos/public/for_rent/"
                      f"{ym}/{lid}/{slug1600}-{pid}.jpg")
    if photos:
        out["images"] = photos

    vt = du.virtual_tour(html)      # visite virtuelle (Matterport/iGUIDE…)
    if vt:
        out.setdefault("details", {})["virtual_tour"] = vt
    return out


class DuProprioConnector(BaseConnector):
    source_id = "duproprio"
    request_delay = 0.8

    def _gz(self, url: str) -> str:
        resp = self.get(url)
        try:
            return gzip.decompress(resp.content).decode("utf-8", "replace")
        except (OSError, EOFError):
            return resp.text

    def _fiche_urls(self) -> list[tuple[str, str, str, str]]:
        """(url, ville, type_slug, id) des fiches location, tous sitemaps."""
        index = self._gz(SITEMAP_INDEX)
        subs = [u for u in _LOC_RE.findall(index) if u.endswith("-listings.xml.gz")]
        out, seen = [], set()
        for sub in subs:
            try:
                xml = self._gz(sub)
            except Exception:
                continue
            for u in _LOC_RE.findall(xml):
                m = _FICHE_RE.match(u)
                if m and m.group(3).split("-")[0] in _EXCLUDED_TYPES:
                    continue
                if m and m.group(4) not in seen:
                    seen.add(m.group(4))
                    out.append((u, m.group(2), m.group(3), m.group(4)))
        return out

    def _fetch_fiche(self, url: str) -> str:
        """GET d'une fiche ; une redirection vers la page parente = retirée."""
        resp = self.get(url, allow_redirects=True)
        if "/hab-" not in resp.url:
            return ""                     # 302 « annonce disparue »
        return resp.text

    @staticmethod
    def _unit_type(type_slug: str, bedrooms: str | None) -> str:
        m = _TYPE_HALF_RE.match(type_slug)
        if m:
            n = int(m.group(1))
            return "6½+" if n >= 6 else f"{n}½"
        base = {"studio": "Studio", "loft": "Loft", "chambre": "Chambre",
                "maison": "Maison", "condo": "", "appartement": ""}
        label = base.get(type_slug.split("-")[0], "")
        if not label and bedrooms and bedrooms.isdigit():
            n = int(bedrooms) + 2
            return "6½+" if n >= 6 else f"{n}½"
        return label or type_slug.replace("-", " ").capitalize()

    def fetch(self) -> list[Listing]:
        fiches = self._fiche_urls()
        if MAX_FICHES:
            fiches = fiches[:MAX_FICHES]
        cache = du.TtlDetailCache(self, budget=DETAIL_LIMIT, ttl_days=TTL_DAYS,
                                  fetch_html=self._fetch_fiche)
        out: list[Listing] = []
        try:
            for url, _ville, type_slug, eid in fiches:
                payload = cache.get(eid, url, _parse_fiche)
                if payload is None:       # jamais visitée + budget épuisé
                    continue
                if not payload or payload.get("gone"):
                    continue              # fiche retirée (302) ou page vide
                price = payload.get("price")
                # availabilityStarts dans le passé = déjà libre aujourd'hui
                avail = payload.get("availability_date")
                if avail:
                    import datetime as _dt
                    if avail <= _dt.date.today().isoformat():
                        avail = "now"
                # « Ahuntsic (Montréal) » -> secteur + ville ; « X (X) » -> X
                locality = payload.get("city") or ""
                sector = ""
                m = re.match(r"^(.*?)\s*\((.+)\)\s*$", locality)
                if m:
                    inner, outer = m.group(1).strip(), m.group(2).strip()
                    if inner.lower() == outer.lower():
                        locality, sector = inner, ""
                    else:
                        locality, sector = outer, inner
                lst = Listing(
                    source=self.source_id,
                    external_id=str(eid),
                    url=url,
                    title=payload.get("title") or "",
                    address=payload.get("address") or "",
                    sector=sector,
                    city=locality,
                    unit_type=self._unit_type(type_slug, payload.get("bedrooms")),
                    price=price,
                    price_label=(f"{price:,.0f} $/mois".replace(",", " ")
                                 if price else ""),
                    availability_date=avail,
                    availability=("Libre immédiatement" if avail == "now"
                                  else f"Libre le {avail}" if avail else ""),
                    area_sqft=payload.get("area_sqft"),
                    description=payload.get("description") or "",
                    amenities=payload.get("amenities") or [],
                    details=payload.get("details") or {},
                    images=payload.get("images") or [],
                    lat=payload.get("lat"),
                    lng=payload.get("lng"),
                )
                low = [a.lower() for a in lst.amenities]
                if any("animaux" in a and "permis" in a for a in low):
                    lst.pets = "oui"
                if any(a.startswith("meublé") for a in low):
                    lst.furnished = True
                out.append(lst)
        finally:
            cache.close()
        return out