# -----------------------------------------------------------------------------
# Auto-Ka — Agrégateur de voitures usagées à vendre (province de Québec)
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/sm360_dealers.py : connecteurs des concessionnaires sur plateforme
#   SM360 / 360.agency (fournisseur de sites très répandu chez les
#   concessionnaires franchisés au Québec).
#
#   Stratégie : le sitemap (/fr/sitemap-xml) recense chaque page véhicule
#   d'occasion (/fr/inventaire-occasion/<marque>/<modèle>/<slug>-id<digits>).
#   Chaque page détail est server-rendered et expose :
#     1. un JSON-LD schema.org (@type Car) : prix, km, VIN, transmission,
#        carrosserie, couleurs, portes/places, moteur, succursale vendeuse ;
#     2. un dataLayer JS : stockNo, isNew/isDemo, catégorie de véhicule,
#        carburant, succursale (utile pour les groupes multi-succursales) ;
#     3. la galerie photo complète (img.sm360.ca/…/images/inventory/…) ;
#     4. le lien Carfax le cas échéant.
#   Les pages détail sont mises en cache BD (clé hebdomadaire) : seuls les
#   nouveaux véhicules — et une revalidation par semaine pour les prix —
#   génèrent de vraies requêtes.
#
#   En complément, l'API GraphQL publique que le site utilise lui-même
#   (webauto-supplier-api.sm360.ca — celle des widgets VDP) donne pour chaque
#   id : la liste d'équipements (options.labels), la galerie complète
#   (multimedia.pictures — indispensable pour les gabarits dont la galerie
#   n'est PAS server-rendered : touteslesmarques, spinelli, rivesudchrysler,
#   toyotagatineau) et une description/tagline de repli. Un POST par véhicule,
#   mis en cache avec le même payload hebdomadaire.
#
#   Le listing lui-même est un widget JS (GraphQL webauto-supplier-api) —
#   inutile ici, le sitemap suffit. robots.txt : Allow /*?page=* et /*/api/*,
#   Crawl-delay 10 → délai généreux + cache détail.
# -----------------------------------------------------------------------------
from __future__ import annotations

import datetime
import html as htmllib
import json
import os
import re
import time

import requests

from ..schema import Vehicle
from .base import BaseConnector

# URLs de véhicules d'occasion dans le sitemap (jamais catalogue-neuf /
# inventaire-neuf, donc jamais de véhicules neufs)
_VEH_URL_RE = re.compile(r"/fr/inventaire-occasion/.+-id(\d+)/?$")

# segments marque/modèle qui ne sont PAS des automobiles (récréatif)
_EXCLUDE_SEGMENTS = re.compile(
    r"ski-?doo|sea-?doo|can-?am|polaris|arctic-?cat|lynx|brp|harley|"
    r"kawasaki|suzuki-moto|ktm|husqvarna|triumph|ducati|vespa|"
    r"motoneige|vtt|moto(?:s)?\b|motomarine|bateau|ponton|roulotte|"
    r"remorque|\bvr\b|campeur|spyder|ryker|side-?by-?side|atv|snow|"
    r"powersport|nautique", re.I)

# catégories dataLayer non automobiles (par prudence — normalement le
# sitemap /fr/inventaire-occasion ne contient que des autos)
_EXCLUDE_CATEGORIES = {
    "motorcycle", "atv", "snowmobile", "watercraft", "boat", "rv",
    "trailer", "sxs", "utv", "scooter", "powersport",
}

_LD_RE = re.compile(
    r'<script type=["\']application/ld\+json["\']>(.*?)</script>', re.S)

# galerie photo : URLs img.sm360.ca, avec ou sans préfixe resizer /ir/wNNN/
_IMG_RE = re.compile(
    r'https://img\.sm360\.ca/(?:ir/[a-zA-Z0-9]+/)?'
    r'(images/inventory/[^"\'\s<>\\]+\.(?:jpe?g|png|webp))', re.I)

_CARFAX_RE = re.compile(r'https?://(?:vhr\.)?carfax\.ca/[^\s"\'<>]+', re.I)

# API GraphQL publique des widgets SM360 (celle que la page détail utilise
# elle-même pour afficher équipements et galerie) — id = id inventaire global
_GRAPHQL_API = "https://webauto-supplier-api.sm360.ca/webauto/graphql"
_GQL_VEHICLE = ("{ vehicle(id: %d) { description tagline "
                "options { labels } multimedia { pictures { url } } } }")

# clés extraites du dataLayer JS de la page détail (valeurs 'entre quotes')
_DL_KEYS = (
    "salePrice", "regularPrice", "mileage", "stockNo", "vehicleVin",
    "isNew", "isDemo", "vehicleCategory", "condition", "vehicleLocation",
    "vehicleFuelType", "transmission", "trim", "make", "model", "year",
)

_DRIVE_MAP = {
    "FrontWheelDriveConfiguration": "Traction",
    "RearWheelDriveConfiguration": "Propulsion",
    "AllWheelDriveConfiguration": "Intégrale / 4x4",
    "FourWheelDriveConfiguration": "Intégrale / 4x4",
}


def _clean(text: str | None) -> str:
    """Désencode les entités HTML et retire balises / « s/o » SM360."""
    if not text:
        return ""
    out = htmllib.unescape(htmllib.unescape(str(text)))
    out = re.sub(r"<[^>]+>", " ", out)
    out = re.sub(r"\s{2,}", " ", out).strip()
    return "" if out.lower() in ("s/o", "n/d", "n/a") else out


def _num(value) -> float | None:
    """float() tolérant ; 0 et négatif → None (SM360 met 0 = prix caché)."""
    try:
        f = float(value)
    except (TypeError, ValueError):
        return None
    return f if f > 0 else None


class SM360Connector(BaseConnector):
    """Base commune aux sites SM360 / 360.agency. Sous-classes : base_url…"""

    base_url: str = ""
    dealer_name: str = ""
    city: str = ""
    request_delay: float = 1.5        # robots.txt SM360 demande de la patience
    use_firecrawl: bool = False       # au cas où un site passe sous Cloudflare

    # plafond de vraies requêtes détail par sync (AUTOKA_MAX_DETAILS le réduit
    # pour les tests) — le reste sera complété aux syncs suivants
    max_details = 600

    def _get_text(self, url: str) -> str:
        if self.use_firecrawl:
            return self.get_rendered(url)
        try:
            return self.get(url).text
        except requests.exceptions.ConnectionError:
            # le serveur SM360 ferme parfois la connexion keep-alive : on
            # repart sur une connexion neuve et on réessaie une fois
            self.session.close()
            time.sleep(2.0)
            return self.get(url).text

    # -- sitemap ---------------------------------------------------------------
    def _vehicle_urls(self) -> list[str]:
        xml = self._get_text(f"{self.base_url}/fr/sitemap-xml")
        urls: list[str] = []
        for loc in re.findall(r"<loc>([^<]+)</loc>", xml):
            loc = loc.strip()
            if not _VEH_URL_RE.search(loc):
                continue
            # segments /fr/inventaire-occasion/<marque>/<modèle>/<slug>
            path = loc.split("/fr/inventaire-occasion/", 1)[1]
            segments = path.split("/")[:-1]
            if any(_EXCLUDE_SEGMENTS.search(s) for s in segments):
                continue
            urls.append(loc)
        return urls

    # -- page détail -> payload (JSON-LD + dataLayer + galerie + carfax) --------
    def _fetch_detail(self, url: str, ext_id: str) -> dict:
        html = self._get_text(url)
        payload: dict = {"ld": {}, "dl": {}, "images": [], "carfax": ""}

        for block in _LD_RE.findall(html):
            try:
                data = json.loads(block.strip(), strict=False)
            except ValueError:
                continue
            items = data if isinstance(data, list) else [data]
            for item in items:
                if not isinstance(item, dict):
                    continue
                types = item.get("@type")
                types = types if isinstance(types, list) else [types]
                if "Car" in types or "Vehicle" in types:
                    payload["ld"] = item
                    break
            if payload["ld"]:
                break

        for key in _DL_KEYS:
            m = re.search(r"\b" + key + r"\s*:\s*'((?:[^'\\]|\\.)*)'", html)
            if m:
                payload["dl"][key] = m.group(1).replace("\\'", "'").strip()

        # galerie : uniquement les photos de CE véhicule, sans préfixe resizer,
        # triées par index _NNN de fin de nom de fichier
        seen: set[str] = set()
        images: list[str] = []
        for path in _IMG_RE.findall(html):
            if f"/{ext_id}/" not in path:
                continue
            full = "https://img.sm360.ca/" + path
            if full not in seen:
                seen.add(full)
                images.append(full)

        def _idx(u: str) -> int:
            m = re.search(r"_(\d+)\.\w+$", u)
            return int(m.group(1)) if m else 999

        payload["images"] = sorted(images, key=_idx)[:20]

        m = _CARFAX_RE.search(html)
        if m:
            payload["carfax"] = htmllib.unescape(m.group(0))

        payload["gql"] = self._fetch_gql(ext_id)
        return payload

    # -- API GraphQL : équipements + galerie + description de repli -------------
    def _fetch_gql(self, ext_id: str) -> dict:
        """options.labels, multimedia.pictures et description/tagline du
        véhicule via l'API publique des widgets. Tolérant : {} en cas d'échec
        (le payload HTML reste utilisable seul)."""
        wait = self.request_delay - (time.time() - self._last_request)
        if wait > 0:
            time.sleep(wait)
        try:
            resp = self.session.post(
                _GRAPHQL_API,
                json={"query": _GQL_VEHICLE % int(ext_id)},
                timeout=self.timeout)
            self._last_request = time.time()
            resp.raise_for_status()
            vehicle = (resp.json().get("data") or {}).get("vehicle") or {}
        except Exception:
            self._last_request = time.time()
            return {}
        if not isinstance(vehicle, dict):
            return {}
        labels = (vehicle.get("options") or {}).get("labels") or []
        options = [_clean(lb) for lb in labels if _clean(lb)]

        images: list[str] = []
        pictures = (vehicle.get("multimedia") or {}).get("pictures") or []
        for pic in pictures:
            u = str((pic or {}).get("url") or "").strip()
            if not u:
                continue
            if not u.startswith("http"):     # chemin relatif au serveur d'images
                u = "https://img.sm360.ca/images/inventory" + u
            images.append(u)

        return {
            "options": options[:60],
            "images": images[:20],
            "description": _clean(vehicle.get("description")),
            "tagline": _clean(vehicle.get("tagline")),
        }

    # -- multi-succursales (hook, surchargé par TLM) -----------------------------
    def _dealer_city(self, dl: dict, ld: dict) -> tuple[str, str]:
        return self.dealer_name, self.city

    # -- contrat ---------------------------------------------------------------
    def fetch(self) -> list[Vehicle]:
        urls = self._vehicle_urls()
        week = datetime.date.today().isocalendar()
        # v2 : + équipements/galerie GraphQL (bump = re-crawl complet forcé)
        cache_key = f"v2:{week.year}w{week.week}"   # revalidation hebdomadaire
        cap = int(os.environ.get("AUTOKA_MAX_DETAILS", self.max_details))

        vehicles: list[Vehicle] = []
        real_fetches = 0
        for url in urls:
            ext_id = _VEH_URL_RE.search(url).group(1)

            def _fetch(u=url, e=ext_id):
                return self._fetch_detail(u, e)

            # plafond atteint : ne servir que le cache (fetch réel interdit)
            if real_fetches >= cap:
                from .. import db
                if self._detail_con is None:
                    self._detail_con = db.connect()
                data = db.get_cached_detail(self._detail_con, self.source_id,
                                            ext_id, cache_key)
                if data is None:
                    continue
            else:
                before = self._last_request
                try:
                    data = self.detail(ext_id, cache_key, _fetch)
                except Exception:
                    # page disparue (vendu) ou erreur ponctuelle : ne jamais
                    # bloquer la source entière
                    if self._last_request != before:
                        real_fetches += 1
                    continue
                if self._last_request != before:   # une vraie requête a eu lieu
                    real_fetches += 1
            if not data:
                continue
            veh = self._to_vehicle(ext_id, url, data)
            if veh is not None:
                vehicles.append(veh)
        return vehicles

    def _to_vehicle(self, ext_id: str, url: str, data: dict) -> Vehicle | None:
        ld: dict = data.get("ld") or {}
        dl: dict = data.get("dl") or {}
        if not ld and not dl:
            return None

        # filtres de sécurité : jamais de neuf, jamais de récréatif
        if dl.get("isNew", "").lower() == "true":
            return None
        if dl.get("vehicleCategory", "").lower() in _EXCLUDE_CATEGORIES:
            return None

        brand = ld.get("brand") or {}
        make = _clean(brand.get("name") if isinstance(brand, dict) else brand)
        model = _clean(ld.get("model")) or _clean(dl.get("model")).title()

        year = None
        for raw in (ld.get("vehicleModelDate"), ld.get("modelDate"),
                    dl.get("year")):
            if raw:
                try:
                    year = int(str(raw)[:4])
                    break
                except ValueError:
                    pass

        offers = ld.get("offers") or {}
        price = _num(offers.get("price")) or _num(dl.get("salePrice")) \
            or _num(dl.get("regularPrice"))

        odo = ld.get("mileageFromOdometer") or {}
        km = _num(odo.get("value") if isinstance(odo, dict) else odo) \
            or _num(dl.get("mileage"))

        # titre : le "name" JSON-LD varie (parfois "CR-V", parfois
        # "Chevrolet Colorado 4WD Trail Boss 2025") — on complète au besoin
        title = _clean(ld.get("name")) or f"{make} {model}".strip()
        if make and make.lower() not in title.lower():
            title = f"{make} {title}"
        if year and str(year) not in title:
            title = f"{title} {year}"

        drive_raw = str(ld.get("driveWheelConfiguration") or "")
        drivetrain = ""
        for key, canon in _DRIVE_MAP.items():
            if key in drive_raw:
                drivetrain = canon
                break

        engines = ld.get("vehicleEngine") or []
        if isinstance(engines, dict):
            engines = [engines]
        fuel, engine_txt = "", ""
        if engines and isinstance(engines[0], dict):
            fuel = _clean(engines[0].get("fuelType"))
            e_type = _clean(engines[0].get("engineType"))
            if len(e_type) > 3:            # "4" = nb cylindres, pas descriptif
                engine_txt = e_type
        fuel = fuel or _clean(dl.get("vehicleFuelType"))

        transmission = _clean(ld.get("vehicleTransmission")) \
            or _clean(dl.get("transmission"))

        trim = _clean(ld.get("vehicleConfiguration"))
        if not trim:
            trim = _clean(dl.get("trim")).upper()

        gql: dict = data.get("gql") or {}

        images = list(data.get("images") or [])
        # la galerie GraphQL est complète même quand le gabarit ne la rend pas
        # côté serveur (touteslesmarques, spinelli, rive-sud, toyotagatineau)
        gql_images = list(gql.get("images") or [])
        if len(gql_images) > len(images):
            images = gql_images
        if not images:
            ld_img = ld.get("image") or []
            images = [ld_img] if isinstance(ld_img, str) else list(ld_img)
            # écarter les stubs sans nom de fichier (véhicule sans photo)
            images = [u for u in images if isinstance(u, str)
                      and re.search(r"\.(?:jpe?g|png|webp)$", u, re.I)]

        def _int(v):
            try:
                return int(str(v))
            except (TypeError, ValueError):
                return None

        dealer_name, city = self._dealer_city(dl, ld)

        details: dict = {}
        if dl.get("isDemo", "").lower() == "true":
            details["demo"] = True
        if dl.get("condition"):
            details["condition"] = dl["condition"]

        return Vehicle(
            source=self.source_id,
            external_id=ext_id,
            url=url,
            title=title,
            make=make,
            model=model,
            trim=trim,
            year=year,
            price=price,
            price_label=f"{price:,.0f} $".replace(",", " ") if price else "",
            mileage_km=km,
            mileage_label=f"{km:,.0f} km".replace(",", " ") if km else "",
            transmission=transmission,
            fuel=fuel,
            drivetrain=drivetrain,
            body_type=_clean(ld.get("bodyType")),
            exterior_color=_clean(ld.get("color")),
            interior_color=_clean(ld.get("vehicleInteriorColor")),
            engine=engine_txt,
            doors=_int(ld.get("numberOfDoors")),
            seats=_int(ld.get("vehicleSeatingCapacity")
                       or ld.get("seatingCapacity")),
            vin=(_clean(ld.get("vehicleIdentificationNumber"))
                 or _clean(dl.get("vehicleVin"))).upper(),
            stock_number=_clean(dl.get("stockNo")) or _clean(ld.get("sku")),
            dealer_name=dealer_name,
            city=city,
            description=(_clean(ld.get("description"))
                         or gql.get("description", "")
                         or gql.get("tagline", ""))[:4000],
            features=list(gql.get("options") or []),
            details=details,
            images=images,
            carfax_url=data.get("carfax") or "",
        )


# ---------------------------------------------------------------------------
# Concessionnaires (1 sous-classe = 1 source)
# ---------------------------------------------------------------------------

class ToutesLesMarques(SM360Connector):
    """Groupe multi-succursales (Chicoutimi, Baie-Comeau, Sept-Îles).

    Le dataLayer `vehicleLocation` donne la succursale qui détient le
    véhicule ; la ville est déduite du nom de la succursale.
    """
    source_id = "touteslesmarques"
    base_url = "https://www.touteslesmarques.ca"
    dealer_name = "Toutes les marques"
    city = "Chicoutimi"

    def _dealer_city(self, dl: dict, ld: dict) -> tuple[str, str]:
        branch = _clean(dl.get("vehicleLocation"))
        if not branch or branch.lower() == "touteslesmarques.ca":
            return self.dealer_name, self.city
        low = branch.lower()
        if "baie" in low and "comeau" in low:
            city = "Baie-Comeau"
        elif "sept" in low and ("iles" in low or "îles" in low):
            city = "Sept-Îles"
        else:                       # Arnold, L'Ami Junior, Intégral Subaru…
            city = "Chicoutimi"
        return branch, city


class StJeromeAutoDepot(SM360Connector):
    source_id = "stjeromeautodepot"
    base_url = "https://www.stjeromeautodepot.com"
    dealer_name = "St-Jérôme Auto Dépôt"
    city = "Saint-Jérôme"


class LamontagneAuto(SM360Connector):
    # concessionnaire Honda — le sitemap /fr/inventaire-occasion ne contient
    # que l'occasion (le neuf vit sous /fr/catalogue-neuf, jamais ramassé)
    source_id = "lamontagneauto"
    base_url = "https://www.lamontagneauto.com"
    dealer_name = "Lamontagne Auto"
    city = "Rimouski"


# ---------------------------------------------------------------------------
# Expansion 2 — concessionnaires SM360 supplémentaires
# ---------------------------------------------------------------------------

class Spinelli(SM360Connector):
    """Groupe multi-concessions de l'île de Montréal (Pointe-Claire, Lachine…)."""
    source_id = "spinelli"
    base_url = "https://www.spinelli.com"
    dealer_name = "Groupe Spinelli"
    city = "Montréal"


class AutoDesjardins(SM360Connector):
    source_id = "autodesjardins"
    base_url = "https://www.autodesjardins.com"
    dealer_name = "Automobiles Desjardins"
    city = "Québec"


class LallierHondaHull(SM360Connector):
    source_id = "lallierhondahull"
    base_url = "https://www.lallierhondahull.com"
    dealer_name = "Lallier Honda Hull"
    city = "Gatineau"


class RiveSudChrysler(SM360Connector):
    source_id = "rivesudchrysler"
    base_url = "https://www.rivesudchrysler.com"
    dealer_name = "Rive-Sud Chrysler"
    city = "Brossard"


class ToyotaGatineau(SM360Connector):
    source_id = "toyotagatineau"
    base_url = "https://www.toyotagatineau.ca"
    dealer_name = "Toyota Gatineau"
    city = "Gatineau"


# ---------------------------------------------------------------------------
# Expansion 3 — Côte-Nord, Bas-Saint-Laurent (Matane) et île de Montréal
# ---------------------------------------------------------------------------

class SeptIlesHonda(SM360Connector):
    source_id = "septileshonda"
    base_url = "https://www.septileshonda.com"
    dealer_name = "Sept-Îles Honda"
    city = "Sept-Îles"


class SeptIlesGM(SM360Connector):
    source_id = "septilesgm"
    base_url = "https://www.sept-ilesgm.com"
    dealer_name = "Sept-Îles Chevrolet Buick GMC"
    city = "Sept-Îles"


class BaieComeauGM(SM360Connector):
    source_id = "baiecomeaugm"
    base_url = "https://www.baiecomeaugm.com"
    dealer_name = "Baie-Comeau Chevrolet Buick GMC"
    city = "Baie-Comeau"


class VolvoCarsSaintLeonard(SM360Connector):
    source_id = "volvosaintleonard"
    base_url = "https://www.volvocarssaintleonard.com"
    dealer_name = "Volvo Cars Saint-Léonard"
    city = "Montréal"


class KiaMatane(SM360Connector):
    source_id = "kiamatane"
    base_url = "https://www.kiamatane.com"
    dealer_name = "Kia Matane"
    city = "Matane"


class MarquisAutomobiles(SM360Connector):
    source_id = "marquisautomobiles"
    base_url = "https://www.marquisautomobiles.com"
    dealer_name = "Marquis Automobiles"
    city = "Matane"


class VilleneuveFord(SM360Connector):
    source_id = "villeneuveford"
    base_url = "https://www.villeneuveford.com"
    dealer_name = "Villeneuve Ford"
    city = "Matane"