# -----------------------------------------------------------------------------
# Fabri-Ka — Agrégateur de produits québécois
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/generic.py : connecteur universel par sitemap + extraction du
#   balisage produit rendu côté serveur (JSON-LD schema.org Product, microdata,
#   Open Graph product, blobs PrestaShop/Magento). Couvre PrestaShop, Magento,
#   BigCommerce, WordPress non-Woo et sites ecommerce sur mesure. Scrapfly en
#   secours pour l'anti-bot. Rendu client-only (Ecwid) non couvert ; Square
#   Online est couvert par connectors/square.py (vague 2).
# -----------------------------------------------------------------------------
from __future__ import annotations

import concurrent.futures as cf
import json
import re
from html import unescape
from urllib.parse import urlparse

from ..schema import Product, parse_price
from .base import BaseConnector

PRODUCT_URL_RE = re.compile(
    r"(/produits?/|/product/|/products/|/boutique/|/shop/|/store/|/p/|/item/"
    r"|/\d+-[a-z0-9]|/achat/|/produit-|-p\d+\.html|\.html$)", re.I)
NON_PRODUCT_RE = re.compile(
    r"(/blog|/blogue|/category|/categorie|/tag/|/page/|/compte|/account|/cart|"
    r"/panier|/checkout|/contact|/a-propos|/about|/cms|/content/|/faq|/policies|"
    r"/politique|sitemap|\.(?:jpg|png|pdf|css|js)$)", re.I)
LOC_RE = re.compile(r"<loc>\s*(?:<!\[CDATA\[)?\s*(.*?)\s*(?:\]\]>)?\s*</loc>", re.I | re.S)
IP_HOST_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?$")


def _clean(s):
    return unescape(re.sub(r"<[^>]+>", " ", re.sub(r"\s+", " ", s or ""))).strip()


def _walk_jsonld(node, out):
    if isinstance(node, list):
        for x in node:
            _walk_jsonld(x, out)
    elif isinstance(node, dict):
        t = node.get("@type")
        types = t if isinstance(t, list) else [t]
        if any(str(x).endswith("Product") for x in types if x):
            out.append(node)
        for v in node.values():
            if isinstance(v, (list, dict)):
                _walk_jsonld(v, out)


def _jsonld_images(img) -> list[str]:
    """Normalise le champ image JSON-LD (str | dict | liste mixte) en URLs."""
    out: list[str] = []
    items = img if isinstance(img, list) else [img]
    for x in items:
        if isinstance(x, dict):
            x = x.get("url") or x.get("contentUrl")
        if isinstance(x, str) and x.startswith("http") and x not in out:
            out.append(x)
    return out[:10]


def extract_product(url, html):
    """Retourne un dict {title, price, price_max, image, images, description,
    currency, available, brand, sku, gtin, rating, review_count} ou None."""
    title = price = price_max = image = desc = None
    brand = sku = gtin = rating = review_count = None
    images: list[str] = []
    currency = "CAD"
    available = None

    # 1) JSON-LD Product
    for block in re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>',
                            html, re.S | re.I):
        try:
            data = json.loads(block.strip())
        except Exception:
            continue
        prods = []
        _walk_jsonld(data, prods)
        for p in prods:
            if not isinstance(p, dict):
                continue
            offers = p.get("offers") or {}
            if isinstance(offers, list):
                offers = next((o for o in offers if isinstance(o, dict)), {})
            if not isinstance(offers, dict):
                offers = {}
            spec = offers.get("priceSpecification") or {}
            if isinstance(spec, list):
                spec = next((s for s in spec if isinstance(s, dict)), {})
            if not isinstance(spec, dict):
                spec = {}
            pr = parse_price(offers.get("price") or offers.get("lowPrice")
                             or spec.get("price"))
            if pr:
                title = title or _clean(p.get("name"))
                price = price or pr
                price_max = price_max or parse_price(offers.get("highPrice"))
                currency = offers.get("priceCurrency") or currency
                if not images:
                    images = _jsonld_images(p.get("image"))
                image = image or (images[0] if images else None)
                desc = desc or _clean(p.get("description"))
                av = str(offers.get("availability") or "")
                available = ("InStock" in av) if av else available
                b = p.get("brand")
                if isinstance(b, dict):
                    b = b.get("name")
                if isinstance(b, str) and b.strip():
                    brand = brand or _clean(b)
                if p.get("sku"):
                    sku = sku or str(p["sku"])[:80]
                for gk in ("gtin13", "gtin", "gtin12", "gtin8", "mpn"):
                    if p.get(gk):
                        gtin = gtin or str(p[gk])[:40]
                        break
                ar = p.get("aggregateRating") or {}
                if isinstance(ar, dict) and ar.get("ratingValue"):
                    try:
                        rating = rating or float(ar["ratingValue"])
                        rc = ar.get("reviewCount") or ar.get("ratingCount")
                        if rc:
                            review_count = review_count or int(float(rc))
                    except (TypeError, ValueError):
                        pass

    # 2) Open Graph product / meta
    if not price:
        m = re.search(r'<meta[^>]+(?:og:price:amount|product:price:amount)"[^>]*content="([^"]+)"', html, re.I) \
            or re.search(r'<meta[^>]+content="([^"]+)"[^>]*(?:og:price:amount|product:price:amount)"', html, re.I)
        if m:
            price = parse_price(m.group(1))
    # 3) microdata itemprop=price
    if not price:
        m = re.search(r'itemprop="price"[^>]*content="([^"]+)"', html, re.I) \
            or re.search(r'content="([^"]+)"[^>]*itemprop="price"', html, re.I)
        if m:
            price = parse_price(m.group(1))
    if not title:
        m = re.search(r'<meta[^>]+property="og:title"[^>]*content="([^"]+)"', html, re.I)
        title = _clean(m.group(1)) if m else None
        if not title:
            m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)
            title = _clean(m.group(1)) if m else None
    if not image:
        m = re.search(r'<meta[^>]+property="og:image"[^>]*content="([^"]+)"', html, re.I)
        image = m.group(1) if m else None
        if image and image not in images:
            images.insert(0, image)
    if not desc:
        m = re.search(r'<meta[^>]+(?:name|property)="(?:description|og:description)"[^>]*content="([^"]+)"', html, re.I)
        desc = _clean(m.group(1)) if m else None
    if not brand:
        m = re.search(r'<meta[^>]+property="(?:og:brand|product:brand)"[^>]*content="([^"]+)"', html, re.I)
        brand = _clean(m.group(1)) if m else None

    if not (title and price):
        return None
    return {"title": title, "price": price, "price_max": price_max,
            "image": image, "images": images or ([image] if image else []),
            "description": desc, "currency": currency, "available": available,
            "brand": brand, "sku": sku, "gtin": gtin,
            "rating": rating, "review_count": review_count}


class GenericConnector(BaseConnector):
    platform = "generic"
    request_delay = 0.2
    max_products = 800
    use_scrapfly = True      # False pendant la détection (vitesse)

    def _rehost(self, u):
        # certains CMS mal configurés (ex. boreale.com, Craft CMS, 2026-09-08)
        # émettent des <loc> sur l'IP brute du serveur → cert TLS invalide.
        # On ramène ces URLs sur le domaine de la boutique, qui sert les
        # mêmes chemins (sitemaps enfants et pages produit vérifiés 200).
        # Variante 2026-09-22 (boreale.com encore) : <loc> émis sur un domaine
        # parasite (sitemaps.instead.beer) — « sitemap » dans l'hôte faisait en
        # plus exclure chaque URL produit via NON_PRODUCT_RE. Réécriture OPT-IN
        # par fiche registre (rehost_hosts), jamais globale : d'autres boutiques
        # génériques pointent légitimement vers un domaine tiers (CDN, miroir).
        p = urlparse(u)
        bad_hosts = {h.lower() for h in (self.store.get("rehost_hosts") or [])}
        if IP_HOST_RE.match(p.netloc) or p.netloc.lower() in bad_hosts:
            return f"{self.base}{p.path}" + (f"?{p.query}" if p.query else "")
        return u

    def _sitemap_products(self):
        from . import scrapfly
        seen, out = set(), []
        roots = [f"{self.base}/sitemap.xml", f"{self.base}/sitemap_index.xml",
                 f"{self.base}/wp-sitemap.xml", f"{self.base}/1_fr_0_sitemap.xml",
                 f"{self.base}/sitemap/sitemap-index.xml", f"{self.base}/media/sitemap.xml",
                 f"{self.base}/pub/media/sitemap.xml", f"{self.base}/sitemap1.xml",
                 f"{self.base}/en/sitemap.xml", f"{self.base}/fr/sitemap.xml"]
        queue, depth_left = list(roots), 3
        fetched_roots = 0
        while queue and fetched_roots < 60:
            u = queue.pop(0)
            if u in seen:
                continue
            seen.add(u)
            try:
                r = self.session.get(u, timeout=self.timeout)
                xml = r.text if r.status_code == 200 else ""
            except Exception:
                xml = ""
            if not xml and self.use_scrapfly and scrapfly.available() and u == roots[0]:
                try:
                    _, xml = scrapfly.scrapfly_get(u)
                except Exception:
                    xml = ""
            if not xml:
                continue
            fetched_roots += 1
            locs = [self._rehost(l.strip()) for l in LOC_RE.findall(xml)]
            child_maps = [l for l in locs if l.endswith(".xml") or "sitemap" in l.lower()]
            if child_maps and depth_left > 0:
                queue = child_maps + queue
                depth_left -= 0
            for l in locs:
                if l.endswith(".xml"):
                    continue
                if PRODUCT_URL_RE.search(l) and not NON_PRODUCT_RE.search(l):
                    out.append(l)
            if len(out) >= self.max_products * 2:
                break
        # sites bilingues (ex. maisondherbes.com, refonte Next.js fr/en) : le
        # sitemap liste chaque produit en double sous un préfixe de langue
        # (/en/…). Exclusion OPT-IN par fiche registre (skip_url_prefixes),
        # jamais globale : d'autres boutiques génériques ont leur catalogue
        # légitimement sous /en/ (ex. pierresdailleurs.ca, 771 produits).
        skip = tuple(self.store.get("skip_url_prefixes") or [])
        if skip:
            out = [u for u in out if not urlparse(u).path.startswith(skip)]
        # dédup en gardant l'ordre
        return list(dict.fromkeys(out))[: self.max_products]

    def _fetch_html(self, url):
        from . import scrapfly
        try:
            r = self.session.get(url, timeout=self.timeout)
            if r.status_code == 200 and len(r.text) > 500:
                return r.text
        except Exception:
            pass
        if self.use_scrapfly and scrapfly.available():
            try:
                st, content = scrapfly.scrapfly_get(url)
                if st == 200:
                    return content
            except Exception:
                pass
        return ""

    def fetch(self) -> list[Product]:
        urls = self._sitemap_products()
        if not urls:
            return []
        out: list[Product] = []
        base_host = urlparse(self.base).netloc.lower().replace("www.", "")

        def work(u):
            html = self._fetch_html(u)
            if not html:
                return None
            info = extract_product(u, html)
            if not info:
                return None
            det: dict = {}
            if info.get("sku"):
                det["sku"] = info["sku"]
            if info.get("gtin"):
                det["gtin"] = info["gtin"]
            if info.get("rating"):
                det["average_rating"] = info["rating"]
            if info.get("review_count"):
                det["review_count"] = info["review_count"]
            return Product(
                store_id=self.store_id,
                external_id=u.rstrip("/").split("/")[-1][:80] or u,
                url=u, title=info["title"], description=info.get("description") or "",
                price=info["price"],
                price_max=info.get("price_max") or info["price"],
                currency=info.get("currency") or "CAD",
                images=info.get("images") or ([info["image"]] if info.get("image") else []),
                vendor=info.get("brand") or "",
                available=info.get("available"),
                details=det)

        with cf.ThreadPoolExecutor(6) as ex:
            for rec in ex.map(work, urls):
                if rec:
                    out.append(rec)
        return out


def probe_generic(domain, session, sample=6):
    """Teste si une boutique est récoltable en générique. Retourne (ok, n_urls, n_hits)."""
    store = {"id": domain, "url": f"https://{domain}"}
    conn = GenericConnector(store)
    conn.session = session
    urls = conn._sitemap_products()
    if not urls:
        return False, 0, 0
    hits = 0
    for u in urls[:sample]:
        html = conn._fetch_html(u)
        if html and extract_product(u, html):
            hits += 1
    return (hits >= max(2, sample // 2)), len(urls), hits