# -----------------------------------------------------------------------------
# Fabri-Ka — Agrégateur de produits québécois
# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
# connectors/generic.py : connecteur universel par sitemap + extraction du
#   balisage produit rendu côté serveur (JSON-LD schema.org Product, microdata,
#   Open Graph product, blobs PrestaShop/Magento). Couvre PrestaShop, Magento,
#   BigCommerce, WordPress non-Woo et sites ecommerce sur mesure. Scrapfly en
#   secours pour l'anti-bot. Rendu client-only (Square, Ecwid) non couvert.
# -----------------------------------------------------------------------------
from __future__ import annotations

import concurrent.futures as cf
import json
import re
from html import unescape
from urllib.parse import urlparse

from ..schema import Product, parse_price
from .base import BaseConnector

PRODUCT_URL_RE = re.compile(
    r"(/produits?/|/product/|/products/|/boutique/|/shop/|/store/|/p/|/item/"
    r"|/\d+-[a-z0-9]|/achat/|/produit-|-p\d+\.html|\.html$)", re.I)
NON_PRODUCT_RE = re.compile(
    r"(/blog|/blogue|/category|/categorie|/tag/|/page/|/compte|/account|/cart|"
    r"/panier|/checkout|/contact|/a-propos|/about|/cms|/content/|/faq|/policies|"
    r"/politique|sitemap|\.(?:jpg|png|pdf|css|js)$)", re.I)
LOC_RE = re.compile(r"<loc>\s*(?:<!\[CDATA\[)?\s*(.*?)\s*(?:\]\]>)?\s*</loc>", re.I | re.S)


def _clean(s):
    return unescape(re.sub(r"<[^>]+>", " ", re.sub(r"\s+", " ", s or ""))).strip()


def _walk_jsonld(node, out):
    if isinstance(node, list):
        for x in node:
            _walk_jsonld(x, out)
    elif isinstance(node, dict):
        t = node.get("@type")
        types = t if isinstance(t, list) else [t]
        if any(str(x).endswith("Product") for x in types if x):
            out.append(node)
        for v in node.values():
            if isinstance(v, (list, dict)):
                _walk_jsonld(v, out)


def extract_product(url, html):
    """Retourne un dict {title, price, image, description, currency, available} ou None."""
    title = price = image = desc = None
    currency = "CAD"
    available = None

    # 1) JSON-LD Product
    for block in re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>',
                            html, re.S | re.I):
        try:
            data = json.loads(block.strip())
        except Exception:
            continue
        prods = []
        _walk_jsonld(data, prods)
        for p in prods:
            if not isinstance(p, dict):
                continue
            offers = p.get("offers") or {}
            if isinstance(offers, list):
                offers = next((o for o in offers if isinstance(o, dict)), {})
            if not isinstance(offers, dict):
                offers = {}
            spec = offers.get("priceSpecification") or {}
            if isinstance(spec, list):
                spec = next((s for s in spec if isinstance(s, dict)), {})
            if not isinstance(spec, dict):
                spec = {}
            pr = parse_price(offers.get("price") or offers.get("lowPrice")
                             or spec.get("price"))
            if pr:
                title = title or _clean(p.get("name"))
                price = price or pr
                currency = offers.get("priceCurrency") or currency
                img = p.get("image")
                if isinstance(img, list):
                    img = img[0] if img else None
                if isinstance(img, dict):
                    img = img.get("url")
                image = image or img
                desc = desc or _clean(p.get("description"))
                av = str(offers.get("availability") or "")
                available = ("InStock" in av) if av else available

    # 2) Open Graph product / meta
    if not price:
        m = re.search(r'<meta[^>]+(?:og:price:amount|product:price:amount)"[^>]*content="([^"]+)"', html, re.I) \
            or re.search(r'<meta[^>]+content="([^"]+)"[^>]*(?:og:price:amount|product:price:amount)"', html, re.I)
        if m:
            price = parse_price(m.group(1))
    # 3) microdata itemprop=price
    if not price:
        m = re.search(r'itemprop="price"[^>]*content="([^"]+)"', html, re.I) \
            or re.search(r'content="([^"]+)"[^>]*itemprop="price"', html, re.I)
        if m:
            price = parse_price(m.group(1))
    if not title:
        m = re.search(r'<meta[^>]+property="og:title"[^>]*content="([^"]+)"', html, re.I)
        title = _clean(m.group(1)) if m else None
        if not title:
            m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)
            title = _clean(m.group(1)) if m else None
    if not image:
        m = re.search(r'<meta[^>]+property="og:image"[^>]*content="([^"]+)"', html, re.I)
        image = m.group(1) if m else None
    if not desc:
        m = re.search(r'<meta[^>]+(?:name|property)="(?:description|og:description)"[^>]*content="([^"]+)"', html, re.I)
        desc = _clean(m.group(1)) if m else None

    if not (title and price):
        return None
    return {"title": title, "price": price, "image": image, "description": desc,
            "currency": currency, "available": available}


class GenericConnector(BaseConnector):
    platform = "generic"
    request_delay = 0.2
    max_products = 800
    use_scrapfly = True      # False pendant la détection (vitesse)

    def _sitemap_products(self):
        from . import scrapfly
        seen, out = set(), []
        roots = [f"{self.base}/sitemap.xml", f"{self.base}/sitemap_index.xml",
                 f"{self.base}/wp-sitemap.xml", f"{self.base}/1_fr_0_sitemap.xml",
                 f"{self.base}/sitemap/sitemap-index.xml", f"{self.base}/media/sitemap.xml",
                 f"{self.base}/pub/media/sitemap.xml", f"{self.base}/sitemap1.xml",
                 f"{self.base}/en/sitemap.xml", f"{self.base}/fr/sitemap.xml"]
        queue, depth_left = list(roots), 3
        fetched_roots = 0
        while queue and fetched_roots < 60:
            u = queue.pop(0)
            if u in seen:
                continue
            seen.add(u)
            try:
                r = self.session.get(u, timeout=self.timeout)
                xml = r.text if r.status_code == 200 else ""
            except Exception:
                xml = ""
            if not xml and self.use_scrapfly and scrapfly.available() and u == roots[0]:
                try:
                    _, xml = scrapfly.scrapfly_get(u)
                except Exception:
                    xml = ""
            if not xml:
                continue
            fetched_roots += 1
            locs = [l.strip() for l in LOC_RE.findall(xml)]
            child_maps = [l for l in locs if l.endswith(".xml") or "sitemap" in l.lower()]
            if child_maps and depth_left > 0:
                queue = child_maps + queue
                depth_left -= 0
            for l in locs:
                if l.endswith(".xml"):
                    continue
                if PRODUCT_URL_RE.search(l) and not NON_PRODUCT_RE.search(l):
                    out.append(l)
            if len(out) >= self.max_products * 2:
                break
        # dédup en gardant l'ordre
        return list(dict.fromkeys(out))[: self.max_products]

    def _fetch_html(self, url):
        from . import scrapfly
        try:
            r = self.session.get(url, timeout=self.timeout)
            if r.status_code == 200 and len(r.text) > 500:
                return r.text
        except Exception:
            pass
        if self.use_scrapfly and scrapfly.available():
            try:
                st, content = scrapfly.scrapfly_get(url)
                if st == 200:
                    return content
            except Exception:
                pass
        return ""

    def fetch(self) -> list[Product]:
        urls = self._sitemap_products()
        if not urls:
            return []
        out: list[Product] = []
        base_host = urlparse(self.base).netloc.lower().replace("www.", "")

        def work(u):
            html = self._fetch_html(u)
            if not html:
                return None
            info = extract_product(u, html)
            if not info:
                return None
            return Product(
                store_id=self.store_id,
                external_id=u.rstrip("/").split("/")[-1][:80] or u,
                url=u, title=info["title"], description=info.get("description") or "",
                price=info["price"], price_max=info["price"],
                currency=info.get("currency") or "CAD",
                images=[info["image"]] if info.get("image") else [],
                available=info.get("available"))

        with cf.ThreadPoolExecutor(6) as ex:
            for rec in ex.map(work, urls):
                if rec:
                    out.append(rec)
        return out


def probe_generic(domain, session, sample=6):
    """Teste si une boutique est récoltable en générique. Retourne (ok, n_urls, n_hits)."""
    store = {"id": domain, "url": f"https://{domain}"}
    conn = GenericConnector(store)
    conn.session = session
    urls = conn._sitemap_products()
    if not urls:
        return False, 0, 0
    hits = 0
    for u in urls[:sample]:
        html = conn._fetch_html(u)
        if html and extract_product(u, html):
            hits += 1
    return (hits >= max(2, sample // 2)), len(urls), hits