#!/usr/bin/env python3
"""Détecte les boutiques récoltables par le connecteur générique.

Pour chaque boutique active SANS endpoint catalogue : teste si son sitemap
expose des pages produit dont le balisage (JSON-LD/microdata/og) est
extractible. Si oui → marque catalog_endpoint='__generic__' dans le cache
de vérification (repris par build_registry). Scrapfly en secours.
"""
import concurrent.futures as cf
import json
import os
import sys

import requests

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
CACHE = os.path.join(ROOT, "data", "verify_cache")

for line in open(os.path.join(ROOT, ".env")).read().splitlines():
    if "=" in line and not line.startswith("#"):
        k, _, v = line.partition("=")
        os.environ.setdefault(k.strip(), v.strip())

from fabrika.connectors.generic import GenericConnector, extract_product  # noqa: E402

UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/126 Safari/537.36"


def test_domain(dom):
    sess = requests.Session()
    sess.headers["User-Agent"] = UA
    conn = GenericConnector({"id": dom, "url": f"https://{dom}"})
    conn.session = sess
    conn.use_scrapfly = False      # détection rapide, sans anti-bot
    conn.timeout = 12
    try:
        urls = conn._sitemap_products()
    except Exception:
        return dom, None, 0
    if len(urls) < 3:
        return dom, None, len(urls)
    hits = 0
    for u in urls[:6]:
        try:
            html = conn._fetch_html(u)
            if html and extract_product(u, html):
                hits += 1
        except Exception:
            pass
        if hits >= 3:
            break
    ok = hits >= 3
    return dom, ok, len(urls)


def main():
    reg = json.load(open(os.path.join(ROOT, "data", "stores.json")))["stores"]
    # candidats : actives, pas déjà connectables, plateforme compatible rendu serveur
    server_plats = {"", "prestashop", "magento", "bigcommerce", "lightspeed",
                    "snipcart", "wordpress", "woocommerce", "squarespace"}
    targets = [s["id"] for s in reg
               if not s.get("catalog_endpoint") and (s.get("platform") or "") in server_plats]
    print(f"test générique sur {len(targets)} boutiques")
    found = 0
    done = 0
    with cf.ThreadPoolExecutor(10) as ex:
        for dom, ok, n in ex.map(test_domain, targets):
            done += 1
            if ok:
                found += 1
                cpath = os.path.join(CACHE, dom + ".json")
                try:
                    rec = json.load(open(cpath))
                except Exception:
                    rec = {"domain": dom, "active": True, "final_domain": dom}
                if not rec.get("platform"):
                    rec["platform"] = "generic"
                rec["catalog_endpoint"] = "__generic__"
                json.dump(rec, open(cpath, "w"))
                print(f"  ✓ {dom} ({n} pages produit)", flush=True)
            if done % 200 == 0:
                print(f"  … {done}/{len(targets)} — {found} récoltables", flush=True)
    print(f"nouvelles boutiques génériques: {found}/{len(targets)}")


if __name__ == "__main__":
    main()