# ==============================================================================
# Author: Simon-Pierre Boucher <contact@spboucher.ai>
# File:   restoka/connectors/yelp_scrape.py
# Desc:   Connecteur d'ENRICHISSEMENT Yelp SANS clé API (avis/notes par
#         scraping léger). Une seule page de RECHERCHE yelp.ca par resto
#         (find_desc=<nom>&find_loc=<ville>) via Scrapfly ASP : la page embarque
#         un cache Apollo (scripts data-apollo-state) avec, PAR commerce,
#         alias, nom, note, nombre d'avis, fourchette de prix, catégories et
#         adresse (ville + civique). Croisement CONSERVATEUR : nom similaire
#         + même numéro civique + ville compatible (jamais fusionner deux
#         établissements). Résultat -> details.yelp {rating, review_count,
#         price, categories, url} (mêmes clés que Yelp Fusion — le connecteur
#         officiel `yelp` reprend la main dès qu'une YELP_API_KEY existe).
#
#         Économe : budget YELP_SCRAPE_BUDGET (défaut 80) pages/cycle,
#         re-visite 30 jours (hit comme miss), priorité aux restos avec menu.
#         Base légale : pages publiques, extraction minimale (note agrégée +
#         lien vers la fiche Yelp, attribution affichée) — voir CLAUDE.md §15.
# ==============================================================================
from __future__ import annotations

import datetime
import html as html_mod
import json
import os
import re
import sys
import urllib.parse

from ..inspections import _CIVIC_RE, _name_similar, norm_name
from ..regions import strip_accents
from ..schema import Restaurant
from .base import BaseConnector, SkipSource

SEARCH_URL = "https://www.yelp.ca/search?find_desc={desc}&find_loc={loc}"
MAX_BUDGET = int(os.environ.get("YELP_SCRAPE_BUDGET", "80"))  # pages/cycle
REFRESH_DAYS = 30                    # re-visite des notes ET des échecs
MAX_CONSECUTIVE_FAILURES = 3         # Yelp bloque même Scrapfly -> on arrête

_APOLLO_RE = re.compile(
    r"<script[^>]*data-apollo-state[^>]*>\s*<!--(.*?)-->\s*</script>", re.S)


def parse_apollo_businesses(page_html: str) -> list[dict]:
    """Extrait les commerces du cache Apollo embarqué dans une page de
    recherche yelp.ca : [{alias, name, rating, review_count, price,
    categories, city, address, closed}]."""
    cache: dict = {}
    for m in _APOLLO_RE.finditer(page_html):
        try:
            cache.update(json.loads(html_mod.unescape(m.group(1))))
        except ValueError:
            continue
    if not cache:
        return []
    categories = {k.split(":", 1)[1]: (v.get("title") or "")
                  for k, v in cache.items()
                  if k.startswith("BusinessCategory:") and isinstance(v, dict)}
    locations = {k.split(":", 1)[1]: (v.get("address") or {})
                 for k, v in cache.items()
                 if k.startswith("BusinessLocation:") and isinstance(v, dict)}
    out = []
    for k, v in cache.items():
        if not k.startswith("Business:") or not isinstance(v, dict):
            continue
        if not v.get("alias") or not v.get("name"):
            continue
        loc_ref = ((v.get("location") or {}).get("__ref") or "")
        addr = locations.get(loc_ref.split(":", 1)[-1], {})
        cats = []
        for c in v.get("categories") or []:
            ref = (c.get("__ref") or "").split(":", 1)[-1]
            if categories.get(ref):
                cats.append(categories[ref])
        closed = any((a or {}).get("type") == "permclosed"
                     for a in [v.get(kk) for kk in v
                               if kk.startswith("activeAlert")])
        price = v.get("priceRange")
        out.append({
            "alias": v["alias"],
            "name": v["name"],
            "rating": v.get("rating"),
            "review_count": v.get("reviewCount") or 0,
            "price": (price or {}).get("display") if isinstance(price, dict)
            else price,
            "categories": cats,
            "city": (addr or {}).get("city") or "",
            "address": (addr or {}).get("addressLine1") or "",
            "closed": closed,
        })
    return out


def _city_key(city: str) -> str:
    """« Quebec City » / « Québec » -> « quebec » ; « Montréal » -> « montreal »."""
    s = strip_accents((city or "").lower())
    s = re.sub(r"[^a-z0-9]+", " ", s).strip()
    return re.sub(r"\bcity\b", "", s).strip()


def match_business(row, candidates: list[dict]) -> dict | None:
    """Croisement CONSERVATEUR d'un resto avec les résultats Yelp :
    nom similaire ET même numéro civique ET ville compatible. Sans civique
    des deux côtés : nom normalisé IDENTIQUE + même ville. Ambigu -> None."""
    rname = norm_name(row["name"])
    rcity = _city_key(row["city"])
    rcivic_m = _CIVIC_RE.match(row["address"] or "")
    rcivic = rcivic_m.group(1) if rcivic_m else ""
    addr_tokens = set(re.sub(r"[^a-z0-9]+", " ", strip_accents(
        (row["address"] or "").lower())).split()) | set(rcity.split())
    hits = []
    for c in candidates:
        if c["closed"]:
            continue
        ccity = _city_key(c["city"])
        if rcity and ccity and rcity != ccity \
                and rcity not in ccity and ccity not in rcity:
            continue
        cname = norm_name(c["name"])
        ccivic_m = _CIVIC_RE.match(c["address"] or "")
        ccivic = ccivic_m.group(1) if ccivic_m else ""
        if rcivic and ccivic:
            if rcivic == ccivic and _name_similar(rname, cname, addr_tokens):
                hits.append(c)
        elif rname and rname == cname and rcity and ccity:
            hits.append(c)            # nom exact + ville, sans civique
    aliases = {h["alias"] for h in hits}
    if len(aliases) == 1:
        return hits[0]
    return None                       # rien ou ambigu : on ne fusionne pas


class YelpScrapeConnector(BaseConnector):
    source_id = "yelp-scrape"
    request_delay = 1.0
    timeout = 60
    use_detail_cache = False
    enrichment_only = True            # n'émet aucune fiche (ingest.run)

    def _now(self) -> str:
        return datetime.datetime.now(datetime.timezone.utc) \
            .strftime("%Y-%m-%dT%H:%M:%SZ")

    def _fresh(self, stamp: str, now: float, stale_s: float) -> bool:
        try:
            ts = datetime.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ") \
                .replace(tzinfo=datetime.timezone.utc).timestamp()
            return ts > now - stale_s
        except (ValueError, TypeError):
            return False

    def fetch(self) -> list[Restaurant]:
        if not os.environ.get("SCRAPFLY_KEY"):
            raise SkipSource("SCRAPFLY_KEY manquant (.env) — scraping Yelp "
                             "impossible sans contournement anti-bot")
        import time as _time
        from .. import db
        con = db.connect()
        now = _time.time()
        stale_s = REFRESH_DAYS * 86400.0
        budget = MAX_BUDGET
        enriched = misses = failures_row = 0
        rows = con.execute(
            "SELECT uid, name, address, city, details,"
            " EXISTS (SELECT 1 FROM menus m WHERE m.uid=restaurants.uid)"
            "   AS has_menu"
            " FROM restaurants WHERE active=1 AND dup_of IS NULL"
            " AND name<>'' AND city<>''"
            # indépendants d'abord : les succursales de chaînes n'ont presque
            # jamais d'avis Yelp (budget mieux investi ailleurs)
            " ORDER BY has_menu DESC, chain IS NULL DESC, phone<>'' DESC,"
            " updated_at DESC"
        ).fetchall()
        for row in rows:
            if budget <= 0:
                break
            if failures_row >= MAX_CONSECUTIVE_FAILURES:
                print("[resto-ka] yelp-scrape: Scrapfly bloqué "
                      f"{failures_row} fois de suite — arrêt du cycle",
                      file=sys.stderr)
                break
            try:
                details = json.loads(row["details"] or "{}")
            except ValueError:
                details = {}
            yelp = details.get("yelp") or {}
            if self._fresh(yelp.get("fetched_at", ""), now, stale_s):
                continue              # note déjà fraîche (<30 j)
            probe = details.get("yelp_scrape") or {}
            if self._fresh(probe.get("fetched_at", ""), now, stale_s):
                continue              # échec récent : re-visite dans 30 j
            url = SEARCH_URL.format(
                desc=urllib.parse.quote(row["name"][:64]),
                loc=urllib.parse.quote(f"{row['city']}, QC"))
            budget -= 1
            try:
                result = self.scrapfly(url, render_js=False)
            except Exception as exc:
                failures_row += 1
                print(f"[resto-ka] yelp-scrape: {row['uid']} erreur: {exc}",
                      file=sys.stderr)
                continue
            if (result.get("status_code") or 0) != 200:
                failures_row += 1
                continue
            failures_row = 0
            candidates = parse_apollo_businesses(result.get("content") or "")
            hit = match_business(row, candidates)
            if hit and hit.get("rating") is not None:
                db.merge_details(con, row["uid"], {"yelp": {
                    "url": f"https://www.yelp.ca/biz/"
                           f"{urllib.parse.quote(hit['alias'])}",
                    "name": hit["name"],
                    "rating": hit["rating"],
                    "review_count": hit["review_count"],
                    "price": hit.get("price"),
                    "categories": hit.get("categories") or [],
                    "matched_by": "nom+adresse",
                    "via": "scrape",
                    "fetched_at": self._now(),
                }})
                enriched += 1
            else:
                reason = ("fiche sans note" if hit else
                          "introuvable ou ambigu")
                db.merge_details(con, row["uid"],
                                 {"yelp_scrape": {"miss": reason,
                                                  "fetched_at": self._now()}})
                misses += 1
            con.commit()
        con.commit()
        con.close()
        self.enriched_count = enriched
        self.enrich_message = (f"{enriched} resto(s) notés (scraping), "
                               f"{misses} sans correspondance, "
                               f"budget restant {max(budget, 0)} page(s)")
        print(f"[resto-ka] yelp-scrape: {self.enrich_message}")
        return []