# QC Élection Forecast — Plateforme de prévision électorale du Québec 2026
# Auteur : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# https://www.qc-election.com
"""Veille web CONTINUE via Firecrawl (v2 « au-delà des sondages »).

Trois missions, exécutées à chaque cycle du pipeline (budget d'appels borné) :

1. **Radar sondages** — recherche d'actualité ciblée : tout nouvel article
   mentionnant un sondage provincial québécois est détecté, archivé avec
   provenance (`web_signals`) et signalé sur la page Signaux et dans l'admin —
   souvent avant son apparition dans l'agrégation Wikipédia.
2. **Satisfaction gouvernementale** — extraction du % de satisfaits envers le
   gouvernement du Québec (baromètres Léger/rapportage de presse). C'est
   l'entrée vivante du prior de fondamentaux (`indicators`).
3. **Presse élargie** — recherche par parti au-delà des flux RSS (JdM/JdQ,
   TVA, 98,5, médias régionaux) : les articles alimentent le moteur de
   sentiment existant (mêmes scoreurs, même provenance).

Chaque page brute est archivée dans data/raw/firecrawl/. Tout est tolérant à
l'échec : sans clé ou sans réseau, le forecast fonctionne exactement comme avant.
"""
from __future__ import annotations

import hashlib
import logging
import re
from datetime import date, datetime, timedelta, timezone

import httpx
from sqlalchemy.orm import Session

from ..config import DATA_DIR, settings
from .. import models as Mo

log = logging.getLogger("firecrawl")

FC_BASE = "https://api.firecrawl.dev/v2"
RAW_DIR = DATA_DIR / "raw" / "firecrawl"

KNOWN_POLLSTERS = ["Léger", "Leger", "Pallas", "Mainstreet", "Segma", "SOM",
                   "Angus Reid", "Research Co", "Abacus", "EKOS", "Ipsos", "CROP",
                   "Synopsis", "Forum"]

PARTY_QUERIES = {
    "CAQ": "Coalition avenir Québec Christine Fréchette",
    "PLQ": "Parti libéral du Québec Charles Milliard",
    "PQ": "Parti québécois Paul St-Pierre Plamondon",
    "QS": "Québec solidaire Ruba Ghazal",
    "PCQ": "Parti conservateur du Québec Éric Duhaime",
}


def _headers() -> dict:
    return {"Authorization": f"Bearer {settings.firecrawl_api_key}",
            "Content-Type": "application/json"}


def _enabled() -> bool:
    return settings.firecrawl_enabled and bool(settings.firecrawl_api_key)


def _post(path: str, payload: dict, timeout: float = 60.0) -> dict | None:
    try:
        r = httpx.post(f"{FC_BASE}{path}", json=payload, headers=_headers(),
                       timeout=timeout)
        r.raise_for_status()
        data = r.json()
        return data if data.get("success", True) else None
    except Exception as e:
        log.warning("firecrawl %s: %s", path, e)
        return None


def _archive(url: str, content: str) -> None:
    try:
        RAW_DIR.mkdir(parents=True, exist_ok=True)
        h = hashlib.sha1(url.encode()).hexdigest()[:16]
        (RAW_DIR / f"{date.today().isoformat()}_{h}.md").write_text(
            f"<!-- {url} — {datetime.now(timezone.utc).isoformat()} -->\n{content}")
    except Exception:
        pass


def scrape_markdown(url: str) -> str | None:
    """Scrape une page → markdown (archivé pour provenance)."""
    if not _enabled():
        return None
    data = _post("/scrape", {"url": url, "formats": ["markdown"],
                             "onlyMainContent": True}, timeout=90.0)
    md = ((data or {}).get("data") or {}).get("markdown")
    if md:
        _archive(url, md[:200000])
    return md


def search(query: str, limit: int = 8, tbs: str | None = "qdr:w",
           news: bool = True) -> list[dict]:
    """Recherche web/actualités → [{url, title, description, date?}]."""
    if not _enabled():
        return []
    payload: dict = {"query": query, "limit": limit, "location": "Montreal,Quebec,Canada"}
    if tbs:
        payload["tbs"] = tbs
    if news:
        payload["sources"] = [{"type": "news"}]
    data = _post("/search", payload)
    if not data:
        return []
    d = data.get("data") or {}
    items = []
    for bucket in ("news", "web"):
        for it in (d.get(bucket) or []):
            if isinstance(it, dict) and it.get("url"):
                items.append(it)
    if not items and isinstance(d, list):  # tolérance : ancien format plat
        items = [it for it in d if isinstance(it, dict) and it.get("url")]
    return items[:limit]


def _add_signal(db: Session, kind: str, title: str, url: str,
                snippet: str | None = None, pollster: str | None = None,
                extra: dict | None = None) -> bool:
    if db.query(Mo.WebSignal).filter_by(url=url, kind=kind).first():
        return False
    db.add(Mo.WebSignal(kind=kind, title=title[:500], url=url,
                        snippet=(snippet or "")[:1000], pollster=pollster,
                        extra=extra or {}))
    return True


# ---------------------------------------------------------------------------
# 1. Radar sondages
# ---------------------------------------------------------------------------
def radar_polls(db: Session) -> int:
    """Détecte les nouveaux sondages provinciaux dans l'actualité."""
    added, seen = 0, set()
    for q in ["sondage intentions de vote Québec provincial",
              "nouveau sondage élections Québec 2026"]:
        for it in search(q, limit=8, tbs="qdr:w"):
            text = f"{it.get('title', '')} {it.get('description', '')}"
            if it["url"] in seen or not re.search(r"sondage|intentions de vote|poll",
                                                  text, re.I):
                continue
            seen.add(it["url"])
            house = next((h for h in KNOWN_POLLSTERS if h.lower() in text.lower()), None)
            if _add_signal(db, "poll-radar", it.get("title", "(sans titre)"),
                           it["url"], it.get("description"), pollster=house,
                           extra={"query": q, "published": it.get("date")}):
                db.flush()
                added += 1
    db.commit()
    return added


# ---------------------------------------------------------------------------
# 2. Satisfaction envers le gouvernement (entrée du prior de fondamentaux)
# ---------------------------------------------------------------------------
SAT_PATTERNS = [
    r"(\d{2})\s?%\s+(?:des\s+\S+\s+){0,4}(?:se\s+disent?\s+|sont\s+)?satisfait",
    r"satisfaction[^.\d]{0,60}?(\d{2})\s?%",
    r"taux de satisfaction[^.\d]{0,40}?(\d{2})\s?%",
]


def fetch_gov_satisfaction(db: Session) -> dict:
    """Cherche la mesure la plus récente du % de satisfaits envers le
    gouvernement du Québec; enregistre un Indicator avec provenance."""
    report = {"found": False}
    hits = search("sondage Léger satisfaction gouvernement du Québec",
                  limit=6, tbs="qdr:m")
    for it in hits:
        blob = f"{it.get('title', '')} {it.get('description', '')}"
        val = _extract_sat(blob)
        source = it["url"]
        if val is None:  # le extrait ne suffit pas → scrape de l'article
            md = scrape_markdown(it["url"])
            if md:
                val = _extract_sat(md[:8000])
        if val is not None:
            today = date.today()
            row = db.query(Mo.Indicator).filter_by(name="gov_satisfaction",
                                                   as_of=today).first()
            if row is None:
                row = Mo.Indicator(name="gov_satisfaction", as_of=today, value=val)
                db.add(row)
            row.value = val
            row.source = it.get("title", "")[:120]
            row.source_url = source
            row.method = "firecrawl"
            _add_signal(db, "satisfaction", it.get("title", ""), source,
                        it.get("description"), extra={"value": val})
            db.commit()
            report = {"found": True, "value": val, "source": source}
            break
    return report


def _extract_sat(text: str) -> float | None:
    low = text.lower()
    if "gouvernement" not in low and "legault" not in low and "fréchette" not in low:
        return None
    for pat in SAT_PATTERNS:
        m = re.search(pat, low)
        if m:
            v = float(m.group(1))
            if 10 <= v <= 80:
                return v
    return None


def latest_satisfaction(db: Session) -> tuple[float, dict]:
    """% de satisfaits le plus récent (ingéré), sinon valeur de repli configurée."""
    row = (db.query(Mo.Indicator).filter_by(name="gov_satisfaction")
           .order_by(Mo.Indicator.as_of.desc()).first())
    if row:
        return row.value, {"as_of": row.as_of.isoformat(), "source": row.source,
                           "source_url": row.source_url, "method": row.method}
    return settings.gov_satisfaction_fallback, {
        "method": "fallback-config",
        "note": "aucune mesure ingérée — valeur de repli de configuration"}


# ---------------------------------------------------------------------------
# 3. Presse élargie → moteur de sentiment existant
# ---------------------------------------------------------------------------
def sweep_news(db: Session) -> dict:
    """Articles au-delà des RSS : recherche par parti, scoring par le moteur
    de sentiment existant (mêmes scoreurs lexique/LLM, même provenance)."""
    from .news_rss import cluster_key, detect_entities, lexicon_score, llm_score
    added, scored, budget = 0, 0, settings.firecrawl_max_docs_per_cycle
    for party, q in PARTY_QUERIES.items():
        if added >= budget:
            break
        for it in search(q, limit=6, tbs="qdr:w"):
            url, title = it.get("url"), it.get("title") or ""
            if not url or not title or added >= budget:
                continue
            if db.query(Mo.SentimentDocument).filter_by(url=url).first():
                continue
            summary = (it.get("description") or "")[:2000]
            pub = None
            if it.get("date"):
                try:
                    pub = datetime.fromisoformat(str(it["date"]).replace("Z", "+00:00"))
                except ValueError:
                    pub = None
            doc = Mo.SentimentDocument(
                source=f"Firecrawl · {url.split('/')[2]}", url=url, title=title,
                summary=summary, published=pub, cluster_key=cluster_key(title))
            text = f"{title}. {summary}"
            for ent in detect_entities(text):
                res = llm_score(text, ent)
                method = "llm"
                if res is None:
                    res, method = lexicon_score(text, ent), "lexicon"
                s, stance = res
                doc.scores.append(Mo.SentimentScore(entity=ent, sentiment=s,
                                                    stance=stance, method=method))
                scored += 1
            db.add(doc)
            added += 1
    db.commit()
    return {"documents": added, "scores": scored}


def reconcile_radar(db: Session) -> dict:
    """Boucle de garantie : chaque sondage détecté par le radar doit finir dans
    la base. Une détection avec maison connue est marquée « intégré » quand un
    sondage de cette maison a été ajouté/actualisé autour de la détection;
    sinon elle reste « nouveau » = alerte visible (admin + point du jour)."""
    cutoff = datetime.now(timezone.utc) - timedelta(days=14)
    pending = (db.query(Mo.WebSignal)
               .filter(Mo.WebSignal.kind == "poll-radar",
                       Mo.WebSignal.status == "nouveau",
                       Mo.WebSignal.detected_at >= cutoff,
                       Mo.WebSignal.pollster.isnot(None)).all())
    integrated, waiting = 0, 0
    for sig in pending:
        d0 = sig.detected_at.date()
        match = (db.query(Mo.Poll).join(Mo.Pollster)
                 .filter(Mo.Pollster.name == sig.pollster,
                         Mo.Poll.field_end >= d0 - timedelta(days=10),
                         Mo.Poll.field_end <= d0 + timedelta(days=3)).first())
        if match:
            sig.status = "intégré"
            sig.extra = {**(sig.extra or {}), "poll_id": match.id,
                         "field_end": match.field_end.isoformat()}
            integrated += 1
        else:
            waiting += 1
    db.commit()
    return {"intégrés": integrated, "en_attente": waiting}


# ---------------------------------------------------------------------------
def run_watch(db: Session) -> dict:
    """Cycle complet de veille — chaque mission est isolée (échec toléré)."""
    if not _enabled():
        return {"skipped": "Firecrawl désactivé ou clé absente"}
    report: dict = {}
    for name, fn in [("radar_sondages", radar_polls),
                     ("réconciliation", reconcile_radar),
                     ("satisfaction", fetch_gov_satisfaction),
                     ("presse_elargie", sweep_news)]:
        try:
            report[name] = fn(db)
        except Exception as e:
            db.rollback()
            report[name] = f"échec: {e}"
    return report