# QC Élection Forecast — Plateforme de prévision électorale du Québec 2026
# Auteur : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# https://www.qc-election.com
"""Connecteurs PRIMAIRES des maisons de sondage (directive maître).

Pour Léger, Pallas Data, Synopsis, Liaison Strategies, Angus Reid, SEGMA et
Mainstreet : surveillance de leurs pages de publications publiques →

  1. détection d'un nouveau rapport (lien inédit, mots-clés Québec/intentions);
  2. archivage LÉGAL du document public (markdown Firecrawl, PDF inclus) dans
     `data/raw/reports/` avec provenance complète;
  3. extraction par RÈGLES des toplines nationales (partis + %, dates de
     terrain, n) avec NIVEAU DE CONFIANCE;
  4. **file de révision** : un sondage extrait est créé `excluded=True`
     (« révision requise ») — il n'entre JAMAIS silencieusement dans le modèle;
     l'admin valide via PATCH /api/admin/poll/{id} (excluded=false);
  5. réconciliation : si Wikipédia a déjà le sondage (maison + fin de terrain),
     la détection est marquée « intégré » sans doublon.

Chaque connecteur est tolérant : une page murée = statut de source consigné,
jamais un pipeline cassé. Qc125/338Canada ne servent jamais de source primaire.
"""
from __future__ import annotations

import logging
import re
from datetime import date, datetime, timedelta, timezone

from sqlalchemy.orm import Session

from ..config import DATA_DIR, settings
from .. import models as Mo

log = logging.getLogger("pollster-reports")

REPORTS_DIR = DATA_DIR / "raw" / "reports"

# Pages de publications publiques (vérifiées best-effort; un 404 est consigné).
PRIMARY_SOURCES: dict[str, dict] = {
    "Léger": {"watch": "https://leger360.com/fr/publications/",
              "mode": "web"},
    "Pallas Data": {"watch": "https://pallasdata.com/",
                    "mode": "ivr"},
    "Synopsis": {"watch": "https://www.synopsisrecherche.com/",
                 "mode": "web"},
    "Liaison Strategies": {"watch": "https://liaisonstrategies.ca/",
                           "mode": "ivr"},
    "Angus Reid": {"watch": "https://angusreid.org/?s=quebec",
                   "mode": "web"},
    "SEGMA": {"watch": "https://www.segma.ca/",
              "mode": "phone"},
    "Mainstreet": {"watch": "https://www.mainstreetresearch.ca/polls/",
                   "mode": "ivr"},
}

KEYWORDS = re.compile(r"qu[ée]bec|intention|sondage|provincial|caq|pq\b|élection",
                      re.I)

PARTY_RES = {
    "CAQ": r"(?:CAQ|Coalition\s+avenir)",
    "PLQ": r"(?:PLQ|Parti\s+lib[ée]ral|lib[ée]raux)",
    "PQ": r"(?:PQ\b|Parti\s+[Qq]u[ée]b[ée]cois)",
    "QS": r"(?:QS\b|Qu[ée]bec\s+[Ss]olidaire)",
    "PCQ": r"(?:PCQ|Parti\s+conservateur|conservateurs?\b)",
}
FIELD_RE = re.compile(
    r"(?:du|from|les)\s+(\d{1,2})(?:er)?\s+(?:([a-zéû]+)\s+)?(?:au|to|et)\s+"
    r"(\d{1,2})(?:er)?\s+([a-zéû]+)(?:\s+(\d{4}))?", re.I)
MONTHS = {"janvier": 1, "février": 2, "mars": 3, "avril": 4, "mai": 5,
          "juin": 6, "juillet": 7, "août": 8, "aout": 8, "septembre": 9,
          "octobre": 10, "novembre": 11, "décembre": 12, "decembre": 12}
N_RE = re.compile(r"(?:n\s*=\s*|aupr[èe]s\s+de\s+|échantillon\s+de\s+)"
                  r"([\d\s ,]{3,6})", re.I)


REGIONAL_RE = re.compile(
    r"grande?\s+région|région\s+de\s+(qu[ée]bec|montr[ée]al)|couronne|\b450\b|"
    r"île\s+de\s+montr[ée]al|chez\s+les\s+(francophones|anglophones|jeunes|"
    r"femmes|hommes|a[îi]n[ée]s)|région\s+métropolitaine", re.I)
NATIONAL_RE = re.compile(
    r"échelle\s+nationale|ensemble\s+du\s+qu[ée]bec|à\s+travers\s+le\s+qu[ée]bec|"
    r"intentions\s+de\s+vote|province", re.I)


def extract_toplines(md: str) -> dict:
    """Extraction PAR RÈGLES des intentions NATIONALES d'un rapport ou article.

    Scopée par PHRASE : une phrase contenant un marqueur régional/démographique
    (« grande région de Québec », « chez les jeunes »…) n'alimente JAMAIS les
    toplines nationales — les articles mêlent souvent les deux. Les phrases à
    ancre nationale (« à l'échelle nationale ») sont lues en priorité."""
    reasons, shares = [], {}
    text = md[:20000]
    sentences = re.split(r"(?<=[.!?])\s+|\n{2,}", text)
    natl = [s for s in sentences if NATIONAL_RE.search(s)
            and not REGIONAL_RE.search(s)]
    neutral = [s for s in sentences if not NATIONAL_RE.search(s)
               and not REGIONAL_RE.search(s)]
    for pool in (natl, neutral):
        for s in pool:
            for party, pat in PARTY_RES.items():
                if party in shares:
                    continue
                m = re.search(pat + r"[^%\n]{0,120}?(\d{1,2})\s*%", s)
                if m:
                    v = float(m.group(1))
                    if 1 <= v <= 60:
                        shares[party] = v
        if len([p for p in shares if p != "AUT"]) >= 4:
            break
    if not shares and REGIONAL_RE.search(text):
        reasons.append("contenu possiblement régional/démographique seulement")
    total = sum(shares.values())
    field_end = None
    m = FIELD_RE.search(md[:6000])
    if m:
        month = MONTHS.get(m.group(4).lower())
        year = int(m.group(5)) if m.group(5) else settings.election_date.year
        if month:
            try:
                field_end = date(year, month, int(m.group(3)))
            except ValueError:
                pass
    field_end_estimated = False
    if field_end is None and re.search(
            r"fin de semaine|cette semaine|derniers jours|week-?end", md[:6000], re.I):
        # dates exactes absentes mais terrain très récent affirmé par l'article :
        # estimation ÉTIQUETÉE (sert au recoupement ±2 j, jamais présentée
        # comme une date officielle — la fiche reste en révision)
        field_end = date.today() - timedelta(days=2)
        field_end_estimated = True
    n = None
    mn = N_RE.search(md[:6000])
    if mn:
        digits = re.sub(r"[^\d]", "", mn.group(1))
        if digits and 100 <= int(digits) <= 100000:
            n = int(digits)
    conf = 0.0
    if len(shares) >= 4:
        conf += 0.4
    else:
        reasons.append(f"seulement {len(shares)} partis extraits")
    if 75 <= total <= 105:
        conf += 0.3
    elif shares:
        reasons.append(f"somme implausible ({total:.0f})")
    if field_end and not field_end_estimated:
        conf += 0.2
    elif field_end_estimated:
        conf += 0.1
        reasons.append("fin de terrain ESTIMÉE (article : terrain très récent)")
    else:
        reasons.append("dates de terrain introuvables")
    if n:
        conf += 0.1
    return {"shares": shares, "field_end": field_end, "sample_size": n,
            "field_end_estimated": field_end_estimated,
            "confidence": round(conf, 2), "reasons": reasons}


def _archive(firm: str, url: str, md: str) -> None:
    try:
        import hashlib
        REPORTS_DIR.mkdir(parents=True, exist_ok=True)
        h = hashlib.sha1(url.encode()).hexdigest()[:12]
        safe = re.sub(r"[^a-z0-9]+", "-", firm.lower())
        (REPORTS_DIR / f"{date.today().isoformat()}_{safe}_{h}.md").write_text(
            f"<!-- {url} — archivé {datetime.now(timezone.utc).isoformat()} -->\n{md}")
    except Exception:
        pass


def _source_status(db: Session, firm: str, url: str, status: str) -> None:
    src = db.query(Mo.DataSource).filter_by(name=f"Primaire — {firm}").first()
    if src is None:
        src = Mo.DataSource(name=f"Primaire — {firm}", url=url, kind="poll-primary")
        db.add(src)
    src.last_fetch = datetime.now(timezone.utc)
    src.last_status = status[:200]


def watch_firm(db: Session, firm: str, cfg: dict) -> dict:
    """Un connecteur : page de publications → nouveaux liens → extraction."""
    from .firecrawl_watch import _enabled, _post, scrape_markdown
    if not _enabled():
        return {"skipped": "Firecrawl requis"}
    data = _post("/scrape", {"url": cfg["watch"], "formats": ["links", "markdown"]},
                 timeout=90.0)
    d = (data or {}).get("data") or {}
    links, page_md = d.get("links") or [], d.get("markdown") or ""
    if not links and not page_md:
        _source_status(db, firm, cfg["watch"], "inaccessible")
        return {"statut": "page inaccessible"}
    _source_status(db, firm, cfg["watch"], "ok")
    # candidats : liens de la même maison contenant les mots-clés
    host = cfg["watch"].split("/")[2].replace("www.", "")
    cands = []
    for l in links:
        if host in l and KEYWORDS.search(l) and len(l) < 220 \
                and not any(x in l for x in ("#", "mailto:", "?share", "/tag/",
                                             "/category/", "/page/")):
            cands.append(l.split("?utm")[0])
    cands = list(dict.fromkeys(cands))[:6]
    new, pending, integrated = 0, 0, 0
    for url in cands:
        if db.query(Mo.WebSignal).filter_by(url=url, kind="rapport-primaire").first():
            continue
        md = scrape_markdown(url)
        if not md or not KEYWORDS.search(md[:4000]):
            continue
        _archive(firm, url, md[:250000])
        ext = extract_toplines(md)
        sig = Mo.WebSignal(kind="rapport-primaire", url=url,
                           title=f"{firm} — rapport détecté",
                           pollster=firm, status="nouveau",
                           snippet=md[:400],
                           extra={"extraction": {**ext,
                                  "field_end": ext["field_end"].isoformat()
                                  if ext["field_end"] else None}})
        db.add(sig)
        new += 1
        # réconciliation Wikipédia / création en file de révision
        if ext["field_end"] and ext["confidence"] >= 0.7:
            pollster = db.query(Mo.Pollster).filter_by(name=firm).first()
            exists = (pollster and db.query(Mo.Poll)
                      .filter_by(pollster_id=pollster.id,
                                 field_end=ext["field_end"]).first())
            if exists:
                sig.status = "intégré"
                sig.extra = {**sig.extra, "poll_id": exists.id}
                integrated += 1
            else:
                from ..seed import get_or_create
                el = db.query(Mo.Election).filter_by(is_target=True).first()
                pol, _ = get_or_create(db, Mo.Pollster, name=firm)
                total = sum(ext["shares"].values())
                poll = Mo.Poll(
                    election_id=el.id, pollster_id=pol.id,
                    field_start=ext["field_end"] - timedelta(days=3),
                    field_end=ext["field_end"],
                    sample_size=ext["sample_size"],
                    mode=cfg.get("mode", "unknown"),
                    source_name=f"Rapport primaire {firm} (extraction auto)",
                    source_url=url, excluded=True,
                    exclusion_reason=("révision requise — extraction "
                                      f"automatique (confiance {ext['confidence']})"))
                for party, v in ext["shares"].items():
                    poll.results.append(Mo.PollResult(
                        party=party, raw_value=v,
                        normalized_value=round(v * 100 / total, 3)))
                db.add(poll)
                db.flush()
                sig.status = "en_révision"
                sig.extra = {**sig.extra, "poll_id": poll.id}
                pending += 1
    db.commit()
    return {"liens": len(cands), "nouveaux": new,
            "en_révision": pending, "déjà_intégrés": integrated}


def extract_from_radar(db: Session, max_articles: int = 6) -> dict:
    """Ferme la boucle du radar : les ARTICLES DE PRESSE détectés (poll-radar)
    qui rapportent un sondage absent de la base sont scrapés et leurs toplines
    extraites → sondage en FILE DE RÉVISION (excluded=True). Pendant la
    campagne, la presse publie les chiffres 24-48 h avant Wikipédia."""
    from .firecrawl_watch import scrape_markdown, _enabled
    if not _enabled():
        return {"skipped": "Firecrawl requis"}
    pending = (db.query(Mo.WebSignal)
               .filter(Mo.WebSignal.kind == "poll-radar",
                       Mo.WebSignal.status == "nouveau")
               .order_by(Mo.WebSignal.detected_at.desc())
               .limit(max_articles).all())
    created, skipped = 0, 0
    for sig in pending:
        md = scrape_markdown(sig.url)
        if not md:
            continue
        _archive(sig.pollster or "presse", sig.url, md[:200000])
        ext = extract_toplines(md)
        sig.extra = {**(sig.extra or {}),
                     "extraction": {**ext, "field_end": ext["field_end"].isoformat()
                                    if ext["field_end"] else None}}
        firm = sig.pollster or next(
            (h for h in ("Léger", "Pallas Data", "Synopsis", "Mainstreet",
                         "SEGMA", "Liaison Strategies", "Angus Reid")
             if h.split()[0].lower() in md[:3000].lower()), None)
        if not firm or ext["confidence"] < 0.7 or not ext["field_end"]:
            skipped += 1
            continue
        from ..seed import get_or_create
        pol, _ = get_or_create(db, Mo.Pollster, name=firm)
        if (db.query(Mo.Poll).filter_by(pollster_id=pol.id,
                                        field_end=ext["field_end"]).first()):
            sig.status = "intégré"
            continue
        el = db.query(Mo.Election).filter_by(is_target=True).first()
        total = sum(ext["shares"].values())
        poll = Mo.Poll(
            election_id=el.id, pollster_id=pol.id,
            field_start=ext["field_end"] - timedelta(days=3),
            field_end=ext["field_end"], sample_size=ext["sample_size"],
            mode=PRIMARY_SOURCES.get(firm, {}).get("mode", "unknown"),
            source_name=f"Article de presse (radar) — extraction auto",
            source_url=sig.url, excluded=True,
            exclusion_reason=("révision requise — extrait d'un article de "
                              f"presse (confiance {ext['confidence']})"))
        for party, v in ext["shares"].items():
            poll.results.append(Mo.PollResult(
                party=party, raw_value=v,
                normalized_value=round(v * 100 / total, 3)))
        db.add(poll)
        db.flush()
        sig.status = "en_révision"
        sig.extra = {**sig.extra, "poll_id": poll.id}
        created += 1
    db.commit()
    return {"articles_scannés": len(pending), "en_révision": created,
            "sans_extraction_fiable": skipped}


def cross_validate_pending(db: Session, tol_pp: float = 1.5) -> dict:
    """AUTO-VALIDATION PAR RECOUPEMENT — jamais silencieux, jamais une seule
    source : un sondage en file de révision est validé (excluded=False)
    UNIQUEMENT si une DEUXIÈME extraction indépendante (autre URL, même maison,
    même fin de terrain ±2 j) rapporte les mêmes chiffres à ±tol_pp près.
    La double provenance et la décision sont journalisées."""
    pending = (db.query(Mo.Poll)
               .filter(Mo.Poll.excluded.is_(True),
                       Mo.Poll.exclusion_reason.like("révision requise%")).all())
    validated = 0
    for poll in pending:
        shares = {r.party: r.raw_value for r in poll.results}
        confirmations = []
        for sig in db.query(Mo.WebSignal).filter(
                Mo.WebSignal.kind.in_(("poll-radar", "rapport-primaire"))).all():
            ext = (sig.extra or {}).get("extraction") or {}
            if not ext.get("shares") or sig.url == poll.source_url:
                continue
            fe = ext.get("field_end")
            if not fe or abs((date.fromisoformat(fe) - poll.field_end).days) > 2:
                continue
            diffs = [abs(ext["shares"].get(p, 0) - v)
                     for p, v in shares.items() if p in ext["shares"]]
            if len(diffs) >= 4 and max(diffs) <= tol_pp:
                confirmations.append(sig.url)
        if confirmations:
            poll.excluded = False
            poll.exclusion_reason = None
            poll.methodology_notes = (
                "Auto-validé par recoupement de sources indépendantes : "
                f"{poll.source_url} + {confirmations[0]}")
            db.add(Mo.PipelineLog(step="auto-validation-sondage", status="ok",
                                  message=f"{poll.pollster.name} "
                                          f"{poll.field_end} validé par "
                                          f"{len(confirmations)} recoupement(s)"))
            validated += 1
    db.commit()
    return {"en_attente": len(pending) - validated, "auto_validés": validated}


def run_primary_watch(db: Session, max_firms: int | None = None) -> dict:
    """Cycle des connecteurs primaires — chaque firme isolée."""
    report = {}
    for i, (firm, cfg) in enumerate(PRIMARY_SOURCES.items()):
        if max_firms is not None and i >= max_firms:
            break
        try:
            report[firm] = watch_firm(db, firm, cfg)
        except Exception as e:
            db.rollback()
            report[firm] = f"échec: {e}"
    try:
        report["articles_radar"] = extract_from_radar(db)
    except Exception as e:
        db.rollback()
        report["articles_radar"] = f"échec: {e}"
    try:
        report["recoupement"] = cross_validate_pending(db)
    except Exception as e:
        db.rollback()
        report["recoupement"] = f"échec: {e}"
    return report