# QC Élection Forecast — Plateforme de prévision électorale du Québec 2026
# Auteur : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# https://www.qc-election.com
"""Pouls social CONTINU — sentiment des médias sociaux à chaque cycle (3-6 h).

Deux couches complémentaires, toutes plateformes confondues :

* **Opinion citoyenne (texte)** — ce que les gens ÉCRIVENT :
    - commentaires publics YouTube des vidéos électorales de la semaine
      (acteur Apify maison `qc-social-pulse`, proxy résidentiel CA);
    - Reddit r/Quebec via l'index Google (Serper) — titres + extraits réels;
    - Mastodon (fils de tags publics) et Lemmy.ca (API ouverte).
  Chaque texte passe par le moteur de stance dirigée (négations, attaques) et
  est PONDÉRÉ PAR L'ENGAGEMENT : w = 1 + ln(1 + votes/likes).

* **Enthousiasme (engagement)** — ce que les gens FONT : abonnés et engagement
  des comptes officiels des partis/chef·fe·s (Instagram/Facebook/TikTok via
  l'acteur maison). La littérature montre que l'engagement relatif corrèle
  avec le vote; publié comme indice, jamais injecté dans le forecast.

Anti-bruit : dédoublonnage URL + empreinte de texte, longueur minimale,
plafonds par plateforme et par cycle. Anomalies de volume → événements.
**Poids dans le forecast : nul** (signal affiché, évalué après le 5 octobre).
"""
from __future__ import annotations

import hashlib
import json
import logging
import math
import re
from datetime import date, datetime, timedelta, timezone

import httpx
import numpy as np
from sqlalchemy.orm import Session

from ..config import settings
from .. import models as Mo
from .news_rss import (ENTITY_ALIASES, NEG_WORDS, POS_WORDS, detect_entities,
                       lexicon_score)

log = logging.getLogger("social-pulse")

APIFY_ACTOR = "aI5AYtFfeDn3KAanf"        # qc-social-pulse (compte gorgeous_thistle)
APIFY_RUN = ("https://api.apify.com/v2/acts/{act}/run-sync-get-dataset-items"
             "?token={token}&timeout=240&memory=1024")

# Registres canoniques (ADDENDUM §31/§37) : comptes vérifiés + requêtes
# versionnées — chargés depuis config/, jamais inventés dans le code.
from ..config import BASE_DIR

_SEEDS_PATH = BASE_DIR / "config" / "social_seeds.json"
_QUERIES_PATH = BASE_DIR / "config" / "social_queries.json"


def _load_registry() -> tuple[dict, dict]:
    """(SOCIAL_TARGETS par plateforme, YT_QUERIES par parti) depuis les
    registres. Un compte `pending_review` reste collecté pour l'ENGAGEMENT
    seulement (owned) — jamais comme opinion; un compte absent est ignoré."""
    targets: dict[str, list] = {"instagram": [], "facebook": [], "tiktok": []}
    yt: dict[str, str] = {}
    try:
        seeds = json.loads(_SEEDS_PATH.read_text())
        for code, party in (seeds.get("parties") or {}).items():
            for group in ("accounts", "leader_accounts"):
                for plat, acc in (party.get(group) or {}).items():
                    if plat in targets and acc.get("handle"):
                        targets[plat].append((code, acc["handle"]))
    except Exception:
        log.warning("social_seeds.json illisible — cibles vides")
    try:
        q = json.loads(_QUERIES_PATH.read_text())
        yt = {code: v["youtube_query"]
              for code, v in (q.get("parties") or {}).items()
              if v.get("youtube_query")}
    except Exception:
        log.warning("social_queries.json illisible — requêtes YouTube vides")
    return targets, yt


SOCIAL_TARGETS, YT_QUERIES = _load_registry()

MIN_TEXT_LEN = 25
MAX_PER_PLATFORM = 80          # plafond de nouveaux posts par plateforme/cycle
SERPER_URL = "https://google.serper.dev/search"
MASTODON_TAGS = ["quebec", "polqc", "assnat"]
MASTODON_HOSTS = ["https://mastodon.social", "https://mstdn.ca"]
LEMMY_SEARCH = ("https://lemmy.ca/api/v3/search?q={q}&type_=Posts"
                "&sort=New&listing_type=All&limit=15")


def _th(text: str) -> str:
    toks = sorted(re.findall(r"[a-zà-ü0-9]{3,}", text.lower()))[:24]
    return hashlib.sha1(" ".join(toks).encode()).hexdigest()[:16]


def _score_text(text: str, fallback_party: str | None = None) -> dict:
    """Stance dirigée par entité détectée; repli sur le parti du contexte
    (ex. commentaire sous une vidéo « CAQ » qui ne nomme aucun parti)."""
    detected = detect_entities(text)
    entities = detected or ([fallback_party] if fallback_party else [])
    scores = {}
    for ent in entities:
        if ent not in ENTITY_ALIASES:
            continue
        if ent in detected:
            s, stance = lexicon_score(text, ent)
        else:
            # entité de contexte (ex. commentaire sous une vidéo « CAQ » qui ne
            # nomme personne) : tonalité générale du texte, dirigée vers elle
            toks = re.findall(r"[a-zà-ü']+", text.lower())
            pos = sum(1 for t in toks if t in POS_WORDS)
            neg = sum(1 for t in toks if t in NEG_WORDS)
            tot = pos + neg
            s = float(np.clip((pos - neg) / tot, -1, 1)) if tot else 0.0
            stance = ("pro" if s > 0.25 else "anti" if s < -0.25
                      else "neutre" if abs(s) < 0.1 else "ambigu")
        scores[ent] = {"sentiment": round(s, 3), "stance": stance,
                       "method": "lexicon"}
    return scores


def _add_post(db: Session, *, platform: str, url: str, text: str,
              engagement: float, created_at: datetime | None,
              community: str | None, fallback_party: str | None,
              query: str | None, seen_hashes: set) -> bool:
    text = (text or "").strip()
    if len(text) < MIN_TEXT_LEN or not url:
        return False
    th = _th(text)
    if th in seen_hashes or f"u:{url}" in seen_hashes:
        return False
    if db.query(Mo.SocialPost).filter_by(url=url).first():
        return False
    if db.query(Mo.SocialPost).filter_by(text_hash=th).first():
        return False
    scores = _score_text(text, fallback_party)
    if not scores:
        return False
    seen_hashes.add(th)
    seen_hashes.add(f"u:{url}")
    db.add(Mo.SocialPost(platform=platform, community=community, url=url,
                         author_hash=None, text=text[:800], text_hash=th,
                         engagement=float(engagement or 0), scores=scores,
                         created_at=created_at, query=query))
    db.flush()
    return True


# ---------------------------------------------------------------------------
# Sources
# ---------------------------------------------------------------------------
def collect_apify(db: Session, seen: set) -> dict:
    """Acteur maison qc-social-pulse : YouTube (commentaires publics) +
    Instagram/Facebook/TikTok (engagement des comptes officiels)."""
    if not settings.apify_token:
        return {"skipped": "APIFY_TOKEN absent"}
    tasks = [{"platform": "youtube", "party": p, "query": q, "limit": 2}
             for p, q in YT_QUERIES.items()]
    for platform, pairs in SOCIAL_TARGETS.items():
        tasks += [{"platform": platform, "party": p, "target": t}
                  for p, t in pairs]
    import time
    t0 = time.monotonic()
    run_log = Mo.ApifyRun(actor_id=APIFY_ACTOR, actor_name="qc-social-pulse",
                          status="error")
    try:
        r = httpx.post(APIFY_RUN.format(act=APIFY_ACTOR,
                                        token=settings.apify_token),
                       json={"tasks": tasks, "maxCommentsPerVideo": 12},
                       timeout=280)
        r.raise_for_status()
        items = r.json()
        run_log.status = "ok"
        run_log.items_collected = len(items)
        run_log.items_valid = sum(1 for i in items if i.get("found"))
    except Exception as e:
        run_log.error = str(e)[:500]
        run_log.runtime_s = round(time.monotonic() - t0, 1)
        db.add(run_log); db.commit()
        return {"échec": str(e)[:200]}
    n_opinion = 0
    engagement: dict[str, dict] = {}
    for it in items:
        if not it.get("found"):
            continue
        party, kind, plat = it.get("party"), it.get("kind"), it["platform"]
        if kind == "comment":
            url = f"{it['url']}#c{_th(it.get('text') or '')[:8]}"
            if _add_post(db, platform="youtube", url=url,
                         text=it.get("text") or "",
                         engagement=it.get("engagement") or 0,
                         created_at=None, community=(it.get("meta") or {}
                                                     ).get("video_title"),
                         fallback_party=party, query=it.get("query"),
                         seen_hashes=seen):
                n_opinion += 1
        elif kind in ("post", "profile"):
            e = engagement.setdefault(party, {"followers": 0, "post_eng": [],
                                              "platforms": set()})
            meta = it.get("meta") or {}
            if kind == "profile" and meta.get("followers"):
                e["followers"] += int(meta["followers"])
                e["platforms"].add(plat)
            if kind == "post":
                e["post_eng"].append(float(it.get("engagement") or 0))
    # indice d'enthousiasme quotidien par parti (Indicator, provenance actor)
    today = date.today()
    for party, e in engagement.items():
        if not e["followers"] and not e["post_eng"]:
            continue
        avg_eng = float(np.mean(e["post_eng"])) if e["post_eng"] else 0.0
        row = (db.query(Mo.Indicator)
               .filter_by(name=f"social_engagement_{party}", as_of=today).first())
        if row is None:
            row = Mo.Indicator(name=f"social_engagement_{party}", as_of=today,
                               value=0.0)
            db.add(row)
        row.value = round(avg_eng, 1)
        row.source = "Acteur Apify qc-social-pulse (comptes officiels)"
        row.method = "apify-actor"
        row.extra = {"followers_total": e["followers"],
                     "avg_post_engagement": round(avg_eng, 1),
                     "n_posts": len(e["post_eng"]),
                     "platforms": sorted(e["platforms"])}
    run_log.items_stored = n_opinion
    run_log.runtime_s = round(time.monotonic() - t0, 1)
    run_log.detail = {"partis_engagement": sorted(engagement)}
    db.add(run_log)
    db.commit()
    return {"items_acteur": len(items), "commentaires_opinion": n_opinion,
            "partis_engagement": sorted(engagement)}


def collect_reddit_serper(db: Session, seen: set) -> dict:
    """Reddit r/Quebec via l'index Google (Serper) : titres + extraits réels."""
    if not settings.serper_api_key:
        return {"skipped": "SERPER_API_KEY absent"}
    added = 0
    for party in ENTITY_ALIASES:
        try:
            r = httpx.post(SERPER_URL, timeout=20,
                           headers={"X-API-KEY": settings.serper_api_key,
                                    "Content-Type": "application/json"},
                           json={"q": f"site:reddit.com/r/Quebec {party}",
                                 "tbs": "qdr:d", "gl": "ca", "hl": "fr",
                                 "num": 10})
            r.raise_for_status()
        except Exception as e:
            log.warning("serper reddit %s: %s", party, e)
            continue
        for res in (r.json().get("organic") or []):
            text = f"{res.get('title', '')}. {res.get('snippet', '')}"
            if _add_post(db, platform="reddit", url=res.get("link", ""),
                         text=text, engagement=1.0, created_at=None,
                         community="r/Quebec", fallback_party=party,
                         query=party, seen_hashes=seen):
                added += 1
    db.commit()
    return {"posts": added}


def collect_mastodon(db: Session, seen: set) -> dict:
    added = 0
    for host in MASTODON_HOSTS:
        for tag in MASTODON_TAGS:
            try:
                r = httpx.get(f"{host}/api/v1/timelines/tag/{tag}?limit=40",
                              timeout=15)
                r.raise_for_status()
            except Exception:
                continue
            for st in r.json():
                text = re.sub(r"<[^>]+>", " ", st.get("content") or "")
                if not detect_entities(text):
                    continue
                eng = (st.get("favourites_count") or 0) + \
                      (st.get("reblogs_count") or 0)
                created = None
                try:
                    created = datetime.fromisoformat(
                        st["created_at"].replace("Z", "+00:00"))
                except (KeyError, ValueError):
                    pass
                if _add_post(db, platform="mastodon", url=st.get("url") or "",
                             text=text, engagement=eng, created_at=created,
                             community=f"#{tag}", fallback_party=None,
                             query=tag, seen_hashes=seen):
                    added += 1
    db.commit()
    return {"posts": added}


def collect_lemmy(db: Session, seen: set) -> dict:
    added = 0
    for party in ENTITY_ALIASES:
        try:
            r = httpx.get(LEMMY_SEARCH.format(q=party), timeout=15,
                          headers={"User-Agent": "qc-election/2.1"})
            r.raise_for_status()
        except Exception:
            continue
        for p in (r.json().get("posts") or []):
            post, counts = p.get("post") or {}, p.get("counts") or {}
            text = f"{post.get('name', '')}. {(post.get('body') or '')[:400]}"
            created = None
            try:
                created = datetime.fromisoformat(
                    (post.get("published") or "").replace("Z", "+00:00"))
            except ValueError:
                pass
            if _add_post(db, platform="lemmy", url=post.get("ap_id") or "",
                         text=text, engagement=counts.get("score") or 0,
                         created_at=created,
                         community=(p.get("community") or {}).get("name"),
                         fallback_party=party, query=party, seen_hashes=seen):
                added += 1
    db.commit()
    return {"posts": added}


# ---------------------------------------------------------------------------
def detect_social_anomalies(db: Session, z_threshold: float = 2.5) -> int:
    """Pic de volume social par parti (z vs 14 jours) → événement."""
    today = date.today()
    created = 0
    for party in ENTITY_ALIASES:
        counts = []
        for back in range(15):
            d0 = datetime(today.year, today.month, today.day,
                          tzinfo=timezone.utc) - timedelta(days=back)
            n = 0
            for post in (db.query(Mo.SocialPost)
                         .filter(Mo.SocialPost.fetched_at >= d0,
                                 Mo.SocialPost.fetched_at < d0 + timedelta(days=1))
                         .all()):
                if party in (post.scores or {}):
                    n += 1
            counts.append(n)
        base = counts[1:]
        if len(base) < 7 or np.std(base) == 0:
            continue
        z = (counts[0] - np.mean(base)) / np.std(base)
        if z >= z_threshold and counts[0] >= 8:
            if not (db.query(Mo.NewsEvent)
                    .filter_by(event_date=today, kind="anomalie-sociale").count()):
                db.add(Mo.NewsEvent(
                    event_date=today, kind="anomalie-sociale",
                    title=f"Pic de conversation sociale — {settings.party_names[party]}",
                    description=f"Volume {counts[0]} vs moyenne {np.mean(base):.1f} (z={z:.1f})",
                    parties=[party], importance=min(0.9, 0.4 + z / 10),
                    detected_by="social-pulse"))
                created += 1
    db.commit()
    return created


def run_pulse(db: Session) -> dict:
    """Cycle complet du pouls social — chaque source est isolée."""
    if not settings.social_pulse_enabled:
        return {"skipped": "pouls social désactivé"}
    seen: set = set()
    report: dict = {}
    for name, fn in [("acteur_apify", collect_apify),
                     ("reddit", collect_reddit_serper),
                     ("mastodon", collect_mastodon),
                     ("lemmy", collect_lemmy)]:
        try:
            report[name] = fn(db, seen)
        except Exception as e:
            db.rollback()
            report[name] = f"échec: {e}"
    try:
        report["anomalies"] = detect_social_anomalies(db)
    except Exception as e:
        report["anomalies"] = f"échec: {e}"
    return report


# ---------------------------------------------------------------------------
def pulse_summary(db: Session, days: int = 14) -> dict:
    """Indice « Pouls social » par parti : volume, sentiment pondéré par
    l'engagement, part de voix, tendance, verbatims les plus engagés."""
    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
    posts = (db.query(Mo.SocialPost)
             .filter(Mo.SocialPost.fetched_at >= cutoff).all())
    parties = [p for p in settings.parties if p != "AUT"]
    out = {p: {"volume": 0, "wsum": 0.0, "wtot": 0.0, "stances": {},
               "timeline": {}, "recent": []} for p in parties}
    platforms: dict[str, int] = {}
    for post in posts:
        w = 1.0 + math.log1p(max(0.0, post.engagement))
        d = (post.created_at or post.fetched_at).date().isoformat()
        platforms[post.platform] = platforms.get(post.platform, 0) + 1
        for party, sc in (post.scores or {}).items():
            if party not in out:
                continue
            o = out[party]
            o["volume"] += 1
            o["wsum"] += sc["sentiment"] * w
            o["wtot"] += w
            o["stances"][sc["stance"]] = o["stances"].get(sc["stance"], 0) + 1
            tl = o["timeline"].setdefault(d, [0.0, 0.0])
            tl[0] += sc["sentiment"] * w
            tl[1] += w
            o["recent"].append((w, post))
    total_volume = sum(o["volume"] for o in out.values()) or 1
    result = {}
    for p in parties:
        o = out[p]
        top = sorted(o["recent"], key=lambda t: -t[0])[:3]
        result[p] = {
            "volume": o["volume"],
            "part_de_voix": round(o["volume"] / total_volume * 100, 1),
            "sentiment_pondere": (round(o["wsum"] / o["wtot"], 3)
                                  if o["wtot"] else None),
            "stances": o["stances"],
            "timeline": [{"date": d, "sentiment": round(v[0] / v[1], 3),
                          "poids": round(v[1], 1)}
                         for d, v in sorted(o["timeline"].items())],
            "verbatims": [{"platform": post.platform, "url": post.url,
                           "engagement": post.engagement,
                           "text": post.text[:220],
                           "stance": (post.scores.get(p) or {}).get("stance")}
                          for _, post in top],
        }
    # enthousiasme (engagement des comptes officiels, dernier point)
    enthusiasm = {}
    for p in parties:
        row = (db.query(Mo.Indicator)
               .filter_by(name=f"social_engagement_{p}")
               .order_by(Mo.Indicator.as_of.desc()).first())
        if row:
            enthusiasm[p] = {"as_of": row.as_of.isoformat(),
                             "avg_post_engagement": row.value, **(row.extra or {})}
    return {"note": ("Pouls social — poids NUL dans le forecast. Sentiment "
                     "pondéré par l'engagement (1+ln(1+votes)); sources : "
                     "commentaires YouTube (acteur maison), Reddit r/Quebec "
                     "(index Google), Mastodon, Lemmy; enthousiasme : comptes "
                     "officiels via acteur Apify maison."),
            "posts_total": len(posts), "platforms": platforms,
            "parties": result, "enthusiasm": enthusiasm,
            "days": days}