spb/qc-election
Public
Python 66.6%
HTML 24.8%
CSS 4.9%
JavaScript 3.6%
1# QC Élection Forecast — Plateforme de prévision électorale du Québec 20262# Auteur : Simon-Pierre Boucher3# Contact : contact@spboucher.ai4# https://www.qc-election.com5"""Pouls social CONTINU — sentiment des médias sociaux à chaque cycle (3-6 h).67Deux couches complémentaires, toutes plateformes confondues :89* **Opinion citoyenne (texte)** — ce que les gens ÉCRIVENT :10 - commentaires publics YouTube des vidéos électorales de la semaine11 (acteur Apify maison `qc-social-pulse`, proxy résidentiel CA);12 - Reddit r/Quebec via l'index Google (Serper) — titres + extraits réels;13 - Mastodon (fils de tags publics) et Lemmy.ca (API ouverte).14 Chaque texte passe par le moteur de stance dirigée (négations, attaques) et15 est PONDÉRÉ PAR L'ENGAGEMENT : w = 1 + ln(1 + votes/likes).1617* **Enthousiasme (engagement)** — ce que les gens FONT : abonnés et engagement18 des comptes officiels des partis/chef·fe·s (Instagram/Facebook/TikTok via19 l'acteur maison). La littérature montre que l'engagement relatif corrèle20 avec le vote; publié comme indice, jamais injecté dans le forecast.2122Anti-bruit : dédoublonnage URL + empreinte de texte, longueur minimale,23plafonds par plateforme et par cycle. Anomalies de volume → événements.24**Poids dans le forecast : nul** (signal affiché, évalué après le 5 octobre).25"""26from __future__ import annotations2728import hashlib29import json30import logging31import math32import re33from datetime import date, datetime, timedelta, timezone3435import httpx36import numpy as np37from sqlalchemy.orm import Session3839from ..config import settings40from .. import models as Mo41from .news_rss import (ENTITY_ALIASES, NEG_WORDS, POS_WORDS, detect_entities,42 lexicon_score)4344log = logging.getLogger("social-pulse")4546APIFY_ACTOR = "aI5AYtFfeDn3KAanf" # qc-social-pulse (compte gorgeous_thistle)47APIFY_RUN = ("https://api.apify.com/v2/acts/{act}/run-sync-get-dataset-items"48 "?token={token}&timeout=240&memory=1024")4950# Registres canoniques (ADDENDUM §31/§37) : comptes vérifiés + requêtes51# versionnées — chargés depuis config/, jamais inventés dans le code.52from ..config import BASE_DIR5354_SEEDS_PATH = BASE_DIR / "config" / "social_seeds.json"55_QUERIES_PATH = BASE_DIR / "config" / "social_queries.json"565758def _load_registry() -> tuple[dict, dict]:59 """(SOCIAL_TARGETS par plateforme, YT_QUERIES par parti) depuis les60 registres. Un compte `pending_review` reste collecté pour l'ENGAGEMENT61 seulement (owned) — jamais comme opinion; un compte absent est ignoré."""62 targets: dict[str, list] = {"instagram": [], "facebook": [], "tiktok": []}63 yt: dict[str, str] = {}64 try:65 seeds = json.loads(_SEEDS_PATH.read_text())66 for code, party in (seeds.get("parties") or {}).items():67 for group in ("accounts", "leader_accounts"):68 for plat, acc in (party.get(group) or {}).items():69 if plat in targets and acc.get("handle"):70 targets[plat].append((code, acc["handle"]))71 except Exception:72 log.warning("social_seeds.json illisible — cibles vides")73 try:74 q = json.loads(_QUERIES_PATH.read_text())75 yt = {code: v["youtube_query"]76 for code, v in (q.get("parties") or {}).items()77 if v.get("youtube_query")}78 except Exception:79 log.warning("social_queries.json illisible — requêtes YouTube vides")80 return targets, yt818283SOCIAL_TARGETS, YT_QUERIES = _load_registry()8485MIN_TEXT_LEN = 2586MAX_PER_PLATFORM = 80 # plafond de nouveaux posts par plateforme/cycle87SERPER_URL = "https://google.serper.dev/search"88MASTODON_TAGS = ["quebec", "polqc", "assnat"]89MASTODON_HOSTS = ["https://mastodon.social", "https://mstdn.ca"]90LEMMY_SEARCH = ("https://lemmy.ca/api/v3/search?q={q}&type_=Posts"91 "&sort=New&listing_type=All&limit=15")929394def _th(text: str) -> str:95 toks = sorted(re.findall(r"[a-zà-ü0-9]{3,}", text.lower()))[:24]96 return hashlib.sha1(" ".join(toks).encode()).hexdigest()[:16]979899def _score_text(text: str, fallback_party: str | None = None) -> dict:100 """Stance dirigée par entité détectée; repli sur le parti du contexte101 (ex. commentaire sous une vidéo « CAQ » qui ne nomme aucun parti)."""102 detected = detect_entities(text)103 entities = detected or ([fallback_party] if fallback_party else [])104 scores = {}105 for ent in entities:106 if ent not in ENTITY_ALIASES:107 continue108 if ent in detected:109 s, stance = lexicon_score(text, ent)110 else:111 # entité de contexte (ex. commentaire sous une vidéo « CAQ » qui ne112 # nomme personne) : tonalité générale du texte, dirigée vers elle113 toks = re.findall(r"[a-zà-ü']+", text.lower())114 pos = sum(1 for t in toks if t in POS_WORDS)115 neg = sum(1 for t in toks if t in NEG_WORDS)116 tot = pos + neg117 s = float(np.clip((pos - neg) / tot, -1, 1)) if tot else 0.0118 stance = ("pro" if s > 0.25 else "anti" if s < -0.25119 else "neutre" if abs(s) < 0.1 else "ambigu")120 scores[ent] = {"sentiment": round(s, 3), "stance": stance,121 "method": "lexicon"}122 return scores123124125def _add_post(db: Session, *, platform: str, url: str, text: str,126 engagement: float, created_at: datetime | None,127 community: str | None, fallback_party: str | None,128 query: str | None, seen_hashes: set) -> bool:129 text = (text or "").strip()130 if len(text) < MIN_TEXT_LEN or not url:131 return False132 th = _th(text)133 if th in seen_hashes or f"u:{url}" in seen_hashes:134 return False135 if db.query(Mo.SocialPost).filter_by(url=url).first():136 return False137 if db.query(Mo.SocialPost).filter_by(text_hash=th).first():138 return False139 scores = _score_text(text, fallback_party)140 if not scores:141 return False142 seen_hashes.add(th)143 seen_hashes.add(f"u:{url}")144 db.add(Mo.SocialPost(platform=platform, community=community, url=url,145 author_hash=None, text=text[:800], text_hash=th,146 engagement=float(engagement or 0), scores=scores,147 created_at=created_at, query=query))148 db.flush()149 return True150151152# ---------------------------------------------------------------------------153# Sources154# ---------------------------------------------------------------------------155def collect_apify(db: Session, seen: set) -> dict:156 """Acteur maison qc-social-pulse : YouTube (commentaires publics) +157 Instagram/Facebook/TikTok (engagement des comptes officiels)."""158 if not settings.apify_token:159 return {"skipped": "APIFY_TOKEN absent"}160 tasks = [{"platform": "youtube", "party": p, "query": q, "limit": 2}161 for p, q in YT_QUERIES.items()]162 for platform, pairs in SOCIAL_TARGETS.items():163 tasks += [{"platform": platform, "party": p, "target": t}164 for p, t in pairs]165 import time166 t0 = time.monotonic()167 run_log = Mo.ApifyRun(actor_id=APIFY_ACTOR, actor_name="qc-social-pulse",168 status="error")169 try:170 r = httpx.post(APIFY_RUN.format(act=APIFY_ACTOR,171 token=settings.apify_token),172 json={"tasks": tasks, "maxCommentsPerVideo": 12},173 timeout=280)174 r.raise_for_status()175 items = r.json()176 run_log.status = "ok"177 run_log.items_collected = len(items)178 run_log.items_valid = sum(1 for i in items if i.get("found"))179 except Exception as e:180 run_log.error = str(e)[:500]181 run_log.runtime_s = round(time.monotonic() - t0, 1)182 db.add(run_log); db.commit()183 return {"échec": str(e)[:200]}184 n_opinion = 0185 engagement: dict[str, dict] = {}186 for it in items:187 if not it.get("found"):188 continue189 party, kind, plat = it.get("party"), it.get("kind"), it["platform"]190 if kind == "comment":191 url = f"{it['url']}#c{_th(it.get('text') or '')[:8]}"192 if _add_post(db, platform="youtube", url=url,193 text=it.get("text") or "",194 engagement=it.get("engagement") or 0,195 created_at=None, community=(it.get("meta") or {}196 ).get("video_title"),197 fallback_party=party, query=it.get("query"),198 seen_hashes=seen):199 n_opinion += 1200 elif kind in ("post", "profile"):201 e = engagement.setdefault(party, {"followers": 0, "post_eng": [],202 "platforms": set()})203 meta = it.get("meta") or {}204 if kind == "profile" and meta.get("followers"):205 e["followers"] += int(meta["followers"])206 e["platforms"].add(plat)207 if kind == "post":208 e["post_eng"].append(float(it.get("engagement") or 0))209 # indice d'enthousiasme quotidien par parti (Indicator, provenance actor)210 today = date.today()211 for party, e in engagement.items():212 if not e["followers"] and not e["post_eng"]:213 continue214 avg_eng = float(np.mean(e["post_eng"])) if e["post_eng"] else 0.0215 row = (db.query(Mo.Indicator)216 .filter_by(name=f"social_engagement_{party}", as_of=today).first())217 if row is None:218 row = Mo.Indicator(name=f"social_engagement_{party}", as_of=today,219 value=0.0)220 db.add(row)221 row.value = round(avg_eng, 1)222 row.source = "Acteur Apify qc-social-pulse (comptes officiels)"223 row.method = "apify-actor"224 row.extra = {"followers_total": e["followers"],225 "avg_post_engagement": round(avg_eng, 1),226 "n_posts": len(e["post_eng"]),227 "platforms": sorted(e["platforms"])}228 run_log.items_stored = n_opinion229 run_log.runtime_s = round(time.monotonic() - t0, 1)230 run_log.detail = {"partis_engagement": sorted(engagement)}231 db.add(run_log)232 db.commit()233 return {"items_acteur": len(items), "commentaires_opinion": n_opinion,234 "partis_engagement": sorted(engagement)}235236237def collect_reddit_serper(db: Session, seen: set) -> dict:238 """Reddit r/Quebec via l'index Google (Serper) : titres + extraits réels."""239 if not settings.serper_api_key:240 return {"skipped": "SERPER_API_KEY absent"}241 added = 0242 for party in ENTITY_ALIASES:243 try:244 r = httpx.post(SERPER_URL, timeout=20,245 headers={"X-API-KEY": settings.serper_api_key,246 "Content-Type": "application/json"},247 json={"q": f"site:reddit.com/r/Quebec {party}",248 "tbs": "qdr:d", "gl": "ca", "hl": "fr",249 "num": 10})250 r.raise_for_status()251 except Exception as e:252 log.warning("serper reddit %s: %s", party, e)253 continue254 for res in (r.json().get("organic") or []):255 text = f"{res.get('title', '')}. {res.get('snippet', '')}"256 if _add_post(db, platform="reddit", url=res.get("link", ""),257 text=text, engagement=1.0, created_at=None,258 community="r/Quebec", fallback_party=party,259 query=party, seen_hashes=seen):260 added += 1261 db.commit()262 return {"posts": added}263264265def collect_mastodon(db: Session, seen: set) -> dict:266 added = 0267 for host in MASTODON_HOSTS:268 for tag in MASTODON_TAGS:269 try:270 r = httpx.get(f"{host}/api/v1/timelines/tag/{tag}?limit=40",271 timeout=15)272 r.raise_for_status()273 except Exception:274 continue275 for st in r.json():276 text = re.sub(r"<[^>]+>", " ", st.get("content") or "")277 if not detect_entities(text):278 continue279 eng = (st.get("favourites_count") or 0) + \280 (st.get("reblogs_count") or 0)281 created = None282 try:283 created = datetime.fromisoformat(284 st["created_at"].replace("Z", "+00:00"))285 except (KeyError, ValueError):286 pass287 if _add_post(db, platform="mastodon", url=st.get("url") or "",288 text=text, engagement=eng, created_at=created,289 community=f"#{tag}", fallback_party=None,290 query=tag, seen_hashes=seen):291 added += 1292 db.commit()293 return {"posts": added}294295296def collect_lemmy(db: Session, seen: set) -> dict:297 added = 0298 for party in ENTITY_ALIASES:299 try:300 r = httpx.get(LEMMY_SEARCH.format(q=party), timeout=15,301 headers={"User-Agent": "qc-election/2.1"})302 r.raise_for_status()303 except Exception:304 continue305 for p in (r.json().get("posts") or []):306 post, counts = p.get("post") or {}, p.get("counts") or {}307 text = f"{post.get('name', '')}. {(post.get('body') or '')[:400]}"308 created = None309 try:310 created = datetime.fromisoformat(311 (post.get("published") or "").replace("Z", "+00:00"))312 except ValueError:313 pass314 if _add_post(db, platform="lemmy", url=post.get("ap_id") or "",315 text=text, engagement=counts.get("score") or 0,316 created_at=created,317 community=(p.get("community") or {}).get("name"),318 fallback_party=party, query=party, seen_hashes=seen):319 added += 1320 db.commit()321 return {"posts": added}322323324# ---------------------------------------------------------------------------325def detect_social_anomalies(db: Session, z_threshold: float = 2.5) -> int:326 """Pic de volume social par parti (z vs 14 jours) → événement."""327 today = date.today()328 created = 0329 for party in ENTITY_ALIASES:330 counts = []331 for back in range(15):332 d0 = datetime(today.year, today.month, today.day,333 tzinfo=timezone.utc) - timedelta(days=back)334 n = 0335 for post in (db.query(Mo.SocialPost)336 .filter(Mo.SocialPost.fetched_at >= d0,337 Mo.SocialPost.fetched_at < d0 + timedelta(days=1))338 .all()):339 if party in (post.scores or {}):340 n += 1341 counts.append(n)342 base = counts[1:]343 if len(base) < 7 or np.std(base) == 0:344 continue345 z = (counts[0] - np.mean(base)) / np.std(base)346 if z >= z_threshold and counts[0] >= 8:347 if not (db.query(Mo.NewsEvent)348 .filter_by(event_date=today, kind="anomalie-sociale").count()):349 db.add(Mo.NewsEvent(350 event_date=today, kind="anomalie-sociale",351 title=f"Pic de conversation sociale — {settings.party_names[party]}",352 description=f"Volume {counts[0]} vs moyenne {np.mean(base):.1f} (z={z:.1f})",353 parties=[party], importance=min(0.9, 0.4 + z / 10),354 detected_by="social-pulse"))355 created += 1356 db.commit()357 return created358359360def run_pulse(db: Session) -> dict:361 """Cycle complet du pouls social — chaque source est isolée."""362 if not settings.social_pulse_enabled:363 return {"skipped": "pouls social désactivé"}364 seen: set = set()365 report: dict = {}366 for name, fn in [("acteur_apify", collect_apify),367 ("reddit", collect_reddit_serper),368 ("mastodon", collect_mastodon),369 ("lemmy", collect_lemmy)]:370 try:371 report[name] = fn(db, seen)372 except Exception as e:373 db.rollback()374 report[name] = f"échec: {e}"375 try:376 report["anomalies"] = detect_social_anomalies(db)377 except Exception as e:378 report["anomalies"] = f"échec: {e}"379 return report380381382# ---------------------------------------------------------------------------383def pulse_summary(db: Session, days: int = 14) -> dict:384 """Indice « Pouls social » par parti : volume, sentiment pondéré par385 l'engagement, part de voix, tendance, verbatims les plus engagés."""386 cutoff = datetime.now(timezone.utc) - timedelta(days=days)387 posts = (db.query(Mo.SocialPost)388 .filter(Mo.SocialPost.fetched_at >= cutoff).all())389 parties = [p for p in settings.parties if p != "AUT"]390 out = {p: {"volume": 0, "wsum": 0.0, "wtot": 0.0, "stances": {},391 "timeline": {}, "recent": []} for p in parties}392 platforms: dict[str, int] = {}393 for post in posts:394 w = 1.0 + math.log1p(max(0.0, post.engagement))395 d = (post.created_at or post.fetched_at).date().isoformat()396 platforms[post.platform] = platforms.get(post.platform, 0) + 1397 for party, sc in (post.scores or {}).items():398 if party not in out:399 continue400 o = out[party]401 o["volume"] += 1402 o["wsum"] += sc["sentiment"] * w403 o["wtot"] += w404 o["stances"][sc["stance"]] = o["stances"].get(sc["stance"], 0) + 1405 tl = o["timeline"].setdefault(d, [0.0, 0.0])406 tl[0] += sc["sentiment"] * w407 tl[1] += w408 o["recent"].append((w, post))409 total_volume = sum(o["volume"] for o in out.values()) or 1410 result = {}411 for p in parties:412 o = out[p]413 top = sorted(o["recent"], key=lambda t: -t[0])[:3]414 result[p] = {415 "volume": o["volume"],416 "part_de_voix": round(o["volume"] / total_volume * 100, 1),417 "sentiment_pondere": (round(o["wsum"] / o["wtot"], 3)418 if o["wtot"] else None),419 "stances": o["stances"],420 "timeline": [{"date": d, "sentiment": round(v[0] / v[1], 3),421 "poids": round(v[1], 1)}422 for d, v in sorted(o["timeline"].items())],423 "verbatims": [{"platform": post.platform, "url": post.url,424 "engagement": post.engagement,425 "text": post.text[:220],426 "stance": (post.scores.get(p) or {}).get("stance")}427 for _, post in top],428 }429 # enthousiasme (engagement des comptes officiels, dernier point)430 enthusiasm = {}431 for p in parties:432 row = (db.query(Mo.Indicator)433 .filter_by(name=f"social_engagement_{p}")434 .order_by(Mo.Indicator.as_of.desc()).first())435 if row:436 enthusiasm[p] = {"as_of": row.as_of.isoformat(),437 "avg_post_engagement": row.value, **(row.extra or {})}438 return {"note": ("Pouls social — poids NUL dans le forecast. Sentiment "439 "pondéré par l'engagement (1+ln(1+votes)); sources : "440 "commentaires YouTube (acteur maison), Reddit r/Quebec "441 "(index Google), Mastodon, Lemmy; enthousiasme : comptes "442 "officiels via acteur Apify maison."),443 "posts_total": len(posts), "platforms": platforms,444 "parties": result, "enthusiasm": enthusiasm,445 "days": days}446