spb/qc-election
Public
Python 66.6%
HTML 24.8%
CSS 4.9%
JavaScript 3.6%
1# QC Élection Forecast — Plateforme de prévision électorale du Québec 20262# Auteur : Simon-Pierre Boucher3# Contact : contact@spboucher.ai4# https://www.qc-election.com5"""Veille web CONTINUE via Firecrawl (v2 « au-delà des sondages »).67Trois missions, exécutées à chaque cycle du pipeline (budget d'appels borné) :891. **Radar sondages** — recherche d'actualité ciblée : tout nouvel article10 mentionnant un sondage provincial québécois est détecté, archivé avec11 provenance (`web_signals`) et signalé sur la page Signaux et dans l'admin —12 souvent avant son apparition dans l'agrégation Wikipédia.132. **Satisfaction gouvernementale** — extraction du % de satisfaits envers le14 gouvernement du Québec (baromètres Léger/rapportage de presse). C'est15 l'entrée vivante du prior de fondamentaux (`indicators`).163. **Presse élargie** — recherche par parti au-delà des flux RSS (JdM/JdQ,17 TVA, 98,5, médias régionaux) : les articles alimentent le moteur de18 sentiment existant (mêmes scoreurs, même provenance).1920Chaque page brute est archivée dans data/raw/firecrawl/. Tout est tolérant à21l'échec : sans clé ou sans réseau, le forecast fonctionne exactement comme avant.22"""23from __future__ import annotations2425import hashlib26import logging27import re28from datetime import date, datetime, timedelta, timezone2930import httpx31from sqlalchemy.orm import Session3233from ..config import DATA_DIR, settings34from .. import models as Mo3536log = logging.getLogger("firecrawl")3738FC_BASE = "https://api.firecrawl.dev/v2"39RAW_DIR = DATA_DIR / "raw" / "firecrawl"4041KNOWN_POLLSTERS = ["Léger", "Leger", "Pallas", "Mainstreet", "Segma", "SOM",42 "Angus Reid", "Research Co", "Abacus", "EKOS", "Ipsos", "CROP",43 "Synopsis", "Forum"]4445PARTY_QUERIES = {46 "CAQ": "Coalition avenir Québec Christine Fréchette",47 "PLQ": "Parti libéral du Québec Charles Milliard",48 "PQ": "Parti québécois Paul St-Pierre Plamondon",49 "QS": "Québec solidaire Ruba Ghazal",50 "PCQ": "Parti conservateur du Québec Éric Duhaime",51}525354def _headers() -> dict:55 return {"Authorization": f"Bearer {settings.firecrawl_api_key}",56 "Content-Type": "application/json"}575859def _enabled() -> bool:60 return settings.firecrawl_enabled and bool(settings.firecrawl_api_key)616263def _post(path: str, payload: dict, timeout: float = 60.0) -> dict | None:64 try:65 r = httpx.post(f"{FC_BASE}{path}", json=payload, headers=_headers(),66 timeout=timeout)67 r.raise_for_status()68 data = r.json()69 return data if data.get("success", True) else None70 except Exception as e:71 log.warning("firecrawl %s: %s", path, e)72 return None737475def _archive(url: str, content: str) -> None:76 try:77 RAW_DIR.mkdir(parents=True, exist_ok=True)78 h = hashlib.sha1(url.encode()).hexdigest()[:16]79 (RAW_DIR / f"{date.today().isoformat()}_{h}.md").write_text(80 f"<!-- {url} — {datetime.now(timezone.utc).isoformat()} -->\n{content}")81 except Exception:82 pass838485def scrape_markdown(url: str) -> str | None:86 """Scrape une page → markdown (archivé pour provenance)."""87 if not _enabled():88 return None89 data = _post("/scrape", {"url": url, "formats": ["markdown"],90 "onlyMainContent": True}, timeout=90.0)91 md = ((data or {}).get("data") or {}).get("markdown")92 if md:93 _archive(url, md[:200000])94 return md959697def search(query: str, limit: int = 8, tbs: str | None = "qdr:w",98 news: bool = True) -> list[dict]:99 """Recherche web/actualités → [{url, title, description, date?}]."""100 if not _enabled():101 return []102 payload: dict = {"query": query, "limit": limit, "location": "Montreal,Quebec,Canada"}103 if tbs:104 payload["tbs"] = tbs105 if news:106 payload["sources"] = [{"type": "news"}]107 data = _post("/search", payload)108 if not data:109 return []110 d = data.get("data") or {}111 items = []112 for bucket in ("news", "web"):113 for it in (d.get(bucket) or []):114 if isinstance(it, dict) and it.get("url"):115 items.append(it)116 if not items and isinstance(d, list): # tolérance : ancien format plat117 items = [it for it in d if isinstance(it, dict) and it.get("url")]118 return items[:limit]119120121def _add_signal(db: Session, kind: str, title: str, url: str,122 snippet: str | None = None, pollster: str | None = None,123 extra: dict | None = None) -> bool:124 if db.query(Mo.WebSignal).filter_by(url=url, kind=kind).first():125 return False126 db.add(Mo.WebSignal(kind=kind, title=title[:500], url=url,127 snippet=(snippet or "")[:1000], pollster=pollster,128 extra=extra or {}))129 return True130131132# ---------------------------------------------------------------------------133# 1. Radar sondages134# ---------------------------------------------------------------------------135def radar_polls(db: Session) -> int:136 """Détecte les nouveaux sondages provinciaux dans l'actualité."""137 added, seen = 0, set()138 for q in ["sondage intentions de vote Québec provincial",139 "nouveau sondage élections Québec 2026"]:140 for it in search(q, limit=8, tbs="qdr:w"):141 text = f"{it.get('title', '')} {it.get('description', '')}"142 if it["url"] in seen or not re.search(r"sondage|intentions de vote|poll",143 text, re.I):144 continue145 seen.add(it["url"])146 house = next((h for h in KNOWN_POLLSTERS if h.lower() in text.lower()), None)147 if _add_signal(db, "poll-radar", it.get("title", "(sans titre)"),148 it["url"], it.get("description"), pollster=house,149 extra={"query": q, "published": it.get("date")}):150 db.flush()151 added += 1152 db.commit()153 return added154155156# ---------------------------------------------------------------------------157# 2. Satisfaction envers le gouvernement (entrée du prior de fondamentaux)158# ---------------------------------------------------------------------------159SAT_PATTERNS = [160 r"(\d{2})\s?%\s+(?:des\s+\S+\s+){0,4}(?:se\s+disent?\s+|sont\s+)?satisfait",161 r"satisfaction[^.\d]{0,60}?(\d{2})\s?%",162 r"taux de satisfaction[^.\d]{0,40}?(\d{2})\s?%",163]164165166def fetch_gov_satisfaction(db: Session) -> dict:167 """Cherche la mesure la plus récente du % de satisfaits envers le168 gouvernement du Québec; enregistre un Indicator avec provenance."""169 report = {"found": False}170 hits = search("sondage Léger satisfaction gouvernement du Québec",171 limit=6, tbs="qdr:m")172 for it in hits:173 blob = f"{it.get('title', '')} {it.get('description', '')}"174 val = _extract_sat(blob)175 source = it["url"]176 if val is None: # le extrait ne suffit pas → scrape de l'article177 md = scrape_markdown(it["url"])178 if md:179 val = _extract_sat(md[:8000])180 if val is not None:181 today = date.today()182 row = db.query(Mo.Indicator).filter_by(name="gov_satisfaction",183 as_of=today).first()184 if row is None:185 row = Mo.Indicator(name="gov_satisfaction", as_of=today, value=val)186 db.add(row)187 row.value = val188 row.source = it.get("title", "")[:120]189 row.source_url = source190 row.method = "firecrawl"191 _add_signal(db, "satisfaction", it.get("title", ""), source,192 it.get("description"), extra={"value": val})193 db.commit()194 report = {"found": True, "value": val, "source": source}195 break196 return report197198199def _extract_sat(text: str) -> float | None:200 low = text.lower()201 if "gouvernement" not in low and "legault" not in low and "fréchette" not in low:202 return None203 for pat in SAT_PATTERNS:204 m = re.search(pat, low)205 if m:206 v = float(m.group(1))207 if 10 <= v <= 80:208 return v209 return None210211212def latest_satisfaction(db: Session) -> tuple[float, dict]:213 """% de satisfaits le plus récent (ingéré), sinon valeur de repli configurée."""214 row = (db.query(Mo.Indicator).filter_by(name="gov_satisfaction")215 .order_by(Mo.Indicator.as_of.desc()).first())216 if row:217 return row.value, {"as_of": row.as_of.isoformat(), "source": row.source,218 "source_url": row.source_url, "method": row.method}219 return settings.gov_satisfaction_fallback, {220 "method": "fallback-config",221 "note": "aucune mesure ingérée — valeur de repli de configuration"}222223224# ---------------------------------------------------------------------------225# 3. Presse élargie → moteur de sentiment existant226# ---------------------------------------------------------------------------227def sweep_news(db: Session) -> dict:228 """Articles au-delà des RSS : recherche par parti, scoring par le moteur229 de sentiment existant (mêmes scoreurs lexique/LLM, même provenance)."""230 from .news_rss import cluster_key, detect_entities, lexicon_score, llm_score231 added, scored, budget = 0, 0, settings.firecrawl_max_docs_per_cycle232 for party, q in PARTY_QUERIES.items():233 if added >= budget:234 break235 for it in search(q, limit=6, tbs="qdr:w"):236 url, title = it.get("url"), it.get("title") or ""237 if not url or not title or added >= budget:238 continue239 if db.query(Mo.SentimentDocument).filter_by(url=url).first():240 continue241 summary = (it.get("description") or "")[:2000]242 pub = None243 if it.get("date"):244 try:245 pub = datetime.fromisoformat(str(it["date"]).replace("Z", "+00:00"))246 except ValueError:247 pub = None248 doc = Mo.SentimentDocument(249 source=f"Firecrawl · {url.split('/')[2]}", url=url, title=title,250 summary=summary, published=pub, cluster_key=cluster_key(title))251 text = f"{title}. {summary}"252 for ent in detect_entities(text):253 res = llm_score(text, ent)254 method = "llm"255 if res is None:256 res, method = lexicon_score(text, ent), "lexicon"257 s, stance = res258 doc.scores.append(Mo.SentimentScore(entity=ent, sentiment=s,259 stance=stance, method=method))260 scored += 1261 db.add(doc)262 added += 1263 db.commit()264 return {"documents": added, "scores": scored}265266267def reconcile_radar(db: Session) -> dict:268 """Boucle de garantie : chaque sondage détecté par le radar doit finir dans269 la base. Une détection avec maison connue est marquée « intégré » quand un270 sondage de cette maison a été ajouté/actualisé autour de la détection;271 sinon elle reste « nouveau » = alerte visible (admin + point du jour)."""272 cutoff = datetime.now(timezone.utc) - timedelta(days=14)273 pending = (db.query(Mo.WebSignal)274 .filter(Mo.WebSignal.kind == "poll-radar",275 Mo.WebSignal.status == "nouveau",276 Mo.WebSignal.detected_at >= cutoff,277 Mo.WebSignal.pollster.isnot(None)).all())278 integrated, waiting = 0, 0279 for sig in pending:280 d0 = sig.detected_at.date()281 match = (db.query(Mo.Poll).join(Mo.Pollster)282 .filter(Mo.Pollster.name == sig.pollster,283 Mo.Poll.field_end >= d0 - timedelta(days=10),284 Mo.Poll.field_end <= d0 + timedelta(days=3)).first())285 if match:286 sig.status = "intégré"287 sig.extra = {**(sig.extra or {}), "poll_id": match.id,288 "field_end": match.field_end.isoformat()}289 integrated += 1290 else:291 waiting += 1292 db.commit()293 return {"intégrés": integrated, "en_attente": waiting}294295296# ---------------------------------------------------------------------------297def run_watch(db: Session) -> dict:298 """Cycle complet de veille — chaque mission est isolée (échec toléré)."""299 if not _enabled():300 return {"skipped": "Firecrawl désactivé ou clé absente"}301 report: dict = {}302 for name, fn in [("radar_sondages", radar_polls),303 ("réconciliation", reconcile_radar),304 ("satisfaction", fetch_gov_satisfaction),305 ("presse_elargie", sweep_news)]:306 try:307 report[name] = fn(db)308 except Exception as e:309 db.rollback()310 report[name] = f"échec: {e}"311 return report312