spb/qc-election

1# QC Élection Forecast — Plateforme de prévision électorale du Québec 20262# Auteur : Simon-Pierre Boucher3# Contact : contact@spboucher.ai4# https://www.qc-election.com5"""Sondages historiques 1998-2018 (Wikipédia EN) — carburant du replay6multi-élections (§52). Chaque élection a son ère de partis (ADQ 2007-2008,7CAQ depuis 2012, PCQ significatif seulement en 2022) : le cœur alr étant8paramétré par la liste de partis, le même moteur rejoue chaque ère.910Les sondages parsés sont mis en cache (`data/historical/polls_{année}.csv`,11provenance URL + date d'accès) — le replay ne refait pas le réseau."""12from __future__ import annotations1314import logging15import re16from datetime import date, datetime, timezone17from io import StringIO18from pathlib import Path1920import httpx21import pandas as pd2223from ..config import DATA_DIR24from .wikipedia import (_flat_cols, _num, _parse_date, norm_pollster, UA)2526log = logging.getLogger("wiki-historical")2728HIST_DIR = DATA_DIR / "historical"2930# Métadonnées des élections rejouables (résultats officiels DGEQ, % du vote).31ELECTIONS: dict[int, dict] = {32    2022: {"date": date(2022, 10, 3),33           "parties": ["CAQ", "PLQ", "PQ", "QS", "PCQ", "AUT"],34           "incumbent": "CAQ", "terms": 1,35           "result": {"CAQ": 40.98, "PLQ": 14.37, "PQ": 14.61, "QS": 15.43,36                      "PCQ": 12.91, "AUT": 1.70},37           "url": "https://en.wikipedia.org/wiki/2022_Quebec_general_election"},38    2018: {"date": date(2018, 10, 1),39           "parties": ["CAQ", "PLQ", "PQ", "QS", "AUT"],40           "incumbent": "PLQ", "terms": 1,41           "result": {"CAQ": 37.42, "PLQ": 24.82, "PQ": 17.06, "QS": 16.10, "AUT": 4.60},42           "url": ("https://en.wikipedia.org/wiki/"43                   "Opinion_polling_for_the_2018_Quebec_general_election")},44    2014: {"date": date(2014, 4, 7),45           "parties": ["CAQ", "PLQ", "PQ", "QS", "AUT"],46           "incumbent": "PQ", "terms": 1,47           "result": {"CAQ": 23.05, "PLQ": 41.52, "PQ": 25.38, "QS": 7.63, "AUT": 2.42},48           "url": "https://en.wikipedia.org/wiki/2014_Quebec_general_election"},49    2012: {"date": date(2012, 9, 4),50           "parties": ["CAQ", "PLQ", "PQ", "QS", "AUT"],51           "incumbent": "PLQ", "terms": 3,52           "result": {"CAQ": 27.05, "PLQ": 31.20, "PQ": 31.95, "QS": 6.03, "AUT": 3.77},53           "url": "https://en.wikipedia.org/wiki/2012_Quebec_general_election"},54    2008: {"date": date(2008, 12, 8),55           "parties": ["ADQ", "PLQ", "PQ", "QS", "AUT"],56           "incumbent": "PLQ", "terms": 2,57           "result": {"ADQ": 16.37, "PLQ": 42.08, "PQ": 35.17, "QS": 3.78, "AUT": 2.60},58           "url": "https://en.wikipedia.org/wiki/2008_Quebec_general_election"},59    2007: {"date": date(2007, 3, 26),60           "parties": ["ADQ", "PLQ", "PQ", "QS", "AUT"],61           "incumbent": "PLQ", "terms": 1,62           "result": {"ADQ": 30.84, "PLQ": 33.08, "PQ": 28.35, "QS": 3.64, "AUT": 4.09},63           "url": "https://en.wikipedia.org/wiki/2007_Quebec_general_election"},64}6566# Correspondance colonnes → partis, toutes ères confondues (minuscule, contient).67PARTY_PATTERNS = [68    ("CAQ", ("caq", "coalition")),69    ("PLQ", ("plq", "qlp", "liberal", "libéral")),70    ("PQ", ("parti québécois", "parti quebecois", "pq")),71    ("QS", ("québec solidaire", "quebec solidaire", "qs")),72    ("ADQ", ("adq", "action démocratique", "action democratique")),73    ("PCQ", ("pcq", "conservative", "conservateur")),74    ("AUT", ("other", "others", "autres")),75]767778def _party_of(col: str) -> str | None:79    c = col.lower().strip()80    for party, pats in PARTY_PATTERNS:81        for pat in pats:82            if pat in c:83                return party84    return None858687def parse_election_polls(html: str, year: int) -> list[dict]:88    """Tables de sondages d'une page d'élection historique → dicts normalisés.89    Tolère les deux mises en page : « Timeline of opinion polls » (multi-niveau)90    et « Date(s)/Firm/Sample » (2008)."""91    meta = ELECTIONS[year]92    parties = meta["parties"]93    out, seen = [], set()94    for t in pd.read_html(StringIO(html)):95        t = t.copy()96        t.columns = _flat_cols(t)97        cols = {str(c).lower().strip(): c for c in t.columns}98        org_col = next((cols[k] for k in cols99                        if "polling" in k and "organi" in k or k == "firm"100                        or "polling firm" in k), None)101        date_col = next((cols[k] for k in cols102                         if "last date" in k or k.startswith("date")), None)103        party_cols = {}104        for k, orig in cols.items():105            p = _party_of(k)106            if p and p in parties and p not in party_cols:107                party_cols[p] = orig108        if org_col is None or date_col is None or len(109                [p for p in party_cols if p != "AUT"]) < 3:110            continue111        n_col = next((cols[k] for k in cols if "sample" in k), None)112        for _, r in t.iterrows():113            org = r.get(org_col)114            if org is None or pd.isna(org):115                continue116            org = str(org)117            if "election" in org.lower() or len(org) > 60:118                continue119            end = _parse_date(r.get(date_col), year)120            if end is None or abs((end - meta["date"]).days) > 4 * 365:121                continue122            shares = {}123            for p, c in party_cols.items():124                v = _num(r.get(c))125                if v is not None and 0 <= v <= 80:126                    shares[p] = v127            if len([p for p in shares if p != "AUT"]) < 3:128                continue129            if "AUT" not in shares:130                s = sum(shares.values())131                if 80 <= s <= 100:132                    shares["AUT"] = round(100 - s, 1)133            n = _num(r.get(n_col)) if n_col is not None else None134            key = (norm_pollster(org), end.isoformat())135            if key in seen:136                continue137            seen.add(key)138            out.append({"pollster": norm_pollster(org),139                        "field_end": end.isoformat(),140                        "sample_size": int(n) if n and n > 40 else None,141                        "mode": "unknown", "shares": shares,142                        "source_url": meta["url"]})143    return out144145146def load_polls(year: int, refresh: bool = False) -> list[dict]:147    """Sondages d'une élection historique (cache CSV, réseau si absent)."""148    HIST_DIR.mkdir(parents=True, exist_ok=True)149    f = HIST_DIR / f"polls_{year}.csv"150    if f.exists() and not refresh:151        df = pd.read_csv(f)152        polls: dict[tuple, dict] = {}153        for _, r in df.iterrows():154            key = (r["pollster"], r["field_end"])155            p = polls.setdefault(key, {156                "pollster": r["pollster"], "field_end": r["field_end"],157                "sample_size": None if pd.isna(r["sample_size"]) else int(r["sample_size"]),158                "mode": r.get("mode", "unknown"), "shares": {}})159            p["shares"][r["party"]] = float(r["raw_value"])160        return list(polls.values())161    html = httpx.get(ELECTIONS[year]["url"], headers=UA, timeout=90,162                     follow_redirects=True).text163    polls = parse_election_polls(html, year)164    recs = []165    accessed = datetime.now(timezone.utc).isoformat()166    for p in polls:167        for party, v in p["shares"].items():168            recs.append({**{k: p[k] for k in p if k != "shares"},169                         "party": party, "raw_value": v, "accessed_at": accessed})170    pd.DataFrame(recs).to_csv(f, index=False)171    log.info("polls %s: %d sondages mis en cache", year, len(polls))172    return polls173174175def to_model_polls(polls: list[dict]) -> list[dict]:176    """Format consommé par fit_trend (field_end en date, parts normalisées)."""177    out = []178    for p in polls:179        total = sum(p["shares"].values()) or 100.0180        out.append({"id": None, "pollster": p["pollster"],181                    "field_end": date.fromisoformat(p["field_end"]),182                    "sample_size": p["sample_size"], "mode": p.get("mode", "unknown"),183                    "shares": {k: v * 100.0 / total for k, v in p["shares"].items()}})184    return out185