spb/qc-election

1# QC Élection Forecast — Plateforme de prévision électorale du Québec 20262# Auteur : Simon-Pierre Boucher3# Contact : contact@spboucher.ai4# https://www.qc-election.com5"""Backtesting du modèle sur l'élection de 2022.67À J-120, J-90, J-60, J-30, J-14, J-7 et J-1, on refait tourner le modèle avec8UNIQUEMENT l'information disponible à cette date (sondages ≤ date; aucune9correction maison, faute d'historique antérieur à 2022 — documenté), puis on10compare aux résultats réels : MAE/RMSE du vote, couverture des intervalles,11sièges, Brier score et calibration des probabilités de victoire locales.12"""13from __future__ import annotations1415import json16from datetime import date, timedelta1718import numpy as np19import pandas as pd20from sqlalchemy.orm import Session2122from ..config import DATA_DIR, settings23from .. import models as Mo24from ..pipeline import load_polls25from . import fundamentals as FUND26from .forecast import distribution_from, forecast as fc27from .simulate import SimulationInput, run_simulation28from .trend import fit_trend2930ELECTION_2022 = date(2022, 10, 3)31HORIZONS = [120, 90, 60, 30, 14, 7, 1]32RESULT_2022 = {"CAQ": 40.98, "PLQ": 14.37, "QS": 15.43, "PQ": 14.61, "PCQ": 12.91, "AUT": 1.70}33RESULT_2018 = {"CAQ": 37.42, "PLQ": 24.82, "QS": 16.10, "PQ": 17.06, "PCQ": 1.46, "AUT": 3.14}34SEATS_2022 = {"CAQ": 90, "PLQ": 21, "QS": 11, "PQ": 3, "PCQ": 0, "AUT": 0}353637def _districts_2022() -> tuple[SimulationInput, dict[str, str]]:38    """Univers 2022 : 125 circonscriptions, référence = résultats 2018 reconstruits."""39    from ..ingest.wikipedia import REGION_RIDINGS, _district_key, canon_district40    region_of = {}41    for reg, names in REGION_RIDINGS.items():42        for n in names:43            region_of[_district_key(n)] = reg44    df = pd.read_csv(DATA_DIR / "riding_results_2022.csv")45    names, baselines, regions, winners = [], [], [], {}46    for _, r in df.iterrows():47        names.append(r["district"])48        b = [max(0.2, float(r.get(f"prev_{p}", 0.0) or 0.0)) for p in settings.parties]49        baselines.append(np.array(b) / sum(b))50        regions.append(region_of.get(_district_key(canon_district(r["district"])), "Autre"))51        actual = {p: float(r[p]) for p in settings.parties if p in r}52        winners[r["district"]] = max(actual, key=actual.get)53    D = len(names)54    inp = SimulationInput(55        x_mean=np.zeros(5), P=np.eye(5),56        baseline_national=np.array([RESULT_2018[p] / 100.0 for p in settings.parties]),57        district_names=names, district_baselines=np.array(baselines),58        district_regions=regions, retirement_flags=np.zeros(D, dtype=int),59        incumbent_party_idx=np.full(D, -1), majority_seats=63)60    return inp, winners616263def run_backtest(db: Session, n_sims: int = 10000, save_runs: bool = True) -> dict:64    el22 = db.query(Mo.Election).filter(Mo.Election.election_date == ELECTION_2022).one()65    polls = load_polls(db, el22.id)66    inp, actual_winners = _districts_2022()67    parties = [p for p in settings.parties if p != "AUT"]6869    horizons_report = []70    all_probs, all_outcomes = [], []   # calibration des probabilités locales71    for h in HORIZONS:72        as_of = ELECTION_2022 - timedelta(days=h)73        trend = fit_trend(polls, {}, as_of)74        if trend is None:75            horizons_report.append({"horizon": h, "error": "sondages insuffisants"})76            continue77        f_polls = fc(trend, as_of, ELECTION_2022)78        # v2 : blend avec le prior de fondamentaux, information ≤ 2022 seulement79        # (sortant CAQ, 1er mandat, satisfaction pré-électorale ~54 %, vote 2018).80        f = f_polls81        if settings.fundamentals_enabled:82            prior22 = FUND.compute_prior(as_of, incumbent="CAQ", terms=1,83                                         satisfaction=54.0, prev_result=RESULT_2018)84            x_b, P_b, _ = FUND.blend(f_polls.x, f_polls.P, prior22,85                                     (ELECTION_2022 - as_of).days)86            f = distribution_from("forecast", as_of, x_b, P_b)87        inp.x_mean, inp.P = f.x, f.P88        sim = run_simulation(inp, n_sims=n_sims)8990        # --- métriques nationales91        errs = [f.summary[p]["mean"] - RESULT_2022[p] for p in parties]92        mae = float(np.mean(np.abs(errs)))93        rmse = float(np.sqrt(np.mean(np.square(errs))))94        coverage = float(np.mean([95            f.summary[p]["lo95"] <= RESULT_2022[p] <= f.summary[p]["hi95"] for p in parties]))96        mae_polls_only = float(np.mean(97            [abs(f_polls.summary[p]["mean"] - RESULT_2022[p]) for p in parties]))9899        # --- sièges100        seat_mae = float(np.mean([abs(sim.seats[p]["mean"] - SEATS_2022[p]) for p in parties]))101        seat_cov = float(np.mean([102            sim.seats[p]["p05"] <= SEATS_2022[p] <= sim.seats[p]["p95"] for p in parties]))103104        # --- circonscriptions : Brier multiclasse + log loss105        briers, lls, correct = [], [], 0106        for d in sim.districts:107            actual = actual_winners.get(d["district"])108            if actual is None:109                continue110            b = sum((d["win_probs"].get(p, 0.0) - (1.0 if p == actual else 0.0)) ** 2111                    for p in settings.parties)112            briers.append(b)113            lls.append(-np.log(max(d["win_probs"].get(actual, 0.0), 1e-6)))114            if d["favorite"] == actual:115                correct += 1116            for p in settings.parties:117                all_probs.append(d["win_probs"].get(p, 0.0))118                all_outcomes.append(1.0 if p == actual else 0.0)119120        rep = {121            "horizon": h, "as_of": as_of.isoformat(), "n_polls": trend.n_polls,122            "national": {"mae_pp": round(mae, 2), "rmse_pp": round(rmse, 2),123                         "mae_pp_sondages_seuls": round(mae_polls_only, 2),124                         "coverage_95": round(coverage, 3),125                         "forecast": {p: f.summary[p]["mean"] for p in parties},126                         "actual": {p: RESULT_2022[p] for p in parties}},127            "seats": {"mae": round(seat_mae, 1), "coverage_90": round(seat_cov, 3),128                      "expected": {p: sim.seats[p]["mean"] for p in parties},129                      "actual": SEATS_2022,130                      "prob_majority_caq": sim.seats["CAQ"]["prob_majority"]},131            "districts": {"brier": round(float(np.mean(briers)), 4),132                          "log_loss": round(float(np.mean(lls)), 4),133                          "winner_accuracy": round(correct / len(briers), 4)},134        }135        horizons_report.append(rep)136137        if save_runs:138            run = Mo.ForecastRun(139                as_of=as_of, model_version=settings.model_version,140                n_polls_used=trend.n_polls, n_simulations=n_sims, is_backtest=True,141                label=f"Backtest 2022 J-{h}",142                national={"forecast": f.summary}, seats={"per_party": sim.seats},143                diagnostics=rep)144            db.add(run)145146    # --- calibration globale (bacs de 10 %)147    probs = np.array(all_probs); outs = np.array(all_outcomes)148    bins = []149    for lo in np.arange(0, 1.0, 0.1):150        m = (probs >= lo) & (probs < lo + 0.1 if lo < 0.9 else probs <= 1.0)151        if m.sum() >= 5:152            bins.append({"bin": f"{int(lo*100)}–{int(lo*100)+10} %",153                         "predicted": round(float(probs[m].mean()), 3),154                         "observed": round(float(outs[m].mean()), 3),155                         "n": int(m.sum())})156    report = {"election": "Québec 2022", "generated": date.today().isoformat(),157              "model_version": settings.model_version,158              "note": ("Aucune correction maison appliquée en backtest (pas de "159                       "données antérieures à 2018). v2 : le prior de fondamentaux "160                       "(information ≤ 2022 : sortant CAQ, 1er mandat, satisfaction "161                       "~54 %, vote 2018) est blendé comme en production; "162                       "mae_pp_sondages_seuls permet la comparaison directe. "163                       "Partielles et ajustement médias non rejouables pour 2022 "164                       "(données indisponibles) — exclus du backtest."),165              "horizons": horizons_report, "calibration": bins}166    (DATA_DIR / "backtest_report.json").write_text(167        json.dumps(report, indent=2, ensure_ascii=False))168    if save_runs:169        db.commit()170    return report171