spb/qc-election

1# QC Élection Forecast — Plateforme de prévision électorale du Québec 20262# Auteur : Simon-Pierre Boucher3# Contact : contact@spboucher.ai4# https://www.qc-election.com5"""Ingestion des données électorales depuis Wikipédia (sondages, résultats 2022).67Chaque exécution sauvegarde une copie brute du HTML dans data/raw/ (provenance)8et produit des CSV normalisés dans data/. Le pipeline recharge ensuite ces CSV9dans la base. Toute valeur conserve : source, URL, date d'accès, valeur brute.10"""11from __future__ import annotations1213import re14import unicodedata15from datetime import date, datetime, timedelta, timezone16from io import StringIO17from pathlib import Path1819import httpx20import pandas as pd2122from ..config import DATA_DIR2324URL_2026 = "https://en.wikipedia.org/wiki/2026_Quebec_general_election"25URL_2022 = "https://en.wikipedia.org/wiki/2022_Quebec_general_election"26UA = {"User-Agent": "Mozilla/5.0 (qc-election-forecast; research/aggregation)"}2728# Renommages carte électorale 2025 (ancienne → nouvelle)29RENAMES_2025 = {30    "Johnson": "Daniel-Johnson",31    "Arthabaska": "Arthabaska–L'Érable",32    "Laporte": "Pierre-Laporte",33    "Matane-Matapédia": "Matane-Matapédia-Mitis",34    "Rivière-du-Loup–Témiscouata": "Rivière-du-Loup–Témiscouata–Les Basques",35    "Vimont": "Vimont-Auteuil",36}3738# Nouvelles circonscriptions 2026 : (nom, région, circonscriptions parentes)39NEW_DISTRICTS_2026 = [40    ("Bellefeuille", "Laurentides", ["Saint-Jérôme", "Mirabel", "Argenteuil"]),41    ("Marie-Lacoste-Gérin-Lajoie", "Chaudière-Appalaches et Centre-du-Québec",42     ["Johnson", "Nicolet-Bécancour", "Drummond–Bois-Francs"]),43]4445# Régions de modélisation (13 groupes — navbox Wikipédia fr, arbitrages documentés46# pour les circonscriptions chevauchant deux régions administratives).47REGION_RIDINGS = {48    "Bas-Saint-Laurent–Gaspésie": [49        "Bonaventure", "Côte-du-Sud", "Gaspé", "Îles-de-la-Madeleine",50        "Matane-Matapédia-Mitis", "Rimouski", "Rivière-du-Loup–Témiscouata–Les Basques"],51    "Saguenay–Lac-Saint-Jean et Côte-Nord": [52        "Chicoutimi", "Dubuc", "Duplessis", "Jonquière", "Lac-Saint-Jean",53        "René-Lévesque", "Roberval"],54    "Capitale-Nationale": [55        "Charlesbourg", "Charlevoix–Côte-de-Beaupré", "Chauveau", "Jean-Lesage",56        "Jean-Talon", "La Peltrie", "Louis-Hébert", "Montmorency", "Portneuf",57        "Taschereau", "Vanier-Les Rivières"],58    "Mauricie": ["Champlain", "Laviolette–Saint-Maurice", "Maskinongé", "Trois-Rivières"],59    "Chaudière-Appalaches et Centre-du-Québec": [60        "Arthabaska–L'Érable", "Beauce-Nord", "Beauce-Sud", "Bellechasse",61        "Chutes-de-la-Chaudière", "Drummond–Bois-Francs", "Daniel-Johnson", "Lévis",62        "Lotbinière-Frontenac", "Marie-Lacoste-Gérin-Lajoie", "Mégantic",63        "Nicolet-Bécancour"],64    "Estrie": ["Brome-Missisquoi", "Granby", "Orford", "Richmond", "Saint-François",65               "Sherbrooke"],66    "Montérégie": [67        "Beauharnois", "Borduas", "Chambly", "Châteauguay", "Huntingdon", "Iberville",68        "La Pinière", "Pierre-Laporte", "La Prairie", "Marie-Victorin", "Montarville",69        "Richelieu", "Saint-Hyacinthe", "Saint-Jean", "Sanguinet", "Soulanges",70        "Taillon", "Vachon", "Vaudreuil", "Verchères"],71    "Montréal": [72        "Acadie", "Anjou–Louis-Riel", "Bourassa-Sauvé", "Camille-Laurin",73        "D'Arcy-McGee", "Gouin", "Hochelaga-Maisonneuve", "Jacques-Cartier",74        "Jeanne-Mance–Viger", "LaFontaine", "Laurier-Dorion", "Marguerite-Bourgeoys",75        "Marquette", "Maurice-Richard", "Mercier", "Mont-Royal–Outremont", "Nelligan",76        "Notre-Dame-de-Grâce", "Pointe-aux-Trembles", "Robert-Baldwin", "Rosemont",77        "Saint-Henri–Sainte-Anne", "Sainte-Marie–Saint-Jacques", "Saint-Laurent",78        "Verdun", "Viau", "Westmount–Saint-Louis"],79    "Laval": ["Chomedey", "Fabre", "Laval-des-Rapides", "Mille-Îles", "Sainte-Rose",80              "Vimont-Auteuil"],81    "Lanaudière": ["Berthier", "Joliette", "L'Assomption", "Les Plaines", "Masson",82                   "Repentigny", "Rousseau", "Terrebonne"],83    "Laurentides": ["Argenteuil", "Bellefeuille", "Bertrand", "Blainville",84                    "Deux-Montagnes", "Groulx", "Labelle", "Mirabel", "Prévost",85                    "Saint-Jérôme"],86    "Outaouais": ["Chapleau", "Gatineau", "Hull", "Papineau", "Pontiac"],87    "Abitibi-Témiscamingue et Nord-du-Québec": [88        "Abitibi-Est", "Abitibi-Ouest", "Rouyn-Noranda–Témiscamingue", "Ungava"],89}9091# Mode de collecte habituel par maison (design effects §2 de la méthodologie).92HOUSE_MODES = {93    "Léger": "web", "Mainstreet": "ivr", "Pallas Data": "ivr", "Synopsis": "web",94    "SEGMA": "phone", "Research Co.": "web", "Angus Reid": "web", "EKOS": "ivr",95    "Liaison Strategies": "ivr", "Innovative Research": "web", "CROP": "web",96    "Forum": "ivr", "Abacus Data": "web", "Ipsos": "web",97}9899POLLSTER_ALIASES = {100    "leger": "Léger", "léger": "Léger", "leger marketing": "Léger",101    "mainstreet research": "Mainstreet", "mainstreet": "Mainstreet",102    "mainstreet (exit poll)": "Mainstreet",103    "pallas data": "Pallas Data", "pallas": "Pallas Data",104    "research co.": "Research Co.", "research co": "Research Co.",105    "angus reid institute": "Angus Reid", "angus reid": "Angus Reid",106    "liaison strategies": "Liaison Strategies",107    "segma recherche": "SEGMA", "segma": "SEGMA",108    "ekos": "EKOS",109    "synopsis recherche": "Synopsis", "synopsis": "Synopsis",110    "innovative research group": "Innovative Research",111    "innovative research": "Innovative Research",112}113114115def norm_name(s: str) -> str:116    """Normalise un nom de circonscription (apostrophes, tirets, espaces)."""117    s = unicodedata.normalize("NFC", str(s)).strip()118    s = s.replace("’", "'").replace("–", "–").replace("—", "–")119    s = re.sub(r"\s+", " ", s)120    s = re.sub(r"\s*\[.*?\]\s*$", "", s)  # notes wiki [a]121    return s122123124def _dash_key(s: str) -> str:125    return norm_name(s).replace("–", "-").lower()126127128_RENAMES_BY_KEY = {}129130131def canon_district(s: str) -> str:132    if not _RENAMES_BY_KEY:133        _RENAMES_BY_KEY.update({_dash_key(k): v for k, v in RENAMES_2025.items()})134    n = norm_name(s)135    return _RENAMES_BY_KEY.get(_dash_key(n), n)136137138def _district_key(s: str) -> str:139    return _dash_key(canon_district(s))140141142def norm_pollster(s: str) -> str:143    key = norm_name(s).lower()144    key = re.sub(r"\s*\(.*\)$", "", key).strip()145    return POLLSTER_ALIASES.get(key, norm_name(s))146147148def fetch(url: str, tag: str) -> str:149    r = httpx.get(url, headers=UA, timeout=60, follow_redirects=True)150    r.raise_for_status()151    raw_dir = DATA_DIR / "raw"152    raw_dir.mkdir(parents=True, exist_ok=True)153    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")154    (raw_dir / f"{tag}_{stamp}.html").write_text(r.text)155    return r.text156157158def _num(x) -> float | None:159    if x is None or (isinstance(x, float) and pd.isna(x)):160        return None161    s = str(x).strip().replace("%", "").replace(",", "")162    s = re.sub(r"\[.*?\]", "", s)163    m = re.search(r"-?\d+(\.\d+)?", s)164    return float(m.group()) if m else None165166167def _parse_date(x, default_year: int | None = None) -> date | None:168    from dateutil import parser as dp169    s = re.sub(r"\[.*?\]", "", str(x)).strip()170    if not s or s.lower() in ("nan", "none"):171        return None172    # plages "September 30 – October 2, 2022" → prendre la fin173    s = re.split(r"[–—]|-(?=\s*[A-Z])", s)[-1].strip()174    try:175        d = dp.parse(s, default=datetime(default_year or 2000, 6, 15))176        return d.date()177    except (ValueError, OverflowError):178        return None179180181def _parse_date_range(x, default_year: int | None = None) -> tuple[date | None, date | None]:182    """Plage de terrain « Sept. 30 – Oct. 2, 2026 » → (début, fin).183    Sans plage : (None, fin)."""184    from dateutil import parser as dp185    s = re.sub(r"\[.*?\]", "", str(x)).strip()186    end = _parse_date(x, default_year)187    parts = re.split(r"[–—]|-(?=\s*[A-Z])", s)188    if end is None or len(parts) < 2:189        return None, end190    lead = parts[0].strip().rstrip(",")191    try:192        # « September 30 » hérite de l'année/du mois de la fin si absents193        start = dp.parse(lead, default=datetime(end.year, end.month, end.day)).date()194        if start <= end and (end - start).days <= 45:195            return start, end196    except (ValueError, OverflowError):197        pass198    return None, end199200201# Sections de la page dont les tables sont des sondages NATIONAUX; on exclut202# explicitement les ventilations (langue, région, âge) et les projections.203SECTION_ACCEPT = re.compile(r"opinion poll|campaign period|pre-?campaign", re.I)204SECTION_REJECT = re.compile(205    r"language|region|montreal|quebec city|age|gender|francophone|anglophone|"206    r"projection|leadership|approval|preferred premier|by-?election", re.I)207208209def iter_section_tables(html: str):210    """Itère (en-tête de section, HTML de la table) pour chaque wikitable —211    le contexte de section permet d'accepter la table nationale et la table de212    campagne (même à 1 rangée) tout en rejetant les ventilations régionales."""213    heads = [(m.start(), re.sub(r"<[^>]+>", "", m.group(2)).strip())214             for m in re.finditer(r"<h([2-5])[^>]*>(.*?)</h\1>", html, re.S)]215    section = "?"216    hi = 0217    for m in re.finditer(r'<table class="wikitable.*?</table>', html, re.S):218        while hi < len(heads) and heads[hi][0] < m.start():219            section = heads[hi][1]220            hi += 1221        yield section, m.group(0)222223224def _flat_cols(df: pd.DataFrame) -> list[str]:225    out = []226    for c in df.columns:227        if isinstance(c, tuple):228            parts = [str(p) for p in c if not str(p).startswith("Unnamed")]229            out.append(parts[-1] if parts else str(c[0]))230        else:231            out.append(str(c))232    return out233234235PARTY_COL_MAP = {"caq": "CAQ", "qs": "QS", "pq": "PQ", "liberal": "PLQ", "plq": "PLQ",236                 "pcq": "PCQ", "conservative": "PCQ", "other": "AUT"}237238239def parse_poll_table(df: pd.DataFrame, election_year: int, source_url: str) -> list[dict]:240    """Table de sondages Wikipédia → liste de dicts normalisés."""241    df = df.copy()242    df.columns = _flat_cols(df)243    cols = {c.lower().strip(): c for c in df.columns}244    org_col = next((cols[k] for k in cols if "polling organisation" in k or k == "firm"), None)245    date_col = next((cols[k] for k in cols if "last date" in k or "date" in k), None)246    if org_col is None or date_col is None:247        return []248    party_cols = {}249    for k, orig in cols.items():250        if k in PARTY_COL_MAP:251            party_cols[PARTY_COL_MAP[k]] = orig252    rows = []253    for _, r in df.iterrows():254        org = r.get(org_col)255        if org is None or pd.isna(org):256            continue257        org = str(org)258        vals = [str(r.get(c)) for c in df.columns]259        if len(set(vals)) <= 2 and len(org) > 40:      # rangée-bannière (événement)260            continue261        if "election" in org.lower():                   # rangée résultat officiel262            continue263        end = _parse_date(r.get(date_col), election_year)264        if end is None:265            continue266        shares = {}267        for party, c in party_cols.items():268            v = _num(r.get(c))269            if v is not None:270                shares[party] = v271        if len([p for p in shares if p != "AUT"]) < 4:272            continue273        n = _num(r.get(cols.get("sample size", cols.get("sample", ""))))274        moe = _num(r.get(cols.get("moe", "")))275        und_col = next((cols[k] for k in cols if "undecided" in k or "und." in k), None)276        undecided = _num(r.get(und_col)) if und_col else None277        start, _ = _parse_date_range(r.get(date_col), election_year)278        house = norm_pollster(org)279        rows.append({280            "pollster": house,281            "field_end": end.isoformat(),282            "field_start": (start or (end - timedelta(days=3))).isoformat(),283            "sample_size": int(n) if n else None,284            "moe": abs(moe) if moe else None,285            "undecided": undecided,286            "population": "adults",287            "mode": HOUSE_MODES.get(house, "unknown"),288            "region": "QC",289            "source_name": "Wikipédia (agrégation)",290            "source_url": source_url,291            "shares": shares,292        })293    return rows294295296def parse_polls_2026(html: str) -> list[dict]:297    """Tables de sondages NATIONAUX de la page 2026, sélectionnées par leur298    section (« Opinion polls », « Campaign period »…) — aucune contrainte de299    taille : la table de campagne est captée dès sa première rangée. Les300    ventilations (langue, région, projections) sont exclues par section."""301    out, seen = [], set()302    for section, thtml in iter_section_tables(html):303        if not SECTION_ACCEPT.search(section) or SECTION_REJECT.search(section):304            continue305        try:306            tables = pd.read_html(StringIO(thtml))307        except ValueError:308            continue309        for t in tables:310            cols = " ".join(str(c).lower() for c in t.columns)311            if ("polling organisation" in cols or "firm" in cols) and "caq" in cols:312                for row in parse_poll_table(t, 2026, URL_2026):313                    key = (row["pollster"], row["field_end"], row["sample_size"])314                    if key not in seen:      # une table peut apparaître deux fois315                        seen.add(key)316                        out.append(row)317    return out318319320URL_2026_FR = ("https://fr.wikipedia.org/wiki/Liste_de_sondages_sur_les_"321               "%C3%A9lections_g%C3%A9n%C3%A9rales_qu%C3%A9b%C3%A9coises_de_2026")322_MOIS = {"janvier": 1, "février": 2, "mars": 3, "avril": 4, "mai": 5, "juin": 6,323         "juillet": 7, "août": 8, "aout": 8, "septembre": 9, "octobre": 10,324         "novembre": 11, "décembre": 12, "decembre": 12}325326327def _parse_date_fr(s: str) -> date | None:328    m = re.search(r"(\d{1,2})(?:er)?\s+([a-zéû]+)\s+(\d{4})", str(s), re.I)329    if not m or m.group(2).lower() not in _MOIS:330        return None331    try:332        return date(int(m.group(3)), _MOIS[m.group(2).lower()], int(m.group(1)))333    except ValueError:334        return None335336337def parse_polls_2026_fr(html: str) -> list[dict]:338    """Wikipédia FR « Liste de sondages… 2026 » — seconde source d'agrégation,339    souvent plus fraîche que la page EN pendant la campagne. Colonnes :340    Dernier jour du sondage | CAQ | PLQ | QS | PQ | PCQ | Autres | Sondeur |341    Échantillon | ME | Source. Dédupliquée avec la page EN par (maison, fin)."""342    out = []343    for t in pd.read_html(StringIO(html)):344        t = t.copy()345        t.columns = _flat_cols(t)346        cols = {str(c).strip(): c for c in t.columns}347        if not ({"CAQ", "PQ", "PLQ"} <= set(cols) and348                any("dernier jour" in str(c).lower() for c in cols)):349            continue350        date_col = next(cols[c] for c in cols if "dernier jour" in c.lower())351        for _, r in t.iterrows():352            end = _parse_date_fr(r.get(date_col))353            org = r.get(cols.get("Sondeur"))354            if end is None or org is None or pd.isna(org):355                continue356            org = str(org)357            # rangée-bannière d'événement (cellule fusionnée) : sondeur qui358            # commence par une date ou contient un événement → ignorer359            if re.match(r"\d", org) or "élection" in org.lower() \360                    or "déclenchement" in org.lower() or len(org) > 60:361                continue362            sponsor = None363            if "/" in org:                      # « Synopsis / La Presse »364                org, sponsor = [x.strip() for x in org.split("/", 1)]365            shares = {}366            for party in ("CAQ", "PLQ", "QS", "PQ", "PCQ"):367                v = _num(r.get(cols.get(party)))368                if v is not None and 0 <= v <= 80:369                    shares[party] = v370            aut = _num(r.get(cols.get("Autres", "")))371            if aut is not None:372                shares["AUT"] = aut373            if len([p for p in shares if p != "AUT"]) < 4:374                continue375            if len(set(shares.values())) <= 2:   # cellule fusionnée répétée376                continue377            n = _num(r.get(cols.get("Échantillon", "")))378            house = norm_pollster(org)379            out.append({"pollster": house,380                        "field_end": end.isoformat(),381                        "field_start": (end - timedelta(days=3)).isoformat(),382                        "sample_size": int(n) if n and n > 40 else None,383                        "moe": abs(_num(r.get(cols.get("ME", ""))) or 0) or None,384                        "population": "adults",385                        "mode": HOUSE_MODES.get(house, "unknown"),386                        "region": "QC",387                        "source_name": "Wikipédia FR (agrégation)"388                                       + (f" · {sponsor}" if sponsor else ""),389                        "source_url": URL_2026_FR,390                        "shares": shares})391    return out392393394def parse_polls_2022(html: str) -> list[dict]:395    """Tables 'Timeline of opinion polls' de l'article 2022 (campagne + pré-campagne)."""396    tables = pd.read_html(StringIO(html))397    out = []398    for t in tables:399        cols = " ".join(str(c).lower() for c in t.columns)400        if "timeline of opinion polls" in cols and "caq" in cols:401            year = 2022402            parsed = parse_poll_table(t, year, URL_2022)403            out.extend(parsed)404    return out405406407def parse_riding_results_2022(html: str) -> pd.DataFrame:408    """Table % par parti et par circonscription (élection 2022)."""409    tables = pd.read_html(StringIO(html))410    target = None411    for t in tables:412        raw = [c for c in t.columns]413        heads = {str(c[0]) if isinstance(c, tuple) else str(c) for c in raw}414        if {"Riding", "CAQ", "QS", "PQ", "PLQ", "PCQ"} <= heads and len(t) > 100:415            target = t416            break417    if target is None:418        raise RuntimeError("Table des résultats par circonscription introuvable")419    # première sous-colonne '%' et 'Change (pp)' de chaque parti420    pct_col: dict[str, object] = {}421    chg_col: dict[str, object] = {}422    name_col = None423    for c in target.columns:424        top = str(c[0]) if isinstance(c, tuple) else str(c)425        sub = str(c[-1]) if isinstance(c, tuple) else ""426        if top == "Riding" and name_col is None:427            name_col = c428        elif top in ("CAQ", "QS", "PQ", "PLQ", "PCQ") and sub == "%" and top not in pct_col:429            pct_col[top] = c430        elif (top in ("CAQ", "QS", "PQ", "PLQ", "PCQ") and sub.startswith("Change")431              and top not in chg_col):432            chg_col[top] = c433    rows = []434    for _, r in target.iterrows():435        name = r.get(name_col)436        if name is None or pd.isna(name):437            continue438        name = norm_name(name)439        if name.lower().startswith(("total", "riding", "source")):440            continue441        rec = {"district": name}442        n_num = 0443        for p, c in pct_col.items():444            v = _num(r.get(c))445            if v is not None:446                n_num += 1447            rec[p] = v if v is not None else 0.0  # parti sans candidat448        if n_num < 3:  # rangée d'en-tête ou de total449            continue450        s = sum(rec[p] for p in pct_col)451        rec["AUT"] = round(max(0.0, 100.0 - s), 2)452        # reconstruction 2018 : % de 2022 − variation (pp)453        for p, c in chg_col.items():454            chg = _num(r.get(c))455            rec[f"prev_{p}"] = round(max(0.0, rec[p] - (chg if chg is not None else rec[p])), 2)456        s18 = sum(rec.get(f"prev_{p}", 0.0) for p in pct_col)457        rec["prev_AUT"] = round(max(0.0, 100.0 - s18), 2)458        rows.append(rec)459    return pd.DataFrame(rows)460461462def parse_turnout_2022(html: str) -> dict[str, float]:463    tables = pd.read_html(StringIO(html))464    for t in tables:465        heads = {str(p) for c in t.columns for p in (c if isinstance(c, tuple) else (c,))}466        if "Turnout" not in heads or len(t) < 100:467            continue468        name_col = t.columns[0]469        turn_col = next((c for c in t.columns470                         if isinstance(c, tuple) and "Turnout" in [str(p) for p in c]471                         and str(c[-1]) == "%"), None)472        if turn_col is not None:473            out = {}474            for _, r in t.iterrows():475                n, v = r.get(name_col), _num(r.get(turn_col))476                if n is not None and not pd.isna(n) and v is not None and 20 < v < 95:477                    out[norm_name(n)] = v478            if len(out) > 100:479                return out480    return {}481482483def parse_retirements_2026(html: str) -> set[str]:484    """Circonscriptions dont le député sortant ne se représente pas."""485    tables = pd.read_html(StringIO(html))486    for t in tables:487        cols = [str(c) for c in t.columns]488        if any("Electoral District" in c for c in cols) and any("Date announced" in c for c in cols):489            col = next(c for c in t.columns if "Electoral District" in str(c))490            return {canon_district(v) for v in t[col].dropna().astype(str)}491    return set()492493494def build_districts(riding_2022: pd.DataFrame, turnout: dict[str, float],495                    retirements: set[str]) -> pd.DataFrame:496    """Construit les 127 circonscriptions 2026 avec référence 2022 (transposition approx.)."""497    parties = ["CAQ", "PCQ", "PLQ", "PQ", "QS", "AUT"]498    base = {norm_name(r["district"]): {p: r[p] for p in parties}499            for _, r in riding_2022.iterrows()}500    region_of = {}501    for reg, names in REGION_RIDINGS.items():502        for n in names:503            region_of[_district_key(n)] = reg504505    rows = []506    for old_name, shares in base.items():507        new_name = canon_district(old_name)508        reg = region_of.get(_district_key(new_name))509        rows.append({510            "district": new_name, "region": reg or "Inconnue",511            **{f"b_{p}": shares[p] for p in parties},512            "turnout_2022": turnout.get(old_name),513            "incumbent_party": max((p for p in parties if p != "AUT"),514                                   key=lambda p: shares[p]),515            "incumbent_running": new_name not in retirements,516            "is_new_2026": False,517            "baseline_source": f"Résultat 2022 ({old_name}) — Wikipédia/DGEQ",518        })519    for name, reg, parents in NEW_DISTRICTS_2026:520        ps = [base[p] for p in parents if p in base]521        shares = {p: round(sum(x[p] for x in ps) / len(ps), 2) for p in parties}522        rows.append({523            "district": name, "region": reg,524            **{f"b_{p}": shares[p] for p in parties},525            "turnout_2022": None,526            "incumbent_party": None, "incumbent_running": False, "is_new_2026": True,527            "baseline_source": "Moyenne des circonscriptions parentes ("528                               + ", ".join(parents) + ") — approximation transposition 2025",529        })530    df = pd.DataFrame(rows)531    missing = df[df.region == "Inconnue"]["district"].tolist()532    if missing:533        raise RuntimeError(f"Régions manquantes pour: {missing}")534    return df535536537def refresh_all(out_dir: Path | None = None, offline_html: dict[str, str] | None = None) -> dict:538    """Télécharge et normalise toutes les données; écrit les CSV de seed."""539    out_dir = out_dir or DATA_DIR540    out_dir.mkdir(parents=True, exist_ok=True)541    html26 = offline_html.get("2026") if offline_html else fetch(URL_2026, "wiki2026")542    html22 = offline_html.get("2022") if offline_html else fetch(URL_2022, "wiki2022")543544    polls26 = parse_polls_2026(html26)545    # seconde agrégation : Wikipédia FR (souvent plus fraîche en campagne);546    # union dédupliquée par (maison, fin de terrain) — la EN fait foi si doublon547    try:548        html_fr = (offline_html.get("2026fr") if offline_html549                   else fetch(URL_2026_FR, "wiki2026fr"))550        from datetime import date as _d551        def _near(house, end_iso, days=1):552            e = _d.fromisoformat(end_iso)553            return any(q["pollster"] == house and554                       abs((_d.fromisoformat(q["field_end"]) - e).days) <= days555                       for q in polls26)556        extra = [p for p in parse_polls_2026_fr(html_fr)557                 if not _near(p["pollster"], p["field_end"])]558        polls26.extend(extra)559    except Exception:560        pass561    polls22 = parse_polls_2022(html22)562    riding = parse_riding_results_2022(html22)563    turnout = parse_turnout_2022(html22)564    retirements = parse_retirements_2026(html26)565    districts = build_districts(riding, turnout, retirements)566567    accessed = datetime.now(timezone.utc).isoformat()568    for polls, tag in ((polls26, "polls_2026"), (polls22, "polls_2022")):569        recs = []570        for p in polls:571            for party, v in p["shares"].items():572                recs.append({**{k: p[k] for k in p if k != "shares"},573                             "party": party, "raw_value": v, "accessed_at": accessed})574        pd.DataFrame(recs).to_csv(out_dir / f"{tag}.csv", index=False)575    districts.to_csv(out_dir / "districts_2026.csv", index=False)576    riding.to_csv(out_dir / "riding_results_2022.csv", index=False)577    return {"polls_2026": len(polls26), "polls_2022": len(polls22),578            "districts": len(districts), "retirements": len(retirements)}579580581if __name__ == "__main__":582    print(refresh_all())583