# QC Élection Forecast — Plateforme de prévision électorale du Québec 2026
# Auteur : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# https://www.qc-election.com
"""Ingestion des données électorales depuis Wikipédia (sondages, résultats 2022).

Chaque exécution sauvegarde une copie brute du HTML dans data/raw/ (provenance)
et produit des CSV normalisés dans data/. Le pipeline recharge ensuite ces CSV
dans la base. Toute valeur conserve : source, URL, date d'accès, valeur brute.
"""
from __future__ import annotations

import re
import unicodedata
from datetime import date, datetime, timedelta, timezone
from io import StringIO
from pathlib import Path

import httpx
import pandas as pd

from ..config import DATA_DIR

URL_2026 = "https://en.wikipedia.org/wiki/2026_Quebec_general_election"
URL_2022 = "https://en.wikipedia.org/wiki/2022_Quebec_general_election"
UA = {"User-Agent": "Mozilla/5.0 (qc-election-forecast; research/aggregation)"}

# Renommages carte électorale 2025 (ancienne → nouvelle)
RENAMES_2025 = {
    "Johnson": "Daniel-Johnson",
    "Arthabaska": "Arthabaska–L'Érable",
    "Laporte": "Pierre-Laporte",
    "Matane-Matapédia": "Matane-Matapédia-Mitis",
    "Rivière-du-Loup–Témiscouata": "Rivière-du-Loup–Témiscouata–Les Basques",
    "Vimont": "Vimont-Auteuil",
}

# Nouvelles circonscriptions 2026 : (nom, région, circonscriptions parentes)
NEW_DISTRICTS_2026 = [
    ("Bellefeuille", "Laurentides", ["Saint-Jérôme", "Mirabel", "Argenteuil"]),
    ("Marie-Lacoste-Gérin-Lajoie", "Chaudière-Appalaches et Centre-du-Québec",
     ["Johnson", "Nicolet-Bécancour", "Drummond–Bois-Francs"]),
]

# Régions de modélisation (13 groupes — navbox Wikipédia fr, arbitrages documentés
# pour les circonscriptions chevauchant deux régions administratives).
REGION_RIDINGS = {
    "Bas-Saint-Laurent–Gaspésie": [
        "Bonaventure", "Côte-du-Sud", "Gaspé", "Îles-de-la-Madeleine",
        "Matane-Matapédia-Mitis", "Rimouski", "Rivière-du-Loup–Témiscouata–Les Basques"],
    "Saguenay–Lac-Saint-Jean et Côte-Nord": [
        "Chicoutimi", "Dubuc", "Duplessis", "Jonquière", "Lac-Saint-Jean",
        "René-Lévesque", "Roberval"],
    "Capitale-Nationale": [
        "Charlesbourg", "Charlevoix–Côte-de-Beaupré", "Chauveau", "Jean-Lesage",
        "Jean-Talon", "La Peltrie", "Louis-Hébert", "Montmorency", "Portneuf",
        "Taschereau", "Vanier-Les Rivières"],
    "Mauricie": ["Champlain", "Laviolette–Saint-Maurice", "Maskinongé", "Trois-Rivières"],
    "Chaudière-Appalaches et Centre-du-Québec": [
        "Arthabaska–L'Érable", "Beauce-Nord", "Beauce-Sud", "Bellechasse",
        "Chutes-de-la-Chaudière", "Drummond–Bois-Francs", "Daniel-Johnson", "Lévis",
        "Lotbinière-Frontenac", "Marie-Lacoste-Gérin-Lajoie", "Mégantic",
        "Nicolet-Bécancour"],
    "Estrie": ["Brome-Missisquoi", "Granby", "Orford", "Richmond", "Saint-François",
               "Sherbrooke"],
    "Montérégie": [
        "Beauharnois", "Borduas", "Chambly", "Châteauguay", "Huntingdon", "Iberville",
        "La Pinière", "Pierre-Laporte", "La Prairie", "Marie-Victorin", "Montarville",
        "Richelieu", "Saint-Hyacinthe", "Saint-Jean", "Sanguinet", "Soulanges",
        "Taillon", "Vachon", "Vaudreuil", "Verchères"],
    "Montréal": [
        "Acadie", "Anjou–Louis-Riel", "Bourassa-Sauvé", "Camille-Laurin",
        "D'Arcy-McGee", "Gouin", "Hochelaga-Maisonneuve", "Jacques-Cartier",
        "Jeanne-Mance–Viger", "LaFontaine", "Laurier-Dorion", "Marguerite-Bourgeoys",
        "Marquette", "Maurice-Richard", "Mercier", "Mont-Royal–Outremont", "Nelligan",
        "Notre-Dame-de-Grâce", "Pointe-aux-Trembles", "Robert-Baldwin", "Rosemont",
        "Saint-Henri–Sainte-Anne", "Sainte-Marie–Saint-Jacques", "Saint-Laurent",
        "Verdun", "Viau", "Westmount–Saint-Louis"],
    "Laval": ["Chomedey", "Fabre", "Laval-des-Rapides", "Mille-Îles", "Sainte-Rose",
              "Vimont-Auteuil"],
    "Lanaudière": ["Berthier", "Joliette", "L'Assomption", "Les Plaines", "Masson",
                   "Repentigny", "Rousseau", "Terrebonne"],
    "Laurentides": ["Argenteuil", "Bellefeuille", "Bertrand", "Blainville",
                    "Deux-Montagnes", "Groulx", "Labelle", "Mirabel", "Prévost",
                    "Saint-Jérôme"],
    "Outaouais": ["Chapleau", "Gatineau", "Hull", "Papineau", "Pontiac"],
    "Abitibi-Témiscamingue et Nord-du-Québec": [
        "Abitibi-Est", "Abitibi-Ouest", "Rouyn-Noranda–Témiscamingue", "Ungava"],
}

# Mode de collecte habituel par maison (design effects §2 de la méthodologie).
HOUSE_MODES = {
    "Léger": "web", "Mainstreet": "ivr", "Pallas Data": "ivr", "Synopsis": "web",
    "SEGMA": "phone", "Research Co.": "web", "Angus Reid": "web", "EKOS": "ivr",
    "Liaison Strategies": "ivr", "Innovative Research": "web", "CROP": "web",
    "Forum": "ivr", "Abacus Data": "web", "Ipsos": "web",
}

POLLSTER_ALIASES = {
    "leger": "Léger", "léger": "Léger", "leger marketing": "Léger",
    "mainstreet research": "Mainstreet", "mainstreet": "Mainstreet",
    "mainstreet (exit poll)": "Mainstreet",
    "pallas data": "Pallas Data", "pallas": "Pallas Data",
    "research co.": "Research Co.", "research co": "Research Co.",
    "angus reid institute": "Angus Reid", "angus reid": "Angus Reid",
    "liaison strategies": "Liaison Strategies",
    "segma recherche": "SEGMA", "segma": "SEGMA",
    "ekos": "EKOS",
    "synopsis recherche": "Synopsis", "synopsis": "Synopsis",
    "innovative research group": "Innovative Research",
    "innovative research": "Innovative Research",
}


def norm_name(s: str) -> str:
    """Normalise un nom de circonscription (apostrophes, tirets, espaces)."""
    s = unicodedata.normalize("NFC", str(s)).strip()
    s = s.replace("’", "'").replace("–", "–").replace("—", "–")
    s = re.sub(r"\s+", " ", s)
    s = re.sub(r"\s*\[.*?\]\s*$", "", s)  # notes wiki [a]
    return s


def _dash_key(s: str) -> str:
    return norm_name(s).replace("–", "-").lower()


_RENAMES_BY_KEY = {}


def canon_district(s: str) -> str:
    if not _RENAMES_BY_KEY:
        _RENAMES_BY_KEY.update({_dash_key(k): v for k, v in RENAMES_2025.items()})
    n = norm_name(s)
    return _RENAMES_BY_KEY.get(_dash_key(n), n)


def _district_key(s: str) -> str:
    return _dash_key(canon_district(s))


def norm_pollster(s: str) -> str:
    key = norm_name(s).lower()
    key = re.sub(r"\s*\(.*\)$", "", key).strip()
    return POLLSTER_ALIASES.get(key, norm_name(s))


def fetch(url: str, tag: str) -> str:
    r = httpx.get(url, headers=UA, timeout=60, follow_redirects=True)
    r.raise_for_status()
    raw_dir = DATA_DIR / "raw"
    raw_dir.mkdir(parents=True, exist_ok=True)
    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    (raw_dir / f"{tag}_{stamp}.html").write_text(r.text)
    return r.text


def _num(x) -> float | None:
    if x is None or (isinstance(x, float) and pd.isna(x)):
        return None
    s = str(x).strip().replace("%", "").replace(",", "")
    s = re.sub(r"\[.*?\]", "", s)
    m = re.search(r"-?\d+(\.\d+)?", s)
    return float(m.group()) if m else None


def _parse_date(x, default_year: int | None = None) -> date | None:
    from dateutil import parser as dp
    s = re.sub(r"\[.*?\]", "", str(x)).strip()
    if not s or s.lower() in ("nan", "none"):
        return None
    # plages "September 30 – October 2, 2022" → prendre la fin
    s = re.split(r"[–—]|-(?=\s*[A-Z])", s)[-1].strip()
    try:
        d = dp.parse(s, default=datetime(default_year or 2000, 6, 15))
        return d.date()
    except (ValueError, OverflowError):
        return None


def _parse_date_range(x, default_year: int | None = None) -> tuple[date | None, date | None]:
    """Plage de terrain « Sept. 30 – Oct. 2, 2026 » → (début, fin).
    Sans plage : (None, fin)."""
    from dateutil import parser as dp
    s = re.sub(r"\[.*?\]", "", str(x)).strip()
    end = _parse_date(x, default_year)
    parts = re.split(r"[–—]|-(?=\s*[A-Z])", s)
    if end is None or len(parts) < 2:
        return None, end
    lead = parts[0].strip().rstrip(",")
    try:
        # « September 30 » hérite de l'année/du mois de la fin si absents
        start = dp.parse(lead, default=datetime(end.year, end.month, end.day)).date()
        if start <= end and (end - start).days <= 45:
            return start, end
    except (ValueError, OverflowError):
        pass
    return None, end


# Sections de la page dont les tables sont des sondages NATIONAUX; on exclut
# explicitement les ventilations (langue, région, âge) et les projections.
SECTION_ACCEPT = re.compile(r"opinion poll|campaign period|pre-?campaign", re.I)
SECTION_REJECT = re.compile(
    r"language|region|montreal|quebec city|age|gender|francophone|anglophone|"
    r"projection|leadership|approval|preferred premier|by-?election", re.I)


def iter_section_tables(html: str):
    """Itère (en-tête de section, HTML de la table) pour chaque wikitable —
    le contexte de section permet d'accepter la table nationale et la table de
    campagne (même à 1 rangée) tout en rejetant les ventilations régionales."""
    heads = [(m.start(), re.sub(r"<[^>]+>", "", m.group(2)).strip())
             for m in re.finditer(r"<h([2-5])[^>]*>(.*?)</h\1>", html, re.S)]
    section = "?"
    hi = 0
    for m in re.finditer(r'<table class="wikitable.*?</table>', html, re.S):
        while hi < len(heads) and heads[hi][0] < m.start():
            section = heads[hi][1]
            hi += 1
        yield section, m.group(0)


def _flat_cols(df: pd.DataFrame) -> list[str]:
    out = []
    for c in df.columns:
        if isinstance(c, tuple):
            parts = [str(p) for p in c if not str(p).startswith("Unnamed")]
            out.append(parts[-1] if parts else str(c[0]))
        else:
            out.append(str(c))
    return out


PARTY_COL_MAP = {"caq": "CAQ", "qs": "QS", "pq": "PQ", "liberal": "PLQ", "plq": "PLQ",
                 "pcq": "PCQ", "conservative": "PCQ", "other": "AUT"}


def parse_poll_table(df: pd.DataFrame, election_year: int, source_url: str) -> list[dict]:
    """Table de sondages Wikipédia → liste de dicts normalisés."""
    df = df.copy()
    df.columns = _flat_cols(df)
    cols = {c.lower().strip(): c for c in df.columns}
    org_col = next((cols[k] for k in cols if "polling organisation" in k or k == "firm"), None)
    date_col = next((cols[k] for k in cols if "last date" in k or "date" in k), None)
    if org_col is None or date_col is None:
        return []
    party_cols = {}
    for k, orig in cols.items():
        if k in PARTY_COL_MAP:
            party_cols[PARTY_COL_MAP[k]] = orig
    rows = []
    for _, r in df.iterrows():
        org = r.get(org_col)
        if org is None or pd.isna(org):
            continue
        org = str(org)
        vals = [str(r.get(c)) for c in df.columns]
        if len(set(vals)) <= 2 and len(org) > 40:      # rangée-bannière (événement)
            continue
        if "election" in org.lower():                   # rangée résultat officiel
            continue
        end = _parse_date(r.get(date_col), election_year)
        if end is None:
            continue
        shares = {}
        for party, c in party_cols.items():
            v = _num(r.get(c))
            if v is not None:
                shares[party] = v
        if len([p for p in shares if p != "AUT"]) < 4:
            continue
        n = _num(r.get(cols.get("sample size", cols.get("sample", ""))))
        moe = _num(r.get(cols.get("moe", "")))
        und_col = next((cols[k] for k in cols if "undecided" in k or "und." in k), None)
        undecided = _num(r.get(und_col)) if und_col else None
        start, _ = _parse_date_range(r.get(date_col), election_year)
        house = norm_pollster(org)
        rows.append({
            "pollster": house,
            "field_end": end.isoformat(),
            "field_start": (start or (end - timedelta(days=3))).isoformat(),
            "sample_size": int(n) if n else None,
            "moe": abs(moe) if moe else None,
            "undecided": undecided,
            "population": "adults",
            "mode": HOUSE_MODES.get(house, "unknown"),
            "region": "QC",
            "source_name": "Wikipédia (agrégation)",
            "source_url": source_url,
            "shares": shares,
        })
    return rows


def parse_polls_2026(html: str) -> list[dict]:
    """Tables de sondages NATIONAUX de la page 2026, sélectionnées par leur
    section (« Opinion polls », « Campaign period »…) — aucune contrainte de
    taille : la table de campagne est captée dès sa première rangée. Les
    ventilations (langue, région, projections) sont exclues par section."""
    out, seen = [], set()
    for section, thtml in iter_section_tables(html):
        if not SECTION_ACCEPT.search(section) or SECTION_REJECT.search(section):
            continue
        try:
            tables = pd.read_html(StringIO(thtml))
        except ValueError:
            continue
        for t in tables:
            cols = " ".join(str(c).lower() for c in t.columns)
            if ("polling organisation" in cols or "firm" in cols) and "caq" in cols:
                for row in parse_poll_table(t, 2026, URL_2026):
                    key = (row["pollster"], row["field_end"], row["sample_size"])
                    if key not in seen:      # une table peut apparaître deux fois
                        seen.add(key)
                        out.append(row)
    return out


URL_2026_FR = ("https://fr.wikipedia.org/wiki/Liste_de_sondages_sur_les_"
               "%C3%A9lections_g%C3%A9n%C3%A9rales_qu%C3%A9b%C3%A9coises_de_2026")
_MOIS = {"janvier": 1, "février": 2, "mars": 3, "avril": 4, "mai": 5, "juin": 6,
         "juillet": 7, "août": 8, "aout": 8, "septembre": 9, "octobre": 10,
         "novembre": 11, "décembre": 12, "decembre": 12}


def _parse_date_fr(s: str) -> date | None:
    m = re.search(r"(\d{1,2})(?:er)?\s+([a-zéû]+)\s+(\d{4})", str(s), re.I)
    if not m or m.group(2).lower() not in _MOIS:
        return None
    try:
        return date(int(m.group(3)), _MOIS[m.group(2).lower()], int(m.group(1)))
    except ValueError:
        return None


def parse_polls_2026_fr(html: str) -> list[dict]:
    """Wikipédia FR « Liste de sondages… 2026 » — seconde source d'agrégation,
    souvent plus fraîche que la page EN pendant la campagne. Colonnes :
    Dernier jour du sondage | CAQ | PLQ | QS | PQ | PCQ | Autres | Sondeur |
    Échantillon | ME | Source. Dédupliquée avec la page EN par (maison, fin)."""
    out = []
    for t in pd.read_html(StringIO(html)):
        t = t.copy()
        t.columns = _flat_cols(t)
        cols = {str(c).strip(): c for c in t.columns}
        if not ({"CAQ", "PQ", "PLQ"} <= set(cols) and
                any("dernier jour" in str(c).lower() for c in cols)):
            continue
        date_col = next(cols[c] for c in cols if "dernier jour" in c.lower())
        for _, r in t.iterrows():
            end = _parse_date_fr(r.get(date_col))
            org = r.get(cols.get("Sondeur"))
            if end is None or org is None or pd.isna(org):
                continue
            org = str(org)
            # rangée-bannière d'événement (cellule fusionnée) : sondeur qui
            # commence par une date ou contient un événement → ignorer
            if re.match(r"\d", org) or "élection" in org.lower() \
                    or "déclenchement" in org.lower() or len(org) > 60:
                continue
            sponsor = None
            if "/" in org:                      # « Synopsis / La Presse »
                org, sponsor = [x.strip() for x in org.split("/", 1)]
            shares = {}
            for party in ("CAQ", "PLQ", "QS", "PQ", "PCQ"):
                v = _num(r.get(cols.get(party)))
                if v is not None and 0 <= v <= 80:
                    shares[party] = v
            aut = _num(r.get(cols.get("Autres", "")))
            if aut is not None:
                shares["AUT"] = aut
            if len([p for p in shares if p != "AUT"]) < 4:
                continue
            if len(set(shares.values())) <= 2:   # cellule fusionnée répétée
                continue
            n = _num(r.get(cols.get("Échantillon", "")))
            house = norm_pollster(org)
            out.append({"pollster": house,
                        "field_end": end.isoformat(),
                        "field_start": (end - timedelta(days=3)).isoformat(),
                        "sample_size": int(n) if n and n > 40 else None,
                        "moe": abs(_num(r.get(cols.get("ME", ""))) or 0) or None,
                        "population": "adults",
                        "mode": HOUSE_MODES.get(house, "unknown"),
                        "region": "QC",
                        "source_name": "Wikipédia FR (agrégation)"
                                       + (f" · {sponsor}" if sponsor else ""),
                        "source_url": URL_2026_FR,
                        "shares": shares})
    return out


def parse_polls_2022(html: str) -> list[dict]:
    """Tables 'Timeline of opinion polls' de l'article 2022 (campagne + pré-campagne)."""
    tables = pd.read_html(StringIO(html))
    out = []
    for t in tables:
        cols = " ".join(str(c).lower() for c in t.columns)
        if "timeline of opinion polls" in cols and "caq" in cols:
            year = 2022
            parsed = parse_poll_table(t, year, URL_2022)
            out.extend(parsed)
    return out


def parse_riding_results_2022(html: str) -> pd.DataFrame:
    """Table % par parti et par circonscription (élection 2022)."""
    tables = pd.read_html(StringIO(html))
    target = None
    for t in tables:
        raw = [c for c in t.columns]
        heads = {str(c[0]) if isinstance(c, tuple) else str(c) for c in raw}
        if {"Riding", "CAQ", "QS", "PQ", "PLQ", "PCQ"} <= heads and len(t) > 100:
            target = t
            break
    if target is None:
        raise RuntimeError("Table des résultats par circonscription introuvable")
    # première sous-colonne '%' et 'Change (pp)' de chaque parti
    pct_col: dict[str, object] = {}
    chg_col: dict[str, object] = {}
    name_col = None
    for c in target.columns:
        top = str(c[0]) if isinstance(c, tuple) else str(c)
        sub = str(c[-1]) if isinstance(c, tuple) else ""
        if top == "Riding" and name_col is None:
            name_col = c
        elif top in ("CAQ", "QS", "PQ", "PLQ", "PCQ") and sub == "%" and top not in pct_col:
            pct_col[top] = c
        elif (top in ("CAQ", "QS", "PQ", "PLQ", "PCQ") and sub.startswith("Change")
              and top not in chg_col):
            chg_col[top] = c
    rows = []
    for _, r in target.iterrows():
        name = r.get(name_col)
        if name is None or pd.isna(name):
            continue
        name = norm_name(name)
        if name.lower().startswith(("total", "riding", "source")):
            continue
        rec = {"district": name}
        n_num = 0
        for p, c in pct_col.items():
            v = _num(r.get(c))
            if v is not None:
                n_num += 1
            rec[p] = v if v is not None else 0.0  # parti sans candidat
        if n_num < 3:  # rangée d'en-tête ou de total
            continue
        s = sum(rec[p] for p in pct_col)
        rec["AUT"] = round(max(0.0, 100.0 - s), 2)
        # reconstruction 2018 : % de 2022 − variation (pp)
        for p, c in chg_col.items():
            chg = _num(r.get(c))
            rec[f"prev_{p}"] = round(max(0.0, rec[p] - (chg if chg is not None else rec[p])), 2)
        s18 = sum(rec.get(f"prev_{p}", 0.0) for p in pct_col)
        rec["prev_AUT"] = round(max(0.0, 100.0 - s18), 2)
        rows.append(rec)
    return pd.DataFrame(rows)


def parse_turnout_2022(html: str) -> dict[str, float]:
    tables = pd.read_html(StringIO(html))
    for t in tables:
        heads = {str(p) for c in t.columns for p in (c if isinstance(c, tuple) else (c,))}
        if "Turnout" not in heads or len(t) < 100:
            continue
        name_col = t.columns[0]
        turn_col = next((c for c in t.columns
                         if isinstance(c, tuple) and "Turnout" in [str(p) for p in c]
                         and str(c[-1]) == "%"), None)
        if turn_col is not None:
            out = {}
            for _, r in t.iterrows():
                n, v = r.get(name_col), _num(r.get(turn_col))
                if n is not None and not pd.isna(n) and v is not None and 20 < v < 95:
                    out[norm_name(n)] = v
            if len(out) > 100:
                return out
    return {}


def parse_retirements_2026(html: str) -> set[str]:
    """Circonscriptions dont le député sortant ne se représente pas."""
    tables = pd.read_html(StringIO(html))
    for t in tables:
        cols = [str(c) for c in t.columns]
        if any("Electoral District" in c for c in cols) and any("Date announced" in c for c in cols):
            col = next(c for c in t.columns if "Electoral District" in str(c))
            return {canon_district(v) for v in t[col].dropna().astype(str)}
    return set()


def build_districts(riding_2022: pd.DataFrame, turnout: dict[str, float],
                    retirements: set[str]) -> pd.DataFrame:
    """Construit les 127 circonscriptions 2026 avec référence 2022 (transposition approx.)."""
    parties = ["CAQ", "PCQ", "PLQ", "PQ", "QS", "AUT"]
    base = {norm_name(r["district"]): {p: r[p] for p in parties}
            for _, r in riding_2022.iterrows()}
    region_of = {}
    for reg, names in REGION_RIDINGS.items():
        for n in names:
            region_of[_district_key(n)] = reg

    rows = []
    for old_name, shares in base.items():
        new_name = canon_district(old_name)
        reg = region_of.get(_district_key(new_name))
        rows.append({
            "district": new_name, "region": reg or "Inconnue",
            **{f"b_{p}": shares[p] for p in parties},
            "turnout_2022": turnout.get(old_name),
            "incumbent_party": max((p for p in parties if p != "AUT"),
                                   key=lambda p: shares[p]),
            "incumbent_running": new_name not in retirements,
            "is_new_2026": False,
            "baseline_source": f"Résultat 2022 ({old_name}) — Wikipédia/DGEQ",
        })
    for name, reg, parents in NEW_DISTRICTS_2026:
        ps = [base[p] for p in parents if p in base]
        shares = {p: round(sum(x[p] for x in ps) / len(ps), 2) for p in parties}
        rows.append({
            "district": name, "region": reg,
            **{f"b_{p}": shares[p] for p in parties},
            "turnout_2022": None,
            "incumbent_party": None, "incumbent_running": False, "is_new_2026": True,
            "baseline_source": "Moyenne des circonscriptions parentes ("
                               + ", ".join(parents) + ") — approximation transposition 2025",
        })
    df = pd.DataFrame(rows)
    missing = df[df.region == "Inconnue"]["district"].tolist()
    if missing:
        raise RuntimeError(f"Régions manquantes pour: {missing}")
    return df


def refresh_all(out_dir: Path | None = None, offline_html: dict[str, str] | None = None) -> dict:
    """Télécharge et normalise toutes les données; écrit les CSV de seed."""
    out_dir = out_dir or DATA_DIR
    out_dir.mkdir(parents=True, exist_ok=True)
    html26 = offline_html.get("2026") if offline_html else fetch(URL_2026, "wiki2026")
    html22 = offline_html.get("2022") if offline_html else fetch(URL_2022, "wiki2022")

    polls26 = parse_polls_2026(html26)
    # seconde agrégation : Wikipédia FR (souvent plus fraîche en campagne);
    # union dédupliquée par (maison, fin de terrain) — la EN fait foi si doublon
    try:
        html_fr = (offline_html.get("2026fr") if offline_html
                   else fetch(URL_2026_FR, "wiki2026fr"))
        from datetime import date as _d
        def _near(house, end_iso, days=1):
            e = _d.fromisoformat(end_iso)
            return any(q["pollster"] == house and
                       abs((_d.fromisoformat(q["field_end"]) - e).days) <= days
                       for q in polls26)
        extra = [p for p in parse_polls_2026_fr(html_fr)
                 if not _near(p["pollster"], p["field_end"])]
        polls26.extend(extra)
    except Exception:
        pass
    polls22 = parse_polls_2022(html22)
    riding = parse_riding_results_2022(html22)
    turnout = parse_turnout_2022(html22)
    retirements = parse_retirements_2026(html26)
    districts = build_districts(riding, turnout, retirements)

    accessed = datetime.now(timezone.utc).isoformat()
    for polls, tag in ((polls26, "polls_2026"), (polls22, "polls_2022")):
        recs = []
        for p in polls:
            for party, v in p["shares"].items():
                recs.append({**{k: p[k] for k in p if k != "shares"},
                             "party": party, "raw_value": v, "accessed_at": accessed})
        pd.DataFrame(recs).to_csv(out_dir / f"{tag}.csv", index=False)
    districts.to_csv(out_dir / "districts_2026.csv", index=False)
    riding.to_csv(out_dir / "riding_results_2022.csv", index=False)
    return {"polls_2026": len(polls26), "polls_2022": len(polls22),
            "districts": len(districts), "retirements": len(retirements)}


if __name__ == "__main__":
    print(refresh_all())