dédup inter-agrégateurs : même sondage daté à ±1 j (EN vs FR) — union + purge
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2 changed files +19 −2
modified
backend/app/ingest/wikipedia.py
+7 −2
@@ -547,9 +547,14 @@ def refresh_all(out_dir: Path | None = None, offline_html: dict[str, str] | None | ||
| 547 | 547 | try: |
| 548 | 548 | html_fr = (offline_html.get("2026fr") if offline_html |
| 549 | 549 | else fetch(URL_2026_FR, "wiki2026fr")) |
| 550 | − seen = {(p["pollster"], p["field_end"]) for p in polls26} | |
| 550 | + from datetime import date as _d | |
| 551 | + def _near(house, end_iso, days=1): | |
| 552 | + e = _d.fromisoformat(end_iso) | |
| 553 | + return any(q["pollster"] == house and | |
| 554 | + abs((_d.fromisoformat(q["field_end"]) - e).days) <= days | |
| 555 | + for q in polls26) | |
| 551 | 556 | extra = [p for p in parse_polls_2026_fr(html_fr) |
| 552 | − if (p["pollster"], p["field_end"]) not in seen] | |
| 557 | + if not _near(p["pollster"], p["field_end"])] | |
| 553 | 558 | polls26.extend(extra) |
| 554 | 559 | except Exception: |
| 555 | 560 | pass |
modified
backend/app/seed.py
+12 −0
@@ -109,6 +109,18 @@ def seed_all(db: Session | None = None) -> dict: | ||
| 109 | 109 | seen_polls[key], poll = poll, kept |
| 110 | 110 | db.delete(poll) |
| 111 | 111 | n_dupes += 1 |
| 112 | + # même maison, même taille d'échantillon, fins de terrain à ±1 jour : | |
| 113 | + # même sondage daté différemment par deux agrégateurs (EN vs FR) | |
| 114 | + by_house: dict[tuple, list] = {} | |
| 115 | + for poll in db.query(Mo.Poll).order_by(Mo.Poll.field_end.asc()).all(): | |
| 116 | + if poll.sample_size: | |
| 117 | + by_house.setdefault((poll.pollster_id, poll.election_id, | |
| 118 | + poll.sample_size), []).append(poll) | |
| 119 | + for group in by_house.values(): | |
| 120 | + for a, b in zip(group, group[1:]): | |
| 121 | + if abs((b.field_end - a.field_end).days) <= 1 and a in db: | |
| 122 | + db.delete(a) # garder la date la plus récente (FR corrige) | |
| 123 | + n_dupes += 1 | |
| 112 | 124 | if n_dupes: |
| 113 | 125 | db.flush() |
| 114 | 126 | stats["dupes_purgés"] = n_dupes |
| 115 | 127 | |