dédup inter-agrégateurs : même sondage daté à ±1 j (EN vs FR) — union + purge
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2 changed files +19 −2
modified
backend/app/ingest/wikipedia.py
+7 −2
@@ -547,9 +547,14 @@ def refresh_all(out_dir: Path | None = None, offline_html: dict[str, str] | None | |||||
| 547 | try: | 547 | try: | ||
| 548 | html_fr = (offline_html.get("2026fr") if offline_html | 548 | html_fr = (offline_html.get("2026fr") if offline_html | ||
| 549 | else fetch(URL_2026_FR, "wiki2026fr")) | 549 | else fetch(URL_2026_FR, "wiki2026fr")) | ||
| 550 | − seen = {(p["pollster"], p["field_end"]) for p in polls26} | 550 | + from datetime import date as _d | ||
| 551 | + def _near(house, end_iso, days=1): | ||||
| 552 | + e = _d.fromisoformat(end_iso) | ||||
| 553 | + return any(q["pollster"] == house and | ||||
| 554 | + abs((_d.fromisoformat(q["field_end"]) - e).days) <= days | ||||
| 555 | + for q in polls26) | ||||
| 551 | extra = [p for p in parse_polls_2026_fr(html_fr) | 556 | extra = [p for p in parse_polls_2026_fr(html_fr) | ||
| 552 | − if (p["pollster"], p["field_end"]) not in seen] | 557 | + if not _near(p["pollster"], p["field_end"])] | ||
| 553 | polls26.extend(extra) | 558 | polls26.extend(extra) | ||
| 554 | except Exception: | 559 | except Exception: | ||
| 555 | pass | 560 | pass | ||
modified
backend/app/seed.py
+12 −0
@@ -109,6 +109,18 @@ def seed_all(db: Session | None = None) -> dict: | |||||
| 109 | seen_polls[key], poll = poll, kept | 109 | seen_polls[key], poll = poll, kept | ||
| 110 | db.delete(poll) | 110 | db.delete(poll) | ||
| 111 | n_dupes += 1 | 111 | n_dupes += 1 | ||
| 112 | + # même maison, même taille d'échantillon, fins de terrain à ±1 jour : | ||||
| 113 | + # même sondage daté différemment par deux agrégateurs (EN vs FR) | ||||
| 114 | + by_house: dict[tuple, list] = {} | ||||
| 115 | + for poll in db.query(Mo.Poll).order_by(Mo.Poll.field_end.asc()).all(): | ||||
| 116 | + if poll.sample_size: | ||||
| 117 | + by_house.setdefault((poll.pollster_id, poll.election_id, | ||||
| 118 | + poll.sample_size), []).append(poll) | ||||
| 119 | + for group in by_house.values(): | ||||
| 120 | + for a, b in zip(group, group[1:]): | ||||
| 121 | + if abs((b.field_end - a.field_end).days) <= 1 and a in db: | ||||
| 122 | + db.delete(a) # garder la date la plus récente (FR corrige) | ||||
| 123 | + n_dupes += 1 | ||||
| 112 | if n_dupes: | 124 | if n_dupes: | ||
| 113 | db.flush() | 125 | db.flush() | ||
| 114 | stats["dupes_purgés"] = n_dupes | 126 | stats["dupes_purgés"] = n_dupes | ||
| 115 | 127 | ||||