# ============================================================================== # Author: Simon-Pierre Boucher # File: restoka/geocode.py # Desc: Géocodage des adresses de restaurants (lat/lng OBLIGATOIRES, §5) via # Nominatim (OpenStreetMap) avec repli Adresses Québec (MERN/ArcGIS). # Cache persistant (geocode_cache), politesse 1 req/s, validation # provinciale des coordonnées. Adapté de louka/geocode.py. # ============================================================================== from __future__ import annotations import re import time import requests from . import db from .regions import attach_region, strip_accents NOMINATIM_URL = "https://nominatim.openstreetmap.org/search" # Repli officiel du gouvernement du Québec (couvre les adresses récentes) AQ_URL = ("https://servicescarto.mern.gouv.qc.ca/pes/rest/services/Territoire/" "Adresse_Geocodage/GeocodeServer/findAddressCandidates") AQ_MIN_SCORE = 75 # ASCII pur : le serveur ArcGIS retourne 500 si le User-Agent contient des accents USER_AGENT = "RestoKaBot/1.0 (agregateur restaurants Quebec; +contact@spboucher.ai)" REQUEST_DELAY = 1.1 # règle Nominatim : max 1 req/s RETRY_FAILED_AFTER = 30 * 86400 # garde-fou provincial (même boîte que schema.finalize()) _BBOX_QC = (44.50, 63.00, -80.00, -56.00) def norm_key(address: str, city: str) -> str: """Clé de cache : adresse+ville normalisées (casse, accents, espaces).""" s = strip_accents(f"{address} {city}".lower()) s = re.sub(r"\b(?:app?t?|suite|local|bureau|#)\.?\s*[\w-]+\b", " ", s) s = re.sub(r"[^a-z0-9]+", " ", s) return re.sub(r"\s+", " ", s).strip() class Geocoder: def __init__(self, con) -> None: self.con = con self.session = requests.Session() self.session.headers["User-Agent"] = USER_AGENT self._last = 0.0 def _throttle(self) -> None: wait = REQUEST_DELAY - (time.time() - self._last) if wait > 0: time.sleep(wait) @staticmethod def _clean(address: str) -> str: """Nettoyages qui aident Nominatim sur les adresses de commerces : suffixes de local/suite, mentions d'unité après virgule.""" s = (address or "").strip() s = re.sub(r",?\s*(?:local|suite|bureau|unite|app\.?|apt\.?|#|s?suite)\s*" r"[\w-]+\s*$", "", s, flags=re.I) s = re.sub(r",\s*[A-Z]?\d+[A-Z]?\s*$", "", s) # « …, F10 » return s.strip().strip(",") def _query(self, address: str, city: str) -> str: q = self._clean(address) if city and strip_accents(city.lower()) not in strip_accents(q.lower()): q += f", {city}" if "quebec" not in strip_accents(q.lower()) and "qc" not in q.lower(): q += ", Québec" return q + ", Canada" def _nominatim(self, q: str) -> tuple[float, float] | None: self._throttle() try: resp = self.session.get(NOMINATIM_URL, params={ "q": q, "format": "jsonv2", "limit": 1, "countrycodes": "ca", }, timeout=20) self._last = time.time() resp.raise_for_status() hits = resp.json() except Exception: self._last = time.time() return None if not hits: return None try: return float(hits[0]["lat"]), float(hits[0]["lon"]) except (KeyError, ValueError): return None def _adresses_quebec(self, address: str, city: str) -> tuple[float, float] | None: self._throttle() try: resp = self.session.get(AQ_URL, params={ "SingleLine": f"{self._clean(address)}, {city}", "f": "json", "outSR": '{"wkid":4326}', "maxLocations": 1, }, timeout=20) self._last = time.time() resp.raise_for_status() cands = resp.json().get("candidates") or [] except Exception: self._last = time.time() return None if not cands or cands[0].get("score", 0) < AQ_MIN_SCORE: return None loc = cands[0].get("location") or {} try: return float(loc["y"]), float(loc["x"]) except (KeyError, ValueError): return None def geocode(self, address: str, city: str) -> tuple[float, float] | None: """Géocode avec cache. None si échec (JAMAIS de coordonnées bidon).""" if not address: return None key = norm_key(address, city) row = self.con.execute( "SELECT lat, lng, failed, ts FROM geocode_cache WHERE address=?", (key,)).fetchone() if row is not None: if not row["failed"]: return row["lat"], row["lng"] if time.time() - (row["ts"] or 0) < RETRY_FAILED_AFTER: return None q = self._query(address, city) hit = self._nominatim(q) provider = "nominatim" if hit is None or not (_BBOX_QC[0] <= hit[0] <= _BBOX_QC[1] and _BBOX_QC[2] <= hit[1] <= _BBOX_QC[3]): hit = self._adresses_quebec(address, city) provider = "adresses-quebec" if hit is not None and (_BBOX_QC[0] <= hit[0] <= _BBOX_QC[1] and _BBOX_QC[2] <= hit[1] <= _BBOX_QC[3]): self.con.execute( "INSERT OR REPLACE INTO geocode_cache (address, lat, lng," " provider, failed, ts) VALUES (?,?,?,?,0,?)", (key, hit[0], hit[1], provider, time.time())) self.con.commit() return hit self.con.execute( "INSERT OR REPLACE INTO geocode_cache (address, lat, lng, provider," " failed, ts) VALUES (?,NULL,NULL,NULL,1,?)", (key, time.time())) self.con.commit() return None def run_batch(limit: int = 200) -> dict: """Géocode les restos actifs sans coordonnées (max `limit` requêtes), puis rattache la région manquante à partir des coordonnées obtenues.""" con = db.connect() geo = Geocoder(con) done = failed = 0 rows = con.execute( "SELECT uid, address, city, postal_code, region FROM restaurants" " WHERE active=1 AND (lat IS NULL OR lng IS NULL) AND geocode_failed=0" " LIMIT ?", (limit,)).fetchall() for r in rows: hit = geo.geocode(r["address"], r["city"]) if hit is None: con.execute("UPDATE restaurants SET geocode_failed=1 WHERE uid=?", (r["uid"],)) failed += 1 continue region = r["region"] or attach_region(r["city"], r["postal_code"], hit[0], hit[1]) con.execute( "UPDATE restaurants SET lat=?, lng=?, region=?, geocode_failed=0" " WHERE uid=?", (hit[0], hit[1], region, r["uid"])) done += 1 con.commit() con.close() stats = {"geocoded": done, "failed": failed, "pending": max(0, len(rows) - done - failed)} print(f"[resto-ka] geocode: {stats}") return stats