spb/resto-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   restoka/geocode.py4# Desc:   Géocodage des adresses de restaurants (lat/lng OBLIGATOIRES, §5) via5#         Nominatim (OpenStreetMap) avec repli Adresses Québec (MERN/ArcGIS).6#         Cache persistant (geocode_cache), politesse 1 req/s, validation7#         provinciale des coordonnées. Adapté de louka/geocode.py.8# ==============================================================================9from __future__ import annotations1011import re12import time1314import requests1516from . import db17from .regions import attach_region, strip_accents1819NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"20# Repli officiel du gouvernement du Québec (couvre les adresses récentes)21AQ_URL = ("https://servicescarto.mern.gouv.qc.ca/pes/rest/services/Territoire/"22          "Adresse_Geocodage/GeocodeServer/findAddressCandidates")23AQ_MIN_SCORE = 7524# ASCII pur : le serveur ArcGIS retourne 500 si le User-Agent contient des accents25USER_AGENT = "RestoKaBot/1.0 (agregateur restaurants Quebec; +contact@spboucher.ai)"26REQUEST_DELAY = 1.1          # règle Nominatim : max 1 req/s27RETRY_FAILED_AFTER = 30 * 864002829# garde-fou provincial (même boîte que schema.finalize())30_BBOX_QC = (44.50, 63.00, -80.00, -56.00)313233def norm_key(address: str, city: str) -> str:34    """Clé de cache : adresse+ville normalisées (casse, accents, espaces)."""35    s = strip_accents(f"{address} {city}".lower())36    s = re.sub(r"\b(?:app?t?|suite|local|bureau|#)\.?\s*[\w-]+\b", " ", s)37    s = re.sub(r"[^a-z0-9]+", " ", s)38    return re.sub(r"\s+", " ", s).strip()394041class Geocoder:42    def __init__(self, con) -> None:43        self.con = con44        self.session = requests.Session()45        self.session.headers["User-Agent"] = USER_AGENT46        self._last = 0.04748    def _throttle(self) -> None:49        wait = REQUEST_DELAY - (time.time() - self._last)50        if wait > 0:51            time.sleep(wait)5253    @staticmethod54    def _clean(address: str) -> str:55        """Nettoyages qui aident Nominatim sur les adresses de commerces :56        suffixes de local/suite, mentions d'unité après virgule."""57        s = (address or "").strip()58        s = re.sub(r",?\s*(?:local|suite|bureau|unite|app\.?|apt\.?|#|s?suite)\s*"59                   r"[\w-]+\s*$", "", s, flags=re.I)60        s = re.sub(r",\s*[A-Z]?\d+[A-Z]?\s*$", "", s)     # « …, F10 »61        return s.strip().strip(",")6263    def _query(self, address: str, city: str) -> str:64        q = self._clean(address)65        if city and strip_accents(city.lower()) not in strip_accents(q.lower()):66            q += f", {city}"67        if "quebec" not in strip_accents(q.lower()) and "qc" not in q.lower():68            q += ", Québec"69        return q + ", Canada"7071    def _nominatim(self, q: str) -> tuple[float, float] | None:72        self._throttle()73        try:74            resp = self.session.get(NOMINATIM_URL, params={75                "q": q, "format": "jsonv2", "limit": 1, "countrycodes": "ca",76            }, timeout=20)77            self._last = time.time()78            resp.raise_for_status()79            hits = resp.json()80        except Exception:81            self._last = time.time()82            return None83        if not hits:84            return None85        try:86            return float(hits[0]["lat"]), float(hits[0]["lon"])87        except (KeyError, ValueError):88            return None8990    def _adresses_quebec(self, address: str, city: str) -> tuple[float, float] | None:91        self._throttle()92        try:93            resp = self.session.get(AQ_URL, params={94                "SingleLine": f"{self._clean(address)}, {city}", "f": "json",95                "outSR": '{"wkid":4326}', "maxLocations": 1,96            }, timeout=20)97            self._last = time.time()98            resp.raise_for_status()99            cands = resp.json().get("candidates") or []100        except Exception:101            self._last = time.time()102            return None103        if not cands or cands[0].get("score", 0) < AQ_MIN_SCORE:104            return None105        loc = cands[0].get("location") or {}106        try:107            return float(loc["y"]), float(loc["x"])108        except (KeyError, ValueError):109            return None110111    def geocode(self, address: str, city: str) -> tuple[float, float] | None:112        """Géocode avec cache. None si échec (JAMAIS de coordonnées bidon)."""113        if not address:114            return None115        key = norm_key(address, city)116        row = self.con.execute(117            "SELECT lat, lng, failed, ts FROM geocode_cache WHERE address=?",118            (key,)).fetchone()119        if row is not None:120            if not row["failed"]:121                return row["lat"], row["lng"]122            if time.time() - (row["ts"] or 0) < RETRY_FAILED_AFTER:123                return None124        q = self._query(address, city)125        hit = self._nominatim(q)126        provider = "nominatim"127        if hit is None or not (_BBOX_QC[0] <= hit[0] <= _BBOX_QC[1]128                               and _BBOX_QC[2] <= hit[1] <= _BBOX_QC[3]):129            hit = self._adresses_quebec(address, city)130            provider = "adresses-quebec"131        if hit is not None and (_BBOX_QC[0] <= hit[0] <= _BBOX_QC[1]132                                and _BBOX_QC[2] <= hit[1] <= _BBOX_QC[3]):133            self.con.execute(134                "INSERT OR REPLACE INTO geocode_cache (address, lat, lng,"135                " provider, failed, ts) VALUES (?,?,?,?,0,?)",136                (key, hit[0], hit[1], provider, time.time()))137            self.con.commit()138            return hit139        self.con.execute(140            "INSERT OR REPLACE INTO geocode_cache (address, lat, lng, provider,"141            " failed, ts) VALUES (?,NULL,NULL,NULL,1,?)", (key, time.time()))142        self.con.commit()143        return None144145146def run_batch(limit: int = 200) -> dict:147    """Géocode les restos actifs sans coordonnées (max `limit` requêtes),148    puis rattache la région manquante à partir des coordonnées obtenues."""149    con = db.connect()150    geo = Geocoder(con)151    done = failed = 0152    rows = con.execute(153        "SELECT uid, address, city, postal_code, region FROM restaurants"154        " WHERE active=1 AND (lat IS NULL OR lng IS NULL) AND geocode_failed=0"155        " LIMIT ?", (limit,)).fetchall()156    for r in rows:157        hit = geo.geocode(r["address"], r["city"])158        if hit is None:159            con.execute("UPDATE restaurants SET geocode_failed=1 WHERE uid=?",160                        (r["uid"],))161            failed += 1162            continue163        region = r["region"] or attach_region(r["city"], r["postal_code"],164                                              hit[0], hit[1])165        con.execute(166            "UPDATE restaurants SET lat=?, lng=?, region=?, geocode_failed=0"167            " WHERE uid=?", (hit[0], hit[1], region, r["uid"]))168        done += 1169    con.commit()170    con.close()171    stats = {"geocoded": done, "failed": failed, "pending": max(0, len(rows) - done - failed)}172    print(f"[resto-ka] geocode: {stats}")173    return stats174