spb/resto-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   restoka/connectors/ubereats.py4# Desc:   Connecteur d'ENRICHISSEMENT Uber Eats (menus livraison + notes) —5#         SANS clé API. Découverte par les sitemaps publics (robots.txt ->6#         sitemap-store-*.xml.gz, URLs /ca/store/<slug>/<uuid>, index local7#         data/ubereats-stores.json rafraîchi tous les 30 j). Chaque page de8#         resto embarque un état React Query (__REACT_QUERY_STATE__) avec le9#         menu complet (sections + items + prix en cents), la note Uber Eats,10#         l'adresse, le téléphone et les GPS. Croisement CONSERVATEUR avec un11#         resto EXISTANT seulement (téléphone identique, OU code postal + numéro12#         civique identiques, OU GPS <120 m + nom similaire) : n'émet AUCUNE13#         fiche. Menu -> table menus en price_context `delivery` (prix majorés14#         ~25-30 %, CLAUDE.md §6.2), note -> details.ubereats.15#16#         Économe : Scrapfly ASP (1 page ≈ 1 crédit constaté), budget17#         UBEREATS_BUDGET (défaut 60) pages resto/cycle, verdict par magasin18#         mis en cache (detail_cache), re-visite 30 j. Conformité §15 :19#         extraction minimale, usage d'appoint (restos sans menu ailleurs),20#         prix toujours étiquetés `delivery`, lien source conservé.21# ==============================================================================22from __future__ import annotations2324import base6425import datetime26import gzip27import json28import os29import re30import sys31import time32import urllib.parse33from pathlib import Path3435from ..inspections import _CIVIC_RE, _name_similar, norm_name36from ..normalize import normalize_phone37from ..regions import strip_accents38from ..schema import Restaurant39from .base import BaseConnector, SkipSource4041ROBOTS_URL = "https://www.ubereats.com/robots.txt"42INDEX_PATH = Path(__file__).resolve().parents[2] / "data" / "ubereats-stores.json"43INDEX_REFRESH_DAYS = 30              # les sitemaps bougent peu44MAX_BUDGET = int(os.environ.get("UBEREATS_BUDGET", "60"))  # pages resto/cycle45REFRESH_DAYS = 30                    # re-visite d'un resto (hit comme miss)46MAX_CANDIDATES = 3                   # magasins testés par resto, max47MAX_CONSECUTIVE_FAILURES = 348MAX_GPS_M = 120.04950_STATE_RE = re.compile(r"__REACT_QUERY_STATE__\">(.*?)</script>", re.S)51_STORE_URL_RE = re.compile(r"/ca/store/([^/]+)/([A-Za-z0-9_-]{20,})")525354def decode_state(page_html: str) -> dict | None:55    """Décode le bloc __REACT_QUERY_STATE__ (JSON avec « " » encodé \\u002256    et « \\ » encodé %5C) d'une page Uber Eats."""57    m = _STATE_RE.search(page_html)58    if not m:59        return None60    t = m.group(1).strip().replace("\\u0022", '"').replace("%5C", "\\")61    try:62        return json.loads(t)63    except ValueError:64        return None656667def store_payload(state: dict) -> dict | None:68    """Extrait le payload getStoreV1 (fiche resto complète) de l'état."""69    for q in (state or {}).get("queries") or []:70        qk = q.get("queryKey")71        if isinstance(qk, list) and qk and qk[0] == "getStoreV1":72            data = (q.get("state") or {}).get("data")73            if isinstance(data, dict):74                return data75    return None767778def build_menu(store: dict, captured_at: str) -> dict | None:79    """Menu standard Resto·Ka (CLAUDE.md §5.2) depuis catalogSectionsMap.80    Prix Uber Eats en cents -> dollars, contexte `delivery` obligatoire."""81    csm = store.get("catalogSectionsMap") or {}82    sections_out: list[dict] = []83    seen: set[str] = set()84    for sections in csm.values():85        for sec in sections or []:86            payload = ((sec.get("payload") or {})87                       .get("standardItemsPayload") or {})88            name = ((payload.get("title") or {}).get("text") or "").strip()89            items_in = payload.get("catalogItems") or []90            if not name or not items_in or name in seen:91                continue92            items = []93            for it in items_in:94                title = (it.get("title") or "").strip()95                if not title:96                    continue97                price = it.get("price")98                items.append({99                    "name": title,100                    "description": (it.get("itemDescription") or "").strip(),101                    "price": round(price / 100.0, 2)102                    if isinstance(price, (int, float)) and price > 0 else None,103                })104            if items:105                seen.add(name)106                sections_out.append({"name": name, "items": items})107        break                          # une seule vue du catalogue suffit108    if not sections_out:109        return None110    return {111        "price_context": "delivery",112        "price_source": "ubereats",113        "currency": store.get("currencyCode") or "CAD",114        "captured_at": captured_at,115        "sections": sections_out,116    }117118119def _review_count(store: dict) -> int:120    raw = str((store.get("rating") or {}).get("reviewCount") or "")121    m = re.search(r"\d+", raw.replace(",", "").replace(" ", ""))122    return int(m.group(0)) if m else 0123124125def _haversine_m(lat1, lng1, lat2, lng2) -> float:126    import math127    r = 6371000.0128    p1, p2 = math.radians(lat1), math.radians(lat2)129    dp, dl = math.radians(lat2 - lat1), math.radians(lng2 - lng1)130    a = (math.sin(dp / 2) ** 2131         + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2)132    return 2 * r * math.asin(math.sqrt(a))133134135def verify_store(row, store: dict) -> str:136    """Vérifie qu'une page magasin Uber Eats correspond bien au resto de la137    base — CONSERVATEUR : téléphone identique, OU code postal + numéro civique138    identiques, OU GPS <120 m + nom similaire. Retourne le mode de croisement139    ('' = pas le même établissement)."""140    loc = store.get("location") or {}141    phone = normalize_phone(store.get("phoneNumber"))142    if phone and normalize_phone(row["phone"]) == phone:143        return "telephone"144    postal = (loc.get("postalCode") or "").replace(" ", "").upper()145    rpostal = (row["postal_code"] or "").replace(" ", "").upper()146    civic_m = _CIVIC_RE.match(loc.get("streetAddress") or "")147    rcivic_m = _CIVIC_RE.match(row["address"] or "")148    if postal and rpostal == postal and civic_m and rcivic_m \149            and civic_m.group(1) == rcivic_m.group(1):150        return "postal+civique"151    lat, lng = loc.get("latitude"), loc.get("longitude")152    if (lat is not None and lng is not None153            and row["lat"] is not None and row["lng"] is not None154            and _haversine_m(row["lat"], row["lng"], lat, lng) <= MAX_GPS_M):155        addr_tokens = set(re.sub(r"[^a-z0-9]+", " ", strip_accents(156            (loc.get("address") or "").lower())).split())157        if _name_similar(norm_name(row["name"]),158                         norm_name(store.get("title") or ""), addr_tokens):159            return "gps+nom"160    return ""161162163def slugify(name: str) -> str:164    """Slug façon Uber Eats : « McDonald's » -> « mcdonalds »,165    « Café Dépôt » -> « cafe-depot » (apostrophes supprimées, pas remplacées)."""166    s = strip_accents((name or "").lower())167    s = re.sub(r"['’´`.]", "", s)168    s = re.sub(r"&", " and ", s)169    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")170    return s171172173class UberEatsConnector(BaseConnector):174    source_id = "ubereats"175    request_delay = 1.0176    timeout = 60177    use_detail_cache = False          # verdicts gérés à la main (detail_cache)178    enrichment_only = True            # n'émet aucune fiche (ingest.run)179180    # -- index sitemap ---------------------------------------------------------181    def _fetch_sitemap(self, url: str) -> str:182        result = self.scrapfly(url, render_js=False)183        content = result.get("content") or ""184        if url.endswith(".gz") or content[:20].startswith("H4sI"):185            try:186                return gzip.decompress(base64.b64decode(content)) \187                    .decode("utf-8", "replace")188            except (ValueError, OSError):189                return content190        return content191192    def load_index(self) -> dict[str, list[str]]:193        """Index slug -> [URLs /ca/store/…] depuis les sitemaps publics,194        en cache local 30 jours (data/ubereats-stores.json)."""195        if INDEX_PATH.exists():196            try:197                data = json.loads(INDEX_PATH.read_text(encoding="utf-8"))198                if time.time() - float(data.get("fetched_at") or 0) \199                        < INDEX_REFRESH_DAYS * 86400:200                    return data.get("stores") or {}201            except (ValueError, OSError):202                pass203        robots = self._fetch_sitemap(ROBOTS_URL)204        sitemaps = re.findall(r"Sitemap:\s*(\S*sitemap-store\S*)", robots)205        if not sitemaps:206            raise RuntimeError("robots.txt Uber Eats sans sitemap-store "207                               "(blocage ?)")208        stores: dict[str, list[str]] = {}209        for sm_url in sitemaps:210            xml = self._fetch_sitemap(sm_url)211            for loc in re.findall(r"<loc>([^<]+)</loc>", xml):212                mm = _STORE_URL_RE.search(loc)213                if mm:214                    slug = urllib.parse.unquote(mm.group(1)).lower()215                    stores.setdefault(slug, [])216                    if loc not in stores[slug]:217                        stores[slug].append(loc)218        INDEX_PATH.parent.mkdir(parents=True, exist_ok=True)219        INDEX_PATH.write_text(json.dumps(220            {"fetched_at": time.time(), "stores": stores},221            ensure_ascii=False), encoding="utf-8")222        print(f"[resto-ka] ubereats: index sitemap rafraîchi — "223              f"{len(stores)} slug(s) canadiens")224        return stores225226    @staticmethod227    def candidates_for(name: str, slugs: list[str]) -> list[str]:228        """Slugs Uber Eats candidats pour un nom de resto (préfixe strict)."""229        base = slugify(name)230        if len(base) < 5:231            return []232        import bisect233        i = bisect.bisect_left(slugs, base)234        out = []235        while i < len(slugs) and len(out) < MAX_CANDIDATES:236            s = slugs[i]237            if s == base or s.startswith(base + "-"):238                out.append(s)239                i += 1240            else:241                break242        return out243244    # -- cycle -----------------------------------------------------------------245    def _now(self) -> str:246        return datetime.datetime.now(datetime.timezone.utc) \247            .strftime("%Y-%m-%dT%H:%M:%SZ")248249    def _fresh(self, stamp: str, now: float, stale_s: float) -> bool:250        try:251            ts = datetime.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ") \252                .replace(tzinfo=datetime.timezone.utc).timestamp()253            return ts > now - stale_s254        except (ValueError, TypeError):255            return False256257    def _details_payload(self, store: dict, url: str, how: str) -> dict:258        rating = store.get("rating") or {}259        return {260            "rating": rating.get("ratingValue"),261            "review_count": _review_count(store),262            "price_bucket": store.get("priceBucket"),263            "url": url.split("?")[0],264            "uuid": store.get("uuid"),265            "matched_by": how,266            "fetched_at": self._now(),267        }268269    def _probe_store(self, con, row, url: str) -> tuple[bool, bool]:270        """Visite une page magasin et l'attache au resto si c'est le même271        établissement. Retourne (matched, menu_added)."""272        from .. import db273        mm = _STORE_URL_RE.search(url)274        store_uuid = mm.group(2) if mm else url275        cached = db.get_cached_detail(con, self.source_id, store_uuid,276                                      "verdict-v1")277        if cached is not None and cached.get("matched_uid") != row["uid"]:278            return False, False       # déjà identifié comme un autre resto279        result = self.scrapfly(url, render_js=False)280        status = result.get("status_code") or 0281        if status in (400, 404, 410, 451):     # magasin retiré d'Uber Eats282            db.put_cached_detail(con, self.source_id, store_uuid,283                                 "verdict-v1", {"matched_uid": None,284                                                "gone": status})285            return False, False286        if status != 200:287            raise RuntimeError(f"HTTP {status}")288        state = decode_state(result.get("content") or "")289        store = store_payload(state or {})290        if not store:291            raise RuntimeError("payload getStoreV1 absent")292        how = verify_store(row, store)293        if not how:294            db.put_cached_detail(con, self.source_id, store_uuid,295                                 "verdict-v1", {"matched_uid": None})296            return False, False297        db.put_cached_detail(con, self.source_id, store_uuid, "verdict-v1",298                             {"matched_uid": row["uid"]})299        menu = build_menu(store, self._now())300        menu_added = False301        if menu:302            # validation stricte du schéma menu (prix implausibles, etc.)303            Restaurant(source=self.source_id, external_id=store_uuid,304                       name=row["name"], menu=menu)._validate_menu()305            db.upsert_menu(con, row["uid"], menu, time.time())306            menu_added = True307        db.merge_details(con, row["uid"],308                         {"ubereats": self._details_payload(store, url, how)})309        con.commit()310        return True, menu_added311312    def fetch(self) -> list[Restaurant]:313        if not os.environ.get("SCRAPFLY_KEY"):314            raise SkipSource("SCRAPFLY_KEY manquant (.env) — Uber Eats est "315                             "derrière Cloudflare, scraping direct impossible")316        from .. import db317        index = self.load_index()318        slugs = sorted(index.keys())319        con = db.connect()320        now = time.time()321        stale_s = REFRESH_DAYS * 86400.0322        budget = MAX_BUDGET323        matched = menus = misses = failures_row = 0324        rows = con.execute(325            "SELECT uid, name, address, city, postal_code, phone, lat, lng,"326            " details,"327            " EXISTS (SELECT 1 FROM menus m WHERE m.uid=restaurants.uid)"328            "   AS has_menu"329            " FROM restaurants WHERE active=1 AND dup_of IS NULL AND name<>''"330            " AND (phone<>'' OR (postal_code<>'' AND address<>'')"331            "      OR (lat IS NOT NULL AND lng IS NOT NULL))"332            " ORDER BY has_menu ASC, phone<>'' DESC, updated_at DESC"333        ).fetchall()334        for row in rows:335            if budget <= 0:336                break337            if failures_row >= MAX_CONSECUTIVE_FAILURES:338                print("[resto-ka] ubereats: Scrapfly bloqué "339                      f"{failures_row} fois de suite — arrêt du cycle",340                      file=sys.stderr)341                break342            try:343                details = json.loads(row["details"] or "{}")344            except ValueError:345                details = {}346            ue = details.get("ubereats") or {}347            if self._fresh(ue.get("fetched_at", ""), now, stale_s):348                continue              # déjà frais (<30 j)349            probe = details.get("ubereats_probe") or {}350            if self._fresh(probe.get("fetched_at", ""), now, stale_s):351                continue              # échec récent : re-visite dans 30 j352            if ue.get("url"):         # déjà croisé : rafraîchir directement353                urls = [ue["url"]]354            else:355                cand = self.candidates_for(row["name"], slugs)356                urls = [u for s in cand for u in index.get(s, [])]357            if not urls:358                continue              # aucun candidat : pas de marqueur,359                                      # l'index du mois prochain peut changer360            found = False361            for url in urls[:MAX_CANDIDATES]:362                if budget <= 0:363                    break364                budget -= 1365                try:366                    ok, menu_added = self._probe_store(con, row, url)367                except Exception as exc:368                    failures_row += 1369                    print(f"[resto-ka] ubereats: {row['uid']} erreur: {exc}",370                          file=sys.stderr)371                    continue372                failures_row = 0373                if ok:374                    matched += 1375                    menus += 1 if menu_added else 0376                    found = True377                    break378            if not found and not ue.get("url"):379                db.merge_details(con, row["uid"],380                                 {"ubereats_probe":381                                  {"miss": "aucun magasin correspondant",382                                   "fetched_at": self._now()}})383                con.commit()384                misses += 1385        con.commit()386        con.close()387        self.enriched_count = matched388        self.enrich_message = (f"{matched} resto(s) croisés Uber Eats "389                               f"({menus} menu(s) livraison), {misses} sans "390                               f"correspondance, budget restant "391                               f"{max(budget, 0)} page(s)")392        print(f"[resto-ka] ubereats: {self.enrich_message}")393        return []394