spb/resto-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   restoka/connectors/doordash.py4# Desc:   Connecteur d'ENRICHISSEMENT DoorDash (menus livraison + notes) —5#         même patron conservateur qu'ubereats.py. Découverte par le sitemap6#         public QC (cdn.doordash.com/sitemaps/…/sitemap-doordash-qc-stores.xml,7#         ~11 700 magasins, index local data/doordash-stores.json, 30 j).8#         Chaque page magasin embarque du JSON-LD schema.org : un bloc9#         `Restaurant` (adresse, GPS, cuisines, priceRange, aggregateRating —10#         PAS de téléphone) et un bloc `Menu` (hasMenuSection imbriqué ->11#         hasMenuItem {name, description, offers.price "$X.XX"}).12#         Croisement CONSERVATEUR avec un resto EXISTANT seulement (GPS <120 m13#         + nom similaire, OU numéro civique + ville + nom similaire) : n'émet14#         AUCUNE fiche. Menu -> table menus en price_context `delivery` (prix15#         majorés ~25-30 %, CLAUDE.md §6.2), note -> details.doordash.16#17#         Mode d'extraction : Scrapfly ASP (anti-bot ; les gros contenus18#         reviennent en « large_object » à re-télécharger), budget19#         DOORDASH_BUDGET (défaut 40) pages/cycle, verdict par magasin en20#         cache, re-visite 30 j. Conformité §15 : extraction minimale, usage21#         d'appoint, prix toujours étiquetés `delivery`, lien source conservé.22# ==============================================================================23from __future__ import annotations2425import datetime26import json27import os28import re29import sys30import time31import urllib.parse32from pathlib import Path3334import requests3536from ..inspections import _CIVIC_RE, _name_similar, norm_name37from ..regions import strip_accents38from ..schema import Restaurant39from .base import BaseConnector, SkipSource4041SITEMAP_QC = ("https://cdn.doordash.com/sitemaps/sitemaps/"42              "sitemap-doordash-qc-stores.xml")43INDEX_PATH = Path(__file__).resolve().parents[2] / "data" / "doordash-stores.json"44INDEX_REFRESH_DAYS = 3045MAX_BUDGET = int(os.environ.get("DOORDASH_BUDGET", "40"))   # pages/cycle46REFRESH_DAYS = 3047MAX_CANDIDATES = 348MAX_CONSECUTIVE_FAILURES = 349MAX_GPS_M = 120.05051# /en-CA/store/<slug>-<id>/ — les /convenience/store/ (épiceries) sont exclus52_STORE_URL_RE = re.compile(r"doordash\.com/en-CA/store/([^/]+?)-(\d+)/?$")5354_LDJSON_RE = re.compile(55    r'<script type="application/ld\+json"[^>]*>(.*?)</script>', re.S)56_PRICE_RE = re.compile(r"(\d+(?:[.,]\d{1,2})?)")575859def parse_ldjson(page_html: str) -> tuple[dict | None, dict | None]:60    """Extrait les blocs JSON-LD `Restaurant` et `Menu` d'une page magasin."""61    resto = menu = None62    for raw in _LDJSON_RE.findall(page_html):63        try:64            data = json.loads(raw)65        except ValueError:66            continue67        for obj in (data if isinstance(data, list) else [data]):68            if not isinstance(obj, dict):69                continue70            if obj.get("@type") == "Restaurant" and resto is None:71                resto = obj72            elif obj.get("@type") == "Menu" and menu is None:73                menu = obj74    return resto, menu757677def _parse_price(offers) -> float | None:78    """« $13.20 » -> 13.20 ; None si absent/illisible."""79    raw = str((offers or {}).get("price") or "")80    m = _PRICE_RE.search(raw.replace(",", "."))81    if not m:82        return None83    price = float(m.group(1))84    return price if 0 < price <= 10000 else None858687def _flatten_sections(node, out: list[dict]) -> None:88    """hasMenuSection est parfois une liste de listes — aplatir récursivement."""89    if isinstance(node, list):90        for x in node:91            _flatten_sections(x, out)92    elif isinstance(node, dict):93        out.append(node)949596def build_menu(ld_menu: dict, captured_at: str) -> dict | None:97    """Menu standard Resto·Ka (CLAUDE.md §5.2) depuis le JSON-LD `Menu`."""98    raw_sections: list[dict] = []99    _flatten_sections((ld_menu or {}).get("hasMenuSection"), raw_sections)100    sections_out: list[dict] = []101    seen: set[str] = set()102    for sec in raw_sections:103        name = (sec.get("name") or "").strip()104        if not name or name in seen:105            continue106        items = []107        for it in sec.get("hasMenuItem") or []:108            title = (it.get("name") or "").strip()109            if not title:110                continue111            items.append({112                "name": title,113                "description": (it.get("description") or "").strip(),114                "price": _parse_price(it.get("offers")),115            })116        if items:117            seen.add(name)118            sections_out.append({"name": name, "items": items})119    if not sections_out:120        return None121    return {122        "price_context": "delivery",123        "price_source": "doordash",124        "currency": "CAD",125        "captured_at": captured_at,126        "sections": sections_out,127    }128129130def _haversine_m(lat1, lng1, lat2, lng2) -> float:131    import math132    r = 6371000.0133    p1, p2 = math.radians(lat1), math.radians(lat2)134    dp, dl = math.radians(lat2 - lat1), math.radians(lng2 - lng1)135    a = (math.sin(dp / 2) ** 2136         + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2)137    return 2 * r * math.asin(math.sqrt(a))138139140def _norm_city(city: str) -> str:141    return re.sub(r"[^a-z0-9]+", "", strip_accents((city or "").lower()))142143144def verify_store(row, ld: dict) -> str:145    """Vérifie qu'une page magasin DoorDash correspond bien au resto de la146    base — CONSERVATEUR (le JSON-LD n'a PAS de téléphone) : GPS <120 m + nom147    similaire, OU numéro civique + ville + nom similaire. Retourne le mode de148    croisement ('' = pas le même établissement)."""149    addr = ld.get("address") or {}150    geo = ld.get("geo") or {}151    street = (addr.get("streetAddress") or "").strip()152    addr_tokens = set(re.sub(r"[^a-z0-9]+", " ",153                             strip_accents(street.lower())).split())154    similar = _name_similar(norm_name(row["name"]),155                            norm_name(ld.get("name") or ""), addr_tokens)156    try:157        lat, lng = float(geo.get("latitude")), float(geo.get("longitude"))158    except (TypeError, ValueError):159        lat = lng = None160    if (lat is not None and row["lat"] is not None and row["lng"] is not None161            and _haversine_m(row["lat"], row["lng"], lat, lng) <= MAX_GPS_M162            and similar):163        return "gps+nom"164    civic_m = _CIVIC_RE.match(street)165    rcivic_m = _CIVIC_RE.match(row["address"] or "")166    if (civic_m and rcivic_m and civic_m.group(1) == rcivic_m.group(1)167            and _norm_city(addr.get("addressLocality") or "")168            == _norm_city(row["city"] or "") and similar):169        return "civique+ville+nom"170    return ""171172173def slugify(name: str) -> str:174    """Slug façon DoorDash : minuscules sans accents, tirets."""175    s = strip_accents((name or "").lower())176    s = re.sub(r"['’´`.]", "", s)177    s = re.sub(r"&", " ", s)178    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")179    return s180181182class DoorDashConnector(BaseConnector):183    source_id = "doordash"184    request_delay = 1.0185    timeout = 60186    use_detail_cache = False          # verdicts gérés à la main (detail_cache)187    enrichment_only = True            # n'émet aucune fiche (ingest.run)188189    # -- Scrapfly / large_object -------------------------------------------------190    def _content(self, url: str, **kw) -> tuple[int, str]:191        """Page via Scrapfly ASP ; résout les réponses « large_object »192        (contenu volumineux renvoyé comme URL à re-télécharger avec la clé)."""193        result = self.scrapfly(url, render_js=False, **kw)194        status = result.get("status_code") or 0195        content = result.get("content") or ""196        if content.startswith("https://api.scrapfly.io/scrape/large_object/"):197            resp = requests.get(content,198                                params={"key": os.environ["SCRAPFLY_KEY"]},199                                timeout=120)200            content = resp.text if resp.ok else ""201        return status, content202203    # -- index sitemap -----------------------------------------------------------204    def load_index(self) -> dict[str, str]:205        """Index slug -> URL magasin depuis le sitemap QC public, en cache206        local 30 jours (data/doordash-stores.json). Épiceries/dépanneurs207        (/convenience/store/) exclus."""208        if INDEX_PATH.exists():209            try:210                data = json.loads(INDEX_PATH.read_text(encoding="utf-8"))211                if time.time() - float(data.get("fetched_at") or 0) \212                        < INDEX_REFRESH_DAYS * 86400:213                    return data.get("stores") or {}214            except (ValueError, OSError):215                pass216        # le CDN sitemap répond en direct ; Scrapfly en secours (large_object)217        try:218            xml = self.get(SITEMAP_QC).text219        except Exception:220            _, xml = self._content(SITEMAP_QC)221        stores: dict[str, str] = {}222        for loc in re.findall(r"<loc>([^<]+)</loc>", xml):223            if "/convenience/" in loc:224                continue225            mm = _STORE_URL_RE.search(urllib.parse.unquote(loc))226            if mm:227                stores[f"{mm.group(1)}-{mm.group(2)}".lower()] = loc228        if not stores:229            raise RuntimeError("sitemap QC DoorDash vide (blocage ?)")230        INDEX_PATH.parent.mkdir(parents=True, exist_ok=True)231        INDEX_PATH.write_text(json.dumps(232            {"fetched_at": time.time(), "stores": stores},233            ensure_ascii=False), encoding="utf-8")234        print(f"[resto-ka] doordash: index sitemap rafraîchi — "235              f"{len(stores)} magasin(s) QC")236        return stores237238    @staticmethod239    def candidates_for(name: str, slugs: list[str]) -> list[str]:240        """Slugs DoorDash candidats pour un nom de resto (préfixe strict —241        le slug DoorDash embarque la ville et l'id : nom-ville-123456)."""242        base = slugify(name)243        if len(base) < 5:244            return []245        import bisect246        i = bisect.bisect_left(slugs, base)247        out = []248        while i < len(slugs) and len(out) < MAX_CANDIDATES:249            s = slugs[i]250            if s == base or s.startswith(base + "-"):251                out.append(s)252                i += 1253            else:254                break255        return out256257    # -- cycle -------------------------------------------------------------------258    def _now(self) -> str:259        return datetime.datetime.now(datetime.timezone.utc) \260            .strftime("%Y-%m-%dT%H:%M:%SZ")261262    def _fresh(self, stamp: str, now: float, stale_s: float) -> bool:263        try:264            ts = datetime.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ") \265                .replace(tzinfo=datetime.timezone.utc).timestamp()266            return ts > now - stale_s267        except (ValueError, TypeError):268            return False269270    def _details_payload(self, ld: dict, url: str, how: str) -> dict:271        rating = ld.get("aggregateRating") or {}272        return {273            "rating": rating.get("ratingValue"),274            "review_count": rating.get("reviewCount"),275            "price_range": ld.get("priceRange"),276            "cuisines": ld.get("servesCuisine"),277            "url": url.split("?")[0],278            "matched_by": how,279            "fetched_at": self._now(),280        }281282    def _probe_store(self, con, row, url: str) -> tuple[bool, bool]:283        """Visite une page magasin et l'attache au resto si c'est le même284        établissement. Retourne (matched, menu_added)."""285        from .. import db286        mm = _STORE_URL_RE.search(urllib.parse.unquote(url))287        store_id = mm.group(2) if mm else url288        cached = db.get_cached_detail(con, self.source_id, store_id,289                                      "verdict-v1")290        if cached is not None and cached.get("matched_uid") != row["uid"]:291            return False, False       # déjà identifié comme un autre resto292        status, content = self._content(url)293        if status in (400, 404, 410, 451):     # magasin retiré de DoorDash294            db.put_cached_detail(con, self.source_id, store_id,295                                 "verdict-v1", {"matched_uid": None,296                                                "gone": status})297            return False, False298        if status != 200:299            raise RuntimeError(f"HTTP {status}")300        ld_resto, ld_menu = parse_ldjson(content)301        if not ld_resto:302            if ld_menu or "application/ld+json" in content:303                # variante de page sans bloc Restaurant (constat de terrain) :304                # identité invérifiable -> miss prudent, PAS un blocage305                db.put_cached_detail(con, self.source_id, store_id,306                                     "verdict-v1", {"matched_uid": None,307                                                    "no_restaurant_ld": True})308                return False, False309            raise RuntimeError("JSON-LD absent (page non rendue ?)")310        how = verify_store(row, ld_resto)311        if not how:312            db.put_cached_detail(con, self.source_id, store_id,313                                 "verdict-v1", {"matched_uid": None})314            return False, False315        db.put_cached_detail(con, self.source_id, store_id, "verdict-v1",316                             {"matched_uid": row["uid"]})317        menu = build_menu(ld_menu or {}, self._now())318        menu_added = False319        if menu:320            # validation stricte du schéma menu (prix implausibles, etc.)321            Restaurant(source=self.source_id, external_id=store_id,322                       name=row["name"], menu=menu)._validate_menu()323            db.upsert_menu(con, row["uid"], menu, time.time())324            menu_added = True325        db.merge_details(con, row["uid"],326                         {"doordash": self._details_payload(ld_resto, url, how)})327        con.commit()328        return True, menu_added329330    def fetch(self) -> list[Restaurant]:331        if not os.environ.get("SCRAPFLY_KEY"):332            raise SkipSource("SCRAPFLY_KEY manquant (.env) — DoorDash est "333                             "derrière un anti-bot, scraping direct impossible")334        from .. import db335        index = self.load_index()336        slugs = sorted(index.keys())337        con = db.connect()338        now = time.time()339        stale_s = REFRESH_DAYS * 86400.0340        budget = MAX_BUDGET341        matched = menus = misses = failures_row = 0342        rows = con.execute(343            "SELECT uid, name, address, city, postal_code, phone, lat, lng,"344            " details,"345            " EXISTS (SELECT 1 FROM menus m WHERE m.uid=restaurants.uid)"346            "   AS has_menu"347            " FROM restaurants WHERE active=1 AND dup_of IS NULL AND name<>''"348            " AND ((lat IS NOT NULL AND lng IS NOT NULL)"349            "      OR (address<>'' AND city<>''))"350            " ORDER BY has_menu ASC, lat IS NULL ASC, updated_at DESC"351        ).fetchall()352        for row in rows:353            if budget <= 0:354                break355            if failures_row >= MAX_CONSECUTIVE_FAILURES:356                print("[resto-ka] doordash: Scrapfly bloqué "357                      f"{failures_row} fois de suite — arrêt du cycle",358                      file=sys.stderr)359                break360            try:361                details = json.loads(row["details"] or "{}")362            except ValueError:363                details = {}364            dd = details.get("doordash") or {}365            if self._fresh(dd.get("fetched_at", ""), now, stale_s):366                continue              # déjà frais (<30 j)367            probe = details.get("doordash_probe") or {}368            if self._fresh(probe.get("fetched_at", ""), now, stale_s):369                continue              # échec récent : re-visite dans 30 j370            if dd.get("url"):         # déjà croisé : rafraîchir directement371                urls = [dd["url"]]372            else:373                cand = self.candidates_for(row["name"], slugs)374                urls = [index[s] for s in cand if s in index]375            if not urls:376                continue              # aucun candidat : pas de marqueur,377                                      # l'index du mois prochain peut changer378            found = False379            for url in urls[:MAX_CANDIDATES]:380                if budget <= 0:381                    break382                budget -= 1383                try:384                    ok, menu_added = self._probe_store(con, row, url)385                except Exception as exc:386                    failures_row += 1387                    print(f"[resto-ka] doordash: {row['uid']} erreur: {exc}",388                          file=sys.stderr)389                    continue390                failures_row = 0391                if ok:392                    matched += 1393                    menus += 1 if menu_added else 0394                    found = True395                    break396            if not found and not dd.get("url"):397                db.merge_details(con, row["uid"],398                                 {"doordash_probe":399                                  {"miss": "aucun magasin correspondant",400                                   "fetched_at": self._now()}})401                con.commit()402                misses += 1403        con.commit()404        con.close()405        self.enriched_count = matched406        self.enrich_message = (f"{matched} resto(s) croisés DoorDash "407                               f"({menus} menu(s) livraison), {misses} sans "408                               f"correspondance, budget restant "409                               f"{max(budget, 0)} page(s)")410        print(f"[resto-ka] doordash: {self.enrich_message}")411        return []412