spb/resto-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   restoka/connectors/skip.py4# Desc:   Connecteur d'ENRICHISSEMENT SkipTheDishes (menus livraison + score) —5#         même patron conservateur qu'ubereats.py/doordash.py. Découverte par6#         les sitemaps publics (sitemap2..7.xml, pages resto à la racine7#         /<slug>, doublons /fr/ exclus ; index local data/skip-stores.json,8#         30 j). Chaque page resto embarque un état Next.js (__NEXT_DATA__)9#         avec props.pageProps.partner (nom, rue, « Ville, PROV, POSTAL, CAN »,10#         score Skip, téléphone dans les textes du disclaimerModal) et11#         menuV2.categories[].menuItems[] (name, description, subtotal en12#         cents). Croisement CONSERVATEUR avec un resto EXISTANT seulement13#         (téléphone identique OU code postal + numéro civique identiques ;14#         province ≠ QC rejetée d'emblée) : n'émet AUCUNE fiche. Menu -> table15#         menus en price_context `delivery` (prix majorés ~25-30 %, §6.2),16#         score -> details.skip.17#18#         Mode d'extraction : Scrapfly ASP (anti-bot ; gros contenus résolus19#         via « large_object »), budget SKIP_BUDGET (défaut 40) pages/cycle,20#         verdict par resto en cache, re-visite 30 j. Conformité §15 :21#         extraction minimale, usage d'appoint, prix étiquetés `delivery`.22# ==============================================================================23from __future__ import annotations2425import datetime26import json27import os28import re29import sys30import time31from pathlib import Path3233import requests3435from ..inspections import _CIVIC_RE36from ..normalize import normalize_phone37from ..regions import strip_accents38from ..schema import Restaurant39from .base import BaseConnector, SkipSource4041BASE = "https://www.skipthedishes.com"42SITEMAPS = [f"{BASE}/sitemap{i}.xml" for i in range(2, 8)]43INDEX_PATH = Path(__file__).resolve().parents[2] / "data" / "skip-stores.json"44INDEX_REFRESH_DAYS = 3045MAX_BUDGET = int(os.environ.get("SKIP_BUDGET", "40"))       # pages/cycle46REFRESH_DAYS = 3047MAX_CANDIDATES = 348MAX_CONSECUTIVE_FAILURES = 34950_NEXT_DATA_RE = re.compile(51    r'<script id="__NEXT_DATA__" type="application/json"[^>]*>(.*?)</script>',52    re.S)53# pages resto = un seul segment à la racine ; /fr/... = doublon francophone54_STORE_URL_RE = re.compile(r"skipthedishes\.com/([a-z0-9-]+)/?$")55# « Windsor, ON, N9E 1S3, CAN »56_LOCDETAILS_RE = re.compile(57    r"^\s*(.+?),\s*([A-Z]{2}),\s*([A-Z]\d[A-Z]\s?\d[A-Z]\d),", re.I)585960def decode_next_data(page_html: str) -> dict | None:61    """Décode le bloc __NEXT_DATA__ (JSON brut) d'une page Skip."""62    m = _NEXT_DATA_RE.search(page_html)63    if not m:64        return None65    try:66        return json.loads(m.group(1))67    except ValueError:68        return None697071def partner_payload(state: dict) -> tuple[dict | None, dict | None]:72    """(partner, menuV2) depuis props.pageProps."""73    pp = ((state or {}).get("props") or {}).get("pageProps") or {}74    partner = pp.get("partner")75    menu = pp.get("menuV2") or (partner or {}).get("menuV2")76    return (partner if isinstance(partner, dict) else None,77            menu if isinstance(menu, dict) else None)787980def partner_phone(partner: dict) -> str:81    """Téléphone du resto, enfoui dans les textes du disclaimerModal."""82    dm = (partner or {}).get("disclaimerModal") or {}83    for d in dm.get("disclaimers") or []:84        phone = normalize_phone(d.get("text") or "")85        if phone:86            return phone87    return ""888990def parse_location_details(details: str) -> tuple[str, str, str]:91    """« Windsor, ON, N9E 1S3, CAN » -> (ville, province, code postal)."""92    m = _LOCDETAILS_RE.match(details or "")93    if not m:94        return "", "", ""95    return (m.group(1).strip(), m.group(2).upper(),96            m.group(3).replace(" ", "").upper())979899def build_menu(menu_v2: dict, captured_at: str) -> dict | None:100    """Menu standard Resto·Ka (CLAUDE.md §5.2) depuis menuV2.categories.101    Prix Skip en cents (`subtotal`) -> dollars, contexte `delivery`."""102    sections_out: list[dict] = []103    seen: set[str] = set()104    for cat in (menu_v2 or {}).get("categories") or []:105        name = (cat.get("name") or "").strip()106        if not name or name in seen:107            continue108        items = []109        for it in cat.get("menuItems") or []:110            title = (it.get("name") or "").strip()111            if not title:112                continue113            sub = it.get("subtotal")114            items.append({115                "name": title,116                "description": (it.get("description") or "").strip(),117                "price": round(sub / 100.0, 2)118                if isinstance(sub, (int, float)) and sub > 0 else None,119            })120        if items:121            seen.add(name)122            sections_out.append({"name": name, "items": items})123    if not sections_out:124        return None125    return {126        "price_context": "delivery",127        "price_source": "skip",128        "currency": "CAD",129        "captured_at": captured_at,130        "sections": sections_out,131    }132133134def verify_partner(row, partner: dict) -> str:135    """Vérifie qu'une page Skip correspond bien au resto de la base —136    CONSERVATEUR : province QC obligatoire, puis téléphone identique OU137    code postal + numéro civique identiques. Retourne le mode de croisement138    ('' = pas le même établissement)."""139    _, prov, postal = parse_location_details(140        partner.get("locationDetails") or "")141    if prov and prov != "QC":142        return ""143    phone = partner_phone(partner)144    if phone and normalize_phone(row["phone"]) == phone:145        return "telephone"146    rpostal = (row["postal_code"] or "").replace(" ", "").upper()147    civic_m = _CIVIC_RE.match(partner.get("location") or "")148    rcivic_m = _CIVIC_RE.match(row["address"] or "")149    if postal and rpostal == postal and civic_m and rcivic_m \150            and civic_m.group(1) == rcivic_m.group(1):151        return "postal+civique"152    return ""153154155def slugify(name: str) -> str:156    """Slug façon Skip (cleanUrl) : minuscules sans accents, tirets."""157    s = strip_accents((name or "").lower())158    s = re.sub(r"['’´`.]", "", s)159    s = re.sub(r"&", " ", s)160    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")161    return s162163164class SkipConnector(BaseConnector):165    source_id = "skip"166    request_delay = 1.0167    timeout = 60168    use_detail_cache = False          # verdicts gérés à la main (detail_cache)169    enrichment_only = True            # n'émet aucune fiche (ingest.run)170171    # -- Scrapfly / large_object -------------------------------------------------172    def _content(self, url: str, **kw) -> tuple[int, str]:173        result = self.scrapfly(url, render_js=False, **kw)174        status = result.get("status_code") or 0175        content = result.get("content") or ""176        if content.startswith("https://api.scrapfly.io/scrape/large_object/"):177            resp = requests.get(content,178                                params={"key": os.environ["SCRAPFLY_KEY"]},179                                timeout=120)180            content = resp.text if resp.ok else ""181        return status, content182183    # -- index sitemap -----------------------------------------------------------184    def load_index(self) -> dict[str, str]:185        """Index slug -> URL resto depuis sitemap2..7 (pages à la racine),186        en cache local 30 jours (data/skip-stores.json). Les /fr/… (doublons)187        et sitemap1 (pages villes/cuisines) sont exclus."""188        if INDEX_PATH.exists():189            try:190                data = json.loads(INDEX_PATH.read_text(encoding="utf-8"))191                if time.time() - float(data.get("fetched_at") or 0) \192                        < INDEX_REFRESH_DAYS * 86400:193                    return data.get("stores") or {}194            except (ValueError, OSError):195                pass196        stores: dict[str, str] = {}197        for sm_url in SITEMAPS:198            _, xml = self._content(sm_url)199            for loc in re.findall(r"<loc>([^<]+)</loc>", xml):200                mm = _STORE_URL_RE.search(loc)201                if mm:202                    stores.setdefault(mm.group(1).lower(), loc)203        if not stores:204            raise RuntimeError("sitemaps Skip vides (blocage ?)")205        INDEX_PATH.parent.mkdir(parents=True, exist_ok=True)206        INDEX_PATH.write_text(json.dumps(207            {"fetched_at": time.time(), "stores": stores},208            ensure_ascii=False), encoding="utf-8")209        print(f"[resto-ka] skip: index sitemap rafraîchi — "210              f"{len(stores)} resto(s) Canada")211        return stores212213    @staticmethod214    def candidates_for(name: str, slugs: list[str]) -> list[str]:215        """Slugs Skip candidats pour un nom de resto (préfixe strict — le216        slug Skip est le nom seul, parfois suffixé de la rue/ville)."""217        base = slugify(name)218        if len(base) < 5:219            return []220        import bisect221        i = bisect.bisect_left(slugs, base)222        out = []223        while i < len(slugs) and len(out) < MAX_CANDIDATES:224            s = slugs[i]225            if s == base or s.startswith(base + "-"):226                out.append(s)227                i += 1228            else:229                break230        return out231232    # -- cycle -------------------------------------------------------------------233    def _now(self) -> str:234        return datetime.datetime.now(datetime.timezone.utc) \235            .strftime("%Y-%m-%dT%H:%M:%SZ")236237    def _fresh(self, stamp: str, now: float, stale_s: float) -> bool:238        try:239            ts = datetime.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ") \240                .replace(tzinfo=datetime.timezone.utc).timestamp()241            return ts > now - stale_s242        except (ValueError, TypeError):243            return False244245    def _details_payload(self, partner: dict, url: str, how: str) -> dict:246        score = partner.get("score") or {}247        return {248            "score": score.get("formatted") or score.get("value"),249            "url": url.split("?")[0],250            "partner_id": partner.get("id"),251            "matched_by": how,252            "fetched_at": self._now(),253        }254255    def _probe_store(self, con, row, url: str) -> tuple[bool, bool]:256        """Visite une page resto Skip et l'attache au resto si c'est le même257        établissement. Retourne (matched, menu_added)."""258        from .. import db259        mm = _STORE_URL_RE.search(url)260        slug = mm.group(1) if mm else url261        cached = db.get_cached_detail(con, self.source_id, slug, "verdict-v1")262        if cached is not None and cached.get("matched_uid") != row["uid"]:263            return False, False       # déjà identifié comme un autre resto264        status, content = self._content(url)265        if status in (400, 404, 410, 451):     # resto retiré de Skip266            db.put_cached_detail(con, self.source_id, slug, "verdict-v1",267                                 {"matched_uid": None, "gone": status})268            return False, False269        if status != 200:270            raise RuntimeError(f"HTTP {status}")271        state = decode_next_data(content)272        if not state:273            raise RuntimeError("__NEXT_DATA__ absent (page non rendue ?)")274        partner, menu_v2 = partner_payload(state)275        if not partner:276            # page rendue mais sans fiche partner (page de groupe/chaîne) :277            # identité invérifiable -> miss prudent, PAS un blocage278            db.put_cached_detail(con, self.source_id, slug, "verdict-v1",279                                 {"matched_uid": None, "no_partner": True})280            return False, False281        how = verify_partner(row, partner)282        if not how:283            db.put_cached_detail(con, self.source_id, slug, "verdict-v1",284                                 {"matched_uid": None})285            return False, False286        db.put_cached_detail(con, self.source_id, slug, "verdict-v1",287                             {"matched_uid": row["uid"]})288        menu = build_menu(menu_v2 or {}, self._now())289        menu_added = False290        if menu:291            # validation stricte du schéma menu (prix implausibles, etc.)292            Restaurant(source=self.source_id, external_id=slug,293                       name=row["name"], menu=menu)._validate_menu()294            db.upsert_menu(con, row["uid"], menu, time.time())295            menu_added = True296        db.merge_details(con, row["uid"],297                         {"skip": self._details_payload(partner, url, how)})298        con.commit()299        return True, menu_added300301    def fetch(self) -> list[Restaurant]:302        if not os.environ.get("SCRAPFLY_KEY"):303            raise SkipSource("SCRAPFLY_KEY manquant (.env) — SkipTheDishes "304                             "est derrière un anti-bot, scraping direct "305                             "impossible")306        from .. import db307        index = self.load_index()308        slugs = sorted(index.keys())309        con = db.connect()310        now = time.time()311        stale_s = REFRESH_DAYS * 86400.0312        budget = MAX_BUDGET313        matched = menus = misses = failures_row = 0314        # Skip n'expose ni GPS ni téléphone en clair : le croisement passe par315        # le téléphone (disclaimer) ou postal+civique -> exiger ces champs316        rows = con.execute(317            "SELECT uid, name, address, city, postal_code, phone, lat, lng,"318            " details,"319            " EXISTS (SELECT 1 FROM menus m WHERE m.uid=restaurants.uid)"320            "   AS has_menu"321            " FROM restaurants WHERE active=1 AND dup_of IS NULL AND name<>''"322            " AND (phone<>'' OR (postal_code<>'' AND address<>''))"323            " ORDER BY has_menu ASC, phone<>'' DESC, updated_at DESC"324        ).fetchall()325        for row in rows:326            if budget <= 0:327                break328            if failures_row >= MAX_CONSECUTIVE_FAILURES:329                print("[resto-ka] skip: Scrapfly bloqué "330                      f"{failures_row} fois de suite — arrêt du cycle",331                      file=sys.stderr)332                break333            try:334                details = json.loads(row["details"] or "{}")335            except ValueError:336                details = {}337            sk = details.get("skip") or {}338            if self._fresh(sk.get("fetched_at", ""), now, stale_s):339                continue              # déjà frais (<30 j)340            probe = details.get("skip_probe") or {}341            if self._fresh(probe.get("fetched_at", ""), now, stale_s):342                continue              # échec récent : re-visite dans 30 j343            if sk.get("url"):         # déjà croisé : rafraîchir directement344                urls = [sk["url"]]345            else:346                cand = self.candidates_for(row["name"], slugs)347                urls = [index[s] for s in cand if s in index]348            if not urls:349                continue              # aucun candidat : pas de marqueur350            found = False351            for url in urls[:MAX_CANDIDATES]:352                if budget <= 0:353                    break354                budget -= 1355                try:356                    ok, menu_added = self._probe_store(con, row, url)357                except Exception as exc:358                    failures_row += 1359                    print(f"[resto-ka] skip: {row['uid']} erreur: {exc}",360                          file=sys.stderr)361                    continue362                failures_row = 0363                if ok:364                    matched += 1365                    menus += 1 if menu_added else 0366                    found = True367                    break368            if not found and not sk.get("url"):369                db.merge_details(con, row["uid"],370                                 {"skip_probe":371                                  {"miss": "aucun resto correspondant",372                                   "fetched_at": self._now()}})373                con.commit()374                misses += 1375        con.commit()376        con.close()377        self.enriched_count = matched378        self.enrich_message = (f"{matched} resto(s) croisés Skip "379                               f"({menus} menu(s) livraison), {misses} sans "380                               f"correspondance, budget restant "381                               f"{max(budget, 0)} page(s)")382        print(f"[resto-ka] skip: {self.enrich_message}")383        return []384