spb/resto-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   restoka/connectors/yelp_scrape.py4# Desc:   Connecteur d'ENRICHISSEMENT Yelp SANS clé API (avis/notes par5#         scraping léger). Une seule page de RECHERCHE yelp.ca par resto6#         (find_desc=<nom>&find_loc=<ville>) via Scrapfly ASP : la page embarque7#         un cache Apollo (scripts data-apollo-state) avec, PAR commerce,8#         alias, nom, note, nombre d'avis, fourchette de prix, catégories et9#         adresse (ville + civique). Croisement CONSERVATEUR : nom similaire10#         + même numéro civique + ville compatible (jamais fusionner deux11#         établissements). Résultat -> details.yelp {rating, review_count,12#         price, categories, url} (mêmes clés que Yelp Fusion — le connecteur13#         officiel `yelp` reprend la main dès qu'une YELP_API_KEY existe).14#15#         Économe : budget YELP_SCRAPE_BUDGET (défaut 80) pages/cycle,16#         re-visite 30 jours (hit comme miss), priorité aux restos avec menu.17#         Base légale : pages publiques, extraction minimale (note agrégée +18#         lien vers la fiche Yelp, attribution affichée) — voir CLAUDE.md §15.19# ==============================================================================20from __future__ import annotations2122import datetime23import html as html_mod24import json25import os26import re27import sys28import urllib.parse2930from ..inspections import _CIVIC_RE, _name_similar, norm_name31from ..regions import strip_accents32from ..schema import Restaurant33from .base import BaseConnector, SkipSource3435SEARCH_URL = "https://www.yelp.ca/search?find_desc={desc}&find_loc={loc}"36MAX_BUDGET = int(os.environ.get("YELP_SCRAPE_BUDGET", "80"))  # pages/cycle37REFRESH_DAYS = 30                    # re-visite des notes ET des échecs38MAX_CONSECUTIVE_FAILURES = 3         # Yelp bloque même Scrapfly -> on arrête3940_APOLLO_RE = re.compile(41    r"<script[^>]*data-apollo-state[^>]*>\s*<!--(.*?)-->\s*</script>", re.S)424344def parse_apollo_businesses(page_html: str) -> list[dict]:45    """Extrait les commerces du cache Apollo embarqué dans une page de46    recherche yelp.ca : [{alias, name, rating, review_count, price,47    categories, city, address, closed}]."""48    cache: dict = {}49    for m in _APOLLO_RE.finditer(page_html):50        try:51            cache.update(json.loads(html_mod.unescape(m.group(1))))52        except ValueError:53            continue54    if not cache:55        return []56    categories = {k.split(":", 1)[1]: (v.get("title") or "")57                  for k, v in cache.items()58                  if k.startswith("BusinessCategory:") and isinstance(v, dict)}59    locations = {k.split(":", 1)[1]: (v.get("address") or {})60                 for k, v in cache.items()61                 if k.startswith("BusinessLocation:") and isinstance(v, dict)}62    out = []63    for k, v in cache.items():64        if not k.startswith("Business:") or not isinstance(v, dict):65            continue66        if not v.get("alias") or not v.get("name"):67            continue68        loc_ref = ((v.get("location") or {}).get("__ref") or "")69        addr = locations.get(loc_ref.split(":", 1)[-1], {})70        cats = []71        for c in v.get("categories") or []:72            ref = (c.get("__ref") or "").split(":", 1)[-1]73            if categories.get(ref):74                cats.append(categories[ref])75        closed = any((a or {}).get("type") == "permclosed"76                     for a in [v.get(kk) for kk in v77                               if kk.startswith("activeAlert")])78        price = v.get("priceRange")79        out.append({80            "alias": v["alias"],81            "name": v["name"],82            "rating": v.get("rating"),83            "review_count": v.get("reviewCount") or 0,84            "price": (price or {}).get("display") if isinstance(price, dict)85            else price,86            "categories": cats,87            "city": (addr or {}).get("city") or "",88            "address": (addr or {}).get("addressLine1") or "",89            "closed": closed,90        })91    return out929394def _city_key(city: str) -> str:95    """« Quebec City » / « Québec » -> « quebec » ; « Montréal » -> « montreal »."""96    s = strip_accents((city or "").lower())97    s = re.sub(r"[^a-z0-9]+", " ", s).strip()98    return re.sub(r"\bcity\b", "", s).strip()99100101def match_business(row, candidates: list[dict]) -> dict | None:102    """Croisement CONSERVATEUR d'un resto avec les résultats Yelp :103    nom similaire ET même numéro civique ET ville compatible. Sans civique104    des deux côtés : nom normalisé IDENTIQUE + même ville. Ambigu -> None."""105    rname = norm_name(row["name"])106    rcity = _city_key(row["city"])107    rcivic_m = _CIVIC_RE.match(row["address"] or "")108    rcivic = rcivic_m.group(1) if rcivic_m else ""109    addr_tokens = set(re.sub(r"[^a-z0-9]+", " ", strip_accents(110        (row["address"] or "").lower())).split()) | set(rcity.split())111    hits = []112    for c in candidates:113        if c["closed"]:114            continue115        ccity = _city_key(c["city"])116        if rcity and ccity and rcity != ccity \117                and rcity not in ccity and ccity not in rcity:118            continue119        cname = norm_name(c["name"])120        ccivic_m = _CIVIC_RE.match(c["address"] or "")121        ccivic = ccivic_m.group(1) if ccivic_m else ""122        if rcivic and ccivic:123            if rcivic == ccivic and _name_similar(rname, cname, addr_tokens):124                hits.append(c)125        elif rname and rname == cname and rcity and ccity:126            hits.append(c)            # nom exact + ville, sans civique127    aliases = {h["alias"] for h in hits}128    if len(aliases) == 1:129        return hits[0]130    return None                       # rien ou ambigu : on ne fusionne pas131132133class YelpScrapeConnector(BaseConnector):134    source_id = "yelp-scrape"135    request_delay = 1.0136    timeout = 60137    use_detail_cache = False138    enrichment_only = True            # n'émet aucune fiche (ingest.run)139140    def _now(self) -> str:141        return datetime.datetime.now(datetime.timezone.utc) \142            .strftime("%Y-%m-%dT%H:%M:%SZ")143144    def _fresh(self, stamp: str, now: float, stale_s: float) -> bool:145        try:146            ts = datetime.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ") \147                .replace(tzinfo=datetime.timezone.utc).timestamp()148            return ts > now - stale_s149        except (ValueError, TypeError):150            return False151152    def fetch(self) -> list[Restaurant]:153        if not os.environ.get("SCRAPFLY_KEY"):154            raise SkipSource("SCRAPFLY_KEY manquant (.env) — scraping Yelp "155                             "impossible sans contournement anti-bot")156        import time as _time157        from .. import db158        con = db.connect()159        now = _time.time()160        stale_s = REFRESH_DAYS * 86400.0161        budget = MAX_BUDGET162        enriched = misses = failures_row = 0163        rows = con.execute(164            "SELECT uid, name, address, city, details,"165            " EXISTS (SELECT 1 FROM menus m WHERE m.uid=restaurants.uid)"166            "   AS has_menu"167            " FROM restaurants WHERE active=1 AND dup_of IS NULL"168            " AND name<>'' AND city<>''"169            # indépendants d'abord : les succursales de chaînes n'ont presque170            # jamais d'avis Yelp (budget mieux investi ailleurs)171            " ORDER BY has_menu DESC, chain IS NULL DESC, phone<>'' DESC,"172            " updated_at DESC"173        ).fetchall()174        for row in rows:175            if budget <= 0:176                break177            if failures_row >= MAX_CONSECUTIVE_FAILURES:178                print("[resto-ka] yelp-scrape: Scrapfly bloqué "179                      f"{failures_row} fois de suite — arrêt du cycle",180                      file=sys.stderr)181                break182            try:183                details = json.loads(row["details"] or "{}")184            except ValueError:185                details = {}186            yelp = details.get("yelp") or {}187            if self._fresh(yelp.get("fetched_at", ""), now, stale_s):188                continue              # note déjà fraîche (<30 j)189            probe = details.get("yelp_scrape") or {}190            if self._fresh(probe.get("fetched_at", ""), now, stale_s):191                continue              # échec récent : re-visite dans 30 j192            url = SEARCH_URL.format(193                desc=urllib.parse.quote(row["name"][:64]),194                loc=urllib.parse.quote(f"{row['city']}, QC"))195            budget -= 1196            try:197                result = self.scrapfly(url, render_js=False)198            except Exception as exc:199                failures_row += 1200                print(f"[resto-ka] yelp-scrape: {row['uid']} erreur: {exc}",201                      file=sys.stderr)202                continue203            if (result.get("status_code") or 0) != 200:204                failures_row += 1205                continue206            failures_row = 0207            candidates = parse_apollo_businesses(result.get("content") or "")208            hit = match_business(row, candidates)209            if hit and hit.get("rating") is not None:210                db.merge_details(con, row["uid"], {"yelp": {211                    "url": f"https://www.yelp.ca/biz/"212                           f"{urllib.parse.quote(hit['alias'])}",213                    "name": hit["name"],214                    "rating": hit["rating"],215                    "review_count": hit["review_count"],216                    "price": hit.get("price"),217                    "categories": hit.get("categories") or [],218                    "matched_by": "nom+adresse",219                    "via": "scrape",220                    "fetched_at": self._now(),221                }})222                enriched += 1223            else:224                reason = ("fiche sans note" if hit else225                          "introuvable ou ambigu")226                db.merge_details(con, row["uid"],227                                 {"yelp_scrape": {"miss": reason,228                                                  "fetched_at": self._now()}})229                misses += 1230            con.commit()231        con.commit()232        con.close()233        self.enriched_count = enriched234        self.enrich_message = (f"{enriched} resto(s) notés (scraping), "235                               f"{misses} sans correspondance, "236                               f"budget restant {max(budget, 0)} page(s)")237        print(f"[resto-ka] yelp-scrape: {self.enrich_message}")238        return []239