spb/resto-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   restoka/connectors/yelp.py4# Desc:   Connecteur d'ENRICHISSEMENT Yelp Fusion (avis/notes) — gratuit,5#         500 requêtes/jour. N'émet AUCUNE fiche restaurant : il complète les6#         fiches existantes avec rating, review_count, price, categories et7#         phone dans details.yelp (colonne d'enrichissement, jamais écrasée8#         par les re-crawls des sources).9#10#         Croisement CONSERVATEUR (jamais fusionner deux établissements) :11#           1. par TÉLÉPHONE (/businesses/search/phone) — signal fort ; si12#              plusieurs résultats, on exige le GPS le plus proche (<300 m) ;13#           2. sinon par NOM+ADRESSE+GPS (/businesses/matches) — l'endpoint14#              d'appariement officiel de Yelp, seuil par défaut (conservateur).15#16#         Clé requise : YELP_API_KEY dans .env (https://www.yelp.com/developers17#         — app gratuite). Sans clé, le connecteur lève SkipSource et le18#         pipeline passe son tour SANS marquer d'échec (statut « clé requise »19#         dans data/sources.json).20#         Base légale : API officielle Yelp Fusion, conditions Display21#         Requirements (attribution Yelp affichée avec la note sur la fiche).22# ==============================================================================23from __future__ import annotations2425import datetime26import json27import math28import os29import sys30import time3132from ..normalize import normalize_phone33from ..schema import Restaurant34from .base import BaseConnector, SkipSource3536API = "https://api.yelp.com/v3"37REFRESH_DAYS = 30                     # re-vérification des notes captées38MAX_REQUESTS_PER_RUN = int(os.environ.get("YELP_BUDGET", "450"))  # quota 500/j39MAX_PHONE_DISTANCE_M = 300            # ambiguïté téléphone : GPS requis404142def _haversine_m(lat1, lng1, lat2, lng2) -> float:43    r = 6371000.044    p1, p2 = math.radians(lat1), math.radians(lat2)45    dp, dl = math.radians(lat2 - lat1), math.radians(lng2 - lng1)46    a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 247    return 2 * r * math.asin(math.sqrt(a))484950class YelpConnector(BaseConnector):51    source_id = "yelp"52    request_delay = 0.35              # politesse API (limite 5 QPS)53    timeout = 2054    use_detail_cache = False55    enrichment_only = True            # n'émet aucune fiche (ingest.run)5657    def _api(self, path: str, params: dict) -> dict:58        resp = self.get(f"{API}{path}", params=params,59                        headers={"Authorization": f"Bearer {self._key}"})60        return resp.json()6162    def _payload(self, biz: dict, matched_by: str) -> dict:63        return {64            "id": biz.get("id"),65            "url": (biz.get("url") or "").split("?")[0],66            "name": biz.get("name"),67            "rating": biz.get("rating"),68            "review_count": biz.get("review_count"),69            "price": biz.get("price"),70            "categories": [c.get("title") for c in biz.get("categories") or []],71            "phone": biz.get("phone"),72            "matched_by": matched_by,73            "fetched_at": datetime.datetime.now(datetime.timezone.utc)74            .strftime("%Y-%m-%dT%H:%M:%SZ"),75        }7677    def _match_by_phone(self, phone: str, lat, lng) -> dict | None:78        data = self._api("/businesses/search/phone", {"phone": phone})79        businesses = data.get("businesses") or []80        if not businesses:81            return None82        if len(businesses) == 1:83            return self._payload(businesses[0], "telephone")84        if lat is None or lng is None:85            return None               # plusieurs candidats, pas de GPS : on passe86        best, best_d = None, None87        for b in businesses:88            c = b.get("coordinates") or {}89            if c.get("latitude") is None or c.get("longitude") is None:90                continue91            d = _haversine_m(lat, lng, c["latitude"], c["longitude"])92            if best_d is None or d < best_d:93                best, best_d = b, d94        if best is not None and best_d is not None and best_d <= MAX_PHONE_DISTANCE_M:95            return self._payload(best, "telephone+gps")96        return None9798    def _match_by_name(self, row) -> dict | None:99        if not (row["name"] and row["address"] and row["city"]100                and row["lat"] is not None and row["lng"] is not None):101            return None102        data = self._api("/businesses/matches", {103            "name": row["name"][:64], "address1": row["address"][:64],104            "city": row["city"][:64], "state": "QC", "country": "CA",105            "latitude": row["lat"], "longitude": row["lng"], "limit": 1,106        })107        businesses = data.get("businesses") or []108        if not businesses:109            return None110        biz = businesses[0]111        # /matches ne retourne pas rating/review_count : détail requis112        detail = self._api(f"/businesses/{biz['id']}", {})113        return self._payload(detail or biz, "nom+gps")114115    def fetch(self) -> list[Restaurant]:116        self._key = os.environ.get("YELP_API_KEY", "").strip()117        if not self._key:118            raise SkipSource("YELP_API_KEY manquant (.env) — connecteur prêt, "119                             "clé gratuite : https://www.yelp.com/developers")120        from .. import db121        con = db.connect()122        now = time.time()123        stale = now - REFRESH_DAYS * 86400124        budget = MAX_REQUESTS_PER_RUN125        enriched = skipped = 0126        rows = con.execute(127            "SELECT uid, name, address, city, lat, lng, phone, details"128            " FROM restaurants WHERE active=1 AND dup_of IS NULL"129            " ORDER BY phone<>'' DESC, updated_at DESC").fetchall()130        for row in rows:131            if budget <= 0:132                break133            try:134                details = json.loads(row["details"] or "{}")135            except ValueError:136                details = {}137            yelp = details.get("yelp") or {}138            if yelp.get("fetched_at"):139                try:140                    ts = datetime.datetime.strptime(141                        yelp["fetched_at"], "%Y-%m-%dT%H:%M:%SZ") \142                        .replace(tzinfo=datetime.timezone.utc).timestamp()143                    if ts > stale:144                        continue                  # déjà frais (<30 j)145                except ValueError:146                    pass147            payload = None148            try:149                phone = normalize_phone(row["phone"])150                if phone:151                    budget -= 1152                    payload = self._match_by_phone(phone, row["lat"], row["lng"])153                if payload is None:154                    budget -= 2                  # /matches + détail155                    payload = self._match_by_name(row)156            except Exception as exc:             # un resto raté ne bloque pas157                skipped += 1158                print(f"[resto-ka] yelp: {row['uid']} erreur: {exc}",159                      file=sys.stderr)160                continue161            if payload:162                db.merge_details(con, row["uid"], {"yelp": payload})163                con.commit()164                enriched += 1165        con.commit()166        con.close()167        self.enriched_count = enriched168        self.enrich_message = (f"{enriched} resto(s) enrichis (avis/notes), "169                               f"budget restant {max(budget, 0)} req")170        print(f"[resto-ka] yelp: {self.enrich_message}"171              + (f", {skipped} erreurs" if skipped else ""))172        return []173