spb/food-ka

1# -----------------------------------------------------------------------------2# Food-Ka — Agrégateur de produits d'épicerie (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# nutrition.py : enrichissement nutritionnel via Open Food Facts (OFF)5#   Les bannières n'exposent pas leurs fiches nutritionnelles sans anti-bot6#   (voila.ca : pages PDP derrière Incapsula ; Shopify : pas de code-barres7#   dans /products.json public). On croise donc avec la base ouverte OFF :8#     https://search.openfoodfacts.org/search (« search-a-licious » : plein9#     texte + filtre brands_tags, JSON) — nutriscore, groupe NOVA, nutriments10#     /100 g ; les INGRÉDIENTS viennent d'un second appel /api/v2/product/11#     {code} (l'index de recherche ne les porte pas). NB : l'ancien12#     cgi/search.pl répond 503 (déprécié) — ne pas y revenir.13#   Croisement CONSERVATEUR (mieux vaut rater que fusionner faux) :14#     - marque non vide obligatoire, et retrouvée dans le champ brands d'OFF ;15#     - similarité élevée des noms nettoyés (0,80 si le format concorde16#       aussi, 0,90 sinon) ;17#     - si les deux formats sont connus, ils doivent concorder (±2 %).18#   Budget par cycle (défaut 100 requêtes) + throttle 6 s (limite OFF :19#   10 recherches/min) + cache BD off_cache (une marque+nom+format n'est20#   cherchée qu'une seule fois, trouvée ou non — vider off_cache pour21#   re-tenter). Résultat stocké dans details.off, ré-appliqué à chaque cycle22#   (les mises à jour de produit écrasent details ; le cache est la vérité).23# -----------------------------------------------------------------------------24from __future__ import annotations2526import hashlib27import json28import re29import sqlite330import time3132import requests3334from .matching import brand_key, name_key, name_similarity, size_key35from .normalize import parse_size, strip_accents3637SEARCH_URL = "https://search.openfoodfacts.org/search"38PRODUCT_URL = ("https://world.openfoodfacts.org/api/v2/product/{code}"39               "?fields=ingredients_text_fr,ingredients_text")40USER_AGENT = "FoodKaBot/1.0 (+https://www.food-ka.com/bot; contact@spboucher.ai)"4142_SLUG_RE = re.compile(r"[^a-z0-9]+")4344REQUEST_DELAY = 6.0          # limite OFF : 10 recherches/minute45SIM_WITH_SIZE = 0.80         # marque + format concordants -> le nom confirme46SIM_NO_SIZE = 0.90           # format inconnu -> quasi-identité exigée4748# nutriments retenus (valeurs /100 g ou /100 ml, + énergie)49_NUTRIMENT_KEYS = (50    "energy-kcal_100g", "fat_100g", "saturated-fat_100g", "trans-fat_100g",51    "carbohydrates_100g", "sugars_100g", "fiber_100g", "proteins_100g",52    "salt_100g", "sodium_100g", "calcium_100g", "iron_100g",53)545556def match_key(brand: str, name: str, size_label: str) -> str:57    raw = f"{brand_key(brand)}|{name_key(name, brand)}|{size_key(size_label)}"58    return hashlib.md5(raw.encode("utf-8")).hexdigest()596061def _sizes_agree(ours: str, theirs: str | None) -> bool | None:62    """True/False si les deux formats sont connus, None si l'un manque."""63    a, b = parse_size(ours), parse_size(theirs)64    if not a or not b:65        return None66    (qa, ua), (qb, ub) = a, b67    return ua == ub and abs(qa - qb) <= 0.02 * max(qa, qb)686970def _pick(hits: list[dict], brand: str, name: str, size_label: str) -> tuple[dict | None, float]:71    """Meilleur candidat OFF respectant marque + nom + format — ou None."""72    ours_nk = name_key(name, brand)73    ours_bk = brand_key(brand)74    best, best_sim = None, 0.075    for hit in hits:76        # marque : notre marque doit se retrouver dans le champ brands d'OFF77        # (liste chez search-a-licious, chaîne « A, B » sur l'API produit)78        raw = hit.get("brands")79        brand_list = raw if isinstance(raw, list) else str(raw or "").split(",")80        their_brands = [brand_key(str(b)) for b in brand_list]81        if ours_bk not in [b for b in their_brands if b]:82            continue83        hit_name = hit.get("product_name_fr") or hit.get("product_name") or ""84        sim = name_similarity(ours_nk, name_key(hit_name, brand))85        agree = _sizes_agree(size_label, hit.get("quantity"))86        if agree is False:87            continue                     # formats connus et différents -> rejet88        threshold = SIM_WITH_SIZE if agree else SIM_NO_SIZE89        if sim >= threshold and sim > best_sim:90            best, best_sim = hit, sim91    return best, best_sim929394def _grade(value) -> str | None:95    """Grade OFF nettoyé — « unknown »/« not-applicable » -> None."""96    v = str(value or "").strip().lower()97    return v if v and v not in ("unknown", "not-applicable") else None9899100def _payload(hit: dict, sim: float, ingredients: str = "") -> dict:101    nutr = hit.get("nutriments") or {}102    brands = hit.get("brands")103    if isinstance(brands, list):104        brands = ", ".join(str(b) for b in brands)105    out = {k: v for k, v in {106        "code": hit.get("code"),107        "product_name": hit.get("product_name_fr") or hit.get("product_name"),108        "brands": brands,109        "quantity": hit.get("quantity"),110        "nutriscore": _grade(hit.get("nutriscore_grade")),111        "nova": hit.get("nova_group"),112        "ecoscore": _grade(hit.get("ecoscore_grade")),113        "ingredients": ingredients[:800] or None,114        "allergens": [str(a).split(":")[-1] for a in hit.get("allergens_tags") or []] or None,115        "nutriments": {k: nutr[k] for k in _NUTRIMENT_KEYS if nutr.get(k) is not None} or None,116        "url": (f"https://world.openfoodfacts.org/product/{hit.get('code')}"117                if hit.get("code") else None),118        "match_similarity": round(sim, 3),119    }.items() if v not in (None, "", {}, [])}120    return out121122123def _search(session: requests.Session, brand: str, name: str) -> list[dict]:124    slug = _SLUG_RE.sub("-", strip_accents(brand).lower()).strip("-")125    params = {126        "q": f'{name[:120]} brands_tags:"{slug}"',127        "page_size": "10", "langs": "fr",128    }129    resp = session.get(SEARCH_URL, params=params, timeout=25)130    resp.raise_for_status()131    return (resp.json().get("hits") or [])132133134def _ingredients(session: requests.Session, code) -> str:135    """Texte des ingrédients (FR de préférence) — jamais bloquant."""136    if not code:137        return ""138    try:139        resp = session.get(PRODUCT_URL.format(code=code), timeout=25)140        resp.raise_for_status()141        prod = resp.json().get("product") or {}142        return str(prod.get("ingredients_text_fr")143                   or prod.get("ingredients_text") or "")144    except Exception:145        return ""146147148# ---------------------------------------------------------------------------149# Cycle d'enrichissement (appelé après chaque ingestion)150# ---------------------------------------------------------------------------151152def enrich(con: sqlite3.Connection, budget: int = 100) -> dict:153    """Recherche OFF pour au plus `budget` nouveaux produits (jamais cherchés),154    puis (ré)applique details.off à tous les produits actifs appariés."""155    rows = con.execute(156        """SELECT p.uid, p.brand, p.name, p.size_label,157                  (l.uid IS NOT NULL) AS linked158           FROM products p LEFT JOIN product_links l ON l.uid = p.uid159           WHERE p.active=1 AND p.brand<>'' AND p.name<>''160           ORDER BY linked DESC, p.last_seen DESC""").fetchall()161162    cached: dict[str, sqlite3.Row] = {163        r["match_key"]: r for r in164        con.execute("SELECT match_key, found, payload FROM off_cache")}165166    session = requests.Session()167    session.headers["User-Agent"] = USER_AGENT168    searched = matched = errors = 0169    now = time.time()170    for r in rows:171        if searched >= budget or errors >= 3:   # OFF injoignable -> on n'insiste pas172            break173        key = match_key(r["brand"], r["name"], r["size_label"])174        if key in cached:175            continue176        searched += 1177        try:178            hits = _search(session, r["brand"], r["name"])179            best, sim = _pick(hits, r["brand"], r["name"], r["size_label"])180            errors = 0181        except Exception:182            errors += 1183            searched -= 1184            time.sleep(REQUEST_DELAY)185            continue186        if best:187            time.sleep(1.0)      # politesse entre recherche et fiche produit188            payload = _payload(best, sim, _ingredients(session, best.get("code")))189        else:190            payload = {}191        found = 1 if best else 0192        matched += found193        con.execute(194            "INSERT INTO off_cache (match_key, code, found, payload, fetched_at)"195            " VALUES (?,?,?,?,?) ON CONFLICT(match_key) DO UPDATE SET"196            " code=excluded.code, found=excluded.found,"197            " payload=excluded.payload, fetched_at=excluded.fetched_at",198            (key, payload.get("code"), found,199             json.dumps(payload, ensure_ascii=False), now))200        con.commit()201        cached[key] = {"match_key": key, "found": found,       # type: ignore[assignment]202                       "payload": json.dumps(payload, ensure_ascii=False)}203        time.sleep(REQUEST_DELAY)204205    # (ré)application : le cache est la source de vérité de details.off206    applied = 0207    for r in con.execute(208            """SELECT uid, brand, name, size_label, details FROM products209               WHERE active=1 AND brand<>'' AND name<>''""").fetchall():210        hit = cached.get(match_key(r["brand"], r["name"], r["size_label"]))211        if hit is None or not hit["found"]:212            continue213        try:214            payload = json.loads(hit["payload"] or "{}")215            details = json.loads(r["details"] or "{}")216        except ValueError:217            continue218        if not payload or details.get("off") == payload:219            continue220        details["off"] = payload221        con.execute("UPDATE products SET details=? WHERE uid=?",222                    (json.dumps(details, ensure_ascii=False), r["uid"]))223        applied += 1224    con.commit()225    with_off = con.execute(226        "SELECT COUNT(*) c FROM products WHERE active=1"227        " AND details LIKE '%\"off\"%'").fetchone()["c"]228    return {"searched": searched, "matched": matched, "applied": applied,229            "products_with_off": with_off}230