spb/fabri-ka Public

1#!/usr/bin/env python32"""Enrichissement des boutiques : logo, image de couverture, description.34Pour chaque boutique du registre : télécharge la page d'accueil, extrait5  - logo : apple-touch-icon > icon (plus grande taille) > og:logo6  - couverture : og:image7  - description : meta description > og:description8  - nom d'affichage : og:site_name (si plus propre que le nom actuel)9Écrit dans la table stores (colonnes logo_url, cover_url, description_meta).10Cache disque data/enrich_cache/<domain>.json ; Scrapfly en secours pour les 403.11"""12import concurrent.futures as cf13import json14import os15import re16import sys17from urllib.parse import urljoin1819import requests2021ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))22sys.path.insert(0, ROOT)23CACHE = os.path.join(ROOT, "data", "enrich_cache")24os.makedirs(CACHE, exist_ok=True)2526for line in open(os.path.join(ROOT, ".env")).read().splitlines():27    if "=" in line and not line.startswith("#"):28        k, _, v = line.partition("=")29        os.environ.setdefault(k.strip(), v.strip())3031from fabrika import db as fdb  # noqa: E40232from fabrika.connectors.scrapfly import scrapfly_get, available  # noqa: E4023334HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",35        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}3637LINK_RE = re.compile(r'<link[^>]+>', re.I)38META_RE = re.compile(r'<meta[^>]+>', re.I)394041def attr(tag, name):42    m = re.search(name + r'\s*=\s*["\']([^"\']+)["\']', tag, re.I)43    return m.group(1) if m else None444546def extract(base_url, html):47    head = html[:120000]48    logo, cover, desc, site_name = None, None, None, None49    icons = []50    for tag in LINK_RE.findall(head):51        rel = (attr(tag, "rel") or "").lower()52        href = attr(tag, "href")53        if not href:54            continue55        if "apple-touch-icon" in rel:56            icons.append((200 if not attr(tag, "sizes") else int((attr(tag, "sizes") or "180x").split("x")[0] or 180), href))57        elif rel in ("icon", "shortcut icon"):58            sizes = attr(tag, "sizes") or "32x32"59            try:60                s = int(sizes.split("x")[0])61            except ValueError:62                s = 3263            if not href.endswith(".ico"):64                icons.append((s, href))65    if icons:66        icons.sort(reverse=True)67        logo = urljoin(base_url, icons[0][1])68    for tag in META_RE.findall(head):69        prop = (attr(tag, "property") or attr(tag, "name") or "").lower()70        content = attr(tag, "content")71        if not content:72            continue73        if prop == "og:image" and not cover:74            cover = urljoin(base_url, content)75        elif prop in ("description", "og:description") and not desc:76            desc = content.strip()[:400]77        elif prop == "og:site_name" and not site_name:78            site_name = content.strip()[:80]79        elif prop == "og:logo" and not logo:80            logo = urljoin(base_url, content)81    return {"logo_url": logo, "cover_url": cover, "description_meta": desc, "site_name": site_name}828384def enrich_domain(store):85    dom = store["id"]86    cpath = os.path.join(CACHE, dom + ".json")87    if os.path.exists(cpath):88        return json.load(open(cpath))89    url = store.get("url") or f"https://{dom}"90    html = ""91    try:92        r = requests.get(url, headers=HDRS, timeout=20, allow_redirects=True)93        if r.status_code == 200:94            html, url = r.text, r.url95    except Exception:96        pass97    if not html and available():98        try:99            status, content = scrapfly_get(url)100            if status == 200:101                html = content102        except Exception:103            pass104    rec = {"domain": dom}105    if html:106        rec.update(extract(url, html))107        # fallback logo : favicon.ico si rien trouvé108        if not rec.get("logo_url"):109            rec["logo_url"] = urljoin(url, "/favicon.ico")110    json.dump(rec, open(cpath, "w"))111    return rec112113114def main(only_live=True):115    con = fdb.connect()116    # colonnes d'enrichissement117    for col in ("logo_url", "cover_url", "description_meta"):118        try:119            con.execute(f"ALTER TABLE stores ADD COLUMN {col} TEXT")120        except Exception:121            pass122    con.commit()123    rows = [dict(r) for r in con.execute(124        "SELECT id, url, product_count FROM stores ORDER BY product_count DESC")]125    con.close()126    if only_live:127        pass  # on enrichit tout le registre, les vivantes d'abord (tri ci-dessus)128    print(f"enrichissement de {len(rows)} boutiques…")129    done = 0130    results = []131    with cf.ThreadPoolExecutor(16) as ex:132        for rec in ex.map(enrich_domain, rows):133            results.append(rec)134            done += 1135            if done % 200 == 0:136                print(f"  {done}/{len(rows)}", flush=True)137    con = fdb.connect()138    n = 0139    for rec in results:140        if rec.get("logo_url") or rec.get("description_meta"):141            con.execute("UPDATE stores SET logo_url=?, cover_url=?, description_meta=? WHERE id=?",142                        (rec.get("logo_url"), rec.get("cover_url"),143                         rec.get("description_meta"), rec["domain"]))144            n += 1145    con.commit(); con.close()146    print(f"boutiques enrichies: {n}/{len(results)}")147148149if __name__ == "__main__":150    main()151