SPB Git forge
spboucher.ai ↗

spb/fabri-ka

Public

Agrégateur de produits québécois — www.fabri-ka.com

217commits 1branches 0releases
66.1 MBsize
maindefault branch
5 h agolast push
Python 38.4% HTML 30.3% TypeScript 17.2% CSS 11% JavaScript 3.2%

Enrichissement boutiques : géocodage RTA via Nominatim (cache), extraction courriel/téléphone/réseaux (pages contact + JSON-LD), taxonomie collections Shopify (table de correspondance consignée)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 19, 2026) parent 1f5ecb8

3 changed files +450 −0

added scripts/enrich_collections.py +145 −0
@@ -0,0 +1,145 @@
1 +#!/usr/bin/env python3
2 +"""Phase 2 — taxonomie métier réelle des boutiques Shopify via /collections.json.
3 +
4 +1 requête par boutique Shopify productive (curl, throttle global 0,7 s) :
5 +la liste des collections (titres + handles + published) est la taxonomie
6 +métier déclarée par le marchand. On la mappe vers la taxonomie Fabri-Ka
7 +(infer_category sur les titres de collections) pour :
8 +
9 + - assigner des catégories boutique (registre + DB) quand il n'y en a pas ;
10 + - alimenter l'héritage de catégorie d'enrich_products.py (produits « autre ») ;
11 + - consigner la table de correspondance collection -> catégorie
12 + (data/collections_taxonomy.json, rejouable et auditable).
13 +
14 +WooCommerce : PAS de requête réseau — la Store API renvoie déjà les
15 +catégories par produit (champ product_type), l'endpoint
16 +/wc/store/v1/products/categories serait redondant (décision consignée).
17 +
18 +Cache : data/enrich_cache/collections/<dom>.json.
19 +Usage : .venv/bin/python scripts/enrich_collections.py [--cap 700]
20 +"""
21 +import argparse
22 +import concurrent.futures as cf
23 +import json
24 +import os
25 +import subprocess
26 +import sys
27 +import threading
28 +import time
29 +from collections import Counter
30 +
31 +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
32 +sys.path.insert(0, ROOT)
33 +CACHE = os.path.join(ROOT, "data", "enrich_cache", "collections")
34 +os.makedirs(CACHE, exist_ok=True)
35 +
36 +from fabrika import db as fdb # noqa: E402
37 +from fabrika.schema import infer_category # noqa: E402
38 +
39 +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
40 + "(KHTML, like Gecko) Chrome/126 Safari/537.36")
41 +
42 +_LOCK = threading.Lock()
43 +_last = [0.0]
44 +
45 +# titres de collections sans signal métier
46 +NOISE = {"accueil", "home", "all", "tous", "tout", "frontpage", "nouveautés",
47 + "nouveautes", "new", "soldes", "sale", "promotions", "promo",
48 + "cadeaux", "gifts", "idées cadeaux", "best sellers", "meilleurs vendeurs"}
49 +
50 +
51 +def fetch_collections(dom: str, base: str) -> dict:
52 + cpath = os.path.join(CACHE, dom + ".json")
53 + if os.path.exists(cpath):
54 + return json.load(open(cpath))
55 + with _LOCK:
56 + wait = 0.7 - (time.time() - _last[0])
57 + if wait > 0:
58 + time.sleep(wait)
59 + _last[0] = time.time()
60 + rec = {"domain": dom, "collections": [], "checked_at": time.strftime("%Y-%m-%d")}
61 + p = subprocess.run(["curl", "-sS", "--compressed", "-L", "--max-time", "25",
62 + "-A", UA, "-w", "\n%{http_code}",
63 + f"{base}/collections.json?limit=250"],
64 + capture_output=True, text=True, errors="replace")
65 + body, _, code = p.stdout.rpartition("\n")
66 + if code == "200":
67 + try:
68 + cols = json.loads(body).get("collections", [])
69 + rec["collections"] = [{"handle": c.get("handle", ""),
70 + "title": c.get("title", ""),
71 + "products_count": c.get("products_count")}
72 + for c in cols]
73 + except Exception as exc:
74 + rec["error"] = str(exc)[:120]
75 + else:
76 + rec["error"] = f"HTTP {code}"
77 + json.dump(rec, open(cpath, "w"), ensure_ascii=False)
78 + return rec
79 +
80 +
81 +def main():
82 + ap = argparse.ArgumentParser()
83 + ap.add_argument("--cap", type=int, default=700)
84 + args = ap.parse_args()
85 +
86 + con = fdb.connect()
87 + rows = [dict(r) for r in con.execute(
88 + "SELECT id, url FROM stores WHERE platform='shopify' AND product_count>0 "
89 + "ORDER BY product_count DESC LIMIT ?", (args.cap,))]
90 + con.close()
91 + print(f"[collections] {len(rows)} boutiques Shopify ciblées", flush=True)
92 +
93 + results, done = [], 0
94 + with cf.ThreadPoolExecutor(4) as ex:
95 + for rec in ex.map(lambda s: fetch_collections(s["id"], (s["url"] or "").rstrip("/")), rows):
96 + results.append(rec)
97 + done += 1
98 + if done % 100 == 0:
99 + print(f" {done}/{len(rows)}", flush=True)
100 +
101 + # table de correspondance collection -> catégorie Fabri-Ka (consignée)
102 + mapping = {}
103 + store_cats = {}
104 + n_cols = 0
105 + for rec in results:
106 + cats = Counter()
107 + for c in rec.get("collections", []):
108 + title = (c.get("title") or "").strip()
109 + if not title or title.lower() in NOISE:
110 + continue
111 + n_cols += 1
112 + cat = infer_category(title)
113 + mapping[title] = cat
114 + if cat != "autre":
115 + cats[cat] += max(1, int(c.get("products_count") or 1))
116 + if cats:
117 + store_cats[rec["domain"]] = [k for k, _ in cats.most_common(3)]
118 +
119 + json.dump({"generated": time.strftime("%Y-%m-%d"),
120 + "note": "titres de collections Shopify -> taxonomie Fabri-Ka (infer_category)",
121 + "mapping": dict(sorted(mapping.items()))},
122 + open(os.path.join(ROOT, "data", "collections_taxonomy.json"), "w"),
123 + ensure_ascii=False, indent=1)
124 + print(f"[collections] {n_cols} collections vues, {len(mapping)} mappées, "
125 + f"{len(store_cats)} boutiques avec catégories déduites")
126 +
127 + # registre + DB : catégories boutique si absentes
128 + reg_path = os.path.join(ROOT, "data", "stores.json")
129 + reg = json.load(open(reg_path))
130 + n_reg = 0
131 + con = fdb.connect()
132 + for s in reg["stores"]:
133 + cats = store_cats.get(s["id"])
134 + if cats and not s.get("categories"):
135 + s["categories"] = cats
136 + con.execute("UPDATE stores SET categories=? WHERE id=?",
137 + (json.dumps(cats, ensure_ascii=False), s["id"]))
138 + n_reg += 1
139 + con.commit(); con.close()
140 + json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)
141 + print(f"[collections] catégories boutique assignées (registre+DB) : {n_reg}")
142 +
143 +
144 +if __name__ == "__main__":
145 + main()
added scripts/enrich_contacts.py +192 −0
@@ -0,0 +1,192 @@
1 +#!/usr/bin/env python3
2 +"""Phase 2 — coordonnées boutiques : courriel, téléphone, réseaux sociaux.
3 +
4 +Pour chaque boutique productive sans courriel :
5 + 1. page d'accueil : mailto:, JSON-LD Organization/LocalBusiness
6 + (email, telephone, sameAs), liens Instagram/Facebook du HTML ;
7 + 2. sinon une page contact usuelle (/contact, /pages/contact,
8 + /nous-joindre, /contactez-nous, /pages/nous-joindre, /a-propos).
9 +
10 +Max 2 requêtes réseau par boutique, throttle 0,5 s par worker (Shopify :
11 +verrou global 0,7 s comme le connecteur). Cache disque
12 +data/enrich_cache/contacts/<dom>.json (échecs inclus).
13 +
14 +Écrit : stores.email, stores.phone (si vide) ; réseaux fusionnés dans le
15 +registre (socials) sans doublon.
16 +
17 +Usage : .venv/bin/python scripts/enrich_contacts.py [--cap 400] [--workers 6]
18 +"""
19 +import argparse
20 +import concurrent.futures as cf
21 +import html as _html
22 +import json
23 +import os
24 +import re
25 +import sys
26 +import threading
27 +import time
28 +
29 +import requests
30 +
31 +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
32 +sys.path.insert(0, ROOT)
33 +CACHE = os.path.join(ROOT, "data", "enrich_cache", "contacts")
34 +os.makedirs(CACHE, exist_ok=True)
35 +
36 +from fabrika import db as fdb # noqa: E402
37 +
38 +HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
39 + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
40 + "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}
41 +
42 +CONTACT_PATHS = ["/pages/contact", "/contact", "/nous-joindre", "/contactez-nous",
43 + "/pages/nous-joindre", "/pages/contactez-nous", "/contact-us"]
44 +EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
45 +BAD_EMAIL = re.compile(r"(sentry|example|wixpress|\.png|\.jpe?g|\.gif|\.webp|\.svg"
46 + r"|@2x|@3x|schema\.org|sentry-next|\.js$|\.css$|no-?reply)", re.I)
47 +MAILTO_RE = re.compile(r'mailto:([^"\'?>\s]+)', re.I)
48 +PHONE_RE = re.compile(r"(?:\+1[ .-]?)?\(?\b([2-9]\d{2})\)?[ .-]?(\d{3})[ .-]?(\d{4})\b")
49 +SOCIAL_RE = re.compile(r'https?://(?:www\.)?(?:instagram\.com|facebook\.com)/[A-Za-z0-9_.\-/%]+', re.I)
50 +LD_RE = re.compile(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', re.I | re.S)
51 +
52 +_SHOPIFY_LOCK = threading.Lock()
53 +_last_shopify = [0.0]
54 +
55 +
56 +def fetch(url, shopify=False, timeout=15):
57 + if shopify:
58 + with _SHOPIFY_LOCK:
59 + wait = 0.7 - (time.time() - _last_shopify[0])
60 + if wait > 0:
61 + time.sleep(wait)
62 + _last_shopify[0] = time.time()
63 + try:
64 + r = requests.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)
65 + if r.status_code == 200 and len(r.text) > 200:
66 + return r.text
67 + except Exception:
68 + pass
69 + return None
70 +
71 +
72 +def harvest(html_text: str, rec: dict) -> None:
73 + """Extrait courriel / téléphone / réseaux d'un HTML (JSON-LD d'abord)."""
74 + for m in LD_RE.finditer(html_text):
75 + try:
76 + data = json.loads(m.group(1).strip())
77 + except Exception:
78 + continue
79 + stack = data if isinstance(data, list) else [data]
80 + while stack:
81 + node = stack.pop()
82 + if isinstance(node, list):
83 + stack.extend(node)
84 + continue
85 + if not isinstance(node, dict):
86 + continue
87 + stack.extend(v for v in node.values() if isinstance(v, (dict, list)))
88 + if not rec.get("email") and isinstance(node.get("email"), str):
89 + e = node["email"].replace("mailto:", "").strip()
90 + if EMAIL_RE.fullmatch(e) and not BAD_EMAIL.search(e):
91 + rec["email"] = e
92 + if not rec.get("phone") and isinstance(node.get("telephone"), str):
93 + rec["phone"] = node["telephone"].strip()[:30]
94 + for u in (node.get("sameAs") or []) if isinstance(node.get("sameAs"), list) else []:
95 + if isinstance(u, str) and SOCIAL_RE.match(u):
96 + rec.setdefault("socials", []).append(u.rstrip("/"))
97 + if not rec.get("email"):
98 + m = MAILTO_RE.search(html_text)
99 + if m:
100 + e = _html.unescape(m.group(1)).strip()
101 + if EMAIL_RE.fullmatch(e) and not BAD_EMAIL.search(e):
102 + rec["email"] = e
103 + if not rec.get("email"):
104 + for e in EMAIL_RE.findall(html_text[:200000]):
105 + if not BAD_EMAIL.search(e):
106 + rec["email"] = e
107 + break
108 + if not rec.get("phone"):
109 + m = PHONE_RE.search(re.sub(r"<[^>]+>", " ", html_text[:150000]))
110 + if m:
111 + rec["phone"] = f"{m.group(1)} {m.group(2)}-{m.group(3)}"
112 + for u in SOCIAL_RE.findall(html_text[:200000]):
113 + if "/sharer" in u or "/share?" in u or "/plugins" in u:
114 + continue
115 + rec.setdefault("socials", []).append(u.rstrip("/"))
116 +
117 +
118 +def work(store):
119 + dom = store["id"]
120 + cpath = os.path.join(CACHE, dom + ".json")
121 + if os.path.exists(cpath):
122 + return json.load(open(cpath))
123 + base = (store["url"] or f"https://{dom}").rstrip("/")
124 + sh = store["platform"] == "shopify"
125 + rec = {"domain": dom, "email": None, "phone": None, "socials": [],
126 + "checked_at": time.strftime("%Y-%m-%d"), "requests": 0}
127 + html_text = fetch(base + "/", shopify=sh)
128 + rec["requests"] += 1
129 + if html_text:
130 + harvest(html_text, rec)
131 + if not rec["email"]:
132 + # une seule page contact : la première trouvée dans le HTML d'accueil
133 + m = re.search(r'href="([^"]*(?:contact|nous-joindre|joindre)[^"]*)"',
134 + html_text, re.I)
135 + path = None
136 + if m:
137 + href = _html.unescape(m.group(1))
138 + if href.startswith("/"):
139 + path = href
140 + elif dom in href:
141 + path = "/" + href.split(dom, 1)[1].lstrip("/")
142 + if not path:
143 + path = CONTACT_PATHS[0] if not sh else "/pages/contact"
144 + page = fetch(base + path, shopify=sh)
145 + rec["requests"] += 1
146 + if page:
147 + harvest(page, rec)
148 + rec["socials"] = sorted(set(rec["socials"]))[:6]
149 + json.dump(rec, open(cpath, "w"), ensure_ascii=False)
150 + return rec
151 +
152 +
153 +def main():
154 + ap = argparse.ArgumentParser()
155 + ap.add_argument("--cap", type=int, default=400)
156 + ap.add_argument("--workers", type=int, default=6)
157 + args = ap.parse_args()
158 +
159 + con = fdb.connect()
160 + rows = [dict(r) for r in con.execute(
161 + "SELECT id, url, platform FROM stores WHERE product_count > 0 "
162 + "AND (email IS NULL OR email='') ORDER BY product_count DESC LIMIT ?",
163 + (args.cap,))]
164 + con.close()
165 + print(f"[contacts] {len(rows)} boutiques ciblées (cap {args.cap})", flush=True)
166 +
167 + results, done, nreq = [], 0, 0
168 + with cf.ThreadPoolExecutor(args.workers) as ex:
169 + for rec in ex.map(work, rows):
170 + results.append(rec)
171 + nreq += rec.get("requests", 0)
172 + done += 1
173 + if done % 50 == 0:
174 + print(f" {done}/{len(rows)}", flush=True)
175 +
176 + con = fdb.connect()
177 + n_email = n_phone = 0
178 + for rec in results:
179 + if rec.get("email"):
180 + con.execute("UPDATE stores SET email=? WHERE id=?", (rec["email"], rec["domain"]))
181 + n_email += 1
182 + if rec.get("phone"):
183 + con.execute("UPDATE stores SET phone=CASE WHEN COALESCE(phone,'')='' "
184 + "THEN ? ELSE phone END WHERE id=?", (rec["phone"], rec["domain"]))
185 + n_phone += 1
186 + con.commit(); con.close()
187 + print(f"[contacts] courriels : {n_email}/{len(results)} | téléphones : {n_phone} "
188 + f"| requêtes réseau : {nreq}")
189 +
190 +
191 +if __name__ == "__main__":
192 + main()
added scripts/geocode_stores.py +113 −0
@@ -0,0 +1,113 @@
1 +#!/usr/bin/env python3
2 +"""Phase 2 — géocodage massif des boutiques via leur RTA (postal_prefix).
3 +
4 +Le registre porte un préfixe postal (RTA, ex. « J0V ») pour 86,6 % des
5 +boutiques mais aucune n'était géocodée. On géocode chaque RTA UNIQUE
6 +(≈ 360) via Nominatim — politesse 1 req/s, cache disque
7 +data/geocode_cache/fsa.json (seedé une fois, rejouable sans réseau) —
8 +puis on propage ville approx. + lat/lng à toutes les boutiques porteuses.
9 +
10 +Précision : centroïde de la RTA (quartier/secteur), pas l'adresse civique —
11 +suffisant pour une carte régionale et le tri par proximité. La ville n'est
12 +écrite que si Nominatim la fournit (aucune donnée inventée).
13 +
14 +Écrit : stores.city (si vide) / stores.lat / stores.lng en DB
15 + + champ "geo" {lat,lng,city,precision:"fsa"} dans data/stores.json.
16 +
17 +Usage : .venv/bin/python scripts/geocode_stores.py [--dry-run] [--max-requests N]
18 +"""
19 +import argparse
20 +import json
21 +import os
22 +import sys
23 +import time
24 +
25 +import requests
26 +
27 +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
28 +sys.path.insert(0, ROOT)
29 +CACHE_DIR = os.path.join(ROOT, "data", "geocode_cache")
30 +os.makedirs(CACHE_DIR, exist_ok=True)
31 +CACHE = os.path.join(CACHE_DIR, "fsa.json")
32 +
33 +from fabrika import db as fdb # noqa: E402
34 +
35 +NOMINATIM = "https://nominatim.openstreetmap.org/search"
36 +HDRS = {"User-Agent": "FabriKaBot/1.0 (+https://www.fabri-ka.com/bot; contact@spboucher.ai)"}
37 +
38 +
39 +def geocode_fsa(fsa: str) -> dict | None:
40 + """RTA -> {lat, lng, city} via Nominatim (postalcode + country=Canada)."""
41 + r = requests.get(NOMINATIM, params={
42 + "postalcode": fsa, "country": "Canada", "format": "jsonv2",
43 + "addressdetails": 1, "limit": 1}, headers=HDRS, timeout=20)
44 + r.raise_for_status()
45 + items = r.json()
46 + if not items:
47 + return None
48 + it = items[0]
49 + addr = it.get("address") or {}
50 + city = (addr.get("city") or addr.get("town") or addr.get("village")
51 + or addr.get("municipality") or "")
52 + # garde-fou : rester au Québec/Canada
53 + if addr.get("country_code") not in (None, "ca"):
54 + return None
55 + return {"lat": round(float(it["lat"]), 5), "lng": round(float(it["lon"]), 5),
56 + "city": city}
57 +
58 +
59 +def main():
60 + ap = argparse.ArgumentParser()
61 + ap.add_argument("--dry-run", action="store_true")
62 + ap.add_argument("--max-requests", type=int, default=400)
63 + args = ap.parse_args()
64 +
65 + reg_path = os.path.join(ROOT, "data", "stores.json")
66 + reg = json.load(open(reg_path))
67 + fsas = sorted({(s.get("postal_prefix") or "").strip().upper()[:3]
68 + for s in reg["stores"] if s.get("postal_prefix")})
69 + cache = json.load(open(CACHE)) if os.path.exists(CACHE) else {}
70 + todo = [f for f in fsas if f not in cache][:args.max_requests]
71 + print(f"[geocode] {len(fsas)} RTA uniques, {len(todo)} à résoudre "
72 + f"(cache: {len(cache)})", flush=True)
73 +
74 + n_req = 0
75 + for fsa in todo:
76 + try:
77 + cache[fsa] = geocode_fsa(fsa)
78 + except Exception as exc:
79 + print(f" ! {fsa}: {exc}", flush=True)
80 + n_req += 1
81 + if n_req % 25 == 0:
82 + print(f" {n_req}/{len(todo)}", flush=True)
83 + json.dump(cache, open(CACHE, "w"))
84 + time.sleep(1.1) # politesse Nominatim : 1 req/s max
85 + json.dump(cache, open(CACHE, "w"))
86 + resolved = {k: v for k, v in cache.items() if v}
87 + print(f"[geocode] RTA résolues : {len(resolved)}/{len(fsas)} "
88 + f"({n_req} requêtes réseau)")
89 +
90 + if args.dry_run:
91 + return
92 +
93 + n_store = 0
94 + con = fdb.connect()
95 + for s in reg["stores"]:
96 + fsa = (s.get("postal_prefix") or "").strip().upper()[:3]
97 + geo = resolved.get(fsa)
98 + if not geo:
99 + continue
100 + s["geo"] = {"lat": geo["lat"], "lng": geo["lng"],
101 + "city": geo["city"], "precision": "fsa"}
102 + con.execute("UPDATE stores SET lat=?, lng=?, "
103 + "city=CASE WHEN COALESCE(city,'')='' THEN ? ELSE city END "
104 + "WHERE id=?",
105 + (geo["lat"], geo["lng"], geo["city"], s["id"]))
106 + n_store += 1
107 + con.commit(); con.close()
108 + json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)
109 + print(f"[geocode] boutiques géocodées : {n_store}")
110 +
111 +
112 +if __name__ == "__main__":
113 + main()
114