Passes rejouables : retry 429 backoff long sur collections.json, géocodage idempotent ; logs/ ignoré

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Simon-Pierre Boucher committed 1 mo ago (Aug 19, 2026) parent a004cf0

3 changed files +51 −31

modified .gitignore +1 −0

		`@@ -11,3 +11,4 @@ data/fabrika.db`
11	11	*.db-journal
12	12	*.db-wal
13	13	*.db-shm
	14	+logs/

modified scripts/enrich_collections.py +33 −15

		`@@ -58,22 +58,27 @@ def fetch_collections(dom: str, base: str) -> dict:`
58	58	time.sleep(wait)
59	59	_last[0] = time.time()
60	60	rec = {"domain": dom, "collections": [], "checked_at": time.strftime("%Y-%m-%d")}
61		− p = subprocess.run(["curl", "-sS", "--compressed", "-L", "--max-time", "25",
62		− "-A", UA, "-w", "\n%{http_code}",
63		− f"{base}/collections.json?limit=250"],
64		− capture_output=True, text=True, errors="replace")
65		− body, _, code = p.stdout.rpartition("\n")
66		− if code == "200":
67		− try:
68		− cols = json.loads(body).get("collections", [])
69		− rec["collections"] = [{"handle": c.get("handle", ""),
70		− "title": c.get("title", ""),
71		− "products_count": c.get("products_count")}
72		− for c in cols]
73		− except Exception as exc:
74		− rec["error"] = str(exc)[:120]
75		− else:
	61	+ for attempt in range(4): # retry 429 avec backoff long
	62	+ p = subprocess.run(["curl", "-sS", "--compressed", "-L", "--max-time", "25",
	63	+ "-A", UA, "-w", "\n%{http_code}",
	64	+ f"{base}/collections.json?limit=250"],
	65	+ capture_output=True, text=True, errors="replace")
	66	+ body, _, code = p.stdout.rpartition("\n")
	67	+ if code == "200":
	68	+ try:
	69	+ cols = json.loads(body).get("collections", [])
	70	+ rec["collections"] = [{"handle": c.get("handle", ""),
	71	+ "title": c.get("title", ""),
	72	+ "products_count": c.get("products_count")}
	73	+ for c in cols]
	74	+ rec.pop("error", None)
	75	+ except Exception as exc:
	76	+ rec["error"] = str(exc)[:120]
	77	+ break
76	78	rec["error"] = f"HTTP {code}"
	79	+ if code != "429":
	80	+ break
	81	+ time.sleep(10 * (attempt + 1))
77	82	json.dump(rec, open(cpath, "w"), ensure_ascii=False)
78	83	return rec
79	84
		`@@ -81,8 +86,21 @@ def fetch_collections(dom: str, base: str) -> dict:`
81	86	def main():
82	87	ap = argparse.ArgumentParser()
83	88	ap.add_argument("--cap", type=int, default=700)
	89	+ ap.add_argument("--retry-errors", action="store_true",
	90	+ help="purge du cache les fiches en erreur (ex. 429) avant la passe")
84	91	args = ap.parse_args()
85	92
	93	+ if args.retry_errors:
	94	+ import glob
	95	+ n = 0
	96	+ for f in glob.glob(os.path.join(CACHE, "*.json")):
	97	+ try:
	98	+ if json.load(open(f)).get("error"):
	99	+ os.remove(f); n += 1
	100	+ except Exception:
	101	+ pass
	102	+ print(f"[collections] {n} caches en erreur purgés")
	103	+
86	104	con = fdb.connect()
87	105	rows = [dict(r) for r in con.execute(
88	106	"SELECT id, url FROM stores WHERE platform='shopify' AND product_count>0 "

modified scripts/geocode_stores.py +17 −16

		`@@ -32,28 +32,29 @@ CACHE = os.path.join(CACHE_DIR, "fsa.json")`
32	32
33	33	from fabrika import db as fdb # noqa: E402
34	34
35		−NOMINATIM = "https://nominatim.openstreetmap.org/search"
	35	+# Nominatim ne résout PAS les RTA seules (postalcode=H2X -> []), constaté le
	36	+# 2026-08-19. zippopotam.us les résout nativement pour le Canada (centroïde +
	37	+# nom du secteur). Même politesse : 1 req/s, cache disque.
	38	+ZIPPO = "https://api.zippopotam.us/CA/{fsa}"
36	39	HDRS = {"User-Agent": "FabriKaBot/1.0 (+https://www.fabri-ka.com/bot; contact@spboucher.ai)"}
37	40
38	41
39	42	def geocode_fsa(fsa: str) -> dict \| None:
40		− """RTA -> {lat, lng, city} via Nominatim (postalcode + country=Canada)."""
41		− r = requests.get(NOMINATIM, params={
42		− "postalcode": fsa, "country": "Canada", "format": "jsonv2",
43		− "addressdetails": 1, "limit": 1}, headers=HDRS, timeout=20)
44		− r.raise_for_status()
45		− items = r.json()
46		− if not items:
	43	+ """RTA -> {lat, lng, city} via zippopotam.us (données GeoNames)."""
	44	+ r = requests.get(ZIPPO.format(fsa=fsa), headers=HDRS, timeout=20)
	45	+ if r.status_code == 404:
47	46	return None
48		− it = items[0]
49		− addr = it.get("address") or {}
50		− city = (addr.get("city") or addr.get("town") or addr.get("village")
51		− or addr.get("municipality") or "")
52		− # garde-fou : rester au Québec/Canada
53		− if addr.get("country_code") not in (None, "ca"):
	47	+ r.raise_for_status()
	48	+ data = r.json()
	49	+ places = data.get("places") or []
	50	+ if not places:
54	51	return None
55		− return {"lat": round(float(it["lat"]), 5), "lng": round(float(it["lon"]), 5),
56		− "city": city}
	52	+ p = places[0]
	53	+ if (p.get("state abbreviation") or "") not in ("QC", ""):
	54	+ return None # garde-fou : rester au Québec
	55	+ return {"lat": round(float(p["latitude"]), 5),
	56	+ "lng": round(float(p["longitude"]), 5),
	57	+ "city": (p.get("place name") or "").strip()}
57	58
58	59
59	60	def main():
60	61