Agrégateur de produits québécois — www.fabri-ka.com
Python 38.4%
HTML 30.3%
TypeScript 17.2%
CSS 11%
JavaScript 3.2%
1#!/usr/bin/env python32"""Vague 4 (Phase 3 — expansion) : nouvelles sources de fabricants.34Sources moissonnées (fiche d'évaluation datée du 2026-08-19 dans5docs/CONFORMITE.md ; chaque source a été sondée AVANT connexion) :67 cartv_bio Répertoire public des entreprises certifiées biologiques8 du Québec (CARTV / SIPAB, produitsbioquebec.info).9 Formulaire Struts public « recherche par type10 d'opération » ; 7 types × toutes les régions.11 Fiches complètes : adresse, municipalité, région, CP,12 tél, site web, certificateur, date de certification.13 -> découverte + champ `certifications` structuré.14 ctaq Répertoire des membres du Conseil de la transformation15 alimentaire du Québec (conseiltaq.com,16 /ajax-search-organisation). On ne garde PAS les17 « Associés » (fournisseurs de services) — seulement les18 transformateurs. Fiche /organisation/<id> = site web.19 vendors_collectifs Champ `vendor` des boutiques collectives déjà20 connectées (paperole, galerieiris, wachiya — patron21 Signé Local prouvé) -> résolution prudente du site22 officiel par candidats de domaine dérivés du nom,23 validés par correspondance du titre de page.24 cibim Boulangeries artisanales membres de la CIBIM25 (cibim.org/membres) — répertoire recommandé par l'UPA.26 canardduquebec Producteurs — Éleveurs de canards et d'oies du Québec.27 lebongoutfraisdesiles Producteurs/transformateurs des Îles-de-la-Madeleine28 (région sous-couverte).29 acheterquebecois_mtl Catégories « Montréal / Fabriqué à Montréal »30 d'acheterquebecois.ca (Montréal sous-représentée ;31 PME MTL n'a aucun répertoire public « Fabriqué à32 Montréal » — constaté 2026-08-19).3334Écartées (raison consignée dans docs/CONFORMITE.md) : REQ open data35(WAF Cloudflare + licence CC-BY-NC-SA non commerciale), UPA Mangeons local36(app web décommissionnée — redirections vers upa.qc.ca/citoyen), Salon des37métiers d'art (salondesmetiersdart.com = coquille ; liste des exposants déjà38couverte par le répertoire CMAQ), Goûtez Lanaudière (CRM Eudonet toujours en39404, re-testé), PME MTL (aucun répertoire).4041Usage :42 python3 scripts/wave4_discovery.py harvest [source ...]43 python3 scripts/wave4_discovery.py integrate # additif, sérialisé44 # (vérifier qu'aucune passe45 # n'écrit stores.json)46"""47import argparse48import concurrent.futures as cf49import html as htmllib50import json51import os52import re53import subprocess54import sys55import time56import unicodedata57from collections import Counter58from urllib.parse import urlparse5960import requests6162ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))63RAW = os.path.join(ROOT, "data", "raw")64ENR = os.path.join(ROOT, "data", "enriched")65sys.path.insert(0, ROOT)66sys.path.insert(0, os.path.join(ROOT, "scripts"))6768for line in open(os.path.join(ROOT, ".env")).read().splitlines():69 if "=" in line and not line.startswith("#"):70 k, _, v = line.partition("=")71 os.environ.setdefault(k.strip(), v.strip())7273from verify import HDRS, POSTAL_RE, AREA_RE, verify_domain # noqa: E40274from aggregate import BLOCK, norm_domain # noqa: E4027576TAG_RE = re.compile(r"<[^>]+>")77SESS = requests.Session()787980def strip_tags(s):81 return re.sub(r"\s+", " ", TAG_RE.sub(" ", htmllib.unescape(s or ""))).strip()828384def get(url, timeout=25):85 try:86 r = SESS.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)87 if r.status_code == 200:88 return r.text89 except Exception:90 pass91 return ""929394def write_jsonl(name, records):95 os.makedirs(RAW, exist_ok=True)96 path = os.path.join(RAW, name + ".jsonl")97 with open(path, "w") as f:98 for r in records:99 f.write(json.dumps(r, ensure_ascii=False) + "\n")100 print(f"[harvest] {name}: {len(records)} entrées -> {path}", flush=True)101102103# ------------------------------------------------------------------ CARTV bio104# SIPAB (produitsbioquebec.info) : danse Struts en 4 temps, par type105# d'opération. Réponses en ISO-8859-1.106SIPAB_BASE = "http://www.produitsbioquebec.info"107SIPAB_ACTION = SIPAB_BASE + "/produitsbioquebec/DispatcherInterrogationGrandPublicFr.do"108SIPAB_TYPES = {109 "10": "Préparation alimentaire et transformation",110 "11": "Boissons alcoolisées",111 "20": "Production acéricole",112 "30": "Production animale",113 "40": "Production végétale",114 "50": "Récoltes sauvages et PFNL",115 "90": "Conditionnement (emballage et étiquetage)",116}117FIELD_RES = {118 "address": re.compile(r"Adresse:\s*([^<]+)"),119 "city": re.compile(r"Municipalité:\s*([^<]+)"),120 "region": re.compile(r"Région:\s*([^<]+)"),121 "postal": re.compile(r"Code postal:\s*([^<]+)"),122 "phone": re.compile(r"Tél\.:\s*([^<]+)"),123 "cert_date": re.compile(r"Date de certification:\s*([0-9-]+)"),124 "certifier": re.compile(r"Produits certifiés par:\s*([^<]+)"),125}126WEB_RE = re.compile(r'Site web:.*?<a href="([^"]+)"', re.S)127NAME_RE = re.compile(r'<font class="text1Bleu">([^<]+)</font>')128129130def sipab_search(code, label):131 """Une recherche complète pour un type d'opération -> liste de fiches."""132 sess = requests.Session()133 sess.get(SIPAB_BASE + "/interroGrandPublicFr.do", headers=HDRS, timeout=40)134 common = {"rechercheParProduitOuMunicipalite": "4", "langue": "Fr"}135 for action, extra in (136 ("rechercheParTypeOperation", {}),137 ("initInterrogationGrandPublicTypeOperationForm", {"codeTypesOperation": code}),138 ("rechercherDonneesProduitsCertifiesTypeOperationGrandPublic",139 {"codeTypesOperation": code, "codeRegionForChoixMultiple": "-1"})):140 data = dict(common, actionDemandee=action, **extra)141 r = sess.post(SIPAB_ACTION, data=data, headers=HDRS, timeout=180)142 time.sleep(1.0)143 # le serveur SIPAB sert de l'UTF-8 sans le déclarer (constaté : décodage144 # latin-1 = mojibake sur les champs accentués)145 html = r.content.decode("utf-8", "replace")146 out = []147 # chaque entreprise = une cellule class="celluleEntreprise"148 for cell in re.findall(r'<td class="celluleEntreprise">(.*?)</td>', html, re.S):149 nm = NAME_RE.search(cell)150 if not nm:151 continue152 rec = {"name": strip_tags(nm.group(1))[:100], "operation": label}153 for key, rx in FIELD_RES.items():154 m = rx.search(cell)155 rec[key] = strip_tags(m.group(1)).strip("\xa0 ") if m else ""156 # noms de régions SIPAB : « Saguenay--Lac-Saint-Jean » -> tiret cadratin157 # canonique du registre158 rec["region"] = rec.get("region", "").replace("--", "–")159 w = WEB_RE.search(cell)160 rec["website"] = htmllib.unescape(w.group(1)).strip() if w else ""161 out.append(rec)162 print(f" [cartv] {label}: {len(out)} entreprises", flush=True)163 return out164165166def harvest_cartv_bio():167 """7 recherches (une par type d'opération), toutes régions."""168 seen, records = {}, []169 for code, label in SIPAB_TYPES.items():170 for rec in sipab_search(code, label):171 key = (rec["name"].lower(), rec.get("city", "").lower())172 if key in seen: # même entreprise, autre type d'opération173 if label not in seen[key]["operations"]:174 seen[key]["operations"].append(label)175 continue176 postal = (rec.pop("postal") or "").replace(" ", "")177 fiche = {178 "url": SIPAB_BASE + "/interroGrandPublicFr.do",179 "title": rec["name"],180 "h1": rec["name"],181 "websites": [rec["website"]] if rec["website"] else [],182 "socials": [],183 "postal_prefix": postal[:3] if len(postal) >= 6 else None,184 "phone": rec.get("phone") or None,185 "regions_mentioned": [rec["region"]] if rec.get("region") else [],186 "text_sample": f"{rec.get('address','')} {rec.get('city','')}",187 # champs propres à la certification (exploités par188 # scripts/backfill_certifications.py)189 "city": rec.get("city", ""),190 "certifier": rec.get("certifier", ""),191 "cert_date": rec.get("cert_date", ""),192 "operations": [label],193 }194 seen[key] = fiche195 records.append(fiche)196 write_jsonl("cartv_bio", records)197198199# ----------------------------------------------------------------------- CTAQ200CTAQ_LIST = "https://conseiltaq.com/ajax-search-organisation?page={p}&type=&per_page=50&lang=fr"201CTAQ_CARD = re.compile(202 r'member-item-list js-block-link.*?href="(https://conseiltaq\.com/organisation/\d+)">([^<]+)</a>'203 r'.*?<div class="desc[^"]*">\s*([^<]*).*?(?:<div class="sector[^"]*">\s*([^<]*))?</div>', re.S)204205206def harvest_ctaq():207 fiches = {}208 for p in range(1, 30):209 html = get(CTAQ_LIST.format(p=p), timeout=40)210 if not html:211 break212 seg = html[html.find("Tous les membres"):]213 cards = CTAQ_CARD.findall(seg)214 if not cards:215 break216 for url, name, kind, sector in cards:217 kind = htmllib.unescape(kind.strip())218 # on ne garde que les FABRICANTS : transformateurs alimentaires et219 # fabricants d'ingrédients ; les « Associés », « Fournisseur220 # produits et services », « Affilié » etc. sont des fournisseurs221 # de l'industrie, pas des fabricants de produits québécois222 if kind not in ("Transformateur", "Fournisseur d'ingrédients"):223 continue224 fiches.setdefault(url, {"name": strip_tags(name), "kind": kind,225 "sector": strip_tags(sector or "")})226 time.sleep(0.5)227 print(f"[ctaq] {len(fiches)} fiches membres (hors Associés) à visiter", flush=True)228 records, done = [], 0229 for url, meta in fiches.items():230 done += 1231 html = get(url, timeout=30)232 time.sleep(0.4)233 if done % 50 == 0:234 print(f" [ctaq] {done}/{len(fiches)}", flush=True)235 if not html:236 continue237 body = html[html.find("</header>"):] if "</header>" in html else html238 webs = []239 for u in re.findall(r'href="(https?://[^"]+)"', body):240 host = (urlparse(u).netloc or "").lower()241 if not host or "conseiltaq" in host:242 continue243 if any(s in host for s in ("facebook", "instagram", "linkedin", "youtube",244 "twitter", "google", "w3.org", "jsdelivr",245 "fonts.", "gstatic", "recaptcha")):246 continue247 webs.append(u)248 text = strip_tags(body[:150000])249 postal = POSTAL_RE.search(text)250 phone = AREA_RE.search(text)251 records.append({252 "url": url,253 "title": meta["name"],254 "h1": meta["name"],255 "websites": list(dict.fromkeys(webs))[:2],256 "socials": [],257 "postal_prefix": postal.group(0)[:3] if postal else None,258 "phone": phone.group(0) if phone else None,259 "regions_mentioned": [],260 "text_sample": (meta["kind"] + " — " + meta["sector"])[:200],261 "category_hint": meta["sector"],262 })263 records = [r for r in records if r["websites"]]264 write_jsonl("ctaq", records)265266267# ------------------------------------------------- vendors des collectifs268COLLECTIFS = ["paperole.com", "galerieiris.com", "wachiya.com"]269STOP_TOKENS = {"inc", "enr", "ltee", "les", "the", "and", "et", "de", "du",270 "la", "le", "des", "by", "par", "studio", "atelier", "co"}271272273def _slug(name):274 s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode()275 return re.sub(r"[^a-z0-9]", "", s.lower())276277278def _norm_name(name):279 s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode().lower()280 return " ".join(t for t in re.findall(r"[a-z0-9]+", s) if t not in STOP_TOKENS)281282283def _curl(url, timeout=20):284 """GET via curl (mécanisme du projet pour éviter le 429 TLS de requests)."""285 try:286 r = subprocess.run(["curl", "-sL", "-A", HDRS["User-Agent"], "--max-time",287 str(timeout), "--max-filesize", "400000", url],288 capture_output=True, timeout=timeout + 10)289 return r.stdout.decode("utf-8", "replace")290 except Exception:291 return ""292293294def harvest_vendors_collectifs(cap=400):295 from fabrika import db as fdb296 con = fdb.connect()297 rows = con.execute(298 "SELECT vendor, COUNT(*) AS n, GROUP_CONCAT(DISTINCT store_id) AS sids "299 "FROM products WHERE active=1 AND vendor<>'' AND store_id IN (%s) "300 "GROUP BY vendor ORDER BY n DESC" % ",".join("?" * len(COLLECTIFS)),301 COLLECTIFS).fetchall()302 known_names = {_norm_name(r[0]) for r in303 con.execute("SELECT name FROM stores") if r[0]}304 known_doms = {r[0] for r in con.execute("SELECT id FROM stores")}305 con.close()306 # candidats déjà connus du pipeline (peu importe le statut)307 cand_path = os.path.join(ENR, "candidates.jsonl")308 if os.path.exists(cand_path):309 with open(cand_path) as f:310 known_doms |= {json.loads(l)["domain"] for l in f}311312 records, probes = [], 0313 for vendor, n, sids in rows:314 v = vendor.strip()315 if len(v) < 3 or len(v) > 60 or _norm_name(v) in known_names:316 continue317 slug = _slug(v)318 if not (4 <= len(slug) <= 30):319 continue320 if probes >= cap:321 break322 found = None323 for dom in (slug + ".com", slug + ".ca"):324 if dom in known_doms or BLOCK.search(dom):325 continue326 probes += 1327 html = _curl(f"https://{dom}")328 time.sleep(0.3)329 if not html:330 continue331 title_m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)332 title = _norm_name(strip_tags(title_m.group(1))[:120]) if title_m else ""333 # garde-fou anti-faux-positif : le titre de la page doit334 # recouper le nom de la marque335 vt = set(_norm_name(v).split())336 if vt and title and len(vt & set(title.split())) / len(vt) >= 0.6:337 found = dom338 break339 if found:340 records.append({341 "name": v,342 "domain": found,343 "url": f"https://{found}",344 "region_hint": "",345 "category_hint": "",346 "evidence": f"Marque vendue par les collectifs d'artisans {sids} "347 f"({n} produits) — site officiel résolu et validé par titre",348 "query": f"vendor:{v}",349 })350 print(f"[vendors] {probes} sondes, {len(records)} sites résolus", flush=True)351 write_jsonl("vendors_collectifs", records)352353354# --------------------------------------------- petits répertoires sectoriels355def harvest_single_page(source, page_url, own_domain, region_hint, evidence):356 html = get(page_url, timeout=40)357 records, seen = [], set()358 for url, label in re.findall(r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>',359 html, re.S | re.I):360 url = htmllib.unescape(url)361 host = (urlparse(url).netloc or "").lower()362 if not host or own_domain in host:363 continue364 if any(s in host for s in ("facebook", "instagram", "linkedin", "youtube",365 "tiktok", "pinterest", "twitter", "x.com")):366 continue367 dom = norm_domain(url)368 if not dom or dom in seen or BLOCK.search(dom) or BLOCK.search(url):369 continue370 seen.add(dom)371 label = strip_tags(label)372 records.append({373 "name": label if 2 < len(label) < 80 else "",374 "domain": dom, "url": url, "region_hint": region_hint,375 "category_hint": "", "evidence": evidence, "query": page_url,376 })377 write_jsonl(source, records)378379380def harvest_aq_mtl():381 """Catégories montréalaises d'acheterquebecois.ca (source déjà couverte,382 pages régionales jamais moissonnées). Pagination /page/N/."""383 base = "https://acheterquebecois.ca"384 cats = ["fabrique-a-montreal", "artisanat", "vetements-2", "alimentation"]385 records, seen = [], set()386 for cat in cats:387 for page in range(1, 8):388 url = f"{base}/montreal/{cat}/" + (f"page/{page}/" if page > 1 else "")389 html = get(url, timeout=30)390 if not html:391 break392 found = 0393 for u, label in re.findall(394 r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>', html, re.S | re.I):395 dom = norm_domain(u)396 if (not dom or dom in seen or "acheterquebecois" in dom397 or BLOCK.search(dom) or BLOCK.search(u)):398 continue399 seen.add(dom)400 found += 1401 records.append({402 "name": strip_tags(label)[:80], "domain": dom, "url": u,403 "region_hint": "Montréal", "category_hint": cat,404 "evidence": "Répertorié « Fabriqué à Montréal » / catégorie "405 "montréalaise sur acheterquebecois.ca",406 "query": url,407 })408 time.sleep(0.4)409 if not found:410 break411 write_jsonl("acheterquebecois_mtl", records)412413414HARVESTERS = {415 "cartv_bio": harvest_cartv_bio,416 "ctaq": harvest_ctaq,417 "vendors_collectifs": harvest_vendors_collectifs,418 "cibim": lambda: harvest_single_page(419 "cibim", "https://cibim.org/membres/", "cibim.org", "",420 "Boulangerie artisanale membre de la CIBIM (Corporation des "421 "boulangers-pâtissiers indépendants)"),422 "canardduquebec": lambda: harvest_single_page(423 "canardduquebec", "https://canardduquebec.com/nos-producteurs/",424 "canardduquebec.com", "",425 "Producteur membre des Éleveurs de canards et d'oies du Québec"),426 "lebongoutfraisdesiles": lambda: harvest_single_page(427 "lebongoutfraisdesiles",428 "https://lebongoutfraisdesiles.com/production-et-transformation/",429 "lebongoutfraisdesiles.com", "Gaspésie–Îles-de-la-Madeleine",430 "Producteur/transformateur membre du Bon goût frais des Îles-de-la-Madeleine"),431 "acheterquebecois_mtl": harvest_aq_mtl,432}433434WAVE4_SOURCES = list(HARVESTERS)435436# Boutiques Shopify re-testées via curl (429 TLS python requests = gotcha437# connu) — grosses places de marché de produits québécois, connectées avec438# leur nature affichée (voir docs/CONFORMITE.md).439KNOWN_SHOPIFY = [440 {"id": "epipresto.ca", "name": "EPIPRESTO", "store_kind": "collectif",441 "origin_class": "C", "origin_confidence": 0.7,442 "origin_evidence": "Place de marché regroupant des épiceries et producteurs "443 "locaux du Québec (re-testée via curl le 2026-08-19 ; "444 "l'ancienne vérification avait échoué sur un 429 TLS)",445 "categories": ["epicerie"], "region": ""},446 {"id": "laboiteagrains.com", "name": "La Boite à Grains",447 "store_kind": "revendeur",448 "origin_class": "C", "origin_confidence": 0.7,449 "origin_evidence": "Épicerie santé de Gatineau (revendeur — produits "450 "québécois et autres ; re-testée via curl le 2026-08-19)",451 "categories": ["epicerie"], "region": "Outaouais"},452]453454455def harvest(only=None):456 os.makedirs(RAW, exist_ok=True)457 for name, fn in HARVESTERS.items():458 if only and name not in only:459 continue460 print(f"[harvest] === {name} ===", flush=True)461 fn()462463464def integrate():465 """Intégration additive au registre + DB (mêmes garde-fous que la vague 3)."""466 from datetime import date467 import build_registry as br468 from fabrika import db as fdb469470 for s in WAVE4_SOURCES:471 assert s in br.SOURCE_PRIORS, f"prior manquant dans build_registry: {s}"472473 subprocess.run([sys.executable, os.path.join(ROOT, "scripts", "aggregate.py")],474 check=True)475476 cands = {}477 with open(os.path.join(ENR, "candidates.jsonl")) as f:478 for line in f:479 c = json.loads(line)480 cands[c["domain"]] = c481482 verified_path = os.path.join(ENR, "verified.jsonl")483 verified = {}484 with open(verified_path) as f:485 for line in f:486 v = json.loads(line)487 verified[v["domain"]] = v488489 reg_path = os.path.join(ROOT, "data", "stores.json")490 reg = json.load(open(reg_path))491 existing_ids = {s["id"] for s in reg["stores"]}492493 new_domains = sorted(d for d in cands494 if d not in verified and d not in existing_ids)495 print(f"[integrate] {len(new_domains)} nouveaux domaines à vérifier", flush=True)496497 new_recs = []498 with cf.ThreadPoolExecutor(12) as ex:499 for i, rec in enumerate(ex.map(verify_domain, new_domains)):500 new_recs.append(rec)501 if (i + 1) % 100 == 0:502 print(f" verify {i+1}/{len(new_domains)}", flush=True)503 with open(verified_path, "a") as f:504 for r in new_recs:505 f.write(json.dumps(r, ensure_ascii=False) + "\n")506 verified[r["domain"]] = r507508 added, skipped_dup, skipped_qc, skipped_dead = [], 0, 0, 0509 for dom in new_domains:510 cand, ver = cands[dom], verified.get(dom, {})511 if not ver.get("active"):512 skipped_dead += 1513 continue514 final_dom = ver.get("final_domain") or dom515 if final_dom in existing_ids:516 skipped_dup += 1517 continue518 qc_signal = any(ver.get(k) for k in ("qc_postal", "qc_phone", "tld_quebec",519 "mentions_quebec", "made_in_qc_wording"))520 qc_source = any(s in br.QC_ONLY_SOURCES for s in cand.get("sources", []))521 if not qc_signal and not qc_source:522 skipped_qc += 1523 continue524 cls, conf, ev = br.classify(cand, ver)525 default_cat = next((br.SOURCE_PRIORS[s][3] for s in br.PRIORITY526 if s in cand.get("sources", []) and br.SOURCE_PRIORS[s][3]), None)527 platform = ver.get("platform") or ""528 catalog_endpoint = ver.get("catalog_endpoint") or ""529 if platform == "wix" and not catalog_endpoint:530 catalog_endpoint = "/_api/wix-ecommerce-storefront-web/api"531 fu = urlparse(ver.get("final_url") or f"https://{final_dom}")532 store = {533 "id": final_dom,534 "name": br.clean_name(cand, ver),535 "url": f"{fu.scheme}://{fu.netloc}",536 "platform": platform,537 "catalog_endpoint": catalog_endpoint,538 "city": "",539 "region": br.pick_region(cand) or br.region_from_postal(cand, ver),540 "postal_prefix": cand.get("postal_prefix") or (ver.get("qc_postal") or "")[:3] or None,541 "phone": cand.get("phone") or ver.get("qc_phone"),542 "origin_class": cls,543 "origin_confidence": conf,544 "origin_evidence": ev,545 "categories": [default_cat] if default_cat else [],546 "socials": (cand.get("socials") or [])[:4] or ver.get("socials", []),547 "discovery_sources": cand.get("sources", []),548 "discovery_source_urls": cand.get("source_pages", [])[:5],549 "language": ver.get("language"),550 "ecommerce": bool(ver.get("has_cart") or catalog_endpoint),551 "verification_date": ver.get("checked_at") or str(date.today()),552 "status": "verified" if (conf >= 0.6 and ver.get("mentions_quebec")) else "probable",553 "enabled": bool(catalog_endpoint),554 }555 existing_ids.add(final_dom)556 added.append(store)557558 # --- boutiques Shopify re-testées via curl (hors pipeline verify) ------559 for spec in KNOWN_SHOPIFY:560 if spec["id"] in existing_ids:561 continue562 probe = _curl(f"https://{spec['id']}/products.json?limit=1")563 if '"products"' not in probe[:200]:564 print(f"[integrate] {spec['id']}: /products.json injoignable — ignorée")565 continue566 store = {567 "id": spec["id"], "name": spec["name"], "url": f"https://{spec['id']}",568 "platform": "shopify", "catalog_endpoint": "/products.json",569 "city": "", "region": spec["region"], "postal_prefix": None,570 "phone": None, "origin_class": spec["origin_class"],571 "origin_confidence": spec["origin_confidence"],572 "origin_evidence": spec["origin_evidence"],573 "categories": spec["categories"], "socials": [],574 "discovery_sources": ["retest_curl_2026_08_19"],575 "discovery_source_urls": [], "language": "fr", "ecommerce": True,576 "verification_date": str(date.today()), "status": "verified",577 "enabled": True, "store_kind": spec["store_kind"],578 }579 existing_ids.add(spec["id"])580 added.append(store)581582 reg["stores"].extend(added)583 reg["count"] = len(reg["stores"])584 reg["generated"] = str(date.today())585 json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)586587 con = fdb.connect()588 for s in added:589 fdb.upsert_store(con, s)590 con.commit(); con.close()591592 enabled = [s["id"] for s in added if s["enabled"]]593 per_plat = Counter(s["platform"] or "(aucune)" for s in added)594 per_src = Counter(src for s in added for src in s["discovery_sources"])595 per_cls = Counter(s["origin_class"] for s in added)596 print(f"[integrate] boutiques ajoutées: {len(added)} | connectables (enabled): {len(enabled)}")597 print(f"[integrate] écartées — mortes/injoignables: {skipped_dead}, "598 f"dédup domaine final: {skipped_dup}, sans preuve QC: {skipped_qc}")599 print("[integrate] par plateforme:", dict(per_plat.most_common()))600 print("[integrate] par source:", dict(per_src.most_common()))601 print("[integrate] par classe:", dict(per_cls))602 with open(os.path.join(ROOT, "data", "wave4_new_enabled.txt"), "w") as f:603 f.write("\n".join(enabled) + "\n")604 if enabled:605 print("[integrate] à synchroniser: python run.py sync $(cat data/wave4_new_enabled.txt)")606607608if __name__ == "__main__":609 ap = argparse.ArgumentParser()610 ap.add_argument("cmd", choices=["harvest", "integrate"])611 ap.add_argument("only", nargs="*", help="sources précises (harvest)")612 args = ap.parse_args()613 harvest(args.only or None) if args.cmd == "harvest" else integrate()614