Agrégateur de produits québécois — www.fabri-ka.com
Python 38.4%
HTML 30.3%
TypeScript 17.2%
CSS 11%
JavaScript 3.2%
1#!/usr/bin/env python32"""Vague 5 (expansion sectorielle) : nouvelles sources de fabricants.34Sources moissonnées (fiche d'évaluation datée du 2026-08-21 dans5docs/CONFORMITE.md ; découvertes via Serper/Tavily puis sondées AVANT6connexion — robots.txt lus, répertoires publics de promotion des membres) :78 modemtl Répertoire de la mode locale montréalaise9 (modemtl.com/repertoire-mode-locale-mtl/). Page10 WordPress statique, ~52 liens directs vers les sites11 des marques. robots.txt permissif (WP standard).12 boeufdici Répertoire Bœuf d'ici — producteurs de bœuf vendant13 à la ferme (boeufdici.com/repertoire/). Page WP14 statique, ~43 liens directs. robots.txt permissif.15 afmq Association des fabricants de meubles du Québec16 (afmq.com/liste_compagnies). Liste -> fiches17 /liste_compagnies/profil/<id>-<slug> (~100 profils),18 site web extrait de chaque fiche (patron CTAQ de la19 vague 4). robots.txt : /catalog /scripts /images20 interdits — jamais requêtés.21 apiculteursduquebec Les Apiculteurs et Apicultrices du Québec — « route22 du miel » (apiculteursduquebec.com). Répertoire23 Drupal paginé par région (/membres/region/<15..31>) :24 chaque carte membre expose nom, adresse (code postal)25 et site web (lien du logo). robots.txt Drupal standard.26 vendors_epipresto Champ `vendor` de la place de marché epipresto.ca27 (connectée en vague 4, 273 vendeurs distincts) —28 patron vendors_collectifs de la vague 4 : résolution29 prudente du site officiel par candidats de domaine,30 validés par correspondance du titre. ⚠️ vendeurs31 hétérogènes (épiceries revendeuses ET producteurs) ->32 prior E 0.4 « nature à vérifier », preuve QC exigée.3334Écartées (raison consignée dans docs/CONFORMITE.md, 2026-08-21) :35ebenistes-quebec.com (coquille lead-gen, 0 fiche extractible),36microentreprendre.ca (répertoire JS sans liens vers les sites ; microcrédit37n'est pas une preuve de fabrication), index-design.ca (répertoire non38QC-only — fournisseurs internationaux), chantier.qc.ca (répertoire39admin-ajax ; entreprises d'économie sociale majoritairement non40fabricantes), apdiq.com / designmontreal.com (designers de services).4142Usage :43 python3 scripts/wave5_discovery.py harvest [source ...]44 python3 scripts/wave5_discovery.py integrate # additif, sérialisé45"""46import argparse47import concurrent.futures as cf48import html as htmllib49import json50import os51import re52import subprocess53import sys54import time55import unicodedata56from collections import Counter57from urllib.parse import urlparse5859import requests6061ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))62RAW = os.path.join(ROOT, "data", "raw")63ENR = os.path.join(ROOT, "data", "enriched")64sys.path.insert(0, ROOT)65sys.path.insert(0, os.path.join(ROOT, "scripts"))6667for line in open(os.path.join(ROOT, ".env")).read().splitlines():68 if "=" in line and not line.startswith("#"):69 k, _, v = line.partition("=")70 os.environ.setdefault(k.strip(), v.strip())7172from verify import HDRS, POSTAL_RE, AREA_RE, verify_domain # noqa: E40273from aggregate import BLOCK, norm_domain # noqa: E4027475TAG_RE = re.compile(r"<[^>]+>")76SESS = requests.Session()7778# domaines utilitaires vus sur les pages répertoires de la vague 5 (agences79# web des annuaires, CDN, boilerplate WordPress) — pas couverts par BLOCK80EXTRA_BLOCK = re.compile(81 r"(?i)(browsehappy\.com|cookiedatabase\.org|gmpg\.org|jarold\.ca|"82 r"viglob\.ca|igminformatique|themeforest|jsdelivr|cloudflare|"83 r"googleapis|gstatic|w3\.org|schema\.org|wp\.org|elementor|"84 # ritkey = agence des fiches AFMQ ; mtl.org = site touristique ;85 # achetezalaferme = place de marché (pas un producteur)86 r"ritkey\.com|mtl\.org|achetezalaferme)")8788SOCIALS = ("facebook", "instagram", "linkedin", "youtube", "tiktok",89 "pinterest", "twitter", "x.com")909192def strip_tags(s):93 return re.sub(r"\s+", " ", TAG_RE.sub(" ", htmllib.unescape(s or ""))).strip()949596def get(url, timeout=25):97 try:98 r = SESS.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)99 if r.status_code == 200:100 return r.text101 except Exception:102 pass103 return ""104105106def write_jsonl(name, records):107 os.makedirs(RAW, exist_ok=True)108 path = os.path.join(RAW, name + ".jsonl")109 with open(path, "w") as f:110 for r in records:111 f.write(json.dumps(r, ensure_ascii=False) + "\n")112 print(f"[harvest] {name}: {len(records)} entrées -> {path}", flush=True)113114115def _bad(url, dom):116 return (not dom or BLOCK.search(dom) or BLOCK.search(url)117 or EXTRA_BLOCK.search(url) or any(s in dom for s in SOCIALS))118119120# ------------------------------------------------ répertoires à liens directs121def harvest_single_page(source, page_url, own_domain, region_hint, evidence):122 html = get(page_url, timeout=40)123 records, seen = [], set()124 for url, label in re.findall(r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>',125 html, re.S | re.I):126 url = htmllib.unescape(url)127 host = (urlparse(url).netloc or "").lower()128 if not host or own_domain in host:129 continue130 dom = norm_domain(url)131 if _bad(url, dom) or dom in seen:132 continue133 seen.add(dom)134 label = strip_tags(label)135 generic = label.lower() in {"web", "site", "site web", "voir le site",136 "en savoir plus", "visiter", "boutique"}137 records.append({138 "name": label if 2 < len(label) < 80 and not generic else "",139 "domain": dom, "url": url, "region_hint": region_hint,140 "category_hint": "", "evidence": evidence, "query": page_url,141 })142 write_jsonl(source, records)143144145# ----------------------------------------------------------------------- AFMQ146AFMQ_BASE = "https://www.afmq.com"147AFMQ_PROFIL_RE = re.compile(r'href="(/liste_compagnies/profil/[^"#?]+)"')148149150def harvest_afmq():151 """Liste des membres -> fiches profil -> site web (patron CTAQ vague 4)."""152 html = get(AFMQ_BASE + "/liste_compagnies", timeout=40)153 paths = sorted(set(AFMQ_PROFIL_RE.findall(html)))154 print(f"[afmq] {len(paths)} fiches profil à visiter", flush=True)155 records = []156 for i, path in enumerate(paths, 1):157 page = get(AFMQ_BASE + path, timeout=30)158 time.sleep(0.4)159 if i % 25 == 0:160 print(f" [afmq] {i}/{len(paths)}", flush=True)161 if not page:162 continue163 name = ""164 m = re.search(r"<h1[^>]*>(.*?)</h1>", page, re.S | re.I)165 if m:166 name = strip_tags(m.group(1))167 if not name:168 name = path.rsplit("-", 1)[-1].replace("_", " ").title()169 webs = []170 for u in re.findall(r'href="(https?://[^"]+)"', page):171 dom = norm_domain(u)172 if _bad(u, dom) or "afmq" in (dom or ""):173 continue174 if u not in webs:175 webs.append(u)176 if not webs:177 continue178 text = strip_tags(page[:150000])179 postal = POSTAL_RE.search(text)180 phone = AREA_RE.search(text)181 cat = re.search(r"Catégorie\(s\) de produits:\s*(.{0,120})", text)182 records.append({183 "url": AFMQ_BASE + path,184 "title": name,185 "h1": name,186 "websites": webs[:2],187 "socials": [],188 "postal_prefix": postal.group(0)[:3] if postal else None,189 "phone": phone.group(0) if phone else None,190 "regions_mentioned": [],191 "text_sample": ("Fabricant de meubles membre AFMQ — "192 + (cat.group(1) if cat else ""))[:200],193 "category_hint": "maison",194 })195 write_jsonl("afmq", records)196197198# ------------------------------------------------------- Apiculteurs du Québec199API_BASE = "https://www.apiculteursduquebec.com"200# ids Drupal des vues régionales (constatés sur le sélecteur de la page201# « entreprises apicoles d'ici ») -> nom canonique du registre202API_REGIONS = {203 15: "Abitibi-Témiscamingue", 16: "Bas-Saint-Laurent",204 17: "Capitale-Nationale", 18: "Centre-du-Québec",205 19: "Chaudière-Appalaches", 20: "Côte-Nord", 21: "Estrie",206 22: "Gaspésie–Îles-de-la-Madeleine", 23: "Lanaudière",207 24: "Laurentides", 25: "Laval", 26: "Mauricie", 27: "Montérégie",208 28: "Montréal", 29: "Nord-du-Québec", 30: "Outaouais",209 31: "Saguenay–Lac-Saint-Jean",210}211API_ROW_RE = re.compile(r'<div class="views-row">(.*?)(?=<div class="views-row">|</section>)', re.S)212API_NAME_RE = re.compile(r'field-adresse-organization-1[^>]*>.*?<a href="[^"]*">([^<]+)</a>', re.S)213API_ADDR_RE = re.compile(r'views-field-field-adresse"[^>]*><div class="field-content">(.*?)</div>', re.S)214215216def harvest_apiculteurs():217 records, seen = [], set()218 for rid, region in API_REGIONS.items():219 for page in range(0, 12):220 url = f"{API_BASE}/membres/region/{rid}" + (f"?page={page}" if page else "")221 html = get(url, timeout=30)222 time.sleep(0.5)223 rows = API_ROW_RE.findall(html) if html else []224 if not rows:225 break226 found = 0227 for row in rows:228 webs = []229 for u in re.findall(r'href="(https?://[^"]+)"', row):230 dom = norm_domain(u)231 if _bad(u, dom) or "apiculteursduquebec" in (dom or ""):232 continue233 if u not in webs:234 webs.append(u)235 if not webs:236 continue # membre sans site web -> inconnectable237 nm = API_NAME_RE.search(row)238 name = strip_tags(nm.group(1)) if nm else ""239 key = norm_domain(webs[0])240 if key in seen:241 continue242 seen.add(key)243 found += 1244 addr = API_ADDR_RE.search(row)245 text = strip_tags(addr.group(1)) if addr else ""246 postal = POSTAL_RE.search(text)247 records.append({248 "url": url,249 "title": name,250 "h1": name,251 "websites": webs[:2],252 "socials": [],253 "postal_prefix": postal.group(0)[:3].replace(" ", "") if postal else None,254 "phone": None,255 "regions_mentioned": [region],256 "text_sample": text[:200],257 "category_hint": "erable",258 })259 if not found and page > 0:260 break261 print(f" [apiculteurs] {region}: cumul {len(records)}", flush=True)262 write_jsonl("apiculteursduquebec", records)263264265# --------------------------------------------------- vendors d'EPIPRESTO266COLLECTIFS = ["epipresto.ca"]267STOP_TOKENS = {"inc", "enr", "ltee", "les", "the", "and", "et", "de", "du",268 "la", "le", "des", "by", "par", "studio", "atelier", "co"}269# vendeurs génériques de la place de marché (pas des marques)270VENDOR_SKIP = re.compile(r"(?i)(général|autre choix|mini march|pharmacie|"271 r"dépanneur|depanneur|épicerie|epicerie|march[ée] )")272273274def _slug(name):275 s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode()276 return re.sub(r"[^a-z0-9]", "", s.lower())277278279def _norm_name(name):280 s = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode().lower()281 return " ".join(t for t in re.findall(r"[a-z0-9]+", s) if t not in STOP_TOKENS)282283284def _curl(url, timeout=20):285 """GET via curl (mécanisme du projet pour éviter le 429 TLS de requests)."""286 try:287 r = subprocess.run(["curl", "-sL", "-A", HDRS["User-Agent"], "--max-time",288 str(timeout), "--max-filesize", "400000", url],289 capture_output=True, timeout=timeout + 10)290 return r.stdout.decode("utf-8", "replace")291 except Exception:292 return ""293294295def harvest_vendors_epipresto(cap=400):296 from fabrika import db as fdb297 con = fdb.connect()298 rows = con.execute(299 "SELECT vendor, COUNT(*) AS n, GROUP_CONCAT(DISTINCT store_id) AS sids "300 "FROM products WHERE active=1 AND vendor<>'' AND store_id IN (%s) "301 "GROUP BY vendor ORDER BY n DESC" % ",".join("?" * len(COLLECTIFS)),302 COLLECTIFS).fetchall()303 known_names = {_norm_name(r[0]) for r in304 con.execute("SELECT name FROM stores") if r[0]}305 known_doms = {r[0] for r in con.execute("SELECT id FROM stores")}306 con.close()307 cand_path = os.path.join(ENR, "candidates.jsonl")308 if os.path.exists(cand_path):309 with open(cand_path) as f:310 known_doms |= {json.loads(l)["domain"] for l in f}311312 records, probes = [], 0313 for vendor, n, sids in rows:314 v = vendor.strip()315 if len(v) < 3 or len(v) > 60 or _norm_name(v) in known_names:316 continue317 if VENDOR_SKIP.search(v):318 continue319 slug = _slug(v)320 if not (4 <= len(slug) <= 30):321 continue322 if probes >= cap:323 break324 found = None325 for dom in (slug + ".com", slug + ".ca"):326 if dom in known_doms or BLOCK.search(dom):327 continue328 probes += 1329 html = _curl(f"https://{dom}")330 time.sleep(0.3)331 if not html:332 continue333 title_m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)334 title = _norm_name(strip_tags(title_m.group(1))[:120]) if title_m else ""335 vt = set(_norm_name(v).split())336 if vt and title and len(vt & set(title.split())) / len(vt) >= 0.6:337 found = dom338 break339 if found:340 records.append({341 "name": v,342 "domain": found,343 "url": f"https://{found}",344 "region_hint": "",345 "category_hint": "",346 "evidence": f"Vendeur de la place de marché EPIPRESTO ({n} produits) "347 f"— site officiel résolu et validé par titre",348 "query": f"vendor:{v}",349 })350 print(f"[vendors_epipresto] {probes} sondes, {len(records)} sites résolus", flush=True)351 write_jsonl("vendors_epipresto", records)352353354HARVESTERS = {355 "modemtl": lambda: harvest_single_page(356 "modemtl", "https://modemtl.com/repertoire-mode-locale-mtl/",357 "modemtl.com", "Montréal",358 "Marque de mode locale répertoriée par ModeMTL (répertoire de la "359 "mode montréalaise)"),360 "boeufdici": lambda: harvest_single_page(361 "boeufdici", "https://boeufdici.com/repertoire/",362 "boeufdici.com", "",363 "Producteur membre du répertoire Bœuf d'ici (vente de bœuf à la ferme)"),364 "afmq": harvest_afmq,365 "apiculteursduquebec": harvest_apiculteurs,366 "vendors_epipresto": harvest_vendors_epipresto,367}368369WAVE5_SOURCES = list(HARVESTERS)370371372def harvest(only=None):373 os.makedirs(RAW, exist_ok=True)374 for name, fn in HARVESTERS.items():375 if only and name not in only:376 continue377 print(f"[harvest] === {name} ===", flush=True)378 fn()379380381def integrate():382 """Intégration additive au registre + DB (mêmes garde-fous que la vague 4)."""383 from datetime import date384 import build_registry as br385 from fabrika import db as fdb386387 for s in WAVE5_SOURCES:388 assert s in br.SOURCE_PRIORS, f"prior manquant dans build_registry: {s}"389390 subprocess.run([sys.executable, os.path.join(ROOT, "scripts", "aggregate.py")],391 check=True)392393 cands = {}394 with open(os.path.join(ENR, "candidates.jsonl")) as f:395 for line in f:396 c = json.loads(line)397 cands[c["domain"]] = c398399 verified_path = os.path.join(ENR, "verified.jsonl")400 verified = {}401 with open(verified_path) as f:402 for line in f:403 v = json.loads(line)404 verified[v["domain"]] = v405406 reg_path = os.path.join(ROOT, "data", "stores.json")407 reg = json.load(open(reg_path))408 existing_ids = {s["id"] for s in reg["stores"]}409410 new_domains = sorted(d for d in cands411 if d not in verified and d not in existing_ids)412 print(f"[integrate] {len(new_domains)} nouveaux domaines à vérifier", flush=True)413414 new_recs = []415 with cf.ThreadPoolExecutor(12) as ex:416 for i, rec in enumerate(ex.map(verify_domain, new_domains)):417 new_recs.append(rec)418 if (i + 1) % 100 == 0:419 print(f" verify {i+1}/{len(new_domains)}", flush=True)420 with open(verified_path, "a") as f:421 for r in new_recs:422 f.write(json.dumps(r, ensure_ascii=False) + "\n")423 verified[r["domain"]] = r424425 added, skipped_dup, skipped_qc, skipped_dead = [], 0, 0, 0426 for dom in new_domains:427 cand, ver = cands[dom], verified.get(dom, {})428 if not ver.get("active"):429 skipped_dead += 1430 continue431 final_dom = ver.get("final_domain") or dom432 if final_dom in existing_ids:433 skipped_dup += 1434 continue435 qc_signal = any(ver.get(k) for k in ("qc_postal", "qc_phone", "tld_quebec",436 "mentions_quebec", "made_in_qc_wording"))437 qc_source = any(s in br.QC_ONLY_SOURCES for s in cand.get("sources", []))438 if not qc_signal and not qc_source:439 skipped_qc += 1440 continue441 cls, conf, ev = br.classify(cand, ver)442 default_cat = next((br.SOURCE_PRIORS[s][3] for s in br.PRIORITY443 if s in cand.get("sources", []) and br.SOURCE_PRIORS[s][3]), None)444 platform = ver.get("platform") or ""445 catalog_endpoint = ver.get("catalog_endpoint") or ""446 if platform == "wix" and not catalog_endpoint:447 catalog_endpoint = "/_api/wix-ecommerce-storefront-web/api"448 fu = urlparse(ver.get("final_url") or f"https://{final_dom}")449 store = {450 "id": final_dom,451 "name": br.clean_name(cand, ver),452 "url": f"{fu.scheme}://{fu.netloc}",453 "platform": platform,454 "catalog_endpoint": catalog_endpoint,455 "city": "",456 "region": br.pick_region(cand) or br.region_from_postal(cand, ver),457 "postal_prefix": cand.get("postal_prefix") or (ver.get("qc_postal") or "")[:3] or None,458 "phone": cand.get("phone") or ver.get("qc_phone"),459 "origin_class": cls,460 "origin_confidence": conf,461 "origin_evidence": ev,462 "categories": [default_cat] if default_cat else [],463 "socials": (cand.get("socials") or [])[:4] or ver.get("socials", []),464 "discovery_sources": cand.get("sources", []),465 "discovery_source_urls": cand.get("source_pages", [])[:5],466 "language": ver.get("language"),467 "ecommerce": bool(ver.get("has_cart") or catalog_endpoint),468 "verification_date": ver.get("checked_at") or str(date.today()),469 "status": "verified" if (conf >= 0.6 and ver.get("mentions_quebec")) else "probable",470 "enabled": bool(catalog_endpoint),471 }472 existing_ids.add(final_dom)473 added.append(store)474475 reg["stores"].extend(added)476 reg["count"] = len(reg["stores"])477 reg["generated"] = str(date.today())478 json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)479480 con = fdb.connect()481 for s in added:482 fdb.upsert_store(con, s)483 con.commit(); con.close()484485 enabled = [s["id"] for s in added if s["enabled"]]486 per_plat = Counter(s["platform"] or "(aucune)" for s in added)487 per_src = Counter(src for s in added for src in s["discovery_sources"])488 per_cls = Counter(s["origin_class"] for s in added)489 print(f"[integrate] boutiques ajoutées: {len(added)} | connectables (enabled): {len(enabled)}")490 print(f"[integrate] écartées — mortes/injoignables: {skipped_dead}, "491 f"dédup domaine final: {skipped_dup}, sans preuve QC: {skipped_qc}")492 print("[integrate] par plateforme:", dict(per_plat.most_common()))493 print("[integrate] par source:", dict(per_src.most_common()))494 print("[integrate] par classe:", dict(per_cls))495 with open(os.path.join(ROOT, "data", "wave5_new_enabled.txt"), "w") as f:496 f.write("\n".join(enabled) + "\n")497 if enabled:498 print("[integrate] à synchroniser: python run.py sync $(cat data/wave5_new_enabled.txt)")499500501if __name__ == "__main__":502 ap = argparse.ArgumentParser()503 ap.add_argument("cmd", choices=["harvest", "integrate"])504 ap.add_argument("only", nargs="*", help="sources précises (harvest)")505 args = ap.parse_args()506 harvest(args.only or None) if args.cmd == "harvest" else integrate()507