Agrégateur de produits québécois — www.fabri-ka.com
Python 38.4%
HTML 30.3%
TypeScript 17.2%
CSS 11%
JavaScript 3.2%
1#!/usr/bin/env python32"""Vague 3 — découverte de nouvelles boutiques via 4 annuaires québécois frais.34Annuaires moissonnés (aucune clé API) :5 lespagesvertes Les Pages Vertes — répertoire de l'économie verte QC6 (fiches /entreprise/<slug>/ via business-sitemap*.xml)7 mauriciemiam MIAM Mauricie — identifiant régional agroalimentaire8 (liens externes de /repertoire-des-membres/)9 gardemangerduquebec La Montérégie, le Garde-Manger du Québec — membres10 « Producteurs et transformateurs » (fiches /membres-complices/)11 gardemangerduquebec_det idem, mais détaillants/marchés/restaurants (classe C)12 terroiretsaveurs Terroir et Saveurs (AATGQ) — liste « producteurs et13 artisans pour acheter local » (liens externes directs)1415Écartés après reconnaissance : Etsy (HTTP 403 anti-bot), Ma Zone Québec16(coquille Drupal vide côté serveur), Goûtez Lanaudière (liste membres dans un17CRM Eudonet inaccessible), Signé Local / CMAQ / économusées / Créateurs de18saveurs (déjà couverts : search_sweep_vendors, search_sweep_cmaq,19artisansaloeuvre, createursdesaveurs).2021Usage :22 python3 scripts/wave3_discovery.py harvest # -> data/raw/<source>.jsonl23 python3 scripts/wave3_discovery.py integrate # aggregate + verify + ajout24 # additif au registre + DB2526`integrate` est strictement additif : il n'ajoute que des domaines absents du27registre, ne touche à aucune boutique existante, et applique les mêmes28garde-fous que build_registry (preuve de localisation QC, classes A–E avec29évidence, dédup par domaine final).30"""31import argparse32import concurrent.futures as cf33import html as htmllib34import json35import os36import re37import sys38import time39from collections import Counter40from urllib.parse import unquote, urlparse, parse_qs4142import requests4344ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))45RAW = os.path.join(ROOT, "data", "raw")46ENR = os.path.join(ROOT, "data", "enriched")47sys.path.insert(0, ROOT)48sys.path.insert(0, os.path.join(ROOT, "scripts"))4950for line in open(os.path.join(ROOT, ".env")).read().splitlines():51 if "=" in line and not line.startswith("#"):52 k, _, v = line.partition("=")53 os.environ.setdefault(k.strip(), v.strip())5455from verify import HDRS, POSTAL_RE, AREA_RE, verify_domain # noqa: E40256from aggregate import BLOCK, norm_domain # noqa: E4025758A_RE = re.compile(r'<a\s[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>', re.S | re.I)59H1_RE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S | re.I)60TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)61TAG_RE = re.compile(r"<[^>]+>")62SOCIAL_HOSTS = ("facebook.com", "instagram.com", "linkedin.com", "youtube.com",63 "tiktok.com", "pinterest.", "x.com", "twitter.com")6465SESS = requests.Session()666768def get(url, timeout=20):69 try:70 r = SESS.get(url, headers=HDRS, timeout=timeout, allow_redirects=True)71 if r.status_code == 200:72 return r.text73 except Exception:74 pass75 return ""767778def strip_tags(s):79 return re.sub(r"\s+", " ", TAG_RE.sub(" ", htmllib.unescape(s or ""))).strip()808182def unwrap_safelink(url):83 """Déballe les liens Outlook SafeLinks (vus sur MIAM)."""84 if "safelinks.protection.outlook.com" in url:85 q = parse_qs(urlparse(url).query)86 if q.get("url"):87 return unquote(q["url"][0])88 return url899091def ext_links(html, own_domain):92 out = []93 for url, label in A_RE.findall(html):94 url = unwrap_safelink(htmllib.unescape(url))95 host = (urlparse(url).netloc or "").lower()96 if not host or own_domain in host:97 continue98 if any(s in host for s in SOCIAL_HOSTS):99 continue100 out.append((url, strip_tags(label)))101 return out102103104def fiche_record(url, html, own_domain):105 text = strip_tags(html[:200000])106 h1 = H1_RE.search(html)107 title = TITLE_RE.search(html)108 postal = POSTAL_RE.search(text)109 phone = AREA_RE.search(text)110 return {111 "url": url,112 "title": strip_tags(title.group(1))[:120] if title else "",113 "h1": strip_tags(h1.group(1))[:120] if h1 else "",114 "websites": [u for u, _ in ext_links(html, own_domain)],115 "socials": [],116 "postal_prefix": postal.group(0)[:3] if postal else None,117 "phone": phone.group(0) if phone else None,118 "regions_mentioned": [],119 "text_sample": text[:600],120 }121122123def write_jsonl(name, records):124 path = os.path.join(RAW, name + ".jsonl")125 with open(path, "w") as f:126 for r in records:127 f.write(json.dumps(r, ensure_ascii=False) + "\n")128 print(f"[harvest] {name}: {len(records)} fiches/entrées -> {path}", flush=True)129130131def drop_frequent_domains(records, threshold=0.05):132 """Retire des fiches les domaines « partenaires » présents partout."""133 n = max(len(records), 1)134 freq = Counter()135 for r in records:136 for dom in {norm_domain(w) for w in r["websites"]} - {None}:137 freq[dom] += 1138 noisy = {d for d, c in freq.items() if c / n > threshold and c > 3}139 if noisy:140 print(f"[harvest] domaines partenaires ignorés (>5% des fiches): {sorted(noisy)}")141 for r in records:142 r["websites"] = [w for w in r["websites"] if norm_domain(w) not in noisy]143 return records144145146def crawl_fiches(urls, own_domain, workers=3, delay=0.4, label=""):147 records, done = [], 0148149 def one(u):150 time.sleep(delay)151 html = get(u)152 return fiche_record(u, html, own_domain) if html else None153154 with cf.ThreadPoolExecutor(workers) as ex:155 for rec in ex.map(one, urls):156 done += 1157 if rec and rec["websites"]:158 records.append(rec)159 if done % 100 == 0:160 print(f" [{label}] {done}/{len(urls)}", flush=True)161 return records162163164# ---------------------------------------------------------------- harvesters165166def harvest_lespagesvertes():167 urls = []168 for sm in ("business-sitemap.xml", "business-sitemap2.xml"):169 xml = get(f"https://lespagesvertes.ca/{sm}")170 urls += re.findall(r"<loc>(https://lespagesvertes\.ca/entreprise/[^<]+)</loc>", xml)171 urls = list(dict.fromkeys(urls))172 print(f"[harvest] lespagesvertes: {len(urls)} fiches à visiter", flush=True)173 records = crawl_fiches(urls, "lespagesvertes.ca", workers=5, delay=0.25, label="lpv")174 write_jsonl("lespagesvertes", drop_frequent_domains(records))175176177GMQ_CATS = {178 "producteur": "producteurs",179 "transformateur": "producteurs",180 "épicerie": "detaillants",181 "epicerie": "detaillants",182 "boutique gourmande": "detaillants",183 "marché public": "detaillants",184 "marche public": "detaillants",185 "restaurant": "detaillants",186 "traiteur": "detaillants",187 "aubergiste": "detaillants",188}189190191def harvest_gardemanger():192 page = get("https://gardemangerduquebec.ca/repertoire-des-membres/", timeout=40)193 fiches = sorted(set(re.findall(194 r'href="(https://gardemangerduquebec\.ca/membres-complices/[^"#]+)"', page)))195 print(f"[harvest] gardemangerduquebec: {len(fiches)} fiches à visiter", flush=True)196 prod, det = [], []197198 def classify_cat(html):199 # classe body Avada : portfolio_category-<slug> = catégorie réelle de la fiche200 if "portfolio_category-producteurs-et-transformateurs" in html:201 return "producteurs"202 return "detaillants"203204 done = 0205206 def one(u):207 time.sleep(0.4)208 html = get(u)209 if not html:210 return None211 return classify_cat(html), fiche_record(u, html, "gardemangerduquebec.ca")212213 with cf.ThreadPoolExecutor(3) as ex:214 for res in ex.map(one, fiches):215 done += 1216 if done % 50 == 0:217 print(f" [gmq] {done}/{len(fiches)}", flush=True)218 if not res:219 continue220 cat, rec = res221 rec["regions_mentioned"] = ["Montérégie"]222 if not rec["websites"]:223 continue224 (prod if cat == "producteurs" else det).append(rec)225 write_jsonl("gardemangerduquebec", drop_frequent_domains(prod))226 write_jsonl("gardemangerduquebec_det", drop_frequent_domains(det))227228229def harvest_single_page(source, page_url, own_domain, region_hint, evidence):230 page = get(page_url, timeout=40)231 records, seen = [], set()232 for url, label in ext_links(page, own_domain):233 dom = norm_domain(url)234 if not dom or dom in seen or BLOCK.search(dom) or BLOCK.search(url):235 continue236 seen.add(dom)237 name = label if 2 < len(label) < 80 else ""238 records.append({239 "name": name,240 "domain": dom,241 "url": url,242 "region_hint": region_hint,243 "category_hint": "",244 "evidence": evidence,245 "query": page_url,246 })247 write_jsonl(source, records)248249250def harvest():251 os.makedirs(RAW, exist_ok=True)252 harvest_single_page(253 "mauriciemiam", "https://mauriciemiam.ca/repertoire-des-membres/",254 "mauriciemiam.ca", "Mauricie",255 "Membre MIAM Mauricie (identifiant régional agroalimentaire)")256 harvest_single_page(257 "terroiretsaveurs",258 "https://terroiretsaveurs.com/repertoire-terroir-saveurs-liste-des-producteurs-et-artisans-pour-acheter-local/",259 "terroiretsaveurs.com", "",260 "Producteur/artisan du répertoire Terroir et Saveurs (AATGQ)")261 harvest_gardemanger()262 harvest_lespagesvertes()263264265# ---------------------------------------------------------------- integrate266267WAVE3_SOURCES = ["lespagesvertes", "mauriciemiam", "gardemangerduquebec",268 "gardemangerduquebec_det", "terroiretsaveurs"]269270271def integrate():272 import subprocess273 from datetime import date274 import build_registry as br275 from fabrika import db as fdb276277 for s in WAVE3_SOURCES:278 assert s in br.SOURCE_PRIORS, f"prior manquant dans build_registry: {s}"279280 # 1) ré-agrège tous les raw (inclut la vague 3) -> candidates.jsonl canonique281 subprocess.run([sys.executable, os.path.join(ROOT, "scripts", "aggregate.py")],282 check=True)283284 cands = {}285 with open(os.path.join(ENR, "candidates.jsonl")) as f:286 for line in f:287 c = json.loads(line)288 cands[c["domain"]] = c289290 verified_path = os.path.join(ENR, "verified.jsonl")291 verified = {}292 with open(verified_path) as f:293 for line in f:294 v = json.loads(line)295 verified[v["domain"]] = v296297 reg_path = os.path.join(ROOT, "data", "stores.json")298 reg = json.load(open(reg_path))299 existing_ids = {s["id"] for s in reg["stores"]}300301 # 2) domaines candidats jamais vérifiés (donc nouveaux pour le pipeline)302 new_domains = sorted(d for d in cands303 if d not in verified and d not in existing_ids)304 print(f"[integrate] {len(new_domains)} nouveaux domaines à vérifier", flush=True)305306 new_recs = []307 with cf.ThreadPoolExecutor(16) as ex:308 for i, rec in enumerate(ex.map(verify_domain, new_domains)):309 new_recs.append(rec)310 if (i + 1) % 100 == 0:311 print(f" verify {i+1}/{len(new_domains)}", flush=True)312 with open(verified_path, "a") as f:313 for r in new_recs:314 f.write(json.dumps(r, ensure_ascii=False) + "\n")315 verified[r["domain"]] = r316317 # 3) construit les entrées registre (mêmes règles que build_registry)318 added, skipped_dup, skipped_qc, skipped_dead = [], 0, 0, 0319 for dom in new_domains:320 cand, ver = cands[dom], verified.get(dom, {})321 if not ver.get("active"):322 skipped_dead += 1323 continue324 final_dom = ver.get("final_domain") or dom325 if final_dom in existing_ids:326 skipped_dup += 1327 continue328 qc_signal = any(ver.get(k) for k in ("qc_postal", "qc_phone", "tld_quebec",329 "mentions_quebec", "made_in_qc_wording"))330 qc_source = any(s in br.QC_ONLY_SOURCES for s in cand.get("sources", []))331 if not qc_signal and not qc_source:332 skipped_qc += 1333 continue334 cls, conf, ev = br.classify(cand, ver)335 default_cat = next((br.SOURCE_PRIORS[s][3] for s in br.PRIORITY336 if s in cand.get("sources", []) and br.SOURCE_PRIORS[s][3]), None)337 platform = ver.get("platform") or ""338 catalog_endpoint = ver.get("catalog_endpoint") or ""339 if platform == "wix" and not catalog_endpoint:340 catalog_endpoint = "/_api/wix-ecommerce-storefront-web/api"341 fu = urlparse(ver.get("final_url") or f"https://{final_dom}")342 store = {343 "id": final_dom,344 "name": br.clean_name(cand, ver),345 "url": f"{fu.scheme}://{fu.netloc}",346 "platform": platform,347 "catalog_endpoint": catalog_endpoint,348 "city": "",349 "region": br.pick_region(cand) or br.region_from_postal(cand, ver),350 "postal_prefix": cand.get("postal_prefix") or (ver.get("qc_postal") or "")[:3] or None,351 "phone": cand.get("phone") or ver.get("qc_phone"),352 "origin_class": cls,353 "origin_confidence": conf,354 "origin_evidence": ev,355 "categories": [default_cat] if default_cat else [],356 "socials": (cand.get("socials") or [])[:4] or ver.get("socials", []),357 "discovery_sources": cand.get("sources", []),358 "discovery_source_urls": cand.get("source_pages", [])[:5],359 "language": ver.get("language"),360 "ecommerce": bool(ver.get("has_cart") or catalog_endpoint),361 "verification_date": ver.get("checked_at") or str(date.today()),362 "status": "verified" if (conf >= 0.6 and ver.get("mentions_quebec")) else "probable",363 "enabled": bool(catalog_endpoint),364 }365 existing_ids.add(final_dom)366 added.append(store)367368 reg["stores"].extend(added)369 reg["count"] = len(reg["stores"])370 reg["generated"] = str(date.today())371 json.dump(reg, open(reg_path, "w"), ensure_ascii=False, indent=1)372373 con = fdb.connect()374 for s in added:375 fdb.upsert_store(con, s)376 con.commit(); con.close()377378 enabled = [s["id"] for s in added if s["enabled"]]379 per_plat = Counter(s["platform"] or "(aucune)" for s in added)380 per_src = Counter(src for s in added for src in s["discovery_sources"])381 per_cls = Counter(s["origin_class"] for s in added)382 print(f"[integrate] boutiques ajoutées: {len(added)} | connectables (enabled): {len(enabled)}")383 print(f"[integrate] écartées — mortes/injoignables: {skipped_dead}, "384 f"dédup domaine final: {skipped_dup}, sans preuve QC: {skipped_qc}")385 print("[integrate] par plateforme:", dict(per_plat.most_common()))386 print("[integrate] par source:", dict(per_src.most_common()))387 print("[integrate] par classe:", dict(per_cls))388 with open(os.path.join(ROOT, "data", "wave3_new_enabled.txt"), "w") as f:389 f.write("\n".join(enabled) + "\n")390 if enabled:391 print("[integrate] à synchroniser: python run.py sync $(cat data/wave3_new_enabled.txt)")392393394if __name__ == "__main__":395 ap = argparse.ArgumentParser()396 ap.add_argument("cmd", choices=["harvest", "integrate"])397 args = ap.parse_args()398 harvest() if args.cmd == "harvest" else integrate()399