spb/fabri-ka Public

1#!/usr/bin/env python32"""Récolte du répertoire CMAQ (metiersdart.ca) via Firecrawl.34Le répertoire est une app JS à session : la liste des artisans (~1880) n'existe5dans le DOM qu'après soumission du formulaire, paginée 18/page en ordre6ALÉATOIRE par recherche. Stratégie : sessions Firecrawl répétées, chacune7soumet la recherche puis clique « > » N fois en accumulant {cid, name, metier,8org} ; dédup par cid entre sessions jusqu'à saturation.910Sortie : data/raw/cmaq_artisans.jsonl (+ format harvest pour aggregate.py)11"""12import json13import os14import sys15import time1617import requests1819ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))20OUT = os.path.join(ROOT, "data", "raw", "cmaq_artisans.jsonl")21KEY = os.environ.get("FIRECRAWL_API_KEY", "")2223PAGES_PER_SESSION = int(os.environ.get("CMAQ_PAGES", "40"))24MAX_SESSIONS = int(os.environ.get("CMAQ_SESSIONS", "14"))25STALE_STOP = 2      # arrêt après N sessions sans nouveau cid2627JS = ("(async()=>{const acc=[];const grab=()=>{document.querySelectorAll('.list_rep_arti')"28      ".forEach(c=>{const a=c.querySelector('a[href*=cid]');const n=c.querySelector('h4');"29      "const m=c.querySelector('.artisant_list_metier');const o=c.querySelector('.artisant_list_org');"30      "acc.push({cid:(a?a.href.match(/cid=(\\d+)/):null)?.[1]||null,"31      "name:n?n.textContent.trim():'',metier:m?m.textContent.trim():'',"32      "org:o?o.textContent.trim():''});});};grab();"33      f"for(let i=0;i<{PAGES_PER_SESSION - 1};i++)" + "{"34      "const next=[...document.querySelectorAll('#pagination a.link')]"35      ".find(x=>x.textContent.trim()==='>');if(!next)break;next.click();"36      "await new Promise(r=>setTimeout(r,2200));grab();}"37      "return JSON.stringify(acc);})()")3839PAYLOAD = {40    "url": "https://www.metiersdart.ca/repertoire_artisan.php",41    "formats": ["html"],42    "timeout": 200000,43    "actions": [44        {"type": "wait", "milliseconds": 2500},45        {"type": "click", "selector": "#form_repertoire_membres_form input[type=submit]"},46        {"type": "wait", "milliseconds": 5000},47        {"type": "executeJavascript", "script": JS},48    ],49}505152def load_seen():53    seen = {}54    if os.path.exists(OUT):55        with open(OUT) as f:56            for line in f:57                try:58                    r = json.loads(line)59                    seen[r["cid"]] = r60                except Exception:61                    pass62    return seen636465def main():66    if not KEY:67        sys.exit("FIRECRAWL_API_KEY manquant")68    seen = load_seen()69    stale = 070    for sess in range(1, MAX_SESSIONS + 1):71        try:72            r = requests.post("https://api.firecrawl.dev/v1/scrape",73                              headers={"Authorization": f"Bearer {KEY}",74                                       "Content-Type": "application/json"},75                              json=PAYLOAD, timeout=280)76            d = r.json()77            rets = ((d.get("data") or {}).get("actions") or {}).get("javascriptReturns") or []78            items = []79            for ret in rets:80                v = ret.get("value")81                if isinstance(v, str) and v.startswith("["):82                    items = json.loads(v)83        except Exception as exc:84            print(f"session {sess}: ERREUR {exc}", flush=True)85            time.sleep(10)86            continue87        new = 088        with open(OUT, "a") as f:89            for it in items:90                cid = it.get("cid")91                if not cid or cid in seen:92                    continue93                seen[cid] = it94                f.write(json.dumps(it, ensure_ascii=False) + "\n")95                new += 196        print(f"session {sess}: {len(items)} cartes, +{new} nouveaux, total {len(seen)}", flush=True)97        stale = stale + 1 if new == 0 else 098        if stale >= STALE_STOP:99            print("saturation atteinte", flush=True)100            break101        time.sleep(3)102    print(f"TOTAL artisans CMAQ: {len(seen)}", flush=True)103104105if __name__ == "__main__":106    main()107