#!/usr/bin/env python3
"""Récolte du répertoire CMAQ (metiersdart.ca) via Firecrawl.

Le répertoire est une app JS à session : la liste des artisans (~1880) n'existe
dans le DOM qu'après soumission du formulaire, paginée 18/page en ordre
ALÉATOIRE par recherche. Stratégie : sessions Firecrawl répétées, chacune
soumet la recherche puis clique « > » N fois en accumulant {cid, name, metier,
org} ; dédup par cid entre sessions jusqu'à saturation.

Sortie : data/raw/cmaq_artisans.jsonl (+ format harvest pour aggregate.py)
"""
import json
import os
import sys
import time

import requests

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "data", "raw", "cmaq_artisans.jsonl")
KEY = os.environ.get("FIRECRAWL_API_KEY", "")

PAGES_PER_SESSION = int(os.environ.get("CMAQ_PAGES", "40"))
MAX_SESSIONS = int(os.environ.get("CMAQ_SESSIONS", "14"))
STALE_STOP = 2      # arrêt après N sessions sans nouveau cid

JS = ("(async()=>{const acc=[];const grab=()=>{document.querySelectorAll('.list_rep_arti')"
      ".forEach(c=>{const a=c.querySelector('a[href*=cid]');const n=c.querySelector('h4');"
      "const m=c.querySelector('.artisant_list_metier');const o=c.querySelector('.artisant_list_org');"
      "acc.push({cid:(a?a.href.match(/cid=(\\d+)/):null)?.[1]||null,"
      "name:n?n.textContent.trim():'',metier:m?m.textContent.trim():'',"
      "org:o?o.textContent.trim():''});});};grab();"
      f"for(let i=0;i<{PAGES_PER_SESSION - 1};i++)" + "{"
      "const next=[...document.querySelectorAll('#pagination a.link')]"
      ".find(x=>x.textContent.trim()==='>');if(!next)break;next.click();"
      "await new Promise(r=>setTimeout(r,2200));grab();}"
      "return JSON.stringify(acc);})()")

PAYLOAD = {
    "url": "https://www.metiersdart.ca/repertoire_artisan.php",
    "formats": ["html"],
    "timeout": 200000,
    "actions": [
        {"type": "wait", "milliseconds": 2500},
        {"type": "click", "selector": "#form_repertoire_membres_form input[type=submit]"},
        {"type": "wait", "milliseconds": 5000},
        {"type": "executeJavascript", "script": JS},
    ],
}


def load_seen():
    seen = {}
    if os.path.exists(OUT):
        with open(OUT) as f:
            for line in f:
                try:
                    r = json.loads(line)
                    seen[r["cid"]] = r
                except Exception:
                    pass
    return seen


def main():
    if not KEY:
        sys.exit("FIRECRAWL_API_KEY manquant")
    seen = load_seen()
    stale = 0
    for sess in range(1, MAX_SESSIONS + 1):
        try:
            r = requests.post("https://api.firecrawl.dev/v1/scrape",
                              headers={"Authorization": f"Bearer {KEY}",
                                       "Content-Type": "application/json"},
                              json=PAYLOAD, timeout=280)
            d = r.json()
            rets = ((d.get("data") or {}).get("actions") or {}).get("javascriptReturns") or []
            items = []
            for ret in rets:
                v = ret.get("value")
                if isinstance(v, str) and v.startswith("["):
                    items = json.loads(v)
        except Exception as exc:
            print(f"session {sess}: ERREUR {exc}", flush=True)
            time.sleep(10)
            continue
        new = 0
        with open(OUT, "a") as f:
            for it in items:
                cid = it.get("cid")
                if not cid or cid in seen:
                    continue
                seen[cid] = it
                f.write(json.dumps(it, ensure_ascii=False) + "\n")
                new += 1
        print(f"session {sess}: {len(items)} cartes, +{new} nouveaux, total {len(seen)}", flush=True)
        stale = stale + 1 if new == 0 else 0
        if stale >= STALE_STOP:
            print("saturation atteinte", flush=True)
            break
        time.sleep(3)
    print(f"TOTAL artisans CMAQ: {len(seen)}", flush=True)


if __name__ == "__main__":
    main()