#!/usr/bin/env python3
# =============================================================================
# Job·Ka — Groupe KA
# Auteur  : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# Fichier : scripts/serper_scout.py
# Rôle    : Éclaireur Serper — requêtes Google `site:<domaine ATS>` × termes
#           québécois, extraction des slugs candidats par motifs d'URL,
#           dédoublonnage contre sources.json/connecteurs. Sortie JSONL prête
#           pour scripts/probe_ats.py.
# Créé    : 2026-08-22   Modifié : 2026-08-25
# =============================================================================
"""Usage :
    python3 scripts/serper_scout.py --out candidats.jsonl \
        [--pages 2] [--ats lever,greenhouse,...] [--workers 6]

Nécessite SERPER_API_KEY dans l'environnement. 1 requête Serper = 1 crédit ;
~(nb ATS × nb termes × pages) requêtes au total, throttlées.
"""
from __future__ import annotations

import argparse
import json
import os
import re
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

import requests

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

SERPER_URL = "https://google.serper.dev/search"
KEY = os.environ.get("SERPER_API_KEY", "")

# Termes de recherche : villes/régions québécoises + génériques emploi QC.
TERMS = [
    '"Montréal"', '"Québec"', '"Laval, QC"', '"Gatineau"', '"Sherbrooke"',
    '"Longueuil"', '"Trois-Rivières"', '"Saguenay"', '"Lévis"', '"Brossard"',
    '"Drummondville"', '"Granby"', '"Terrebonne"', '"Boucherville"',
    '"Saint-Hyacinthe"', '"Rimouski"', '"Victoriaville"', '"Rouyn-Noranda"',
    '"Sept-Îles"', '"Vaudreuil"', '"Joliette"', '"Saint-Jérôme"',
    '"Baie-Comeau"', '"Val-d\'Or"', '"Alma"', '"Shawinigan"',
    '"Salaberry-de-Valleyfield"', '"Sainte-Julie"', '"Beloeil"', '"Mirabel"',
    '"Blainville"', '"Repentigny"', '"Chicoutimi"', '"Kirkland"',
    '"Pointe-Claire"', '"Dorval"', '"Anjou"', '"Lachine"', '"Longue-Pointe"',
    '"Québec, QC"', 'emploi Québec', 'carrières Québec',
]

# domaine ATS -> (nom de site pour `site:`, regex d'extraction, ats, groupe(s))
PATTERNS: dict[str, dict] = {
    "lever": {
        "site": "jobs.lever.co",
        "rx": re.compile(r"jobs\.lever\.co/([A-Za-z0-9._-]+)"),
    },
    "greenhouse": {
        "site": "boards.greenhouse.io OR site:job-boards.greenhouse.io",
        "rx": re.compile(r"(?:job-)?boards(?:\.eu)?\.greenhouse\.io/"
                         r"(?:embed/job_board\?for=)?([A-Za-z0-9._-]+)"),
    },
    "smartrecruiters": {
        "site": "jobs.smartrecruiters.com",
        "rx": re.compile(r"jobs\.smartrecruiters\.com/(?:oneclick-ui/company/)?"
                         r"([A-Za-z0-9._-]+)"),
    },
    "workable": {
        "site": "apply.workable.com",
        "rx": re.compile(r"apply\.workable\.com/(?:api/v\d/accounts/)?"
                         r"([A-Za-z0-9._-]+)"),
    },
    "ashby": {
        "site": "jobs.ashbyhq.com",
        "rx": re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9._%-]+)"),
    },
    "recruitee": {
        "site": "recruitee.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com"),
    },
    "breezy": {
        "site": "breezy.hr",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr"),
    },
    "bamboohr": {
        "site": "bamboohr.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/(?:careers|jobs)"),
    },
    "workday": {
        "site": "myworkdayjobs.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/"
                         r"(?:([a-zA-Z]{2}-[a-zA-Z]{2})/)?([A-Za-z0-9_-]+)"),
    },
    "dayforce": {
        "site": "jobs.dayforcehcm.com",
        "rx": re.compile(r"jobs\.dayforcehcm\.com/(?:[a-z]{2}-[A-Za-z]{2}/)?"
                         r"([A-Za-z0-9]+)/([A-Za-z0-9_-]+)"),
    },
    # --- ajouts vague E (2026-08-25) -----------------------------------------
    "jazzhr": {
        "site": "applytojob.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.applytojob\.com"),
    },
    "teamtailor": {
        "site": "teamtailor.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.teamtailor\.com"),
    },
    "zohorecruit": {
        "site": "zohorecruit.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.zohorecruit\.com"
                         r"/jobs/([^/?#\s\"]+)"),
    },
    "talentnest": {
        "site": "talentnest.com",
        "rx": re.compile(r"https?://([a-z0-9-]+)\.talentnest\.com"),
    },
    "njoyn": {
        "site": "njoyn.com",
        "rx": re.compile(r"https?://([a-z0-9.-]+)\.njoyn\.com/"
                         r"(cl\d+|corp)/xweb/xweb\.asp\?[^\"'\s]*"
                         r"clid=(\d+)", re.I),
    },
}

BAD_SLUGS = {
    "login", "signin", "api", "js", "css", "static", "assets", "app", "www",
    "help", "support", "blog", "about", "privacy", "terms", "search", "jobs",
    "job", "embed", "widget", "oneclick-ui", "company", "careers", "fr", "en",
    "mydayforce", "candidateportal",
}

_lock = threading.Lock()
_last = [0.0]
_nreq = [0]


def serper(q: str, page: int) -> list[dict]:
    with _lock:
        wait = 0.15 - (time.time() - _last[0])
        if wait > 0:
            time.sleep(wait)
        _last[0] = time.time()
        _nreq[0] += 1
    r = requests.post(
        SERPER_URL, timeout=20,
        headers={"X-API-KEY": KEY, "Content-Type": "application/json"},
        json={"q": q, "gl": "ca", "hl": "fr", "num": 20, "page": page})
    r.raise_for_status()
    return r.json().get("organic") or []


def known_slugs() -> set[str]:
    known: set[str] = set()
    src = ROOT / "data" / "sources.json"
    if src.exists():
        for s in json.loads(src.read_text())["sources"]:
            known.add(s["id"].lower())
    pat = re.compile(
        r"^\s*(?:ORG|BOARD|COMPANY|TENANT|NS|CLID)\s*=\s*['\"]([^'\"]+)",
        re.M)
    for f in (ROOT / "jobka" / "connectors").glob("*.py"):
        for m in pat.finditer(f.read_text(encoding="utf-8")):
            known.add(m.group(1).lower())
    return known


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", required=True)
    ap.add_argument("--pages", type=int, default=2)
    ap.add_argument("--ats", default=",".join(PATTERNS))
    ap.add_argument("--workers", type=int, default=6)
    ap.add_argument("--terms-file", default="",
                    help="fichier texte : un terme de recherche par ligne "
                         "(remplace la liste de villes par défaut)")
    args = ap.parse_args()
    if not KEY:
        sys.exit("SERPER_API_KEY manquant")
    global TERMS
    if args.terms_file:
        TERMS = [l.strip() for l in Path(args.terms_file)
                 .read_text(encoding="utf-8").splitlines() if l.strip()]

    wanted = [a.strip() for a in args.ats.split(",") if a.strip() in PATTERNS]
    known = known_slugs()
    found: dict[tuple, dict] = {}
    flock = threading.Lock()

    def one(ats: str, term: str, page: int) -> None:
        cfg = PATTERNS[ats]
        q = f"site:{cfg['site']} {term}"
        try:
            organic = serper(q, page)
        except Exception as exc:
            print(f"  ! {q} p{page}: {exc}", flush=True)
            return
        for item in organic:
            url = item.get("link") or ""
            m = cfg["rx"].search(url)
            if not m:
                continue
            title = (item.get("title") or "").split(" - ")[0].split(" | ")[0]
            if ats == "workday":
                tenant, host, _cult, site = m.groups()
                if tenant.lower() in known or tenant.lower() in BAD_SLUGS:
                    continue
                key = ("workday", tenant.lower())
                entry = {"slug": tenant.lower(), "ats": "workday",
                         "tenant": tenant.lower(), "host": host, "site": site,
                         "employer": title.strip(), "src_url": url}
            elif ats == "dayforce":
                ns, board = m.group(1), m.group(2)
                if ns.lower() in known or ns.lower() in BAD_SLUGS:
                    continue
                key = ("dayforce", ns.lower())
                entry = {"slug": ns, "ats": "dayforce", "board": board
                         if board.upper() != "CANDIDATEPORTAL"
                         and not board.startswith("jobs") else "",
                         "employer": title.strip(), "src_url": url}
            elif ats == "zohorecruit":
                slug, page = m.group(1), m.group(2)
                if slug.lower() in known or slug.lower() in BAD_SLUGS \
                        or page.lower() in ("rss",):
                    continue
                key = ("zohorecruit", slug.lower())
                entry = {"slug": slug, "ats": "zohorecruit", "page": page,
                         "employer": title.strip(), "src_url": url}
            elif ats == "njoyn":
                host, cl, clid = m.group(1), m.group(2), m.group(3)
                if clid in known:
                    continue
                key = ("njoyn", clid)
                entry = {"slug": clid, "ats": "njoyn", "host": host.lower(),
                         "cl": cl, "clid": clid,
                         "employer": title.strip(), "src_url": url}
            else:
                slug = m.group(1)
                if slug.lower() in known or slug.lower() in BAD_SLUGS:
                    continue
                key = (ats, slug.lower())
                entry = {"slug": slug, "ats": ats,
                         "employer": title.strip(), "src_url": url}
            with flock:
                found.setdefault(key, entry)

    jobs = [(a, t, p) for a in wanted for t in TERMS
            for p in range(1, args.pages + 1)]
    print(f"[scout] {len(jobs)} requêtes Serper ({len(wanted)} ATS × "
          f"{len(TERMS)} termes × {args.pages} pages)", flush=True)
    t0 = time.time()
    with ThreadPoolExecutor(max_workers=args.workers) as ex:
        futs = [ex.submit(one, a, t, p) for a, t, p in jobs]
        for i, fut in enumerate(as_completed(futs), 1):
            fut.result()
            if i % 100 == 0:
                print(f"[scout] {i}/{len(jobs)} requêtes, "
                      f"{len(found)} candidats, {time.time()-t0:.0f}s",
                      flush=True)

    out = sorted(found.values(), key=lambda e: (e["ats"], e["slug"].lower()))
    Path(args.out).write_text(
        "\n".join(json.dumps(e, ensure_ascii=False) for e in out) + "\n",
        encoding="utf-8")
    per = {}
    for e in out:
        per[e["ats"]] = per.get(e["ats"], 0) + 1
    print(f"[scout] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes, "
          f"{len(out)} candidats -> {args.out}")
    for a, n in sorted(per.items()):
        print(f"    {a:16s} {n}")


if __name__ == "__main__":
    main()