#!/usr/bin/env python3
# =============================================================================
# Job·Ka — Groupe KA
# Auteur  : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# Fichier : scripts/detect_ats.py
# Rôle    : Détecteur d'ATS par page carrières — visite le site de chaque
#           employeur (accueil + chemins carrières usuels), reconnaît l'ATS
#           par motifs d'URL/DOM et extrait le slug. Les 9 ATS supportés
#           deviennent des candidats pour scripts/probe_ats.py ; les autres
#           (njoyn/taleo/ukg/icims/successfactors/adp) sont consignés dans
#           data/detected-other-ats.jsonl pour traitement séparé.
# Créé    : 2026-08-18   Modifié : 2026-08-18
# =============================================================================
"""Usage :
    python3 scripts/detect_ats.py employeurs.jsonl [autres.jsonl …] \
        --out candidats.jsonl --other data/detected-other-ats.jsonl \
        [--cap 1500] [--workers 8]

Entrée : JSONL {"name": "...", "url"/"domain"/"domain_hint": "...",
                "city": "...", "categorie": "..."} (ordre = priorité).
"""
from __future__ import annotations

import argparse
import json
import re
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from urllib.parse import urljoin, urlparse

import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

UA = "JobKaBot/1.0 (+https://www.job-ka.com; contact@spboucher.ai)"
TIMEOUT = 6
PATHS = ["", "/carrieres", "/careers", "/emplois", "/jobs", "/fr/carrieres"]
MAX_HTML = 500_000

# slugs jamais valides (pages génériques des ATS eux-mêmes)
BAD_SLUGS = {"www", "app", "api", "apply", "embed", "boards", "jobs", "job",
             "careers", "career", "fr", "en", "help", "support", "cdn",
             "assets", "status", "docs", "login", "my", "sso", "static",
             "oneclick-ui", "attachments", "hire", "home", "account",
             "js", "css", "images", "img", "media", "email", "click"}

# ATS supportés par Job·Ka : (ats, regex, groupes)
SUPPORTED = [
    ("workday", re.compile(
        r"https?://([\w-]+)\.(wd\d+)\.myworkdayjobs\.com"
        r"(?:/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_\-]+))?", re.I)),
    ("greenhouse", re.compile(
        r"(?:boards|job-boards)\.greenhouse\.io/([A-Za-z0-9_\-]+)", re.I)),
    ("greenhouse", re.compile(
        r"greenhouse\.io/embed/job_board\?[^\"'\s]*for=([A-Za-z0-9_\-]+)", re.I)),
    ("lever", re.compile(r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_\-]+)", re.I)),
    ("smartrecruiters", re.compile(
        r"(?:careers|jobs)\.smartrecruiters\.com/([A-Za-z0-9]+)")),
    ("ashby", re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9_\-.%]+)", re.I)),
    ("workable", re.compile(
        r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_\-]+)", re.I)),
    ("recruitee", re.compile(r"https?://([A-Za-z0-9\-]+)\.recruitee\.com", re.I)),
    ("breezy", re.compile(r"https?://([A-Za-z0-9\-]+)\.breezy\.hr", re.I)),
    ("bamboohr", re.compile(r"https?://([A-Za-z0-9\-]+)\.bamboohr\.com", re.I)),
]

# ATS repérés mais non supportés ici -> data/detected-other-ats.jsonl
OTHER = [
    ("njoyn", re.compile(r"https?://[\w.-]*njoyn\.com[^\"'\s<>]*", re.I)),
    ("taleo", re.compile(r"https?://[\w.-]*taleo\.net[^\"'\s<>]*", re.I)),
    ("ukg_ultipro", re.compile(
        r"https?://[\w.-]*(?:ultipro|ukg)\.com[^\"'\s<>]*", re.I)),
    ("icims", re.compile(r"https?://[\w-]+\.icims\.com[^\"'\s<>]*", re.I)),
    ("successfactors", re.compile(
        r"https?://[\w.-]*successfactors\.(?:com|eu)[^\"'\s<>]*", re.I)),
    ("adp", re.compile(
        r"https?://(?:workforcenow|recruiting|jobs)\.adp\.com[^\"'\s<>]*", re.I)),
]

CAREER_LINK = re.compile(
    r'href=["\']([^"\']*(?:carri|career|emploi|job|recrut|joignez|joindre|'
    r'postul|travaill)[^"\']*)["\']', re.I)

_print_lock = threading.Lock()


def norm_domain(rec: dict) -> str:
    raw = (rec.get("domain_hint") or rec.get("domain") or rec.get("url")
           or "").strip()
    if not raw:
        return ""
    if "://" not in raw:
        raw = "https://" + raw
    host = urlparse(raw).netloc.lower()
    return host[4:] if host.startswith("www.") else host


def fetch(session: requests.Session, url: str) -> tuple[str, str]:
    r = session.get(url, timeout=TIMEOUT, allow_redirects=True, verify=False,
                    headers={"User-Agent": UA})
    if r.status_code >= 400:
        return r.url, ""
    return r.url, (r.text or "")[:MAX_HTML]


def scan_text(text: str) -> tuple[list[tuple], list[tuple]]:
    """-> ([(ats, slug, extra…)], [(ats, url)])."""
    hits, others = [], []
    for ats, rx in SUPPORTED:
        for m in rx.finditer(text):
            if ats == "workday":
                tenant, host = m.group(1).lower(), m.group(2).lower()
                site = m.group(3) or ""
                if tenant in BAD_SLUGS or tenant == "impl":
                    continue
                hits.append((ats, tenant, host, site))
            else:
                slug = m.group(1)
                if slug.lower() in BAD_SLUGS:
                    continue
                hits.append((ats, slug))
    for ats, rx in OTHER:
        m = rx.search(text)
        if m:
            others.append((ats, m.group(0)[:300]))
    return hits, others


def detect_one(rec: dict) -> dict:
    """-> {"candidates": [...], "others": [...], "domain": d}."""
    domain = norm_domain(rec)
    out = {"domain": domain, "candidates": [], "others": []}
    if not domain:
        return out
    session = requests.Session()
    seen_hits: set[tuple] = set()
    seen_others: set[str] = set()
    extra_links: list[str] = []
    pages = 0
    for i, path in enumerate(PATHS + ["@extra1", "@extra2"]):
        if path.startswith("@"):
            if not extra_links:
                continue
            url = extra_links.pop(0)
        else:
            url = f"https://{domain}{path}"
        try:
            final_url, html = fetch(session, url)
        except requests.RequestException:
            if path == "":     # racine injoignable en https -> essai www/http
                for alt in (f"https://www.{domain}", f"http://{domain}"):
                    try:
                        final_url, html = fetch(session, alt)
                        break
                    except requests.RequestException:
                        final_url, html = "", ""
                if not html:
                    continue
            else:
                continue
        pages += 1
        hits, others = scan_text(final_url + "\n" + html)
        for h in hits:
            if h in seen_hits:
                continue
            seen_hits.add(h)
        for ats, u in others:
            if ats not in seen_others:
                seen_others.add(ats)
                out["others"].append({"name": rec.get("name", ""),
                                      "domain": domain, "ats": ats, "url": u})
        if path == "" and html and not seen_hits:
            # relever jusqu'à 2 liens « carrières » à suivre
            base = final_url or f"https://{domain}"
            for link in CAREER_LINK.findall(html):
                if link.startswith(("mailto:", "tel:", "#", "javascript")):
                    continue
                full = urljoin(base, link)
                if full not in extra_links:
                    extra_links.append(full)
                if len(extra_links) >= 2:
                    break
        if seen_hits:
            break              # ATS supporté trouvé : inutile d'aller plus loin
    for h in seen_hits:
        cand = {"employer": rec.get("name") or domain, "domain": domain,
                "url": f"https://{domain}",
                "sectors": [s for s in [rec.get("categorie", "")] if s],
                "cities": [c for c in [rec.get("city", "")] if c],
                "group": rec.get("name") or domain}
        if h[0] == "workday":
            cand.update(ats="workday", slug=h[1], tenant=h[1], host=h[2])
            if h[3] and not re.fullmatch(r"[a-z]{2}-[A-Z]{2}", h[3]):
                cand["site"] = h[3]
        else:
            cand.update(ats=h[0], slug=h[1])
        out["candidates"].append(cand)
    return out


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("inputs", nargs="+")
    ap.add_argument("--out", required=True)
    ap.add_argument("--other", default="data/detected-other-ats.jsonl")
    ap.add_argument("--cap", type=int, default=1500)
    ap.add_argument("--workers", type=int, default=8)
    args = ap.parse_args()

    recs, seen = [], set()
    for f in args.inputs:
        for line in Path(f).read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line:
                continue
            rec = json.loads(line)
            d = norm_domain(rec)
            if d and d not in seen:
                seen.add(d)
                recs.append(rec)
    recs = recs[:args.cap]
    print(f"[detect] {len(recs)} domaines à visiter (cap {args.cap})",
          flush=True)

    cands, others = [], []
    t0 = time.time()
    done = 0
    with ThreadPoolExecutor(max_workers=args.workers) as ex:
        futs = [ex.submit(detect_one, r) for r in recs]
        for fut in as_completed(futs):
            done += 1
            try:
                res = fut.result()
            except Exception:
                continue
            for c in res["candidates"]:
                cands.append(c)
                with _print_lock:
                    print(f"  ✓ {c['employer']} -> {c['ats']}:{c['slug']}",
                          flush=True)
            others.extend(res["others"])
            if done % 100 == 0:
                print(f"[detect] {done}/{len(recs)} domaines, "
                      f"{len(cands)} candidats, {len(others)} autres ATS, "
                      f"{time.time()-t0:.0f}s", flush=True)

    Path(args.out).write_text(
        "\n".join(json.dumps(c, ensure_ascii=False) for c in cands) + "\n",
        encoding="utf-8")
    op = Path(args.other)
    existing = set()
    if op.exists():
        existing = {l for l in op.read_text(encoding="utf-8").splitlines()
                    if l.strip()}
    with op.open("a", encoding="utf-8") as fh:
        for o in others:
            line = json.dumps(o, ensure_ascii=False)
            if line not in existing:
                fh.write(line + "\n")
                existing.add(line)

    per = {}
    for o in others:
        per[o["ats"]] = per.get(o["ats"], 0) + 1
    print(f"\n[detect] terminé en {time.time()-t0:.0f}s : "
          f"{len(cands)} candidats ATS supportés -> {args.out}")
    print(f"[detect] autres ATS ({sum(per.values())}) -> {args.other} : "
          + ", ".join(f"{k}={v}" for k, v in sorted(per.items())))


if __name__ == "__main__":
    main()