1#!/usr/bin/env python32# =============================================================================3# Job·Ka — Groupe KA4# Auteur : Simon-Pierre Boucher5# Contact : contact@spboucher.ai6# Fichier : scripts/probe_ats.py7# Rôle : Sondage massif de slugs candidats sur les 9 ATS supportés —8# un slug est CONFIRMÉ si l'API répond ≥1 offre ET ≥1 offre au9# Québec (réutilise is_quebec_location du projet). Émet des entrées10# prêtes pour data/feeds-*.json (scripts/gen_connectors.py).11# Créé : 2026-08-18 Modifié : 2026-08-1812# =============================================================================13"""Sonde des slugs candidats sur les 9 plateformes ATS de Job·Ka.1415Usage :16 python3 scripts/probe_ats.py candidats.jsonl --out confirmed.json \17 [--rejected rejected.jsonl] [--workers 8]1819Entrée (JSONL, ou JSON liste) — un candidat par ligne :20 {"slug": "acme", "employer": "Acme inc.", "ats": "lever"?,21 "sectors": ["TI"], "cities": ["Montréal"], "group": "Acme"?,22 "no_workday": true?, "tenant"/"host"/"site" (workday)?}2324- sans "ats" : les 9 plateformes sont sondées dans l'ordre DEFAULT_ORDER25 (Workday en dernier — découverte automatique de l'hôte wdN et du site26 par le redirect de la racine du tenant).27- les candidats partageant le même "group" (défaut : employer) sont des28 variantes du même employeur : on s'arrête à la première confirmation.29- les slugs/tenants déjà connectés (sources.json + jobka/connectors/*.py)30 sont sautés automatiquement.3132Politesse : timeout 6 s, throttle 0,3 s par domaine d'API, 8 threads.33"""34from __future__ import annotations3536import argparse37import json38import re39import sys40import threading41import time42from concurrent.futures import ThreadPoolExecutor, as_completed43from pathlib import Path44from urllib.parse import urlparse4546import requests4748ROOT = Path(__file__).resolve().parent.parent49sys.path.insert(0, str(ROOT))5051from jobka.normalize import is_quebec_location # noqa: E4025253UA = "JobKaBot/1.0 (+https://www.job-ka.com; contact@spboucher.ai)"54TIMEOUT = 655THROTTLE = 0.3 # s entre deux requêtes vers un même domaine d'API56WD_HOSTS = ["wd3", "wd1", "wd5", "wd10", "wd12", "wd103", "wd502", "wd504"]5758# Noms de sites carrières Workday fréquents — essayés quand le site est59# inconnu. L'API cxs répond 422 si le tenant/hôte est faux mais 404 si seul60# le site l'est : on découvre d'abord l'hôte, puis on devine le site.61WD_COMMON_SITES = [62 "External", "Careers", "External_Careers", "ExternalCareers", "careers",63 "External_Career_Site", "CareerSite", "Career", "Global", "external",64 "Global_Careers", "GlobalCareers", "jobs", "Jobs", "External_Site",65 "ExternalSite", "Externe", "Carrieres", "carrieres", "Recruiting",66 "Recrutement", "Emplois", "External_Portal", "EXTERNAL", "Ext", "EXT",67]6869DEFAULT_ORDER = ["lever", "greenhouse", "ashby", "smartrecruiters",70 "workable", "recruitee", "breezy", "bamboohr", "workday"]7172# --- throttling par domaine d'API -------------------------------------------73_locks: dict[str, threading.Lock] = {}74_last: dict[str, float] = {}75_glock = threading.Lock()76_nreq = [0]777879def _req(family: str, method: str, url: str, **kw) -> requests.Response:80 """Requête HTTP throttlée (verrou + espacement par famille/domaine)."""81 with _glock:82 lock = _locks.setdefault(family, threading.Lock())83 _nreq[0] += 184 headers = {"User-Agent": UA}85 headers.update(kw.pop("headers", None) or {})86 with lock:87 wait = THROTTLE - (time.time() - _last.get(family, 0.0))88 if wait > 0:89 time.sleep(wait)90 try:91 return requests.request(method, url, timeout=TIMEOUT,92 headers=headers, **kw)93 finally:94 _last[family] = time.time()959697def _qc_stats(labels: list[list[str]]) -> tuple[int, list[str]]:98 """[[libellés d'une offre], …] -> (nb d'offres QC, libellés QC exemples)."""99 qc, ex = 0, []100 for locs in labels:101 hit = next((l for l in locs if l and is_quebec_location(l)), None)102 if hit:103 qc += 1104 if len(ex) < 5 and hit not in ex:105 ex.append(hit)106 return qc, ex107108109# --- sondes par ATS ----------------------------------------------------------110def probe_lever(slug: str, cand: dict) -> dict | None:111 r = _req("api.lever.co", "GET",112 f"https://api.lever.co/v0/postings/{slug}?mode=json")113 if r.status_code != 200:114 return None115 data = r.json()116 if not isinstance(data, list) or not data:117 return None118 labels = []119 for p in data:120 locs = [(p.get("categories") or {}).get("location") or ""]121 locs += [l for l in (p.get("allLocations") or []) if isinstance(l, str)]122 labels.append(locs)123 qc, ex = _qc_stats(labels)124 return {"found": len(data), "qc": qc, "qc_ex": ex, "api_name": ""}125126127def probe_greenhouse(slug: str, cand: dict) -> dict | None:128 r = _req("boards-api.greenhouse.io", "GET",129 f"https://boards-api.greenhouse.io/v1/boards/{slug}/jobs")130 if r.status_code != 200:131 return None132 jobs = r.json().get("jobs") or []133 if not jobs:134 return None135 labels = [[(j.get("location") or {}).get("name") or ""] for j in jobs]136 qc, ex = _qc_stats(labels)137 name = ""138 if qc:139 try:140 r2 = _req("boards-api.greenhouse.io", "GET",141 f"https://boards-api.greenhouse.io/v1/boards/{slug}")142 if r2.status_code == 200:143 name = r2.json().get("name") or ""144 except requests.RequestException:145 pass146 return {"found": len(jobs), "qc": qc, "qc_ex": ex, "api_name": name}147148149def probe_ashby(slug: str, cand: dict) -> dict | None:150 r = _req("api.ashbyhq.com", "GET",151 f"https://api.ashbyhq.com/posting-api/job-board/{slug}")152 if r.status_code != 200:153 return None154 data = r.json()155 jobs = data.get("jobs") or []156 if not jobs:157 return None158 labels = []159 for j in jobs:160 locs = [j.get("location") or ""]161 locs += [(s.get("location") or "") for s in (j.get("secondaryLocations") or [])]162 labels.append(locs)163 qc, ex = _qc_stats(labels)164 return {"found": len(jobs), "qc": qc, "qc_ex": ex,165 "api_name": data.get("name") or ""}166167168def probe_smartrecruiters(slug: str, cand: dict) -> dict | None:169 variants = [slug]170 if slug != slug.capitalize():171 variants.append(slug.capitalize())172 for v in variants:173 r = _req("api.smartrecruiters.com", "GET",174 f"https://api.smartrecruiters.com/v1/companies/{v}/postings?limit=100")175 if r.status_code != 200:176 continue177 data = r.json()178 content = data.get("content") or []179 total = int(data.get("totalFound") or len(content))180 if not content:181 continue182 labels, name = [], ""183 for p in content:184 loc = p.get("location") or {}185 country = (loc.get("country") or "").lower()186 lbl = ", ".join(x for x in (loc.get("city"), loc.get("region"),187 loc.get("country")) if x)188 reg = (loc.get("region") or "").lower()189 ok = reg in ("qc", "quebec", "québec") or (190 country in ("ca", "canada") and is_quebec_location(lbl))191 labels.append([lbl] if ok else [""])192 name = name or (p.get("company") or {}).get("name") or ""193 qc, ex = _qc_stats(labels)194 return {"found": total, "qc": qc, "qc_ex": ex, "api_name": name,195 "org_override": v}196 return None197198199def probe_workable(slug: str, cand: dict) -> dict | None:200 r = _req("apply.workable.com", "GET",201 f"https://apply.workable.com/api/v1/widget/accounts/{slug}?details=false")202 if r.status_code != 200:203 return None204 data = r.json()205 jobs = data.get("jobs") or []206 if not jobs:207 return None208 labels = [[", ".join(x for x in (j.get("city"), j.get("state"),209 j.get("country")) if x)] for j in jobs]210 qc, ex = _qc_stats(labels)211 return {"found": len(jobs), "qc": qc, "qc_ex": ex,212 "api_name": data.get("name") or ""}213214215def probe_recruitee(slug: str, cand: dict) -> dict | None:216 r = _req(f"{slug}.recruitee.com", "GET",217 f"https://{slug}.recruitee.com/api/offers/")218 if r.status_code != 200:219 return None220 offers = r.json().get("offers") or []221 if not offers:222 return None223 labels = []224 for o in offers:225 locs = [o.get("location") or "",226 ", ".join(x for x in (o.get("city"), o.get("country")) if x)]227 labels.append(locs)228 qc, ex = _qc_stats(labels)229 return {"found": len(offers), "qc": qc, "qc_ex": ex, "api_name": ""}230231232def probe_breezy(slug: str, cand: dict) -> dict | None:233 r = _req(f"{slug}.breezy.hr", "GET", f"https://{slug}.breezy.hr/json")234 if r.status_code != 200:235 return None236 try:237 data = r.json()238 except ValueError:239 return None240 if not isinstance(data, list) or not data:241 return None242 labels, name = [], ""243 for p in data:244 loc = p.get("location") or {}245 labels.append([loc.get("name") or "",246 ", ".join(x for x in (loc.get("city"),247 (loc.get("state") or {}).get("name")248 if isinstance(loc.get("state"), dict)249 else loc.get("state"),250 loc.get("country", {}).get("name")251 if isinstance(loc.get("country"), dict)252 else loc.get("country")) if x)])253 name = name or (p.get("company") or {}).get("name", "") \254 if isinstance(p.get("company"), dict) else name255 qc, ex = _qc_stats(labels)256 return {"found": len(data), "qc": qc, "qc_ex": ex, "api_name": name}257258259def probe_bamboohr(slug: str, cand: dict) -> dict | None:260 r = _req(f"{slug}.bamboohr.com", "GET",261 f"https://{slug}.bamboohr.com/careers/list",262 headers={"User-Agent": UA, "Accept": "application/json"})263 if r.status_code != 200:264 return None265 try:266 data = r.json()267 except ValueError:268 return None269 jobs = data.get("result") or []270 if not jobs:271 return None272 labels = []273 for j in jobs:274 loc = j.get("location") or {}275 labels.append([", ".join(x for x in (loc.get("city"),276 loc.get("state")) if x)])277 qc, ex = _qc_stats(labels)278 return {"found": len(jobs), "qc": qc, "qc_ex": ex,279 "api_name": (data.get("meta") or {}).get("companyName") or ""}280281282def _wd_post(tenant: str, host: str, site: str,283 search: str = "") -> requests.Response:284 url = (f"https://{tenant}.{host}.myworkdayjobs.com"285 f"/wday/cxs/{tenant}/{site}/jobs")286 return _req(f"{tenant}.{host}.myworkdayjobs.com", "POST", url,287 json={"appliedFacets": {}, "limit": 20, "offset": 0,288 "searchText": search})289290291def _wd_eval(tenant: str, host: str, site: str) -> dict | None:292 """Site valide -> confirme s'il y a ≥1 offre dont ≥1 au Québec293 (page 1 brute, puis recherches « Québec »)."""294 total = 0295 for st in ("", "Québec", "Quebec, Canada"):296 try:297 r = _wd_post(tenant, host, site, st)298 except requests.RequestException:299 return None300 if r.status_code != 200:301 return None302 try:303 data = r.json()304 except ValueError:305 return None306 items = data.get("jobPostings") or []307 total = max(total, int(data.get("total") or 0))308 if st == "" and not total:309 return None310 labels = [[i.get("locationsText") or ""] for i in items]311 qc, ex = _qc_stats(labels)312 if qc:313 return {"found": total, "qc": qc, "qc_ex": ex, "api_name": "",314 "tenant": tenant, "host": host, "site": site}315 return None316317318def probe_workday(slug: str, cand: dict) -> dict | None:319 """La page racine myworkdayjobs répond 406 aux bots : on passe par l'API320 cxs — 422 = tenant absent de cet hôte, 404 = hôte bon mais site inconnu,321 200 = tenant+site valides. Hôte scanné puis site deviné au besoin."""322 tenant = (cand.get("tenant") or slug).lower()323 hosts = [cand["host"]] if cand.get("host") else WD_HOSTS324 for host in hosts:325 try:326 r = _wd_post(tenant, host, cand.get("site") or "__jobka_probe__")327 except requests.RequestException:328 continue329 if r.status_code == 200: # site fourni et valide330 return _wd_eval(tenant, host, cand["site"])331 if r.status_code != 404:332 continue # 422/5xx : hôte suivant333 # hôte confirmé — deviner le nom du site334 t = tenant335 derived = [f"{t}careers", f"{t}_careers", f"{t.capitalize()}Careers",336 f"{t.capitalize()}_Careers", f"{t.upper()}_CAREERS",337 f"{t.capitalize()}_External_Careers", t.capitalize(),338 f"Carrieres{t.capitalize()}", f"{t}_carrieres"]339 seen: list[str] = []340 for s in ([cand["site"]] if cand.get("site") else []) + derived \341 + WD_COMMON_SITES:342 if s in seen:343 continue344 seen.append(s)345 if len(seen) > 34:346 break347 try:348 r2 = _wd_post(tenant, host, s)349 except requests.RequestException:350 continue351 if r2.status_code == 200:352 return _wd_eval(tenant, host, s)353 return None # hôte trouvé mais nom de site introuvable354 return None355356357PROBES = {358 "lever": probe_lever, "greenhouse": probe_greenhouse, "ashby": probe_ashby,359 "smartrecruiters": probe_smartrecruiters, "workable": probe_workable,360 "recruitee": probe_recruitee, "breezy": probe_breezy,361 "bamboohr": probe_bamboohr, "workday": probe_workday,362}363364365# --- registre existant (pour sauter les slugs déjà connectés) ----------------366def known_slugs() -> set[str]:367 known: set[str] = set()368 src = ROOT / "data" / "sources.json"369 if src.exists():370 for s in json.loads(src.read_text())["sources"]:371 known.add(s["id"])372 pat = re.compile(r"^\s*(?:ORG|BOARD|COMPANY|TENANT)\s*=\s*['\"]([^'\"]+)",373 re.M)374 for f in (ROOT / "jobka" / "connectors").glob("*.py"):375 for m in pat.finditer(f.read_text(encoding="utf-8")):376 known.add(m.group(1).lower())377 return known378379380# --- pipeline -----------------------------------------------------------------381def probe_group(group: str, cands: list[dict], stats: dict,382 slock: threading.Lock) -> tuple[dict | None, dict]:383 """Essaie chaque variante du groupe ; retourne (feed confirmé, rejet)."""384 best = None385 for cand in cands:386 slug = cand["slug"]387 order = [cand["ats"]] if cand.get("ats") else [388 a for a in DEFAULT_ORDER if not (cand.get("no_workday")389 and a == "workday")]390 for ats in order:391 with slock:392 stats.setdefault(ats, [0, 0])[0] += 1393 try:394 res = PROBES[ats](slug, cand)395 except Exception: # API hors-format = slug non confirmé396 res = None397 if not res:398 continue399 if res["qc"] >= 1:400 with slock:401 stats[ats][1] += 1402 feed = {"ats": ats,403 "employer": cand.get("employer") or slug,404 "sectors": cand.get("sectors") or [],405 "cities": cand.get("cities") or [],406 "url": cand.get("url", ""),407 "_found": res["found"], "_qc": res["qc"],408 "_qc_locations": res["qc_ex"],409 "_api_name": res.get("api_name", "")}410 if ats == "workday":411 feed.update(tenant=res["tenant"], host=res["host"],412 site=res["site"])413 else:414 feed["org"] = res.get("org_override", slug)415 return feed, {}416 if best is None or res["found"] > best.get("found", 0):417 best = {"ats": ats, "slug": slug, "found": res["found"]}418 reject = {"group": group, "slugs": [c["slug"] for c in cands],419 "reason": "no_qc" if best else "no_api", "best": best}420 return None, reject421422423def main() -> None:424 ap = argparse.ArgumentParser()425 ap.add_argument("candidates")426 ap.add_argument("--out", required=True)427 ap.add_argument("--rejected", default="")428 ap.add_argument("--workers", type=int, default=8)429 args = ap.parse_args()430431 raw = Path(args.candidates).read_text(encoding="utf-8").strip()432 if raw.startswith("["):433 cands = json.loads(raw)434 else:435 cands = [json.loads(l) for l in raw.splitlines() if l.strip()]436437 known = known_slugs()438 groups: dict[str, list[dict]] = {}439 skipped = 0440 for c in cands:441 slug = (c.get("slug") or "").strip()442 if not slug:443 continue444 if slug.lower() in known or (c.get("tenant") or "").lower() in known:445 skipped += 1446 continue447 groups.setdefault(c.get("group") or c.get("employer") or slug,448 []).append(c)449450 n_slugs = sum(len(v) for v in groups.values())451 print(f"[probe] {len(groups)} groupes / {n_slugs} slugs à sonder "452 f"({skipped} déjà connectés, sautés)", flush=True)453454 stats: dict[str, list[int]] = {}455 slock = threading.Lock()456 confirmed, rejected = [], []457 t0 = time.time()458 with ThreadPoolExecutor(max_workers=args.workers) as ex:459 futs = {ex.submit(probe_group, g, cs, stats, slock): g460 for g, cs in groups.items()}461 done = 0462 for fut in as_completed(futs):463 done += 1464 try:465 feed, reject = fut.result()466 except Exception as exc:467 print(f"[probe] {futs[fut]} : erreur {exc}", flush=True)468 continue469 if feed:470 confirmed.append(feed)471 print(f" ✓ {feed['employer']} -> {feed['ats']}"472 f" ({feed.get('org') or feed.get('tenant')}) "473 f"qc={feed['_qc']}/{feed['_found']}", flush=True)474 elif reject:475 rejected.append(reject)476 if done % 50 == 0:477 print(f"[probe] {done}/{len(groups)} groupes, "478 f"{len(confirmed)} confirmés, {_nreq[0]} requêtes, "479 f"{time.time()-t0:.0f}s", flush=True)480481 confirmed.sort(key=lambda f: (f["ats"], f.get("org") or f.get("tenant", "")))482 Path(args.out).write_text(483 json.dumps(confirmed, ensure_ascii=False, indent=2) + "\n",484 encoding="utf-8")485 if args.rejected:486 Path(args.rejected).write_text(487 "\n".join(json.dumps(r, ensure_ascii=False) for r in rejected)488 + "\n", encoding="utf-8")489490 print(f"\n[probe] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes")491 print(f"[probe] slugs sondés par ATS (sondés/confirmés) :")492 for ats in DEFAULT_ORDER:493 p, c = stats.get(ats, [0, 0])494 print(f" {ats:16s} {p:5d} / {c}")495 print(f"[probe] {len(confirmed)} groupes confirmés, "496 f"{len(rejected)} rejetés -> {args.out}")497498499if __name__ == "__main__":500 main()501