HTML 82.1%
Python 14.6%
TypeScript 1.9%
CSS 1%
JavaScript 0.5%
1#!/usr/bin/env python32# =============================================================================3# Job·Ka — Groupe KA4# Auteur : Simon-Pierre Boucher5# Contact : contact@spboucher.ai6# Fichier : scripts/probe_ats.py7# Rôle : Sondage massif de slugs candidats sur les 10 ATS supportés —8# un slug est CONFIRMÉ si l'API répond ≥1 offre ET ≥1 offre au9# Québec (réutilise is_quebec_location du projet). Émet des entrées10# prêtes pour data/feeds-*.json (scripts/gen_connectors.py).11# Créé : 2026-08-18 Modifié : 2026-08-2212# =============================================================================13"""Sonde des slugs candidats sur les 9 plateformes ATS de Job·Ka.1415Usage :16 python3 scripts/probe_ats.py candidats.jsonl --out confirmed.json \17 [--rejected rejected.jsonl] [--workers 8]1819Entrée (JSONL, ou JSON liste) — un candidat par ligne :20 {"slug": "acme", "employer": "Acme inc.", "ats": "lever"?,21 "sectors": ["TI"], "cities": ["Montréal"], "group": "Acme"?,22 "no_workday": true?, "tenant"/"host"/"site" (workday)?}2324- sans "ats" : les 9 plateformes sont sondées dans l'ordre DEFAULT_ORDER25 (Workday en dernier — découverte automatique de l'hôte wdN et du site26 par le redirect de la racine du tenant).27- les candidats partageant le même "group" (défaut : employer) sont des28 variantes du même employeur : on s'arrête à la première confirmation.29- les slugs/tenants déjà connectés (sources.json + jobka/connectors/*.py)30 sont sautés automatiquement.3132Politesse : timeout 6 s, throttle 0,3 s par domaine d'API, 8 threads.33"""34from __future__ import annotations3536import argparse37import json38import re39import sys40import threading41import time42from concurrent.futures import ThreadPoolExecutor, as_completed43from pathlib import Path44from urllib.parse import urlparse4546import requests4748ROOT = Path(__file__).resolve().parent.parent49sys.path.insert(0, str(ROOT))5051from jobka.normalize import is_quebec_location # noqa: E4025253UA = "JobKaBot/1.0 (+https://www.job-ka.com; contact@spboucher.ai)"54TIMEOUT = 655THROTTLE = 0.3 # s entre deux requêtes vers un même domaine d'API56WD_HOSTS = ["wd3", "wd1", "wd5", "wd10", "wd12", "wd103", "wd502", "wd504"]5758# Noms de sites carrières Workday fréquents — essayés quand le site est59# inconnu. L'API cxs répond 422 si le tenant/hôte est faux mais 404 si seul60# le site l'est : on découvre d'abord l'hôte, puis on devine le site.61WD_COMMON_SITES = [62 "External", "Careers", "External_Careers", "ExternalCareers", "careers",63 "External_Career_Site", "CareerSite", "Career", "Global", "external",64 "Global_Careers", "GlobalCareers", "jobs", "Jobs", "External_Site",65 "ExternalSite", "Externe", "Carrieres", "carrieres", "Recruiting",66 "Recrutement", "Emplois", "External_Portal", "EXTERNAL", "Ext", "EXT",67]6869DEFAULT_ORDER = ["lever", "greenhouse", "ashby", "smartrecruiters",70 "workable", "recruitee", "breezy", "bamboohr", "dayforce",71 "workday"]7273# --- throttling par domaine d'API -------------------------------------------74_locks: dict[str, threading.Lock] = {}75_last: dict[str, float] = {}76_glock = threading.Lock()77_nreq = [0]787980def _req(family: str, method: str, url: str, **kw) -> requests.Response:81 """Requête HTTP throttlée (verrou + espacement par famille/domaine)."""82 with _glock:83 lock = _locks.setdefault(family, threading.Lock())84 _nreq[0] += 185 headers = {"User-Agent": UA}86 headers.update(kw.pop("headers", None) or {})87 with lock:88 wait = THROTTLE - (time.time() - _last.get(family, 0.0))89 if wait > 0:90 time.sleep(wait)91 try:92 return requests.request(method, url, timeout=TIMEOUT,93 headers=headers, **kw)94 finally:95 _last[family] = time.time()969798def _qc_stats(labels: list[list[str]]) -> tuple[int, list[str]]:99 """[[libellés d'une offre], …] -> (nb d'offres QC, libellés QC exemples)."""100 qc, ex = 0, []101 for locs in labels:102 hit = next((l for l in locs if l and is_quebec_location(l)), None)103 if hit:104 qc += 1105 if len(ex) < 5 and hit not in ex:106 ex.append(hit)107 return qc, ex108109110# --- sondes par ATS ----------------------------------------------------------111def probe_lever(slug: str, cand: dict) -> dict | None:112 r = _req("api.lever.co", "GET",113 f"https://api.lever.co/v0/postings/{slug}?mode=json")114 if r.status_code != 200:115 return None116 data = r.json()117 if not isinstance(data, list) or not data:118 return None119 labels = []120 for p in data:121 locs = [(p.get("categories") or {}).get("location") or ""]122 locs += [l for l in (p.get("allLocations") or []) if isinstance(l, str)]123 labels.append(locs)124 qc, ex = _qc_stats(labels)125 return {"found": len(data), "qc": qc, "qc_ex": ex, "api_name": ""}126127128def probe_greenhouse(slug: str, cand: dict) -> dict | None:129 r = _req("boards-api.greenhouse.io", "GET",130 f"https://boards-api.greenhouse.io/v1/boards/{slug}/jobs")131 if r.status_code != 200:132 return None133 jobs = r.json().get("jobs") or []134 if not jobs:135 return None136 labels = [[(j.get("location") or {}).get("name") or ""] for j in jobs]137 qc, ex = _qc_stats(labels)138 name = ""139 if qc:140 try:141 r2 = _req("boards-api.greenhouse.io", "GET",142 f"https://boards-api.greenhouse.io/v1/boards/{slug}")143 if r2.status_code == 200:144 name = r2.json().get("name") or ""145 except requests.RequestException:146 pass147 return {"found": len(jobs), "qc": qc, "qc_ex": ex, "api_name": name}148149150def probe_ashby(slug: str, cand: dict) -> dict | None:151 r = _req("api.ashbyhq.com", "GET",152 f"https://api.ashbyhq.com/posting-api/job-board/{slug}")153 if r.status_code != 200:154 return None155 data = r.json()156 jobs = data.get("jobs") or []157 if not jobs:158 return None159 labels = []160 for j in jobs:161 locs = [j.get("location") or ""]162 locs += [(s.get("location") or "") for s in (j.get("secondaryLocations") or [])]163 labels.append(locs)164 qc, ex = _qc_stats(labels)165 return {"found": len(jobs), "qc": qc, "qc_ex": ex,166 "api_name": data.get("name") or ""}167168169def probe_smartrecruiters(slug: str, cand: dict) -> dict | None:170 variants = [slug]171 if slug != slug.capitalize():172 variants.append(slug.capitalize())173 for v in variants:174 r = _req("api.smartrecruiters.com", "GET",175 f"https://api.smartrecruiters.com/v1/companies/{v}/postings?limit=100")176 if r.status_code != 200:177 continue178 data = r.json()179 content = data.get("content") or []180 total = int(data.get("totalFound") or len(content))181 if not content:182 continue183 labels, name = [], ""184 for p in content:185 loc = p.get("location") or {}186 country = (loc.get("country") or "").lower()187 lbl = ", ".join(x for x in (loc.get("city"), loc.get("region"),188 loc.get("country")) if x)189 reg = (loc.get("region") or "").lower()190 ok = reg in ("qc", "quebec", "québec") or (191 country in ("ca", "canada") and is_quebec_location(lbl))192 labels.append([lbl] if ok else [""])193 name = name or (p.get("company") or {}).get("name") or ""194 qc, ex = _qc_stats(labels)195 return {"found": total, "qc": qc, "qc_ex": ex, "api_name": name,196 "org_override": v}197 return None198199200def probe_workable(slug: str, cand: dict) -> dict | None:201 r = _req("apply.workable.com", "GET",202 f"https://apply.workable.com/api/v1/widget/accounts/{slug}?details=false")203 if r.status_code != 200:204 return None205 data = r.json()206 jobs = data.get("jobs") or []207 if not jobs:208 return None209 labels = [[", ".join(x for x in (j.get("city"), j.get("state"),210 j.get("country")) if x)] for j in jobs]211 qc, ex = _qc_stats(labels)212 return {"found": len(jobs), "qc": qc, "qc_ex": ex,213 "api_name": data.get("name") or ""}214215216def probe_recruitee(slug: str, cand: dict) -> dict | None:217 r = _req(f"{slug}.recruitee.com", "GET",218 f"https://{slug}.recruitee.com/api/offers/")219 if r.status_code != 200:220 return None221 offers = r.json().get("offers") or []222 if not offers:223 return None224 labels = []225 for o in offers:226 locs = [o.get("location") or "",227 ", ".join(x for x in (o.get("city"), o.get("country")) if x)]228 labels.append(locs)229 qc, ex = _qc_stats(labels)230 return {"found": len(offers), "qc": qc, "qc_ex": ex, "api_name": ""}231232233def probe_breezy(slug: str, cand: dict) -> dict | None:234 r = _req(f"{slug}.breezy.hr", "GET", f"https://{slug}.breezy.hr/json")235 if r.status_code != 200:236 return None237 try:238 data = r.json()239 except ValueError:240 return None241 if not isinstance(data, list) or not data:242 return None243 labels, name = [], ""244 for p in data:245 loc = p.get("location") or {}246 labels.append([loc.get("name") or "",247 ", ".join(x for x in (loc.get("city"),248 (loc.get("state") or {}).get("name")249 if isinstance(loc.get("state"), dict)250 else loc.get("state"),251 loc.get("country", {}).get("name")252 if isinstance(loc.get("country"), dict)253 else loc.get("country")) if x)])254 name = name or (p.get("company") or {}).get("name", "") \255 if isinstance(p.get("company"), dict) else name256 qc, ex = _qc_stats(labels)257 return {"found": len(data), "qc": qc, "qc_ex": ex, "api_name": name}258259260def probe_bamboohr(slug: str, cand: dict) -> dict | None:261 r = _req(f"{slug}.bamboohr.com", "GET",262 f"https://{slug}.bamboohr.com/careers/list",263 headers={"User-Agent": UA, "Accept": "application/json"})264 if r.status_code != 200:265 return None266 try:267 data = r.json()268 except ValueError:269 return None270 jobs = data.get("result") or []271 if not jobs:272 return None273 labels = []274 for j in jobs:275 loc = j.get("location") or {}276 labels.append([", ".join(x for x in (loc.get("city"),277 loc.get("state")) if x)])278 qc, ex = _qc_stats(labels)279 return {"found": len(jobs), "qc": qc, "qc_ex": ex,280 "api_name": (data.get("meta") or {}).get("companyName") or ""}281282283def _wd_post(tenant: str, host: str, site: str,284 search: str = "") -> requests.Response:285 url = (f"https://{tenant}.{host}.myworkdayjobs.com"286 f"/wday/cxs/{tenant}/{site}/jobs")287 return _req(f"{tenant}.{host}.myworkdayjobs.com", "POST", url,288 json={"appliedFacets": {}, "limit": 20, "offset": 0,289 "searchText": search})290291292def _wd_eval(tenant: str, host: str, site: str) -> dict | None:293 """Site valide -> confirme s'il y a ≥1 offre dont ≥1 au Québec294 (page 1 brute, puis recherches « Québec »)."""295 total = 0296 for st in ("", "Québec", "Quebec, Canada"):297 try:298 r = _wd_post(tenant, host, site, st)299 except requests.RequestException:300 return None301 if r.status_code != 200:302 return None303 try:304 data = r.json()305 except ValueError:306 return None307 items = data.get("jobPostings") or []308 total = max(total, int(data.get("total") or 0))309 if st == "" and not total:310 return None311 labels = [[i.get("locationsText") or ""] for i in items]312 qc, ex = _qc_stats(labels)313 if qc:314 return {"found": total, "qc": qc, "qc_ex": ex, "api_name": "",315 "tenant": tenant, "host": host, "site": site}316 return None317318319def probe_workday(slug: str, cand: dict) -> dict | None:320 """La page racine myworkdayjobs répond 406 aux bots : on passe par l'API321 cxs — 422 = tenant absent de cet hôte, 404 = hôte bon mais site inconnu,322 200 = tenant+site valides. Hôte scanné puis site deviné au besoin."""323 tenant = (cand.get("tenant") or slug).lower()324 hosts = [cand["host"]] if cand.get("host") else WD_HOSTS325 for host in hosts:326 try:327 r = _wd_post(tenant, host, cand.get("site") or "__jobka_probe__")328 except requests.RequestException:329 continue330 if r.status_code == 200: # site fourni et valide331 return _wd_eval(tenant, host, cand["site"])332 if r.status_code != 404:333 continue # 422/5xx : hôte suivant334 # hôte confirmé — deviner le nom du site335 t = tenant336 derived = [f"{t}careers", f"{t}_careers", f"{t.capitalize()}Careers",337 f"{t.capitalize()}_Careers", f"{t.upper()}_CAREERS",338 f"{t.capitalize()}_External_Careers", t.capitalize(),339 f"Carrieres{t.capitalize()}", f"{t}_carrieres"]340 seen: list[str] = []341 for s in ([cand["site"]] if cand.get("site") else []) + derived \342 + WD_COMMON_SITES:343 if s in seen:344 continue345 seen.append(s)346 if len(seen) > 34:347 break348 try:349 r2 = _wd_post(tenant, host, s)350 except requests.RequestException:351 continue352 if r2.status_code == 200:353 return _wd_eval(tenant, host, s)354 return None # hôte trouvé mais nom de site introuvable355 return None356357358def probe_dayforce(slug: str, cand: dict) -> dict | None:359 """Dayforce HCM — même API que jobka/connectors/dayforce.py : csrf anonyme360 puis recherche paginée. Le namespace est sensible à la casse tel qu'écrit361 dans l'URL ; babillard par défaut CANDIDATEPORTAL."""362 base = "https://jobs.dayforcehcm.com"363 boards = [b for b in (cand.get("board"), "CANDIDATEPORTAL") if b]364 sess = requests.Session()365 sess.headers["User-Agent"] = UA366 with _glock:367 lock = _locks.setdefault("jobs.dayforcehcm.com", threading.Lock())368 for board in dict.fromkeys(boards):369 with lock:370 wait = THROTTLE - (time.time() - _last.get("dayforce", 0.0))371 if wait > 0:372 time.sleep(wait)373 try:374 csrf = (sess.get(f"{base}/api/auth/csrf",375 timeout=TIMEOUT).json() or {}).get(376 "csrfToken") or ""377 r = sess.post(378 f"{base}/api/geo/{slug}/jobposting/search",379 timeout=TIMEOUT,380 headers={"X-CSRF-TOKEN": csrf,381 "Accept": "application/json"},382 json={"clientNamespace": slug, "jobBoardCode": board,383 "cultureCode": "fr-CA", "paginationStart": 0})384 except requests.RequestException:385 return None386 finally:387 _last["dayforce"] = time.time()388 _nreq[0] += 1389 if r.status_code != 200:390 continue391 try:392 data = r.json()393 except ValueError:394 continue395 postings = data.get("jobPostings") or []396 total = int(data.get("maxCount") or len(postings))397 if not postings:398 continue399 labels = []400 for p in postings:401 locs = []402 for l in (p.get("postingLocations") or []):403 if (l.get("stateCode") or "").upper() == "QC" and \404 (l.get("isoCountryCode") or "CA").upper() == "CA":405 locs.append(l.get("formattedAddress")406 or f"{l.get('cityName') or ''}, QC")407 else:408 locs.append(l.get("formattedAddress") or "")409 labels.append(locs)410 qc, ex = _qc_stats(labels)411 return {"found": total, "qc": qc, "qc_ex": ex, "api_name": "",412 "ns": slug, "board": board}413 return None414415416PROBES = {417 "lever": probe_lever, "greenhouse": probe_greenhouse, "ashby": probe_ashby,418 "smartrecruiters": probe_smartrecruiters, "workable": probe_workable,419 "recruitee": probe_recruitee, "breezy": probe_breezy,420 "bamboohr": probe_bamboohr, "workday": probe_workday,421 "dayforce": probe_dayforce,422}423424425# --- registre existant (pour sauter les slugs déjà connectés) ----------------426def known_slugs() -> set[str]:427 known: set[str] = set()428 src = ROOT / "data" / "sources.json"429 if src.exists():430 for s in json.loads(src.read_text())["sources"]:431 known.add(s["id"])432 pat = re.compile(r"^\s*(?:ORG|BOARD|COMPANY|TENANT)\s*=\s*['\"]([^'\"]+)",433 re.M)434 for f in (ROOT / "jobka" / "connectors").glob("*.py"):435 for m in pat.finditer(f.read_text(encoding="utf-8")):436 known.add(m.group(1).lower())437 return known438439440# --- pipeline -----------------------------------------------------------------441def probe_group(group: str, cands: list[dict], stats: dict,442 slock: threading.Lock) -> tuple[dict | None, dict]:443 """Essaie chaque variante du groupe ; retourne (feed confirmé, rejet)."""444 best = None445 for cand in cands:446 slug = cand["slug"]447 order = [cand["ats"]] if cand.get("ats") else [448 a for a in DEFAULT_ORDER if not (cand.get("no_workday")449 and a == "workday")]450 for ats in order:451 with slock:452 stats.setdefault(ats, [0, 0])[0] += 1453 try:454 res = PROBES[ats](slug, cand)455 except Exception: # API hors-format = slug non confirmé456 res = None457 if not res:458 continue459 if res["qc"] >= 1:460 with slock:461 stats[ats][1] += 1462 feed = {"ats": ats,463 "employer": cand.get("employer") or slug,464 "sectors": cand.get("sectors") or [],465 "cities": cand.get("cities") or [],466 "url": cand.get("url", ""),467 "_found": res["found"], "_qc": res["qc"],468 "_qc_locations": res["qc_ex"],469 "_api_name": res.get("api_name", "")}470 if ats == "workday":471 feed.update(tenant=res["tenant"], host=res["host"],472 site=res["site"])473 elif ats == "dayforce":474 feed.update(ns=res["ns"], board=res["board"],475 source_id=res["ns"].lower())476 else:477 feed["org"] = res.get("org_override", slug)478 return feed, {}479 if best is None or res["found"] > best.get("found", 0):480 best = {"ats": ats, "slug": slug, "found": res["found"]}481 reject = {"group": group, "slugs": [c["slug"] for c in cands],482 "reason": "no_qc" if best else "no_api", "best": best}483 return None, reject484485486def main() -> None:487 ap = argparse.ArgumentParser()488 ap.add_argument("candidates")489 ap.add_argument("--out", required=True)490 ap.add_argument("--rejected", default="")491 ap.add_argument("--workers", type=int, default=8)492 args = ap.parse_args()493494 raw = Path(args.candidates).read_text(encoding="utf-8").strip()495 if raw.startswith("["):496 cands = json.loads(raw)497 else:498 cands = [json.loads(l) for l in raw.splitlines() if l.strip()]499500 known = known_slugs()501 groups: dict[str, list[dict]] = {}502 skipped = 0503 for c in cands:504 slug = (c.get("slug") or "").strip()505 if not slug:506 continue507 if slug.lower() in known or (c.get("tenant") or "").lower() in known:508 skipped += 1509 continue510 groups.setdefault(c.get("group") or c.get("employer") or slug,511 []).append(c)512513 n_slugs = sum(len(v) for v in groups.values())514 print(f"[probe] {len(groups)} groupes / {n_slugs} slugs à sonder "515 f"({skipped} déjà connectés, sautés)", flush=True)516517 stats: dict[str, list[int]] = {}518 slock = threading.Lock()519 confirmed, rejected = [], []520 t0 = time.time()521 with ThreadPoolExecutor(max_workers=args.workers) as ex:522 futs = {ex.submit(probe_group, g, cs, stats, slock): g523 for g, cs in groups.items()}524 done = 0525 for fut in as_completed(futs):526 done += 1527 try:528 feed, reject = fut.result()529 except Exception as exc:530 print(f"[probe] {futs[fut]} : erreur {exc}", flush=True)531 continue532 if feed:533 confirmed.append(feed)534 print(f" ✓ {feed['employer']} -> {feed['ats']}"535 f" ({feed.get('org') or feed.get('tenant')}) "536 f"qc={feed['_qc']}/{feed['_found']}", flush=True)537 elif reject:538 rejected.append(reject)539 if done % 50 == 0:540 print(f"[probe] {done}/{len(groups)} groupes, "541 f"{len(confirmed)} confirmés, {_nreq[0]} requêtes, "542 f"{time.time()-t0:.0f}s", flush=True)543544 confirmed.sort(key=lambda f: (f["ats"], f.get("org") or f.get("tenant", "")))545 Path(args.out).write_text(546 json.dumps(confirmed, ensure_ascii=False, indent=2) + "\n",547 encoding="utf-8")548 if args.rejected:549 Path(args.rejected).write_text(550 "\n".join(json.dumps(r, ensure_ascii=False) for r in rejected)551 + "\n", encoding="utf-8")552553 print(f"\n[probe] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes")554 print(f"[probe] slugs sondés par ATS (sondés/confirmés) :")555 for ats in DEFAULT_ORDER:556 p, c = stats.get(ats, [0, 0])557 print(f" {ats:16s} {p:5d} / {c}")558 print(f"[probe] {len(confirmed)} groupes confirmés, "559 f"{len(rejected)} rejetés -> {args.out}")560561562if __name__ == "__main__":563 main()564