HTML 82.1%
Python 14.6%
TypeScript 1.9%
CSS 1%
JavaScript 0.5%
1#!/usr/bin/env python32# =============================================================================3# Job·Ka — Groupe KA4# Auteur : Simon-Pierre Boucher5# Contact : contact@spboucher.ai6# Fichier : scripts/detect_ats.py7# Rôle : Détecteur d'ATS par page carrières — visite le site de chaque8# employeur (accueil + chemins carrières usuels), reconnaît l'ATS9# par motifs d'URL/DOM et extrait le slug. Les 9 ATS supportés10# deviennent des candidats pour scripts/probe_ats.py ; les autres11# (njoyn/taleo/ukg/icims/successfactors/adp) sont consignés dans12# data/detected-other-ats.jsonl pour traitement séparé.13# Créé : 2026-08-18 Modifié : 2026-08-1814# =============================================================================15"""Usage :16 python3 scripts/detect_ats.py employeurs.jsonl [autres.jsonl …] \17 --out candidats.jsonl --other data/detected-other-ats.jsonl \18 [--cap 1500] [--workers 8]1920Entrée : JSONL {"name": "...", "url"/"domain"/"domain_hint": "...",21 "city": "...", "categorie": "..."} (ordre = priorité).22"""23from __future__ import annotations2425import argparse26import json27import re28import sys29import threading30import time31from concurrent.futures import ThreadPoolExecutor, as_completed32from pathlib import Path33from urllib.parse import urljoin, urlparse3435import requests36import urllib33738urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)3940UA = "JobKaBot/1.0 (+https://www.job-ka.com; contact@spboucher.ai)"41TIMEOUT = 642PATHS = ["", "/carrieres", "/careers", "/emplois", "/jobs", "/fr/carrieres"]43MAX_HTML = 500_0004445# slugs jamais valides (pages génériques des ATS eux-mêmes)46BAD_SLUGS = {"www", "app", "api", "apply", "embed", "boards", "jobs", "job",47 "careers", "career", "fr", "en", "help", "support", "cdn",48 "assets", "status", "docs", "login", "my", "sso", "static",49 "oneclick-ui", "attachments", "hire", "home", "account",50 "js", "css", "images", "img", "media", "email", "click"}5152# ATS supportés par Job·Ka : (ats, regex, groupes)53SUPPORTED = [54 ("workday", re.compile(55 r"https?://([\w-]+)\.(wd\d+)\.myworkdayjobs\.com"56 r"(?:/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_\-]+))?", re.I)),57 ("greenhouse", re.compile(58 r"(?:boards|job-boards)\.greenhouse\.io/([A-Za-z0-9_\-]+)", re.I)),59 ("greenhouse", re.compile(60 r"greenhouse\.io/embed/job_board\?[^\"'\s]*for=([A-Za-z0-9_\-]+)", re.I)),61 ("lever", re.compile(r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_\-]+)", re.I)),62 ("smartrecruiters", re.compile(63 r"(?:careers|jobs)\.smartrecruiters\.com/([A-Za-z0-9]+)")),64 ("ashby", re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9_\-.%]+)", re.I)),65 ("workable", re.compile(66 r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_\-]+)", re.I)),67 ("recruitee", re.compile(r"https?://([A-Za-z0-9\-]+)\.recruitee\.com", re.I)),68 ("breezy", re.compile(r"https?://([A-Za-z0-9\-]+)\.breezy\.hr", re.I)),69 ("bamboohr", re.compile(r"https?://([A-Za-z0-9\-]+)\.bamboohr\.com", re.I)),70]7172# ATS repérés mais non supportés ici -> data/detected-other-ats.jsonl73OTHER = [74 ("njoyn", re.compile(r"https?://[\w.-]*njoyn\.com[^\"'\s<>]*", re.I)),75 ("taleo", re.compile(r"https?://[\w.-]*taleo\.net[^\"'\s<>]*", re.I)),76 ("ukg_ultipro", re.compile(77 r"https?://[\w.-]*(?:ultipro|ukg)\.com[^\"'\s<>]*", re.I)),78 ("icims", re.compile(r"https?://[\w-]+\.icims\.com[^\"'\s<>]*", re.I)),79 ("successfactors", re.compile(80 r"https?://[\w.-]*successfactors\.(?:com|eu)[^\"'\s<>]*", re.I)),81 ("adp", re.compile(82 r"https?://(?:workforcenow|recruiting|jobs)\.adp\.com[^\"'\s<>]*", re.I)),83]8485CAREER_LINK = re.compile(86 r'href=["\']([^"\']*(?:carri|career|emploi|job|recrut|joignez|joindre|'87 r'postul|travaill)[^"\']*)["\']', re.I)8889_print_lock = threading.Lock()909192def norm_domain(rec: dict) -> str:93 raw = (rec.get("domain_hint") or rec.get("domain") or rec.get("url")94 or "").strip()95 if not raw:96 return ""97 if "://" not in raw:98 raw = "https://" + raw99 host = urlparse(raw).netloc.lower()100 return host[4:] if host.startswith("www.") else host101102103def fetch(session: requests.Session, url: str) -> tuple[str, str]:104 r = session.get(url, timeout=TIMEOUT, allow_redirects=True, verify=False,105 headers={"User-Agent": UA})106 if r.status_code >= 400:107 return r.url, ""108 return r.url, (r.text or "")[:MAX_HTML]109110111def scan_text(text: str) -> tuple[list[tuple], list[tuple]]:112 """-> ([(ats, slug, extra…)], [(ats, url)])."""113 hits, others = [], []114 for ats, rx in SUPPORTED:115 for m in rx.finditer(text):116 if ats == "workday":117 tenant, host = m.group(1).lower(), m.group(2).lower()118 site = m.group(3) or ""119 if tenant in BAD_SLUGS or tenant == "impl":120 continue121 hits.append((ats, tenant, host, site))122 else:123 slug = m.group(1)124 if slug.lower() in BAD_SLUGS:125 continue126 hits.append((ats, slug))127 for ats, rx in OTHER:128 m = rx.search(text)129 if m:130 others.append((ats, m.group(0)[:300]))131 return hits, others132133134def detect_one(rec: dict) -> dict:135 """-> {"candidates": [...], "others": [...], "domain": d}."""136 domain = norm_domain(rec)137 out = {"domain": domain, "candidates": [], "others": []}138 if not domain:139 return out140 session = requests.Session()141 seen_hits: set[tuple] = set()142 seen_others: set[str] = set()143 extra_links: list[str] = []144 pages = 0145 for i, path in enumerate(PATHS + ["@extra1", "@extra2"]):146 if path.startswith("@"):147 if not extra_links:148 continue149 url = extra_links.pop(0)150 else:151 url = f"https://{domain}{path}"152 try:153 final_url, html = fetch(session, url)154 except requests.RequestException:155 if path == "": # racine injoignable en https -> essai www/http156 for alt in (f"https://www.{domain}", f"http://{domain}"):157 try:158 final_url, html = fetch(session, alt)159 break160 except requests.RequestException:161 final_url, html = "", ""162 if not html:163 continue164 else:165 continue166 pages += 1167 hits, others = scan_text(final_url + "\n" + html)168 for h in hits:169 if h in seen_hits:170 continue171 seen_hits.add(h)172 for ats, u in others:173 if ats not in seen_others:174 seen_others.add(ats)175 out["others"].append({"name": rec.get("name", ""),176 "domain": domain, "ats": ats, "url": u})177 if path == "" and html and not seen_hits:178 # relever jusqu'à 2 liens « carrières » à suivre179 base = final_url or f"https://{domain}"180 for link in CAREER_LINK.findall(html):181 if link.startswith(("mailto:", "tel:", "#", "javascript")):182 continue183 full = urljoin(base, link)184 if full not in extra_links:185 extra_links.append(full)186 if len(extra_links) >= 2:187 break188 if seen_hits:189 break # ATS supporté trouvé : inutile d'aller plus loin190 for h in seen_hits:191 cand = {"employer": rec.get("name") or domain, "domain": domain,192 "url": f"https://{domain}",193 "sectors": [s for s in [rec.get("categorie", "")] if s],194 "cities": [c for c in [rec.get("city", "")] if c],195 "group": rec.get("name") or domain}196 if h[0] == "workday":197 cand.update(ats="workday", slug=h[1], tenant=h[1], host=h[2])198 if h[3] and not re.fullmatch(r"[a-z]{2}-[A-Z]{2}", h[3]):199 cand["site"] = h[3]200 else:201 cand.update(ats=h[0], slug=h[1])202 out["candidates"].append(cand)203 return out204205206def main() -> None:207 ap = argparse.ArgumentParser()208 ap.add_argument("inputs", nargs="+")209 ap.add_argument("--out", required=True)210 ap.add_argument("--other", default="data/detected-other-ats.jsonl")211 ap.add_argument("--cap", type=int, default=1500)212 ap.add_argument("--workers", type=int, default=8)213 args = ap.parse_args()214215 recs, seen = [], set()216 for f in args.inputs:217 for line in Path(f).read_text(encoding="utf-8").splitlines():218 line = line.strip()219 if not line:220 continue221 rec = json.loads(line)222 d = norm_domain(rec)223 if d and d not in seen:224 seen.add(d)225 recs.append(rec)226 recs = recs[:args.cap]227 print(f"[detect] {len(recs)} domaines à visiter (cap {args.cap})",228 flush=True)229230 cands, others = [], []231 t0 = time.time()232 done = 0233 with ThreadPoolExecutor(max_workers=args.workers) as ex:234 futs = [ex.submit(detect_one, r) for r in recs]235 for fut in as_completed(futs):236 done += 1237 try:238 res = fut.result()239 except Exception:240 continue241 for c in res["candidates"]:242 cands.append(c)243 with _print_lock:244 print(f" ✓ {c['employer']} -> {c['ats']}:{c['slug']}",245 flush=True)246 others.extend(res["others"])247 if done % 100 == 0:248 print(f"[detect] {done}/{len(recs)} domaines, "249 f"{len(cands)} candidats, {len(others)} autres ATS, "250 f"{time.time()-t0:.0f}s", flush=True)251252 Path(args.out).write_text(253 "\n".join(json.dumps(c, ensure_ascii=False) for c in cands) + "\n",254 encoding="utf-8")255 op = Path(args.other)256 existing = set()257 if op.exists():258 existing = {l for l in op.read_text(encoding="utf-8").splitlines()259 if l.strip()}260 with op.open("a", encoding="utf-8") as fh:261 for o in others:262 line = json.dumps(o, ensure_ascii=False)263 if line not in existing:264 fh.write(line + "\n")265 existing.add(line)266267 per = {}268 for o in others:269 per[o["ats"]] = per.get(o["ats"], 0) + 1270 print(f"\n[detect] terminé en {time.time()-t0:.0f}s : "271 f"{len(cands)} candidats ATS supportés -> {args.out}")272 print(f"[detect] autres ATS ({sum(per.values())}) -> {args.other} : "273 + ", ".join(f"{k}={v}" for k, v in sorted(per.items())))274275276if __name__ == "__main__":277 main()278