1#!/usr/bin/env python32# =============================================================================3# Job·Ka — Groupe KA4# Auteur : Simon-Pierre Boucher5# Contact : contact@spboucher.ai6# Fichier : scripts/serper_scout.py7# Rôle : Éclaireur Serper — requêtes Google `site:<domaine ATS>` × termes8# québécois, extraction des slugs candidats par motifs d'URL,9# dédoublonnage contre sources.json/connecteurs. Sortie JSONL prête10# pour scripts/probe_ats.py.11# Créé : 2026-08-22 Modifié : 2026-08-2212# =============================================================================13"""Usage :14 python3 scripts/serper_scout.py --out candidats.jsonl \15 [--pages 2] [--ats lever,greenhouse,...] [--workers 6]1617Nécessite SERPER_API_KEY dans l'environnement. 1 requête Serper = 1 crédit ;18~(nb ATS × nb termes × pages) requêtes au total, throttlées.19"""20from __future__ import annotations2122import argparse23import json24import os25import re26import sys27import threading28import time29from concurrent.futures import ThreadPoolExecutor, as_completed30from pathlib import Path3132import requests3334ROOT = Path(__file__).resolve().parent.parent35sys.path.insert(0, str(ROOT))3637SERPER_URL = "https://google.serper.dev/search"38KEY = os.environ.get("SERPER_API_KEY", "")3940# Termes de recherche : villes/régions québécoises + génériques emploi QC.41TERMS = [42 '"Montréal"', '"Québec"', '"Laval, QC"', '"Gatineau"', '"Sherbrooke"',43 '"Longueuil"', '"Trois-Rivières"', '"Saguenay"', '"Lévis"', '"Brossard"',44 '"Drummondville"', '"Granby"', '"Terrebonne"', '"Boucherville"',45 '"Saint-Hyacinthe"', '"Rimouski"', '"Victoriaville"', '"Rouyn-Noranda"',46 '"Sept-Îles"', '"Vaudreuil"', '"Joliette"', '"Saint-Jérôme"',47 '"Baie-Comeau"', '"Val-d\'Or"', '"Alma"', '"Shawinigan"',48 '"Salaberry-de-Valleyfield"', '"Sainte-Julie"', '"Beloeil"', '"Mirabel"',49 '"Blainville"', '"Repentigny"', '"Chicoutimi"', '"Kirkland"',50 '"Pointe-Claire"', '"Dorval"', '"Anjou"', '"Lachine"', '"Longue-Pointe"',51 '"Québec, QC"', 'emploi Québec', 'carrières Québec',52]5354# domaine ATS -> (nom de site pour `site:`, regex d'extraction, ats, groupe(s))55PATTERNS: dict[str, dict] = {56 "lever": {57 "site": "jobs.lever.co",58 "rx": re.compile(r"jobs\.lever\.co/([A-Za-z0-9._-]+)"),59 },60 "greenhouse": {61 "site": "boards.greenhouse.io OR site:job-boards.greenhouse.io",62 "rx": re.compile(r"(?:job-)?boards(?:\.eu)?\.greenhouse\.io/"63 r"(?:embed/job_board\?for=)?([A-Za-z0-9._-]+)"),64 },65 "smartrecruiters": {66 "site": "jobs.smartrecruiters.com",67 "rx": re.compile(r"jobs\.smartrecruiters\.com/(?:oneclick-ui/company/)?"68 r"([A-Za-z0-9._-]+)"),69 },70 "workable": {71 "site": "apply.workable.com",72 "rx": re.compile(r"apply\.workable\.com/(?:api/v\d/accounts/)?"73 r"([A-Za-z0-9._-]+)"),74 },75 "ashby": {76 "site": "jobs.ashbyhq.com",77 "rx": re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9._%-]+)"),78 },79 "recruitee": {80 "site": "recruitee.com",81 "rx": re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com"),82 },83 "breezy": {84 "site": "breezy.hr",85 "rx": re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr"),86 },87 "bamboohr": {88 "site": "bamboohr.com",89 "rx": re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/(?:careers|jobs)"),90 },91 "workday": {92 "site": "myworkdayjobs.com",93 "rx": re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/"94 r"(?:([a-zA-Z]{2}-[a-zA-Z]{2})/)?([A-Za-z0-9_-]+)"),95 },96 "dayforce": {97 "site": "jobs.dayforcehcm.com",98 "rx": re.compile(r"jobs\.dayforcehcm\.com/(?:[a-z]{2}-[A-Za-z]{2}/)?"99 r"([A-Za-z0-9]+)/([A-Za-z0-9_-]+)"),100 },101}102103BAD_SLUGS = {104 "login", "signin", "api", "js", "css", "static", "assets", "app", "www",105 "help", "support", "blog", "about", "privacy", "terms", "search", "jobs",106 "job", "embed", "widget", "oneclick-ui", "company", "careers", "fr", "en",107 "mydayforce", "candidateportal",108}109110_lock = threading.Lock()111_last = [0.0]112_nreq = [0]113114115def serper(q: str, page: int) -> list[dict]:116 with _lock:117 wait = 0.15 - (time.time() - _last[0])118 if wait > 0:119 time.sleep(wait)120 _last[0] = time.time()121 _nreq[0] += 1122 r = requests.post(123 SERPER_URL, timeout=20,124 headers={"X-API-KEY": KEY, "Content-Type": "application/json"},125 json={"q": q, "gl": "ca", "hl": "fr", "num": 20, "page": page})126 r.raise_for_status()127 return r.json().get("organic") or []128129130def known_slugs() -> set[str]:131 known: set[str] = set()132 src = ROOT / "data" / "sources.json"133 if src.exists():134 for s in json.loads(src.read_text())["sources"]:135 known.add(s["id"].lower())136 pat = re.compile(137 r"^\s*(?:ORG|BOARD|COMPANY|TENANT|NS)\s*=\s*['\"]([^'\"]+)", re.M)138 for f in (ROOT / "jobka" / "connectors").glob("*.py"):139 for m in pat.finditer(f.read_text(encoding="utf-8")):140 known.add(m.group(1).lower())141 return known142143144def main() -> None:145 ap = argparse.ArgumentParser()146 ap.add_argument("--out", required=True)147 ap.add_argument("--pages", type=int, default=2)148 ap.add_argument("--ats", default=",".join(PATTERNS))149 ap.add_argument("--workers", type=int, default=6)150 ap.add_argument("--terms-file", default="",151 help="fichier texte : un terme de recherche par ligne "152 "(remplace la liste de villes par défaut)")153 args = ap.parse_args()154 if not KEY:155 sys.exit("SERPER_API_KEY manquant")156 global TERMS157 if args.terms_file:158 TERMS = [l.strip() for l in Path(args.terms_file)159 .read_text(encoding="utf-8").splitlines() if l.strip()]160161 wanted = [a.strip() for a in args.ats.split(",") if a.strip() in PATTERNS]162 known = known_slugs()163 found: dict[tuple, dict] = {}164 flock = threading.Lock()165166 def one(ats: str, term: str, page: int) -> None:167 cfg = PATTERNS[ats]168 q = f"site:{cfg['site']} {term}"169 try:170 organic = serper(q, page)171 except Exception as exc:172 print(f" ! {q} p{page}: {exc}", flush=True)173 return174 for item in organic:175 url = item.get("link") or ""176 m = cfg["rx"].search(url)177 if not m:178 continue179 title = (item.get("title") or "").split(" - ")[0].split(" | ")[0]180 if ats == "workday":181 tenant, host, _cult, site = m.groups()182 if tenant.lower() in known or tenant.lower() in BAD_SLUGS:183 continue184 key = ("workday", tenant.lower())185 entry = {"slug": tenant.lower(), "ats": "workday",186 "tenant": tenant.lower(), "host": host, "site": site,187 "employer": title.strip(), "src_url": url}188 elif ats == "dayforce":189 ns, board = m.group(1), m.group(2)190 if ns.lower() in known or ns.lower() in BAD_SLUGS:191 continue192 key = ("dayforce", ns.lower())193 entry = {"slug": ns, "ats": "dayforce", "board": board194 if board.upper() != "CANDIDATEPORTAL"195 and not board.startswith("jobs") else "",196 "employer": title.strip(), "src_url": url}197 else:198 slug = m.group(1)199 if slug.lower() in known or slug.lower() in BAD_SLUGS:200 continue201 key = (ats, slug.lower())202 entry = {"slug": slug, "ats": ats,203 "employer": title.strip(), "src_url": url}204 with flock:205 found.setdefault(key, entry)206207 jobs = [(a, t, p) for a in wanted for t in TERMS208 for p in range(1, args.pages + 1)]209 print(f"[scout] {len(jobs)} requêtes Serper ({len(wanted)} ATS × "210 f"{len(TERMS)} termes × {args.pages} pages)", flush=True)211 t0 = time.time()212 with ThreadPoolExecutor(max_workers=args.workers) as ex:213 futs = [ex.submit(one, a, t, p) for a, t, p in jobs]214 for i, fut in enumerate(as_completed(futs), 1):215 fut.result()216 if i % 100 == 0:217 print(f"[scout] {i}/{len(jobs)} requêtes, "218 f"{len(found)} candidats, {time.time()-t0:.0f}s",219 flush=True)220221 out = sorted(found.values(), key=lambda e: (e["ats"], e["slug"].lower()))222 Path(args.out).write_text(223 "\n".join(json.dumps(e, ensure_ascii=False) for e in out) + "\n",224 encoding="utf-8")225 per = {}226 for e in out:227 per[e["ats"]] = per.get(e["ats"], 0) + 1228 print(f"[scout] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes, "229 f"{len(out)} candidats -> {args.out}")230 for a, n in sorted(per.items()):231 print(f" {a:16s} {n}")232233234if __name__ == "__main__":235 main()236