HTML 82.1%
Python 14.6%
TypeScript 1.9%
CSS 1%
JavaScript 0.5%
1#!/usr/bin/env python32# =============================================================================3# Job·Ka — Groupe KA4# Auteur : Simon-Pierre Boucher5# Contact : contact@spboucher.ai6# Fichier : scripts/serper_scout.py7# Rôle : Éclaireur Serper — requêtes Google `site:<domaine ATS>` × termes8# québécois, extraction des slugs candidats par motifs d'URL,9# dédoublonnage contre sources.json/connecteurs. Sortie JSONL prête10# pour scripts/probe_ats.py.11# Créé : 2026-08-22 Modifié : 2026-08-2512# =============================================================================13"""Usage :14 python3 scripts/serper_scout.py --out candidats.jsonl \15 [--pages 2] [--ats lever,greenhouse,...] [--workers 6]1617Nécessite SERPER_API_KEY dans l'environnement. 1 requête Serper = 1 crédit ;18~(nb ATS × nb termes × pages) requêtes au total, throttlées.19"""20from __future__ import annotations2122import argparse23import json24import os25import re26import sys27import threading28import time29from concurrent.futures import ThreadPoolExecutor, as_completed30from pathlib import Path3132import requests3334ROOT = Path(__file__).resolve().parent.parent35sys.path.insert(0, str(ROOT))3637SERPER_URL = "https://google.serper.dev/search"38KEY = os.environ.get("SERPER_API_KEY", "")3940# Termes de recherche : villes/régions québécoises + génériques emploi QC.41TERMS = [42 '"Montréal"', '"Québec"', '"Laval, QC"', '"Gatineau"', '"Sherbrooke"',43 '"Longueuil"', '"Trois-Rivières"', '"Saguenay"', '"Lévis"', '"Brossard"',44 '"Drummondville"', '"Granby"', '"Terrebonne"', '"Boucherville"',45 '"Saint-Hyacinthe"', '"Rimouski"', '"Victoriaville"', '"Rouyn-Noranda"',46 '"Sept-Îles"', '"Vaudreuil"', '"Joliette"', '"Saint-Jérôme"',47 '"Baie-Comeau"', '"Val-d\'Or"', '"Alma"', '"Shawinigan"',48 '"Salaberry-de-Valleyfield"', '"Sainte-Julie"', '"Beloeil"', '"Mirabel"',49 '"Blainville"', '"Repentigny"', '"Chicoutimi"', '"Kirkland"',50 '"Pointe-Claire"', '"Dorval"', '"Anjou"', '"Lachine"', '"Longue-Pointe"',51 '"Québec, QC"', 'emploi Québec', 'carrières Québec',52]5354# domaine ATS -> (nom de site pour `site:`, regex d'extraction, ats, groupe(s))55PATTERNS: dict[str, dict] = {56 "lever": {57 "site": "jobs.lever.co",58 "rx": re.compile(r"jobs\.lever\.co/([A-Za-z0-9._-]+)"),59 },60 "greenhouse": {61 "site": "boards.greenhouse.io OR site:job-boards.greenhouse.io",62 "rx": re.compile(r"(?:job-)?boards(?:\.eu)?\.greenhouse\.io/"63 r"(?:embed/job_board\?for=)?([A-Za-z0-9._-]+)"),64 },65 "smartrecruiters": {66 "site": "jobs.smartrecruiters.com",67 "rx": re.compile(r"jobs\.smartrecruiters\.com/(?:oneclick-ui/company/)?"68 r"([A-Za-z0-9._-]+)"),69 },70 "workable": {71 "site": "apply.workable.com",72 "rx": re.compile(r"apply\.workable\.com/(?:api/v\d/accounts/)?"73 r"([A-Za-z0-9._-]+)"),74 },75 "ashby": {76 "site": "jobs.ashbyhq.com",77 "rx": re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9._%-]+)"),78 },79 "recruitee": {80 "site": "recruitee.com",81 "rx": re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com"),82 },83 "breezy": {84 "site": "breezy.hr",85 "rx": re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr"),86 },87 "bamboohr": {88 "site": "bamboohr.com",89 "rx": re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/(?:careers|jobs)"),90 },91 "workday": {92 "site": "myworkdayjobs.com",93 "rx": re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/"94 r"(?:([a-zA-Z]{2}-[a-zA-Z]{2})/)?([A-Za-z0-9_-]+)"),95 },96 "dayforce": {97 "site": "jobs.dayforcehcm.com",98 "rx": re.compile(r"jobs\.dayforcehcm\.com/(?:[a-z]{2}-[A-Za-z]{2}/)?"99 r"([A-Za-z0-9]+)/([A-Za-z0-9_-]+)"),100 },101 # --- ajouts vague E (2026-08-25) -----------------------------------------102 "jazzhr": {103 "site": "applytojob.com",104 "rx": re.compile(r"https?://([a-z0-9-]+)\.applytojob\.com"),105 },106 "teamtailor": {107 "site": "teamtailor.com",108 "rx": re.compile(r"https?://([a-z0-9-]+)\.teamtailor\.com"),109 },110 "zohorecruit": {111 "site": "zohorecruit.com",112 "rx": re.compile(r"https?://([a-z0-9-]+)\.zohorecruit\.com"113 r"/jobs/([^/?#\s\"]+)"),114 },115 "talentnest": {116 "site": "talentnest.com",117 "rx": re.compile(r"https?://([a-z0-9-]+)\.talentnest\.com"),118 },119 "njoyn": {120 "site": "njoyn.com",121 "rx": re.compile(r"https?://([a-z0-9.-]+)\.njoyn\.com/"122 r"(cl\d+|corp)/xweb/xweb\.asp\?[^\"'\s]*"123 r"clid=(\d+)", re.I),124 },125}126127BAD_SLUGS = {128 "login", "signin", "api", "js", "css", "static", "assets", "app", "www",129 "help", "support", "blog", "about", "privacy", "terms", "search", "jobs",130 "job", "embed", "widget", "oneclick-ui", "company", "careers", "fr", "en",131 "mydayforce", "candidateportal",132}133134_lock = threading.Lock()135_last = [0.0]136_nreq = [0]137138139def serper(q: str, page: int) -> list[dict]:140 with _lock:141 wait = 0.15 - (time.time() - _last[0])142 if wait > 0:143 time.sleep(wait)144 _last[0] = time.time()145 _nreq[0] += 1146 r = requests.post(147 SERPER_URL, timeout=20,148 headers={"X-API-KEY": KEY, "Content-Type": "application/json"},149 json={"q": q, "gl": "ca", "hl": "fr", "num": 20, "page": page})150 r.raise_for_status()151 return r.json().get("organic") or []152153154def known_slugs() -> set[str]:155 known: set[str] = set()156 src = ROOT / "data" / "sources.json"157 if src.exists():158 for s in json.loads(src.read_text())["sources"]:159 known.add(s["id"].lower())160 pat = re.compile(161 r"^\s*(?:ORG|BOARD|COMPANY|TENANT|NS|CLID)\s*=\s*['\"]([^'\"]+)",162 re.M)163 for f in (ROOT / "jobka" / "connectors").glob("*.py"):164 for m in pat.finditer(f.read_text(encoding="utf-8")):165 known.add(m.group(1).lower())166 return known167168169def main() -> None:170 ap = argparse.ArgumentParser()171 ap.add_argument("--out", required=True)172 ap.add_argument("--pages", type=int, default=2)173 ap.add_argument("--ats", default=",".join(PATTERNS))174 ap.add_argument("--workers", type=int, default=6)175 ap.add_argument("--terms-file", default="",176 help="fichier texte : un terme de recherche par ligne "177 "(remplace la liste de villes par défaut)")178 args = ap.parse_args()179 if not KEY:180 sys.exit("SERPER_API_KEY manquant")181 global TERMS182 if args.terms_file:183 TERMS = [l.strip() for l in Path(args.terms_file)184 .read_text(encoding="utf-8").splitlines() if l.strip()]185186 wanted = [a.strip() for a in args.ats.split(",") if a.strip() in PATTERNS]187 known = known_slugs()188 found: dict[tuple, dict] = {}189 flock = threading.Lock()190191 def one(ats: str, term: str, page: int) -> None:192 cfg = PATTERNS[ats]193 q = f"site:{cfg['site']} {term}"194 try:195 organic = serper(q, page)196 except Exception as exc:197 print(f" ! {q} p{page}: {exc}", flush=True)198 return199 for item in organic:200 url = item.get("link") or ""201 m = cfg["rx"].search(url)202 if not m:203 continue204 title = (item.get("title") or "").split(" - ")[0].split(" | ")[0]205 if ats == "workday":206 tenant, host, _cult, site = m.groups()207 if tenant.lower() in known or tenant.lower() in BAD_SLUGS:208 continue209 key = ("workday", tenant.lower())210 entry = {"slug": tenant.lower(), "ats": "workday",211 "tenant": tenant.lower(), "host": host, "site": site,212 "employer": title.strip(), "src_url": url}213 elif ats == "dayforce":214 ns, board = m.group(1), m.group(2)215 if ns.lower() in known or ns.lower() in BAD_SLUGS:216 continue217 key = ("dayforce", ns.lower())218 entry = {"slug": ns, "ats": "dayforce", "board": board219 if board.upper() != "CANDIDATEPORTAL"220 and not board.startswith("jobs") else "",221 "employer": title.strip(), "src_url": url}222 elif ats == "zohorecruit":223 slug, page = m.group(1), m.group(2)224 if slug.lower() in known or slug.lower() in BAD_SLUGS \225 or page.lower() in ("rss",):226 continue227 key = ("zohorecruit", slug.lower())228 entry = {"slug": slug, "ats": "zohorecruit", "page": page,229 "employer": title.strip(), "src_url": url}230 elif ats == "njoyn":231 host, cl, clid = m.group(1), m.group(2), m.group(3)232 if clid in known:233 continue234 key = ("njoyn", clid)235 entry = {"slug": clid, "ats": "njoyn", "host": host.lower(),236 "cl": cl, "clid": clid,237 "employer": title.strip(), "src_url": url}238 else:239 slug = m.group(1)240 if slug.lower() in known or slug.lower() in BAD_SLUGS:241 continue242 key = (ats, slug.lower())243 entry = {"slug": slug, "ats": ats,244 "employer": title.strip(), "src_url": url}245 with flock:246 found.setdefault(key, entry)247248 jobs = [(a, t, p) for a in wanted for t in TERMS249 for p in range(1, args.pages + 1)]250 print(f"[scout] {len(jobs)} requêtes Serper ({len(wanted)} ATS × "251 f"{len(TERMS)} termes × {args.pages} pages)", flush=True)252 t0 = time.time()253 with ThreadPoolExecutor(max_workers=args.workers) as ex:254 futs = [ex.submit(one, a, t, p) for a, t, p in jobs]255 for i, fut in enumerate(as_completed(futs), 1):256 fut.result()257 if i % 100 == 0:258 print(f"[scout] {i}/{len(jobs)} requêtes, "259 f"{len(found)} candidats, {time.time()-t0:.0f}s",260 flush=True)261262 out = sorted(found.values(), key=lambda e: (e["ats"], e["slug"].lower()))263 Path(args.out).write_text(264 "\n".join(json.dumps(e, ensure_ascii=False) for e in out) + "\n",265 encoding="utf-8")266 per = {}267 for e in out:268 per[e["ats"]] = per.get(e["ats"], 0) + 1269 print(f"[scout] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes, "270 f"{len(out)} candidats -> {args.out}")271 for a, n in sorted(per.items()):272 print(f" {a:16s} {n}")273274275if __name__ == "__main__":276 main()277