| |
1 | +#!/usr/bin/env python3 |
| |
2 | +# ============================================================================= |
| |
3 | +# Job·Ka — Groupe KA |
| |
4 | +# Auteur : Simon-Pierre Boucher |
| |
5 | +# Contact : contact@spboucher.ai |
| |
6 | +# Fichier : scripts/serper_scout.py |
| |
7 | +# Rôle : Éclaireur Serper — requêtes Google `site:<domaine ATS>` × termes |
| |
8 | +# québécois, extraction des slugs candidats par motifs d'URL, |
| |
9 | +# dédoublonnage contre sources.json/connecteurs. Sortie JSONL prête |
| |
10 | +# pour scripts/probe_ats.py. |
| |
11 | +# Créé : 2026-08-22 Modifié : 2026-08-22 |
| |
12 | +# ============================================================================= |
| |
13 | +"""Usage : |
| |
14 | + python3 scripts/serper_scout.py --out candidats.jsonl \ |
| |
15 | + [--pages 2] [--ats lever,greenhouse,...] [--workers 6] |
| |
16 | + |
| |
17 | +Nécessite SERPER_API_KEY dans l'environnement. 1 requête Serper = 1 crédit ; |
| |
18 | +~(nb ATS × nb termes × pages) requêtes au total, throttlées. |
| |
19 | +""" |
| |
20 | +from __future__ import annotations |
| |
21 | + |
| |
22 | +import argparse |
| |
23 | +import json |
| |
24 | +import os |
| |
25 | +import re |
| |
26 | +import sys |
| |
27 | +import threading |
| |
28 | +import time |
| |
29 | +from concurrent.futures import ThreadPoolExecutor, as_completed |
| |
30 | +from pathlib import Path |
| |
31 | + |
| |
32 | +import requests |
| |
33 | + |
| |
34 | +ROOT = Path(__file__).resolve().parent.parent |
| |
35 | +sys.path.insert(0, str(ROOT)) |
| |
36 | + |
| |
37 | +SERPER_URL = "https://google.serper.dev/search" |
| |
38 | +KEY = os.environ.get("SERPER_API_KEY", "") |
| |
39 | + |
| |
40 | +# Termes de recherche : villes/régions québécoises + génériques emploi QC. |
| |
41 | +TERMS = [ |
| |
42 | + '"Montréal"', '"Québec"', '"Laval, QC"', '"Gatineau"', '"Sherbrooke"', |
| |
43 | + '"Longueuil"', '"Trois-Rivières"', '"Saguenay"', '"Lévis"', '"Brossard"', |
| |
44 | + '"Drummondville"', '"Granby"', '"Terrebonne"', '"Boucherville"', |
| |
45 | + '"Saint-Hyacinthe"', '"Rimouski"', '"Victoriaville"', '"Rouyn-Noranda"', |
| |
46 | + '"Sept-Îles"', '"Vaudreuil"', '"Joliette"', '"Saint-Jérôme"', |
| |
47 | + '"Baie-Comeau"', '"Val-d\'Or"', '"Alma"', '"Shawinigan"', |
| |
48 | + '"Salaberry-de-Valleyfield"', '"Sainte-Julie"', '"Beloeil"', '"Mirabel"', |
| |
49 | + '"Blainville"', '"Repentigny"', '"Chicoutimi"', '"Kirkland"', |
| |
50 | + '"Pointe-Claire"', '"Dorval"', '"Anjou"', '"Lachine"', '"Longue-Pointe"', |
| |
51 | + '"Québec, QC"', 'emploi Québec', 'carrières Québec', |
| |
52 | +] |
| |
53 | + |
| |
54 | +# domaine ATS -> (nom de site pour `site:`, regex d'extraction, ats, groupe(s)) |
| |
55 | +PATTERNS: dict[str, dict] = { |
| |
56 | + "lever": { |
| |
57 | + "site": "jobs.lever.co", |
| |
58 | + "rx": re.compile(r"jobs\.lever\.co/([A-Za-z0-9._-]+)"), |
| |
59 | + }, |
| |
60 | + "greenhouse": { |
| |
61 | + "site": "boards.greenhouse.io OR site:job-boards.greenhouse.io", |
| |
62 | + "rx": re.compile(r"(?:job-)?boards(?:\.eu)?\.greenhouse\.io/" |
| |
63 | + r"(?:embed/job_board\?for=)?([A-Za-z0-9._-]+)"), |
| |
64 | + }, |
| |
65 | + "smartrecruiters": { |
| |
66 | + "site": "jobs.smartrecruiters.com", |
| |
67 | + "rx": re.compile(r"jobs\.smartrecruiters\.com/(?:oneclick-ui/company/)?" |
| |
68 | + r"([A-Za-z0-9._-]+)"), |
| |
69 | + }, |
| |
70 | + "workable": { |
| |
71 | + "site": "apply.workable.com", |
| |
72 | + "rx": re.compile(r"apply\.workable\.com/(?:api/v\d/accounts/)?" |
| |
73 | + r"([A-Za-z0-9._-]+)"), |
| |
74 | + }, |
| |
75 | + "ashby": { |
| |
76 | + "site": "jobs.ashbyhq.com", |
| |
77 | + "rx": re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9._%-]+)"), |
| |
78 | + }, |
| |
79 | + "recruitee": { |
| |
80 | + "site": "recruitee.com", |
| |
81 | + "rx": re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com"), |
| |
82 | + }, |
| |
83 | + "breezy": { |
| |
84 | + "site": "breezy.hr", |
| |
85 | + "rx": re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr"), |
| |
86 | + }, |
| |
87 | + "bamboohr": { |
| |
88 | + "site": "bamboohr.com", |
| |
89 | + "rx": re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/(?:careers|jobs)"), |
| |
90 | + }, |
| |
91 | + "workday": { |
| |
92 | + "site": "myworkdayjobs.com", |
| |
93 | + "rx": re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/" |
| |
94 | + r"(?:([a-zA-Z]{2}-[a-zA-Z]{2})/)?([A-Za-z0-9_-]+)"), |
| |
95 | + }, |
| |
96 | + "dayforce": { |
| |
97 | + "site": "jobs.dayforcehcm.com", |
| |
98 | + "rx": re.compile(r"jobs\.dayforcehcm\.com/(?:[a-z]{2}-[A-Za-z]{2}/)?" |
| |
99 | + r"([A-Za-z0-9]+)/([A-Za-z0-9_-]+)"), |
| |
100 | + }, |
| |
101 | +} |
| |
102 | + |
| |
103 | +BAD_SLUGS = { |
| |
104 | + "login", "signin", "api", "js", "css", "static", "assets", "app", "www", |
| |
105 | + "help", "support", "blog", "about", "privacy", "terms", "search", "jobs", |
| |
106 | + "job", "embed", "widget", "oneclick-ui", "company", "careers", "fr", "en", |
| |
107 | + "mydayforce", "candidateportal", |
| |
108 | +} |
| |
109 | + |
| |
110 | +_lock = threading.Lock() |
| |
111 | +_last = [0.0] |
| |
112 | +_nreq = [0] |
| |
113 | + |
| |
114 | + |
| |
115 | +def serper(q: str, page: int) -> list[dict]: |
| |
116 | + with _lock: |
| |
117 | + wait = 0.15 - (time.time() - _last[0]) |
| |
118 | + if wait > 0: |
| |
119 | + time.sleep(wait) |
| |
120 | + _last[0] = time.time() |
| |
121 | + _nreq[0] += 1 |
| |
122 | + r = requests.post( |
| |
123 | + SERPER_URL, timeout=20, |
| |
124 | + headers={"X-API-KEY": KEY, "Content-Type": "application/json"}, |
| |
125 | + json={"q": q, "gl": "ca", "hl": "fr", "num": 20, "page": page}) |
| |
126 | + r.raise_for_status() |
| |
127 | + return r.json().get("organic") or [] |
| |
128 | + |
| |
129 | + |
| |
130 | +def known_slugs() -> set[str]: |
| |
131 | + known: set[str] = set() |
| |
132 | + src = ROOT / "data" / "sources.json" |
| |
133 | + if src.exists(): |
| |
134 | + for s in json.loads(src.read_text())["sources"]: |
| |
135 | + known.add(s["id"].lower()) |
| |
136 | + pat = re.compile( |
| |
137 | + r"^\s*(?:ORG|BOARD|COMPANY|TENANT|NS)\s*=\s*['\"]([^'\"]+)", re.M) |
| |
138 | + for f in (ROOT / "jobka" / "connectors").glob("*.py"): |
| |
139 | + for m in pat.finditer(f.read_text(encoding="utf-8")): |
| |
140 | + known.add(m.group(1).lower()) |
| |
141 | + return known |
| |
142 | + |
| |
143 | + |
| |
144 | +def main() -> None: |
| |
145 | + ap = argparse.ArgumentParser() |
| |
146 | + ap.add_argument("--out", required=True) |
| |
147 | + ap.add_argument("--pages", type=int, default=2) |
| |
148 | + ap.add_argument("--ats", default=",".join(PATTERNS)) |
| |
149 | + ap.add_argument("--workers", type=int, default=6) |
| |
150 | + ap.add_argument("--terms-file", default="", |
| |
151 | + help="fichier texte : un terme de recherche par ligne " |
| |
152 | + "(remplace la liste de villes par défaut)") |
| |
153 | + args = ap.parse_args() |
| |
154 | + if not KEY: |
| |
155 | + sys.exit("SERPER_API_KEY manquant") |
| |
156 | + global TERMS |
| |
157 | + if args.terms_file: |
| |
158 | + TERMS = [l.strip() for l in Path(args.terms_file) |
| |
159 | + .read_text(encoding="utf-8").splitlines() if l.strip()] |
| |
160 | + |
| |
161 | + wanted = [a.strip() for a in args.ats.split(",") if a.strip() in PATTERNS] |
| |
162 | + known = known_slugs() |
| |
163 | + found: dict[tuple, dict] = {} |
| |
164 | + flock = threading.Lock() |
| |
165 | + |
| |
166 | + def one(ats: str, term: str, page: int) -> None: |
| |
167 | + cfg = PATTERNS[ats] |
| |
168 | + q = f"site:{cfg['site']} {term}" |
| |
169 | + try: |
| |
170 | + organic = serper(q, page) |
| |
171 | + except Exception as exc: |
| |
172 | + print(f" ! {q} p{page}: {exc}", flush=True) |
| |
173 | + return |
| |
174 | + for item in organic: |
| |
175 | + url = item.get("link") or "" |
| |
176 | + m = cfg["rx"].search(url) |
| |
177 | + if not m: |
| |
178 | + continue |
| |
179 | + title = (item.get("title") or "").split(" - ")[0].split(" | ")[0] |
| |
180 | + if ats == "workday": |
| |
181 | + tenant, host, _cult, site = m.groups() |
| |
182 | + if tenant.lower() in known or tenant.lower() in BAD_SLUGS: |
| |
183 | + continue |
| |
184 | + key = ("workday", tenant.lower()) |
| |
185 | + entry = {"slug": tenant.lower(), "ats": "workday", |
| |
186 | + "tenant": tenant.lower(), "host": host, "site": site, |
| |
187 | + "employer": title.strip(), "src_url": url} |
| |
188 | + elif ats == "dayforce": |
| |
189 | + ns, board = m.group(1), m.group(2) |
| |
190 | + if ns.lower() in known or ns.lower() in BAD_SLUGS: |
| |
191 | + continue |
| |
192 | + key = ("dayforce", ns.lower()) |
| |
193 | + entry = {"slug": ns, "ats": "dayforce", "board": board |
| |
194 | + if board.upper() != "CANDIDATEPORTAL" |
| |
195 | + and not board.startswith("jobs") else "", |
| |
196 | + "employer": title.strip(), "src_url": url} |
| |
197 | + else: |
| |
198 | + slug = m.group(1) |
| |
199 | + if slug.lower() in known or slug.lower() in BAD_SLUGS: |
| |
200 | + continue |
| |
201 | + key = (ats, slug.lower()) |
| |
202 | + entry = {"slug": slug, "ats": ats, |
| |
203 | + "employer": title.strip(), "src_url": url} |
| |
204 | + with flock: |
| |
205 | + found.setdefault(key, entry) |
| |
206 | + |
| |
207 | + jobs = [(a, t, p) for a in wanted for t in TERMS |
| |
208 | + for p in range(1, args.pages + 1)] |
| |
209 | + print(f"[scout] {len(jobs)} requêtes Serper ({len(wanted)} ATS × " |
| |
210 | + f"{len(TERMS)} termes × {args.pages} pages)", flush=True) |
| |
211 | + t0 = time.time() |
| |
212 | + with ThreadPoolExecutor(max_workers=args.workers) as ex: |
| |
213 | + futs = [ex.submit(one, a, t, p) for a, t, p in jobs] |
| |
214 | + for i, fut in enumerate(as_completed(futs), 1): |
| |
215 | + fut.result() |
| |
216 | + if i % 100 == 0: |
| |
217 | + print(f"[scout] {i}/{len(jobs)} requêtes, " |
| |
218 | + f"{len(found)} candidats, {time.time()-t0:.0f}s", |
| |
219 | + flush=True) |
| |
220 | + |
| |
221 | + out = sorted(found.values(), key=lambda e: (e["ats"], e["slug"].lower())) |
| |
222 | + Path(args.out).write_text( |
| |
223 | + "\n".join(json.dumps(e, ensure_ascii=False) for e in out) + "\n", |
| |
224 | + encoding="utf-8") |
| |
225 | + per = {} |
| |
226 | + for e in out: |
| |
227 | + per[e["ats"]] = per.get(e["ats"], 0) + 1 |
| |
228 | + print(f"[scout] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes, " |
| |
229 | + f"{len(out)} candidats -> {args.out}") |
| |
230 | + for a, n in sorted(per.items()): |
| |
231 | + print(f" {a:16s} {n}") |
| |
232 | + |
| |
233 | + |
| |
234 | +if __name__ == "__main__": |
| |
235 | + main() |
| |
236 | |