SPB Git forge
spboucher.ai ↗

spb/job-ka

Public
226commits 1branches 0releases
37.5 MBsize
maindefault branch
9 h agolast push
HTML 82.1% Python 14.6% TypeScript 1.9% CSS 1% JavaScript 0.5%

scripts: éclaireur Serper + support Dayforce dans probe_ats (campagne connecteurs du 2026-08-22)

Outils laissés non commités par la session précédente : serper_scout.py
(requêtes site:<ATS> × termes québécois → slugs candidats JSONL) et
probe_ats.py étendu à 10 ATS (ajout dayforce).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 23, 2026) parent 7b871e8

2 changed files +301 −3

modified scripts/probe_ats.py +66 −3
@@ -4,11 +4,11 @@
4 4 # Auteur : Simon-Pierre Boucher
5 5 # Contact : contact@spboucher.ai
6 6 # Fichier : scripts/probe_ats.py
7 −# Rôle : Sondage massif de slugs candidats sur les 9 ATS supportés —
7 +# Rôle : Sondage massif de slugs candidats sur les 10 ATS supportés —
8 8 # un slug est CONFIRMÉ si l'API répond ≥1 offre ET ≥1 offre au
9 9 # Québec (réutilise is_quebec_location du projet). Émet des entrées
10 10 # prêtes pour data/feeds-*.json (scripts/gen_connectors.py).
11 −# Créé : 2026-08-18 Modifié : 2026-08-18
11 +# Créé : 2026-08-18 Modifié : 2026-08-22
12 12 # =============================================================================
13 13 """Sonde des slugs candidats sur les 9 plateformes ATS de Job·Ka.
14 14
@@ -67,7 +67,8 @@ WD_COMMON_SITES = [
67 67 ]
68 68
69 69 DEFAULT_ORDER = ["lever", "greenhouse", "ashby", "smartrecruiters",
70 − "workable", "recruitee", "breezy", "bamboohr", "workday"]
70 + "workable", "recruitee", "breezy", "bamboohr", "dayforce",
71 + "workday"]
71 72
72 73 # --- throttling par domaine d'API -------------------------------------------
73 74 _locks: dict[str, threading.Lock] = {}
@@ -354,11 +355,70 @@ def probe_workday(slug: str, cand: dict) -> dict | None:
354 355 return None
355 356
356 357
358 +def probe_dayforce(slug: str, cand: dict) -> dict | None:
359 + """Dayforce HCM — même API que jobka/connectors/dayforce.py : csrf anonyme
360 + puis recherche paginée. Le namespace est sensible à la casse tel qu'écrit
361 + dans l'URL ; babillard par défaut CANDIDATEPORTAL."""
362 + base = "https://jobs.dayforcehcm.com"
363 + boards = [b for b in (cand.get("board"), "CANDIDATEPORTAL") if b]
364 + sess = requests.Session()
365 + sess.headers["User-Agent"] = UA
366 + with _glock:
367 + lock = _locks.setdefault("jobs.dayforcehcm.com", threading.Lock())
368 + for board in dict.fromkeys(boards):
369 + with lock:
370 + wait = THROTTLE - (time.time() - _last.get("dayforce", 0.0))
371 + if wait > 0:
372 + time.sleep(wait)
373 + try:
374 + csrf = (sess.get(f"{base}/api/auth/csrf",
375 + timeout=TIMEOUT).json() or {}).get(
376 + "csrfToken") or ""
377 + r = sess.post(
378 + f"{base}/api/geo/{slug}/jobposting/search",
379 + timeout=TIMEOUT,
380 + headers={"X-CSRF-TOKEN": csrf,
381 + "Accept": "application/json"},
382 + json={"clientNamespace": slug, "jobBoardCode": board,
383 + "cultureCode": "fr-CA", "paginationStart": 0})
384 + except requests.RequestException:
385 + return None
386 + finally:
387 + _last["dayforce"] = time.time()
388 + _nreq[0] += 1
389 + if r.status_code != 200:
390 + continue
391 + try:
392 + data = r.json()
393 + except ValueError:
394 + continue
395 + postings = data.get("jobPostings") or []
396 + total = int(data.get("maxCount") or len(postings))
397 + if not postings:
398 + continue
399 + labels = []
400 + for p in postings:
401 + locs = []
402 + for l in (p.get("postingLocations") or []):
403 + if (l.get("stateCode") or "").upper() == "QC" and \
404 + (l.get("isoCountryCode") or "CA").upper() == "CA":
405 + locs.append(l.get("formattedAddress")
406 + or f"{l.get('cityName') or ''}, QC")
407 + else:
408 + locs.append(l.get("formattedAddress") or "")
409 + labels.append(locs)
410 + qc, ex = _qc_stats(labels)
411 + return {"found": total, "qc": qc, "qc_ex": ex, "api_name": "",
412 + "ns": slug, "board": board}
413 + return None
414 +
415 +
357 416 PROBES = {
358 417 "lever": probe_lever, "greenhouse": probe_greenhouse, "ashby": probe_ashby,
359 418 "smartrecruiters": probe_smartrecruiters, "workable": probe_workable,
360 419 "recruitee": probe_recruitee, "breezy": probe_breezy,
361 420 "bamboohr": probe_bamboohr, "workday": probe_workday,
421 + "dayforce": probe_dayforce,
362 422 }
363 423
364 424
@@ -410,6 +470,9 @@ def probe_group(group: str, cands: list[dict], stats: dict,
410 470 if ats == "workday":
411 471 feed.update(tenant=res["tenant"], host=res["host"],
412 472 site=res["site"])
473 + elif ats == "dayforce":
474 + feed.update(ns=res["ns"], board=res["board"],
475 + source_id=res["ns"].lower())
413 476 else:
414 477 feed["org"] = res.get("org_override", slug)
415 478 return feed, {}
added scripts/serper_scout.py +235 −0
@@ -0,0 +1,235 @@
1 +#!/usr/bin/env python3
2 +# =============================================================================
3 +# Job·Ka — Groupe KA
4 +# Auteur : Simon-Pierre Boucher
5 +# Contact : contact@spboucher.ai
6 +# Fichier : scripts/serper_scout.py
7 +# Rôle : Éclaireur Serper — requêtes Google `site:<domaine ATS>` × termes
8 +# québécois, extraction des slugs candidats par motifs d'URL,
9 +# dédoublonnage contre sources.json/connecteurs. Sortie JSONL prête
10 +# pour scripts/probe_ats.py.
11 +# Créé : 2026-08-22 Modifié : 2026-08-22
12 +# =============================================================================
13 +"""Usage :
14 + python3 scripts/serper_scout.py --out candidats.jsonl \
15 + [--pages 2] [--ats lever,greenhouse,...] [--workers 6]
16 +
17 +Nécessite SERPER_API_KEY dans l'environnement. 1 requête Serper = 1 crédit ;
18 +~(nb ATS × nb termes × pages) requêtes au total, throttlées.
19 +"""
20 +from __future__ import annotations
21 +
22 +import argparse
23 +import json
24 +import os
25 +import re
26 +import sys
27 +import threading
28 +import time
29 +from concurrent.futures import ThreadPoolExecutor, as_completed
30 +from pathlib import Path
31 +
32 +import requests
33 +
34 +ROOT = Path(__file__).resolve().parent.parent
35 +sys.path.insert(0, str(ROOT))
36 +
37 +SERPER_URL = "https://google.serper.dev/search"
38 +KEY = os.environ.get("SERPER_API_KEY", "")
39 +
40 +# Termes de recherche : villes/régions québécoises + génériques emploi QC.
41 +TERMS = [
42 + '"Montréal"', '"Québec"', '"Laval, QC"', '"Gatineau"', '"Sherbrooke"',
43 + '"Longueuil"', '"Trois-Rivières"', '"Saguenay"', '"Lévis"', '"Brossard"',
44 + '"Drummondville"', '"Granby"', '"Terrebonne"', '"Boucherville"',
45 + '"Saint-Hyacinthe"', '"Rimouski"', '"Victoriaville"', '"Rouyn-Noranda"',
46 + '"Sept-Îles"', '"Vaudreuil"', '"Joliette"', '"Saint-Jérôme"',
47 + '"Baie-Comeau"', '"Val-d\'Or"', '"Alma"', '"Shawinigan"',
48 + '"Salaberry-de-Valleyfield"', '"Sainte-Julie"', '"Beloeil"', '"Mirabel"',
49 + '"Blainville"', '"Repentigny"', '"Chicoutimi"', '"Kirkland"',
50 + '"Pointe-Claire"', '"Dorval"', '"Anjou"', '"Lachine"', '"Longue-Pointe"',
51 + '"Québec, QC"', 'emploi Québec', 'carrières Québec',
52 +]
53 +
54 +# domaine ATS -> (nom de site pour `site:`, regex d'extraction, ats, groupe(s))
55 +PATTERNS: dict[str, dict] = {
56 + "lever": {
57 + "site": "jobs.lever.co",
58 + "rx": re.compile(r"jobs\.lever\.co/([A-Za-z0-9._-]+)"),
59 + },
60 + "greenhouse": {
61 + "site": "boards.greenhouse.io OR site:job-boards.greenhouse.io",
62 + "rx": re.compile(r"(?:job-)?boards(?:\.eu)?\.greenhouse\.io/"
63 + r"(?:embed/job_board\?for=)?([A-Za-z0-9._-]+)"),
64 + },
65 + "smartrecruiters": {
66 + "site": "jobs.smartrecruiters.com",
67 + "rx": re.compile(r"jobs\.smartrecruiters\.com/(?:oneclick-ui/company/)?"
68 + r"([A-Za-z0-9._-]+)"),
69 + },
70 + "workable": {
71 + "site": "apply.workable.com",
72 + "rx": re.compile(r"apply\.workable\.com/(?:api/v\d/accounts/)?"
73 + r"([A-Za-z0-9._-]+)"),
74 + },
75 + "ashby": {
76 + "site": "jobs.ashbyhq.com",
77 + "rx": re.compile(r"jobs\.ashbyhq\.com/([A-Za-z0-9._%-]+)"),
78 + },
79 + "recruitee": {
80 + "site": "recruitee.com",
81 + "rx": re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com"),
82 + },
83 + "breezy": {
84 + "site": "breezy.hr",
85 + "rx": re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr"),
86 + },
87 + "bamboohr": {
88 + "site": "bamboohr.com",
89 + "rx": re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/(?:careers|jobs)"),
90 + },
91 + "workday": {
92 + "site": "myworkdayjobs.com",
93 + "rx": re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/"
94 + r"(?:([a-zA-Z]{2}-[a-zA-Z]{2})/)?([A-Za-z0-9_-]+)"),
95 + },
96 + "dayforce": {
97 + "site": "jobs.dayforcehcm.com",
98 + "rx": re.compile(r"jobs\.dayforcehcm\.com/(?:[a-z]{2}-[A-Za-z]{2}/)?"
99 + r"([A-Za-z0-9]+)/([A-Za-z0-9_-]+)"),
100 + },
101 +}
102 +
103 +BAD_SLUGS = {
104 + "login", "signin", "api", "js", "css", "static", "assets", "app", "www",
105 + "help", "support", "blog", "about", "privacy", "terms", "search", "jobs",
106 + "job", "embed", "widget", "oneclick-ui", "company", "careers", "fr", "en",
107 + "mydayforce", "candidateportal",
108 +}
109 +
110 +_lock = threading.Lock()
111 +_last = [0.0]
112 +_nreq = [0]
113 +
114 +
115 +def serper(q: str, page: int) -> list[dict]:
116 + with _lock:
117 + wait = 0.15 - (time.time() - _last[0])
118 + if wait > 0:
119 + time.sleep(wait)
120 + _last[0] = time.time()
121 + _nreq[0] += 1
122 + r = requests.post(
123 + SERPER_URL, timeout=20,
124 + headers={"X-API-KEY": KEY, "Content-Type": "application/json"},
125 + json={"q": q, "gl": "ca", "hl": "fr", "num": 20, "page": page})
126 + r.raise_for_status()
127 + return r.json().get("organic") or []
128 +
129 +
130 +def known_slugs() -> set[str]:
131 + known: set[str] = set()
132 + src = ROOT / "data" / "sources.json"
133 + if src.exists():
134 + for s in json.loads(src.read_text())["sources"]:
135 + known.add(s["id"].lower())
136 + pat = re.compile(
137 + r"^\s*(?:ORG|BOARD|COMPANY|TENANT|NS)\s*=\s*['\"]([^'\"]+)", re.M)
138 + for f in (ROOT / "jobka" / "connectors").glob("*.py"):
139 + for m in pat.finditer(f.read_text(encoding="utf-8")):
140 + known.add(m.group(1).lower())
141 + return known
142 +
143 +
144 +def main() -> None:
145 + ap = argparse.ArgumentParser()
146 + ap.add_argument("--out", required=True)
147 + ap.add_argument("--pages", type=int, default=2)
148 + ap.add_argument("--ats", default=",".join(PATTERNS))
149 + ap.add_argument("--workers", type=int, default=6)
150 + ap.add_argument("--terms-file", default="",
151 + help="fichier texte : un terme de recherche par ligne "
152 + "(remplace la liste de villes par défaut)")
153 + args = ap.parse_args()
154 + if not KEY:
155 + sys.exit("SERPER_API_KEY manquant")
156 + global TERMS
157 + if args.terms_file:
158 + TERMS = [l.strip() for l in Path(args.terms_file)
159 + .read_text(encoding="utf-8").splitlines() if l.strip()]
160 +
161 + wanted = [a.strip() for a in args.ats.split(",") if a.strip() in PATTERNS]
162 + known = known_slugs()
163 + found: dict[tuple, dict] = {}
164 + flock = threading.Lock()
165 +
166 + def one(ats: str, term: str, page: int) -> None:
167 + cfg = PATTERNS[ats]
168 + q = f"site:{cfg['site']} {term}"
169 + try:
170 + organic = serper(q, page)
171 + except Exception as exc:
172 + print(f" ! {q} p{page}: {exc}", flush=True)
173 + return
174 + for item in organic:
175 + url = item.get("link") or ""
176 + m = cfg["rx"].search(url)
177 + if not m:
178 + continue
179 + title = (item.get("title") or "").split(" - ")[0].split(" | ")[0]
180 + if ats == "workday":
181 + tenant, host, _cult, site = m.groups()
182 + if tenant.lower() in known or tenant.lower() in BAD_SLUGS:
183 + continue
184 + key = ("workday", tenant.lower())
185 + entry = {"slug": tenant.lower(), "ats": "workday",
186 + "tenant": tenant.lower(), "host": host, "site": site,
187 + "employer": title.strip(), "src_url": url}
188 + elif ats == "dayforce":
189 + ns, board = m.group(1), m.group(2)
190 + if ns.lower() in known or ns.lower() in BAD_SLUGS:
191 + continue
192 + key = ("dayforce", ns.lower())
193 + entry = {"slug": ns, "ats": "dayforce", "board": board
194 + if board.upper() != "CANDIDATEPORTAL"
195 + and not board.startswith("jobs") else "",
196 + "employer": title.strip(), "src_url": url}
197 + else:
198 + slug = m.group(1)
199 + if slug.lower() in known or slug.lower() in BAD_SLUGS:
200 + continue
201 + key = (ats, slug.lower())
202 + entry = {"slug": slug, "ats": ats,
203 + "employer": title.strip(), "src_url": url}
204 + with flock:
205 + found.setdefault(key, entry)
206 +
207 + jobs = [(a, t, p) for a in wanted for t in TERMS
208 + for p in range(1, args.pages + 1)]
209 + print(f"[scout] {len(jobs)} requêtes Serper ({len(wanted)} ATS × "
210 + f"{len(TERMS)} termes × {args.pages} pages)", flush=True)
211 + t0 = time.time()
212 + with ThreadPoolExecutor(max_workers=args.workers) as ex:
213 + futs = [ex.submit(one, a, t, p) for a, t, p in jobs]
214 + for i, fut in enumerate(as_completed(futs), 1):
215 + fut.result()
216 + if i % 100 == 0:
217 + print(f"[scout] {i}/{len(jobs)} requêtes, "
218 + f"{len(found)} candidats, {time.time()-t0:.0f}s",
219 + flush=True)
220 +
221 + out = sorted(found.values(), key=lambda e: (e["ats"], e["slug"].lower()))
222 + Path(args.out).write_text(
223 + "\n".join(json.dumps(e, ensure_ascii=False) for e in out) + "\n",
224 + encoding="utf-8")
225 + per = {}
226 + for e in out:
227 + per[e["ats"]] = per.get(e["ats"], 0) + 1
228 + print(f"[scout] terminé en {time.time()-t0:.0f}s — {_nreq[0]} requêtes, "
229 + f"{len(out)} candidats -> {args.out}")
230 + for a, n in sorted(per.items()):
231 + print(f" {a:16s} {n}")
232 +
233 +
234 +if __name__ == "__main__":
235 + main()
236