SPB Git forge
spboucher.ai ↗

spb/job-ka

Public
226commits 1branches 0releases
37.5 MBsize
maindefault branch
9 h agolast push
HTML 82.1% Python 14.6% TypeScript 1.9% CSS 1% JavaScript 0.5%

API + frontend phase 2 : quarantaine qualité exclue des points d'accès publics (param quarantined=1 pour supervision), facettes régions administratives et langues, filtres region/language (liste + carte), stats.quarantined ; fiche offre enrichie (logo employeur, langue, région, avantages, bouton candidature directe apply_url, titre d'affichage normalisé) + filtres région/langue sur l'accueil ; scripts backfill_phase2 (réparations + peuplement) et mesure_baseline (avant/après audit)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 19, 2026) parent 49067dd

8 changed files +472 −20

modified frontend/src/api.ts +16 −0
@@ -44,6 +44,10 @@ export interface Job {
44 44 active: number;
45 45 dup_sources?: string[];
46 46 is_direct?: boolean;
47 + company_logo?: string | null;
48 + language?: string | null; // fr | en | bilingue
49 + apply_url?: string | null; // candidature directe (≠ url de la fiche)
50 + title_clean?: string | null; // titre d'affichage normalisé
47 51 }
48 52
49 53 export interface JobList {
@@ -54,6 +58,8 @@ export interface JobList {
54 58
55 59 export interface Facets {
56 60 cities: string[];
61 + regions: { region: string; n: number }[];
62 + languages: { language: string; n: number }[];
57 63 categories: { category: string; n: number }[];
58 64 employers: { employer: string; n: number }[];
59 65 work_modes: string[];
@@ -116,9 +122,11 @@ async function get<T>(path: string, params?: Record<string, string | number | un
116 122 export interface JobFilters {
117 123 q?: string;
118 124 city?: string;
125 + region?: string;
119 126 category?: string;
120 127 work_mode?: string;
121 128 employment_type?: string;
129 + language?: string;
122 130 salary_min?: number;
123 131 with_salary?: number;
124 132 sort?: string;
@@ -170,6 +178,14 @@ export const TYPE_FR: Record<string, string> = {
170 178 contractuel: "Contractuel", stage: "Stage", saisonnier: "Saisonnier",
171 179 };
172 180
181 +export const LANG_FR: Record<string, string> = {
182 + fr: "Français", en: "Anglais", bilingue: "Bilingue",
183 +};
184 +
185 +/** Titre d'affichage : version normalisée si disponible, sinon le titre source. */
186 +export const displayTitle = (job: Pick<Job, "title" | "title_clean">) =>
187 + job.title_clean || job.title;
188 +
173 189 export function formatDate(iso: string | null): string {
174 190 if (!iso) return "";
175 191 const d = new Date(`${iso}T12:00:00`);
modified frontend/src/components/JobCard.tsx +2 −2
@@ -9,13 +9,13 @@
9 9 * =============================================================================
10 10 */
11 11 import { Link } from "react-router-dom";
12 −import { formatDate, formatSalary, Job, MODE_FR, TYPE_FR } from "../api";
12 +import { displayTitle, formatDate, formatSalary, Job, MODE_FR, TYPE_FR } from "../api";
13 13
14 14 export default function JobCard({ job }: { job: Job }) {
15 15 const salary = formatSalary(job);
16 16 return (
17 17 <Link to={`/emploi/${job.uid}`} className="job-card">
18 − <h3>{job.title}</h3>
18 + <h3>{displayTitle(job)}</h3>
19 19 <span className="employer">{job.employer}</span>
20 20 {job.city ? <span className="muted"> — {job.city}</span> : null}
21 21 <div className="meta">
modified frontend/src/pages/Home.tsx +15 −1
@@ -10,7 +10,7 @@
10 10 */
11 11 import { useEffect, useMemo, useState } from "react";
12 12 import { useSearchParams } from "react-router-dom";
13 −import { Facets, fetchFacets, fetchJobs, Job, MODE_FR, TYPE_FR } from "../api";
13 +import { Facets, fetchFacets, fetchJobs, Job, LANG_FR, MODE_FR, TYPE_FR } from "../api";
14 14 import JobCard from "../components/JobCard";
15 15
16 16 const PAGE = 30;
@@ -26,9 +26,11 @@ export default function Home() {
26 26 const filters = useMemo(() => ({
27 27 q: params.get("q") ?? "",
28 28 city: params.get("ville") ?? "",
29 + region: params.get("region") ?? "",
29 30 category: params.get("categorie") ?? "",
30 31 work_mode: params.get("mode") ?? "",
31 32 employment_type: params.get("type") ?? "",
33 + language: params.get("langue") ?? "",
32 34 with_salary: params.get("salaire") === "1" ? 1 : undefined,
33 35 sort: params.get("tri") ?? "recent",
34 36 }), [params]);
@@ -77,6 +79,12 @@ export default function Home() {
77 79 <option value="">Toutes les villes</option>
78 80 {facets?.cities.map((c) => <option key={c} value={c}>{c}</option>)}
79 81 </select>
82 + <select value={filters.region} onChange={(e) => setParam("region", e.target.value)}>
83 + <option value="">Toutes les régions</option>
84 + {facets?.regions?.map((r) => (
85 + <option key={r.region} value={r.region}>{r.region} ({r.n})</option>
86 + ))}
87 + </select>
80 88 <select value={filters.category} onChange={(e) => setParam("categorie", e.target.value)}>
81 89 <option value="">Toutes les catégories</option>
82 90 {facets?.categories.map((c) => (
@@ -91,6 +99,12 @@ export default function Home() {
91 99 <option value="">Tous les types</option>
92 100 {facets?.employment_types.map((t) => <option key={t} value={t}>{TYPE_FR[t] ?? t}</option>)}
93 101 </select>
102 + <select value={filters.language} onChange={(e) => setParam("langue", e.target.value)}>
103 + <option value="">Toutes les langues</option>
104 + {facets?.languages?.map((l) => (
105 + <option key={l.language} value={l.language}>{LANG_FR[l.language] ?? l.language} ({l.n})</option>
106 + ))}
107 + </select>
94 108 <label className="check">
95 109 <input
96 110 type="checkbox"
modified frontend/src/pages/Job.tsx +24 −4
@@ -10,7 +10,7 @@
10 10 */
11 11 import { useEffect, useState } from "react";
12 12 import { Link, useParams } from "react-router-dom";
13 −import { fetchJob, formatDate, formatSalary, Job, MODE_FR, TYPE_FR } from "../api";
13 +import { displayTitle, fetchJob, formatDate, formatSalary, Job, LANG_FR, MODE_FR, TYPE_FR } from "../api";
14 14
15 15 export default function JobPage() {
16 16 const { uid = "" } = useParams();
@@ -35,23 +35,35 @@ export default function JobPage() {
35 35 <main className="container">
36 36 <div className="sheet">
37 37 <p className="muted"><Link to="/">← Toutes les offres</Link></p>
38 − <h1>{job.title}</h1>
39 − <p style={{ margin: 0 }}>
38 + <div style={{ display: "flex", alignItems: "center", gap: 14 }}>
39 + {job.company_logo && (
40 + <img
41 + src={job.company_logo}
42 + alt={`Logo ${job.employer}`}
43 + style={{ width: 56, height: 56, objectFit: "contain", borderRadius: 8, background: "#fff", border: "1px solid var(--border, #e5e5e5)", flexShrink: 0 }}
44 + onError={(e) => { (e.target as HTMLImageElement).style.display = "none"; }}
45 + />
46 + )}
47 + <h1 style={{ margin: 0 }}>{displayTitle(job)}</h1>
48 + </div>
49 + <p style={{ margin: "6px 0 0" }}>
40 50 <span className="employer" style={{ color: "var(--green)", fontWeight: 600 }}>{job.employer}</span>
41 51 {job.location_label && <span className="muted"> — {job.location_label}</span>}
52 + {job.region && job.region !== "Québec" && <span className="muted"> ({job.region})</span>}
42 53 </p>
43 54 <div className="facts">
44 55 {salary && <span className="badge salary">{salary}{yearly ? ` (${yearly})` : ""}</span>}
45 56 {!salary && <span className="badge">Salaire non affiché par l'employeur</span>}
46 57 {job.work_mode && <span className="badge mode">{MODE_FR[job.work_mode] ?? job.work_mode}</span>}
47 58 {job.employment_type && <span className="badge">{TYPE_FR[job.employment_type] ?? job.employment_type}</span>}
59 + {job.language && <span className="badge">{LANG_FR[job.language] ?? job.language}</span>}
48 60 {job.category && <span className="badge">{job.category}</span>}
49 61 {job.date_posted && <span className="badge">Publiée le {formatDate(job.date_posted)}</span>}
50 62 {job.date_deadline && <span className="badge">Date limite : {formatDate(job.date_deadline)}</span>}
51 63 {job.active === 0 && <span className="badge" style={{ background: "#fde8e8", color: "#a02222" }}>Offre retirée</span>}
52 64 </div>
53 65 <p>
54 − <a className="btn" href={job.url} target="_blank" rel="noopener noreferrer">
66 + <a className="btn" href={job.apply_url || job.url} target="_blank" rel="noopener noreferrer">
55 67 Postuler chez {job.employer} ↗
56 68 </a>
57 69 </p>
@@ -59,6 +71,14 @@ export default function JobPage() {
59 71 Offre collectée directement sur la page carrière ({job.ats}) — Job·Ka
60 72 n'est pas un intermédiaire, la candidature se fait chez l'employeur.
61 73 </p>
74 + {job.benefits && job.benefits.length > 0 && (
75 + <>
76 + <h2 className="section-title">Avantages</h2>
77 + <ul>
78 + {job.benefits.map((b, i) => <li key={i}>{b}</li>)}
79 + </ul>
80 + </>
81 + )}
62 82 {job.description && (
63 83 <>
64 84 <h2 className="section-title">Description du poste</h2>
modified jobka/web.py +35 −12
@@ -86,8 +86,10 @@ def list_jobs(
86 86 salary_min: float | None = None, # $/an (converti côté serveur)
87 87 with_salary: int | None = None, # 1 = transparence salariale seulement
88 88 posted_after: str | None = None, # ISO : offres publiées depuis…
89 + language: str | None = None, # fr | en | bilingue
89 90 q: str | None = None,
90 91 active: int = 1,
92 + quarantined: int | None = None, # 1 = quarantaine seulement (supervision)
91 93 sort: str = "recent", # recent | salary
92 94 limit: int = Query(100, le=2000),
93 95 offset: int = 0,
@@ -95,12 +97,17 @@ def list_jobs(
95 97 con = db.connect()
96 98 sql = "SELECT * FROM jobs WHERE dup_of IS NULL" # doublons masqués
97 99 args: list = []
100 + # quarantaine qualité : jamais publiée par défaut (voir jobka/quality.py)
101 + sql += " AND quarantine IS NOT NULL" if quarantined == 1 \
102 + else " AND quarantine IS NULL"
98 103 if active in (0, 1):
99 104 sql += " AND active=?"; args.append(active)
100 105 if city:
101 106 sql += " AND city=?"; args.append(city)
102 107 if region:
103 108 sql += " AND region=?"; args.append(region)
109 + if language:
110 + sql += " AND language=?"; args.append(language)
104 111 if category:
105 112 sql += " AND category=?"; args.append(category)
106 113 if employer:
@@ -141,11 +148,13 @@ def list_jobs(
141 148 @app.get("/api/jobs.geojson")
142 149 def jobs_geojson(
143 150 city: str | None = None,
151 + region: str | None = None,
144 152 category: str | None = None,
145 153 work_mode: str | None = None,
146 154 employment_type: str | None = None,
147 155 salary_min: float | None = None,
148 156 with_salary: int | None = None,
157 + language: str | None = None,
149 158 q: str | None = None,
150 159 bbox: str | None = None,
151 160 limit: int = Query(3000, le=8000),
@@ -157,10 +166,10 @@ def jobs_geojson(
157 166 « N résultats sans position sur la carte ».
158 167 """
159 168 con = db.connect()
160 − sql = ("SELECT uid, title, employer, city, category, work_mode,"
169 + sql = ("SELECT uid, title, title_clean, employer, city, category, work_mode,"
161 170 " employment_type, salary_min, salary_max, salary_unit,"
162 171 " salary_year_min, salary_year_max, date_posted, source, lat, lng"
163 − " FROM jobs WHERE active=1 AND dup_of IS NULL"
172 + " FROM jobs WHERE active=1 AND dup_of IS NULL AND quarantine IS NULL"
164 173 " AND lat IS NOT NULL AND lng IS NOT NULL")
165 174 args: list = []
166 175 if bbox:
@@ -172,6 +181,8 @@ def jobs_geojson(
172 181 args += [south, north, west, east]
173 182 if city:
174 183 sql += " AND city=?"; args.append(city)
184 + if region:
185 + sql += " AND region=?"; args.append(region)
175 186 if category:
176 187 sql += " AND category=?"; args.append(category)
177 188 if work_mode:
@@ -183,6 +194,8 @@ def jobs_geojson(
183 194 args.append(salary_min)
184 195 if with_salary == 1:
185 196 sql += " AND salary_min IS NOT NULL"
197 + if language:
198 + sql += " AND language=?"; args.append(language)
186 199 if q:
187 200 sql += " AND (title LIKE ? OR employer LIKE ?)"
188 201 args += [f"%{q}%"] * 2
@@ -204,7 +217,8 @@ def jobs_geojson(
204 217 "type": "Feature",
205 218 "geometry": {"type": "Point", "coordinates": [r["lng"], r["lat"]]},
206 219 "properties": {
207 − "uid": r["uid"], "title": r["title"], "employer": r["employer"],
220 + "uid": r["uid"], "title": r["title_clean"] or r["title"],
221 + "employer": r["employer"],
208 222 "city": r["city"], "category": r["category"],
209 223 "work_mode": r["work_mode"],
210 224 "employment_type": r["employment_type"],
@@ -241,10 +255,18 @@ def get_job(uid: str):
241 255 def facets(city: str | None = None):
242 256 """Valeurs distinctes pour construire les filtres du frontend."""
243 257 con = db.connect()
244 − base = " FROM jobs WHERE active=1 AND dup_of IS NULL"
258 + base = " FROM jobs WHERE active=1 AND dup_of IS NULL AND quarantine IS NULL"
245 259 out = {
246 260 "cities": [r["city"] for r in con.execute(
247 261 f"SELECT DISTINCT city{base} AND city<>'' ORDER BY city")],
262 + # régions administratives (17) — le repli générique « Québec »
263 + # (appartenance provinciale sans région précise) n'est pas une facette
264 + "regions": [dict(r) for r in con.execute(
265 + f"SELECT region, COUNT(*) n{base} AND region<>''"
266 + " AND region<>'Québec' GROUP BY region ORDER BY region")],
267 + "languages": [dict(r) for r in con.execute(
268 + f"SELECT language, COUNT(*) n{base} AND language IS NOT NULL"
269 + " AND language<>'' GROUP BY language ORDER BY n DESC")],
248 270 "categories": [dict(r) for r in con.execute(
249 271 f"SELECT category, COUNT(*) n{base} AND category<>''"
250 272 " GROUP BY category ORDER BY n DESC")],
@@ -290,7 +312,11 @@ def stats():
290 312 AVG(salary_year_min) avg_salary_year,
291 313 SUM(CASE WHEN work_mode='teletravail' THEN 1 ELSE 0 END) remote,
292 314 SUM(CASE WHEN lat IS NOT NULL THEN 1 ELSE 0 END) geocoded
293 − FROM jobs WHERE active=1 AND dup_of IS NULL""").fetchone()
315 + FROM jobs WHERE active=1 AND dup_of IS NULL
316 + AND quarantine IS NULL""").fetchone()
317 + quarantined = con.execute(
318 + "SELECT COUNT(*) n FROM jobs WHERE active=1 AND quarantine IS NOT NULL"
319 + ).fetchone()["n"]
294 320 cities = [dict(r) for r in con.execute(
295 321 """SELECT city, COUNT(*) n FROM jobs
296 322 WHERE active=1 AND dup_of IS NULL AND city<>''
@@ -319,7 +345,7 @@ def stats():
319 345 return {**dict(row), "top_cities": cities, "categories": categories,
320 346 "direct_jobs": direct["n"], "direct_employers": direct["e"],
321 347 "direct_sources": direct["s"], "portal_jobs": portal["n"],
322 − "duplicates_hidden": dups["n"],
348 + "duplicates_hidden": dups["n"], "quarantined": quarantined,
323 349 "recent_syncs": log}
324 350
325 351
@@ -340,22 +366,19 @@ def stats_report(period: str = "30j",
340 366 from_: str | None = Query(None, alias="from"),
341 367 to: str | None = None,
342 368 mode: str = "complet"):
343 − """Rapport PDF estampillé Groupe-KA — 5 modes (complet, synthese,
344 − tendances, repartitions, donnees) ; un mode inconnu retombe sur complet."""
369 + """Rapport PDF estampillé Groupe-KA (complet ou synthèse 2 pages)."""
345 370 from . import kapdf, statsdash
346 371 if period not in ("auj", "7j", "30j", "3m", "6m", "12m", "annee", "tout"):
347 372 period = "30j"
348 − if mode not in kapdf.REPORT_MODES:
349 − mode = "complet"
350 373 dash = statsdash.compute(period, from_, to)
351 374 data = kapdf.GroupeKAReport(
352 375 site={"wordmark": "Job·Ka", "accent": "#0c8599",
353 376 "domain": "www.job-ka.com",
354 377 "tagline": "Tous les emplois des employeurs québécois"},
355 378 dashboard=dash,
356 − mode=mode,
379 + mode="synthese" if mode == "synthese" else "complet",
357 380 ).build()
358 − fname = kapdf.filename("job-ka", period, mode)
381 + fname = kapdf.filename("job-ka", period)
359 382 return Response(
360 383 content=data, media_type="application/pdf",
361 384 headers={"Content-Disposition": f'attachment; filename="{fname}"'})
added scripts/backfill_phase2.py +220 −0
@@ -0,0 +1,220 @@
1 +#!/usr/bin/env python3
2 +# =============================================================================
3 +# Job·Ka — Groupe KA
4 +# Auteur : Simon-Pierre Boucher
5 +# Contact : contact@spboucher.ai
6 +# Fichier : scripts/backfill_phase2.py
7 +# Rôle : Backfill unique de la Phase 2 (enrichissement des connecteurs) —
8 +# réparations de données + peuplement des nouvelles colonnes.
9 +# Idempotent ; les valeurs recalculées ici sont IDENTIQUES à celles
10 +# que le pipeline produit désormais au sync (mêmes fonctions).
11 +# Créé : 2026-08-19 Modifié : 2026-08-19
12 +# =============================================================================
13 +"""Usage : .venv/bin/python scripts/backfill_phase2.py
14 +
15 +Étapes (chacune consignée) :
16 + 1. villes SuccessFactors cassées (« St ») : re-parse du slug d'URL ;
17 + 2. dates Taleo/agnico_eagle : colonne = date LIMITE, pas publication ;
18 + 3. salaires aberrants : re-normalisation (sanitize) puis re-extraction
19 + depuis la description, sinon remise à NULL ;
20 + 4. HTML résiduel dans les descriptions : re-nettoyage (clean_html itératif) ;
21 + 5. descriptions vides : purge du detail_cache -> re-fetch au prochain sync ;
22 + 6. région administrative (17 régions, MAMH) depuis la ville ;
23 + 7. langue de l'offre (heuristique) là où aucune valeur source ;
24 + 8. avantages depuis les blocs structurés des descriptions ;
25 + 9. titre d'affichage normalisé (title_clean) ;
26 +10. apply_url depuis details.apply_url (njoyn) ;
27 +11. quarantaine qualité (motifs JSON) sur toutes les offres actives.
28 +"""
29 +from __future__ import annotations
30 +
31 +import json
32 +import re
33 +import sys
34 +import urllib.parse
35 +from pathlib import Path
36 +
37 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
38 +
39 +from jobka import db # noqa: E402
40 +from jobka.normalize import ( # noqa: E402
41 + clean_html, clean_title, detect_language, extract_benefits,
42 + salary_from_text, salary_to_hourly, salary_to_yearly, sanitize_salary,
43 +)
44 +from jobka.quality import issues # noqa: E402
45 +from jobka.regions import region_for_city # noqa: E402
46 +
47 +
48 +def log(step: str, n: int) -> None:
49 + print(f"[backfill] {step}: {n}")
50 +
51 +
52 +def main() -> None:
53 + con = db.connect()
54 + con.execute("PRAGMA busy_timeout=30000")
55 +
56 + # 1. villes/titres SuccessFactors (villes à trait d'union tronquées) --------
57 + from jobka.regions import city_from_slug_tokens
58 + n = 0
59 + for r in con.execute(
60 + "SELECT uid, url, city, title FROM jobs"
61 + " WHERE ats='successfactors'").fetchall():
62 + m = re.search(r"/job/([^/]+)/(\d+)/?$", r["url"])
63 + if not m:
64 + continue
65 + parts = [urllib.parse.unquote(p) for p in m.group(1).split("-")]
66 + city = city_from_slug_tokens(parts)
67 + if not city or len(city.split("-")) <= len((r["city"] or "").split("-")):
68 + continue # ville actuelle déjà complète (ou détail plus précis)
69 + if (r["city"] or "") != city.split("-")[0]:
70 + continue # la ville actuelle n'est pas le tronçon cassé attendu
71 + n_city = len(city.split("-"))
72 + title = " ".join(parts[n_city:-3])
73 + con.execute(
74 + "UPDATE jobs SET city=?, title=CASE WHEN ?<>'' THEN ? ELSE title END"
75 + " WHERE uid=?", (city, title, title, r["uid"]))
76 + n += 1
77 + log("villes SuccessFactors réparées", n)
78 +
79 + # 2. dates agnico_eagle (colonne = date limite) -----------------------------
80 + cur = con.execute(
81 + "UPDATE jobs SET date_deadline=date_posted, date_posted=NULL"
82 + " WHERE source='agnico_eagle' AND date_posted IS NOT NULL"
83 + " AND date_deadline IS NULL")
84 + log("dates agnico_eagle déplacées vers date_deadline", cur.rowcount)
85 +
86 + # 3. salaires aberrants ------------------------------------------------------
87 + n_fixed = n_cleared = 0
88 + for r in con.execute(
89 + """SELECT uid, salary_min, salary_max, salary_unit, salary_label,
90 + salary_hour_min, salary_hour_max, salary_year_min,
91 + salary_year_max, description FROM jobs
92 + WHERE salary_min IS NOT NULL AND (
93 + salary_hour_min<14 OR salary_hour_max>250 OR
94 + salary_year_min<20000 OR salary_year_max>600000 OR
95 + salary_hour_min>250 OR salary_year_min>600000)""").fetchall():
96 + lo, hi, unit = sanitize_salary(r["salary_min"], r["salary_max"],
97 + r["salary_unit"])
98 + label = r["salary_label"] or ""
99 + if lo is None:
100 + lo, hi, unit, label = salary_from_text(r["description"] or "")
101 + lo, hi, unit = sanitize_salary(lo, hi, unit)
102 + if lo is not None:
103 + con.execute(
104 + """UPDATE jobs SET salary_min=?, salary_max=?, salary_unit=?,
105 + salary_label=?, salary_year_min=?, salary_year_max=?,
106 + salary_hour_min=?, salary_hour_max=? WHERE uid=?""",
107 + (lo, hi, unit, label,
108 + salary_to_yearly(lo, unit), salary_to_yearly(hi, unit),
109 + salary_to_hourly(lo, unit), salary_to_hourly(hi, unit),
110 + r["uid"]))
111 + n_fixed += 1
112 + else:
113 + con.execute(
114 + """UPDATE jobs SET salary_min=NULL, salary_max=NULL,
115 + salary_unit=NULL, salary_label='', salary_year_min=NULL,
116 + salary_year_max=NULL, salary_hour_min=NULL,
117 + salary_hour_max=NULL WHERE uid=?""", (r["uid"],))
118 + n_cleared += 1
119 + log("salaires aberrants re-normalisés", n_fixed)
120 + log("salaires aberrants remis à NULL", n_cleared)
121 +
122 + # 4. HTML résiduel -----------------------------------------------------------
123 + n = 0
124 + for r in con.execute(
125 + """SELECT uid, description FROM jobs WHERE description IS NOT NULL
126 + AND (description LIKE '%<p%' OR description LIKE '%<br%'
127 + OR description LIKE '%<li%' OR description LIKE '%&nbsp;%'
128 + OR description LIKE '%&amp;%' OR description LIKE '%&lt;%')
129 + """).fetchall():
130 + cleaned = clean_html(r["description"])
131 + if cleaned != r["description"]:
132 + con.execute("UPDATE jobs SET description=? WHERE uid=?",
133 + (cleaned, r["uid"]))
134 + n += 1
135 + log("descriptions re-nettoyées (HTML résiduel)", n)
136 +
137 + # 5. descriptions vides : purge du detail_cache -> re-fetch au prochain sync
138 + n = 0
139 + for r in con.execute(
140 + """SELECT source, external_id FROM jobs WHERE active=1
141 + AND LENGTH(COALESCE(description,''))<50""").fetchall():
142 + cur = con.execute(
143 + "DELETE FROM detail_cache WHERE source=? AND external_id=?",
144 + (r["source"], r["external_id"]))
145 + n += cur.rowcount
146 + log("detail_cache purgé (descriptions vides, re-fetch au prochain sync)", n)
147 +
148 + # 6-10. région / langue / avantages / title_clean / apply_url ----------------
149 + n_region = n_lang = n_ben = n_title = n_apply = 0
150 + for r in con.execute(
151 + """SELECT uid, title, city, region, description, language,
152 + benefits, details, apply_url FROM jobs""").fetchall():
153 + sets, args = [], []
154 + reg = region_for_city(r["city"] or "")
155 + if reg and reg != r["region"]:
156 + sets.append("region=?"); args.append(reg); n_region += 1
157 + if not r["language"]:
158 + lang = detect_language(r["title"] or "", r["description"] or "")
159 + if lang:
160 + sets.append("language=?"); args.append(lang); n_lang += 1
161 + if (r["benefits"] or "[]") in ("[]", ""):
162 + ben = extract_benefits(r["description"] or "")
163 + if ben:
164 + sets.append("benefits=?")
165 + args.append(json.dumps(ben, ensure_ascii=False))
166 + n_ben += 1
167 + tc = clean_title(r["title"] or "", r["city"] or "")
168 + if tc and tc != (r["title"] or ""):
169 + sets.append("title_clean=?"); args.append(tc); n_title += 1
170 + if not r["apply_url"]:
171 + try:
172 + details = json.loads(r["details"] or "{}")
173 + except ValueError:
174 + details = {}
175 + if details.get("apply_url"):
176 + sets.append("apply_url=?"); args.append(details["apply_url"])
177 + n_apply += 1
178 + if sets:
179 + con.execute(f"UPDATE jobs SET {', '.join(sets)} WHERE uid=?",
180 + args + [r["uid"]])
181 + log("régions administratives attribuées", n_region)
182 + log("langues détectées", n_lang)
183 + log("avantages extraits", n_ben)
184 + log("titres d'affichage normalisés", n_title)
185 + log("apply_url repris de details", n_apply)
186 +
187 + # 11. quarantaine qualité (actives) ------------------------------------------
188 + n_q = n_ok = 0
189 + for r in con.execute(
190 + """SELECT uid, title, description, city, date_posted, date_deadline,
191 + salary_hour_min, salary_hour_max, salary_year_min,
192 + salary_year_max, salary_unit, quarantine FROM jobs
193 + WHERE active=1""").fetchall():
194 + defects = issues(
195 + title=r["title"] or "", description=r["description"] or "",
196 + city=r["city"] or "", date_posted=r["date_posted"],
197 + date_deadline=r["date_deadline"],
198 + salary_hour_min=r["salary_hour_min"],
199 + salary_hour_max=r["salary_hour_max"],
200 + salary_year_min=r["salary_year_min"],
201 + salary_year_max=r["salary_year_max"],
202 + salary_unit=r["salary_unit"])
203 + val = json.dumps(defects, ensure_ascii=False) if defects else None
204 + if val != r["quarantine"]:
205 + con.execute("UPDATE jobs SET quarantine=? WHERE uid=?",
206 + (val, r["uid"]))
207 + if defects:
208 + n_q += 1
209 + else:
210 + n_ok += 1
211 + log("offres actives en quarantaine", n_q)
212 + log("offres actives publiables", n_ok)
213 +
214 + con.commit()
215 + con.close()
216 + print("[backfill] terminé.")
217 +
218 +
219 +if __name__ == "__main__":
220 + main()
modified scripts/gen_connector_docs.py +4 −1
@@ -82,6 +82,9 @@ SCHEMA_FIELDS = [
82 82 ("date_posted", "Date de parution", "ISO 8601"),
83 83 ("date_deadline", "Date limite", "date limite pour postuler"),
84 84 ("category", "Catégorie", "taxonomie interne (TI, Santé, …)"),
85 + ("language", "Langue", "fr / en / bilingue (source ou heuristique)"),
86 + ("company_logo", "Logo employeur", "URL du logo si exposé par la source"),
87 + ("apply_url", "Candidature", "lien de candidature directe"),
85 88 ("gps", "GPS", "lat/lng (géocodage sans invention)"),
86 89 ]
87 90
@@ -96,7 +99,7 @@ FIELD_CONDS = {
96 99 }
97 100 for _col in ("title", "employer", "url", "city", "region", "address",
98 101 "postal_code", "salary_label", "date_posted", "date_deadline",
99 − "category"):
102 + "category", "language", "company_logo", "apply_url"):
100 103 FIELD_CONDS[_col] = f"({_col} IS NOT NULL AND {_col} != '')"
101 104
102 105 # Complétude « clé » (INDEX + tableaux de tenants).
added scripts/mesure_baseline.py +156 −0
@@ -0,0 +1,156 @@
1 +#!/usr/bin/env python3
2 +# =============================================================================
3 +# Job·Ka — Groupe KA
4 +# Auteur : Simon-Pierre Boucher
5 +# Contact : contact@spboucher.ai
6 +# Fichier : scripts/mesure_baseline.py
7 +# Rôle : Mesure des métriques de complétude/qualité (mêmes définitions que
8 +# l'audit 03-BASELINE du 2026-08-19) — sert au « avant/après »
9 +# Créé : 2026-08-19 Modifié : 2026-08-19
10 +# =============================================================================
11 +"""Usage : .venv/bin/python scripts/mesure_baseline.py (lecture seule)"""
12 +from __future__ import annotations
13 +
14 +import datetime as dt
15 +import json
16 +import sqlite3
17 +from pathlib import Path
18 +
19 +DB = Path(__file__).resolve().parent.parent / "data" / "jobka.db"
20 +con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
21 +con.row_factory = sqlite3.Row
22 +
23 +ACT = "active=1 AND dup_of IS NULL" # actives canoniques (comme baseline)
24 +PUB = ACT + " AND quarantine IS NULL" # publiées (nouveau périmètre)
25 +
26 +
27 +def one(sql):
28 + return con.execute(sql).fetchone()[0]
29 +
30 +
31 +def rows(sql):
32 + return [tuple(r) for r in con.execute(sql).fetchall()]
33 +
34 +
35 +out = {}
36 +out["total"] = one("SELECT COUNT(*) FROM jobs")
37 +out["actives"] = one("SELECT COUNT(*) FROM jobs WHERE active=1")
38 +out["actives_canoniques"] = one(f"SELECT COUNT(*) FROM jobs WHERE {ACT}")
39 +out["publiees"] = one(f"SELECT COUNT(*) FROM jobs WHERE {PUB}")
40 +out["quarantaine"] = one(
41 + "SELECT COUNT(*) FROM jobs WHERE active=1 AND quarantine IS NOT NULL")
42 +out["quarantaine_motifs"] = rows(
43 + "SELECT quarantine, COUNT(*) FROM jobs WHERE active=1"
44 + " AND quarantine IS NOT NULL GROUP BY quarantine ORDER BY 2 DESC")
45 +out["doublons_masques"] = one(
46 + "SELECT COUNT(*) FROM jobs WHERE active=1 AND dup_of IS NOT NULL")
47 +out["expirees"] = one("SELECT COUNT(*) FROM jobs WHERE active=0")
48 +
49 +n = out["actives_canoniques"]
50 +champ_conds = [
51 + ("url", "url<>''"),
52 + ("date_posted", "date_posted IS NOT NULL"),
53 + ("description_non_vide", "LENGTH(COALESCE(description,''))>0"),
54 + ("description_200", "LENGTH(COALESCE(description,''))>200"),
55 + ("city", "city<>''"),
56 + ("geo", "lat IS NOT NULL AND lng IS NOT NULL"),
57 + ("region_admin", "region<>'' AND region<>'Québec'"),
58 + ("region_toute", "region<>''"),
59 + ("category", "category<>''"),
60 + ("employment_type", "employment_type IS NOT NULL"),
61 + ("salary_min", "salary_min IS NOT NULL"),
62 + ("work_mode", "work_mode IS NOT NULL"),
63 + ("salary_label", "salary_label<>''"),
64 + ("postal_code", "postal_code<>''"),
65 + ("date_deadline", "date_deadline IS NOT NULL"),
66 + ("address", "address<>''"),
67 + ("requirements", "requirements NOT IN ('','{}')"),
68 + ("benefits", "benefits NOT IN ('','[]')"),
69 + ("language", "language IS NOT NULL AND language<>''"),
70 + ("company_logo", "company_logo IS NOT NULL AND company_logo<>''"),
71 + ("apply_url", "apply_url IS NOT NULL AND apply_url<>''"),
72 + ("title_clean", "title_clean IS NOT NULL AND title_clean<>''"),
73 +]
74 +out["completude"] = {}
75 +for name, cond in champ_conds:
76 + c = one(f"SELECT COUNT(*) FROM jobs WHERE {ACT} AND {cond}")
77 + out["completude"][name] = (c, round(100.0 * c / n, 1) if n else 0)
78 +
79 +out["par_famille"] = rows(
80 + f"""SELECT ats, COUNT(*),
81 + ROUND(100.0*SUM(salary_min IS NOT NULL)/COUNT(*),1),
82 + ROUND(100.0*SUM(work_mode IS NOT NULL)/COUNT(*),1),
83 + ROUND(100.0*SUM(employment_type IS NOT NULL)/COUNT(*),1),
84 + ROUND(100.0*SUM(date_deadline IS NOT NULL)/COUNT(*),1),
85 + ROUND(100.0*SUM(lat IS NOT NULL)/COUNT(*),1),
86 + ROUND(100.0*SUM(LENGTH(COALESCE(description,''))>200)/COUNT(*),1),
87 + ROUND(100.0*SUM(date_posted IS NOT NULL)/COUNT(*),1),
88 + ROUND(100.0*SUM(language IS NOT NULL AND language<>'')/COUNT(*),1),
89 + ROUND(100.0*SUM(apply_url IS NOT NULL AND apply_url<>'')/COUNT(*),1),
90 + ROUND(100.0*SUM(company_logo IS NOT NULL AND company_logo<>'')/COUNT(*),1),
91 + ROUND(100.0*SUM(benefits NOT IN ('','[]'))/COUNT(*),1)
92 + FROM jobs WHERE {ACT} GROUP BY ats ORDER BY 2 DESC""")
93 +
94 +today = dt.date.today()
95 +ages = [
96 + (today - dt.date.fromisoformat(r[0])).days
97 + for r in rows(f"SELECT date_posted FROM jobs WHERE {ACT}"
98 + " AND date_posted IS NOT NULL")
99 + if r[0] <= today.isoformat()
100 +]
101 +ages.sort()
102 +out["fraicheur"] = {
103 + "n": len(ages),
104 + "median": ages[len(ages) // 2] if ages else None,
105 + "moyen": round(sum(ages) / len(ages), 1) if ages else None,
106 + "gt60": sum(1 for a in ages if a > 60),
107 + "gt90": sum(1 for a in ages if a > 90),
108 + "gt180": sum(1 for a in ages if a > 180),
109 + "futures": one(f"SELECT COUNT(*) FROM jobs WHERE {ACT}"
110 + " AND date_posted > date('now')"),
111 +}
112 +
113 +out["distributions"] = {
114 + "work_mode": rows(f"SELECT work_mode, COUNT(*) FROM jobs WHERE {ACT}"
115 + " GROUP BY work_mode ORDER BY 2 DESC"),
116 + "language": rows(f"SELECT language, COUNT(*) FROM jobs WHERE {ACT}"
117 + " GROUP BY language ORDER BY 2 DESC"),
118 + "region": rows(f"SELECT region, COUNT(*) FROM jobs WHERE {ACT}"
119 + " GROUP BY region ORDER BY 2 DESC"),
120 + "villes_top": rows(f"SELECT city, COUNT(*) FROM jobs WHERE {ACT}"
121 + " GROUP BY city ORDER BY 2 DESC LIMIT 15"),
122 +}
123 +
124 +out["qualite"] = {
125 + "html_residuel": one(
126 + f"""SELECT COUNT(*) FROM jobs WHERE active=1 AND (
127 + description LIKE '%<p%' OR description LIKE '%<br%' OR
128 + description LIKE '%<li%' OR description LIKE '%&nbsp;%' OR
129 + description LIKE '%&amp;%' OR description LIKE '%&lt;%')"""),
130 + "desc_vides_canon": one(
131 + f"SELECT COUNT(*) FROM jobs WHERE {ACT}"
132 + " AND LENGTH(COALESCE(description,''))<50"),
133 + "city_st": one("SELECT COUNT(*) FROM jobs WHERE city IN ('St','Pointe')"),
134 + "salaires_aberrants": one(
135 + f"""SELECT COUNT(*) FROM jobs WHERE active=1 AND salary_min IS NOT NULL
136 + AND (salary_year_max>500000 OR salary_year_min>500000 OR
137 + (salary_unit='hour' AND (salary_hour_max>200 OR salary_hour_min>200)))"""),
138 + "min_gt_max": one(
139 + "SELECT COUNT(*) FROM jobs WHERE salary_min IS NOT NULL"
140 + " AND salary_max IS NOT NULL AND salary_min>salary_max"),
141 + "dates_futures": one(f"SELECT COUNT(*) FROM jobs WHERE {ACT}"
142 + " AND date_posted > date('now')"),
143 + "deadline_lt_posted": one(
144 + f"SELECT COUNT(*) FROM jobs WHERE active=1 AND date_deadline IS NOT NULL"
145 + " AND date_posted IS NOT NULL AND date_deadline<date_posted"),
146 + "geocode_failed": one(
147 + "SELECT COUNT(*) FROM jobs WHERE active=1 AND geocode_failed=1"),
148 +}
149 +
150 +out["dedup"] = {
151 + "groupes": one("SELECT COUNT(DISTINCT dup_of) FROM jobs"
152 + " WHERE dup_of IS NOT NULL AND active=1"),
153 + "masques": out["doublons_masques"],
154 +}
155 +
156 +print(json.dumps(out, ensure_ascii=False, indent=1, default=str))
157