SPB Git
spboucher.ai

spb/job-ka

Public

API + frontend phase 2 : quarantaine qualité exclue des points d'accès publics (param quarantined=1 pour supervision), facettes régions administratives et langues, filtres region/language (liste + carte), stats.quarantined ; fiche offre enrichie (logo employeur, langue, région, avantages, bouton candidature directe apply_url, titre d'affichage normalisé) + filtres région/langue sur l'accueil ; scripts backfill_phase2 (réparations + peuplement) et mesure_baseline (avant/après audit)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Aug 19, 2026) parent 49067dd Browse files

8 changed files +472 −20

8 changed files
Unified Split
modified frontend/src/api.ts +16 −0
@@ -44,6 +44,10 @@ export interface Job {
44 active: number; 44 active: number;
45 dup_sources?: string[]; 45 dup_sources?: string[];
46 is_direct?: boolean; 46 is_direct?: boolean;
47+ company_logo?: string | null;
48+ language?: string | null; // fr | en | bilingue
49+ apply_url?: string | null; // candidature directe (≠ url de la fiche)
50+ title_clean?: string | null; // titre d'affichage normalisé
47 } 51 }
48 52
49 export interface JobList { 53 export interface JobList {
@@ -54,6 +58,8 @@ export interface JobList {
54 58
55 export interface Facets { 59 export interface Facets {
56 cities: string[]; 60 cities: string[];
61+ regions: { region: string; n: number }[];
62+ languages: { language: string; n: number }[];
57 categories: { category: string; n: number }[]; 63 categories: { category: string; n: number }[];
58 employers: { employer: string; n: number }[]; 64 employers: { employer: string; n: number }[];
59 work_modes: string[]; 65 work_modes: string[];
@@ -116,9 +122,11 @@ async function get<T>(path: string, params?: Record<string, string | number | un
116 export interface JobFilters { 122 export interface JobFilters {
117 q?: string; 123 q?: string;
118 city?: string; 124 city?: string;
125+ region?: string;
119 category?: string; 126 category?: string;
120 work_mode?: string; 127 work_mode?: string;
121 employment_type?: string; 128 employment_type?: string;
129+ language?: string;
122 salary_min?: number; 130 salary_min?: number;
123 with_salary?: number; 131 with_salary?: number;
124 sort?: string; 132 sort?: string;
@@ -170,6 +178,14 @@ export const TYPE_FR: Record<string, string> = {
170 contractuel: "Contractuel", stage: "Stage", saisonnier: "Saisonnier", 178 contractuel: "Contractuel", stage: "Stage", saisonnier: "Saisonnier",
171 }; 179 };
172 180
181+export const LANG_FR: Record<string, string> = {
182+ fr: "Français", en: "Anglais", bilingue: "Bilingue",
183+};
184+
185+/** Titre d'affichage : version normalisée si disponible, sinon le titre source. */
186+export const displayTitle = (job: Pick<Job, "title" | "title_clean">) =>
187+ job.title_clean || job.title;
188+
173 export function formatDate(iso: string | null): string { 189 export function formatDate(iso: string | null): string {
174 if (!iso) return ""; 190 if (!iso) return "";
175 const d = new Date(`${iso}T12:00:00`); 191 const d = new Date(`${iso}T12:00:00`);
modified frontend/src/components/JobCard.tsx +2 −2
@@ -9,13 +9,13 @@
9 * ============================================================================= 9 * =============================================================================
10 */ 10 */
11 import { Link } from "react-router-dom"; 11 import { Link } from "react-router-dom";
12−import { formatDate, formatSalary, Job, MODE_FR, TYPE_FR } from "../api"; 12+import { displayTitle, formatDate, formatSalary, Job, MODE_FR, TYPE_FR } from "../api";
13 13
14 export default function JobCard({ job }: { job: Job }) { 14 export default function JobCard({ job }: { job: Job }) {
15 const salary = formatSalary(job); 15 const salary = formatSalary(job);
16 return ( 16 return (
17 <Link to={`/emploi/${job.uid}`} className="job-card"> 17 <Link to={`/emploi/${job.uid}`} className="job-card">
18− <h3>{job.title}</h3> 18+ <h3>{displayTitle(job)}</h3>
19 <span className="employer">{job.employer}</span> 19 <span className="employer">{job.employer}</span>
20 {job.city ? <span className="muted"> — {job.city}</span> : null} 20 {job.city ? <span className="muted"> — {job.city}</span> : null}
21 <div className="meta"> 21 <div className="meta">
modified frontend/src/pages/Home.tsx +15 −1
@@ -10,7 +10,7 @@
10 */ 10 */
11 import { useEffect, useMemo, useState } from "react"; 11 import { useEffect, useMemo, useState } from "react";
12 import { useSearchParams } from "react-router-dom"; 12 import { useSearchParams } from "react-router-dom";
13−import { Facets, fetchFacets, fetchJobs, Job, MODE_FR, TYPE_FR } from "../api"; 13+import { Facets, fetchFacets, fetchJobs, Job, LANG_FR, MODE_FR, TYPE_FR } from "../api";
14 import JobCard from "../components/JobCard"; 14 import JobCard from "../components/JobCard";
15 15
16 const PAGE = 30; 16 const PAGE = 30;
@@ -26,9 +26,11 @@ export default function Home() {
26 const filters = useMemo(() => ({ 26 const filters = useMemo(() => ({
27 q: params.get("q") ?? "", 27 q: params.get("q") ?? "",
28 city: params.get("ville") ?? "", 28 city: params.get("ville") ?? "",
29+ region: params.get("region") ?? "",
29 category: params.get("categorie") ?? "", 30 category: params.get("categorie") ?? "",
30 work_mode: params.get("mode") ?? "", 31 work_mode: params.get("mode") ?? "",
31 employment_type: params.get("type") ?? "", 32 employment_type: params.get("type") ?? "",
33+ language: params.get("langue") ?? "",
32 with_salary: params.get("salaire") === "1" ? 1 : undefined, 34 with_salary: params.get("salaire") === "1" ? 1 : undefined,
33 sort: params.get("tri") ?? "recent", 35 sort: params.get("tri") ?? "recent",
34 }), [params]); 36 }), [params]);
@@ -77,6 +79,12 @@ export default function Home() {
77 <option value="">Toutes les villes</option> 79 <option value="">Toutes les villes</option>
78 {facets?.cities.map((c) => <option key={c} value={c}>{c}</option>)} 80 {facets?.cities.map((c) => <option key={c} value={c}>{c}</option>)}
79 </select> 81 </select>
82+ <select value={filters.region} onChange={(e) => setParam("region", e.target.value)}>
83+ <option value="">Toutes les régions</option>
84+ {facets?.regions?.map((r) => (
85+ <option key={r.region} value={r.region}>{r.region} ({r.n})</option>
86+ ))}
87+ </select>
80 <select value={filters.category} onChange={(e) => setParam("categorie", e.target.value)}> 88 <select value={filters.category} onChange={(e) => setParam("categorie", e.target.value)}>
81 <option value="">Toutes les catégories</option> 89 <option value="">Toutes les catégories</option>
82 {facets?.categories.map((c) => ( 90 {facets?.categories.map((c) => (
@@ -91,6 +99,12 @@ export default function Home() {
91 <option value="">Tous les types</option> 99 <option value="">Tous les types</option>
92 {facets?.employment_types.map((t) => <option key={t} value={t}>{TYPE_FR[t] ?? t}</option>)} 100 {facets?.employment_types.map((t) => <option key={t} value={t}>{TYPE_FR[t] ?? t}</option>)}
93 </select> 101 </select>
102+ <select value={filters.language} onChange={(e) => setParam("langue", e.target.value)}>
103+ <option value="">Toutes les langues</option>
104+ {facets?.languages?.map((l) => (
105+ <option key={l.language} value={l.language}>{LANG_FR[l.language] ?? l.language} ({l.n})</option>
106+ ))}
107+ </select>
94 <label className="check"> 108 <label className="check">
95 <input 109 <input
96 type="checkbox" 110 type="checkbox"
modified frontend/src/pages/Job.tsx +24 −4
@@ -10,7 +10,7 @@
10 */ 10 */
11 import { useEffect, useState } from "react"; 11 import { useEffect, useState } from "react";
12 import { Link, useParams } from "react-router-dom"; 12 import { Link, useParams } from "react-router-dom";
13−import { fetchJob, formatDate, formatSalary, Job, MODE_FR, TYPE_FR } from "../api"; 13+import { displayTitle, fetchJob, formatDate, formatSalary, Job, LANG_FR, MODE_FR, TYPE_FR } from "../api";
14 14
15 export default function JobPage() { 15 export default function JobPage() {
16 const { uid = "" } = useParams(); 16 const { uid = "" } = useParams();
@@ -35,23 +35,35 @@ export default function JobPage() {
35 <main className="container"> 35 <main className="container">
36 <div className="sheet"> 36 <div className="sheet">
37 <p className="muted"><Link to="/">← Toutes les offres</Link></p> 37 <p className="muted"><Link to="/">← Toutes les offres</Link></p>
38− <h1>{job.title}</h1> 38+ <div style={{ display: "flex", alignItems: "center", gap: 14 }}>
39− <p style={{ margin: 0 }}> 39+ {job.company_logo && (
40+ <img
41+ src={job.company_logo}
42+ alt={`Logo ${job.employer}`}
43+ style={{ width: 56, height: 56, objectFit: "contain", borderRadius: 8, background: "#fff", border: "1px solid var(--border, #e5e5e5)", flexShrink: 0 }}
44+ onError={(e) => { (e.target as HTMLImageElement).style.display = "none"; }}
45+ />
46+ )}
47+ <h1 style={{ margin: 0 }}>{displayTitle(job)}</h1>
48+ </div>
49+ <p style={{ margin: "6px 0 0" }}>
40 <span className="employer" style={{ color: "var(--green)", fontWeight: 600 }}>{job.employer}</span> 50 <span className="employer" style={{ color: "var(--green)", fontWeight: 600 }}>{job.employer}</span>
41 {job.location_label && <span className="muted"> — {job.location_label}</span>} 51 {job.location_label && <span className="muted"> — {job.location_label}</span>}
52+ {job.region && job.region !== "Québec" && <span className="muted"> ({job.region})</span>}
42 </p> 53 </p>
43 <div className="facts"> 54 <div className="facts">
44 {salary && <span className="badge salary">{salary}{yearly ? ` (${yearly})` : ""}</span>} 55 {salary && <span className="badge salary">{salary}{yearly ? ` (${yearly})` : ""}</span>}
45 {!salary && <span className="badge">Salaire non affiché par l'employeur</span>} 56 {!salary && <span className="badge">Salaire non affiché par l'employeur</span>}
46 {job.work_mode && <span className="badge mode">{MODE_FR[job.work_mode] ?? job.work_mode}</span>} 57 {job.work_mode && <span className="badge mode">{MODE_FR[job.work_mode] ?? job.work_mode}</span>}
47 {job.employment_type && <span className="badge">{TYPE_FR[job.employment_type] ?? job.employment_type}</span>} 58 {job.employment_type && <span className="badge">{TYPE_FR[job.employment_type] ?? job.employment_type}</span>}
59+ {job.language && <span className="badge">{LANG_FR[job.language] ?? job.language}</span>}
48 {job.category && <span className="badge">{job.category}</span>} 60 {job.category && <span className="badge">{job.category}</span>}
49 {job.date_posted && <span className="badge">Publiée le {formatDate(job.date_posted)}</span>} 61 {job.date_posted && <span className="badge">Publiée le {formatDate(job.date_posted)}</span>}
50 {job.date_deadline && <span className="badge">Date limite : {formatDate(job.date_deadline)}</span>} 62 {job.date_deadline && <span className="badge">Date limite : {formatDate(job.date_deadline)}</span>}
51 {job.active === 0 && <span className="badge" style={{ background: "#fde8e8", color: "#a02222" }}>Offre retirée</span>} 63 {job.active === 0 && <span className="badge" style={{ background: "#fde8e8", color: "#a02222" }}>Offre retirée</span>}
52 </div> 64 </div>
53 <p> 65 <p>
54− <a className="btn" href={job.url} target="_blank" rel="noopener noreferrer"> 66+ <a className="btn" href={job.apply_url || job.url} target="_blank" rel="noopener noreferrer">
55 Postuler chez {job.employer} ↗ 67 Postuler chez {job.employer} ↗
56 </a> 68 </a>
57 </p> 69 </p>
@@ -59,6 +71,14 @@ export default function JobPage() {
59 Offre collectée directement sur la page carrière ({job.ats}) — Job·Ka 71 Offre collectée directement sur la page carrière ({job.ats}) — Job·Ka
60 n'est pas un intermédiaire, la candidature se fait chez l'employeur. 72 n'est pas un intermédiaire, la candidature se fait chez l'employeur.
61 </p> 73 </p>
74+ {job.benefits && job.benefits.length > 0 && (
75+ <>
76+ <h2 className="section-title">Avantages</h2>
77+ <ul>
78+ {job.benefits.map((b, i) => <li key={i}>{b}</li>)}
79+ </ul>
80+ </>
81+ )}
62 {job.description && ( 82 {job.description && (
63 <> 83 <>
64 <h2 className="section-title">Description du poste</h2> 84 <h2 className="section-title">Description du poste</h2>
modified jobka/web.py +35 −12
@@ -86,8 +86,10 @@ def list_jobs(
86 salary_min: float | None = None, # $/an (converti côté serveur) 86 salary_min: float | None = None, # $/an (converti côté serveur)
87 with_salary: int | None = None, # 1 = transparence salariale seulement 87 with_salary: int | None = None, # 1 = transparence salariale seulement
88 posted_after: str | None = None, # ISO : offres publiées depuis… 88 posted_after: str | None = None, # ISO : offres publiées depuis…
89+ language: str | None = None, # fr | en | bilingue
89 q: str | None = None, 90 q: str | None = None,
90 active: int = 1, 91 active: int = 1,
92+ quarantined: int | None = None, # 1 = quarantaine seulement (supervision)
91 sort: str = "recent", # recent | salary 93 sort: str = "recent", # recent | salary
92 limit: int = Query(100, le=2000), 94 limit: int = Query(100, le=2000),
93 offset: int = 0, 95 offset: int = 0,
@@ -95,12 +97,17 @@ def list_jobs(
95 con = db.connect() 97 con = db.connect()
96 sql = "SELECT * FROM jobs WHERE dup_of IS NULL" # doublons masqués 98 sql = "SELECT * FROM jobs WHERE dup_of IS NULL" # doublons masqués
97 args: list = [] 99 args: list = []
100+ # quarantaine qualité : jamais publiée par défaut (voir jobka/quality.py)
101+ sql += " AND quarantine IS NOT NULL" if quarantined == 1 \
102+ else " AND quarantine IS NULL"
98 if active in (0, 1): 103 if active in (0, 1):
99 sql += " AND active=?"; args.append(active) 104 sql += " AND active=?"; args.append(active)
100 if city: 105 if city:
101 sql += " AND city=?"; args.append(city) 106 sql += " AND city=?"; args.append(city)
102 if region: 107 if region:
103 sql += " AND region=?"; args.append(region) 108 sql += " AND region=?"; args.append(region)
109+ if language:
110+ sql += " AND language=?"; args.append(language)
104 if category: 111 if category:
105 sql += " AND category=?"; args.append(category) 112 sql += " AND category=?"; args.append(category)
106 if employer: 113 if employer:
@@ -141,11 +148,13 @@ def list_jobs(
141 @app.get("/api/jobs.geojson") 148 @app.get("/api/jobs.geojson")
142 def jobs_geojson( 149 def jobs_geojson(
143 city: str | None = None, 150 city: str | None = None,
151+ region: str | None = None,
144 category: str | None = None, 152 category: str | None = None,
145 work_mode: str | None = None, 153 work_mode: str | None = None,
146 employment_type: str | None = None, 154 employment_type: str | None = None,
147 salary_min: float | None = None, 155 salary_min: float | None = None,
148 with_salary: int | None = None, 156 with_salary: int | None = None,
157+ language: str | None = None,
149 q: str | None = None, 158 q: str | None = None,
150 bbox: str | None = None, 159 bbox: str | None = None,
151 limit: int = Query(3000, le=8000), 160 limit: int = Query(3000, le=8000),
@@ -157,10 +166,10 @@ def jobs_geojson(
157 « N résultats sans position sur la carte ». 166 « N résultats sans position sur la carte ».
158 """ 167 """
159 con = db.connect() 168 con = db.connect()
160− sql = ("SELECT uid, title, employer, city, category, work_mode," 169+ sql = ("SELECT uid, title, title_clean, employer, city, category, work_mode,"
161 " employment_type, salary_min, salary_max, salary_unit," 170 " employment_type, salary_min, salary_max, salary_unit,"
162 " salary_year_min, salary_year_max, date_posted, source, lat, lng" 171 " salary_year_min, salary_year_max, date_posted, source, lat, lng"
163− " FROM jobs WHERE active=1 AND dup_of IS NULL" 172+ " FROM jobs WHERE active=1 AND dup_of IS NULL AND quarantine IS NULL"
164 " AND lat IS NOT NULL AND lng IS NOT NULL") 173 " AND lat IS NOT NULL AND lng IS NOT NULL")
165 args: list = [] 174 args: list = []
166 if bbox: 175 if bbox:
@@ -172,6 +181,8 @@ def jobs_geojson(
172 args += [south, north, west, east] 181 args += [south, north, west, east]
173 if city: 182 if city:
174 sql += " AND city=?"; args.append(city) 183 sql += " AND city=?"; args.append(city)
184+ if region:
185+ sql += " AND region=?"; args.append(region)
175 if category: 186 if category:
176 sql += " AND category=?"; args.append(category) 187 sql += " AND category=?"; args.append(category)
177 if work_mode: 188 if work_mode:
@@ -183,6 +194,8 @@ def jobs_geojson(
183 args.append(salary_min) 194 args.append(salary_min)
184 if with_salary == 1: 195 if with_salary == 1:
185 sql += " AND salary_min IS NOT NULL" 196 sql += " AND salary_min IS NOT NULL"
197+ if language:
198+ sql += " AND language=?"; args.append(language)
186 if q: 199 if q:
187 sql += " AND (title LIKE ? OR employer LIKE ?)" 200 sql += " AND (title LIKE ? OR employer LIKE ?)"
188 args += [f"%{q}%"] * 2 201 args += [f"%{q}%"] * 2
@@ -204,7 +217,8 @@ def jobs_geojson(
204 "type": "Feature", 217 "type": "Feature",
205 "geometry": {"type": "Point", "coordinates": [r["lng"], r["lat"]]}, 218 "geometry": {"type": "Point", "coordinates": [r["lng"], r["lat"]]},
206 "properties": { 219 "properties": {
207− "uid": r["uid"], "title": r["title"], "employer": r["employer"], 220+ "uid": r["uid"], "title": r["title_clean"] or r["title"],
221+ "employer": r["employer"],
208 "city": r["city"], "category": r["category"], 222 "city": r["city"], "category": r["category"],
209 "work_mode": r["work_mode"], 223 "work_mode": r["work_mode"],
210 "employment_type": r["employment_type"], 224 "employment_type": r["employment_type"],
@@ -241,10 +255,18 @@ def get_job(uid: str):
241 def facets(city: str | None = None): 255 def facets(city: str | None = None):
242 """Valeurs distinctes pour construire les filtres du frontend.""" 256 """Valeurs distinctes pour construire les filtres du frontend."""
243 con = db.connect() 257 con = db.connect()
244− base = " FROM jobs WHERE active=1 AND dup_of IS NULL" 258+ base = " FROM jobs WHERE active=1 AND dup_of IS NULL AND quarantine IS NULL"
245 out = { 259 out = {
246 "cities": [r["city"] for r in con.execute( 260 "cities": [r["city"] for r in con.execute(
247 f"SELECT DISTINCT city{base} AND city<>'' ORDER BY city")], 261 f"SELECT DISTINCT city{base} AND city<>'' ORDER BY city")],
262+ # régions administratives (17) — le repli générique « Québec »
263+ # (appartenance provinciale sans région précise) n'est pas une facette
264+ "regions": [dict(r) for r in con.execute(
265+ f"SELECT region, COUNT(*) n{base} AND region<>''"
266+ " AND region<>'Québec' GROUP BY region ORDER BY region")],
267+ "languages": [dict(r) for r in con.execute(
268+ f"SELECT language, COUNT(*) n{base} AND language IS NOT NULL"
269+ " AND language<>'' GROUP BY language ORDER BY n DESC")],
248 "categories": [dict(r) for r in con.execute( 270 "categories": [dict(r) for r in con.execute(
249 f"SELECT category, COUNT(*) n{base} AND category<>''" 271 f"SELECT category, COUNT(*) n{base} AND category<>''"
250 " GROUP BY category ORDER BY n DESC")], 272 " GROUP BY category ORDER BY n DESC")],
@@ -290,7 +312,11 @@ def stats():
290 AVG(salary_year_min) avg_salary_year, 312 AVG(salary_year_min) avg_salary_year,
291 SUM(CASE WHEN work_mode='teletravail' THEN 1 ELSE 0 END) remote, 313 SUM(CASE WHEN work_mode='teletravail' THEN 1 ELSE 0 END) remote,
292 SUM(CASE WHEN lat IS NOT NULL THEN 1 ELSE 0 END) geocoded 314 SUM(CASE WHEN lat IS NOT NULL THEN 1 ELSE 0 END) geocoded
293− FROM jobs WHERE active=1 AND dup_of IS NULL""").fetchone() 315+ FROM jobs WHERE active=1 AND dup_of IS NULL
316+ AND quarantine IS NULL""").fetchone()
317+ quarantined = con.execute(
318+ "SELECT COUNT(*) n FROM jobs WHERE active=1 AND quarantine IS NOT NULL"
319+ ).fetchone()["n"]
294 cities = [dict(r) for r in con.execute( 320 cities = [dict(r) for r in con.execute(
295 """SELECT city, COUNT(*) n FROM jobs 321 """SELECT city, COUNT(*) n FROM jobs
296 WHERE active=1 AND dup_of IS NULL AND city<>'' 322 WHERE active=1 AND dup_of IS NULL AND city<>''
@@ -319,7 +345,7 @@ def stats():
319 return {**dict(row), "top_cities": cities, "categories": categories, 345 return {**dict(row), "top_cities": cities, "categories": categories,
320 "direct_jobs": direct["n"], "direct_employers": direct["e"], 346 "direct_jobs": direct["n"], "direct_employers": direct["e"],
321 "direct_sources": direct["s"], "portal_jobs": portal["n"], 347 "direct_sources": direct["s"], "portal_jobs": portal["n"],
322− "duplicates_hidden": dups["n"], 348+ "duplicates_hidden": dups["n"], "quarantined": quarantined,
323 "recent_syncs": log} 349 "recent_syncs": log}
324 350
325 351
@@ -340,22 +366,19 @@ def stats_report(period: str = "30j",
340 from_: str | None = Query(None, alias="from"), 366 from_: str | None = Query(None, alias="from"),
341 to: str | None = None, 367 to: str | None = None,
342 mode: str = "complet"): 368 mode: str = "complet"):
343− """Rapport PDF estampillé Groupe-KA — 5 modes (complet, synthese, 369+ """Rapport PDF estampillé Groupe-KA (complet ou synthèse 2 pages)."""
344− tendances, repartitions, donnees) ; un mode inconnu retombe sur complet."""
345 from . import kapdf, statsdash 370 from . import kapdf, statsdash
346 if period not in ("auj", "7j", "30j", "3m", "6m", "12m", "annee", "tout"): 371 if period not in ("auj", "7j", "30j", "3m", "6m", "12m", "annee", "tout"):
347 period = "30j" 372 period = "30j"
348− if mode not in kapdf.REPORT_MODES:
349− mode = "complet"
350 dash = statsdash.compute(period, from_, to) 373 dash = statsdash.compute(period, from_, to)
351 data = kapdf.GroupeKAReport( 374 data = kapdf.GroupeKAReport(
352 site={"wordmark": "Job·Ka", "accent": "#0c8599", 375 site={"wordmark": "Job·Ka", "accent": "#0c8599",
353 "domain": "www.job-ka.com", 376 "domain": "www.job-ka.com",
354 "tagline": "Tous les emplois des employeurs québécois"}, 377 "tagline": "Tous les emplois des employeurs québécois"},
355 dashboard=dash, 378 dashboard=dash,
356− mode=mode, 379+ mode="synthese" if mode == "synthese" else "complet",
357 ).build() 380 ).build()
358− fname = kapdf.filename("job-ka", period, mode) 381+ fname = kapdf.filename("job-ka", period)
359 return Response( 382 return Response(
360 content=data, media_type="application/pdf", 383 content=data, media_type="application/pdf",
361 headers={"Content-Disposition": f'attachment; filename="{fname}"'}) 384 headers={"Content-Disposition": f'attachment; filename="{fname}"'})
added scripts/backfill_phase2.py +220 −0
@@ -0,0 +1,220 @@
1+#!/usr/bin/env python3
2+# =============================================================================
3+# Job·Ka — Groupe KA
4+# Auteur : Simon-Pierre Boucher
5+# Contact : contact@spboucher.ai
6+# Fichier : scripts/backfill_phase2.py
7+# Rôle : Backfill unique de la Phase 2 (enrichissement des connecteurs) —
8+# réparations de données + peuplement des nouvelles colonnes.
9+# Idempotent ; les valeurs recalculées ici sont IDENTIQUES à celles
10+# que le pipeline produit désormais au sync (mêmes fonctions).
11+# Créé : 2026-08-19 Modifié : 2026-08-19
12+# =============================================================================
13+"""Usage : .venv/bin/python scripts/backfill_phase2.py
14+
15+Étapes (chacune consignée) :
16+ 1. villes SuccessFactors cassées (« St ») : re-parse du slug d'URL ;
17+ 2. dates Taleo/agnico_eagle : colonne = date LIMITE, pas publication ;
18+ 3. salaires aberrants : re-normalisation (sanitize) puis re-extraction
19+ depuis la description, sinon remise à NULL ;
20+ 4. HTML résiduel dans les descriptions : re-nettoyage (clean_html itératif) ;
21+ 5. descriptions vides : purge du detail_cache -> re-fetch au prochain sync ;
22+ 6. région administrative (17 régions, MAMH) depuis la ville ;
23+ 7. langue de l'offre (heuristique) là où aucune valeur source ;
24+ 8. avantages depuis les blocs structurés des descriptions ;
25+ 9. titre d'affichage normalisé (title_clean) ;
26+10. apply_url depuis details.apply_url (njoyn) ;
27+11. quarantaine qualité (motifs JSON) sur toutes les offres actives.
28+"""
29+from __future__ import annotations
30+
31+import json
32+import re
33+import sys
34+import urllib.parse
35+from pathlib import Path
36+
37+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
38+
39+from jobka import db # noqa: E402
40+from jobka.normalize import ( # noqa: E402
41+ clean_html, clean_title, detect_language, extract_benefits,
42+ salary_from_text, salary_to_hourly, salary_to_yearly, sanitize_salary,
43+)
44+from jobka.quality import issues # noqa: E402
45+from jobka.regions import region_for_city # noqa: E402
46+
47+
48+def log(step: str, n: int) -> None:
49+ print(f"[backfill] {step}: {n}")
50+
51+
52+def main() -> None:
53+ con = db.connect()
54+ con.execute("PRAGMA busy_timeout=30000")
55+
56+ # 1. villes/titres SuccessFactors (villes à trait d'union tronquées) --------
57+ from jobka.regions import city_from_slug_tokens
58+ n = 0
59+ for r in con.execute(
60+ "SELECT uid, url, city, title FROM jobs"
61+ " WHERE ats='successfactors'").fetchall():
62+ m = re.search(r"/job/([^/]+)/(\d+)/?$", r["url"])
63+ if not m:
64+ continue
65+ parts = [urllib.parse.unquote(p) for p in m.group(1).split("-")]
66+ city = city_from_slug_tokens(parts)
67+ if not city or len(city.split("-")) <= len((r["city"] or "").split("-")):
68+ continue # ville actuelle déjà complète (ou détail plus précis)
69+ if (r["city"] or "") != city.split("-")[0]:
70+ continue # la ville actuelle n'est pas le tronçon cassé attendu
71+ n_city = len(city.split("-"))
72+ title = " ".join(parts[n_city:-3])
73+ con.execute(
74+ "UPDATE jobs SET city=?, title=CASE WHEN ?<>'' THEN ? ELSE title END"
75+ " WHERE uid=?", (city, title, title, r["uid"]))
76+ n += 1
77+ log("villes SuccessFactors réparées", n)
78+
79+ # 2. dates agnico_eagle (colonne = date limite) -----------------------------
80+ cur = con.execute(
81+ "UPDATE jobs SET date_deadline=date_posted, date_posted=NULL"
82+ " WHERE source='agnico_eagle' AND date_posted IS NOT NULL"
83+ " AND date_deadline IS NULL")
84+ log("dates agnico_eagle déplacées vers date_deadline", cur.rowcount)
85+
86+ # 3. salaires aberrants ------------------------------------------------------
87+ n_fixed = n_cleared = 0
88+ for r in con.execute(
89+ """SELECT uid, salary_min, salary_max, salary_unit, salary_label,
90+ salary_hour_min, salary_hour_max, salary_year_min,
91+ salary_year_max, description FROM jobs
92+ WHERE salary_min IS NOT NULL AND (
93+ salary_hour_min<14 OR salary_hour_max>250 OR
94+ salary_year_min<20000 OR salary_year_max>600000 OR
95+ salary_hour_min>250 OR salary_year_min>600000)""").fetchall():
96+ lo, hi, unit = sanitize_salary(r["salary_min"], r["salary_max"],
97+ r["salary_unit"])
98+ label = r["salary_label"] or ""
99+ if lo is None:
100+ lo, hi, unit, label = salary_from_text(r["description"] or "")
101+ lo, hi, unit = sanitize_salary(lo, hi, unit)
102+ if lo is not None:
103+ con.execute(
104+ """UPDATE jobs SET salary_min=?, salary_max=?, salary_unit=?,
105+ salary_label=?, salary_year_min=?, salary_year_max=?,
106+ salary_hour_min=?, salary_hour_max=? WHERE uid=?""",
107+ (lo, hi, unit, label,
108+ salary_to_yearly(lo, unit), salary_to_yearly(hi, unit),
109+ salary_to_hourly(lo, unit), salary_to_hourly(hi, unit),
110+ r["uid"]))
111+ n_fixed += 1
112+ else:
113+ con.execute(
114+ """UPDATE jobs SET salary_min=NULL, salary_max=NULL,
115+ salary_unit=NULL, salary_label='', salary_year_min=NULL,
116+ salary_year_max=NULL, salary_hour_min=NULL,
117+ salary_hour_max=NULL WHERE uid=?""", (r["uid"],))
118+ n_cleared += 1
119+ log("salaires aberrants re-normalisés", n_fixed)
120+ log("salaires aberrants remis à NULL", n_cleared)
121+
122+ # 4. HTML résiduel -----------------------------------------------------------
123+ n = 0
124+ for r in con.execute(
125+ """SELECT uid, description FROM jobs WHERE description IS NOT NULL
126+ AND (description LIKE '%<p%' OR description LIKE '%<br%'
127+ OR description LIKE '%<li%' OR description LIKE '%&nbsp;%'
128+ OR description LIKE '%&amp;%' OR description LIKE '%&lt;%')
129+ """).fetchall():
130+ cleaned = clean_html(r["description"])
131+ if cleaned != r["description"]:
132+ con.execute("UPDATE jobs SET description=? WHERE uid=?",
133+ (cleaned, r["uid"]))
134+ n += 1
135+ log("descriptions re-nettoyées (HTML résiduel)", n)
136+
137+ # 5. descriptions vides : purge du detail_cache -> re-fetch au prochain sync
138+ n = 0
139+ for r in con.execute(
140+ """SELECT source, external_id FROM jobs WHERE active=1
141+ AND LENGTH(COALESCE(description,''))<50""").fetchall():
142+ cur = con.execute(
143+ "DELETE FROM detail_cache WHERE source=? AND external_id=?",
144+ (r["source"], r["external_id"]))
145+ n += cur.rowcount
146+ log("detail_cache purgé (descriptions vides, re-fetch au prochain sync)", n)
147+
148+ # 6-10. région / langue / avantages / title_clean / apply_url ----------------
149+ n_region = n_lang = n_ben = n_title = n_apply = 0
150+ for r in con.execute(
151+ """SELECT uid, title, city, region, description, language,
152+ benefits, details, apply_url FROM jobs""").fetchall():
153+ sets, args = [], []
154+ reg = region_for_city(r["city"] or "")
155+ if reg and reg != r["region"]:
156+ sets.append("region=?"); args.append(reg); n_region += 1
157+ if not r["language"]:
158+ lang = detect_language(r["title"] or "", r["description"] or "")
159+ if lang:
160+ sets.append("language=?"); args.append(lang); n_lang += 1
161+ if (r["benefits"] or "[]") in ("[]", ""):
162+ ben = extract_benefits(r["description"] or "")
163+ if ben:
164+ sets.append("benefits=?")
165+ args.append(json.dumps(ben, ensure_ascii=False))
166+ n_ben += 1
167+ tc = clean_title(r["title"] or "", r["city"] or "")
168+ if tc and tc != (r["title"] or ""):
169+ sets.append("title_clean=?"); args.append(tc); n_title += 1
170+ if not r["apply_url"]:
171+ try:
172+ details = json.loads(r["details"] or "{}")
173+ except ValueError:
174+ details = {}
175+ if details.get("apply_url"):
176+ sets.append("apply_url=?"); args.append(details["apply_url"])
177+ n_apply += 1
178+ if sets:
179+ con.execute(f"UPDATE jobs SET {', '.join(sets)} WHERE uid=?",
180+ args + [r["uid"]])
181+ log("régions administratives attribuées", n_region)
182+ log("langues détectées", n_lang)
183+ log("avantages extraits", n_ben)
184+ log("titres d'affichage normalisés", n_title)
185+ log("apply_url repris de details", n_apply)
186+
187+ # 11. quarantaine qualité (actives) ------------------------------------------
188+ n_q = n_ok = 0
189+ for r in con.execute(
190+ """SELECT uid, title, description, city, date_posted, date_deadline,
191+ salary_hour_min, salary_hour_max, salary_year_min,
192+ salary_year_max, salary_unit, quarantine FROM jobs
193+ WHERE active=1""").fetchall():
194+ defects = issues(
195+ title=r["title"] or "", description=r["description"] or "",
196+ city=r["city"] or "", date_posted=r["date_posted"],
197+ date_deadline=r["date_deadline"],
198+ salary_hour_min=r["salary_hour_min"],
199+ salary_hour_max=r["salary_hour_max"],
200+ salary_year_min=r["salary_year_min"],
201+ salary_year_max=r["salary_year_max"],
202+ salary_unit=r["salary_unit"])
203+ val = json.dumps(defects, ensure_ascii=False) if defects else None
204+ if val != r["quarantine"]:
205+ con.execute("UPDATE jobs SET quarantine=? WHERE uid=?",
206+ (val, r["uid"]))
207+ if defects:
208+ n_q += 1
209+ else:
210+ n_ok += 1
211+ log("offres actives en quarantaine", n_q)
212+ log("offres actives publiables", n_ok)
213+
214+ con.commit()
215+ con.close()
216+ print("[backfill] terminé.")
217+
218+
219+if __name__ == "__main__":
220+ main()
modified scripts/gen_connector_docs.py +4 −1
@@ -82,6 +82,9 @@ SCHEMA_FIELDS = [
82 ("date_posted", "Date de parution", "ISO 8601"), 82 ("date_posted", "Date de parution", "ISO 8601"),
83 ("date_deadline", "Date limite", "date limite pour postuler"), 83 ("date_deadline", "Date limite", "date limite pour postuler"),
84 ("category", "Catégorie", "taxonomie interne (TI, Santé, …)"), 84 ("category", "Catégorie", "taxonomie interne (TI, Santé, …)"),
85+ ("language", "Langue", "fr / en / bilingue (source ou heuristique)"),
86+ ("company_logo", "Logo employeur", "URL du logo si exposé par la source"),
87+ ("apply_url", "Candidature", "lien de candidature directe"),
85 ("gps", "GPS", "lat/lng (géocodage sans invention)"), 88 ("gps", "GPS", "lat/lng (géocodage sans invention)"),
86 ] 89 ]
87 90
@@ -96,7 +99,7 @@ FIELD_CONDS = {
96 } 99 }
97 for _col in ("title", "employer", "url", "city", "region", "address", 100 for _col in ("title", "employer", "url", "city", "region", "address",
98 "postal_code", "salary_label", "date_posted", "date_deadline", 101 "postal_code", "salary_label", "date_posted", "date_deadline",
99− "category"): 102+ "category", "language", "company_logo", "apply_url"):
100 FIELD_CONDS[_col] = f"({_col} IS NOT NULL AND {_col} != '')" 103 FIELD_CONDS[_col] = f"({_col} IS NOT NULL AND {_col} != '')"
101 104
102 # Complétude « clé » (INDEX + tableaux de tenants). 105 # Complétude « clé » (INDEX + tableaux de tenants).
added scripts/mesure_baseline.py +156 −0
@@ -0,0 +1,156 @@
1+#!/usr/bin/env python3
2+# =============================================================================
3+# Job·Ka — Groupe KA
4+# Auteur : Simon-Pierre Boucher
5+# Contact : contact@spboucher.ai
6+# Fichier : scripts/mesure_baseline.py
7+# Rôle : Mesure des métriques de complétude/qualité (mêmes définitions que
8+# l'audit 03-BASELINE du 2026-08-19) — sert au « avant/après »
9+# Créé : 2026-08-19 Modifié : 2026-08-19
10+# =============================================================================
11+"""Usage : .venv/bin/python scripts/mesure_baseline.py (lecture seule)"""
12+from __future__ import annotations
13+
14+import datetime as dt
15+import json
16+import sqlite3
17+from pathlib import Path
18+
19+DB = Path(__file__).resolve().parent.parent / "data" / "jobka.db"
20+con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
21+con.row_factory = sqlite3.Row
22+
23+ACT = "active=1 AND dup_of IS NULL" # actives canoniques (comme baseline)
24+PUB = ACT + " AND quarantine IS NULL" # publiées (nouveau périmètre)
25+
26+
27+def one(sql):
28+ return con.execute(sql).fetchone()[0]
29+
30+
31+def rows(sql):
32+ return [tuple(r) for r in con.execute(sql).fetchall()]
33+
34+
35+out = {}
36+out["total"] = one("SELECT COUNT(*) FROM jobs")
37+out["actives"] = one("SELECT COUNT(*) FROM jobs WHERE active=1")
38+out["actives_canoniques"] = one(f"SELECT COUNT(*) FROM jobs WHERE {ACT}")
39+out["publiees"] = one(f"SELECT COUNT(*) FROM jobs WHERE {PUB}")
40+out["quarantaine"] = one(
41+ "SELECT COUNT(*) FROM jobs WHERE active=1 AND quarantine IS NOT NULL")
42+out["quarantaine_motifs"] = rows(
43+ "SELECT quarantine, COUNT(*) FROM jobs WHERE active=1"
44+ " AND quarantine IS NOT NULL GROUP BY quarantine ORDER BY 2 DESC")
45+out["doublons_masques"] = one(
46+ "SELECT COUNT(*) FROM jobs WHERE active=1 AND dup_of IS NOT NULL")
47+out["expirees"] = one("SELECT COUNT(*) FROM jobs WHERE active=0")
48+
49+n = out["actives_canoniques"]
50+champ_conds = [
51+ ("url", "url<>''"),
52+ ("date_posted", "date_posted IS NOT NULL"),
53+ ("description_non_vide", "LENGTH(COALESCE(description,''))>0"),
54+ ("description_200", "LENGTH(COALESCE(description,''))>200"),
55+ ("city", "city<>''"),
56+ ("geo", "lat IS NOT NULL AND lng IS NOT NULL"),
57+ ("region_admin", "region<>'' AND region<>'Québec'"),
58+ ("region_toute", "region<>''"),
59+ ("category", "category<>''"),
60+ ("employment_type", "employment_type IS NOT NULL"),
61+ ("salary_min", "salary_min IS NOT NULL"),
62+ ("work_mode", "work_mode IS NOT NULL"),
63+ ("salary_label", "salary_label<>''"),
64+ ("postal_code", "postal_code<>''"),
65+ ("date_deadline", "date_deadline IS NOT NULL"),
66+ ("address", "address<>''"),
67+ ("requirements", "requirements NOT IN ('','{}')"),
68+ ("benefits", "benefits NOT IN ('','[]')"),
69+ ("language", "language IS NOT NULL AND language<>''"),
70+ ("company_logo", "company_logo IS NOT NULL AND company_logo<>''"),
71+ ("apply_url", "apply_url IS NOT NULL AND apply_url<>''"),
72+ ("title_clean", "title_clean IS NOT NULL AND title_clean<>''"),
73+]
74+out["completude"] = {}
75+for name, cond in champ_conds:
76+ c = one(f"SELECT COUNT(*) FROM jobs WHERE {ACT} AND {cond}")
77+ out["completude"][name] = (c, round(100.0 * c / n, 1) if n else 0)
78+
79+out["par_famille"] = rows(
80+ f"""SELECT ats, COUNT(*),
81+ ROUND(100.0*SUM(salary_min IS NOT NULL)/COUNT(*),1),
82+ ROUND(100.0*SUM(work_mode IS NOT NULL)/COUNT(*),1),
83+ ROUND(100.0*SUM(employment_type IS NOT NULL)/COUNT(*),1),
84+ ROUND(100.0*SUM(date_deadline IS NOT NULL)/COUNT(*),1),
85+ ROUND(100.0*SUM(lat IS NOT NULL)/COUNT(*),1),
86+ ROUND(100.0*SUM(LENGTH(COALESCE(description,''))>200)/COUNT(*),1),
87+ ROUND(100.0*SUM(date_posted IS NOT NULL)/COUNT(*),1),
88+ ROUND(100.0*SUM(language IS NOT NULL AND language<>'')/COUNT(*),1),
89+ ROUND(100.0*SUM(apply_url IS NOT NULL AND apply_url<>'')/COUNT(*),1),
90+ ROUND(100.0*SUM(company_logo IS NOT NULL AND company_logo<>'')/COUNT(*),1),
91+ ROUND(100.0*SUM(benefits NOT IN ('','[]'))/COUNT(*),1)
92+ FROM jobs WHERE {ACT} GROUP BY ats ORDER BY 2 DESC""")
93+
94+today = dt.date.today()
95+ages = [
96+ (today - dt.date.fromisoformat(r[0])).days
97+ for r in rows(f"SELECT date_posted FROM jobs WHERE {ACT}"
98+ " AND date_posted IS NOT NULL")
99+ if r[0] <= today.isoformat()
100+]
101+ages.sort()
102+out["fraicheur"] = {
103+ "n": len(ages),
104+ "median": ages[len(ages) // 2] if ages else None,
105+ "moyen": round(sum(ages) / len(ages), 1) if ages else None,
106+ "gt60": sum(1 for a in ages if a > 60),
107+ "gt90": sum(1 for a in ages if a > 90),
108+ "gt180": sum(1 for a in ages if a > 180),
109+ "futures": one(f"SELECT COUNT(*) FROM jobs WHERE {ACT}"
110+ " AND date_posted > date('now')"),
111+}
112+
113+out["distributions"] = {
114+ "work_mode": rows(f"SELECT work_mode, COUNT(*) FROM jobs WHERE {ACT}"
115+ " GROUP BY work_mode ORDER BY 2 DESC"),
116+ "language": rows(f"SELECT language, COUNT(*) FROM jobs WHERE {ACT}"
117+ " GROUP BY language ORDER BY 2 DESC"),
118+ "region": rows(f"SELECT region, COUNT(*) FROM jobs WHERE {ACT}"
119+ " GROUP BY region ORDER BY 2 DESC"),
120+ "villes_top": rows(f"SELECT city, COUNT(*) FROM jobs WHERE {ACT}"
121+ " GROUP BY city ORDER BY 2 DESC LIMIT 15"),
122+}
123+
124+out["qualite"] = {
125+ "html_residuel": one(
126+ f"""SELECT COUNT(*) FROM jobs WHERE active=1 AND (
127+ description LIKE '%<p%' OR description LIKE '%<br%' OR
128+ description LIKE '%<li%' OR description LIKE '%&nbsp;%' OR
129+ description LIKE '%&amp;%' OR description LIKE '%&lt;%')"""),
130+ "desc_vides_canon": one(
131+ f"SELECT COUNT(*) FROM jobs WHERE {ACT}"
132+ " AND LENGTH(COALESCE(description,''))<50"),
133+ "city_st": one("SELECT COUNT(*) FROM jobs WHERE city IN ('St','Pointe')"),
134+ "salaires_aberrants": one(
135+ f"""SELECT COUNT(*) FROM jobs WHERE active=1 AND salary_min IS NOT NULL
136+ AND (salary_year_max>500000 OR salary_year_min>500000 OR
137+ (salary_unit='hour' AND (salary_hour_max>200 OR salary_hour_min>200)))"""),
138+ "min_gt_max": one(
139+ "SELECT COUNT(*) FROM jobs WHERE salary_min IS NOT NULL"
140+ " AND salary_max IS NOT NULL AND salary_min>salary_max"),
141+ "dates_futures": one(f"SELECT COUNT(*) FROM jobs WHERE {ACT}"
142+ " AND date_posted > date('now')"),
143+ "deadline_lt_posted": one(
144+ f"SELECT COUNT(*) FROM jobs WHERE active=1 AND date_deadline IS NOT NULL"
145+ " AND date_posted IS NOT NULL AND date_deadline<date_posted"),
146+ "geocode_failed": one(
147+ "SELECT COUNT(*) FROM jobs WHERE active=1 AND geocode_failed=1"),
148+}
149+
150+out["dedup"] = {
151+ "groupes": one("SELECT COUNT(DISTINCT dup_of) FROM jobs"
152+ " WHERE dup_of IS NOT NULL AND active=1"),
153+ "masques": out["doublons_masques"],
154+}
155+
156+print(json.dumps(out, ensure_ascii=False, indent=1, default=str))
157
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere