spb/job-ka

1#!/usr/bin/env python32# =============================================================================3# Job·Ka — Groupe KA4# Auteur  : Simon-Pierre Boucher5# Contact : contact@spboucher.ai6# Fichier : scripts/merge_confirmed.py7# Rôle    : Fusion + nettoyage des sorties de probe_ats.py (plusieurs vagues)8#           en un feeds-*.json propre pour gen_connectors.py — dédoublonnage,9#           filtre des faux positifs « Québec » (Alma Michigan…), nettoyage10#           des noms d'employeurs issus des titres Google, villes normalisées11# Créé    : 2026-08-23   Modifié : 2026-08-2312# =============================================================================13"""Usage :14    python3 scripts/merge_confirmed.py OUT.json BRUT1.json [BRUT2.json ...]15"""16from __future__ import annotations1718import json19import re20import sys21import unicodedata22from pathlib import Path2324ROOT = Path(__file__).resolve().parent.parent2526# lieux qui ont trompé is_quebec_location (villes homonymes hors Québec)27_BAD_LOC = re.compile(28    r"michigan|ohio|georgia|texas|florida|california|colorado|illinois|"29    r"indiana|iowa|kansas|kentucky|maine\b|maryland|missouri|nebraska|"30    r"nevada|oregon|tennessee|utah|virginia|wisconsin|arkansas|alabama|"31    r"oklahoma|louisiana|pennsylvania|new (york|jersey|mexico|hampshire)|"32    r"(north|south) (carolina|dakota)|arizona|washington|idaho|montana\b|"33    r"wyoming|delaware|connecticut|massachusetts|rhode island|vermont\b|"34    r"minnesota|mississippi|hawaii|alaska|, [a-z]{2} \d{5}|"35    r", (mi|oh|ga|tx|fl|co|il|ia|ks|ky|md|mo|ne|nv|tn|ut|va|wi|ar|al|ok|"36    r"pa|nj|nh|nm|az|wa|id|mt|wy|de|ct|ma|ri|vt|mn|ms|hi|ak|ny), "37    r"?(usa?|united states|us)|"38    r"france|belgique|belgium|suisse|switzerland|maroc|morocco|"39    r"united states|estados|deutschland",40    re.I)4142_QC_HINT = re.compile(43    r"qu[ée]bec|qc\b|montr[ée]al|laval|gatineau|sherbrooke|longueuil|"44    r"trois.rivi|saguenay|l[ée]vis|brossard|drummondville|granby|"45    r"terrebonne|boucherville|saint-|ste?-|rimouski|victoriaville|"46    r"rouyn|sept-[îi]les|joliette|mirabel|blainville|repentigny|"47    r"chicoutimi|kirkland|pointe-claire|dorval|anjou|lachine|varennes|"48    r"bromont|magog|thetford|boisbriand|sorel|alma, q|shawinigan|val-d|beloeil|vaudreuil|ch[âa]teauguay|candiac|"49    r"remote.*(canada|qu[ée]bec)|canada.*remote", re.I)5051_NOISE = re.compile(52    r"^(jobs?|careers?|emplois?|carri[èe]res?|current openings|"53    r"offres? d'emplois?|open positions|job openings|opportunit[ée]s?)"54    r"( (at|@|chez|-|:|\|))?\s*", re.I)55_TAIL = re.compile(56    r"\s*(\||–|—|-|:)?\s*(jobs?|careers?|emplois?|carri[èe]res?|"57    r"job board|hiring|apply|recrutement|current openings)\s*$", re.I)5859# noms qui sont en fait des titres de poste (titre Google de la page détail)60_TITLE_LIKE = re.compile(61    r"planificat|technicien|conseill|directeur|directrice|coordonn|"62    r"d[ée]veloppeur|ing[ée]nieur|analyste|adjoint|pr[ée]pos|superviseur|"63    r"gestionnaire|repr[ée]sentant|sp[ée]cialiste|op[ée]rateur|m[ée]canicien|"64    r"soudeur|chauffeur|commis|caissi|engineer\b|manager\b|developer\b|"65    r"analyst\b|specialist\b|technician\b|supervisor\b|coordinator\b|"66    r"director\b|lead\b|intern(e|ship)?\b|stagiaire|temps (plein|partiel)|"67    r"full.time|part.time|\(h/f\)|remote\b", re.I)686970def clean_employer(e: str, api_name: str, org: str) -> str:71    if api_name and len(api_name) > 2:72        e = api_name73    if " @ " in e:                       # « Titre du poste @ Employeur »74        e = e.split(" @ ")[-1]75    e = re.sub(r"\s+", " ", e).strip(" '\"·•…")76    prev = None77    while prev != e:78        prev = e79        e = _NOISE.sub("", e).strip()80        e = _TAIL.sub("", e).strip()81    if len(e) < 3 or _TITLE_LIKE.search(e):82        e = org.replace("-", " ").replace("_", " ").title()83    return e848586def city_label(loc: str) -> str:87    c = loc.split(",")[0].split("(")[0].strip()88    c = re.sub(r"\s*-\s*(hq|si[èe]ge|office|bureau).*$", "", c, flags=re.I)89    c = re.sub(r"\s+(office|bureau|campus|hq|studio)s?$", "", c, flags=re.I)90    fix = {"montreal": "Montréal", "quebec": "Québec",91           "quebec city": "Québec", "levis": "Lévis",92           "trois-rivieres": "Trois-Rivières"}93    return fix.get(c.lower(), c[:1].upper() + c[1:]) if c else ""949596def main() -> None:97    out_path, *ins = sys.argv[1:]98    # slugs déjà connectés99    pat = re.compile(100        r"^\s*(?:ORG|BOARD|COMPANY|TENANT|NS)\s*=\s*['\"]([^'\"]+)", re.M)101    known: set[str] = set()102    for f in (ROOT / "jobka" / "connectors").glob("*.py"):103        for m in pat.finditer(f.read_text(encoding="utf-8")):104            known.add(m.group(1).lower())105106    merged: dict[tuple, dict] = {}107    dropped_qc, dropped_known = [], 0108    for p in ins:109        if not Path(p).exists():110            print(f"  ! absent : {p}")111            continue112        for e in json.loads(Path(p).read_text(encoding="utf-8")):113            org = (e.get("org") or e.get("tenant") or e.get("ns")114                   or e.get("slug") or "")115            key = (e["ats"], org.lower())116            if re.search(r"demo|sandbox|test[0-9]|sample", org, re.I):117                continue118            if org.lower() in {"ashby", "greenhouse", "lever", "workable",119                               "recruitee", "breezy", "bamboohr", "workday",120                               "smartrecruiters", "dayforce"}:121                continue122            if org.lower() in known:123                dropped_known += 1124                continue125            if key in merged:126                continue127            locs = e.get("_qc_locations") or []128            good = [l for l in locs129                    if not _BAD_LOC.search(l) and _QC_HINT.search(l)]130            if locs and not good:131                dropped_qc.append((e.get("employer", org), locs[:2]))132                continue133            e["employer"] = clean_employer(134                e.get("employer", ""), e.get("_api_name") or "", org)135            cities = []136            for l in good[:4]:137                c = city_label(l)138                if c and c not in cities and not re.search(139                        r"remote|t[ée]l[ée]travail|canada$|qu[ée]bec - ", c,140                        re.I):141                    cities.append(c)142            e["cities"] = cities or e.get("cities") or []143            e.setdefault("sectors", [])144            e.setdefault("url", "")145            for k in list(e):146                if k.startswith("_") or k == "slug":147                    e.pop(k)148            merged[key] = e149150    feeds = sorted(merged.values(),151                   key=lambda x: (x["ats"], x.get("org") or x.get("tenant", "")))152    Path(out_path).write_text(153        json.dumps(feeds, ensure_ascii=False, indent=1) + "\n",154        encoding="utf-8")155    print(f"[merge] {len(feeds)} flux -> {out_path} "156          f"({dropped_known} déjà connectés, {len(dropped_qc)} faux QC)")157    for n, locs in dropped_qc[:15]:158        print(f"    faux QC : {n} {locs}")159160161if __name__ == "__main__":162    main()163