Python 67%
TypeScript 18.2%
CSS 14.4%
1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# Groupe KA — Annuaire des inspecteurs en bâtiment du Québec4# build_inspecteurs.py : balayage Serper Places sur ~200 villes des 17 régions5# administratives, filtre « inspection de bâtiment », dédoublonnage6# CID/téléphone, sortie canonique data/inspecteurs.json (consommée par Immo-Ka).7# -----------------------------------------------------------------------------8from __future__ import annotations910import json11import os12import re13import sys14import time15import unicodedata16from concurrent.futures import ThreadPoolExecutor, as_completed17from pathlib import Path18from threading import Lock19from urllib.request import Request, urlopen2021API_KEY = os.environ.get("SERPER_API_KEY", "")22OUT = Path(__file__).parent / "inspecteurs.json"2324# Région administrative → villes balayées (couverture provinciale complète)25REGIONS: dict[str, list[str]] = {26 "Bas-Saint-Laurent": [27 "Rimouski", "Rivière-du-Loup", "Matane", "Mont-Joli", "Amqui",28 "La Pocatière", "Témiscouata-sur-le-Lac", "Trois-Pistoles", "Dégelis"],29 "Saguenay–Lac-Saint-Jean": [30 "Saguenay", "Chicoutimi", "Jonquière", "La Baie", "Alma",31 "Dolbeau-Mistassini", "Roberval", "Saint-Félicien", "Normandin"],32 "Capitale-Nationale": [33 "Québec", "L'Ancienne-Lorette", "Saint-Augustin-de-Desmaures",34 "Beaupré", "Baie-Saint-Paul", "La Malbaie", "Donnacona", "Pont-Rouge",35 "Saint-Raymond", "Sainte-Catherine-de-la-Jacques-Cartier",36 "Stoneham-et-Tewkesbury", "Boischatel", "Château-Richer"],37 "Mauricie": [38 "Trois-Rivières", "Shawinigan", "La Tuque", "Louiseville",39 "Saint-Tite", "Saint-Étienne-des-Grès"],40 "Estrie": [41 "Sherbrooke", "Magog", "Granby", "Cowansville", "Bromont",42 "Coaticook", "Lac-Mégantic", "Windsor", "East Angus",43 "Val-des-Sources", "Waterloo"],44 "Montréal": [45 "Montréal", "Montréal-Nord", "Saint-Laurent Montréal", "LaSalle",46 "Anjou", "Ahuntsic", "Rosemont", "Hochelaga-Maisonneuve",47 "Le Plateau-Mont-Royal", "Verdun", "Lachine", "Pierrefonds",48 "Rivière-des-Prairies", "Côte-des-Neiges", "Westmount",49 "Côte-Saint-Luc", "Dollard-des-Ormeaux", "Pointe-Claire", "Dorval",50 "Kirkland", "Beaconsfield", "Sainte-Anne-de-Bellevue", "Outremont",51 "Mont-Royal"],52 "Outaouais": [53 "Gatineau", "Hull Gatineau", "Aylmer Gatineau", "Buckingham Gatineau",54 "Cantley", "Chelsea", "Val-des-Monts", "Maniwaki", "Papineauville",55 "La Pêche"],56 "Abitibi-Témiscamingue": [57 "Rouyn-Noranda", "Val-d'Or", "Amos", "La Sarre", "Ville-Marie QC",58 "Senneterre", "Malartic"],59 "Côte-Nord": [60 "Baie-Comeau", "Sept-Îles", "Port-Cartier", "Forestville",61 "Havre-Saint-Pierre", "Fermont"],62 "Nord-du-Québec": [63 "Chibougamau", "Lebel-sur-Quévillon", "Matagami", "Chapais"],64 "Gaspésie–Îles-de-la-Madeleine": [65 "Gaspé", "Chandler", "New Richmond", "Bonaventure",66 "Carleton-sur-Mer", "Sainte-Anne-des-Monts", "Percé",67 "Les Îles-de-la-Madeleine"],68 "Chaudière-Appalaches": [69 "Lévis", "Saint-Georges", "Thetford Mines", "Montmagny",70 "Sainte-Marie", "Beauceville", "Saint-Joseph-de-Beauce", "L'Islet",71 "Lac-Etchemin", "Saint-Anselme", "Saint-Henri"],72 "Laval": ["Laval", "Chomedey Laval", "Sainte-Rose Laval"],73 "Lanaudière": [74 "Terrebonne", "Repentigny", "Mascouche", "Joliette", "L'Assomption",75 "Rawdon", "Saint-Lin-Laurentides", "Berthierville", "Lavaltrie",76 "Saint-Charles-Borromée", "Chertsey"],77 "Laurentides": [78 "Saint-Jérôme", "Blainville", "Boisbriand", "Sainte-Thérèse",79 "Mirabel", "Saint-Eustache", "Deux-Montagnes", "Rosemère",80 "Bois-des-Filion", "Sainte-Adèle", "Sainte-Agathe-des-Monts",81 "Mont-Tremblant", "Mont-Laurier", "Prévost", "Saint-Sauveur",82 "Lachute", "Saint-Colomban"],83 "Montérégie": [84 "Longueuil", "Brossard", "Saint-Lambert", "Boucherville",85 "Saint-Bruno-de-Montarville", "Saint-Hubert Longueuil",86 "Châteauguay", "Saint-Jean-sur-Richelieu", "Salaberry-de-Valleyfield",87 "Vaudreuil-Dorion", "Saint-Hyacinthe", "Sorel-Tracy", "Beloeil",88 "Mont-Saint-Hilaire", "Chambly", "La Prairie", "Candiac",89 "Sainte-Catherine QC", "Saint-Constant", "Mercier", "Beauharnois",90 "Varennes", "Sainte-Julie", "Contrecoeur", "Marieville", "Farnham",91 "Acton Vale", "Rigaud", "L'Île-Perrot", "Pincourt", "Saint-Rémi"],92 "Centre-du-Québec": [93 "Drummondville", "Victoriaville", "Bécancour", "Nicolet",94 "Plessisville", "Princeville", "Kingsey Falls", "Warwick QC"],95}9697# Grandes villes : pagination plus profonde + variante de requête98BIG = {"Montréal", "Québec", "Laval", "Gatineau", "Longueuil", "Sherbrooke",99 "Trois-Rivières", "Saguenay", "Lévis", "Terrebonne", "Saint-Jérôme",100 "Brossard", "Repentigny", "Drummondville", "Saint-Jean-sur-Richelieu"}101102MOVER_RE = re.compile(r"inspect", re.I)103EXCLUDE_RE = re.compile(r"auto|vehicul|camion|mecanique|pneu|extermin|parasit|ascenseur|saaq|voiture|urbanisme|securite publique|metallurg|agricole|sanitaire", re.I)104105_lock = Lock()106_stats = {"queries": 0, "kept": 0, "raw": 0}107108109def norm(s: str) -> str:110 return unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode()111112113def slugify(s: str) -> str:114 s = norm(s).lower()115 s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")116 return s or "x"117118119def norm_phone(p: str | None) -> str:120 d = re.sub(r"\D", "", p or "")121 return d[-10:] if len(d) >= 10 else d122123124def serper_places(q: str, page: int) -> list[dict]:125 body = json.dumps({"q": q, "gl": "ca", "hl": "fr", "page": page}).encode()126 req = Request("https://google.serper.dev/places", data=body,127 headers={"X-API-KEY": API_KEY, "Content-Type": "application/json"})128 for attempt in range(3):129 try:130 with urlopen(req, timeout=30) as r:131 with _lock:132 _stats["queries"] += 1133 return json.loads(r.read()).get("places", [])134 except Exception:135 time.sleep(2 * (attempt + 1))136 return []137138139def is_mover(p: dict) -> bool:140 blob = norm(p.get("category", "")) + " " + norm(p.get("title", ""))141 return bool(MOVER_RE.search(blob)) and not EXCLUDE_RE.search(blob)142143144def in_quebec(p: dict) -> bool:145 addr = p.get("address", "")146 return (", QC" in addr or " QC " in addr or addr.endswith(" QC")147 or not addr) # certaines fiches n'ont pas d'adresse : région du balayage148149150CITY_RE = re.compile(r",\s*([^,]+?),\s*(?:QC|Québec)\b")151152153def parse_city(addr: str) -> str:154 m = CITY_RE.search(addr or "")155 return m.group(1).strip() if m else ""156157158def sweep_city(region: str, city: str) -> list[dict]:159 label = city.replace(" QC", "").replace(" Montréal", "").replace(" Gatineau", "").replace(" Laval", "").replace(" Longueuil", "")160 queries = [f"inspecteur en bâtiment {city} QC"]161 max_pages = 5 if city in BIG else 3162 if city in BIG:163 queries.append(f"inspection préachat {city} QC")164 out = []165 for q in queries:166 for page in range(1, max_pages + 1):167 places = serper_places(q, page)168 with _lock:169 _stats["raw"] += len(places)170 for p in places:171 if is_mover(p) and in_quebec(p):172 p["_region"] = region173 p["_query_city"] = label174 out.append(p)175 if len(places) < 10:176 break177 return out178179180def main() -> None:181 if not API_KEY:182 sys.exit("SERPER_API_KEY manquante")183 jobs = [(r, c) for r, cities in REGIONS.items() for c in cities]184 # ville balayée → région (pour rattacher la ville réelle de l'adresse)185 city_region = {}186 for r, cities in REGIONS.items():187 for c in cities:188 city_region[norm(c.split(" QC")[0]).lower()] = r189190 raw: list[dict] = []191 with ThreadPoolExecutor(max_workers=8) as ex:192 futs = {ex.submit(sweep_city, r, c): (r, c) for r, c in jobs}193 done = 0194 for f in as_completed(futs):195 raw.extend(f.result())196 done += 1197 if done % 25 == 0:198 print(f" … {done}/{len(jobs)} villes, {len(raw)} fiches brutes, "199 f"{_stats['queries']} requêtes", flush=True)200201 # Dédoublonnage : CID d'abord, sinon téléphone, sinon (nom, ville)202 by_key: dict[str, dict] = {}203 for p in raw:204 key = ("cid:" + p["cid"]) if p.get("cid") else ""205 if not key:206 ph = norm_phone(p.get("phoneNumber"))207 key = ("tel:" + ph) if ph else ("nc:" + slugify(p.get("title", "")) + ":" + slugify(parse_city(p.get("address", ""))))208 cur = by_key.get(key)209 if cur is None or (not cur.get("website") and p.get("website")):210 if cur:211 p.setdefault("_region", cur["_region"])212 by_key[key] = p213214 movers = []215 seen_slug: set[str] = set()216 for p in by_key.values():217 addr_city = parse_city(p.get("address", ""))218 region = city_region.get(norm(addr_city).lower(), p["_region"])219 city = addr_city or p["_query_city"]220 base = slugify(f"{p.get('title','')}-{city}")221 slug = base222 i = 2223 while slug in seen_slug:224 slug = f"{base}-{i}"225 i += 1226 seen_slug.add(slug)227 movers.append({228 "id": slug,229 "name": (p.get("title") or "").strip(),230 "city": city,231 "region": region,232 "address": p.get("address", ""),233 "phone": p.get("phoneNumber", ""),234 "website": p.get("website", ""),235 "rating": p.get("rating"),236 "reviews": p.get("ratingCount"),237 "lat": p.get("latitude"),238 "lng": p.get("longitude"),239 "category": p.get("category", ""),240 "cid": p.get("cid", ""),241 })242243 movers.sort(key=lambda m: (norm(m["region"]), norm(m["city"]), norm(m["name"])))244 doc = {245 "generated": time.strftime("%Y-%m-%d"),246 "source": "Google Places via Serper — balayage " + str(len(jobs)) + " villes, 17 régions",247 "count": len(movers),248 "movers": movers,249 }250 OUT.write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")251 regions = {}252 for m in movers:253 regions[m["region"]] = regions.get(m["region"], 0) + 1254 print(f"\n✅ {len(movers)} inspecteurs uniques ({_stats['raw']} fiches brutes, "255 f"{_stats['queries']} requêtes Serper) → {OUT}")256 for r, n in sorted(regions.items(), key=lambda x: -x[1]):257 print(f" {r}: {n}")258259260if __name__ == "__main__":261 main()262