Python 67%
TypeScript 18.2%
CSS 14.4%
1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# Groupe KA — Annuaire des déménageurs du Québec4# build_demenageurs.py : balayage Serper Places sur ~200 villes des 17 régions5# administratives, filtre « déménagement », dédoublonnage CID/téléphone,6# sortie canonique data/demenageurs.json (consommée par Lou-Ka et Immo-Ka).7# -----------------------------------------------------------------------------8from __future__ import annotations910import json11import os12import re13import sys14import time15import unicodedata16from concurrent.futures import ThreadPoolExecutor, as_completed17from pathlib import Path18from threading import Lock19from urllib.request import Request, urlopen2021API_KEY = os.environ.get("SERPER_API_KEY", "")22OUT = Path(__file__).parent / "demenageurs.json"2324# Région administrative → villes balayées (couverture provinciale complète)25REGIONS: dict[str, list[str]] = {26 "Bas-Saint-Laurent": [27 "Rimouski", "Rivière-du-Loup", "Matane", "Mont-Joli", "Amqui",28 "La Pocatière", "Témiscouata-sur-le-Lac", "Trois-Pistoles", "Dégelis"],29 "Saguenay–Lac-Saint-Jean": [30 "Saguenay", "Chicoutimi", "Jonquière", "La Baie", "Alma",31 "Dolbeau-Mistassini", "Roberval", "Saint-Félicien", "Normandin"],32 "Capitale-Nationale": [33 "Québec", "L'Ancienne-Lorette", "Saint-Augustin-de-Desmaures",34 "Beaupré", "Baie-Saint-Paul", "La Malbaie", "Donnacona", "Pont-Rouge",35 "Saint-Raymond", "Sainte-Catherine-de-la-Jacques-Cartier",36 "Stoneham-et-Tewkesbury", "Boischatel", "Château-Richer"],37 "Mauricie": [38 "Trois-Rivières", "Shawinigan", "La Tuque", "Louiseville",39 "Saint-Tite", "Saint-Étienne-des-Grès"],40 "Estrie": [41 "Sherbrooke", "Magog", "Granby", "Cowansville", "Bromont",42 "Coaticook", "Lac-Mégantic", "Windsor", "East Angus",43 "Val-des-Sources", "Waterloo"],44 "Montréal": [45 "Montréal", "Montréal-Nord", "Saint-Laurent Montréal", "LaSalle",46 "Anjou", "Ahuntsic", "Rosemont", "Hochelaga-Maisonneuve",47 "Le Plateau-Mont-Royal", "Verdun", "Lachine", "Pierrefonds",48 "Rivière-des-Prairies", "Côte-des-Neiges", "Westmount",49 "Côte-Saint-Luc", "Dollard-des-Ormeaux", "Pointe-Claire", "Dorval",50 "Kirkland", "Beaconsfield", "Sainte-Anne-de-Bellevue", "Outremont",51 "Mont-Royal"],52 "Outaouais": [53 "Gatineau", "Hull Gatineau", "Aylmer Gatineau", "Buckingham Gatineau",54 "Cantley", "Chelsea", "Val-des-Monts", "Maniwaki", "Papineauville",55 "La Pêche"],56 "Abitibi-Témiscamingue": [57 "Rouyn-Noranda", "Val-d'Or", "Amos", "La Sarre", "Ville-Marie QC",58 "Senneterre", "Malartic"],59 "Côte-Nord": [60 "Baie-Comeau", "Sept-Îles", "Port-Cartier", "Forestville",61 "Havre-Saint-Pierre", "Fermont"],62 "Nord-du-Québec": [63 "Chibougamau", "Lebel-sur-Quévillon", "Matagami", "Chapais"],64 "Gaspésie–Îles-de-la-Madeleine": [65 "Gaspé", "Chandler", "New Richmond", "Bonaventure",66 "Carleton-sur-Mer", "Sainte-Anne-des-Monts", "Percé",67 "Les Îles-de-la-Madeleine"],68 "Chaudière-Appalaches": [69 "Lévis", "Saint-Georges", "Thetford Mines", "Montmagny",70 "Sainte-Marie", "Beauceville", "Saint-Joseph-de-Beauce", "L'Islet",71 "Lac-Etchemin", "Saint-Anselme", "Saint-Henri"],72 "Laval": ["Laval", "Chomedey Laval", "Sainte-Rose Laval"],73 "Lanaudière": [74 "Terrebonne", "Repentigny", "Mascouche", "Joliette", "L'Assomption",75 "Rawdon", "Saint-Lin-Laurentides", "Berthierville", "Lavaltrie",76 "Saint-Charles-Borromée", "Chertsey"],77 "Laurentides": [78 "Saint-Jérôme", "Blainville", "Boisbriand", "Sainte-Thérèse",79 "Mirabel", "Saint-Eustache", "Deux-Montagnes", "Rosemère",80 "Bois-des-Filion", "Sainte-Adèle", "Sainte-Agathe-des-Monts",81 "Mont-Tremblant", "Mont-Laurier", "Prévost", "Saint-Sauveur",82 "Lachute", "Saint-Colomban"],83 "Montérégie": [84 "Longueuil", "Brossard", "Saint-Lambert", "Boucherville",85 "Saint-Bruno-de-Montarville", "Saint-Hubert Longueuil",86 "Châteauguay", "Saint-Jean-sur-Richelieu", "Salaberry-de-Valleyfield",87 "Vaudreuil-Dorion", "Saint-Hyacinthe", "Sorel-Tracy", "Beloeil",88 "Mont-Saint-Hilaire", "Chambly", "La Prairie", "Candiac",89 "Sainte-Catherine QC", "Saint-Constant", "Mercier", "Beauharnois",90 "Varennes", "Sainte-Julie", "Contrecoeur", "Marieville", "Farnham",91 "Acton Vale", "Rigaud", "L'Île-Perrot", "Pincourt", "Saint-Rémi"],92 "Centre-du-Québec": [93 "Drummondville", "Victoriaville", "Bécancour", "Nicolet",94 "Plessisville", "Princeville", "Kingsey Falls", "Warwick QC"],95}9697# Grandes villes : pagination plus profonde + variante de requête98BIG = {"Montréal", "Québec", "Laval", "Gatineau", "Longueuil", "Sherbrooke",99 "Trois-Rivières", "Saguenay", "Lévis", "Terrebonne", "Saint-Jérôme",100 "Brossard", "Repentigny", "Drummondville", "Saint-Jean-sur-Richelieu"}101102MOVER_RE = re.compile(r"demenag|moving|movers|transport et demenagement", re.I)103104_lock = Lock()105_stats = {"queries": 0, "kept": 0, "raw": 0}106107108def norm(s: str) -> str:109 return unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode()110111112def slugify(s: str) -> str:113 s = norm(s).lower()114 s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")115 return s or "x"116117118def norm_phone(p: str | None) -> str:119 d = re.sub(r"\D", "", p or "")120 return d[-10:] if len(d) >= 10 else d121122123def serper_places(q: str, page: int) -> list[dict]:124 body = json.dumps({"q": q, "gl": "ca", "hl": "fr", "page": page}).encode()125 req = Request("https://google.serper.dev/places", data=body,126 headers={"X-API-KEY": API_KEY, "Content-Type": "application/json"})127 for attempt in range(3):128 try:129 with urlopen(req, timeout=30) as r:130 with _lock:131 _stats["queries"] += 1132 return json.loads(r.read()).get("places", [])133 except Exception:134 time.sleep(2 * (attempt + 1))135 return []136137138def is_mover(p: dict) -> bool:139 return bool(MOVER_RE.search(norm(p.get("category", "")))140 or MOVER_RE.search(norm(p.get("title", ""))))141142143def in_quebec(p: dict) -> bool:144 addr = p.get("address", "")145 return (", QC" in addr or " QC " in addr or addr.endswith(" QC")146 or not addr) # certaines fiches n'ont pas d'adresse : région du balayage147148149CITY_RE = re.compile(r",\s*([^,]+?),\s*(?:QC|Québec)\b")150151152def parse_city(addr: str) -> str:153 m = CITY_RE.search(addr or "")154 return m.group(1).strip() if m else ""155156157def sweep_city(region: str, city: str) -> list[dict]:158 label = city.replace(" QC", "").replace(" Montréal", "").replace(" Gatineau", "").replace(" Laval", "").replace(" Longueuil", "")159 queries = [f"déménageur {city} QC"]160 max_pages = 5 if city in BIG else 3161 if city in BIG:162 queries.append(f"entreprise de déménagement {city} QC")163 out = []164 for q in queries:165 for page in range(1, max_pages + 1):166 places = serper_places(q, page)167 with _lock:168 _stats["raw"] += len(places)169 for p in places:170 if is_mover(p) and in_quebec(p):171 p["_region"] = region172 p["_query_city"] = label173 out.append(p)174 if len(places) < 10:175 break176 return out177178179def main() -> None:180 if not API_KEY:181 sys.exit("SERPER_API_KEY manquante")182 jobs = [(r, c) for r, cities in REGIONS.items() for c in cities]183 # ville balayée → région (pour rattacher la ville réelle de l'adresse)184 city_region = {}185 for r, cities in REGIONS.items():186 for c in cities:187 city_region[norm(c.split(" QC")[0]).lower()] = r188189 raw: list[dict] = []190 with ThreadPoolExecutor(max_workers=8) as ex:191 futs = {ex.submit(sweep_city, r, c): (r, c) for r, c in jobs}192 done = 0193 for f in as_completed(futs):194 raw.extend(f.result())195 done += 1196 if done % 25 == 0:197 print(f" … {done}/{len(jobs)} villes, {len(raw)} fiches brutes, "198 f"{_stats['queries']} requêtes", flush=True)199200 # Dédoublonnage : CID d'abord, sinon téléphone, sinon (nom, ville)201 by_key: dict[str, dict] = {}202 for p in raw:203 key = ("cid:" + p["cid"]) if p.get("cid") else ""204 if not key:205 ph = norm_phone(p.get("phoneNumber"))206 key = ("tel:" + ph) if ph else ("nc:" + slugify(p.get("title", "")) + ":" + slugify(parse_city(p.get("address", ""))))207 cur = by_key.get(key)208 if cur is None or (not cur.get("website") and p.get("website")):209 if cur:210 p.setdefault("_region", cur["_region"])211 by_key[key] = p212213 movers = []214 seen_slug: set[str] = set()215 for p in by_key.values():216 addr_city = parse_city(p.get("address", ""))217 region = city_region.get(norm(addr_city).lower(), p["_region"])218 city = addr_city or p["_query_city"]219 base = slugify(f"{p.get('title','')}-{city}")220 slug = base221 i = 2222 while slug in seen_slug:223 slug = f"{base}-{i}"224 i += 1225 seen_slug.add(slug)226 movers.append({227 "id": slug,228 "name": (p.get("title") or "").strip(),229 "city": city,230 "region": region,231 "address": p.get("address", ""),232 "phone": p.get("phoneNumber", ""),233 "website": p.get("website", ""),234 "rating": p.get("rating"),235 "reviews": p.get("ratingCount"),236 "lat": p.get("latitude"),237 "lng": p.get("longitude"),238 "category": p.get("category", ""),239 "cid": p.get("cid", ""),240 })241242 movers.sort(key=lambda m: (norm(m["region"]), norm(m["city"]), norm(m["name"])))243 doc = {244 "generated": time.strftime("%Y-%m-%d"),245 "source": "Google Places via Serper — balayage " + str(len(jobs)) + " villes, 17 régions",246 "count": len(movers),247 "movers": movers,248 }249 OUT.write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")250 regions = {}251 for m in movers:252 regions[m["region"]] = regions.get(m["region"], 0) + 1253 print(f"\n✅ {len(movers)} déménageurs uniques ({_stats['raw']} fiches brutes, "254 f"{_stats['queries']} requêtes Serper) → {OUT}")255 for r, n in sorted(regions.items(), key=lambda x: -x[1]):256 print(f" {r}: {n}")257258259if __name__ == "__main__":260 main()261