SPB Git forge
spboucher.ai ↗

spb/lou-ka

Public

Lou·Ka — tous les logements à louer du Québec, un seul endroit.

www.lou-ka.com
webappreal-estatequebecrental
232commits 1branches 0releases
172.9 MBsize
maindefault branch
3 days agolast push
HTML 98.9% Python 0.6%

geocode : mode EN LOT Adresses Québec (porté d'Immo-Ka) — peuple Lou-Ka Maps

run.py geocode = lots de 200 adresses normalisées (geocodeAddresses),
1 entrée par immeuble, cache partagé, validation bbox par ville ;
l'ancien mode 1-par-1 (Nominatim) reste sous « geocode1 ».
Résultat : 11 033 → 14 206 annonces géolocalisées (77 % des actives).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 13, 2026) parent 390a807

2 changed files +120 −1

modified louka/geocode.py +114 −0
@@ -256,3 +256,117 @@ def run(limit: int | None = None) -> dict:
256 256 "unique_addresses": len(groupes), "api_requests": requests_made}
257 257 print(f"[lou-ka] geocode {stats}")
258 258 return stats
259 +
260 +
261 +# --------------------------------------------------------------------------
262 +# Géocodage EN LOT (Adresses Québec `geocodeAddresses`) — porté d'Immo-Ka.
263 +# ~40 immeubles/requête POST au lieu de 1 adresse/1,1 s : peuple la carte
264 +# Lou-Ka Maps (framework Ka Maps) en minutes plutôt qu'en heures.
265 +# --------------------------------------------------------------------------
266 +AQ_BATCH_URL = ("https://servicescarto.mern.gouv.qc.ca/pes/rest/services/Territoire/"
267 + "Adresse_Geocodage/GeocodeServer/geocodeAddresses")
268 +BATCH_SIZE = 200
269 +
270 +
271 +def run_batch(limit: int | None = None) -> dict:
272 + """Géocode toutes les annonces sans coordonnées, par lots de 200 adresses
273 + normalisées (1 entrée par immeuble — le cache sert toutes les unités)."""
274 + import json as _json
275 + import sqlite3 as _sqlite3
276 +
277 + con = db.connect()
278 + geo = Geocoder(con) # réutilise _clean / bbox
279 + rows = con.execute(
280 + """SELECT uid, address, city FROM listings
281 + WHERE active=1 AND lat IS NULL AND address IS NOT NULL
282 + AND address<>'' AND geocode_failed=0 ORDER BY address""").fetchall()
283 + uniq: dict[str, dict] = {}
284 + for r in rows:
285 + k = norm_key(r["address"])
286 + if not k or len(k) < 6:
287 + continue
288 + u = uniq.setdefault(k, {"address": r["address"], "city": r["city"],
289 + "members": []})
290 + u["members"].append(r["uid"])
291 + pending = []
292 + for k, u in uniq.items():
293 + c = con.execute("SELECT lat,lng,failed FROM geocode_cache WHERE address=?",
294 + (k,)).fetchone()
295 + if c is not None and not c["failed"]:
296 + for uid in u["members"]:
297 + con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",
298 + (c["lat"], c["lng"], uid))
299 + continue
300 + if c is not None and c["failed"]:
301 + continue
302 + pending.append((k, u))
303 + con.commit()
304 + if limit is not None:
305 + pending = pending[:limit]
306 +
307 + session = requests.Session()
308 + session.headers["User-Agent"] = USER_AGENT
309 + done = failed = 0
310 + for i in range(0, len(pending), BATCH_SIZE):
311 + chunk = pending[i:i + BATCH_SIZE]
312 + records = {"records": [
313 + {"attributes": {"OBJECTID": j,
314 + "SingleLine": f"{geo._clean(u['address']).split(',')[0].strip()}, "
315 + f"{(u['city'] or 'Québec').strip()}"}}
316 + for j, (_k, u) in enumerate(chunk)]}
317 + try:
318 + resp = session.post(AQ_BATCH_URL, data={
319 + "addresses": _json.dumps(records, ensure_ascii=False),
320 + "f": "json", "outSR": 4326}, timeout=90)
321 + locs = resp.json().get("locations", [])
322 + except Exception as e:
323 + print(f"[lou-ka] geocode-batch lot {i//BATCH_SIZE} ERREUR: {str(e)[:80]}")
324 + time.sleep(1.0)
325 + continue
326 + by_id = {l["attributes"].get("ResultID"): l for l in locs}
327 + for j, (k, u) in enumerate(chunk):
328 + loc = by_id.get(j)
329 + coords = None
330 + if loc and loc["attributes"].get("Score", 0) >= AQ_MIN_SCORE:
331 + lc = loc.get("location") or {}
332 + try:
333 + cand = (float(lc["y"]), float(lc["x"]))
334 + if _in_bbox(*cand, _bbox_for(u["city"] or "")):
335 + coords = cand
336 + except (KeyError, ValueError, TypeError):
337 + coords = None
338 + try:
339 + if coords:
340 + for uid in u["members"]:
341 + con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",
342 + (coords[0], coords[1], uid))
343 + con.execute(
344 + "INSERT INTO geocode_cache (address,lat,lng,provider,failed,ts)"
345 + " VALUES (?,?,?,?,0,?) ON CONFLICT(address) DO UPDATE SET"
346 + " lat=excluded.lat, lng=excluded.lng, provider=excluded.provider,"
347 + " failed=0, ts=excluded.ts",
348 + (k, coords[0], coords[1], "adresses_quebec_batch", time.time()))
349 + done += len(u["members"])
350 + else:
351 + for uid in u["members"]:
352 + con.execute("UPDATE listings SET geocode_failed=1 WHERE uid=?", (uid,))
353 + con.execute(
354 + "INSERT INTO geocode_cache (address,lat,lng,provider,failed,ts)"
355 + " VALUES (?,?,?,?,1,?) ON CONFLICT(address) DO UPDATE SET"
356 + " failed=1, ts=excluded.ts",
357 + (k, None, None, "adresses_quebec_batch", time.time()))
358 + failed += len(u["members"])
359 + con.commit()
360 + except _sqlite3.OperationalError:
361 + try:
362 + con.rollback()
363 + except _sqlite3.Error:
364 + pass
365 + time.sleep(1.0)
366 + print(f"[lou-ka] geocode-batch {i+len(chunk)}/{len(pending)} "
367 + f"(résolues {done}, échecs {failed})")
368 + con.close()
369 + stats = {"geocoded": done, "failed": failed,
370 + "batches": (len(pending)+BATCH_SIZE-1)//BATCH_SIZE}
371 + print(f"[lou-ka] geocode-batch {stats}")
372 + return stats
modified run.py +6 −1
@@ -37,10 +37,15 @@ def main() -> None:
37 37 from louka import ingest
38 38 minutes = int(sys.argv[2]) if len(sys.argv) > 2 else 60
39 39 ingest.watch(minutes * 60)
40 − elif cmd == "geocode":
40 + elif cmd == "geocode1":
41 41 from louka import geocode
42 42 limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
43 43 geocode.run(limit)
44 + elif cmd == "geocode":
45 + # lot Adresses Québec (rapide) ; « geocode1 » = ancien mode 1-par-1
46 + from louka import geocode
47 + limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
48 + geocode.run_batch(limit)
44 49 elif cmd == "poi":
45 50 from louka import poi
46 51 limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
47 52