geocode : mode EN LOT Adresses Québec (porté d'Immo-Ka) — peuple Lou-Ka Maps · 2fc73d6 · lou-ka

geocode : mode EN LOT Adresses Québec (porté d'Immo-Ka) — peuple Lou-Ka Maps

run.py geocode = lots de 200 adresses normalisées (geocodeAddresses),
1 entrée par immeuble, cache partagé, validation bbox par ville ;
l'ancien mode 1-par-1 (Nominatim) reste sous « geocode1 ».
Résultat : 11 033 → 14 206 annonces géolocalisées (77 % des actives).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Simon-Pierre Boucher committed 2 mo ago (Aug 13, 2026) parent 390a807 Browse files

2 changed files +120 −1

modified louka/geocode.py +114 −0

+              "unique_addresses": len(groupes), "api_requests": requests_made}
+     print(f"[lou-ka] geocode {stats}")
+     return stats
++
++
++# --------------------------------------------------------------------------
++# Géocodage EN LOT (Adresses Québec `geocodeAddresses`) — porté d'Immo-Ka.
++# ~40 immeubles/requête POST au lieu de 1 adresse/1,1 s : peuple la carte
++# Lou-Ka Maps (framework Ka Maps) en minutes plutôt qu'en heures.
++# --------------------------------------------------------------------------
++AQ_BATCH_URL = ("https://servicescarto.mern.gouv.qc.ca/pes/rest/services/Territoire/"
++                "Adresse_Geocodage/GeocodeServer/geocodeAddresses")
++BATCH_SIZE = 200
++
++
++def run_batch(limit: int | None = None) -> dict:
++    """Géocode toutes les annonces sans coordonnées, par lots de 200 adresses
++    normalisées (1 entrée par immeuble — le cache sert toutes les unités)."""
++    import json as _json
++    import sqlite3 as _sqlite3
++
++    con = db.connect()
++    geo = Geocoder(con)     # réutilise _clean / bbox
++    rows = con.execute(
++        """SELECT uid, address, city FROM listings
++           WHERE active=1 AND lat IS NULL AND address IS NOT NULL
++             AND address<>'' AND geocode_failed=0 ORDER BY address""").fetchall()
++    uniq: dict[str, dict] = {}
++    for r in rows:
++        k = norm_key(r["address"])
++        if not k or len(k) < 6:
++            continue
++        u = uniq.setdefault(k, {"address": r["address"], "city": r["city"],
++                                "members": []})
++        u["members"].append(r["uid"])
++    pending = []
++    for k, u in uniq.items():
++        c = con.execute("SELECT lat,lng,failed FROM geocode_cache WHERE address=?",
++                        (k,)).fetchone()
++        if c is not None and not c["failed"]:
++            for uid in u["members"]:
++                con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",
++                            (c["lat"], c["lng"], uid))
++            continue
++        if c is not None and c["failed"]:
++            continue
++        pending.append((k, u))
++    con.commit()
++    if limit is not None:
++        pending = pending[:limit]
++
++    session = requests.Session()
++    session.headers["User-Agent"] = USER_AGENT
++    done = failed = 0
++    for i in range(0, len(pending), BATCH_SIZE):
++        chunk = pending[i:i + BATCH_SIZE]
++        records = {"records": [
++            {"attributes": {"OBJECTID": j,
++                            "SingleLine": f"{geo._clean(u['address']).split(',')[0].strip()}, "
++                                          f"{(u['city'] or 'Québec').strip()}"}}
++            for j, (_k, u) in enumerate(chunk)]}
++        try:
++            resp = session.post(AQ_BATCH_URL, data={
++                "addresses": _json.dumps(records, ensure_ascii=False),
++                "f": "json", "outSR": 4326}, timeout=90)
++            locs = resp.json().get("locations", [])
++        except Exception as e:
++            print(f"[lou-ka] geocode-batch lot {i//BATCH_SIZE} ERREUR: {str(e)[:80]}")
++            time.sleep(1.0)
++            continue
++        by_id = {l["attributes"].get("ResultID"): l for l in locs}
++        for j, (k, u) in enumerate(chunk):
++            loc = by_id.get(j)
++            coords = None
++            if loc and loc["attributes"].get("Score", 0) >= AQ_MIN_SCORE:
++                lc = loc.get("location") or {}
++                try:
++                    cand = (float(lc["y"]), float(lc["x"]))
++                    if _in_bbox(*cand, _bbox_for(u["city"] or "")):
++                        coords = cand
++                except (KeyError, ValueError, TypeError):
++                    coords = None
++            try:
++                if coords:
++                    for uid in u["members"]:
++                        con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",
++                                    (coords[0], coords[1], uid))
++                    con.execute(
++                        "INSERT INTO geocode_cache (address,lat,lng,provider,failed,ts)"
++                        " VALUES (?,?,?,?,0,?) ON CONFLICT(address) DO UPDATE SET"
++                        " lat=excluded.lat, lng=excluded.lng, provider=excluded.provider,"
++                        " failed=0, ts=excluded.ts",
++                        (k, coords[0], coords[1], "adresses_quebec_batch", time.time()))
++                    done += len(u["members"])
++                else:
++                    for uid in u["members"]:
++                        con.execute("UPDATE listings SET geocode_failed=1 WHERE uid=?", (uid,))
++                    con.execute(
++                        "INSERT INTO geocode_cache (address,lat,lng,provider,failed,ts)"
++                        " VALUES (?,?,?,?,1,?) ON CONFLICT(address) DO UPDATE SET"
++                        " failed=1, ts=excluded.ts",
++                        (k, None, None, "adresses_quebec_batch", time.time()))
++                    failed += len(u["members"])
++                con.commit()
++            except _sqlite3.OperationalError:
++                try:
++                    con.rollback()
++                except _sqlite3.Error:
++                    pass
++                time.sleep(1.0)
++        print(f"[lou-ka] geocode-batch {i+len(chunk)}/{len(pending)} "
++              f"(résolues {done}, échecs {failed})")
++    con.close()
++    stats = {"geocoded": done, "failed": failed,
++             "batches": (len(pending)+BATCH_SIZE-1)//BATCH_SIZE}
++    print(f"[lou-ka] geocode-batch {stats}")
++    return stats

modified run.py +6 −1

				`@@ -37,10 +37,15 @@ def main() -> None:`
37	from louka import ingest	37	from louka import ingest
38	minutes = int(sys.argv[2]) if len(sys.argv) > 2 else 60	38	minutes = int(sys.argv[2]) if len(sys.argv) > 2 else 60
39	ingest.watch(minutes * 60)	39	ingest.watch(minutes * 60)
40	− elif cmd == "geocode":	40	+ elif cmd == "geocode1":
41	from louka import geocode	41	from louka import geocode
42	limit = int(sys.argv[2]) if len(sys.argv) > 2 else None	42	limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
43	geocode.run(limit)	43	geocode.run(limit)
		44	+ elif cmd == "geocode":
		45	+ # lot Adresses Québec (rapide) ; « geocode1 » = ancien mode 1-par-1
		46	+ from louka import geocode
		47	+ limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
		48	+ geocode.run_batch(limit)
44	elif cmd == "poi":	49	elif cmd == "poi":
45	from louka import poi	50	from louka import poi
46	limit = int(sys.argv[2]) if len(sys.argv) > 2 else None	51	limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
47		52

spb/lou-ka

geocode : mode EN LOT Adresses Québec (porté d'Immo-Ka) — peuple Lou-Ka Maps

Theme