Python 49.6%
TypeScript 25.5%
CSS 24.1%
1# -----------------------------------------------------------------------------2# Home-Ka — US real-estate aggregator (Groupe KA)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# geocode.py : US geocoding — Census Bureau batch geocoder (free, no key,5# 10 000 addresses per request) with persistent cache, Nominatim fallback6# on a small budget. Lowest-level source available, zero proxy.7# -----------------------------------------------------------------------------8from __future__ import annotations910import csv11import io12import time1314import requests1516from . import db1718CENSUS_BATCH = ("https://geocoding.geo.census.gov/geocoder/locations/"19 "addressbatch")20NOMINATIM = "https://nominatim.openstreetmap.org/search"21UA = "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)"2223BATCH_SIZE = 5000242526def _cache_key(street: str, city: str, state: str, zip_code: str) -> str:27 return f"{street}|{city}|{state}|{zip_code}".lower().strip()282930def run_batch(limit: int | None = None, nominatim_budget: int = 25) -> dict:31 """Geocode active listings without coordinates.3233 1) persistent cache; 2) Census batch (5k/req); 3) Nominatim fallback for34 the few that Census can't match (budgeted, 1 req/s politeness).35 """36 con = db.connect()37 rows = con.execute(38 "SELECT uid, street_address, city, state, zip_code FROM listings"39 " WHERE active=1 AND lat IS NULL AND geocode_failed=0"40 " AND street_address<>'' AND (city<>'' OR zip_code<>'')"41 " LIMIT ?", (limit or 20000,)).fetchall()42 if not rows:43 con.close()44 return {"pending": 0, "geocoded": 0}4546 hits = 047 todo = []48 for r in rows:49 key = _cache_key(r["street_address"], r["city"], r["state"], r["zip_code"])50 c = con.execute("SELECT lat, lng, failed FROM geocode_cache WHERE address=?",51 (key,)).fetchone()52 if c is not None:53 if c["lat"] is not None:54 con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",55 (c["lat"], c["lng"], r["uid"]))56 hits += 157 elif c["failed"]:58 con.execute("UPDATE listings SET geocode_failed=1 WHERE uid=?",59 (r["uid"],))60 else:61 todo.append(r)62 con.commit()6364 geocoded = failed = 065 for i in range(0, len(todo), BATCH_SIZE):66 chunk = todo[i:i + BATCH_SIZE]67 results = _census_batch(chunk)68 now = time.time()69 unmatched = []70 for r in chunk:71 key = _cache_key(r["street_address"], r["city"], r["state"], r["zip_code"])72 hit = results.get(r["uid"])73 if hit:74 lat, lng = hit75 con.execute(76 "INSERT OR REPLACE INTO geocode_cache (address, lat, lng,"77 " provider, failed, ts) VALUES (?,?,?,?,0,?)",78 (key, lat, lng, "census", now))79 con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",80 (lat, lng, r["uid"]))81 geocoded += 182 else:83 unmatched.append(r)84 # Nominatim fallback (budgeted, polite)85 for r in unmatched:86 key = _cache_key(r["street_address"], r["city"], r["state"], r["zip_code"])87 if nominatim_budget > 0:88 nominatim_budget -= 189 hit = _nominatim(r)90 if hit:91 con.execute(92 "INSERT OR REPLACE INTO geocode_cache (address, lat,"93 " lng, provider, failed, ts) VALUES (?,?,?,?,0,?)",94 (key, hit[0], hit[1], "nominatim", time.time()))95 con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",96 (hit[0], hit[1], r["uid"]))97 geocoded += 198 continue99 con.execute(100 "INSERT OR REPLACE INTO geocode_cache (address, lat, lng,"101 " provider, failed, ts) VALUES (?, NULL, NULL, 'census', 1, ?)",102 (key, time.time()))103 con.execute("UPDATE listings SET geocode_failed=1 WHERE uid=?",104 (r["uid"],))105 failed += 1106 con.commit()107 print(f"[home-ka] geocode: batch {i // BATCH_SIZE + 1} — "108 f"{geocoded} geocoded, {failed} failed")109 con.close()110 return {"pending": len(rows), "cache_hits": hits,111 "geocoded": geocoded, "failed": failed}112113114def _census_batch(rows) -> dict[str, tuple[float, float]]:115 """POST a CSV batch to the Census geocoder; {uid: (lat, lng)}."""116 buf = io.StringIO()117 w = csv.writer(buf)118 for r in rows:119 w.writerow([r["uid"], r["street_address"], r["city"], r["state"],120 r["zip_code"]])121 try:122 resp = requests.post(123 CENSUS_BATCH,124 files={"addressFile": ("batch.csv", buf.getvalue().encode(), "text/csv")},125 data={"benchmark": "Public_AR_Current"},126 headers={"User-Agent": UA}, timeout=300)127 resp.raise_for_status()128 except requests.RequestException as exc:129 print(f"[home-ka] geocode: census batch failed: {exc}")130 return {}131 out: dict[str, tuple[float, float]] = {}132 for row in csv.reader(io.StringIO(resp.text)):133 # id, input, match, exact, matched addr, "lng,lat", tigerline, side134 if len(row) >= 6 and row[2].strip().lower() == "match" and row[5]:135 try:136 lng, lat = (float(v) for v in row[5].split(","))137 out[row[0]] = (lat, lng)138 except ValueError:139 continue140 return out141142143def _nominatim(r) -> tuple[float, float] | None:144 q = ", ".join(filter(None, (r["street_address"], r["city"],145 r["state"], r["zip_code"], "USA")))146 try:147 time.sleep(1.1) # Nominatim usage policy148 resp = requests.get(NOMINATIM, params={149 "q": q, "format": "json", "limit": 1, "countrycodes": "us"},150 headers={"User-Agent": UA}, timeout=30)151 resp.raise_for_status()152 data = resp.json()153 if data:154 return float(data[0]["lat"]), float(data[0]["lon"])155 except (requests.RequestException, ValueError, KeyError, IndexError):156 pass157 return None158