Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# geocode.py : address geocoding via Nominatim (OpenStreetMap)5# - persistent cache (geocode_cache table): an address is geocoded once,6# per building (normalized address), not per listing; keys are prefixed7# by the province code ("on|", "bc|", …) to avoid cross-province8# collisions between homonymous addresses9# - politeness: max 1 request/second, identifiable User-Agent10# - validation: coordinates must fall inside the expected province11# bounding box, otherwise geocode_failed is flagged — never bogus12# coordinates13# -----------------------------------------------------------------------------14from __future__ import annotations1516import re17import time1819import requests2021from . import db22from .normalize import strip_accents23from .schema import CANADA_BBOX, PROVINCE_BBOX2425NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"26USER_AGENT = "RentKaBot/1.0 (Canadian rentals aggregator; +contact@spboucher.ai)"27REQUEST_DELAY = 1.1 # Nominatim rule: max 1 req/s28RETRY_FAILED_AFTER = 30 * 86400 # retry failures after 30 days2930PROVINCE_NAMES = {31 "ON": "Ontario", "BC": "British Columbia", "AB": "Alberta",32 "SK": "Saskatchewan", "MB": "Manitoba", "NB": "New Brunswick",33 "NS": "Nova Scotia", "PE": "Prince Edward Island",34 "NL": "Newfoundland and Labrador", "YT": "Yukon",35 "NT": "Northwest Territories", "NU": "Nunavut", "QC": "Québec",36}373839def _bbox_for(province: str = "ON") -> tuple[float, float, float, float]:40 return PROVINCE_BBOX.get((province or "ON").upper(), CANADA_BBOX)414243def _in_bbox(lat: float, lng: float, bbox: tuple) -> bool:44 return bbox[0] <= lat <= bbox[1] and bbox[2] <= lng <= bbox[3]454647def norm_key(address: str) -> str:48 """Cache key: normalized address (case, accents, spaces, unit numbers)."""49 s = strip_accents((address or "").lower())50 s = re.sub(r"\b(?:app?t?|appartement|unite|suite|#)\.?\s*[\w-]+\b", " ", s)51 s = re.sub(r"[^a-z0-9]+", " ", s)52 return re.sub(r"\s+", " ", s).strip()535455def cache_key(address: str, province: str = "ON") -> str:56 """Province-aware geocode_cache key: «on|…», «bc|…» — avoids collisions57 between homonymous addresses across provinces. The existing Ontario58 cache («on|» prefix, inherited from the Rent-Ka era) is preserved."""59 k = norm_key(address)60 if not k:61 return k62 return f"{(province or 'ON').lower()}|{k}"636465class Geocoder:66 def __init__(self, con) -> None:67 self.con = con68 self.session = requests.Session()69 self.session.headers["User-Agent"] = USER_AGENT70 self._last = 0.07172 @staticmethod73 def _clean(address: str) -> str:74 """Cleanups that help Nominatim on Canadian addresses."""75 s = address.strip()76 # «14340, Main St» (comma after the civic number, French style)77 s = re.sub(r"^(\d+[A-Za-z]?),\s+", r"\1 ", s)78 # «101-2905 Main St» = unit 101, civic 2905 -> keep the civic79 s = re.sub(r"^(\d+)-(\d+)\s", r"\2 ", s)80 # «6275 et 6375 boulevard X» / «6275 and 6375 X» -> first civic81 s = re.sub(r"^(\d+)\s+(?:et|and)\s+\d+\s", r"\1 ", s)82 # «bureau 105» / «suite 3» / «unit B»: office/unit suffixes83 s = re.sub(r",?\s*(?:bureau|suite|local|unit|app\.?|apt\.?)\s*[\w-]+\b",84 "", s, flags=re.I)85 return s8687 def _build_query(self, address: str, city: str,88 province: str = "ON") -> str:89 q = self._clean(address)90 prov = (province or "ON").upper()91 key = strip_accents(q.lower())92 if city and strip_accents(city.lower()) not in key:93 q += f", {city}"94 prov_name = PROVINCE_NAMES.get(prov, "")95 if prov_name and strip_accents(prov_name.lower()) not in key \96 and not re.search(rf"\b{prov}\b", q):97 q += f", {prov_name}"98 return q + ", Canada"99100 def _query_nominatim(self, params: dict) -> tuple[float, float] | None:101 """One throttled Nominatim request."""102 wait = REQUEST_DELAY - (time.time() - self._last)103 if wait > 0:104 time.sleep(wait)105 try:106 resp = self.session.get(NOMINATIM_URL, params={107 "format": "jsonv2", "limit": 1, "countrycodes": "ca", **params,108 }, timeout=20)109 self._last = time.time()110 resp.raise_for_status()111 hits = resp.json()112 except Exception:113 self._last = time.time()114 return None115 if not hits:116 return None117 try:118 return float(hits[0]["lat"]), float(hits[0]["lon"])119 except (KeyError, ValueError):120 return None121122 def _attempts(self, address: str, city: str,123 province: str = "ON") -> list[dict]:124 """Query strategies, most precise first.125126 1) structured civic+street127 2) full free-text search128 3) structured street only -> street centroid (acceptable map129 fallback when the civic number is missing from OpenStreetMap)130 """131 prov = (province or "ON").upper()132 premier = self._clean(address).split(",")[0].strip()133 ville = (city or "").strip()134 commun = {"state": PROVINCE_NAMES.get(prov, ""), "country": "Canada"}135 if ville:136 commun["city"] = ville137138 tries: list[dict] = []139 m = re.match(r"^(\d+)[,\s]+(.{4,})$", premier)140 if m:141 tries.append({"street": f"{m.group(1)} {m.group(2)}", **commun})142 tries.append({"q": self._build_query(address, city, prov)})143 if m:144 tries.append({"street": m.group(2), **commun})145 return tries146147 def resolve(self, address: str, city: str,148 province: str = "ON") -> tuple[float, float] | None:149 """Address -> (lat, lng), through the cache then Nominatim."""150 prov = (province or "ON").upper()151 nk = norm_key(address)152 if not nk or len(nk) < 6:153 return None154 key = cache_key(address, prov)155 row = self.con.execute(156 "SELECT lat, lng, failed, ts FROM geocode_cache WHERE address=?",157 (key,)).fetchone()158 if row is not None:159 if not row["failed"]:160 return (row["lat"], row["lng"])161 if time.time() - (row["ts"] or 0) < RETRY_FAILED_AFTER:162 return None # recent failure: don't hammer the API163164 bbox = _bbox_for(prov)165 coords = None166 for params in self._attempts(address, city, prov):167 c = self._query_nominatim(params)168 if c is not None and _in_bbox(*c, bbox):169 coords = c170 break171 ok = coords is not None172 self.con.execute(173 "INSERT INTO geocode_cache (address, lat, lng, provider, failed, ts)"174 " VALUES (?,?,?,?,?,?)"175 " ON CONFLICT(address) DO UPDATE SET lat=excluded.lat,"176 " lng=excluded.lng, provider=excluded.provider,"177 " failed=excluded.failed, ts=excluded.ts",178 (key, coords[0] if ok else None, coords[1] if ok else None,179 "nominatim", 0 if ok else 1, time.time()))180 self.con.commit()181 return coords if ok else None182183184def run(limit: int | None = None) -> dict:185 """Geocode active listings without coordinates, per unique address.186187 `limit` caps the number of NEW Nominatim requests (cache hits are free188 and always applied).189 """190 con = db.connect()191 geo = Geocoder(con)192 rows = con.execute(193 """SELECT uid, address, city, province FROM listings194 WHERE active=1 AND lat IS NULL AND address<>'' AND geocode_failed=0195 ORDER BY address""").fetchall()196197 # group per building (normalized address, province-prefixed)198 groupes: dict[str, list] = {}199 for r in rows:200 groupes.setdefault(cache_key(r["address"], r["province"]), []).append(r)201202 done = failed = requests_made = 0203 for key, members in groupes.items():204 if not key:205 continue206 cached = con.execute(207 "SELECT failed FROM geocode_cache WHERE address=?", (key,)).fetchone()208 if cached is None:209 if limit is not None and requests_made >= limit:210 continue211 requests_made += 1212 coords = geo.resolve(members[0]["address"], members[0]["city"],213 members[0]["province"])214 if coords:215 for r in members:216 con.execute("UPDATE listings SET lat=?, lng=? WHERE uid=?",217 (coords[0], coords[1], r["uid"]))218 done += len(members)219 else:220 # not found/out of zone: flag for review, never bogus coordinates221 for r in members:222 con.execute("UPDATE listings SET geocode_failed=1 WHERE uid=?",223 (r["uid"],))224 failed += len(members)225 con.commit()226227 con.close()228 stats = {"geocoded": done, "failed": failed,229 "unique_addresses": len(groupes), "api_requests": requests_made}230 print(f"[rent-ka] geocode {stats}")231 return stats232233234def run_batch(limit: int | None = None) -> dict:235 """Batch entry point kept for the CLI/watch loop.236237 The Rent-Ka era used the Adresses Québec bulk endpoint (Québec-only238 dataset); outside Québec everything goes through unit Nominatim239 resolution — this simply forwards to run() with the same signature.240 """241 return run(limit)242