Python 49.6%
TypeScript 25.5%
CSS 24.1%
1# -----------------------------------------------------------------------------2# Home-Ka — US real-estate aggregator (Groupe KA)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/public_data/arcgis.py : county assessor / GIS parcel connector.5#6# US counties publish parcel/assessor layers on ArcGIS FeatureServers (open7# data portals) — the lowest-level public source there is. This connector8# feeds the PROPERTIES table (physical assets: APN, address, year built,9# areas, assessed values), NOT the listings table: it enriches property10# matching and keeps properties alive after listings go off-market.11#12# {"id": "county_travis_tx", "connector_type": "arcgis", "config": {13# "layer_url": "https://services.arcgis.com/.../FeatureServer/0",14# "where": "1=1", "out_fields": "*", "page_size": 2000,15# "max_records": 200000,16# "mapping": {"apn": "PROP_ID", "street_address": "SITUS_ADDR",17# "city": "SITUS_CITY", "zip_code": "SITUS_ZIP",18# "year_built": "YEAR_BUILT", "living_area_sqft": "LIVING_AREA",19# "assessed_value": "ASSESSED_VAL"},20# "static": {"state": "TX", "county": "Travis"}}}21# -----------------------------------------------------------------------------22from __future__ import annotations2324import json25import time2627from ..base import BaseConnector28from .._maputil import dig29from ...schema import Listing3031# properties-table columns a mapping can target; everything else goes to details32_PROP_FIELDS = {"apn", "street_address", "unit", "city", "state", "zip_code",33 "county", "property_type", "year_built", "living_area_sqft",34 "lot_size_sqft", "lat", "lng"}353637class ArcGISParcelConnector(BaseConnector):38 family = "arcgis"39 request_delay = 0.440 is_public_records = True # ingest.py routes fetch_records() → properties4142 def fetch(self) -> list[Listing]:43 return [] # public records never create listings4445 def fetch_records(self) -> list[dict]:46 """Yield property records {column: value, 'details': {...}}."""47 cfg = self.config48 layer = (cfg.get("layer_url") or "").rstrip("/")49 if not layer:50 raise RuntimeError(f"{self.source_id}: layer_url missing")51 mapping = cfg.get("mapping") or {}52 static = cfg.get("static") or {}53 page = int(cfg.get("page_size", 2000))54 max_records = int(cfg.get("max_records", 200_000))55 offset = 056 out: list[dict] = []57 while offset < max_records:58 params = {59 "f": "json", "where": cfg.get("where", "1=1"),60 "outFields": cfg.get("out_fields", "*"),61 "resultOffset": offset, "resultRecordCount": page,62 # polygons are heavy: ask for centroids only by default63 "returnGeometry": "true" if cfg.get("return_geometry") else "false",64 "returnCentroid": "true", "outSR": 4326,65 }66 data = self.get(f"{layer}/query", params=params).json()67 feats = data.get("features") or []68 if not feats:69 break70 for f in feats:71 attrs = f.get("attributes") or {}72 rec: dict = {"details": {}}73 for field, path in mapping.items():74 if "{" in str(path): # template: "{LOCN} {LOCS} {LOCT}"75 safe = {k: ("" if v is None else v)76 for k, v in attrs.items()}77 try:78 v = " ".join(str(path).format(**safe).split())79 except (KeyError, IndexError):80 v = None81 else:82 v = dig(attrs, path)83 if v in (None, ""):84 continue85 if field in _PROP_FIELDS:86 rec[field] = v87 else:88 rec["details"][field] = v89 for field, v in static.items():90 rec.setdefault(field, v)91 geom = f.get("centroid") or f.get("geometry") or {}92 if "lat" not in rec and geom.get("y") is not None:93 rec["lat"], rec["lng"] = geom.get("y"), geom.get("x")94 if rec.get("apn") or rec.get("street_address"):95 out.append(rec)96 if not data.get("exceededTransferLimit") and len(feats) < page:97 break98 offset += len(feats)99 return out100101102def sync_records(con, source_id: str, records: list[dict]) -> dict:103 """Upsert public-records rows into `properties` (match APN, then address)."""104 from ... import propertymatch105 from ...normalize import clean_title, normalize_zip106 now = time.time()107 added = updated = 0108 for rec in records:109 # assessor exports: ZIP as float ("34446.0"), SHOUTING city names110 if rec.get("zip_code") is not None:111 rec["zip_code"] = normalize_zip(rec["zip_code"])112 if rec.get("city"):113 rec["city"] = clean_title(str(rec["city"]).strip().title())114 if rec.get("street_address"):115 rec["street_address"] = str(rec["street_address"]).strip().title()116 state = str(rec.get("state") or "")117 apn = str(rec.get("apn") or "").replace(" ", "")118 # address+unit key FIRST — APNs are not unit-unique for condos119 # (whole buildings share one assessor account in several counties)120 key = propertymatch.addr_key(str(rec.get("street_address") or ""),121 str(rec.get("unit") or ""),122 str(rec.get("city") or ""), state,123 str(rec.get("zip_code") or ""))124 row = None125 if key:126 row = con.execute("SELECT id, details FROM properties"127 " WHERE addr_key=?", (key,)).fetchone()128 elif apn and state:129 row = con.execute("SELECT id, details FROM properties"130 " WHERE apn=? AND state=?", (apn, state)).fetchone()131 details = rec.get("details") or {}132 if row is None:133 if not key and not apn:134 continue135 con.execute(136 """INSERT OR IGNORE INTO properties (addr_key, apn,137 street_address, unit, city, state, zip_code, county,138 property_type, year_built, living_area_sqft, lot_size_sqft,139 lat, lng, details, first_seen, last_seen)140 VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",141 (key, apn or None, rec.get("street_address"), rec.get("unit"),142 rec.get("city"), state, rec.get("zip_code"), rec.get("county"),143 rec.get("property_type"), rec.get("year_built"),144 rec.get("living_area_sqft"), rec.get("lot_size_sqft"),145 rec.get("lat"), rec.get("lng"),146 json.dumps(details, ensure_ascii=False, default=str), now, now))147 added += 1148 else:149 old = json.loads(row["details"] or "{}")150 old.update(details)151 con.execute(152 """UPDATE properties SET153 apn=COALESCE(NULLIF(apn,''), ?),154 year_built=COALESCE(year_built, ?),155 living_area_sqft=COALESCE(living_area_sqft, ?),156 lot_size_sqft=COALESCE(lot_size_sqft, ?),157 lat=COALESCE(lat, ?), lng=COALESCE(lng, ?),158 details=?, last_seen=? WHERE id=?""",159 (apn or None, rec.get("year_built"),160 rec.get("living_area_sqft"), rec.get("lot_size_sqft"),161 rec.get("lat"), rec.get("lng"),162 json.dumps(old, ensure_ascii=False, default=str), now,163 row["id"]))164 updated += 1165 con.commit()166 return {"source": source_id, "found": len(records),167 "added": added, "updated": updated, "removed": 0,168 "kind": "public-records"}169