SPB Git forge
spboucher.ai ↗

spb/home-ka

Public
10commits 1branches 0releases
793.0 KBsize
maindefault branch
20 days agolast push
Python 49.6% TypeScript 25.5% CSS 24.1%

Public records + source prospecting: Sarasota FL parcels source (ArcGIS, 250k), address-template mapping, centroid fetch, ZIP/city normalization; property matching now keys address+unit BEFORE APN (condo buildings share one assessor account); per-source sync_interval_hours; probe_sources.py auto-registers JSON-LD-ready brokerages as sources

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 27 days ago (Aug 28, 2026) parent 6ad4660

4 changed files +204 −16

modified homeka/connectors/public_data/arcgis.py +28 −8
@@ -59,7 +59,9 @@ class ArcGISParcelConnector(BaseConnector):
59 59 "f": "json", "where": cfg.get("where", "1=1"),
60 60 "outFields": cfg.get("out_fields", "*"),
61 61 "resultOffset": offset, "resultRecordCount": page,
62 − "returnGeometry": "true", "outSR": 4326,
62 + # polygons are heavy: ask for centroids only by default
63 + "returnGeometry": "true" if cfg.get("return_geometry") else "false",
64 + "returnCentroid": "true", "outSR": 4326,
63 65 }
64 66 data = self.get(f"{layer}/query", params=params).json()
65 67 feats = data.get("features") or []
@@ -69,7 +71,15 @@ class ArcGISParcelConnector(BaseConnector):
69 71 attrs = f.get("attributes") or {}
70 72 rec: dict = {"details": {}}
71 73 for field, path in mapping.items():
72 − v = dig(attrs, path)
74 + if "{" in str(path): # template: "{LOCN} {LOCS} {LOCT}"
75 + safe = {k: ("" if v is None else v)
76 + for k, v in attrs.items()}
77 + try:
78 + v = " ".join(str(path).format(**safe).split())
79 + except (KeyError, IndexError):
80 + v = None
81 + else:
82 + v = dig(attrs, path)
73 83 if v in (None, ""):
74 84 continue
75 85 if field in _PROP_FIELDS:
@@ -78,7 +88,7 @@ class ArcGISParcelConnector(BaseConnector):
78 88 rec["details"][field] = v
79 89 for field, v in static.items():
80 90 rec.setdefault(field, v)
81 − geom = f.get("geometry") or {}
91 + geom = f.get("centroid") or f.get("geometry") or {}
82 92 if "lat" not in rec and geom.get("y") is not None:
83 93 rec["lat"], rec["lng"] = geom.get("y"), geom.get("x")
84 94 if rec.get("apn") or rec.get("street_address"):
@@ -92,22 +102,32 @@ class ArcGISParcelConnector(BaseConnector):
92 102 def sync_records(con, source_id: str, records: list[dict]) -> dict:
93 103 """Upsert public-records rows into `properties` (match APN, then address)."""
94 104 from ... import propertymatch
105 + from ...normalize import clean_title, normalize_zip
95 106 now = time.time()
96 107 added = updated = 0
97 108 for rec in records:
109 + # assessor exports: ZIP as float ("34446.0"), SHOUTING city names
110 + if rec.get("zip_code") is not None:
111 + rec["zip_code"] = normalize_zip(rec["zip_code"])
112 + if rec.get("city"):
113 + rec["city"] = clean_title(str(rec["city"]).strip().title())
114 + if rec.get("street_address"):
115 + rec["street_address"] = str(rec["street_address"]).strip().title()
98 116 state = str(rec.get("state") or "")
99 117 apn = str(rec.get("apn") or "").replace(" ", "")
100 − row = None
101 − if apn and state:
102 − row = con.execute("SELECT id, details FROM properties"
103 − " WHERE apn=? AND state=?", (apn, state)).fetchone()
118 + # address+unit key FIRST — APNs are not unit-unique for condos
119 + # (whole buildings share one assessor account in several counties)
104 120 key = propertymatch.addr_key(str(rec.get("street_address") or ""),
105 121 str(rec.get("unit") or ""),
106 122 str(rec.get("city") or ""), state,
107 123 str(rec.get("zip_code") or ""))
108 − if row is None and key:
124 + row = None
125 + if key:
109 126 row = con.execute("SELECT id, details FROM properties"
110 127 " WHERE addr_key=?", (key,)).fetchone()
128 + elif apn and state:
129 + row = con.execute("SELECT id, details FROM properties"
130 + " WHERE apn=? AND state=?", (apn, state)).fetchone()
111 131 details = rec.get("details") or {}
112 132 if row is None:
113 133 if not key and not apn:
modified homeka/ingest.py +9 −0
@@ -51,6 +51,15 @@ def run(sources: list[str] | None = None) -> list[dict]:
51 51 print(f"[home-ka] no connector for {sid} (type {ctype!r})",
52 52 file=sys.stderr)
53 53 continue
54 + # slow sources (public records...): config.sync_interval_hours skips
55 + # the source while its last successful sync is fresh enough
56 + interval_h = float((config or {}).get("sync_interval_hours") or 0)
57 + if interval_h and not sources:
58 + last = con.execute(
59 + "SELECT MAX(ts) ts FROM sync_log WHERE source=? AND ok=1",
60 + (sid,)).fetchone()["ts"]
61 + if last and time.time() - last < interval_h * 3600:
62 + continue
54 63 t0 = time.time()
55 64 print(f"[home-ka] sync {sid} ...")
56 65 try:
modified homeka/propertymatch.py +11 −8
@@ -63,14 +63,11 @@ def addr_key(street: str, unit: str, city: str, state: str,
63 63
64 64 def match_or_create(con: sqlite3.Connection, lst: Listing,
65 65 now: float) -> int | None:
66 − """Return the property id for a listing, creating the property if new."""
67 − # 1) APN match (same state) — assessor id is authoritative
68 − if lst.apn and lst.state:
69 − row = con.execute(
70 − "SELECT id FROM properties WHERE apn=? AND state=?",
71 − (lst.apn, lst.state)).fetchone()
72 − if row:
73 − return row["id"]
66 + """Return the property id for a listing, creating the property if new.
67 +
68 + Address+unit key first: an APN is NOT always unit-unique (whole condo
69 + buildings share one assessor account in several counties), so the APN is
70 + only a fallback when no usable address key exists."""
74 71 key = addr_key(lst.street_address, lst.unit, lst.city, lst.state,
75 72 lst.zip_code)
76 73 if key:
@@ -78,6 +75,12 @@ def match_or_create(con: sqlite3.Connection, lst: Listing,
78 75 "SELECT id FROM properties WHERE addr_key=?", (key,)).fetchone()
79 76 if row:
80 77 return row["id"]
78 + elif lst.apn and lst.state:
79 + row = con.execute(
80 + "SELECT id FROM properties WHERE apn=? AND state=?",
81 + (lst.apn, lst.state)).fetchone()
82 + if row:
83 + return row["id"]
81 84 if not key and not lst.apn:
82 85 return None # nothing reliable to key the property on
83 86 cur = con.execute(
added scripts/probe_sources.py +156 −0
@@ -0,0 +1,156 @@
1 +#!/usr/bin/env python3
2 +# -----------------------------------------------------------------------------
3 +# Home-Ka — probe_sources.py : turn discovered brokerages into SOURCES.
4 +#
5 +# For every brokerage in the registry (not yet a source), test the cheapest
6 +# no-proxy path: sitemap.xml → listing URLs → one page → JSON-LD listing node.
7 +# When a site qualifies, register a `jsonld` source (config inferred: sitemap,
8 +# url_include, external_id_regex, state) and stamp the brokerage
9 +# (possible_feed_type=jsonld-ready, evidence.jsonld_probe).
10 +#
11 +# .venv/bin/python scripts/probe_sources.py [limit] [--register]
12 +#
13 +# Direct requests only (2-3 per site, 0.6s politeness) — probing must stay
14 +# cheap; the resilient/proxy chain is reserved for registered sources.
15 +# -----------------------------------------------------------------------------
16 +from __future__ import annotations
17 +
18 +import json
19 +import re
20 +import sys
21 +import time
22 +from pathlib import Path
23 +
24 +import requests
25 +
26 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
27 +from homeka import db # noqa: E402
28 +from homeka.connectors.json.jsonld_site import iter_ld, _LISTING_TYPES # noqa: E402
29 +
30 +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
31 + "(KHTML, like Gecko) Chrome/126 Safari/537.36 "
32 + "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)")
33 +LOC_RE = re.compile(r"<loc>\s*(.*?)\s*</loc>", re.S | re.I)
34 +LISTING_URL_RE = re.compile(
35 + r"/(listing|property|properties|homedetails|homes-for-sale|home-for-sale|idx)/", re.I)
36 +
37 +
38 +def probe(website: str, s: requests.Session) -> dict | None:
39 + base = website if website.startswith("http") else f"https://{website}"
40 + base = base.rstrip("/")
41 + try:
42 + r = s.get(f"{base}/sitemap.xml", timeout=15, allow_redirects=True)
43 + if r.status_code != 200 or "<" not in r.text[:200]:
44 + return None
45 + except requests.RequestException:
46 + return None
47 + locs = LOC_RE.findall(r.text)
48 + nested = [u for u in locs if u.endswith(".xml")][:8]
49 + listing_urls = [u for u in locs if LISTING_URL_RE.search(u)]
50 + for sm in nested:
51 + if listing_urls:
52 + break
53 + try:
54 + time.sleep(0.6)
55 + body = s.get(sm, timeout=15).text
56 + except requests.RequestException:
57 + continue
58 + listing_urls = [u for u in LOC_RE.findall(body)
59 + if LISTING_URL_RE.search(u)]
60 + if not listing_urls:
61 + return None
62 + sample = listing_urls[len(listing_urls) // 2]
63 + try:
64 + time.sleep(0.6)
65 + page = s.get(sample, timeout=20).text
66 + except requests.RequestException:
67 + return None
68 + nodes = [n for n in iter_ld(page)
69 + if str(n.get("@type") or "").lower() in _LISTING_TYPES]
70 + if not nodes:
71 + return None
72 + has_addr = any(isinstance(n.get("address"), dict) and
73 + n["address"].get("streetAddress") for n in nodes)
74 + has_price = any(n.get("offers") for n in nodes)
75 + if not (has_addr or has_price):
76 + return None
77 + # infer url_include + external_id_regex from the sample URL
78 + m = re.search(r"/(listing|property|properties|homedetails)/", sample, re.I)
79 + seg = m.group(1).lower() if m else "listing"
80 + include = f"/{seg}/"
81 + ext_re = ""
82 + mid = re.search(rf"/{seg}/([A-Za-z0-9_-]+)/", sample, re.I)
83 + if mid and len(mid.group(1)) <= 24:
84 + ext_re = rf"/{seg}/([A-Za-z0-9_-]+)/"
85 + return {"sample": sample, "n_urls": len(listing_urls),
86 + "url_include": include, "external_id_regex": ext_re,
87 + "has_addr": has_addr, "has_price": has_price,
88 + "types": sorted({str(n.get("@type")) for n in nodes})}
89 +
90 +
91 +def main() -> None:
92 + limit = next((int(a) for a in sys.argv[1:] if a.isdigit()), 100)
93 + register = "--register" in sys.argv
94 + con = db.connect()
95 + existing_sites = set()
96 + for src in db.get_sources(con):
97 + u = (src["config"].get("base_url") or "").replace("https://", "")
98 + existing_sites.add(u.replace("http://", "").rstrip("/").lower())
99 + rows = con.execute(
100 + "SELECT id, name, website, states FROM brokerages"
101 + " WHERE website<>'' AND (inspect_error IS NULL OR inspect_error='')"
102 + " AND (json_extract(COALESCE(evidence,'{}'),'$.jsonld_probe') IS NULL)"
103 + " ORDER BY priority_score DESC LIMIT ?", (limit,)).fetchall()
104 + s = requests.Session()
105 + s.headers["User-Agent"] = UA
106 + found = 0
107 + for r in rows:
108 + site = r["website"].lower().lstrip("www.")
109 + if r["website"].lower().replace("www.", "") in {e.replace("www.", "") for e in existing_sites}:
110 + continue
111 + time.sleep(0.6)
112 + try:
113 + res = probe(r["website"], s)
114 + except Exception:
115 + res = None
116 + ev_row = con.execute("SELECT evidence FROM brokerages WHERE id=?",
117 + (r["id"],)).fetchone()
118 + ev = json.loads(ev_row["evidence"] or "{}")
119 + ev["jsonld_probe"] = {"ok": bool(res), "ts": int(time.time()),
120 + **({k: res[k] for k in ("n_urls", "types")} if res else {})}
121 + sets = ["evidence=?", "updated=?"]
122 + args: list = [json.dumps(ev, ensure_ascii=False), time.time()]
123 + if res:
124 + found += 1
125 + sets += ["possible_feed_type='jsonld-ready'",
126 + "feed_probability_score=MAX(COALESCE(feed_probability_score,0), 55)"]
127 + print(f"[OK] {r['name']} — {res['n_urls']} listing URLs, "
128 + f"types={res['types']}, sample={res['sample'][:80]}")
129 + if register:
130 + states = json.loads(r["states"] or "[]")
131 + sid = re.sub(r"[^a-z0-9]+", "_",
132 + r["name"].lower()).strip("_")[:40]
133 + db.upsert_source(
134 + con, sid, r["name"], "jsonld",
135 + config={"base_url": f"https://{r['website']}",
136 + "url_include": res["url_include"],
137 + **({"external_id_regex": res["external_id_regex"]}
138 + if res["external_id_regex"] else {}),
139 + "max_pages": 500,
140 + **({"static": {"state": states[0]}}
141 + if len(states) == 1 else {})},
142 + authority=2, states=states, brokerage_id=r["id"],
143 + notes="auto-registered by probe_sources.py (JSON-LD ready)")
144 + con.execute("UPDATE brokerages SET source_id=?,"
145 + " partnership_status='prospect' WHERE id=?",
146 + (sid, r["id"]))
147 + con.execute(f"UPDATE brokerages SET {', '.join(sets)} WHERE id=?",
148 + args + [r["id"]])
149 + con.commit()
150 + con.close()
151 + print(f"probed {len(rows)} brokerage(s), {found} JSON-LD ready"
152 + + (" (registered)" if register else ""))
153 +
154 +
155 +if __name__ == "__main__":
156 + main()
157