quality: plausible price is a hard publication requirement; jsonld: parse address from Product.name (Real Geeks, @properties platforms); probe www-normalization fix

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Simon-Pierre Boucher committed 27 days ago (Aug 28, 2026) parent 83a3f0d

3 changed files +25 −3

modified homeka/connectors/json/jsonld_site.py +18 −1

		`@@ -43,6 +43,11 @@ _LISTING_TYPES = {`
43	43	# URL slug fallback: ".../7039-s-straight-avenue-homosassa-fl-34446/"
44	44	_SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I)
45	45
	46	+# `name` fallback: several platforms (Real Geeks, @properties/Christie's)
	47	+# put the full address in Product.name — "123 Main St #4, Austin, TX 78701"
	48	+_NAME_ADDR_RE = re.compile(
	49	+ r"^(.+?),\s([^,]+?),\s([A-Za-z]{2})[,\s]+(\d{5})(?:-\d{4})?\s*$")
	50	+
46	51
47	52	# Next.js "flight" streaming: some SSR sites (Douglas Elliman) embed the
48	53	# JSON-LD as self.__next_s.push([0,{"type":"application/ld+json",
		`@@ -116,9 +121,16 @@ class JSONLDSiteConnector(BaseConnector):`
116	121	if cfg.get("sitemap_exclude") else None
117	122	depth = 0
118	123	max_pages = int(cfg.get("max_pages", 400))
	124	+ max_children = int(cfg.get("sitemap_max_children", 80))
	125	+ _PRIORITY = re.compile(r"active\|listing\|residential\|propert\|homes",
	126	+ re.I)
119	127	while sitemaps and depth < 3 and len(urls) < max_pages:
	128	+ # likely-listing shards first — big indexes (@properties: 69
	129	+ # children) would otherwise exhaust the budget on agent/office maps
	130	+ sitemaps.sort(key=lambda u: 0 if _PRIORITY.search(
	131	+ u.rsplit("/", 1)[-1]) else 1)
120	132	nxt: list[str] = []
121		− for sm in sitemaps[:30]:
	133	+ for sm in sitemaps[:max_children]:
122	134	if len(urls) >= max_pages:
123	135	break
124	136	if sm_exclude and sm_exclude.search(sm):
		`@@ -308,6 +320,11 @@ class JSONLDSiteConnector(BaseConnector):`
308	320	city = str(addr.get("addressLocality") or "")
309	321	state = str(addr.get("addressRegion") or "")
310	322	zip_code = str(addr.get("postalCode") or "")
	323	+ if not street: # name fallback: "123 Main St, Austin, TX 78701"
	324	+ m = _NAME_ADDR_RE.match(str(merged.get("name") or "").strip())
	325	+ if m:
	326	+ street, city = m.group(1), city or m.group(2)
	327	+ state, zip_code = state or m.group(3).upper(), zip_code or m.group(4)
311	328	if not street: # slug fallback: .../123-main-street-austin-tx-78701/
312	329	m = _SLUG_ADDR_RE.search(url)
313	330	if m:

modified homeka/quality.py +5 −1

		`@@ -50,7 +50,11 @@ def refresh(con: sqlite3.Connection) -> dict:`
50	50	" property_type, description, features, images"
51	51	" FROM listings WHERE active=1"):
52	52	q = score_row(r)
53		− pub = 1 if q >= PUBLISH_THRESHOLD else 0
	53	+ # a plausible asking price is a HARD requirement for publication —
	54	+ # crawled sold/off-market shells must never reach the grid
	55	+ price_ok = (r["list_price"] is not None
	56	+ and PRICE_MIN <= r["list_price"] <= PRICE_MAX)
	57	+ pub = 1 if (q >= PUBLISH_THRESHOLD and price_ok) else 0
54	58	published += pub
55	59	quarantined += 1 - pub
56	60	con.execute("UPDATE listings SET quality=?, published=? WHERE uid=?",

modified scripts/probe_sources.py +2 −1

		`@@ -95,7 +95,8 @@ def main() -> None:`
95	95	existing_sites = set()
96	96	for src in db.get_sources(con):
97	97	u = (src["config"].get("base_url") or "").replace("https://", "")
98		− existing_sites.add(u.replace("http://", "").rstrip("/").lower())
	98	+ u = u.replace("http://", "").rstrip("/").lower()
	99	+ existing_sites.add(u.removeprefix("www."))
99	100	rows = con.execute(
100	101	"SELECT id, name, website, states FROM brokerages"
101	102	" WHERE website<>'' AND (inspect_error IS NULL OR inspect_error='')"
102	103