# ----------------------------------------------------------------------------- # Home-Ka — US real-estate aggregator (Groupe KA) # Author: Simon-Pierre Boucher — contact@spboucher.ai # quality.py : data-quality gate — completeness score and publication # threshold (quarantine), same principle as immo-ka/quality.py. # A listing without a plausible price, location, type or usable content stays # out of the grid/map/stats until later cycles complete it; its detail page # by uid remains reachable. # ----------------------------------------------------------------------------- from __future__ import annotations import sqlite3 PRICE_MIN, PRICE_MAX = 5_000, 200_000_000 PUBLISH_THRESHOLD = 0.55 _WEIGHTS = { "price": 0.30, # plausible list price "location": 0.25, # city+state or zip "address": 0.15, "type": 0.10, "content": 0.10, # description or features "images": 0.10, } def score_row(r) -> float: s = 0.0 price = r["list_price"] if price is not None and PRICE_MIN <= price <= PRICE_MAX: s += _WEIGHTS["price"] if (r["city"] and r["state"]) or r["zip_code"]: s += _WEIGHTS["location"] if r["street_address"]: s += _WEIGHTS["address"] if r["property_type"]: s += _WEIGHTS["type"] if (r["description"] and len(r["description"]) > 40) or \ (r["features"] and r["features"] != "[]"): s += _WEIGHTS["content"] if r["images"] and r["images"] != "[]": s += _WEIGHTS["images"] return round(s, 3) def refresh(con: sqlite3.Connection) -> dict: published = quarantined = 0 for r in con.execute( "SELECT uid, list_price, city, state, zip_code, street_address," " property_type, description, features, images" " FROM listings WHERE active=1"): q = score_row(r) # a plausible asking price is a HARD requirement for publication — # crawled sold/off-market shells must never reach the grid price_ok = (r["list_price"] is not None and PRICE_MIN <= r["list_price"] <= PRICE_MAX) pub = 1 if (q >= PUBLISH_THRESHOLD and price_ok) else 0 published += pub quarantined += 1 - pub con.execute("UPDATE listings SET quality=?, published=? WHERE uid=?", (q, pub, r["uid"])) con.commit() return {"published": published, "quarantined": quarantined} def summary(con: sqlite3.Connection) -> dict: rows = [dict(r) for r in con.execute( "SELECT source, COUNT(*) n, SUM(published) published," " ROUND(AVG(quality), 3) avg_quality," " SUM(CASE WHEN list_price IS NULL THEN 1 ELSE 0 END) no_price," " SUM(CASE WHEN lat IS NULL THEN 1 ELSE 0 END) no_geo" " FROM listings WHERE active=1 GROUP BY source ORDER BY n DESC")] return {"by_source": rows}