1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# Home-Ka — probe_sources.py : turn discovered brokerages into SOURCES.4#5# For every brokerage in the registry (not yet a source), test the cheapest6# no-proxy path: sitemap.xml → listing URLs → one page → JSON-LD listing node.7# When a site qualifies, register a `jsonld` source (config inferred: sitemap,8# url_include, external_id_regex, state) and stamp the brokerage9# (possible_feed_type=jsonld-ready, evidence.jsonld_probe).10#11# .venv/bin/python scripts/probe_sources.py [limit] [--register]12#13# Direct requests only (2-3 per site, 0.6s politeness) — probing must stay14# cheap; the resilient/proxy chain is reserved for registered sources.15# -----------------------------------------------------------------------------16from __future__ import annotations1718import json19import re20import sys21import time22from pathlib import Path2324import requests2526sys.path.insert(0, str(Path(__file__).resolve().parent.parent))27from homeka import db # noqa: E40228from homeka.connectors.json.jsonld_site import iter_ld, _LISTING_TYPES # noqa: E4022930UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "31 "(KHTML, like Gecko) Chrome/126 Safari/537.36 "32 "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)")33LOC_RE = re.compile(r"<loc>\s*(.*?)\s*</loc>", re.S | re.I)34LISTING_URL_RE = re.compile(35 r"/(listing|property|properties|homedetails|homes-for-sale|home-for-sale|idx)/", re.I)363738def probe(website: str, s: requests.Session) -> dict | None:39 base = website if website.startswith("http") else f"https://{website}"40 base = base.rstrip("/")41 try:42 r = s.get(f"{base}/sitemap.xml", timeout=15, allow_redirects=True)43 if r.status_code != 200 or "<" not in r.text[:200]:44 return None45 except requests.RequestException:46 return None47 locs = LOC_RE.findall(r.text)48 nested = [u for u in locs if u.endswith(".xml")][:8]49 listing_urls = [u for u in locs if LISTING_URL_RE.search(u)]50 for sm in nested:51 if listing_urls:52 break53 try:54 time.sleep(0.6)55 body = s.get(sm, timeout=15).text56 except requests.RequestException:57 continue58 listing_urls = [u for u in LOC_RE.findall(body)59 if LISTING_URL_RE.search(u)]60 if not listing_urls:61 return None62 sample = listing_urls[len(listing_urls) // 2]63 try:64 time.sleep(0.6)65 page = s.get(sample, timeout=20).text66 except requests.RequestException:67 return None68 nodes = [n for n in iter_ld(page)69 if str(n.get("@type") or "").lower() in _LISTING_TYPES]70 if not nodes:71 return None72 has_addr = any(isinstance(n.get("address"), dict) and73 n["address"].get("streetAddress") for n in nodes)74 has_price = any(n.get("offers") for n in nodes)75 if not (has_addr or has_price):76 return None77 # infer url_include + external_id_regex from the sample URL78 m = re.search(r"/(listing|property|properties|homedetails)/", sample, re.I)79 seg = m.group(1).lower() if m else "listing"80 include = f"/{seg}/"81 ext_re = ""82 mid = re.search(rf"/{seg}/([A-Za-z0-9_-]+)/", sample, re.I)83 if mid and len(mid.group(1)) <= 24:84 ext_re = rf"/{seg}/([A-Za-z0-9_-]+)/"85 return {"sample": sample, "n_urls": len(listing_urls),86 "url_include": include, "external_id_regex": ext_re,87 "has_addr": has_addr, "has_price": has_price,88 "types": sorted({str(n.get("@type")) for n in nodes})}899091def main() -> None:92 limit = next((int(a) for a in sys.argv[1:] if a.isdigit()), 100)93 register = "--register" in sys.argv94 con = db.connect()95 existing_sites = set()96 for src in db.get_sources(con):97 u = (src["config"].get("base_url") or "").replace("https://", "")98 existing_sites.add(u.replace("http://", "").rstrip("/").lower())99 rows = con.execute(100 "SELECT id, name, website, states FROM brokerages"101 " WHERE website<>'' AND (inspect_error IS NULL OR inspect_error='')"102 " AND (json_extract(COALESCE(evidence,'{}'),'$.jsonld_probe') IS NULL)"103 " ORDER BY priority_score DESC LIMIT ?", (limit,)).fetchall()104 s = requests.Session()105 s.headers["User-Agent"] = UA106 found = 0107 for r in rows:108 site = r["website"].lower().lstrip("www.")109 if r["website"].lower().replace("www.", "") in {e.replace("www.", "") for e in existing_sites}:110 continue111 time.sleep(0.6)112 try:113 res = probe(r["website"], s)114 except Exception:115 res = None116 ev_row = con.execute("SELECT evidence FROM brokerages WHERE id=?",117 (r["id"],)).fetchone()118 ev = json.loads(ev_row["evidence"] or "{}")119 ev["jsonld_probe"] = {"ok": bool(res), "ts": int(time.time()),120 **({k: res[k] for k in ("n_urls", "types")} if res else {})}121 sets = ["evidence=?", "updated=?"]122 args: list = [json.dumps(ev, ensure_ascii=False), time.time()]123 if res:124 found += 1125 sets += ["possible_feed_type='jsonld-ready'",126 "feed_probability_score=MAX(COALESCE(feed_probability_score,0), 55)"]127 print(f"[OK] {r['name']} — {res['n_urls']} listing URLs, "128 f"types={res['types']}, sample={res['sample'][:80]}")129 if register:130 states = json.loads(r["states"] or "[]")131 sid = re.sub(r"[^a-z0-9]+", "_",132 r["name"].lower()).strip("_")[:40]133 db.upsert_source(134 con, sid, r["name"], "jsonld",135 config={"base_url": f"https://{r['website']}",136 "url_include": res["url_include"],137 **({"external_id_regex": res["external_id_regex"]}138 if res["external_id_regex"] else {}),139 "max_pages": 500,140 **({"static": {"state": states[0]}}141 if len(states) == 1 else {})},142 authority=2, states=states, brokerage_id=r["id"],143 notes="auto-registered by probe_sources.py (JSON-LD ready)")144 con.execute("UPDATE brokerages SET source_id=?,"145 " partnership_status='prospect' WHERE id=?",146 (sid, r["id"]))147 con.execute(f"UPDATE brokerages SET {', '.join(sets)} WHERE id=?",148 args + [r["id"]])149 con.commit()150 con.close()151 print(f"probed {len(rows)} brokerage(s), {found} JSON-LD ready"152 + (" (registered)" if register else ""))153154155if __name__ == "__main__":156 main()157