Python 49.6%
TypeScript 25.5%
CSS 24.1%
1#!/usr/bin/env python32# -----------------------------------------------------------------------------3# Home-Ka — probe_sources.py : turn discovered brokerages into SOURCES.4#5# For every brokerage in the registry (not yet a source), test the cheapest6# no-proxy path: sitemap.xml → listing URLs → one page → JSON-LD listing node.7# When a site qualifies, register a `jsonld` source (config inferred: sitemap,8# url_include, external_id_regex, state) and stamp the brokerage9# (possible_feed_type=jsonld-ready, evidence.jsonld_probe).10#11# .venv/bin/python scripts/probe_sources.py [limit] [--register]12#13# Direct requests only (2-3 per site, 0.6s politeness) — probing must stay14# cheap; the resilient/proxy chain is reserved for registered sources.15# -----------------------------------------------------------------------------16from __future__ import annotations1718import json19import re20import sys21import time22from pathlib import Path2324import requests2526sys.path.insert(0, str(Path(__file__).resolve().parent.parent))27from homeka import db # noqa: E40228from homeka.connectors.json.jsonld_site import iter_ld, _LISTING_TYPES # noqa: E4022930UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "31 "(KHTML, like Gecko) Chrome/126 Safari/537.36 "32 "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)")33LOC_RE = re.compile(r"<loc>\s*(.*?)\s*</loc>", re.S | re.I)34LISTING_URL_RE = re.compile(35 r"/(listing|property|properties|homedetails|homes-for-sale|home-for-sale|idx)/", re.I)363738def probe(website: str, s: requests.Session) -> dict | None:39 base = website if website.startswith("http") else f"https://{website}"40 base = base.rstrip("/")41 try:42 r = s.get(f"{base}/sitemap.xml", timeout=15, allow_redirects=True)43 if r.status_code != 200 or "<" not in r.text[:200]:44 return None45 except requests.RequestException:46 return None47 locs = LOC_RE.findall(r.text)48 nested = [u for u in locs if u.endswith(".xml")][:8]49 listing_urls = [u for u in locs if LISTING_URL_RE.search(u)]50 for sm in nested:51 if listing_urls:52 break53 try:54 time.sleep(0.6)55 body = s.get(sm, timeout=15).text56 except requests.RequestException:57 continue58 listing_urls = [u for u in LOC_RE.findall(body)59 if LISTING_URL_RE.search(u)]60 if not listing_urls:61 return None62 sample = listing_urls[len(listing_urls) // 2]63 try:64 time.sleep(0.6)65 page = s.get(sample, timeout=20).text66 except requests.RequestException:67 return None68 nodes = [n for n in iter_ld(page)69 if str(n.get("@type") or "").lower() in _LISTING_TYPES]70 if not nodes:71 return None72 has_addr = any(isinstance(n.get("address"), dict) and73 n["address"].get("streetAddress") for n in nodes)74 has_price = any(n.get("offers") for n in nodes)75 if not (has_addr or has_price):76 return None77 # infer url_include + external_id_regex from the sample URL78 m = re.search(r"/(listing|property|properties|homedetails)/", sample, re.I)79 seg = m.group(1).lower() if m else "listing"80 include = f"/{seg}/"81 ext_re = ""82 mid = re.search(rf"/{seg}/([A-Za-z0-9_-]+)/", sample, re.I)83 if mid and len(mid.group(1)) <= 24:84 ext_re = rf"/{seg}/([A-Za-z0-9_-]+)/"85 return {"sample": sample, "n_urls": len(listing_urls),86 "url_include": include, "external_id_regex": ext_re,87 "has_addr": has_addr, "has_price": has_price,88 "types": sorted({str(n.get("@type")) for n in nodes})}899091def main() -> None:92 limit = next((int(a) for a in sys.argv[1:] if a.isdigit()), 100)93 register = "--register" in sys.argv94 con = db.connect()95 existing_sites = set()96 for src in db.get_sources(con):97 u = (src["config"].get("base_url") or "").replace("https://", "")98 u = u.replace("http://", "").rstrip("/").lower()99 existing_sites.add(u.removeprefix("www."))100 rows = con.execute(101 "SELECT id, name, website, states FROM brokerages"102 " WHERE website<>'' AND (inspect_error IS NULL OR inspect_error='')"103 " AND (json_extract(COALESCE(evidence,'{}'),'$.jsonld_probe') IS NULL)"104 " ORDER BY priority_score DESC LIMIT ?", (limit,)).fetchall()105 s = requests.Session()106 s.headers["User-Agent"] = UA107 found = 0108 for r in rows:109 site = r["website"].lower().lstrip("www.")110 if r["website"].lower().replace("www.", "") in {e.replace("www.", "") for e in existing_sites}:111 continue112 time.sleep(0.6)113 try:114 res = probe(r["website"], s)115 except Exception:116 res = None117 ev_row = con.execute("SELECT evidence FROM brokerages WHERE id=?",118 (r["id"],)).fetchone()119 ev = json.loads(ev_row["evidence"] or "{}")120 ev["jsonld_probe"] = {"ok": bool(res), "ts": int(time.time()),121 **({k: res[k] for k in ("n_urls", "types")} if res else {})}122 sets = ["evidence=?", "updated=?"]123 args: list = [json.dumps(ev, ensure_ascii=False), time.time()]124 if res:125 found += 1126 sets += ["possible_feed_type='jsonld-ready'",127 "feed_probability_score=MAX(COALESCE(feed_probability_score,0), 55)"]128 print(f"[OK] {r['name']} — {res['n_urls']} listing URLs, "129 f"types={res['types']}, sample={res['sample'][:80]}")130 if register:131 states = json.loads(r["states"] or "[]")132 sid = re.sub(r"[^a-z0-9]+", "_",133 r["name"].lower()).strip("_")[:40]134 db.upsert_source(135 con, sid, r["name"], "jsonld",136 config={"base_url": f"https://{r['website']}",137 "url_include": res["url_include"],138 **({"external_id_regex": res["external_id_regex"]}139 if res["external_id_regex"] else {}),140 "max_pages": 500,141 **({"static": {"state": states[0]}}142 if len(states) == 1 else {})},143 authority=2, states=states, brokerage_id=r["id"],144 notes="auto-registered by probe_sources.py (JSON-LD ready)")145 con.execute("UPDATE brokerages SET source_id=?,"146 " partnership_status='prospect' WHERE id=?",147 (sid, r["id"]))148 con.execute(f"UPDATE brokerages SET {', '.join(sets)} WHERE id=?",149 args + [r["id"]])150 con.commit()151 con.close()152 print(f"probed {len(rows)} brokerage(s), {found} JSON-LD ready"153 + (" (registered)" if register else ""))154155156if __name__ == "__main__":157 main()158