Python 49.6%
TypeScript 25.5%
CSS 24.1%
1# -----------------------------------------------------------------------------2# Home-Ka — US real-estate aggregator (Groupe KA)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# discovery.py : automatic brokerage pipeline5#6# discover brokerage → inspect website → detect IDX/feed/provider7# → find public contact information → classify potential connector8# → score brokerage → add to admin (/admin/brokerages)9#10# Crawling is used HERE — to discover brokerages and their technology — then11# every interesting source should be converted into a DIRECT FEED (RESO Web12# API first). Direct requests with polite throttling; the anti-bot fallback13# chain (base.py) only fires when a site blocks.14# -----------------------------------------------------------------------------15from __future__ import annotations1617import json18import re19import time2021import requests2223from . import db2425UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "26 "(KHTML, like Gecko) Chrome/126 Safari/537.36 "27 "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)")2829# --------------------------------------------------------------------------30# IDX / website-platform fingerprints. Each entry:31# (provider name, regex on the homepage HTML/headers, feed hint, feed prob.)32# feed hint = the most realistic connector once a partnership exists.33# Probabilities reflect how easily that platform's brokerages hand out a34# usable feed (kvCORE/iHomefinder shops usually have MLS/RESO access; pure35# website builders rarely do).36# --------------------------------------------------------------------------37_FINGERPRINTS: list[tuple[str, str, str, int]] = [38 ("kvCORE / BoldTrail", r"kvcore|boldtrail|insiderealestate|idxhome\.com", "reso", 70),39 ("iHomefinder", r"ihomefinder|idxhome|optimahomes|ihfkestrel", "reso", 65),40 ("IDX Broker", r"idxbroker\.com|idx-broker|IDX Broker", "reso", 60),41 ("Showcase IDX", r"showcaseidx", "reso", 55),42 ("Real Geeks", r"realgeeks", "json-api", 55),43 ("Sierra Interactive", r"sierrastatic|sierrainteractive", "json-api", 60),44 ("Ylopo", r"ylopo", "json-api", 55),45 ("CINC", r"cincpro|cinc\.com", "json-api", 50),46 ("Lofty (Chime)", r"chime\.me|lofty\.com|chimeroi", "json-api", 50),47 ("Luxury Presence", r"luxurypresence", "idx-partner", 45),48 ("AgentFire", r"agentfire", "idx-partner", 45),49 ("Placester", r"placester", "idx-partner", 40),50 ("Real Estate Webmasters", r"realestatewebmasters|rew\.ca|rew-", "json-api", 55),51 ("Delta Media Group", r"deltamediagroup|deltagroup", "xml", 55),52 ("Union Street Media", r"unionstreetmedia", "idx-partner", 45),53 ("Moxi Works", r"moxiworks|moxi-", "idx-partner", 50),54 ("Tribus", r"tribus", "json-api", 50),55 ("BoomTown", r"boomtownroi|boomtown", "json-api", 50),56 ("Homes.com / Homesnap", r"homesnap|homes\.com/widget", "idx-partner", 35),57 ("WordPress + IDX plugin", r"wp-content/plugins/(idx|impress|wpl|realtyna|optima)", "idx-partner", 50),58 ("WordPress", r"wp-content|wp-includes", "crawl", 30),59 ("Squarespace", r"squarespace", "crawl", 15),60 ("Wix", r"wix\.com|wixstatic", "crawl", 15),61]6263# Signals that a RESO/RETS pipeline exists behind the site64_RESO_RE = re.compile(65 r"reso\b|api\.mlsgrid|trestle[.-]corelogic|api\.bridgedataoutput"66 r"|sparkapi\.com|spark\.paragonrels|retsly|rets\b|webapi.*odata|odata.*Property",67 re.I)6869_MLS_HINTS = [70 ("ACTRIS", r"actris"), ("ARMLS", r"armls"), ("BRIGHT", r"bright\s*mls"),71 ("CRMLS", r"crmls"), ("Canopy", r"canopy\s*mls"), ("CharlestonMLS", r"charleston.*mls"),72 ("FMLS", r"\bfmls\b"), ("GAMLS", r"gamls"), ("HAR", r"\bhar\.com|houston association of realtors"),73 ("Heartland", r"heartland\s*mls"), ("MRED", r"\bmred\b|midwest real estate data"),74 ("MLSPIN", r"mls\s*pin|mlspin"), ("NTREIS", r"ntreis"), ("NWMLS", r"nwmls"),75 ("OneKey", r"onekey"), ("RMLS", r"\brmls\b"), ("SABOR", r"sabor"),76 ("Stellar", r"stellar\s*mls"), ("TRREB", r"trreb"), ("UtahRealEstate", r"utahrealestate"),77 ("Realtracs", r"realtracs"), ("BeachesMLS", r"beaches\s*mls"),78 ("MiamiMLS", r"miami.*(association|mls)"), ("Metrolist", r"metrolist"),79]8081_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")82_CONTACT_HREF_RE = re.compile(83 r'href=["\']([^"\']*(?:contact|about|our-team|team|offices?)[^"\']*)["\']', re.I)84_BAD_EMAIL = re.compile(r"example\.|sentry|wixpress|schema|\.png|\.jpg|\.gif|"85 r"godaddy|placeholder|yourdomain|@2x|email@domain|"86 r"^(user|name|test|john\.?doe|jane\.?doe)@", re.I)878889def _fetch_page(s: requests.Session, url: str, timeout: int = 25):90 """Direct GET; on an anti-bot block, escalate through the fallback chain91 (Oxylabs → Scrapfly → Bright Data) like the connectors do."""92 from .connectors import _resilient as kar93 try:94 resp = s.get(url, timeout=timeout, allow_redirects=True)95 resp.raise_for_status()96 return resp97 except requests.HTTPError as exc:98 r = getattr(exc, "response", None)99 if r is not None and kar.is_blocked(r):100 better = kar.escalate_if_blocked(r, str(r.url) or url,101 timeout=timeout)102 if better is not None and getattr(better, "status_code", 0) == 200:103 return better104 raise105 except (requests.ConnectionError, requests.Timeout):106 better = kar.escalate(url, timeout=timeout)107 if better is not None and getattr(better, "status_code", 0) == 200:108 return better109 raise110111112def inspect_website(website: str, session: requests.Session | None = None) -> dict:113 """Inspect one brokerage website: platform/IDX detection, RESO hints,114 MLS affiliations, contact discovery. Direct GET only (one page + the115 contact page); returns an evidence dict."""116 s = session or requests.Session()117 s.headers["User-Agent"] = UA118 url = website if website.startswith("http") else f"https://{website}"119 out: dict = {"final_url": "", "idx_present": 0, "idx_provider": "",120 "reso_detected": 0, "possible_feed_type": "crawl",121 "feed_probability_score": 20, "mls_affiliations": [],122 "contact_page": "", "partnership_contact": "",123 "technical_contact": "", "evidence": {}}124 resp = _fetch_page(s, url)125 html = resp.text[:800_000]126 out["final_url"] = str(resp.url)127 server = " ".join(f"{k}: {v}" for k, v in resp.headers.items()128 if k.lower() in ("server", "x-powered-by", "x-generator"))129 haystack = html + " " + server130131 hits = []132 for name, pattern, feed, prob in _FINGERPRINTS:133 if re.search(pattern, haystack, re.I):134 hits.append({"provider": name, "feed": feed, "prob": prob})135 if hits:136 best = max(hits, key=lambda h: h["prob"])137 out["idx_present"] = 1138 out["idx_provider"] = best["provider"]139 out["possible_feed_type"] = best["feed"]140 out["feed_probability_score"] = best["prob"]141 out["evidence"]["providers"] = [h["provider"] for h in hits]142143 if _RESO_RE.search(haystack):144 out["reso_detected"] = 1145 out["possible_feed_type"] = "reso"146 out["feed_probability_score"] = max(out["feed_probability_score"], 75)147148 # any IDX search page at all (listings embedded) counts as IDX present149 if not out["idx_present"] and re.search(150 r"/idx/|idx-search|property-search|listing-search|mls-search",151 haystack, re.I):152 out["idx_present"] = 1153 out["idx_provider"] = out["idx_provider"] or "unknown IDX"154 out["feed_probability_score"] = max(out["feed_probability_score"], 40)155 out["possible_feed_type"] = ("idx-partner"156 if out["possible_feed_type"] == "crawl"157 else out["possible_feed_type"])158159 for mls, pattern in _MLS_HINTS:160 if re.search(pattern, haystack, re.I):161 out["mls_affiliations"].append(mls)162163 # contacts: emails on the homepage, then the contact page164 emails = [e for e in _EMAIL_RE.findall(html) if not _BAD_EMAIL.search(e)]165 m = _CONTACT_HREF_RE.search(html)166 if m:167 href = m.group(1)168 contact_url = href if href.startswith("http") else \169 str(resp.url).rstrip("/") + "/" + href.lstrip("/")170 out["contact_page"] = contact_url171 try:172 time.sleep(0.5)173 c = _fetch_page(s, contact_url, timeout=20)174 emails += [e for e in _EMAIL_RE.findall(c.text)175 if not _BAD_EMAIL.search(e)]176 except requests.RequestException:177 pass178 emails = list(dict.fromkeys(e.lower() for e in emails))179 if emails:180 tech = [e for e in emails if re.match(181 r"(it|tech|web|webmaster|dev|api|data|idx)[@.]", e)]182 biz = [e for e in emails if re.match(183 r"(info|contact|hello|admin|office|broker|partnerships?|media)[@.]", e)]184 out["technical_contact"] = tech[0] if tech else ""185 out["partnership_contact"] = biz[0] if biz else emails[0]186 out["evidence"]["emails"] = emails[:8]187 return out188189190def priority(estimated_listings: int | None, estimated_agents: int | None,191 feed_prob: float, n_states: int) -> float:192 """0-100 priority: size × feed probability × geographic coverage."""193 import math194 listings = estimated_listings or (estimated_agents or 0) * 8195 size = min(60.0, 12 * math.log10(max(listings, 1) + 1))196 return round(min(100.0, size * (0.4 + 0.6 * feed_prob / 100)197 + min(n_states, 5) * 3), 1)198199200def inspect_one(con, brokerage_id: int) -> dict:201 """Run the inspection for one brokerage row and persist the results."""202 row = con.execute("SELECT * FROM brokerages WHERE id=?",203 (brokerage_id,)).fetchone()204 if row is None:205 raise ValueError(f"brokerage {brokerage_id} not found")206 now = time.time()207 try:208 info = inspect_website(row["website"])209 error = ""210 except Exception as exc:211 info = {}212 error = str(exc)[:300]213 if info:214 mls = list(dict.fromkeys(215 json.loads(row["mls_affiliations"] or "[]") + info["mls_affiliations"]))216 states = json.loads(row["states"] or "[]")217 prio = priority(row["estimated_listings"], row["estimated_agents"],218 info["feed_probability_score"], len(states))219 con.execute(220 """UPDATE brokerages SET idx_present=?, idx_provider=?,221 reso_detected=?, possible_feed_type=?, feed_probability_score=?,222 priority_score=?, mls_affiliations=?,223 contact_page=COALESCE(NULLIF(?,''), contact_page),224 partnership_contact=COALESCE(NULLIF(?,''), partnership_contact),225 technical_contact=COALESCE(NULLIF(?,''), technical_contact),226 evidence=?, last_inspected=?, inspect_error='', updated=?227 WHERE id=?""",228 (info["idx_present"], info["idx_provider"], info["reso_detected"],229 info["possible_feed_type"], info["feed_probability_score"], prio,230 json.dumps(mls, ensure_ascii=False), info["contact_page"],231 info["partnership_contact"], info["technical_contact"],232 json.dumps(info["evidence"], ensure_ascii=False), now, now,233 brokerage_id))234 else:235 con.execute(236 "UPDATE brokerages SET last_inspected=?, inspect_error=?, updated=?"237 " WHERE id=?", (now, error, now, brokerage_id))238 con.commit()239 return {"id": brokerage_id, "ok": not error, "error": error, **(info or {})}240241242def run_batch(limit: int = 25, reinspect_days: int = 90) -> dict:243 """Inspect the next never-inspected (then stalest) prospects — called244 after each sync cycle and by `run.py discover`."""245 con = db.connect()246 cutoff = time.time() - reinspect_days * 86400247 rows = con.execute(248 "SELECT id FROM brokerages WHERE website IS NOT NULL AND website<>''"249 " AND (last_inspected IS NULL OR"250 " (last_inspected < ? AND inspect_error=''))"251 " ORDER BY last_inspected IS NOT NULL, priority_score DESC, id"252 " LIMIT ?", (cutoff, limit)).fetchall()253 inspected = errors = 0254 for r in rows:255 res = inspect_one(con, r["id"])256 inspected += 1257 errors += 0 if res["ok"] else 1258 time.sleep(1.0) # politeness between brokerage sites259 con.close()260 return {"inspected": inspected, "errors": errors}261