Python 49.6%
TypeScript 25.5%
CSS 24.1%
1# -----------------------------------------------------------------------------2# Home-Ka — US real-estate aggregator (Groupe KA)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# connectors/json/jsonld_site.py : schema.org JSON-LD site connector.5#6# Many brokerage sites (Real Estate Webmasters and other server-rendered7# platforms) publish each listing as JSON-LD (`Product` + `RealEstateListing`8# / `SingleFamilyResidence` nodes) — clean, stable data with full address,9# geo, price and photos, no proxy needed. This is the SECONDARY crawler path10# (authority 3): use it to bootstrap coverage while converting the brokerage11# to a direct feed. Ported from immo-ka connectors/jsonld.py.12#13# {"id": "michael_saunders", "connector_type": "jsonld", "config": {14# "base_url": "https://www.michaelsaunders.com",15# "url_include": "/listing/",16# "external_id_regex": "/listing/([A-Z0-9-]+)/", # from the URL17# "max_pages": 400, "static": {"state": "FL"}}}18# -----------------------------------------------------------------------------19from __future__ import annotations2021import hashlib22import json23import re24import time2526from ..base import BaseConnector27from ...normalize import parse_price28from ...schema import Listing2930_LD_RE = re.compile(r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',31 re.S | re.I)32_LOC_RE = re.compile(r"<loc>\s*(?:<!\[CDATA\[)?\s*(.*?)\s*(?:\]\]>)?\s*</loc>",33 re.S | re.I)3435_LISTING_TYPES = {36 "realestatelisting", "singlefamilyresidence", "house", "residence",37 "apartment", "product", "accommodation", "condominium", "townhouse",38 # some SSR platforms (Repliers/Next.js) mark the listing as a plain Place;39 # safe here because pages are pre-filtered by url_include40 "place",41}4243# URL slug fallback: ".../7039-s-straight-avenue-homosassa-fl-34446/"44_SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I)4546# `name` fallback: several platforms (Real Geeks, @properties/Christie's)47# put the full address in Product.name — "123 Main St #4, Austin, TX 78701"48_NAME_ADDR_RE = re.compile(49 r"^(.+?),\s*([^,]+?),\s*([A-Za-z]{2})[,\s]+(\d{5})(?:-\d{4})?\s*$")505152# Next.js "flight" streaming: some SSR sites (Douglas Elliman) embed the53# JSON-LD as self.__next_s.push([0,{"type":"application/ld+json",54# "children":"<escaped JSON>"}]) instead of a <script> tag.55_NEXT_LD_RE = re.compile(56 r'application/ld\+json\\?"[^}]*?\\?"children\\?":\\?"((?:[^"\\]|\\.)*)"',57 re.S)585960def _blocks(html: str):61 yield from _LD_RE.findall(html)62 for esc in _NEXT_LD_RE.findall(html):63 try:64 yield json.loads(f'"{esc}"') # unescape the JS string65 except ValueError:66 continue676869def iter_ld(html: str):70 """Iterate JSON-LD objects of a page (each block, flattened from @graph).71 Tolerates non-standard keys without the '@' prefix (John L. Scott)."""72 for block in _blocks(html):73 block = block.strip()74 if not block:75 continue76 try:77 # strict=False: some platforms leave raw control chars in remarks78 data = json.loads(block, strict=False)79 except ValueError:80 continue81 if isinstance(data, dict):82 nodes = data.get("@graph") or data.get("graph") or [data]83 else:84 nodes = data85 for node in (nodes if isinstance(nodes, list) else [nodes]):86 if isinstance(node, dict):87 if "@type" not in node and "type" in node:88 node = {**node, "@type": node["type"]}89 yield node909192def _qv(value):93 """Unwrap schema.org QuantitativeValue ({'value': 2, ...} → 2)."""94 if isinstance(value, dict):95 return value.get("value")96 return value979899class JSONLDSiteConnector(BaseConnector):100 family = "jsonld"101 request_delay = 0.8102103 # -- URL discovery -----------------------------------------------------104 def _listing_urls(self) -> list[str]:105 cfg = self.config106 include = re.compile(cfg.get("url_include", "/listing/|/property/"))107 urls: list[str] = []108 seen: set[str] = set()109110 def add(u: str) -> None:111 if u not in seen and include.search(u):112 seen.add(u)113 urls.append(u)114115 sitemaps = [cfg["sitemap_url"]] if cfg.get("sitemap_url") else []116 if not sitemaps and cfg.get("base_url"):117 sitemaps = [cfg["base_url"].rstrip("/") + "/sitemap.xml"]118 # skip nested sitemaps by name (e.g. Union Street Media's huge119 # "off-market-sitemap-N.xml" full of sold listings)120 sm_exclude = re.compile(cfg["sitemap_exclude"]) \121 if cfg.get("sitemap_exclude") else None122 depth = 0123 max_pages = int(cfg.get("max_pages", 400))124 max_children = int(cfg.get("sitemap_max_children", 80))125 _PRIORITY = re.compile(r"active|listing|residential|propert|homes",126 re.I)127 while sitemaps and depth < 3 and len(urls) < max_pages:128 # likely-listing shards first — big indexes (@properties: 69129 # children) would otherwise exhaust the budget on agent/office maps130 sitemaps.sort(key=lambda u: 0 if _PRIORITY.search(131 u.rsplit("/", 1)[-1]) else 1)132 nxt: list[str] = []133 for sm in sitemaps[:max_children]:134 if len(urls) >= max_pages:135 break136 if sm_exclude and sm_exclude.search(sm):137 continue138 try:139 resp = self.get(sm)140 if sm.split("?")[0].endswith(".gz"):141 import gzip142 body = gzip.decompress(resp.content).decode(143 "utf-8", errors="replace")144 else:145 body = resp.text146 # bot wall on the sitemap itself → Firecrawl fallback147 if ("<loc" not in body and "<urlset" not in body148 and self.config.get("backend") == "firecrawl"):149 body = self.fc_scrape(sm)150 except Exception:151 if self.config.get("backend") != "firecrawl":152 continue153 try:154 body = self.fc_scrape(sm)155 except Exception:156 continue157 for loc in _LOC_RE.findall(body):158 # nested sitemap: .xml/.gz files, or dynamic shard URLs159 # (sitemapindex.aspx / sitemaplistings.ashx à la Weichert)160 low = loc.split("?")[0].lower()161 if (low.endswith((".xml", ".gz"))162 or "sitemap" in loc.lower().rsplit("/", 1)[-1]):163 nxt.append(loc)164 else:165 add(loc)166 sitemaps = nxt167 depth += 1168 for lp in cfg.get("list_urls", []):169 try:170 body = self.get(lp).text171 except Exception:172 continue173 base = cfg.get("base_url", "").rstrip("/")174 for href in re.findall(r'href=["\']([^"\']+)["\']', body):175 u = href if href.startswith("http") else base + href176 add(u.split("#")[0])177 return urls[:max_pages]178179 # -- page fetch backends -------------------------------------------------180 def _page(self, url: str) -> str:181 """One listing page: Firecrawl backend (config backend=firecrawl —182 parallel-safe, bypasses bot walls, spares the source) or polite183 direct GET."""184 if self.config.get("backend") == "firecrawl":185 return self.fc_scrape(url,186 proxy=self.config.get("fc_proxy", "auto"),187 render=bool(self.config.get("fc_render")))188 return self.get(url).text189190 # -- fetch --------------------------------------------------------------191 def fetch(self) -> list[Listing]:192 cfg = self.config193 static = cfg.get("static") or {}194 ext_re = re.compile(cfg["external_id_regex"]) \195 if cfg.get("external_id_regex") else None196 # weekly cache bucket: each detail page is re-fetched at most once a197 # week (price/status refresh) and never re-fetched inside a cycle198 week = int(time.time() // (7 * 86400))199 items: list[tuple[str, str]] = []200 seen_ext: set[str] = set()201 for url in self._listing_urls():202 ext = ""203 if ext_re:204 m = ext_re.search(url)205 if m:206 ext = m.group(1)207 if not ext:208 ext = hashlib.sha1(url.encode()).hexdigest()[:16]209 if ext not in seen_ext:210 seen_ext.add(ext)211 items.append((url, ext))212213 from ... import db214 con = db.connect()215 payloads: dict[str, dict] = {}216 misses: list[tuple[str, str]] = []217 for url, ext in items:218 cached = db.get_cached_detail(con, self.source_id, ext,219 f"w{week}:{url}")220 if cached is not None:221 payloads[ext] = cached222 else:223 misses.append((url, ext))224225 def _grab(url: str) -> dict:226 try:227 return {"ld": list(iter_ld(self._page(url)))}228 except Exception:229 return {}230231 if cfg.get("backend") == "firecrawl" and misses:232 # Firecrawl absorbs the load — fetch in parallel, cache serially233 from concurrent.futures import ThreadPoolExecutor234 workers = int(cfg.get("fc_workers", 8))235 with ThreadPoolExecutor(max_workers=workers) as pool:236 results = pool.map(lambda t: _grab(t[0]), misses)237 for (url, ext), payload in zip(misses, results):238 db.put_cached_detail(con, self.source_id, ext,239 f"w{week}:{url}", payload)240 payloads[ext] = payload241 else:242 for url, ext in misses:243 payload = _grab(url)244 db.put_cached_detail(con, self.source_id, ext,245 f"w{week}:{url}", payload)246 payloads[ext] = payload247 con.close()248249 out: list[Listing] = []250 for url, ext in items:251 lst = self._from_nodes(payloads.get(ext, {}).get("ld") or [],252 url, ext, static)253 if lst is not None:254 out.append(lst)255 return out256257 # -- mapping --------------------------------------------------------------258 def _from_nodes(self, nodes: list[dict], url: str, ext: str,259 static: dict) -> Listing | None:260 """Merge every listing-typed JSON-LD node of the page (REW splits the261 data across Product + SingleFamilyResidence)."""262 def _t(n):263 t = n.get("@type")264 if isinstance(t, list): # e.g. [Land, RealEstateListing, Product]265 for x in t:266 if str(x).lower() in _LISTING_TYPES:267 return str(x).lower()268 return str(t[0]).lower() if t else ""269 return str(t or "").lower()270271 picked = [n for n in nodes if _t(n) in _LISTING_TYPES]272 if not picked:273 return None274 # standalone Offer nodes (John L. Scott, Greybeard publish the price275 # in a sibling node instead of an `offers` property)276 loose_offers = [n for n in nodes if _t(n) == "offer"]277 merged: dict = {}278 addr: dict = {}279 geo: dict = {}280 offer: dict = {}281 subtype = ""282 # only meaningful dwelling types qualify as a property subtype —283 # generic wrappers (Product, RealEstateListing, Place...) say nothing284 _MEANINGFUL = {"singlefamilyresidence", "house", "condominium",285 "townhouse", "apartment"}286 for n in picked:287 t = _t(n)288 if t in _MEANINGFUL and not subtype:289 subtype = t290 for k, v in n.items():291 if v in (None, "", [], {}):292 continue293 if k == "address" and isinstance(v, dict):294 addr = {**v, **addr}295 elif k == "geo" and isinstance(v, dict):296 geo = {**v, **geo}297 elif k == "offers":298 o = v[0] if isinstance(v, list) and v else v299 if isinstance(o, dict):300 offer = {**o, **offer}301 else:302 merged.setdefault(k, v)303 for o in loose_offers:304 offer = {**o, **offer}305 if isinstance(merged.get("address"), str):306 addr.setdefault("streetAddress", merged["address"])307308 imgs = merged.get("image") or merged.get("photo") or []309 if isinstance(imgs, str):310 imgs = [imgs]311 imgs = [i.get("url") if isinstance(i, dict) else i for i in imgs]312313 price = parse_price(_qv(offer.get("price")))314 availability = str(offer.get("availability") or "")315 status = "active"316 if "soldout" in availability.lower().replace("_", ""):317 status = "sold"318319 street = str(addr.get("streetAddress") or "")320 city = str(addr.get("addressLocality") or "")321 state = str(addr.get("addressRegion") or "")322 zip_code = str(addr.get("postalCode") or "")323 if not street: # name fallback: "123 Main St, Austin, TX 78701"324 m = _NAME_ADDR_RE.match(str(merged.get("name") or "").strip())325 if m:326 street, city = m.group(1), city or m.group(2)327 state, zip_code = state or m.group(3).upper(), zip_code or m.group(4)328 if not street: # slug fallback: .../123-main-street-austin-tx-78701/329 m = _SLUG_ADDR_RE.search(url)330 if m:331 state = state or m.group(2).upper()332 zip_code = zip_code or m.group(3)333334 beds = _qv(merged.get("numberOfBedrooms") or merged.get("numberOfRooms"))335 baths = _qv(merged.get("numberOfBathroomsTotal")336 or merged.get("numberOfFullBathrooms"))337 floor = _qv(merged.get("floorSize"))338339 kw = dict(340 source=self.source_id, external_id=ext, url=url,341 title=str(merged.get("name") or ""),342 street_address=street, city=city, state=state, zip_code=zip_code,343 list_price=price, status=status,344 mls_id=ext if not ext.startswith(("http", "sha")) and len(ext) <= 20 else "",345 property_type=subtype,346 property_subtype=subtype,347 bedrooms=beds, bathrooms=baths,348 living_area_sqft=floor,349 year_built=_qv(merged.get("yearBuilt")),350 description=str(merged.get("description") or ""),351 images=[i for i in imgs if isinstance(i, str)],352 lat=_qv(geo.get("latitude")), lng=_qv(geo.get("longitude")),353 details={"jsonld_types": [str(n.get("@type")) for n in picked],354 "availability": availability},355 )356 for k, v in static.items():357 if not kw.get(k):358 kw[k] = v359 if not kw["street_address"] and not kw["title"]:360 return None361 return Listing(**kw)362