jsonld connector: Next.js flight JSON-LD (Elliman), non-@ keys + list @type (John L. Scott), standalone Offer nodes, .gz + dynamic sitemap shards (Weichert), meaningful-subtype filter
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 changed file +60 −10
modified
homeka/connectors/json/jsonld_site.py
+60 −10
@@ -44,9 +44,27 @@ _LISTING_TYPES = { | ||
| 44 | 44 | _SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I) |
| 45 | 45 | |
| 46 | 46 | |
| 47 | +# Next.js "flight" streaming: some SSR sites (Douglas Elliman) embed the | |
| 48 | +# JSON-LD as self.__next_s.push([0,{"type":"application/ld+json", | |
| 49 | +# "children":"<escaped JSON>"}]) instead of a <script> tag. | |
| 50 | +_NEXT_LD_RE = re.compile( | |
| 51 | + r'application/ld\+json\\?"[^}]*?\\?"children\\?":\\?"((?:[^"\\]|\\.)*)"', | |
| 52 | + re.S) | |
| 53 | + | |
| 54 | + | |
| 55 | +def _blocks(html: str): | |
| 56 | + yield from _LD_RE.findall(html) | |
| 57 | + for esc in _NEXT_LD_RE.findall(html): | |
| 58 | + try: | |
| 59 | + yield json.loads(f'"{esc}"') # unescape the JS string | |
| 60 | + except ValueError: | |
| 61 | + continue | |
| 62 | + | |
| 63 | + | |
| 47 | 64 | def iter_ld(html: str): |
| 48 | − """Iterate JSON-LD objects of a page (each block, flattened from @graph).""" | |
| 49 | − for block in _LD_RE.findall(html): | |
| 65 | + """Iterate JSON-LD objects of a page (each block, flattened from @graph). | |
| 66 | + Tolerates non-standard keys without the '@' prefix (John L. Scott).""" | |
| 67 | + for block in _blocks(html): | |
| 50 | 68 | block = block.strip() |
| 51 | 69 | if not block: |
| 52 | 70 | continue |
@@ -55,9 +73,14 @@ def iter_ld(html: str): | ||
| 55 | 73 | data = json.loads(block, strict=False) |
| 56 | 74 | except ValueError: |
| 57 | 75 | continue |
| 58 | − nodes = data.get("@graph", [data]) if isinstance(data, dict) else data | |
| 76 | + if isinstance(data, dict): | |
| 77 | + nodes = data.get("@graph") or data.get("graph") or [data] | |
| 78 | + else: | |
| 79 | + nodes = data | |
| 59 | 80 | for node in (nodes if isinstance(nodes, list) else [nodes]): |
| 60 | 81 | if isinstance(node, dict): |
| 82 | + if "@type" not in node and "type" in node: | |
| 83 | + node = {**node, "@type": node["type"]} | |
| 61 | 84 | yield node |
| 62 | 85 | |
| 63 | 86 | |
@@ -101,11 +124,21 @@ class JSONLDSiteConnector(BaseConnector): | ||
| 101 | 124 | if sm_exclude and sm_exclude.search(sm): |
| 102 | 125 | continue |
| 103 | 126 | try: |
| 104 | − body = self.get(sm).text | |
| 127 | + resp = self.get(sm) | |
| 128 | + if sm.split("?")[0].endswith(".gz"): | |
| 129 | + import gzip | |
| 130 | + body = gzip.decompress(resp.content).decode( | |
| 131 | + "utf-8", errors="replace") | |
| 132 | + else: | |
| 133 | + body = resp.text | |
| 105 | 134 | except Exception: |
| 106 | 135 | continue |
| 107 | 136 | for loc in _LOC_RE.findall(body): |
| 108 | − if loc.endswith(".xml"): | |
| 137 | + # nested sitemap: .xml/.gz files, or dynamic shard URLs | |
| 138 | + # (sitemapindex.aspx / sitemaplistings.ashx à la Weichert) | |
| 139 | + low = loc.split("?")[0].lower() | |
| 140 | + if (low.endswith((".xml", ".gz")) | |
| 141 | + or "sitemap" in loc.lower().rsplit("/", 1)[-1]): | |
| 109 | 142 | nxt.append(loc) |
| 110 | 143 | else: |
| 111 | 144 | add(loc) |
@@ -157,19 +190,34 @@ class JSONLDSiteConnector(BaseConnector): | ||
| 157 | 190 | static: dict) -> Listing | None: |
| 158 | 191 | """Merge every listing-typed JSON-LD node of the page (REW splits the |
| 159 | 192 | data across Product + SingleFamilyResidence).""" |
| 160 | − picked = [n for n in nodes | |
| 161 | − if str(n.get("@type") or "").lower() in _LISTING_TYPES] | |
| 193 | + def _t(n): | |
| 194 | + t = n.get("@type") | |
| 195 | + if isinstance(t, list): # e.g. [Land, RealEstateListing, Product] | |
| 196 | + for x in t: | |
| 197 | + if str(x).lower() in _LISTING_TYPES: | |
| 198 | + return str(x).lower() | |
| 199 | + return str(t[0]).lower() if t else "" | |
| 200 | + return str(t or "").lower() | |
| 201 | + | |
| 202 | + picked = [n for n in nodes if _t(n) in _LISTING_TYPES] | |
| 162 | 203 | if not picked: |
| 163 | 204 | return None |
| 205 | + # standalone Offer nodes (John L. Scott, Greybeard publish the price | |
| 206 | + # in a sibling node instead of an `offers` property) | |
| 207 | + loose_offers = [n for n in nodes if _t(n) == "offer"] | |
| 164 | 208 | merged: dict = {} |
| 165 | 209 | addr: dict = {} |
| 166 | 210 | geo: dict = {} |
| 167 | 211 | offer: dict = {} |
| 168 | 212 | subtype = "" |
| 213 | + # only meaningful dwelling types qualify as a property subtype — | |
| 214 | + # generic wrappers (Product, RealEstateListing, Place...) say nothing | |
| 215 | + _MEANINGFUL = {"singlefamilyresidence", "house", "condominium", | |
| 216 | + "townhouse", "apartment"} | |
| 169 | 217 | for n in picked: |
| 170 | − t = str(n.get("@type") or "").lower() | |
| 171 | − if t not in ("product",) and not subtype: | |
| 172 | − subtype = str(n.get("@type") or "") | |
| 218 | + t = _t(n) | |
| 219 | + if t in _MEANINGFUL and not subtype: | |
| 220 | + subtype = t | |
| 173 | 221 | for k, v in n.items(): |
| 174 | 222 | if v in (None, "", [], {}): |
| 175 | 223 | continue |
@@ -183,6 +231,8 @@ class JSONLDSiteConnector(BaseConnector): | ||
| 183 | 231 | offer = {**o, **offer} |
| 184 | 232 | else: |
| 185 | 233 | merged.setdefault(k, v) |
| 234 | + for o in loose_offers: | |
| 235 | + offer = {**o, **offer} | |
| 186 | 236 | if isinstance(merged.get("address"), str): |
| 187 | 237 | addr.setdefault("streetAddress", merged["address"]) |
| 188 | 238 | |
| 189 | 239 | |