jsonld connector: Next.js flight JSON-LD (Elliman), non-@ keys + list @type (John L. Scott), standalone Offer nodes, .gz + dynamic sitemap shards (Weichert), meaningful-subtype filter
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 changed file +60 −10
modified
homeka/connectors/json/jsonld_site.py
+60 −10
@@ -44,9 +44,27 @@ _LISTING_TYPES = { | |||||
| 44 | _SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I) | 44 | _SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I) | ||
| 45 | 45 | ||||
| 46 | 46 | ||||
| 47 | +# Next.js "flight" streaming: some SSR sites (Douglas Elliman) embed the | ||||
| 48 | +# JSON-LD as self.__next_s.push([0,{"type":"application/ld+json", | ||||
| 49 | +# "children":"<escaped JSON>"}]) instead of a <script> tag. | ||||
| 50 | +_NEXT_LD_RE = re.compile( | ||||
| 51 | + r'application/ld\+json\\?"[^}]*?\\?"children\\?":\\?"((?:[^"\\]|\\.)*)"', | ||||
| 52 | + re.S) | ||||
| 53 | + | ||||
| 54 | + | ||||
| 55 | +def _blocks(html: str): | ||||
| 56 | + yield from _LD_RE.findall(html) | ||||
| 57 | + for esc in _NEXT_LD_RE.findall(html): | ||||
| 58 | + try: | ||||
| 59 | + yield json.loads(f'"{esc}"') # unescape the JS string | ||||
| 60 | + except ValueError: | ||||
| 61 | + continue | ||||
| 62 | + | ||||
| 63 | + | ||||
| 47 | def iter_ld(html: str): | 64 | def iter_ld(html: str): | ||
| 48 | − """Iterate JSON-LD objects of a page (each block, flattened from @graph).""" | 65 | + """Iterate JSON-LD objects of a page (each block, flattened from @graph). | ||
| 49 | − for block in _LD_RE.findall(html): | 66 | + Tolerates non-standard keys without the '@' prefix (John L. Scott).""" | ||
| 67 | + for block in _blocks(html): | ||||
| 50 | block = block.strip() | 68 | block = block.strip() | ||
| 51 | if not block: | 69 | if not block: | ||
| 52 | continue | 70 | continue | ||
@@ -55,9 +73,14 @@ def iter_ld(html: str): | |||||
| 55 | data = json.loads(block, strict=False) | 73 | data = json.loads(block, strict=False) | ||
| 56 | except ValueError: | 74 | except ValueError: | ||
| 57 | continue | 75 | continue | ||
| 58 | − nodes = data.get("@graph", [data]) if isinstance(data, dict) else data | 76 | + if isinstance(data, dict): | ||
| 77 | + nodes = data.get("@graph") or data.get("graph") or [data] | ||||
| 78 | + else: | ||||
| 79 | + nodes = data | ||||
| 59 | for node in (nodes if isinstance(nodes, list) else [nodes]): | 80 | for node in (nodes if isinstance(nodes, list) else [nodes]): | ||
| 60 | if isinstance(node, dict): | 81 | if isinstance(node, dict): | ||
| 82 | + if "@type" not in node and "type" in node: | ||||
| 83 | + node = {**node, "@type": node["type"]} | ||||
| 61 | yield node | 84 | yield node | ||
| 62 | 85 | ||||
| 63 | 86 | ||||
@@ -101,11 +124,21 @@ class JSONLDSiteConnector(BaseConnector): | |||||
| 101 | if sm_exclude and sm_exclude.search(sm): | 124 | if sm_exclude and sm_exclude.search(sm): | ||
| 102 | continue | 125 | continue | ||
| 103 | try: | 126 | try: | ||
| 104 | − body = self.get(sm).text | 127 | + resp = self.get(sm) | ||
| 128 | + if sm.split("?")[0].endswith(".gz"): | ||||
| 129 | + import gzip | ||||
| 130 | + body = gzip.decompress(resp.content).decode( | ||||
| 131 | + "utf-8", errors="replace") | ||||
| 132 | + else: | ||||
| 133 | + body = resp.text | ||||
| 105 | except Exception: | 134 | except Exception: | ||
| 106 | continue | 135 | continue | ||
| 107 | for loc in _LOC_RE.findall(body): | 136 | for loc in _LOC_RE.findall(body): | ||
| 108 | − if loc.endswith(".xml"): | 137 | + # nested sitemap: .xml/.gz files, or dynamic shard URLs | ||
| 138 | + # (sitemapindex.aspx / sitemaplistings.ashx à la Weichert) | ||||
| 139 | + low = loc.split("?")[0].lower() | ||||
| 140 | + if (low.endswith((".xml", ".gz")) | ||||
| 141 | + or "sitemap" in loc.lower().rsplit("/", 1)[-1]): | ||||
| 109 | nxt.append(loc) | 142 | nxt.append(loc) | ||
| 110 | else: | 143 | else: | ||
| 111 | add(loc) | 144 | add(loc) | ||
@@ -157,19 +190,34 @@ class JSONLDSiteConnector(BaseConnector): | |||||
| 157 | static: dict) -> Listing | None: | 190 | static: dict) -> Listing | None: | ||
| 158 | """Merge every listing-typed JSON-LD node of the page (REW splits the | 191 | """Merge every listing-typed JSON-LD node of the page (REW splits the | ||
| 159 | data across Product + SingleFamilyResidence).""" | 192 | data across Product + SingleFamilyResidence).""" | ||
| 160 | − picked = [n for n in nodes | 193 | + def _t(n): | ||
| 161 | − if str(n.get("@type") or "").lower() in _LISTING_TYPES] | 194 | + t = n.get("@type") | ||
| 195 | + if isinstance(t, list): # e.g. [Land, RealEstateListing, Product] | ||||
| 196 | + for x in t: | ||||
| 197 | + if str(x).lower() in _LISTING_TYPES: | ||||
| 198 | + return str(x).lower() | ||||
| 199 | + return str(t[0]).lower() if t else "" | ||||
| 200 | + return str(t or "").lower() | ||||
| 201 | + | ||||
| 202 | + picked = [n for n in nodes if _t(n) in _LISTING_TYPES] | ||||
| 162 | if not picked: | 203 | if not picked: | ||
| 163 | return None | 204 | return None | ||
| 205 | + # standalone Offer nodes (John L. Scott, Greybeard publish the price | ||||
| 206 | + # in a sibling node instead of an `offers` property) | ||||
| 207 | + loose_offers = [n for n in nodes if _t(n) == "offer"] | ||||
| 164 | merged: dict = {} | 208 | merged: dict = {} | ||
| 165 | addr: dict = {} | 209 | addr: dict = {} | ||
| 166 | geo: dict = {} | 210 | geo: dict = {} | ||
| 167 | offer: dict = {} | 211 | offer: dict = {} | ||
| 168 | subtype = "" | 212 | subtype = "" | ||
| 213 | + # only meaningful dwelling types qualify as a property subtype — | ||||
| 214 | + # generic wrappers (Product, RealEstateListing, Place...) say nothing | ||||
| 215 | + _MEANINGFUL = {"singlefamilyresidence", "house", "condominium", | ||||
| 216 | + "townhouse", "apartment"} | ||||
| 169 | for n in picked: | 217 | for n in picked: | ||
| 170 | − t = str(n.get("@type") or "").lower() | 218 | + t = _t(n) | ||
| 171 | − if t not in ("product",) and not subtype: | 219 | + if t in _MEANINGFUL and not subtype: | ||
| 172 | − subtype = str(n.get("@type") or "") | 220 | + subtype = t | ||
| 173 | for k, v in n.items(): | 221 | for k, v in n.items(): | ||
| 174 | if v in (None, "", [], {}): | 222 | if v in (None, "", [], {}): | ||
| 175 | continue | 223 | continue | ||
@@ -183,6 +231,8 @@ class JSONLDSiteConnector(BaseConnector): | |||||
| 183 | offer = {**o, **offer} | 231 | offer = {**o, **offer} | ||
| 184 | else: | 232 | else: | ||
| 185 | merged.setdefault(k, v) | 233 | merged.setdefault(k, v) | ||
| 234 | + for o in loose_offers: | ||||
| 235 | + offer = {**o, **offer} | ||||
| 186 | if isinstance(merged.get("address"), str): | 236 | if isinstance(merged.get("address"), str): | ||
| 187 | addr.setdefault("streetAddress", merged["address"]) | 237 | addr.setdefault("streetAddress", merged["address"]) | ||
| 188 | 238 | ||||
| 189 | 239 | ||||