| 43 | # URL slug fallback: ".../7039-s-straight-avenue-homosassa-fl-34446/" |
43 | # URL slug fallback: ".../7039-s-straight-avenue-homosassa-fl-34446/" |
| 44 | _SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I) |
44 | _SLUG_ADDR_RE = re.compile(r"/([\w-]+)-([a-z]{2})-(\d{5})/?$", re.I) |
| 45 | |
45 | |
| |
46 | +# `name` fallback: several platforms (Real Geeks, @properties/Christie's) |
| |
47 | +# put the full address in Product.name — "123 Main St #4, Austin, TX 78701" |
| |
48 | +_NAME_ADDR_RE = re.compile( |
| |
49 | + r"^(.+?),\s*([^,]+?),\s*([A-Za-z]{2})[,\s]+(\d{5})(?:-\d{4})?\s*$") |
| |
50 | + |
| 46 | |
51 | |
| 47 | # Next.js "flight" streaming: some SSR sites (Douglas Elliman) embed the |
52 | # Next.js "flight" streaming: some SSR sites (Douglas Elliman) embed the |
| 48 | # JSON-LD as self.__next_s.push([0,{"type":"application/ld+json", |
53 | # JSON-LD as self.__next_s.push([0,{"type":"application/ld+json", |
| 116 | if cfg.get("sitemap_exclude") else None |
121 | if cfg.get("sitemap_exclude") else None |
| 117 | depth = 0 |
122 | depth = 0 |
| 118 | max_pages = int(cfg.get("max_pages", 400)) |
123 | max_pages = int(cfg.get("max_pages", 400)) |
| |
124 | + max_children = int(cfg.get("sitemap_max_children", 80)) |
| |
125 | + _PRIORITY = re.compile(r"active|listing|residential|propert|homes", |
| |
126 | + re.I) |
| 119 | while sitemaps and depth < 3 and len(urls) < max_pages: |
127 | while sitemaps and depth < 3 and len(urls) < max_pages: |
| |
128 | + # likely-listing shards first — big indexes (@properties: 69 |
| |
129 | + # children) would otherwise exhaust the budget on agent/office maps |
| |
130 | + sitemaps.sort(key=lambda u: 0 if _PRIORITY.search( |
| |
131 | + u.rsplit("/", 1)[-1]) else 1) |
| 120 | nxt: list[str] = [] |
132 | nxt: list[str] = [] |
| 121 | − for sm in sitemaps[:30]: |
133 | + for sm in sitemaps[:max_children]: |
| 122 | if len(urls) >= max_pages: |
134 | if len(urls) >= max_pages: |
| 123 | break |
135 | break |
| 124 | if sm_exclude and sm_exclude.search(sm): |
136 | if sm_exclude and sm_exclude.search(sm): |
| 308 | city = str(addr.get("addressLocality") or "") |
320 | city = str(addr.get("addressLocality") or "") |
| 309 | state = str(addr.get("addressRegion") or "") |
321 | state = str(addr.get("addressRegion") or "") |
| 310 | zip_code = str(addr.get("postalCode") or "") |
322 | zip_code = str(addr.get("postalCode") or "") |
| |
323 | + if not street: # name fallback: "123 Main St, Austin, TX 78701" |
| |
324 | + m = _NAME_ADDR_RE.match(str(merged.get("name") or "").strip()) |
| |
325 | + if m: |
| |
326 | + street, city = m.group(1), city or m.group(2) |
| |
327 | + state, zip_code = state or m.group(3).upper(), zip_code or m.group(4) |
| 311 | if not street: # slug fallback: .../123-main-street-austin-tx-78701/ |
328 | if not street: # slug fallback: .../123-main-street-austin-tx-78701/ |
| 312 | m = _SLUG_ADDR_RE.search(url) |
329 | m = _SLUG_ADDR_RE.search(url) |
| 313 | if m: |
330 | if m: |