jsonld connector: CDATA sitemap locs, lenient JSON-LD parsing (strict=False), Place type, sitemap_exclude (skip USM off-market shards)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Simon-Pierre Boucher committed 27 days ago (Aug 28, 2026) parent 118ebcf

1 changed file +13 −2

modified homeka/connectors/json/jsonld_site.py +13 −2

		`@@ -29,11 +29,15 @@ from ...schema import Listing`
29	29
30	30	_LD_RE = re.compile(r'<script[^>]+type=["\']application/ld\+json["\'][^>]>(.?)</script>',
31	31	re.S \| re.I)
32		−_LOC_RE = re.compile(r"<loc>\s(.?)\s*</loc>", re.S \| re.I)
	32	+_LOC_RE = re.compile(r"<loc>\s(?:<!\[CDATA\[)?\s(.?)\s(?:\]\]>)?\s*</loc>",
	33	+ re.S \| re.I)
33	34
34	35	_LISTING_TYPES = {
35	36	"realestatelisting", "singlefamilyresidence", "house", "residence",
36	37	"apartment", "product", "accommodation", "condominium", "townhouse",
	38	+ # some SSR platforms (Repliers/Next.js) mark the listing as a plain Place;
	39	+ # safe here because pages are pre-filtered by url_include
	40	+ "place",
37	41	}
38	42
39	43	# URL slug fallback: ".../7039-s-straight-avenue-homosassa-fl-34446/"
		`@@ -47,7 +51,8 @@ def iter_ld(html: str):`
47	51	if not block:
48	52	continue
49	53	try:
50		− data = json.loads(block)
	54	+ # strict=False: some platforms leave raw control chars in remarks
	55	+ data = json.loads(block, strict=False)
51	56	except ValueError:
52	57	continue
53	58	nodes = data.get("@graph", [data]) if isinstance(data, dict) else data
		`@@ -82,6 +87,10 @@ class JSONLDSiteConnector(BaseConnector):`
82	87	sitemaps = [cfg["sitemap_url"]] if cfg.get("sitemap_url") else []
83	88	if not sitemaps and cfg.get("base_url"):
84	89	sitemaps = [cfg["base_url"].rstrip("/") + "/sitemap.xml"]
	90	+ # skip nested sitemaps by name (e.g. Union Street Media's huge
	91	+ # "off-market-sitemap-N.xml" full of sold listings)
	92	+ sm_exclude = re.compile(cfg["sitemap_exclude"]) \
	93	+ if cfg.get("sitemap_exclude") else None
85	94	depth = 0
86	95	max_pages = int(cfg.get("max_pages", 400))
87	96	while sitemaps and depth < 3 and len(urls) < max_pages:
		`@@ -89,6 +98,8 @@ class JSONLDSiteConnector(BaseConnector):`
89	98	for sm in sitemaps[:30]:
90	99	if len(urls) >= max_pages:
91	100	break
	101	+ if sm_exclude and sm_exclude.search(sm):
	102	+ continue
92	103	try:
93	104	body = self.get(sm).text
94	105	except Exception:
95	106