# ----------------------------------------------------------------------------- # Home-Ka — US real-estate aggregator (Groupe KA) # Author: Simon-Pierre Boucher — contact@spboucher.ai # connectors/xml/feed.py : generic XML feed connector (brokerage exports, # franchise syndication feeds, legacy XML drops). Config-driven: # # {"id": "acme_xml", "connector_type": "xml", "config": { # "url": "https://feeds.acmerealty.com/listings.xml", # "item_tag": "listing", # repeated element # "mapping": {"external_id": "mlsNumber", "list_price": "price", # "street_address": "address/street", "city": "address/city", # "state": "address/state", "zip_code": "address/zip", # "images": "photos/photo", ...}, # ElementTree paths # "static": {"state": "TX"}, # "headers": {}, "auth_user": "", "auth_pass": ""}} # ----------------------------------------------------------------------------- from __future__ import annotations import xml.etree.ElementTree as ET from ..base import BaseConnector from ...schema import Listing _LIST_FIELDS = {"images", "features"} def _strip_ns(tag: str) -> str: return tag.rsplit("}", 1)[-1] class XMLFeedConnector(BaseConnector): family = "xml" request_delay = 0.2 def fetch(self) -> list[Listing]: cfg = self.config kw = {} if cfg.get("headers"): kw["headers"] = cfg["headers"] if cfg.get("auth_user"): kw["auth"] = (cfg["auth_user"], cfg.get("auth_pass", "")) if cfg.get("path"): body = open(cfg["path"], "rb").read() else: body = self.get(cfg["url"], **kw).content root = ET.fromstring(body) item_tag = cfg.get("item_tag", "listing") mapping = cfg.get("mapping") or {} static = cfg.get("static") or {} out: list[Listing] = [] for el in root.iter(): if _strip_ns(el.tag) != item_tag: continue lst = self._to_listing(el, mapping, static) if lst is not None: out.append(lst) return out def _find(self, el: ET.Element, path: str): """Namespace-tolerant lookup ('address/city' or '@attr' or 'tag@attr').""" if path.startswith("@"): return [el.get(path[1:])] attr = None if "@" in path: path, _, attr = path.partition("@") parts = path.split("/") nodes = [el] for p in parts: nxt = [] for n in nodes: nxt.extend(c for c in n if _strip_ns(c.tag) == p) nodes = nxt if attr: return [n.get(attr) for n in nodes] return [(n.text or "").strip() for n in nodes] def _to_listing(self, el: ET.Element, mapping: dict, static: dict) -> Listing | None: kw: dict = {"source": self.source_id, "external_id": "", "url": "", "details": {}} for field, path in mapping.items(): vals = [v for v in self._find(el, path) if v] if not vals: continue kw[field] = vals if field in _LIST_FIELDS else vals[0] for field, v in static.items(): kw.setdefault(field, v) if not kw.get("external_id"): return None kw["external_id"] = str(kw["external_id"]) # flat leaf values preserved for debugging/passthrough for c in el: if len(c) == 0 and c.text and c.text.strip(): kw["details"].setdefault(_strip_ns(c.tag), c.text.strip()) allowed = set(Listing.__dataclass_fields__) return Listing(**{k: v for k, v in kw.items() if k in allowed})