# ----------------------------------------------------------------------------- # Home-Ka — US real-estate aggregator (Groupe KA) # Author: Simon-Pierre Boucher — contact@spboucher.ai # connectors/sftp/drop.py : SFTP drop connector — brokerages/MLS vendors that # push periodic CSV/XML exports to an SFTP folder. Downloads the newest # matching file then dispatches to the csv/xml parser. # # {"id": "acme_sftp", "connector_type": "sftp", "config": { # "host": "sftp.acmefeeds.com", "port": 22, # "username": "$ENV:ACME_SFTP_USER", "password": "$ENV:ACME_SFTP_PASS", # "key_path": "", # or private key file # "remote_dir": "/exports", "pattern": "listings_*.csv", # "format": "csv", # csv | xml # ... then the csv/xml family options (mapping, static, ...)}} # # Requires `paramiko` (in requirements.txt). # ----------------------------------------------------------------------------- from __future__ import annotations import fnmatch import io from ..base import BaseConnector from ...schema import Listing class SFTPDropConnector(BaseConnector): family = "sftp" def _download(self) -> bytes: try: import paramiko except ImportError: raise RuntimeError("paramiko missing — pip install paramiko") cfg = self.config client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) kw: dict = {"username": cfg.get("username", "")} if cfg.get("key_path"): kw["key_filename"] = cfg["key_path"] else: kw["password"] = cfg.get("password", "") client.connect(cfg["host"], port=int(cfg.get("port", 22)), timeout=30, **kw) try: sftp = client.open_sftp() remote_dir = cfg.get("remote_dir", ".") pattern = cfg.get("pattern", "*") names = [n for n in sftp.listdir(remote_dir) if fnmatch.fnmatch(n, pattern)] if not names: raise RuntimeError(f"{self.source_id}: no file matching " f"{pattern} in {remote_dir}") # newest file by mtime newest = max(names, key=lambda n: sftp.stat(f"{remote_dir}/{n}").st_mtime) buf = io.BytesIO() sftp.getfo(f"{remote_dir}/{newest}", buf) return buf.getvalue() finally: client.close() def fetch(self) -> list[Listing]: data = self._download() fmt = self.config.get("format", "csv") if fmt == "csv": from ..csv.feed import parse_csv return parse_csv(self.source_id, data.decode(self.config.get("encoding", "utf-8-sig"), errors="replace"), self.config) if fmt == "xml": from ..xml.feed import XMLFeedConnector conn = XMLFeedConnector(self.source_id, {**self.config, "path": None}) import xml.etree.ElementTree as ET root = ET.fromstring(data) item_tag = self.config.get("item_tag", "listing") mapping = self.config.get("mapping") or {} static = self.config.get("static") or {} out = [] for el in root.iter(): if el.tag.rsplit("}", 1)[-1] != item_tag: continue lst = conn._to_listing(el, mapping, static) if lst is not None: out.append(lst) return out raise RuntimeError(f"{self.source_id}: unknown sftp format {fmt!r}")