# ----------------------------------------------------------------------------- # Home-Ka — US real-estate aggregator (Groupe KA) # Author: Simon-Pierre Boucher — contact@spboucher.ai # connectors/csv/feed.py : CSV feed connector — brokerage exports, scheduled # drops (also consumed by the sftp family after download). # # {"id": "acme_csv", "connector_type": "csv", "config": { # "url": "https://feeds.acme.com/export.csv", # or "path": local file # "delimiter": ",", "encoding": "utf-8-sig", # "mapping": {"external_id": "MLS #", "list_price": "Price", # "street_address": "Address", "city": "City", ...}, # "images_split": "|", # column with delimiter-joined URLs # "static": {"state": "FL"}}} # ----------------------------------------------------------------------------- from __future__ import annotations import csv import io from ..base import BaseConnector from .._maputil import apply_mapping from ...schema import Listing class CSVFeedConnector(BaseConnector): family = "csv" request_delay = 0.2 def fetch(self) -> list[Listing]: cfg = self.config if cfg.get("path"): text = open(cfg["path"], encoding=cfg.get("encoding", "utf-8-sig"), newline="").read() else: resp = self.get(cfg["url"]) resp.encoding = cfg.get("encoding", "utf-8-sig") text = resp.text return parse_csv(self.source_id, text, cfg) def parse_csv(source_id: str, text: str, cfg: dict) -> list[Listing]: """Shared with the sftp family.""" mapping = cfg.get("mapping") or {} static = cfg.get("static") or {} split = cfg.get("images_split", "|") out: list[Listing] = [] reader = csv.DictReader(io.StringIO(text), delimiter=cfg.get("delimiter", ",")) for rec in reader: lst = apply_mapping(source_id, rec, mapping, static) if lst is None: continue if lst.images and len(lst.images) == 1 and split in (lst.images[0] or ""): lst.images = [u.strip() for u in lst.images[0].split(split) if u.strip()] out.append(lst) return out