# -----------------------------------------------------------------------------
# Home-Ka — US real-estate aggregator (Groupe KA)
# Author: Simon-Pierre Boucher — contact@spboucher.ai
# connectors/base.py : connector base class + fetch backends.
#
# Same philosophy as immo-ka: SOURCE → connector → raw ingestion →
# normalization → property matching → deduplication → canonical listing →
# search index → Home-Ka. One connector adapts ONE source; the pipeline
# (ingest.py) handles the diff with the database.
#
# Two kinds of connectors:
#   - FAMILY connectors (reso/, rets/, xml/, json/, csv/, sftp/, public_data/):
#     generic, config-driven — one class serves many sources. Instantiated
#     with (source_id, config) from a row of the `sources` table. They carry
#     a class attribute `family` (e.g. "reso").
#   - CUSTOM connectors (custom_broker/*.py): one class per site, class
#     attribute `source_id` set, zero-arg constructible — exactly like
#     immo-ka's per-agency connectors.
#
# Data-sourcing policy (Home-Ka): favor the LOWEST-LEVEL source available —
# direct feeds (RESO Web API, RETS, XML/JSON/CSV drops) over site crawling.
# Crawling is a discovery/bootstrap tool, not the foundation; the proxy
# backends (Firecrawl/Scrapfly) exist as a LAST RESORT only.
# -----------------------------------------------------------------------------
from __future__ import annotations

import json
import os
import time

import requests

from ..schema import Listing

USER_AGENT = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
              "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36 "
              "HomeKaBot/1.0 (+https://www.home-ka.com/bot; contact@spboucher.ai)")

FIRECRAWL_API = "https://api.firecrawl.dev/v1/scrape"
SCRAPFLY_API = "https://api.scrapfly.io/scrape"


def expand_env(config: dict) -> dict:
    """Resolve '$ENV:NAME' placeholders in a source config — credentials live
    in the environment (.env), never in the database."""
    out = {}
    for k, v in (config or {}).items():
        if isinstance(v, str) and v.startswith("$ENV:"):
            out[k] = os.environ.get(v[5:], "")
        elif isinstance(v, dict):
            out[k] = expand_env(v)
        else:
            out[k] = v
    return out


class BaseConnector:
    """A connector = one adapter for one listing source.

    Family subclasses: set `family`, accept (source_id, config) and implement
    `fetch()`. Custom subclasses: set `source_id` and implement `fetch()`.
    `fetch()` returns the complete list of listings currently visible at the
    source; the pipeline handles additions/changes/removals.
    """

    family: str = ""                     # set on generic, config-driven families
    source_id: str = ""                  # set on custom one-site connectors
    request_delay: float = 0.6           # politeness between requests
    timeout: int = 30
    use_detail_cache: bool = True        # DB cache of detail pages

    def __init__(self, source_id: str | None = None,
                 config: dict | None = None) -> None:
        if source_id:
            self.source_id = source_id
        self.config = expand_env(config or {})
        if self.config.get("request_delay"):
            self.request_delay = float(self.config["request_delay"])
        self.session = requests.Session()
        self.session.headers["User-Agent"] = USER_AGENT
        self._last_request = 0.0
        self._detail_con = None

    # -- backends -------------------------------------------------------------
    def _throttle(self) -> None:
        wait = self.request_delay - (time.time() - self._last_request)
        if wait > 0:
            time.sleep(wait)

    def get(self, url: str, **kw) -> requests.Response:
        """Polite direct GET."""
        self._throttle()
        resp = self.session.get(url, timeout=self.timeout, **kw)
        self._last_request = time.time()
        resp.raise_for_status()
        return resp

    def post(self, url: str, **kw) -> requests.Response:
        """Polite direct POST (internal search APIs)."""
        self._throttle()
        resp = self.session.post(url, timeout=self.timeout, **kw)
        self._last_request = time.time()
        resp.raise_for_status()
        return resp

    def fc_scrape(self, url: str, proxy: str = "auto", render: bool = False,
                  wait_for: int = 0, retries: int = 2) -> str:
        """Raw HTML via Firecrawl — used as a THROUGHPUT backend (parallel
        fetching without hammering the source directly) and to reach
        bot-challenged sites. `maxAge` leans on Firecrawl's cache so weekly
        re-syncs stay cheap. Retries on 429/5xx with backoff."""
        key = os.environ.get("FIRECRAWL_API_KEY")
        if not key:
            raise RuntimeError("FIRECRAWL_API_KEY missing (see .env)")
        payload: dict = {"url": url, "timeout": 60000,
                         "formats": ["html"] if render else ["rawHtml"],
                         "proxy": proxy,
                         "maxAge": 6 * 86400 * 1000}
        if wait_for:
            payload["waitFor"] = wait_for
        last = ""
        for attempt in range(retries + 1):
            try:
                resp = requests.post(
                    FIRECRAWL_API, json=payload,
                    headers={"Authorization": f"Bearer {key}"}, timeout=120)
                if resp.status_code in (429, 500, 502, 503):
                    time.sleep(3 * (attempt + 1))
                    continue
                resp.raise_for_status()
                data = resp.json().get("data") or {}
                return data.get("rawHtml") or data.get("html") or ""
            except requests.RequestException as exc:
                last = str(exc)
                time.sleep(2 * (attempt + 1))
        raise RuntimeError(f"firecrawl failed for {url}: {last}")

    def get_rendered(self, url: str, wait_for: int = 0,
                     proxy: str | None = None) -> str:
        """LAST RESORT — JS-rendered HTML via Firecrawl (FIRECRAWL_API_KEY)."""
        key = os.environ.get("FIRECRAWL_API_KEY")
        if not key:
            raise RuntimeError("FIRECRAWL_API_KEY missing (see .env)")
        payload: dict = {"url": url, "formats": ["html"], "timeout": 90000}
        if wait_for:
            payload["waitFor"] = wait_for
        if proxy:
            payload["proxy"] = proxy
        resp = requests.post(FIRECRAWL_API, json=payload,
                             headers={"Authorization": f"Bearer {key}"},
                             timeout=150)
        resp.raise_for_status()
        return (resp.json().get("data") or {}).get("html", "")

    def scrapfly(self, url: str, render_js: bool = True, asp: bool = True,
                 rendering_wait: int = 0, country: str = "us",
                 wait_for_selector: str | None = None,
                 js_scenario: list | str | None = None,
                 proxy_pool: str | None = None, headers: dict | None = None,
                 method: str = "GET", body: str | None = None) -> dict:
        """LAST RESORT — full Scrapfly call, returns the `result` dict.
        Home-Ka policy: only for sources with no lower-level path; convert
        every interesting source to a direct feed instead."""
        import base64
        key = os.environ.get("SCRAPFLY_KEY")
        if not key:
            raise RuntimeError("SCRAPFLY_KEY missing (see .env)")
        params: dict = {"key": key, "url": url, "country": country}
        if asp:
            params["asp"] = "true"
        if render_js:
            params["render_js"] = "true"
        if rendering_wait:
            params["rendering_wait"] = rendering_wait
        if wait_for_selector:
            params["wait_for_selector"] = wait_for_selector
        if proxy_pool:
            params["proxy_pool"] = proxy_pool
        if js_scenario is not None:
            js = js_scenario if isinstance(js_scenario, str) else json.dumps(js_scenario)
            params["js_scenario"] = base64.urlsafe_b64encode(js.encode()).decode()
        if headers:
            for k, v in headers.items():
                params[f"headers[{k}]"] = v
        self._throttle()
        if method.upper() == "POST":
            resp = requests.post(SCRAPFLY_API, params=params,
                                 data=(body or ""), timeout=180)
        else:
            resp = requests.get(SCRAPFLY_API, params=params, timeout=180)
        self._last_request = time.time()
        try:
            return resp.json().get("result") or {}
        except ValueError:
            return {}

    def detail(self, external_id: str, key: str, fetch_fn) -> dict:
        """Detail-page payload with cache: `fetch_fn` only runs when the
        listing is new or its list-content hash changed."""
        if not self.use_detail_cache:
            return fetch_fn() or {}
        from .. import db
        if self._detail_con is None:
            self._detail_con = db.connect()
        cached = db.get_cached_detail(self._detail_con, self.source_id,
                                      str(external_id), key)
        if cached is not None:
            return cached
        payload = fetch_fn() or {}
        db.put_cached_detail(self._detail_con, self.source_id,
                             str(external_id), key, payload)
        return payload

    # -- contract --------------------------------------------------------------
    def fetch(self) -> list[Listing]:
        raise NotImplementedError


# =============================================================================
# Anti-bot resilience (Groupe KA) — auto-escalation of get() without touching
# connector bodies: an anti-bot block (403/429/503/challenge) or a network cut
# triggers the fallback chain (Oxylabs residential -> Scrapfly ASP -> Bright
# Data). See connectors/_resilient.py. Idempotent (_KA_RESILIENT_WRAPPED).
# Direct fetch stays the default — proxies only fire when blocked.
# =============================================================================
if not getattr(BaseConnector, "_KA_RESILIENT_WRAPPED", False):
    import requests as _ka_requests  # noqa: E402
    from . import _resilient as _kar  # noqa: E402

    _ka_orig_get = BaseConnector.get

    def _ka_full_url(url, kw):
        try:
            return _ka_requests.Request("GET", url,
                                        params=kw.get("params")).prepare().url
        except Exception:  # noqa: BLE001
            return url

    def _ka_resilient_get(self, url, **kw):
        timeout = getattr(self, "timeout", 30)
        headers = kw.get("headers")
        try:
            return _ka_orig_get(self, url, **kw)
        except _ka_requests.HTTPError as exc:
            r = getattr(exc, "response", None)
            if r is not None and _kar.is_blocked(r):
                target = getattr(r, "url", None) or _ka_full_url(url, kw)
                better = _kar.escalate_if_blocked(
                    r, target, timeout=timeout, headers=headers)
                if better is not None and getattr(better, "status_code", 0) == 200:
                    return better
            raise
        except (_ka_requests.ConnectionError, _ka_requests.Timeout):
            better = _kar.escalate(_ka_full_url(url, kw),
                                   timeout=timeout, headers=headers)
            if better is not None and getattr(better, "status_code", 0) == 200:
                return better
            raise

    def _ka_get_resilient(self, url, *, render_js=False, country="us", **kw):
        """Explicit anti-bot fetch: forces the fallback chain when needed."""
        timeout = getattr(self, "timeout", 30)
        headers = kw.get("headers")
        try:
            resp = _ka_orig_get(self, url, **kw)
        except _ka_requests.HTTPError as exc:
            resp = getattr(exc, "response", None)
        except (_ka_requests.ConnectionError, _ka_requests.Timeout):
            resp = None
        target = _ka_full_url(url, kw)
        if resp is not None and getattr(resp, "url", None):
            target = resp.url
        return _kar.escalate_if_blocked(resp, target, timeout=timeout,
                                        country=country, render_js=render_js,
                                        headers=headers)

    BaseConnector.get = _ka_resilient_get
    BaseConnector.get_resilient = _ka_get_resilient
    BaseConnector._KA_RESILIENT_WRAPPED = True