# -----------------------------------------------------------------------------
# Rent-Ka — Rental listings aggregator (Canada, outside Québec)
# Author: Simon-Pierre Boucher — contact@spboucher.ai
# connectors/realtor_ca.py : REALTOR.ca (CREA) — annonces de location publiées
#   par les COURTIERS immobiliers (MLS). C'est LA source pancanadienne des
#   logements mis en location via courtier (condos en particulier), absente des
#   portails de gestionnaires.
#
#   API interne : POST https://api2.realtor.ca/Listing.svc/PropertySearch_Post
#   (form-urlencoded, TransactionTypeId=3 = à louer, PropertySearchTypeId=1 =
#   résidentiel). Protégée par Incapsula/Imperva → chaque requête passe par
#   Scrapfly ASP + pool résidentiel (validé en live le 2026-08-28 : 200 OK,
#   7 779 locations dans le seul Grand Toronto).
#
#   L'API plafonne à MaxRecords=600 par zone (3 pages × 200) : le fetch part de
#   boîtes provinciales (ROC) et les DIVISE récursivement en quadrants tant que
#   TotalRecords > 600 — couverture complète garantie, dédup par Id (les boîtes
#   se chevauchent aux frontières). Les requêtes sont parallélisées (pool de
#   threads, compteur verrouillé, garde-fou max_requests).
#
#   Une annonce = un résultat de recherche : prix (LeaseRent), adresse complète
#   avec GPS, chambres/SDB, photo, remarques publiques, et surtout le COURTIER
#   (Individual + Organization = agence) exposé dans details.contact. Pas de
#   visite des pages détail (40 k fiches × Scrapfly = hors budget) — la photo
#   de couverture et les champs du résultat suffisent à publier.
#   Échec HONNÊTE : si une part significative des requêtes échoue, on lève
#   plutôt que de laisser l'ingestion archiver l'inventaire manquant.
# -----------------------------------------------------------------------------
from __future__ import annotations

import json
import os
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlencode

import requests

from ..schema import Listing, normalize_unit_type
from .base import SCRAPFLY_API, BaseConnector

_API = "https://api2.realtor.ca/Listing.svc/PropertySearch_Post"
_MAX_RECORDS = 600          # plafond serveur (Paging.MaxRecords)

# Boîtes de départ — Canada hors Québec, généreuses (le Québec limitrophe est
# filtré au parse via ProvinceName). Chevauchements sans conséquence : dédup
# par Id.
_SEED_BOXES: list[tuple[float, float, float, float]] = [
    # (lat_min, lat_max, lng_min, lng_max)
    (41.6, 47.6, -83.7, -74.3),     # Ontario sud (Windsor→Ottawa)
    (46.0, 57.0, -95.4, -79.0),     # Ontario nord
    (48.2, 60.0, -139.1, -114.0),   # Colombie-Britannique
    (48.9, 60.0, -120.0, -110.0),   # Alberta
    (48.9, 60.0, -110.0, -101.4),   # Saskatchewan
    (48.9, 60.0, -102.2, -95.0),    # Manitoba
    (43.3, 48.3, -69.1, -59.6),     # NB + NS + IPE
    (46.5, 60.5, -67.9, -52.5),     # Terre-Neuve-et-Labrador
    (60.0, 70.5, -141.0, -61.0),    # YT + TNO + NU
]

_PROVINCE_CODES = {
    "ontario": "ON", "british columbia": "BC", "alberta": "AB",
    "saskatchewan": "SK", "manitoba": "MB", "new brunswick": "NB",
    "nova scotia": "NS", "prince edward island": "PE",
    "newfoundland & labrador": "NL", "newfoundland and labrador": "NL",
    "yukon": "YT", "northwest territories": "NT", "nunavut": "NU",
}

# loyers non mensuels (rarissimes en résidentiel) : hors sujet
_NON_MONTHLY_RE = re.compile(r"/(week|night|dai|day|year|annum)", re.I)

# « 600 sqft » ou « 500-599 sqft » — borne basse plausible seulement
_AREA_RE = re.compile(r"(\d[\d,]*)(?:\s*-\s*(\d[\d,]*))?\s*sqft", re.I)

# types de bâtiment non résidentiels (filet — PropertySearchTypeId=1 est déjà
# résidentiel, mais quelques stationnements/locaux passent la maille)
_NON_RESIDENTIAL_RE = re.compile(
    r"\b(parking|locker|storage|office|retail|commercial|warehouse|"
    r"vacant land|land|farm|business|agriculture)\b", re.I)


class RealtorCaConnector(BaseConnector):
    """REALTOR.ca — locations MLS des courtiers, Canada hors Québec."""

    source_id = "realtor_ca"
    use_detail_cache = False    # tout vient des résultats de recherche
    workers = 6                 # requêtes Scrapfly simultanées
    max_requests = 900          # garde-fou budget Scrapfly par sync
    max_depth = 10              # profondeur de subdivision des quadrants
    max_images = 1              # les résultats n'exposent que la couverture

    def fetch(self) -> list[Listing]:
        self._lock = threading.Lock()
        self._nreq = 0
        self._failures = 0
        seen: dict[str, Listing] = {}
        queue: list[tuple[tuple[float, float, float, float], int]] = \
            [(b, 0) for b in _SEED_BOXES]
        with ThreadPoolExecutor(max_workers=self.workers) as ex:
            while queue:
                wave, queue = queue[:24], queue[24:]
                probes = list(ex.map(
                    lambda bd: self._search(bd[0], 1), wave))
                page_jobs: list[tuple[tuple, int]] = []
                for (box, depth), data in zip(wave, probes):
                    paging = (data or {}).get("Paging") or {}
                    total = int(paging.get("TotalRecords") or 0)
                    if not total:
                        continue
                    if total > _MAX_RECORDS and depth < self.max_depth:
                        queue.extend((q, depth + 1)
                                     for q in _quadrants(box))
                        continue
                    self._collect(data, seen)
                    pages = min(int(paging.get("TotalPages") or 1), 3)
                    page_jobs += [(box, p) for p in range(2, pages + 1)]
                for d in ex.map(lambda bp: self._search(bp[0], bp[1]),
                                page_jobs):
                    self._collect(d, seen)
        # échec honnête : trop d'échecs = inventaire incomplet, on n'archive pas
        if self._failures > max(3, self._nreq // 20):
            raise RuntimeError(
                f"realtor_ca : {self._failures} échecs sur {self._nreq} "
                f"requêtes — sync abandonné (inventaire incomplet)")
        if self._nreq >= self.max_requests:
            raise RuntimeError(
                f"realtor_ca : garde-fou max_requests ({self.max_requests}) "
                f"atteint — couverture incomplète, sync abandonné")
        return list(seen.values())

    # -- une requête de recherche (Scrapfly ASP, thread-safe) ------------------
    def _search(self, box: tuple, page: int) -> dict:
        with self._lock:
            if self._nreq >= self.max_requests:
                return {}
            self._nreq += 1
        key = os.environ.get("SCRAPFLY_KEY")
        if not key:
            raise RuntimeError("SCRAPFLY_KEY manquant (voir .env)")
        lat_min, lat_max, lng_min, lng_max = box
        body = urlencode({
            "ZoomLevel": "11",
            "LatitudeMin": f"{lat_min:.5f}", "LatitudeMax": f"{lat_max:.5f}",
            "LongitudeMin": f"{lng_min:.5f}", "LongitudeMax": f"{lng_max:.5f}",
            "Sort": "6-D", "PropertyTypeGroupID": "1",
            "TransactionTypeId": "3", "PropertySearchTypeId": "1",
            "Currency": "CAD", "IncludeHiddenListings": "false",
            "RecordsPerPage": "200", "ApplicationId": "1",
            "CultureId": "1", "Version": "7.0", "CurrentPage": str(page),
        })
        params = {
            "key": key, "url": _API, "asp": "true", "country": "ca",
            "proxy_pool": "public_residential_pool",
            "headers[Content-Type]":
                "application/x-www-form-urlencoded; charset=UTF-8",
            "headers[Referer]": "https://www.realtor.ca/",
            "headers[Origin]": "https://www.realtor.ca",
        }
        for attempt in range(3):
            try:
                resp = requests.post(SCRAPFLY_API, params=params, data=body,
                                     timeout=180)
                res = resp.json().get("result") or {}
                if (res.get("status_code") or 0) == 200:
                    return json.loads(res.get("content") or "{}")
            except (requests.RequestException, ValueError):
                pass
            if attempt < 2:
                time.sleep(4 * (attempt + 1))
        with self._lock:
            self._failures += 1
        return {}

    # -- accumulation dédupliquée (les boîtes se chevauchent) ------------------
    def _collect(self, data: dict, seen: dict[str, Listing]) -> None:
        for r in (data or {}).get("Results") or []:
            try:
                lst = self._listing(r)
            except Exception:
                continue
            if lst is not None and lst.external_id not in seen:
                seen[lst.external_id] = lst

    # -- un résultat de recherche -> Listing -----------------------------------
    def _listing(self, r: dict) -> Listing | None:
        prov = _PROVINCE_CODES.get(
            (r.get("ProvinceName") or "").strip().lower())
        if not prov:
            return None                       # Québec (ou inconnu) : hors champ
        prop = r.get("Property") or {}
        bld = r.get("Building") or {}
        btype = (bld.get("Type") or prop.get("Type") or "").strip()
        if _NON_RESIDENTIAL_RE.search(btype):
            return None
        rent = (prop.get("LeaseRent") or "").strip()
        if _NON_MONTHLY_RE.search(rent):
            return None                       # loyer non mensuel : hors sujet

        pid = str(r.get("Id") or "").strip()
        mls = (r.get("MlsNumber") or "").strip()
        if not pid:
            return None

        # adresse « unité - rue|Ville (Secteur), Province CodePostal »
        addr = prop.get("Address") or {}
        text = (addr.get("AddressText") or "").strip()
        street, _, tail = text.partition("|")
        city, sector = "", ""
        m = re.match(r"^([^(,]+?)\s*(?:\(([^)]+)\))?\s*,", tail)
        if m:
            city = m.group(1).strip()
            sector = (m.group(2) or "").strip()
        postal = (r.get("PostalCode") or "").strip()
        full_addr = ", ".join(x for x in (street.strip(), city) if x)
        if full_addr:
            full_addr += f", {prov} {postal}".rstrip()

        try:
            lat = float(addr.get("Latitude") or "")
            lng = float(addr.get("Longitude") or "")
        except (TypeError, ValueError):
            lat = lng = None

        # prix : valeur brute du flux (mensuelle)
        price = None
        try:
            v = float(str(prop.get("LeaseRentUnformattedValue") or "")
                      .replace(",", ""))
            if 200 <= v <= 100000:
                price = v
        except (TypeError, ValueError):
            pass
        price_label = rent.replace("/Monthly", "/mo") if rent else ""

        # chambres « 2 + 1 » (TRREB : + den) -> 2 ; « 0 » -> studio
        bedrooms = None
        mb = re.match(r"\s*(\d+)", str(bld.get("Bedrooms") or ""))
        if mb:
            bedrooms = float(mb.group(1))
        bathrooms = None
        try:
            bathrooms = float(bld.get("BathroomTotal") or "")
        except (TypeError, ValueError):
            pass
        if bedrooms is not None:
            unit_type = "Studio" if bedrooms == 0 else \
                normalize_unit_type(f"{int(bedrooms)} bedrooms")
        else:
            unit_type = normalize_unit_type(btype)

        # superficie : « 600 sqft » ou plage « 500-599 sqft » (borne basse)
        area = None
        for src in (bld.get("SizeInterior") or "",
                    " ".join((fm or {}).get("Area") or ""
                             for fm in bld.get("FloorAreaMeasurements")
                             or [])):
            ma = _AREA_RE.search(src)
            if ma:
                try:
                    v = float(ma.group(1).replace(",", ""))
                    if 80 <= v <= 20000:
                        area = v
                        break
                except ValueError:
                    pass

        amenities = [a.strip() for a in
                     (bld.get("Ammenities") or "").split(",") if a.strip()]

        # courtier + agence — la valeur ajoutée « annonce par courtier »
        details: dict = {"mls": mls} if mls else {}
        agents = []
        for ind in r.get("Individual") or []:
            nm = (ind.get("Name") or "").strip()
            if nm and nm not in agents:
                agents.append(nm)
            org = ind.get("Organization") or {}
            if org.get("Name") and "organization" not in details:
                details["organization"] = org["Name"].strip()
                for ph in org.get("Phones") or []:
                    if ph.get("PhoneNumber"):
                        details.setdefault("contact", {})["phone"] = \
                            f"{ph.get('AreaCode', '')}-{ph['PhoneNumber']}" \
                            .strip("-")
                        break
        if agents:
            details.setdefault("contact", {})["name"] = ", ".join(agents[:3])
        if prop.get("ParkingType"):
            details["parking"] = prop["ParkingType"]
        if prop.get("OwnershipType"):
            details["ownership"] = prop["OwnershipType"]

        images = [p.get("HighResPath") or p.get("MedResPath") or ""
                  for p in prop.get("Photo") or []]
        images = [u for u in images if u][: self.max_images]

        rel = r.get("RelativeDetailsURL") or r.get("RelativeURLEn") or ""
        title = street.strip() or text
        if unit_type and city:
            title = f"{title} — {city}"

        return Listing(
            source=self.source_id,
            external_id=pid,
            url=f"https://www.realtor.ca{rel}" if rel else
                "https://www.realtor.ca",
            title=title,
            address=full_addr,
            sector=sector,
            city=city,
            province=prov,
            unit_type=unit_type,
            bedrooms=bedrooms,
            bathrooms=bathrooms,
            price=price,
            price_label=price_label,
            area_sqft=area,
            description=(r.get("PublicRemarks") or "").strip()[:900],
            amenities=amenities[:25],
            details=details,
            images=images,
            lat=lat,
            lng=lng,
        )


def _quadrants(box: tuple) -> list[tuple]:
    lat_min, lat_max, lng_min, lng_max = box
    lat_mid = (lat_min + lat_max) / 2
    lng_mid = (lng_min + lng_max) / 2
    return [
        (lat_min, lat_mid, lng_min, lng_mid),
        (lat_min, lat_mid, lng_mid, lng_max),
        (lat_mid, lat_max, lng_min, lng_mid),
        (lat_mid, lat_max, lng_mid, lng_max),
    ]