# -----------------------------------------------------------------------------
# Rent-Ka — Rental listings aggregator (Canada, outside Québec)
# Author: Simon-Pierre Boucher — contact@spboucher.ai
# connectors/realstar.py : Realstar (realstar.ca)
#   Cloudflare-protected (403 for robots) client-rendered site (RentCafe/
#   Yardi engine): everything goes through Scrapfly rendering with a wait.
#   1) /searchlisting?province=<name> -> property cards (name, address,
#      beds/baths/sqft, price range, phone, thumbnail) — one render per
#      province, looped over Realstar's markets outside Québec (Ontario,
#      Alberta, British Columbia, Nova Scotia, Newfoundland and Labrador);
#   2) each property page -> photo gallery, description, highlights;
#   3) /floorplans page -> structured plans (type, beds, sqft, price,
#      number of available units) — real availability and prices.
#   One listing per property (stable uids). Detail pages go through
#   self.detail(...) (DB cache): the render is only re-done when the list
#   card changed.
# -----------------------------------------------------------------------------
from __future__ import annotations

import hashlib
import os
import re

from bs4 import BeautifulSoup

from ..schema import Listing, parse_price
from .base import BaseConnector

# Realstar markets outside Québec: (search province name, URL path code,
# province code). One search render each.
_PROVINCES = [
    ("Ontario", "on", "ON"),
    ("Alberta", "ab", "AB"),
    ("British Columbia", "bc", "BC"),
    ("Nova Scotia", "ns", "NS"),
    ("Newfoundland and Labrador", "nl", "NL"),
]
SEARCH_URL = "https://www.realstar.ca/searchlisting?province={name}"

# City slug in /apartments/<prov>/<city>/<slug> -> (display city, sector).
# Former Toronto boroughs fold into Toronto; unknown slugs pass through as
# Title Case (whole-province coverage).
_CITY_NORM = {
    "north-york": ("Toronto", "North York"),
    "etobicoke": ("Toronto", "Etobicoke"),
    "scarborough": ("Toronto", "Scarborough"),
    "east-york": ("Toronto", "East York"),
}

_BED_TYPES = {"0": "Studio", "1": "1 bedroom", "2": "2 bedrooms",
              "3": "3 bedrooms", "4": "4 bedrooms"}
_SKIP_IMG = re.compile(r"logo|icon|favicon|placeholder", re.I)
_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?")


class _BudgetReached(Exception):
    """Plafond de rendus atteint pour cette synchronisation."""


class RealstarConnector(BaseConnector):
    source_id = "realstar"
    request_delay = 1.0
    max_properties = 60      # global safety cap (2 renders per NEW property)
    max_images = 25
    max_renders = 60         # render cap per sync (cache hits are free)

    # -- JS render with wait (SPA + Cloudflare) — Scrapfly ---------------------
    # (migrated from Firecrawl 2026-08-27; the short retry on 5xx/empty lives
    #  in BaseConnector.get_rendered, which the fixtures monkeypatch)
    def _rendered(self, url: str, wait_ms: int = 9000) -> str:
        return self.get_rendered(url, wait_ms)

    def fetch(self) -> list[Listing]:
        self._renders = 0
        listings: list[Listing] = []
        seen: set[str] = set()
        count = 0
        for prov_name, code, prov in _PROVINCES:
            try:
                html = self._rendered(
                    SEARCH_URL.format(name=prov_name.replace(" ", "%20")),
                    12000)
                soup = BeautifulSoup(html, "html.parser")
                # [class*=…]: the Scrapfly render captures the DOM before the
                # RentCafe JS reveals the cards (property-box-hidden) — the
                # hidden cards are complete
                cards = soup.select('li[class*="property-box"]')
                if not cards:   # incomplete render: one more chance
                    html = self._rendered(
                        SEARCH_URL.format(name=prov_name.replace(" ", "%20")),
                        15000)
                    soup = BeautifulSoup(html, "html.parser")
                    cards = soup.select('li[class*="property-box"]')
            except Exception:
                continue
            for card in cards:
                try:
                    a = card.select_one(f"a[href*='/apartments/{code}/']")
                    if not a:
                        continue    # another province's card
                    url = (a.get("href") or "").split("?")[0]
                    url = url.replace("http://", "https://")
                    m = re.search(
                        rf"/apartments/{code}/([a-z0-9\-.]+)/([a-z0-9\-]+)",
                        url)
                    if not m or url in seen:
                        continue
                    seen.add(url)
                    city_slug, slug = m.group(1), m.group(2)
                    if count >= self.max_properties:
                        break
                    count += 1
                    listings.append(self._property_listing(
                        card, url, city_slug, slug, province=prov))
                except Exception:
                    continue
        return listings

    def _property_listing(self, card, url: str, city_slug: str,
                          slug: str, province: str = "ON") -> Listing:
        city, sector = _CITY_NORM.get(
            city_slug, (city_slug.replace("-", " ").title(), ""))
        name = ""
        fav = card.select_one("[data-property]")
        if fav:
            name = (fav.get("data-property") or "").strip()
        if not name:
            h = card.select_one(".property-name a")
            if h:
                name = h.get_text(" ", strip=True)
        name = re.sub(r"\s*opens in a new tab\s*", "", name).strip()
        name = name or slug.replace("-", " ").title()

        addr_el = card.select_one(".card-prop-address")
        address = addr_el.get_text(" ", strip=True) if addr_el else ""
        if address and not re.search(rf",?\s+{province}\b", address):
            address = f"{address}, {province}"

        meta = card.select_one(".card-bed-bath-rent")
        beds = baths = sqft = ""
        if meta:
            items = [li.get_text(" ", strip=True)
                     for li in meta.select("li")]
            for it in items:
                if "Bed" in it:
                    beds = it
                elif "Bath" in it:
                    baths = it
                elif "Sq" in it:
                    sqft = it
        unit_type = ""
        bm = re.match(r"^(\d)(?:\s|-)?.*Bed", beds or "")
        if bm and "-" not in beds.split("Bed")[0]:
            unit_type = _BED_TYPES.get(bm.group(1), "")

        # Fourchette de prix « $1,645.00 - $2,630.00 »
        card_text = card.get_text(" ", strip=True)
        price = None
        price_label = ""
        pm = re.search(r"\$[\d,]+(?:\.\d{2})?(?:(?:\s*(?:-|to))+\s*"
                       r"\$[\d,]+(?:\.\d{2})?)?", card_text)
        if pm:
            price_label = re.sub(r"\s*to\s*-\s*", " - ", pm.group(0))
            first = price_label.split("-")[0].replace("$", "").replace(
                ",", "").replace("to", "").strip()
            try:
                price = float(first)
            except ValueError:
                price = parse_price(price_label)
            if "-" in price_label:
                price_label = "From " + price_label

        # Téléphone du bureau de location (lien tel: structuré de la carte)
        phone = ""
        tel = card.select_one("a[href^='tel:']")
        if tel:
            tm = re.search(r"\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})",
                           tel.get("href", ""))
            if tm:
                phone = f"{tm.group(1)}-{tm.group(2)}-{tm.group(3)}"

        # Vignette de la carte
        images: list[str] = []
        img = card.select_one("img[src*='rentcafe']")
        if img and img.get("src"):
            images.append(img["src"])

        # Pages détail (fiche + plans) via cache BD : rendu seulement si la
        # carte liste a changé (prix/dispo inclus dans le hash).
        key = hashlib.sha1(
            f"{name}|{address}|{beds}|{baths}|{sqft}|{price_label}"
            .encode("utf-8")).hexdigest()
        # uid: province-prefixed to avoid cross-province slug collisions.
        # Ontario keeps the historical "on-" prefix (uids in the seeded DB).
        ext = f"{province.lower()}-{slug}"
        try:
            payload = self.detail(ext, key, lambda: self._fetch_detail(url))
        except _BudgetReached:
            payload = {}
        except Exception:
            payload = {}

        desc = payload.get("description", "")
        amenities = list(payload.get("amenities") or [])
        for im in (payload.get("images") or []):
            if im not in images:
                images.append(im)

        # Plans structurés -> disponibilité, prix « à partir de », superficie
        availability = ""
        area_sqft = None
        plans = payload.get("floorplans") or []
        avail_plans = [p for p in plans if p.get("available", 0) > 0]
        if plans:
            total = sum(p.get("available", 0) for p in avail_plans)
            if total > 0:
                availability = (f"{total} unit(s) available — "
                                + ", ".join(p["name"] for p in avail_plans[:6]))
            prices = [p["price"] for p in avail_plans
                      if p.get("price") and 100 <= p["price"] <= 20000]
            if prices:
                price = min(prices)
                price_label = (f"From ${price:,.0f}/month"
                               if len(avail_plans) > 1 or len(prices) > 1
                               else f"${price:,.0f}/month")
            if len(avail_plans) == 1 and avail_plans[0].get("sqft"):
                # une seule unité type disponible : sa superficie est fiable
                area_sqft = avail_plans[0]["sqft"]
                if avail_plans[0].get("unit_type"):
                    unit_type = avail_plans[0]["unit_type"]

        # Summary of available plans in the description (faithful text)
        plan_bits = []
        for p in avail_plans[:8]:
            seg = p["name"]
            if p.get("sqft"):
                seg += f" ({p['sqft']:.0f} sq ft)"
            if p.get("price"):
                seg += f": ${p['price']:,.0f}/month"
            plan_bits.append(seg)

        details: dict = {}
        if phone:
            details["contact"] = {"phone": phone}

        bits = [b for b in [beds, baths, sqft] if b]
        desc_parts = ([desc] if desc else []) + bits
        if plan_bits:
            desc_parts.append("Available: " + "; ".join(plan_bits))
        return Listing(
            source=self.source_id,
            external_id=ext,
            url=url,
            title=name,
            address=address,
            sector=sector,
            city=city,
            province=province,
            unit_type=unit_type,
            price=price,
            price_label=price_label,
            availability=availability,
            area_sqft=area_sqft,
            description=" — ".join(desc_parts)[:900],
            amenities=amenities,
            details=details,
            images=images[: self.max_images],
        )

    # -- pages détail (fiche propriété + plans) --------------------------------
    def _fetch_detail(self, url: str) -> dict:
        """2 rendus Scrapfly : fiche (photos, description, points forts) et
        /floorplans (plans structurés). Appelé seulement hors cache."""
        if self._renders + 2 > self.max_renders:
            raise _BudgetReached()
        self._renders += 2

        payload: dict = {"description": "", "amenities": [], "images": [],
                         "floorplans": []}
        try:
            ph = self._rendered(url, 8000)
            psoup = BeautifulSoup(ph, "html.parser")
            for im in psoup.select("img[src*='resource.rentcafe.com']"):
                src = im.get("src", "")
                if src and not _SKIP_IMG.search(src) \
                        and src not in payload["images"]:
                    payload["images"].append(src)
            # description : premiers paragraphes substantiels
            paras = [p.get_text(" ", strip=True)
                     for p in psoup.find_all("p")]
            paras = [p for p in paras if len(p) > 80]
            if paras:
                payload["description"] = " ".join(paras[:2])[:600]
            # points forts de la propriété (courtes mentions après le titre)
            text = psoup.get_text("\n", strip=True)
            hm = re.search(r"Points forts de la propri[ée]t[ée]\n(.*?)\n"
                           r"(?:Photos|Emplacement|Votre)", text, re.S)
            if not hm:  # gabarit anglais (fiches Ontario)
                hm = re.search(r"Property Highlights\n(.*?)\n"
                               r"(?:Photos|Location|Your)", text, re.S)
            if hm:
                amenities = []
                for t in hm.group(1).split("\n"):
                    t = t.strip()
                    if 2 < len(t) < 50 and t not in amenities:
                        amenities.append(t)
                payload["amenities"] = amenities[:15]
        except Exception:
            pass

        try:
            fh = self._rendered(url.rstrip("/") + "/floorplans", 10000)
            payload["floorplans"] = self._parse_floorplans(fh)
        except Exception:
            pass
        return payload

    @staticmethod
    def _parse_floorplans(html: str) -> list[dict]:
        """Cartes de plans RentCafe : nom (« 4 ½ D »), chambres, pi², prix,
        nombre d'unités disponibles (structuré : .fp-availability)."""
        soup = BeautifulSoup(html, "html.parser")
        plans: list[dict] = []
        for cont in soup.select("div[id^='fp-container-']"):
            try:
                name_el = cont.select_one("span[data-selenium-id$='Name']")
                name = name_el.get_text(" ", strip=True) if name_el else ""
                if not name:
                    continue
                avail = 0
                av_el = cont.select_one(".fp-availability")
                if av_el:
                    am = re.search(r"(\d+)", av_el.get_text(" ", strip=True))
                    if am:
                        avail = int(am.group(1))
                sqft = None
                sq_el = cont.select_one("span[data-selenium-id$='SqFt']")
                if sq_el:
                    # « Pi. Ca. » (gabarit FR) ou « Sq. Ft. » (gabarit EN/ON)
                    sm = re.search(r"([\d,]{2,})\s*(?:Pi|Sq)",
                                   sq_el.get_text(" ", strip=True), re.I)
                    if sm:
                        v = float(sm.group(1).replace(",", ""))
                        if 80 <= v <= 20000:
                            sqft = v
                price = None
                pm = _PRICE_RE.search(cont.get_text(" ", strip=True))
                if pm:
                    v = float(pm.group(0).replace("$", "").replace(",", ""))
                    if 100 <= v <= 20000:
                        price = v
                unit_type = ""
                um = re.match(r"^\s*(\d)\s*½", name)
                if um:
                    # French template names («4 ½ D»): n½ -> n-2 bedrooms
                    n = max(int(um.group(1)) - 2, 0)
                    unit_type = _BED_TYPES.get(str(min(n, 4)), "")
                elif re.match(r"(?i)^\s*(?:studio|bachelor)", name):
                    unit_type = "Studio"
                else:
                    # English templates: «2 Bed 1 Bath A», «One Bedroom»…
                    wm = re.match(r"(?i)^\s*(\d|one|two|three|four)\s*bed",
                                  name)
                    if wm:
                        n = {"one": 1, "two": 2, "three": 3, "four": 4}.get(
                            wm.group(1).lower()) or int(wm.group(1))
                        unit_type = _BED_TYPES.get(str(min(n, 4)), "")
                plans.append({"name": name, "available": avail,
                              "sqft": sqft, "price": price,
                              "unit_type": unit_type})
            except Exception:
                continue
        return plans