# ==============================================================================
# Author: Simon-Pierre Boucher <contact@spboucher.ai>
# File:   src/main.py (ka-fb-marketplace)
# Desc:   Facebook Marketplace — logements à louer (propertyrentals), Québec.
#         Scraping public DÉCONNECTÉ, AUCUN rendu JS : les données vivent dans
#         les <script type="application/json"> du HTML brut.
#         - Recherche : objets marketplace_listing_title (~25/URL, on multiplie
#           les URLs ville × tranche de prix pour élargir).
#         - Fiche /marketplace/item/<id> : le target GraphQL préchargé
#           (marketplace_product_details_page) expose GPS, ville
#           (reverse_geocode_detailed), description, statut is_live/is_sold ;
#           la GALERIE complète vient du target MediaViewer préchargé
#           (listing_photos du même id) — validé live, aucun rendu requis.
#         Sortie dataset : {kind:"listing"} (recherche) et {kind:"detail"}.
#         ⚠️ Données personnelles (Loi 25) : ne jamais republier nom/téléphone
#         du vendeur ; le champ seller n'est PAS extrait.
# ==============================================================================
from __future__ import annotations

import asyncio
import html as _html
import json
import re

from apify import Actor

from .net import Fetcher

BASE = "https://www.facebook.com/marketplace"

# Couverture Québec par défaut (mode autonome, sans searchUrls) :
# slug vanity OU id numérique Marketplace (validés live 2026-08-22 — les slugs
# inventés retombent sur des villes aléatoires, seuls ceux-ci sont sûrs).
# Chaque recherche couvre un rayon d'environ 65 km.
BIG_CITIES = {  # gros parcs : tranches de prix fines (flux ~25 annonces/URL)
    "montreal": "Montréal", "laval": "Laval", "longueuil": "Longueuil",
    "quebec": "Québec", "gatineau": "Gatineau", "sherbrooke": "Sherbrooke",
    "trois-rivieres": "Trois-Rivières", "levis": "Lévis",
    "saguenay": "Saguenay", "drummondville": "Drummondville",
}
REGIONAL_CITIES = {  # couverture régions : ids numériques (Serper + validation)
    "105790762794728": "Rimouski", "110925022261358": "Rivière-du-Loup",
    "106528256049537": "Matane", "109502305742889": "Gaspé",
    "100246746683723": "Sept-Îles", "105615529471535": "Baie-Comeau",
    "106016152771864": "Rouyn-Noranda", "110594102301488": "Val-d'Or",
    "105598479473872": "Amos", "109699385723610": "Saint-Georges",
    "110304502331480": "Thetford Mines", "111938355489908": "Granby",
    "108126699220609": "Victoriaville", "109753249050671": "Joliette",
    "109429165750459": "Saint-Hyacinthe",
    "108645709159963": "Saint-Jean-sur-Richelieu",
    "112946182053396": "Saint-Jérôme", "113011352052118": "Salaberry-de-Valleyfield",
    "107992819229760": "Magog", "109514615734434": "Alma",
    "109664672393669": "Mont-Laurier", "108557812502323": "La Tuque",
    "108296932538245": "Dolbeau-Mistassini",
}
PRICE_BANDS = [(0, 800), (800, 1100), (1100, 1400), (1400, 1700),
               (1700, 2100), (2100, 2800), (2800, 6000)]
REGIONAL_BANDS = [(0, 1200), (1200, 6000)]   # parcs plus petits : 2 tranches

_SCRIPT_RE = re.compile(
    r'<script type="application/json"[^>]*>(.*?)</script>', re.S)


def default_search_urls() -> list[str]:
    urls = []
    for slug in BIG_CITIES:
        for lo, hi in PRICE_BANDS:
            urls.append(f"{BASE}/{slug}/propertyrentals"
                        f"?minPrice={lo}&maxPrice={hi}"
                        f"&sortBy=creation_time_descend")
    for cid in REGIONAL_CITIES:
        for lo, hi in REGIONAL_BANDS:
            urls.append(f"{BASE}/{cid}/propertyrentals"
                        f"?minPrice={lo}&maxPrice={hi}"
                        f"&sortBy=creation_time_descend")
    return urls


def _iter_json_scripts(html: str):
    for m in _SCRIPT_RE.finditer(html):
        try:
            yield json.loads(m.group(1))
        except ValueError:
            continue


def walk_listings(html: str) -> dict:
    """Objets d'annonce (marketplace_listing_title) des <script> JSON."""
    out: dict = {}

    def walk(o):
        if isinstance(o, dict):
            if "marketplace_listing_title" in o and o.get("id"):
                out.setdefault(str(o["id"]), o)
            for v in o.values():
                walk(v)
        elif isinstance(o, list):
            for v in o:
                walk(v)

    for data in _iter_json_scripts(html):
        walk(data)
    return out


def price_of(obj: dict) -> float | None:
    """Montant du loyer depuis listing_price (offset en cents géré)."""
    pr = obj.get("listing_price") or {}
    for k in ("amount", "amount_with_offset_in_currency"):
        v = pr.get(k)
        if v is None:
            continue
        try:
            val = float(v)
        except (TypeError, ValueError):
            continue
        if k == "amount_with_offset_in_currency" and val > 100000:
            val /= 100.0
        if val > 0:
            return round(val, 0)
    return None


def _pdp_target(html: str) -> dict | None:
    """Nœud cible le plus RICHE (description/GPS/statut) parmi les targets
    marketplace_product_details_page préchargés (stubs inclus)."""
    found: list[dict] = []

    def walk(o):
        if isinstance(o, dict):
            pdp = o.get("marketplace_product_details_page")
            if isinstance(pdp, dict) and isinstance(pdp.get("target"), dict):
                found.append(pdp["target"])
            for v in o.values():
                walk(v)
        elif isinstance(o, list):
            for v in o:
                walk(v)

    for data in _iter_json_scripts(html):
        walk(data)

    def score(t: dict) -> tuple:
        desc = (t.get("redacted_description") or {}).get("text") or ""
        loc = t.get("location") or {}
        return (1 if desc else 0) + (1 if loc.get("latitude") else 0), len(t)
    return max(found, key=score) if found else None


def _gallery(html: str, lid: str) -> list[str]:
    """Galerie complète : plus grand tableau listing_photos porté par un objet
    du MÊME id (le target MediaViewer préchargé la contient hors connexion)."""
    best: list[str] = []

    def walk(o):
        nonlocal best
        if isinstance(o, dict):
            lp = o.get("listing_photos")
            if (isinstance(lp, list) and lp
                    and str(o.get("id") or "") == lid):
                uris = []
                for p in lp:
                    uri = ((p.get("image") or {}).get("uri")
                           if isinstance(p, dict) else "") or ""
                    if uri and uri not in uris:
                        uris.append(uri)
                if len(uris) > len(best):
                    best = uris
            for v in o.values():
                walk(v)
        elif isinstance(o, list):
            for v in o:
                walk(v)

    for data in _iter_json_scripts(html):
        walk(data)
    return best


def _clean_desc(text: str) -> str:
    t = text.replace("\r", "\n")
    t = re.sub(r"[ \t ]+", " ", t)
    t = re.sub(r" ?\n ?", "\n", t)
    t = re.sub(r"\n{3,}", "\n\n", t)
    return t.strip()


def parse_detail(html: str, lid: str) -> dict:
    """Champs riches d'une fiche /marketplace/item/<id> hors connexion.

    {} si la page n'a pas pu être rendue (échec réseau : l'appelant ne doit
    PAS mettre en cache), {"nopdp": True} si rendue mais sans objet cible
    (fiche supprimée / mur de login : cacher pour ne pas boucler)."""
    if not html:
        return {}
    o = _pdp_target(html)
    if not o:
        return {"nopdp": True}
    out: dict = {}
    if o.get("is_sold") or o.get("is_pending") or o.get("is_live") is False:
        out["gone"] = True              # loué / en attente / retiré
    loc = o.get("location") or {}
    if loc.get("latitude") and loc.get("longitude"):
        out["lat"] = loc["latitude"]
        out["lng"] = loc["longitude"]
    geo = loc.get("reverse_geocode_detailed") or {}
    if geo.get("city"):
        out["city"] = geo["city"]
    if geo.get("state"):
        out["state"] = geo["state"]
    lt = (o.get("location_text") or {}).get("text") or ""
    if lt:                              # ex. « Québec, QC » / « Norfolk, ON »
        out["location_text"] = lt
        parts = [p.strip() for p in lt.split(",") if p.strip()]
        if parts and not out.get("city"):
            out["city"] = parts[0]
        if len(parts) > 1 and not out.get("state"):
            out["state"] = parts[1]
    desc = (o.get("redacted_description") or {}).get("text") or ""
    if desc:
        out["description"] = _clean_desc(desc)[:6000]
    addr = ((o.get("home_address") or {}).get("street") or "").strip()
    unit_fields: list[str] = []
    for sec in (o.get("pdp_display_sections") or []):
        stype = sec.get("section_type") or ""
        for f in (sec.get("pdp_fields") or []):
            lbl = (f.get("display_label") or "").strip()
            if not lbl:
                continue
            if stype == "UNIT_INCLUDES":
                unit_fields.append(lbl)
            elif stype == "UNIT_SUBTITLE":
                if (f.get("icon_name") or "") == "pin" and not addr:
                    addr = lbl.split(",")[0].strip()
                elif (f.get("icon_name") or "") == "clock":
                    out["listed_text"] = lbl
    if addr:
        out["address"] = addr
    if unit_fields:
        out["unit_fields"] = unit_fields
    for src_k, dst_k in (("walk_score_info", "walk_score"),
                         ("transit_score_info", "transit_score"),
                         ("bike_score_info", "bike_score")):
        sc = (o.get(src_k) or {}).get("score")
        if sc is not None:
            out[dst_k] = sc
    if o.get("virtual_tour_url"):
        out["virtual_tour_url"] = o["virtual_tour_url"]
    photos = _gallery(html, lid)
    if photos:
        out["images"] = photos
    return out


async def main() -> None:
    async with Actor:
        inp = await Actor.get_input() or {}
        search_urls = [u for u in (inp.get("searchUrls") or []) if u]
        if not search_urls:
            search_urls = default_search_urls()
        get_details = inp.get("getDetails", True)
        max_details = int(inp.get("maxDetails") or 200)
        skip_ids = {str(i) for i in (inp.get("skipDetailIds") or [])}
        extra_ids = [str(i) for i in (inp.get("extraDetailIds") or [])]
        template = (inp.get("proxyUrlTemplate") or "").strip() or None
        proxy = None
        if not template:
            proxy = await Actor.create_proxy_configuration(
                actor_proxy_input=inp.get("proxyConfiguration"))
        fetcher = Fetcher(proxy, delay=float(inp.get("requestDelay") or 1),
                          proxy_template=template)
        sem = asyncio.Semaphore(int(inp.get("concurrency") or 8))

        # -- phase recherche ---------------------------------------------------
        found: dict[str, tuple[dict, str]] = {}   # id -> (objet, source_url)
        fails = 0

        async def search(url: str) -> None:
            nonlocal fails
            async with sem:
                try:
                    resp = await fetcher.get(url, session_id=f"q{hash(url)}")
                except Exception as exc:
                    fails += 1
                    Actor.log.warning(f"recherche KO {url} : {exc}")
                    return
                if resp.status_code != 200:
                    fails += 1
                    return
                for lid, obj in walk_listings(resp.text).items():
                    found.setdefault(lid, (obj, url))

        await asyncio.gather(*[search(u) for u in search_urls])
        Actor.log.info(f"recherche : {len(found)} annonces uniques "
                       f"({len(search_urls)} URLs, {fails} échecs)")

        for lid, (obj, src) in found.items():
            photo = ((obj.get("primary_listing_photo") or {})
                     .get("image") or {}).get("uri") or ""
            loc = obj.get("location") or {}
            geo = loc.get("reverse_geocode") or {}
            await Actor.push_data({
                "kind": "listing",
                "id": lid,
                "title": (obj.get("marketplace_listing_title") or "")[:300],
                "price": price_of(obj),
                "city": (geo.get("city")
                         or (geo.get("city_page") or {}).get("display_name")
                         or ""),
                "state": geo.get("state") or "",
                "primary_photo": photo,
                "is_sold": bool(obj.get("is_sold")),
                "is_pending": bool(obj.get("is_pending")),
                "source_url": src,
                "url": f"{BASE}/item/{lid}/",
            })

        # -- phase détail ------------------------------------------------------
        if not get_details or max_details <= 0:
            return
        todo: list[str] = []
        for lid, (obj, _src) in found.items():
            if lid in skip_ids or obj.get("is_sold") or obj.get("is_pending"):
                continue
            if price_of(obj) is None:
                continue               # sans prix : jamais publiable, épargner
            # le flux public glisse des annonces d'autres provinces (suggestions)
            geo = (obj.get("location") or {}).get("reverse_geocode") or {}
            st = (geo.get("state") or "").strip().lower()
            if st and st not in ("qc", "quebec", "québec"):
                continue
            todo.append(lid)
        for lid in extra_ids:          # rattrapage : après les nouveautés
            if lid not in todo and lid not in skip_ids:
                todo.append(lid)
        todo = todo[:max_details]

        async def detail(lid: str) -> None:
            async with sem:
                try:
                    resp = await fetcher.get(f"{BASE}/item/{lid}/",
                                             session_id=f"d{lid}")
                    payload = parse_detail(
                        resp.text if resp.status_code == 200 else "", lid)
                except Exception as exc:
                    Actor.log.warning(f"détail KO {lid} : {exc}")
                    payload = {}
                await Actor.push_data({"kind": "detail", "id": lid,
                                       "ok": bool(payload), **payload})

        await asyncio.gather(*[detail(lid) for lid in todo])
        Actor.log.info(f"détail : {len(todo)} fiches visitées")