# -----------------------------------------------------------------------------
# Rent-Ka — Rental listings aggregator (Canada, outside Québec)
# Author: Simon-Pierre Boucher — contact@spboucher.ai
# scripts/probe_liftsystem_gaps.py : probe LiftSystem client_ids NEVER probed —
#   the gaps left by liftsystem_national.txt (enumeration was incomplete) plus
#   the range beyond its max id. Unlike probe_liftsystem.py there is no name/
#   website input: both are read from the feed itself (each property embeds
#   "client": {id, name, website}). Appends to the same resumable
#   data/liftsystem_probe.jsonl; feed build_lift_registry.py afterwards.
# Usage: python scripts/probe_liftsystem_gaps.py [id_max] (default 3200)
# -----------------------------------------------------------------------------
from __future__ import annotations

import json
import sys
import time
from collections import Counter
from pathlib import Path

import requests

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "data" / "liftsystem_probe.jsonl"
LIFT = ROOT / "data" / "liftsystem_clients.json"
API = "https://api.theliftsystem.com/v2/search"
TOKEN = "sswpREkUtyeYjeoahA2i"
UA = {"User-Agent": "RentKaBot/1.0 (+https://www.rent-ka.com/bot; "
                    "contact@spboucher.ai)",
      "Accept": "application/json"}
NONRES = {"office", "retail", "warehouse", "industrial", "commercial", "land",
          "construction", "motel", "hotel", "parking", "storage"}


def probe(cid: int, tries: int = 2) -> dict | None:
    """None = pas de client (404/vide) ; dict = record probe.jsonl."""
    for i in range(tries):
        try:
            r = requests.get(API, params={
                "client_id": str(cid), "auth_token": TOKEN,
                "show_all_properties": "true", "limit": "1000",
            }, headers=UA, timeout=25)
            if r.status_code == 404:
                return None
            if r.status_code != 200:
                time.sleep(1.5 * (i + 1))
                continue
            props = r.json()
            if not isinstance(props, list) or not props:
                return None
            client = (props[0].get("client") or {})
            provs: Counter = Counter()
            residential = 0
            for p in props:
                pc = ((p.get("address") or {}).get("province_code")
                      or "").upper()
                if str(p.get("property_type") or "").strip().lower() in NONRES:
                    continue
                residential += 1
                if pc:
                    provs[pc] += 1
            return {"client_id": cid,
                    "name": (client.get("name") or f"LiftSystem {cid}").strip(),
                    "website": (client.get("website") or "").strip(),
                    "n": len(props), "res": residential,
                    "provinces": dict(provs)}
        except Exception:
            time.sleep(1.5 * (i + 1))
    return {"client_id": cid, "error": True}


def main() -> None:
    id_max = int(sys.argv[1]) if len(sys.argv) > 1 else 3200
    done: set[int] = set()
    if OUT.exists():
        for line in OUT.read_text().splitlines():
            try:
                done.add(json.loads(line)["client_id"])
            except Exception:
                pass
    try:
        reg = json.loads(LIFT.read_text("utf-8"))
        done |= {int(c["client_id"]) for c in reg.get("clients") or []
                 if c.get("client_id")}
    except (OSError, ValueError):
        pass
    todo = [i for i in range(1, id_max + 1) if i not in done]
    print(f"[gaps] {len(todo)} ids to probe ({len(done)} already covered)",
          flush=True)
    hits = 0
    with OUT.open("a") as fh:
        for i, cid in enumerate(todo, 1):
            rec = probe(cid)
            if rec is not None:
                rec["ts"] = time.strftime("%Y-%m-%dT%H:%M:%S")
                fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
                fh.flush()
                if not rec.get("error"):
                    hits += 1
            if i % 100 == 0:
                print(f"[gaps] {i}/{len(todo)} — {hits} live clients",
                      flush=True)
            time.sleep(0.3)
    print(f"[gaps] done — {hits} live clients found → {OUT}", flush=True)


if __name__ == "__main__":
    main()