# -----------------------------------------------------------------------------
# Rent-Ka — Rental listings aggregator (Canada, outside Québec)
# Author: Simon-Pierre Boucher — contact@spboucher.ai
# scripts/probe_liftsystem.py : probe every LiftSystem client enumerated in
#   data/liftsystem_national.txt against api.theliftsystem.com/v2/search and
#   record, per client, how many properties it exposes and in which provinces.
#   Resumable: results append to data/liftsystem_probe.jsonl (one JSON/line);
#   already-probed ids are skipped on re-run. ~3 req/s.
# Usage: python scripts/probe_liftsystem.py [limit]
# -----------------------------------------------------------------------------
from __future__ import annotations

import json
import re
import sys
import time
from collections import Counter
from pathlib import Path

import requests

ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "data" / "liftsystem_national.txt"
OUT = ROOT / "data" / "liftsystem_probe.jsonl"
API = "https://api.theliftsystem.com/v2/search"
TOKEN = "sswpREkUtyeYjeoahA2i"
UA = {"User-Agent": "RentKaBot/1.0 (+https://www.rent-ka.com/bot; contact@spboucher.ai)",
      "Accept": "application/json"}

LINE_RE = re.compile(r'^(\d+)\|"client":\{"id":\d+,"name":"(.*?)","website":"([^"]*)')


def clients() -> list[dict]:
    out = []
    for line in SRC.read_text("utf-8").splitlines():
        m = LINE_RE.match(line.strip())
        if m:
            out.append({"client_id": int(m.group(1)),
                        "name": m.group(2), "website": m.group(3)})
    return out


def probe(cid: int, tries: int = 3) -> dict:
    for i in range(tries):
        try:
            r = requests.get(API, params={
                "client_id": str(cid), "auth_token": TOKEN,
                "show_all_properties": "true", "limit": "1000",
            }, headers=UA, timeout=25)
            if r.status_code != 200:
                time.sleep(1.5 * (i + 1))
                continue
            props = r.json()
            if not isinstance(props, list):
                return {"n": 0, "provinces": {}}
            provs = Counter()
            residential = 0
            nonres = {"office", "retail", "warehouse", "industrial",
                      "commercial", "land", "construction", "motel", "hotel",
                      "parking", "storage"}
            for p in props:
                pc = ((p.get("address") or {}).get("province_code") or "").upper()
                if str(p.get("property_type") or "").strip().lower() in nonres:
                    continue
                residential += 1
                if pc:
                    provs[pc] += 1
            return {"n": len(props), "res": residential, "provinces": dict(provs)}
        except Exception:
            time.sleep(1.5 * (i + 1))
    return {"error": True}


def main() -> None:
    limit = int(sys.argv[1]) if len(sys.argv) > 1 else 10**9
    done = set()
    if OUT.exists():
        for line in OUT.read_text().splitlines():
            try:
                done.add(json.loads(line)["client_id"])
            except Exception:
                pass
    todo = [c for c in clients() if c["client_id"] not in done][:limit]
    print(f"[probe] {len(todo)} clients to probe ({len(done)} already done)")
    with OUT.open("a") as fh:
        for i, c in enumerate(todo, 1):
            res = probe(c["client_id"])
            rec = {**c, **res, "ts": time.strftime("%Y-%m-%dT%H:%M:%S")}
            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
            fh.flush()
            if i % 50 == 0:
                print(f"[probe] {i}/{len(todo)}")
            time.sleep(0.33)
    print("[probe] done →", OUT)


if __name__ == "__main__":
    main()