spb/rent-ka

1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# scripts/probe_liftsystem_gaps.py : probe LiftSystem client_ids NEVER probed —5#   the gaps left by liftsystem_national.txt (enumeration was incomplete) plus6#   the range beyond its max id. Unlike probe_liftsystem.py there is no name/7#   website input: both are read from the feed itself (each property embeds8#   "client": {id, name, website}). Appends to the same resumable9#   data/liftsystem_probe.jsonl; feed build_lift_registry.py afterwards.10# Usage: python scripts/probe_liftsystem_gaps.py [id_max] (default 3200)11# -----------------------------------------------------------------------------12from __future__ import annotations1314import json15import sys16import time17from collections import Counter18from pathlib import Path1920import requests2122ROOT = Path(__file__).resolve().parents[1]23OUT = ROOT / "data" / "liftsystem_probe.jsonl"24LIFT = ROOT / "data" / "liftsystem_clients.json"25API = "https://api.theliftsystem.com/v2/search"26TOKEN = "sswpREkUtyeYjeoahA2i"27UA = {"User-Agent": "RentKaBot/1.0 (+https://www.rent-ka.com/bot; "28                    "contact@spboucher.ai)",29      "Accept": "application/json"}30NONRES = {"office", "retail", "warehouse", "industrial", "commercial", "land",31          "construction", "motel", "hotel", "parking", "storage"}323334def probe(cid: int, tries: int = 2) -> dict | None:35    """None = pas de client (404/vide) ; dict = record probe.jsonl."""36    for i in range(tries):37        try:38            r = requests.get(API, params={39                "client_id": str(cid), "auth_token": TOKEN,40                "show_all_properties": "true", "limit": "1000",41            }, headers=UA, timeout=25)42            if r.status_code == 404:43                return None44            if r.status_code != 200:45                time.sleep(1.5 * (i + 1))46                continue47            props = r.json()48            if not isinstance(props, list) or not props:49                return None50            client = (props[0].get("client") or {})51            provs: Counter = Counter()52            residential = 053            for p in props:54                pc = ((p.get("address") or {}).get("province_code")55                      or "").upper()56                if str(p.get("property_type") or "").strip().lower() in NONRES:57                    continue58                residential += 159                if pc:60                    provs[pc] += 161            return {"client_id": cid,62                    "name": (client.get("name") or f"LiftSystem {cid}").strip(),63                    "website": (client.get("website") or "").strip(),64                    "n": len(props), "res": residential,65                    "provinces": dict(provs)}66        except Exception:67            time.sleep(1.5 * (i + 1))68    return {"client_id": cid, "error": True}697071def main() -> None:72    id_max = int(sys.argv[1]) if len(sys.argv) > 1 else 320073    done: set[int] = set()74    if OUT.exists():75        for line in OUT.read_text().splitlines():76            try:77                done.add(json.loads(line)["client_id"])78            except Exception:79                pass80    try:81        reg = json.loads(LIFT.read_text("utf-8"))82        done |= {int(c["client_id"]) for c in reg.get("clients") or []83                 if c.get("client_id")}84    except (OSError, ValueError):85        pass86    todo = [i for i in range(1, id_max + 1) if i not in done]87    print(f"[gaps] {len(todo)} ids to probe ({len(done)} already covered)",88          flush=True)89    hits = 090    with OUT.open("a") as fh:91        for i, cid in enumerate(todo, 1):92            rec = probe(cid)93            if rec is not None:94                rec["ts"] = time.strftime("%Y-%m-%dT%H:%M:%S")95                fh.write(json.dumps(rec, ensure_ascii=False) + "\n")96                fh.flush()97                if not rec.get("error"):98                    hits += 199            if i % 100 == 0:100                print(f"[gaps] {i}/{len(todo)} — {hits} live clients",101                      flush=True)102            time.sleep(0.3)103    print(f"[gaps] done — {hits} live clients found → {OUT}", flush=True)104105106if __name__ == "__main__":107    main()108