spb/rent-ka

1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# scripts/probe_liftsystem.py : probe every LiftSystem client enumerated in5#   data/liftsystem_national.txt against api.theliftsystem.com/v2/search and6#   record, per client, how many properties it exposes and in which provinces.7#   Resumable: results append to data/liftsystem_probe.jsonl (one JSON/line);8#   already-probed ids are skipped on re-run. ~3 req/s.9# Usage: python scripts/probe_liftsystem.py [limit]10# -----------------------------------------------------------------------------11from __future__ import annotations1213import json14import re15import sys16import time17from collections import Counter18from pathlib import Path1920import requests2122ROOT = Path(__file__).resolve().parents[1]23SRC = ROOT / "data" / "liftsystem_national.txt"24OUT = ROOT / "data" / "liftsystem_probe.jsonl"25API = "https://api.theliftsystem.com/v2/search"26TOKEN = "sswpREkUtyeYjeoahA2i"27UA = {"User-Agent": "RentKaBot/1.0 (+https://www.rent-ka.com/bot; contact@spboucher.ai)",28      "Accept": "application/json"}2930LINE_RE = re.compile(r'^(\d+)\|"client":\{"id":\d+,"name":"(.*?)","website":"([^"]*)')313233def clients() -> list[dict]:34    out = []35    for line in SRC.read_text("utf-8").splitlines():36        m = LINE_RE.match(line.strip())37        if m:38            out.append({"client_id": int(m.group(1)),39                        "name": m.group(2), "website": m.group(3)})40    return out414243def probe(cid: int, tries: int = 3) -> dict:44    for i in range(tries):45        try:46            r = requests.get(API, params={47                "client_id": str(cid), "auth_token": TOKEN,48                "show_all_properties": "true", "limit": "1000",49            }, headers=UA, timeout=25)50            if r.status_code != 200:51                time.sleep(1.5 * (i + 1))52                continue53            props = r.json()54            if not isinstance(props, list):55                return {"n": 0, "provinces": {}}56            provs = Counter()57            residential = 058            nonres = {"office", "retail", "warehouse", "industrial",59                      "commercial", "land", "construction", "motel", "hotel",60                      "parking", "storage"}61            for p in props:62                pc = ((p.get("address") or {}).get("province_code") or "").upper()63                if str(p.get("property_type") or "").strip().lower() in nonres:64                    continue65                residential += 166                if pc:67                    provs[pc] += 168            return {"n": len(props), "res": residential, "provinces": dict(provs)}69        except Exception:70            time.sleep(1.5 * (i + 1))71    return {"error": True}727374def main() -> None:75    limit = int(sys.argv[1]) if len(sys.argv) > 1 else 10**976    done = set()77    if OUT.exists():78        for line in OUT.read_text().splitlines():79            try:80                done.add(json.loads(line)["client_id"])81            except Exception:82                pass83    todo = [c for c in clients() if c["client_id"] not in done][:limit]84    print(f"[probe] {len(todo)} clients to probe ({len(done)} already done)")85    with OUT.open("a") as fh:86        for i, c in enumerate(todo, 1):87            res = probe(c["client_id"])88            rec = {**c, **res, "ts": time.strftime("%Y-%m-%dT%H:%M:%S")}89            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")90            fh.flush()91            if i % 50 == 0:92                print(f"[probe] {i}/{len(todo)}")93            time.sleep(0.33)94    print("[probe] done →", OUT)959697if __name__ == "__main__":98    main()99