spb/rent-ka

1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# scripts/probe_rentfaster_cities.py : discover RentFaster numeric city_ids.5#   /api/search.json only filters by numeric city_id (1=Calgary, 2=Edmonton,6#   6=Vancouver…) but /api/map.json's `cities` array exposes names without ids.7#   This script probes city_id 1..N once, records {id, city, province, total}8#   and writes data/rentfaster_cities.json (consumed by connectors/rentfaster.py).9#   Province comes from the map.json cities array (name match) with a bbox10#   fallback on the first listing's coordinates. Resumable via the output file.11# Usage: python scripts/probe_rentfaster_cities.py [max_id=700]12# -----------------------------------------------------------------------------13from __future__ import annotations1415import json16import sys17import time18from pathlib import Path1920import requests2122ROOT = Path(__file__).resolve().parents[1]23OUT = ROOT / "data" / "rentfaster_cities.json"24UA = {"User-Agent": "RentKaBot/1.0 (+https://www.rent-ka.com/bot; contact@spboucher.ai)",25      "Accept": "application/json"}2627sys.path.insert(0, str(ROOT))28from rentka.schema import PROVINCE_BBOX  # noqa: E402293031def bbox_province(lat: float, lng: float) -> str:32    for prov, (lo_lat, hi_lat, lo_lng, hi_lng) in PROVINCE_BBOX.items():33        if lo_lat <= lat <= hi_lat and lo_lng <= lng <= hi_lng:34            return prov35    return ""363738def city_provinces() -> dict[str, str]:39    """name(lower) -> province from map.json's cities array (incl. markers)."""40    try:41        d = requests.post("https://www.rentfaster.ca/api/map.json",42                          data={"x": "1"}, headers=UA, timeout=30).json()43    except Exception:44        return {}45    out: dict[str, str] = {}4647    def walk(items):48        for c in items or []:49            name = (c.get("city") or "").strip().lower()50            if name and c.get("province"):51                out.setdefault(name, c["province"])52            walk(c.get("markers"))53    walk(d.get("cities"))54    return out555657def main() -> None:58    max_id = int(sys.argv[1]) if len(sys.argv) > 1 else 70059    known: dict[str, dict] = {}60    if OUT.exists():61        try:62            known = {str(c["id"]): c63                     for c in json.loads(OUT.read_text())["cities"]}64        except Exception:65            known = {}66    provs = city_provinces()67    print(f"[rf-probe] {len(provs)} city names with province from map.json")68    for cid in range(1, max_id + 1):69        if str(cid) in known:70            continue71        try:72            r = requests.get("https://www.rentfaster.ca/api/search.json",73                             params={"city_id": str(cid), "cur_page": "0"},74                             headers=UA, timeout=25)75            d = r.json()76        except Exception:77            time.sleep(2)78            continue79        ls = d.get("listings") or []80        total = int(d.get("total") or 0)81        if not ls:82            known[str(cid)] = {"id": cid, "city": "", "province": "", "total": 0}83        else:84            city = (ls[0].get("city") or "").strip()85            prov = provs.get(city.lower(), "")86            if not prov:87                try:88                    prov = bbox_province(float(ls[0]["latitude"]),89                                         float(ls[0]["longitude"]))90                except (KeyError, TypeError, ValueError):91                    prov = ""92            known[str(cid)] = {"id": cid, "city": city,93                               "province": prov, "total": total}94        if cid % 50 == 0:95            print(f"[rf-probe] {cid}/{max_id}")96            OUT.write_text(json.dumps(97                {"_comment": "RentFaster city_id map — probed live; consumed "98                             "by connectors/rentfaster.py (QC skipped there).",99                 "updated": time.strftime("%Y-%m-%d"),100                 "cities": sorted(known.values(), key=lambda c: c["id"])},101                ensure_ascii=False, indent=1))102        time.sleep(0.4)103    OUT.write_text(json.dumps(104        {"_comment": "RentFaster city_id map — probed live; consumed by "105                     "connectors/rentfaster.py (QC skipped there).",106         "updated": time.strftime("%Y-%m-%d"),107         "cities": sorted(known.values(), key=lambda c: c["id"])},108        ensure_ascii=False, indent=1))109    active = [c for c in known.values() if c["total"] > 0]110    print(f"[rf-probe] done: {len(active)} active city_ids → {OUT}")111112113if __name__ == "__main__":114    main()115