# ----------------------------------------------------------------------------- # Rent-Ka — Rental listings aggregator (Canada, outside Québec) # Author: Simon-Pierre Boucher — contact@spboucher.ai # scripts/build_lift_registry.py : turn data/liftsystem_probe.jsonl (output of # probe_liftsystem.py) into liftsystem_clients.json entries + sources.json # entries. Keeps clients with >= MIN_LISTINGS residential properties outside # Québec; excludes clients already covered (existing lift_* registry entries, # dedicated connectors, other platform registries) by client_id and by # normalized name. Idempotent: re-running refreshes the generated entries # (marker "canada-expansion" in notes) without touching hand-written ones. # ----------------------------------------------------------------------------- from __future__ import annotations import json import re from pathlib import Path ROOT = Path(__file__).resolve().parents[1] PROBE = ROOT / "data" / "liftsystem_probe.jsonl" LIFT = ROOT / "data" / "liftsystem_clients.json" SOURCES = ROOT / "data" / "sources.json" MIN_LISTINGS = 3 MARKER = "canada-expansion 2026-08-27" # client_ids covered by dedicated Rent-Ka connectors (or the platform vendor # itself) — verified against docs/canada-sources.md. EXCLUDE_IDS = { 1, # Landlord Web Solutions (the vendor) 203, # Boardwalk (dedicated) 228, # CAPREIT (dedicated) 99, # Killam (dedicated) 153, # Minto (dedicated) 6, # Homestead (dedicated) 96, # Skyline (dedicated) 436, # Realstar (dedicated) 497, # Hazelview (dedicated) 21, # Centurion (dedicated) 518, # InterRent / CLV (dedicated) 5, # Osgoode (rc_osgoode) 17, # Effort Trust (rc_effort) 53, # Sterling Karamar (rsgw_sterlingkaramar) 4, # Drewlo (rsgw_drewlo) 1746, # « Sterling Karamar » (2e flux LiftSystem — déjà rsgw_sterlingkaramar) 1198, # « (NEW WEBSITE) Drewlo Holdings2 » (déjà rsgw_drewlo) 467, # « Minto Corp Services (Investors) » (déjà minto dédié) } PROV_NAMES = { "ON": "Ontario", "BC": "British Columbia", "AB": "Alberta", "SK": "Saskatchewan", "MB": "Manitoba", "NB": "New Brunswick", "NS": "Nova Scotia", "PE": "Prince Edward Island", "NL": "Newfoundland and Labrador", "YT": "Yukon", "NT": "Northwest Territories", "NU": "Nunavut", } def norm(s: str) -> str: return re.sub(r"[^a-z0-9]", "", (s or "").lower()) def slug(s: str) -> str: s = re.sub(r"[^a-z0-9]+", "", (s or "").lower() .replace("properties", "").replace("property", "") .replace("management", "").replace("apartments", "") .replace("rentals", "").replace("group", "") .replace("realestate", "").replace("inc", "")) return s[:24] or "client" def main() -> None: lift = json.loads(LIFT.read_text("utf-8")) sources = json.loads(SOURCES.read_text("utf-8")) # drop previously generated entries FIRST (idempotence), keep hand-written # ones — coverage sets must be computed on the FILTERED lists, otherwise a # re-run sees its own past output in have_names and re-adds nothing (bug # found 2026-08-28: sources.json fell from 675 to 100 entries). lift["clients"] = [c for c in lift["clients"] if MARKER not in (c.get("notes") or "")] sources["sources"] = [s for s in sources["sources"] if MARKER not in (s.get("notes") or "")] # existing coverage: ids + normalized names from lift registry and sources have_cids = {c.get("client_id") for c in lift["clients"]} have_slugs = {c["id"] for c in lift["clients"]} have_names = {norm(c.get("name")) for c in lift["clients"]} have_names |= {norm(s.get("name")) for s in sources["sources"]} for reg in ("rentcafe_clients.json", "buildium_clients.json", "appfolio_clients.json", "rentsyncgw_clients.json"): try: d = json.loads((ROOT / "data" / reg).read_text("utf-8")) have_names |= {norm(c.get("name")) for c in d.get("clients") or []} except (OSError, ValueError): pass added, skipped_cov, skipped_small = [], 0, 0 for line in PROBE.read_text("utf-8").splitlines(): try: rec = json.loads(line) except ValueError: continue cid = rec.get("client_id") if not cid or rec.get("error"): continue provs = {p: n for p, n in (rec.get("provinces") or {}).items() if p != "QC" and p in PROV_NAMES} n_roc = sum(provs.values()) if cid in EXCLUDE_IDS or cid in have_cids or \ norm(rec.get("name")) in have_names: skipped_cov += 1 continue if n_roc < MIN_LISTINGS: skipped_small += 1 continue base = slug(rec["name"]) sid = base i = 2 while sid in have_slugs: sid = f"{base}{i}" i += 1 have_slugs.add(sid) have_names.add(norm(rec["name"])) dominant = max(provs, key=provs.get) site = rec.get("website") or "" if site and not site.startswith("http"): site = "https://" + site prov_txt = ", ".join(f"{p} {n}" for p, n in sorted(provs.items(), key=lambda x: -x[1])) lift["clients"].append({ "id": sid, "name": rec["name"], "site": site, "client_id": cid, "listing_url": site, "regions": [PROV_NAMES[p] for p in sorted(provs, key=provs.get, reverse=True)], "province": dominant, "status": "valide", "notes": f"{MARKER} — probe live: {n_roc} propriétés " f"résidentielles hors QC ({prov_txt})", }) sources["sources"].append({ "id": f"lift_{sid}", "name": rec["name"], "url": site, "listing_url": site, "sectors": [PROV_NAMES[p] for p in provs], "connector": f"lift_{sid}", "status": "actif", "region": " / ".join(PROV_NAMES[p] for p in sorted(provs, key=provs.get, reverse=True)) + " — LiftSystem", "notes": f"{MARKER} — API LiftSystem client_id {cid}", }) added.append((sid, cid, dominant, n_roc)) lift["updated"] = "2026-08-27" n = len(sources["sources"]) sources["_comment"] = re.sub(r"^Rent-Ka source registry[^.]*\.", f"Rent-Ka source registry — {n} rental " f"sources for Canada outside Québec " f"(national expansion 2026-08-27).", sources["_comment"]) LIFT.write_text(json.dumps(lift, ensure_ascii=False, indent=2)) SOURCES.write_text(json.dumps(sources, ensure_ascii=False, indent=2)) from collections import Counter per_prov = Counter(d for _, _, d, _ in added) print(f"[lift-registry] +{len(added)} clients " f"(covered elsewhere: {skipped_cov}, <{MIN_LISTINGS} listings: " f"{skipped_small}) — dominant province: {dict(per_prov)}") print(f"[lift-registry] sources.json now {n} entries") if __name__ == "__main__": main()