1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# scripts/build_lift_registry.py : turn data/liftsystem_probe.jsonl (output of5# probe_liftsystem.py) into liftsystem_clients.json entries + sources.json6# entries. Keeps clients with >= MIN_LISTINGS residential properties outside7# Québec; excludes clients already covered (existing lift_* registry entries,8# dedicated connectors, other platform registries) by client_id and by9# normalized name. Idempotent: re-running refreshes the generated entries10# (marker "canada-expansion" in notes) without touching hand-written ones.11# -----------------------------------------------------------------------------12from __future__ import annotations1314import json15import re16from pathlib import Path1718ROOT = Path(__file__).resolve().parents[1]19PROBE = ROOT / "data" / "liftsystem_probe.jsonl"20LIFT = ROOT / "data" / "liftsystem_clients.json"21SOURCES = ROOT / "data" / "sources.json"2223MIN_LISTINGS = 324MARKER = "canada-expansion 2026-08-27"2526# client_ids covered by dedicated Rent-Ka connectors (or the platform vendor27# itself) — verified against docs/canada-sources.md.28EXCLUDE_IDS = {29 1, # Landlord Web Solutions (the vendor)30 203, # Boardwalk (dedicated)31 228, # CAPREIT (dedicated)32 99, # Killam (dedicated)33 153, # Minto (dedicated)34 6, # Homestead (dedicated)35 96, # Skyline (dedicated)36 436, # Realstar (dedicated)37 497, # Hazelview (dedicated)38 21, # Centurion (dedicated)39 518, # InterRent / CLV (dedicated)40 5, # Osgoode (rc_osgoode)41 17, # Effort Trust (rc_effort)42 53, # Sterling Karamar (rsgw_sterlingkaramar)43 4, # Drewlo (rsgw_drewlo)44}45PROV_NAMES = {46 "ON": "Ontario", "BC": "British Columbia", "AB": "Alberta",47 "SK": "Saskatchewan", "MB": "Manitoba", "NB": "New Brunswick",48 "NS": "Nova Scotia", "PE": "Prince Edward Island",49 "NL": "Newfoundland and Labrador", "YT": "Yukon",50 "NT": "Northwest Territories", "NU": "Nunavut",51}525354def norm(s: str) -> str:55 return re.sub(r"[^a-z0-9]", "", (s or "").lower())565758def slug(s: str) -> str:59 s = re.sub(r"[^a-z0-9]+", "", (s or "").lower()60 .replace("properties", "").replace("property", "")61 .replace("management", "").replace("apartments", "")62 .replace("rentals", "").replace("group", "")63 .replace("realestate", "").replace("inc", ""))64 return s[:24] or "client"656667def main() -> None:68 lift = json.loads(LIFT.read_text("utf-8"))69 sources = json.loads(SOURCES.read_text("utf-8"))7071 # existing coverage: ids + normalized names from lift registry and sources72 have_cids = {c.get("client_id") for c in lift["clients"]}73 have_slugs = {c["id"] for c in lift["clients"]}74 have_names = {norm(c.get("name")) for c in lift["clients"]}75 have_names |= {norm(s.get("name")) for s in sources["sources"]}76 for reg in ("rentcafe_clients.json", "buildium_clients.json",77 "appfolio_clients.json", "rentsyncgw_clients.json"):78 try:79 d = json.loads((ROOT / "data" / reg).read_text("utf-8"))80 have_names |= {norm(c.get("name")) for c in d.get("clients") or []}81 except (OSError, ValueError):82 pass8384 # drop previously generated entries (idempotence), keep hand-written ones85 lift["clients"] = [c for c in lift["clients"]86 if MARKER not in (c.get("notes") or "")]87 sources["sources"] = [s for s in sources["sources"]88 if MARKER not in (s.get("notes") or "")]89 have_cids = {c.get("client_id") for c in lift["clients"]}9091 added, skipped_cov, skipped_small = [], 0, 092 for line in PROBE.read_text("utf-8").splitlines():93 try:94 rec = json.loads(line)95 except ValueError:96 continue97 cid = rec.get("client_id")98 if not cid or rec.get("error"):99 continue100 provs = {p: n for p, n in (rec.get("provinces") or {}).items()101 if p != "QC" and p in PROV_NAMES}102 n_roc = sum(provs.values())103 if cid in EXCLUDE_IDS or cid in have_cids or \104 norm(rec.get("name")) in have_names:105 skipped_cov += 1106 continue107 if n_roc < MIN_LISTINGS:108 skipped_small += 1109 continue110 base = slug(rec["name"])111 sid = base112 i = 2113 while sid in have_slugs:114 sid = f"{base}{i}"115 i += 1116 have_slugs.add(sid)117 have_names.add(norm(rec["name"]))118 dominant = max(provs, key=provs.get)119 site = rec.get("website") or ""120 if site and not site.startswith("http"):121 site = "https://" + site122 prov_txt = ", ".join(f"{p} {n}" for p, n in123 sorted(provs.items(), key=lambda x: -x[1]))124 lift["clients"].append({125 "id": sid,126 "name": rec["name"],127 "site": site,128 "client_id": cid,129 "listing_url": site,130 "regions": [PROV_NAMES[p] for p in131 sorted(provs, key=provs.get, reverse=True)],132 "province": dominant,133 "status": "valide",134 "notes": f"{MARKER} — probe live: {n_roc} propriétés "135 f"résidentielles hors QC ({prov_txt})",136 })137 sources["sources"].append({138 "id": f"lift_{sid}",139 "name": rec["name"],140 "url": site,141 "listing_url": site,142 "sectors": [PROV_NAMES[p] for p in provs],143 "connector": f"lift_{sid}",144 "status": "actif",145 "region": " / ".join(PROV_NAMES[p] for p in146 sorted(provs, key=provs.get, reverse=True))147 + " — LiftSystem",148 "notes": f"{MARKER} — API LiftSystem client_id {cid}",149 })150 added.append((sid, cid, dominant, n_roc))151152 lift["updated"] = "2026-08-27"153 n = len(sources["sources"])154 sources["_comment"] = re.sub(r"^Rent-Ka source registry[^.]*\.",155 f"Rent-Ka source registry — {n} rental "156 f"sources for Canada outside Québec "157 f"(national expansion 2026-08-27).",158 sources["_comment"])159 LIFT.write_text(json.dumps(lift, ensure_ascii=False, indent=2))160 SOURCES.write_text(json.dumps(sources, ensure_ascii=False, indent=2))161 from collections import Counter162 per_prov = Counter(d for _, _, d, _ in added)163 print(f"[lift-registry] +{len(added)} clients "164 f"(covered elsewhere: {skipped_cov}, <{MIN_LISTINGS} listings: "165 f"{skipped_small}) — dominant province: {dict(per_prov)}")166 print(f"[lift-registry] sources.json now {n} entries")167168169if __name__ == "__main__":170 main()171