spb/rent-ka

1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# scripts/build_lift_registry.py : turn data/liftsystem_probe.jsonl (output of5#   probe_liftsystem.py) into liftsystem_clients.json entries + sources.json6#   entries. Keeps clients with >= MIN_LISTINGS residential properties outside7#   Québec; excludes clients already covered (existing lift_* registry entries,8#   dedicated connectors, other platform registries) by client_id and by9#   normalized name. Idempotent: re-running refreshes the generated entries10#   (marker "canada-expansion" in notes) without touching hand-written ones.11# -----------------------------------------------------------------------------12from __future__ import annotations1314import json15import re16from pathlib import Path1718ROOT = Path(__file__).resolve().parents[1]19PROBE = ROOT / "data" / "liftsystem_probe.jsonl"20LIFT = ROOT / "data" / "liftsystem_clients.json"21SOURCES = ROOT / "data" / "sources.json"2223MIN_LISTINGS = 324MARKER = "canada-expansion 2026-08-27"2526# client_ids covered by dedicated Rent-Ka connectors (or the platform vendor27# itself) — verified against docs/canada-sources.md.28EXCLUDE_IDS = {29    1,      # Landlord Web Solutions (the vendor)30    203,    # Boardwalk (dedicated)31    228,    # CAPREIT (dedicated)32    99,     # Killam (dedicated)33    153,    # Minto (dedicated)34    6,      # Homestead (dedicated)35    96,     # Skyline (dedicated)36    436,    # Realstar (dedicated)37    497,    # Hazelview (dedicated)38    21,     # Centurion (dedicated)39    518,    # InterRent / CLV (dedicated)40    5,      # Osgoode (rc_osgoode)41    17,     # Effort Trust (rc_effort)42    53,     # Sterling Karamar (rsgw_sterlingkaramar)43    4,      # Drewlo (rsgw_drewlo)44    1746,   # « Sterling Karamar » (2e flux LiftSystem — déjà rsgw_sterlingkaramar)45    1198,   # « (NEW WEBSITE) Drewlo Holdings2 » (déjà rsgw_drewlo)46    467,    # « Minto Corp Services (Investors) » (déjà minto dédié)47}48PROV_NAMES = {49    "ON": "Ontario", "BC": "British Columbia", "AB": "Alberta",50    "SK": "Saskatchewan", "MB": "Manitoba", "NB": "New Brunswick",51    "NS": "Nova Scotia", "PE": "Prince Edward Island",52    "NL": "Newfoundland and Labrador", "YT": "Yukon",53    "NT": "Northwest Territories", "NU": "Nunavut",54}555657def norm(s: str) -> str:58    return re.sub(r"[^a-z0-9]", "", (s or "").lower())596061def slug(s: str) -> str:62    s = re.sub(r"[^a-z0-9]+", "", (s or "").lower()63               .replace("properties", "").replace("property", "")64               .replace("management", "").replace("apartments", "")65               .replace("rentals", "").replace("group", "")66               .replace("realestate", "").replace("inc", ""))67    return s[:24] or "client"686970def main() -> None:71    lift = json.loads(LIFT.read_text("utf-8"))72    sources = json.loads(SOURCES.read_text("utf-8"))7374    # drop previously generated entries FIRST (idempotence), keep hand-written75    # ones — coverage sets must be computed on the FILTERED lists, otherwise a76    # re-run sees its own past output in have_names and re-adds nothing (bug77    # found 2026-08-28: sources.json fell from 675 to 100 entries).78    lift["clients"] = [c for c in lift["clients"]79                       if MARKER not in (c.get("notes") or "")]80    sources["sources"] = [s for s in sources["sources"]81                          if MARKER not in (s.get("notes") or "")]8283    # existing coverage: ids + normalized names from lift registry and sources84    have_cids = {c.get("client_id") for c in lift["clients"]}85    have_slugs = {c["id"] for c in lift["clients"]}86    have_names = {norm(c.get("name")) for c in lift["clients"]}87    have_names |= {norm(s.get("name")) for s in sources["sources"]}88    for reg in ("rentcafe_clients.json", "buildium_clients.json",89                "appfolio_clients.json", "rentsyncgw_clients.json"):90        try:91            d = json.loads((ROOT / "data" / reg).read_text("utf-8"))92            have_names |= {norm(c.get("name")) for c in d.get("clients") or []}93        except (OSError, ValueError):94            pass9596    added, skipped_cov, skipped_small = [], 0, 097    for line in PROBE.read_text("utf-8").splitlines():98        try:99            rec = json.loads(line)100        except ValueError:101            continue102        cid = rec.get("client_id")103        if not cid or rec.get("error"):104            continue105        provs = {p: n for p, n in (rec.get("provinces") or {}).items()106                 if p != "QC" and p in PROV_NAMES}107        n_roc = sum(provs.values())108        if cid in EXCLUDE_IDS or cid in have_cids or \109                norm(rec.get("name")) in have_names:110            skipped_cov += 1111            continue112        if n_roc < MIN_LISTINGS:113            skipped_small += 1114            continue115        base = slug(rec["name"])116        sid = base117        i = 2118        while sid in have_slugs:119            sid = f"{base}{i}"120            i += 1121        have_slugs.add(sid)122        have_names.add(norm(rec["name"]))123        dominant = max(provs, key=provs.get)124        site = rec.get("website") or ""125        if site and not site.startswith("http"):126            site = "https://" + site127        prov_txt = ", ".join(f"{p} {n}" for p, n in128                             sorted(provs.items(), key=lambda x: -x[1]))129        lift["clients"].append({130            "id": sid,131            "name": rec["name"],132            "site": site,133            "client_id": cid,134            "listing_url": site,135            "regions": [PROV_NAMES[p] for p in136                        sorted(provs, key=provs.get, reverse=True)],137            "province": dominant,138            "status": "valide",139            "notes": f"{MARKER} — probe live: {n_roc} propriétés "140                     f"résidentielles hors QC ({prov_txt})",141        })142        sources["sources"].append({143            "id": f"lift_{sid}",144            "name": rec["name"],145            "url": site,146            "listing_url": site,147            "sectors": [PROV_NAMES[p] for p in provs],148            "connector": f"lift_{sid}",149            "status": "actif",150            "region": " / ".join(PROV_NAMES[p] for p in151                                 sorted(provs, key=provs.get, reverse=True))152                      + " — LiftSystem",153            "notes": f"{MARKER} — API LiftSystem client_id {cid}",154        })155        added.append((sid, cid, dominant, n_roc))156157    lift["updated"] = "2026-08-27"158    n = len(sources["sources"])159    sources["_comment"] = re.sub(r"^Rent-Ka source registry[^.]*\.",160                                 f"Rent-Ka source registry — {n} rental "161                                 f"sources for Canada outside Québec "162                                 f"(national expansion 2026-08-27).",163                                 sources["_comment"])164    LIFT.write_text(json.dumps(lift, ensure_ascii=False, indent=2))165    SOURCES.write_text(json.dumps(sources, ensure_ascii=False, indent=2))166    from collections import Counter167    per_prov = Counter(d for _, _, d, _ in added)168    print(f"[lift-registry] +{len(added)} clients "169          f"(covered elsewhere: {skipped_cov}, <{MIN_LISTINGS} listings: "170          f"{skipped_small}) — dominant province: {dict(per_prov)}")171    print(f"[lift-registry] sources.json now {n} entries")172173174if __name__ == "__main__":175    main()176