1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Rent-based proxies: lease matching, median rents, crude gross yields and the user-cost diagnostic3(paper §sec:fundamental; docs/fundamental_value.md §3–§4).45Outputs (results/fundamental/):6 leases_matched.parquet cleaned leases with municipality / DA attribution7 leases_matching.json cleaning funnel + matching counts8 rents_by_mun_year.csv median rent by municipality × rooms bucket × year9 gross_yields.parquet one row per condo / plex sale matched to a municipality-year rent cell10 price_rent.csv region group × class × year: yield distribution + user cost + implied value / price11 usercost_parameters.csv parameter table (mortgage rate from the Bank of Canada, assumptions labelled)12 paper/tables/tab_usercost_params.tex13 figures/fundamental_price_rent.pdf, figures/fundamental_usercost.pdf1415 python3 scripts/52_rents_usercost.py [--no-download] [--condo-bucket all] [--plex-bucket all]16"""17from __future__ import annotations1819import argparse20import json21import sys22import time23from datetime import datetime, timezone24from pathlib import Path2526import matplotlib27matplotlib.use("Agg")28import matplotlib.pyplot as plt # noqa: E40229import numpy as np # noqa: E40230import pandas as pd # noqa: E40231import polars as pl # noqa: E4023233from agentic_valuation.config import FIGURES, PAPER, PROCESSED, RESULTS # noqa: E40234from agentic_valuation.evaluation.tables import escape, num, tabular # noqa: E40235from agentic_valuation.fundamental.rents import (BOC_CACHE, BOC_SERIES, QUARTIER_DB, RDL_DB, USER_COST_PARAMS, # noqa: E40236 annual_mortgage_rate, fetch_boc_mortgage_rate, gross_yields,37 implied_value_ratio, load_leases, match_leases, median_rents, user_cost)3839OUT = RESULTS / "fundamental"40REGIONS = ["Montréal CMA", "Québec CMA", "Outaouais", "other"]41PALETTE = ["#1f5fa8", "#c8541c", "#0b7a5a", "#8b4e8d"]42GREY, INK = "#7a7a7a", "#222222"43plt.rcParams.update({"font.size": 8.5, "axes.spines.top": False, "axes.spines.right": False, "axes.edgecolor": GREY,44 "axes.grid": True, "grid.color": "#e3e3e3", "grid.linewidth": 0.5, "axes.axisbelow": True,45 "pdf.fonttype": 42, "legend.frameon": False, "text.color": INK, "font.family": "sans-serif"})464748def write_fragment(tex: str, path: Path) -> None:49 path.parent.mkdir(parents=True, exist_ok=True)50 path.write_text("% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca\n% generated by scripts/52_rents_usercost.py — do not edit\n"51 + tex.rstrip() + "\n", encoding="utf-8")525354def fig_price_rent(gy: pd.DataFrame, path: Path) -> None:55 years = sorted(gy["year"].unique())56 fig, axes = plt.subplots(1, 2, figsize=(7.2, 2.6), sharey=True)57 for ax, cls in zip(axes, ["condo", "plex"]):58 sub = gy[gy["property_class"] == cls]59 width = 0.1860 for j, reg in enumerate(REGIONS):61 data = [sub.loc[(sub["region_group"] == reg) & (sub["year"] == y), "gross_yield"].to_numpy() for y in years]62 pos = [i + (j - 1.5) * width for i in range(len(years))]63 keep = [k for k, d in enumerate(data) if len(d) >= 30]64 if not keep:65 continue66 bp = ax.boxplot([data[k] for k in keep], positions=[pos[k] for k in keep], widths=width * 0.9, whis=(5, 95),67 showfliers=False, patch_artist=True, medianprops={"color": INK, "linewidth": 1.0},68 whiskerprops={"color": GREY, "linewidth": 0.6}, capprops={"color": GREY, "linewidth": 0.6})69 for patch in bp["boxes"]:70 patch.set(facecolor=PALETTE[j], alpha=0.6, edgecolor=PALETTE[j])71 ax.set_xticks(range(len(years)), [str(y) for y in years])72 ax.set_title(f"{cls} (n = {len(sub):,})", fontsize=8.5, loc="left", color=INK)73 ax.set_xlabel("year of sale")74 axes[0].set_ylabel("gross rental yield (annual rent / price)")75 axes[0].set_ylim(0, 0.16)76 handles = [plt.Rectangle((0, 0), 1, 1, facecolor=PALETTE[j], alpha=0.6) for j in range(len(REGIONS))]77 axes[1].legend(handles, REGIONS, loc="upper right", ncol=2, fontsize=7)78 fig.tight_layout()79 fig.savefig(path, bbox_inches="tight", pad_inches=0.02)80 plt.close(fig)818283def fig_usercost(pr: pd.DataFrame, g_scen: list[float], path: Path) -> None:84 fig, axes = plt.subplots(1, 2, figsize=(7.2, 2.6), sharey=True)85 styles = ["-", "--", ":"]86 for ax, cls in zip(axes, ["condo", "plex"]):87 sub = pr[(pr["property_class"] == cls) & (pr["region_group"] == "ALL")].sort_values("year")88 for k, g in enumerate(g_scen):89 col = f"implied_ratio_median_g{int(round(g * 100))}"90 ax.plot(sub["year"], sub[col], linestyle=styles[k], color=PALETTE[k], marker="o", markersize=3, label=f"$\\pi^e$ = {g:.0%}")91 lo, hi = sub[f"implied_ratio_p25_g{int(round(g_scen[1] * 100))}"], sub[f"implied_ratio_p75_g{int(round(g_scen[1] * 100))}"]92 ax.fill_between(sub["year"], lo, hi, color=PALETTE[1], alpha=0.12, linewidth=0, label=f"IQR, $\\pi^e$ = {g_scen[1]:.0%}")93 ax.axhline(1.0, color=GREY, linewidth=0.7, linestyle="--")94 ax.set_title(f"{cls} — illustrative", fontsize=8.5, loc="left", color=INK)95 ax.set_xlabel("year of sale")96 ax.set_xticks(sub["year"].to_numpy())97 axes[0].set_ylabel("rent-implied value / price (= gross yield / $u$)")98 axes[0].legend(loc="upper left", fontsize=7)99 fig.tight_layout()100 fig.savefig(path, bbox_inches="tight", pad_inches=0.02)101 plt.close(fig)102103104def main(argv: list[str] | None = None) -> int:105 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)106 ap.add_argument("--analytic", default=str(PROCESSED / "analytic.parquet"))107 ap.add_argument("--rdl", default=str(RDL_DB))108 ap.add_argument("--quartier", default=str(QUARTIER_DB))109 ap.add_argument("--no-download", action="store_true", help="do not call the Bank of Canada API (use the cache or NaN)")110 ap.add_argument("--condo-bucket", default="all", help="rooms bucket of the leases matched to condo sales (0-1, 2, 3, 4+, all)")111 ap.add_argument("--plex-bucket", default="all")112 ap.add_argument("--min-leases", type=int, default=10)113 ap.add_argument("--out", default=str(OUT))114 args = ap.parse_args(argv)115 out = Path(args.out)116 out.mkdir(parents=True, exist_ok=True)117 t0 = time.time()118119 analytic = pl.read_parquet(args.analytic)120 leases = load_leases(Path(args.rdl))121 funnel = dict(leases.attrs["funnel"])122 print(f"leases: {funnel}")123 matched = match_leases(leases, analytic, quartier_db=Path(args.quartier) if args.quartier else None)124 n_match = int(matched["matched"].sum())125 n_da = int(matched["dauid"].notna().sum())126 print(f"matched to a municipality (≤ 2 km from a sale): {n_match:,} / {len(matched):,}; in a DA polygon: {n_da:,} ({time.time() - t0:.1f} s)")127 matched.to_parquet(out / "leases_matched.parquet", index=False)128129 rents = median_rents(matched[matched["matched"]])130 rents.to_csv(out / "rents_by_mun_year.csv", index=False)131 gy = gross_yields(analytic, rents, condo_bucket=args.condo_bucket, plex_bucket=args.plex_bucket, min_leases=args.min_leases)132 gy.to_parquet(out / "gross_yields.parquet", index=False)133 print(f"gross yields: {len(gy):,} condo/plex sales matched to a municipality-year rent cell with ≥ {args.min_leases} leases "134 f"(condo {int((gy['property_class'] == 'condo').sum()):,}, plex {int((gy['property_class'] == 'plex').sum()):,}); "135 f"median yield condo {gy.loc[gy['property_class'] == 'condo', 'gross_yield'].median():.3f}, plex {gy.loc[gy['property_class'] == 'plex', 'gross_yield'].median():.3f}")136137 # mortgage rate (Bank of Canada Valet, verified series id) — NaN when unavailable, never invented138 weekly = fetch_boc_mortgage_rate(BOC_CACHE, download=not args.no_download)139 years = sorted(gy["year"].unique().tolist())140 i_year = annual_mortgage_rate(weekly, years)141 rate_src = f"Bank of Canada Valet {BOC_SERIES} ({weekly.attrs.get('label', '')}), {len(weekly)} weekly obs {weekly['date'].min().date() if len(weekly) else ''}–{weekly['date'].max().date() if len(weekly) else ''}" if len(weekly) else "UNAVAILABLE (download failed, no cache) — rate left NaN"142 print(f"mortgage rate: {rate_src}; annual means {i_year.round(4).to_dict()}")143144 tau_p, delta, gamma = (USER_COST_PARAMS[k]["value"] for k in ("tau_p", "delta", "gamma"))145 g_scen = list(USER_COST_PARAMS["g_scenarios"]["value"])146 gy = gy.merge(i_year.rename("i").reset_index().rename(columns={"index": "year"}), on="year", how="left")147 for g in g_scen:148 u = user_cost(gy["i"], tau_p, delta, gamma, g)149 gy[f"u_g{int(round(g * 100))}"] = u150 gy[f"implied_ratio_g{int(round(g * 100))}"] = implied_value_ratio(gy["gross_yield"], u)151152 rows = []153 for (reg, cls, y), gsub in list(gy.groupby(["region_group", "property_class", "year"])) + [(("ALL", c, y), s) for (c, y), s in gy.groupby(["property_class", "year"])]:154 r = {"region_group": reg, "property_class": cls, "year": int(y), "n_sales": int(len(gsub)), "n_mun": int(gsub["mun_code"].nunique()),155 "price_median": float(gsub["price"].median()), "rent_median": float(gsub["rent_median"].median()),156 "gross_yield_p25": float(gsub["gross_yield"].quantile(0.25)), "gross_yield_median": float(gsub["gross_yield"].median()),157 "gross_yield_p75": float(gsub["gross_yield"].quantile(0.75)), "mortgage_rate_5y": float(i_year.get(int(y), np.nan))}158 for g in g_scen:159 tag = f"g{int(round(g * 100))}"160 r[f"user_cost_{tag}"] = float(gsub[f"u_{tag}"].iloc[0]) if len(gsub) else np.nan161 r[f"implied_ratio_p25_{tag}"] = float(gsub[f"implied_ratio_{tag}"].quantile(0.25))162 r[f"implied_ratio_median_{tag}"] = float(gsub[f"implied_ratio_{tag}"].median())163 r[f"implied_ratio_p75_{tag}"] = float(gsub[f"implied_ratio_{tag}"].quantile(0.75))164 rows.append(r)165 pr = pd.DataFrame(rows).sort_values(["property_class", "region_group", "year"]).reset_index(drop=True)166 pr.to_csv(out / "price_rent.csv", index=False)167168 # parameter table169 prow = [{"parameter": "i (5-year conventional mortgage rate)", "value": "; ".join(f"{y}: {v:.2%}" if np.isfinite(v) else f"{y}: NaN" for y, v in i_year.items()),170 "status": "measured", "source": rate_src}]171 for k in ("tau_p", "delta", "gamma"):172 prow.append({"parameter": USER_COST_PARAMS[k]["label"], "value": f"{USER_COST_PARAMS[k]['value']:.1%}", "status": "assumption", "source": USER_COST_PARAMS[k]["source"]})173 prow.append({"parameter": USER_COST_PARAMS["g_scenarios"]["label"], "value": ", ".join(f"{g:.0%}" for g in g_scen), "status": "scenario", "source": USER_COST_PARAMS["g_scenarios"]["source"]})174 params = pd.DataFrame(prow)175 params.to_csv(out / "usercost_parameters.csv", index=False)176 header = ["Parameter", "Value", "Status"]177 tex_names = {USER_COST_PARAMS["g_scenarios"]["label"]: r"expected nominal appreciation $\pi^{e}$ (three scenarios)",178 prow[0]["parameter"]: r"$i$: 5-year conventional mortgage rate (annual mean of weekly observations)",179 USER_COST_PARAMS["tau_p"]["label"]: r"$\tau_p$: property tax rate (municipal and school, share of market value)",180 USER_COST_PARAMS["delta"]["label"]: r"$\delta$: maintenance and depreciation",181 USER_COST_PARAMS["gamma"]["label"]: r"$\gamma$: risk and liquidity premium"}182 trows = [[tex_names.get(r["parameter"], escape(r["parameter"])), escape(r["value"]), escape(r["status"])] for r in prow]183 write_fragment(tabular(header, trows, "p{5.6cm}p{6.2cm}l"), PAPER / "tables" / "tab_usercost_params.tex")184185 FIGURES.mkdir(parents=True, exist_ok=True)186 fig_price_rent(gy, FIGURES / "fundamental_price_rent.pdf")187 fig_usercost(pr, g_scen, FIGURES / "fundamental_usercost.pdf")188189 summary = {"funnel": funnel, "n_matched_municipality": n_match, "n_in_da_polygon": n_da, "n_unmatched": int(len(matched) - n_match),190 "rooms_bucket_counts": matched.loc[matched["matched"], "rooms_bucket"].value_counts().to_dict(),191 "start_date_present_share": float(matched.loc[matched["matched"], "start_date_present"].mean()),192 "region_counts": matched.loc[matched["matched"], "region_group"].value_counts().to_dict(),193 "year_counts": {int(k): int(v) for k, v in matched.loc[matched["matched"], "year"].value_counts().sort_index().items()},194 "n_rent_cells": int(len(rents)), "n_gross_yield_sales": int(len(gy)),195 "condo_bucket": args.condo_bucket, "plex_bucket": args.plex_bucket, "min_leases": args.min_leases,196 "mortgage_rate_source": rate_src, "mortgage_rate_annual": {int(k): (None if not np.isfinite(v) else float(v)) for k, v in i_year.items()},197 "user_cost_params": {k: v["value"] for k, v in USER_COST_PARAMS.items()},198 "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "seconds": round(time.time() - t0, 1)}199 (out / "leases_matching.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False, default=str), encoding="utf-8")200 print(f"done in {time.time() - t0:.1f} s")201 return 0202203204if __name__ == "__main__":205 sys.exit(main())206