1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Descriptive statistics of the analytic sample for the Data section of the paper.45Reads data/processed/analytic.parquet and data/processed/splits.parquet (real data; ``--synthetic`` switches6to data/synthetic/ and suffixes the outputs) and writes78* results/tables/descriptives.csv — one row per property class + total9* paper/tables/tab_descriptives.tex — booktabs ``tabular`` fragment (no caption; wrapped by sections/data.tex)10* results/tables/splits_by_region.csv — split × region group counts (+ totals)11* paper/tables/tab_splits.tex — booktabs ``tabular`` fragment1213Numbers are written as plain text (``siunitx`` is loaded with ``parse-numbers=false`` in the paper).14"""15from __future__ import annotations1617import argparse18import sys19from pathlib import Path2021import polars as pl2223ROOT = Path(__file__).resolve().parents[1]24sys.path.insert(0, str(ROOT / "src"))2526CLASS_ORDER = ["house", "condo", "plex"]27CLASS_LABEL = {"house": "House", "condo": "Condominium", "plex": "Plex", "total": "All classes"}28SPLIT_ORDER = ["train", "validation", "test", "holdout"]29SPLIT_LABEL = {"train": "Train", "validation": "Validation", "test": "Test", "holdout": "Holdout", "total": "Total"}30REGION_ORDER = ["Montréal CMA", "Québec CMA", "Outaouais", "other"]31REGION_LABEL = {"Montréal CMA": "Montr\\'eal CMA", "Québec CMA": "Qu\\'ebec CMA", "Outaouais": "Outaouais", "other": "Other regions", "total": "Total"}323334def _fmt_int(x: float | int | None) -> str:35 return "--" if x is None else f"{int(round(x)):,}"363738def _fmt_1(x: float | None) -> str:39 return "--" if x is None else f"{x:,.1f}"404142def _fmt_year(x: float | None) -> str:43 return "--" if x is None else f"{int(round(x))}"444546def _fmt_pct(x: float | None) -> str:47 return "--" if x is None else f"{100 * x:.1f}"484950def descriptives(analytic: pl.DataFrame) -> pl.DataFrame:51 def summarise(df: pl.DataFrame, label: str) -> dict:52 return {53 "property_class": label,54 "n": df.height,55 "median_price": float(df["price"].median()),56 "mean_floor_area_m2": float(df["floor_area_m2"].mean()),57 "median_floor_area_m2": float(df["floor_area_m2"].median()),58 "median_land_area_m2": float(df["land_area_m2"].median()),59 "median_year_built": float(df["year_built"].median()),60 "median_roll_total_value": float(df["roll_total_value"].median()),61 "share_prior_sale": float(df["prior_sale_price"].is_not_null().mean()),62 "share_rows": df.height / analytic.height,63 }6465 rows = [summarise(analytic.filter(pl.col("property_class") == c), c) for c in CLASS_ORDER]66 rows.append(summarise(analytic, "total"))67 return pl.DataFrame(rows)686970def write_descriptives_tex(desc: pl.DataFrame, path: Path) -> None:71 lines = [72 "% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca",73 "% Generated by scripts/15_descriptives.py — do not edit by hand.",74 "\\begin{tabular}{@{}lrrrrrrrr@{}}",75 "\\toprule",76 " & & Median & \\multicolumn{2}{c}{Floor area (m$^2$)} & Land area & Year & Roll value & Prior sale \\\\",77 "\\cmidrule(lr){4-5}",78 "Class & $n$ & price (CAD) & mean & median & median (m$^2$) & built & median (CAD) & (\\%) \\\\",79 "\\midrule",80 ]81 for r in desc.iter_rows(named=True):82 if r["property_class"] == "total":83 lines.append("\\midrule")84 lines.append(85 f"{CLASS_LABEL[r['property_class']]} & {_fmt_int(r['n'])} & {_fmt_int(r['median_price'])} & "86 f"{_fmt_1(r['mean_floor_area_m2'])} & {_fmt_1(r['median_floor_area_m2'])} & {_fmt_int(r['median_land_area_m2'])} & "87 f"{_fmt_year(r['median_year_built'])} & {_fmt_int(r['median_roll_total_value'])} & {_fmt_pct(r['share_prior_sale'])} \\\\"88 )89 lines += ["\\bottomrule", "\\end{tabular}"]90 path.write_text("\n".join(lines) + "\n", encoding="utf-8")919293def splits_by_region(analytic: pl.DataFrame, splits: pl.DataFrame) -> pl.DataFrame:94 joined = analytic.select("tx_id", "region_group").join(splits.select("tx_id", "split"), on="tx_id", how="inner")95 counts = joined.group_by("split", "region_group").len().rename({"len": "n"})96 table = {(r["split"], r["region_group"]): r["n"] for r in counts.iter_rows(named=True)}97 rows = []98 for s in SPLIT_ORDER + ["total"]:99 row = {"split": s}100 for g in REGION_ORDER + ["total"]:101 if s == "total" and g == "total":102 v = sum(table.values())103 elif s == "total":104 v = sum(n for (ss, gg), n in table.items() if gg == g)105 elif g == "total":106 v = sum(n for (ss, gg), n in table.items() if ss == s)107 else:108 v = table.get((s, g), 0)109 row[g] = v110 rows.append(row)111 return pl.DataFrame(rows)112113114def write_splits_tex(tab: pl.DataFrame, windows: dict[str, tuple[str, str]], path: Path) -> None:115 lines = [116 "% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca",117 "% Generated by scripts/15_descriptives.py — do not edit by hand.",118 "\\begin{tabular}{@{}llrrrrr@{}}",119 "\\toprule",120 "Partition & Sale dates & " + " & ".join(REGION_LABEL[g] for g in REGION_ORDER) + " & Total \\\\",121 "\\midrule",122 ]123 for r in tab.iter_rows(named=True):124 s = r["split"]125 if s == "total":126 lines.append("\\midrule")127 window = "2021-01-04 to 2026-10-07"128 else:129 window = f"{windows[s][0]} to {windows[s][1]}"130 cells = " & ".join(_fmt_int(r[g]) for g in REGION_ORDER + ["total"])131 lines.append(f"{SPLIT_LABEL[s]} & {window} & {cells} \\\\")132 lines += ["\\bottomrule", "\\end{tabular}"]133 path.write_text("\n".join(lines) + "\n", encoding="utf-8")134135136def main() -> None:137 ap = argparse.ArgumentParser()138 ap.add_argument("--synthetic", action="store_true")139 args = ap.parse_args()140141 from agentic_valuation.config import load_config142143 base = ROOT / ("data/synthetic" if args.synthetic else "data/processed")144 suffix = "_synthetic" if args.synthetic else ""145 analytic = pl.read_parquet(base / "analytic.parquet")146 splits = pl.read_parquet(base / "splits.parquet")147 cfg = load_config("data")148 windows = {k: (v["start"], v["end"]) for k, v in cfg["splits"].items() if isinstance(v, dict) and "start" in v}149150 out_res = ROOT / "results" / "tables"151 out_tex = ROOT / "paper" / "tables"152 out_res.mkdir(parents=True, exist_ok=True)153 out_tex.mkdir(parents=True, exist_ok=True)154155 desc = descriptives(analytic)156 desc.write_csv(out_res / f"descriptives{suffix}.csv")157 write_descriptives_tex(desc, out_tex / f"tab_descriptives{suffix}.tex")158159 sbr = splits_by_region(analytic, splits)160 sbr.write_csv(out_res / f"splits_by_region{suffix}.csv")161 write_splits_tex(sbr, windows, out_tex / f"tab_splits{suffix}.tex")162163 print(desc)164 print(sbr)165 print(f"wrote {out_res / f'descriptives{suffix}.csv'}, {out_tex / f'tab_descriptives{suffix}.tex'}")166 print(f"wrote {out_res / f'splits_by_region{suffix}.csv'}, {out_tex / f'tab_splits{suffix}.tex'}")167168169if __name__ == "__main__":170 main()171