#!/usr/bin/env python3 # Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca """Descriptive statistics of the analytic sample for the Data section of the paper. Reads data/processed/analytic.parquet and data/processed/splits.parquet (real data; ``--synthetic`` switches to data/synthetic/ and suffixes the outputs) and writes * results/tables/descriptives.csv — one row per property class + total * paper/tables/tab_descriptives.tex — booktabs ``tabular`` fragment (no caption; wrapped by sections/data.tex) * results/tables/splits_by_region.csv — split × region group counts (+ totals) * paper/tables/tab_splits.tex — booktabs ``tabular`` fragment Numbers are written as plain text (``siunitx`` is loaded with ``parse-numbers=false`` in the paper). """ from __future__ import annotations import argparse import sys from pathlib import Path import polars as pl ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) CLASS_ORDER = ["house", "condo", "plex"] CLASS_LABEL = {"house": "House", "condo": "Condominium", "plex": "Plex", "total": "All classes"} SPLIT_ORDER = ["train", "validation", "test", "holdout"] SPLIT_LABEL = {"train": "Train", "validation": "Validation", "test": "Test", "holdout": "Holdout", "total": "Total"} REGION_ORDER = ["Montréal CMA", "Québec CMA", "Outaouais", "other"] REGION_LABEL = {"Montréal CMA": "Montr\\'eal CMA", "Québec CMA": "Qu\\'ebec CMA", "Outaouais": "Outaouais", "other": "Other regions", "total": "Total"} def _fmt_int(x: float | int | None) -> str: return "--" if x is None else f"{int(round(x)):,}" def _fmt_1(x: float | None) -> str: return "--" if x is None else f"{x:,.1f}" def _fmt_year(x: float | None) -> str: return "--" if x is None else f"{int(round(x))}" def _fmt_pct(x: float | None) -> str: return "--" if x is None else f"{100 * x:.1f}" def descriptives(analytic: pl.DataFrame) -> pl.DataFrame: def summarise(df: pl.DataFrame, label: str) -> dict: return { "property_class": label, "n": df.height, "median_price": float(df["price"].median()), "mean_floor_area_m2": float(df["floor_area_m2"].mean()), "median_floor_area_m2": float(df["floor_area_m2"].median()), "median_land_area_m2": float(df["land_area_m2"].median()), "median_year_built": float(df["year_built"].median()), "median_roll_total_value": float(df["roll_total_value"].median()), "share_prior_sale": float(df["prior_sale_price"].is_not_null().mean()), "share_rows": df.height / analytic.height, } rows = [summarise(analytic.filter(pl.col("property_class") == c), c) for c in CLASS_ORDER] rows.append(summarise(analytic, "total")) return pl.DataFrame(rows) def write_descriptives_tex(desc: pl.DataFrame, path: Path) -> None: lines = [ "% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca", "% Generated by scripts/15_descriptives.py — do not edit by hand.", "\\begin{tabular}{@{}lrrrrrrrr@{}}", "\\toprule", " & & Median & \\multicolumn{2}{c}{Floor area (m$^2$)} & Land area & Year & Roll value & Prior sale \\\\", "\\cmidrule(lr){4-5}", "Class & $n$ & price (CAD) & mean & median & median (m$^2$) & built & median (CAD) & (\\%) \\\\", "\\midrule", ] for r in desc.iter_rows(named=True): if r["property_class"] == "total": lines.append("\\midrule") lines.append( f"{CLASS_LABEL[r['property_class']]} & {_fmt_int(r['n'])} & {_fmt_int(r['median_price'])} & " f"{_fmt_1(r['mean_floor_area_m2'])} & {_fmt_1(r['median_floor_area_m2'])} & {_fmt_int(r['median_land_area_m2'])} & " f"{_fmt_year(r['median_year_built'])} & {_fmt_int(r['median_roll_total_value'])} & {_fmt_pct(r['share_prior_sale'])} \\\\" ) lines += ["\\bottomrule", "\\end{tabular}"] path.write_text("\n".join(lines) + "\n", encoding="utf-8") def splits_by_region(analytic: pl.DataFrame, splits: pl.DataFrame) -> pl.DataFrame: joined = analytic.select("tx_id", "region_group").join(splits.select("tx_id", "split"), on="tx_id", how="inner") counts = joined.group_by("split", "region_group").len().rename({"len": "n"}) table = {(r["split"], r["region_group"]): r["n"] for r in counts.iter_rows(named=True)} rows = [] for s in SPLIT_ORDER + ["total"]: row = {"split": s} for g in REGION_ORDER + ["total"]: if s == "total" and g == "total": v = sum(table.values()) elif s == "total": v = sum(n for (ss, gg), n in table.items() if gg == g) elif g == "total": v = sum(n for (ss, gg), n in table.items() if ss == s) else: v = table.get((s, g), 0) row[g] = v rows.append(row) return pl.DataFrame(rows) def write_splits_tex(tab: pl.DataFrame, windows: dict[str, tuple[str, str]], path: Path) -> None: lines = [ "% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca", "% Generated by scripts/15_descriptives.py — do not edit by hand.", "\\begin{tabular}{@{}llrrrrr@{}}", "\\toprule", "Partition & Sale dates & " + " & ".join(REGION_LABEL[g] for g in REGION_ORDER) + " & Total \\\\", "\\midrule", ] for r in tab.iter_rows(named=True): s = r["split"] if s == "total": lines.append("\\midrule") window = "2021-01-04 to 2026-10-07" else: window = f"{windows[s][0]} to {windows[s][1]}" cells = " & ".join(_fmt_int(r[g]) for g in REGION_ORDER + ["total"]) lines.append(f"{SPLIT_LABEL[s]} & {window} & {cells} \\\\") lines += ["\\bottomrule", "\\end{tabular}"] path.write_text("\n".join(lines) + "\n", encoding="utf-8") def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--synthetic", action="store_true") args = ap.parse_args() from agentic_valuation.config import load_config base = ROOT / ("data/synthetic" if args.synthetic else "data/processed") suffix = "_synthetic" if args.synthetic else "" analytic = pl.read_parquet(base / "analytic.parquet") splits = pl.read_parquet(base / "splits.parquet") cfg = load_config("data") windows = {k: (v["start"], v["end"]) for k, v in cfg["splits"].items() if isinstance(v, dict) and "start" in v} out_res = ROOT / "results" / "tables" out_tex = ROOT / "paper" / "tables" out_res.mkdir(parents=True, exist_ok=True) out_tex.mkdir(parents=True, exist_ok=True) desc = descriptives(analytic) desc.write_csv(out_res / f"descriptives{suffix}.csv") write_descriptives_tex(desc, out_tex / f"tab_descriptives{suffix}.tex") sbr = splits_by_region(analytic, splits) sbr.write_csv(out_res / f"splits_by_region{suffix}.csv") write_splits_tex(sbr, windows, out_tex / f"tab_splits{suffix}.tex") print(desc) print(sbr) print(f"wrote {out_res / f'descriptives{suffix}.csv'}, {out_tex / f'tab_descriptives{suffix}.tex'}") print(f"wrote {out_res / f'splits_by_region{suffix}.csv'}, {out_tex / f'tab_splits{suffix}.tex'}") if __name__ == "__main__": main()