"""LaTeX table fragments (booktabs, no vertical rules) and the numbers.tex macro file, generated from saved results only.
Every number that appears in the manuscript text comes from paper/tables/numbers.tex."""
from __future__ import annotations

import json
import re
from pathlib import Path

import numpy as np
import pandas as pd
import yaml

ROOT = Path(__file__).resolve().parents[2]
P = ROOT / "data" / "processed"
T = ROOT / "results" / "tables"
OUT = ROOT / "results" / "tables" / "tex"
OUT.mkdir(parents=True, exist_ok=True)
NUM = {}


def esc(s) -> str:
    s = str(s).replace("−", "$-$").replace("→", "$\\rightarrow$").replace("≈", "$\\approx$").replace("×", "$\\times$").replace("≥", "$\\geq$").replace("≤", "$\\leq$")
    for ch in ["&", "%", "_", "#"]:
        s = re.sub(r"(?<!\\)" + re.escape(ch), "\\" + ch, s)
    return s


def fmt(x, nd=1, pct=False, money=False, comma=True):
    if x is None or (isinstance(x, float) and np.isnan(x)):
        return "--"
    if money:
        return f"\\${x:,.0f}"
    if pct:
        return f"{x * 100:.{nd}f}"
    if isinstance(x, (int, np.integer)) or (isinstance(x, float) and x.is_integer() and abs(x) >= 100):
        return f"{int(x):,}" if comma else str(int(x))
    return f"{x:.{nd}f}"


DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}


def key(s) -> str:
    s = str(s).replace("+", "Plus").replace("−", "Minus").replace("-", "Minus")
    return "".join(DIGITS.get(ch, ch) for ch in s)


def macro(name: str, value: str):
    name = key(name)
    name = re.sub(r"[^A-Za-z]", "", name)
    if name in NUM and NUM[name] != value:
        print("WARNING macro collision", name, NUM[name], value)
    NUM[name] = value


def tabular(df: pd.DataFrame, colspec: str, header: list[str] | None = None, note_cols=None) -> str:
    header = header or list(df.columns)
    lines = [f"\\begin{{tabular}}{{{colspec}}}", "\\toprule", " & ".join(esc(h) for h in header) + " \\\\", "\\midrule"]
    for _, r in df.iterrows():
        lines.append(" & ".join(esc(v) for v in r.values) + " \\\\")
    lines += ["\\bottomrule", "\\end{tabular}"]
    return "\n".join(lines)


def write(name: str, content: str):
    (OUT / f"{name}.tex").write_text(content + "\n", encoding="utf-8")


# ------------------------------------------------------------------ T1 exclusions
def t_exclusions():
    df = pd.read_csv(T / "frame_exclusions.csv")
    df["description"] = df["description"].str.replace(r": \d+$", "", regex=True).str.replace("('Maison', 'Détachée', 'Jumelé', 'Maison de ville')", "house types", regex=False)
    d = df[["rule", "description", "removed", "remaining"]].copy()
    d["removed"] = d["removed"].map(lambda v: f"{v:,}"); d["remaining"] = d["remaining"].map(lambda v: f"{v:,}")
    write("frame_exclusions", tabular(d, "lp{10.5cm}rr", ["Rule", "Description", "Removed", "Remaining"]))
    macro("nFrameStart", f"{int(df.iloc[0]['remaining']):,}"); macro("nFrame", f"{int(df.iloc[-1]['remaining']):,}")
    r10 = df[df.rule == "R10"]
    if len(r10):
        macro("nRobotsExcluded", f"{int(r10.iloc[0]['removed']):,}")


# ------------------------------------------------------------------ T2 sample descriptives
def t_sample():
    smp = pd.read_parquet(P / "sample.parquet"); frame = pd.read_parquet(P / "frame.parquet")
    rows = []
    for col, lab in [("cma", "Metropolitan area"), ("age_band", "Building age (years)"), ("price_tier", "Asking-price quartile")]:
        for v in (sorted(frame[col].astype(str).unique()) if col != "cma" else frame[col].value_counts().index):
            rows.append({"Stratum": f"{lab}: {v}", "Frame n": f"{int((frame[col].astype(str) == str(v)).sum()):,}",
                         "Frame %": f"{(frame[col].astype(str) == str(v)).mean() * 100:.1f}",
                         "Sample n": f"{int((smp[col].astype(str) == str(v)).sum()):,}", "Sample %": f"{(smp[col].astype(str) == str(v)).mean() * 100:.1f}"})
    write("sample_strata", tabular(pd.DataFrame(rows), "lrrrr"))
    desc = []
    for col, lab, f in [("price", "Asking price (\\$)", lambda s: s.quantile([.1, .5, .9])), ("roll_floor_area_m2", "Floor area, roll (m$^2$)", lambda s: s.quantile([.1, .5, .9])),
                        ("roll_lot_m2", "Lot area, roll (m$^2$)", lambda s: s.quantile([.1, .5, .9])), ("age_2026", "Age in 2026 (years)", lambda s: s.quantile([.1, .5, .9])),
                        ("roll_land_value", "Roll land value (\\$)", lambda s: s.quantile([.1, .5, .9])), ("roll_building_value", "Roll building value (\\$)", lambda s: s.quantile([.1, .5, .9])),
                        ("n_images", "Images available", lambda s: s.quantile([.1, .5, .9])), ("desc_len", "Description length (chars)", lambda s: s.quantile([.1, .5, .9]))]:
        q = f(smp[col].astype(float)); qf = f(frame[col].astype(float))
        desc.append({"Variable": lab, "Sample P10": fmt(q.iloc[0], 0), "Sample median": fmt(q.iloc[1], 0), "Sample P90": fmt(q.iloc[2], 0), "Frame median": fmt(qf.iloc[1], 0)})
    write("sample_descriptives", tabular(pd.DataFrame(desc), "lrrrr"))
    macro("nSample", f"{len(smp):,}"); macro("nImagesSample", f"{int(pd.read_parquet(P / 'sample_images.parquet').shape[0]):,}")
    macro("medianPriceSample", f"{smp.price.median():,.0f}"); macro("medianAreaSample", f"{smp.roll_floor_area_m2.median():.0f}")
    macro("nStrata", str(smp["stratum"].nunique())); macro("nMunicipalitiesSample", str(smp["roll_municipalite"].nunique()))
    macro("medianImagesSample", f"{smp.n_images.median():.0f}")
    bal = pd.read_csv(T / "robots_exclusion_balance.csv", index_col=0)
    macro("robotsExcludedPriceMedian", f"{bal.loc['price', 'excluded_robots_median']:,.0f}"); macro("robotsRetainedPriceMedian", f"{bal.loc['price', 'retained_median']:,.0f}")
    macro("robotsExcludedLandMedian", f"{bal.loc['roll_land_value', 'excluded_robots_median']:,.0f}"); macro("robotsRetainedLandMedian", f"{bal.loc['roll_land_value', 'retained_median']:,.0f}")
    for c in ["claim_siding", "claim_roof", "claim_basement", "claim_garage", "claim_year_built", "claim_assess_building"]:
        macro("cov" + c.replace("claim_", "").title().replace("_", ""), f"{smp[c].notna().mean() * 100:.0f}")


# ------------------------------------------------------------------ T3 extraction run summary
def t_run():
    rows = []
    usage = pd.read_json(ROOT / "results" / "raw" / "api_usage.jsonl", lines=True)
    price = {"claude-haiku-5-5": (0.10, 0.50), "claude-sonnet-5-5": (2.0, 10.0), "claude-opus-5-5": (4.0, 20.0)}
    total = 0.0
    for tier in ["haiku", "sonnet", "opus"]:
        f = P / f"images_{tier}.parquet"
        if not f.exists():
            continue
        img = pd.read_parquet(f); txt = pd.read_parquet(P / f"text_{tier}.parquet") if (P / f"text_{tier}.parquet").exists() else None
        u = usage[usage.model == f"claude-{tier}-5-5"]
        usd = ((u.input_tokens.sum() * price[f"claude-{tier}-5-5"][0] + u.output_tokens.sum() * price[f"claude-{tier}-5-5"][1]) / 1e6) * 0.5
        total += usd
        err = img[~img.ok]
        rows.append({"Extractor": f"Claude {tier.capitalize()} 5.5", "Listings": f"{img.uid.nunique():,}", "Image calls": f"{len(img):,}",
                     "Succeeded": f"{img.ok.mean() * 100:.1f}\\%", "URL failures": f"{int(err.error_message.fillna('').str.contains('download').sum()):,}",
                     "robots.txt refusals": f"{int(err.error_message.fillna('').str.contains('robots').sum()):,}",
                     "Text calls": f"{len(txt):,}" if txt is not None else "--",
                     "Mean tokens in / out": f"{img.input_tokens.mean():,.0f} / {img.output_tokens.mean():,.0f}", "Cost (US\\$, batch)": f"{usd:,.0f}"})
        macro(f"n{tier.capitalize()}Images", f"{len(img):,}"); macro(f"n{tier.capitalize()}Listings", f"{img.uid.nunique():,}")
        macro(f"ok{tier.capitalize()}Pct", f"{img.ok.mean() * 100:.1f}")
    write("extraction_runs", tabular(pd.DataFrame(rows), "lrrrrrrrr"))
    macro("apiCostUSD", f"{total:,.0f}")


# ------------------------------------------------------------------ T4 agreement
def t_agreement():
    df = pd.read_csv(T / "exp1_agreement.csv")
    df = df[~df.attribute.str.contains("kitchen")]
    d = df.copy()
    d["attribute"] = d["attribute"].str.replace("∈", "in").str.replace("TEXT ", "Text: ")
    out = d[["tier", "channel", "attribute", "n_ref", "coverage", "n_compared", "accuracy", "kappa"]].copy()
    out["coverage"] = out["coverage"].map(lambda v: fmt(v, 1, pct=True)); out["accuracy"] = out["accuracy"].map(lambda v: fmt(v, 1, pct=True))
    out["kappa"] = out["kappa"].map(lambda v: fmt(v, 2)); out["n_ref"] = out["n_ref"].map(lambda v: f"{int(v):,}"); out["n_compared"] = out["n_compared"].map(lambda v: f"{int(v):,}")
    write("agreement", tabular(out, "lllrrrrr", ["Extractor", "Channel", "Attribute vs reference", "Ref. n", "Coverage (\\%)", "Compared", "Agreement (\\%)", "$\\kappa$"]))
    main = "sonnet" if "sonnet" in df.tier.values else df.tier.iloc[0]
    for _, r in df[df.tier == main].iterrows():
        kk = re.sub(r"[^A-Za-z]", "", r["attribute"].split(" vs ")[0].split(" in ")[0].split(" where")[0].replace("TEXT", "Text"))[:28]
        macro(f"acc{kk}", fmt(r["accuracy"], 1, pct=True)); macro(f"cov{kk}", fmt(r["coverage"], 1, pct=True)); macro(f"n{kk}", f"{int(r['n_compared']):,}")
        if not np.isnan(r.get("kappa", np.nan)):
            macro(f"kappa{kk}", fmt(r["kappa"], 2))
        if "accuracy_strict" in r and not np.isnan(r["accuracy_strict"]):
            macro(f"accStrict{kk}", fmt(r["accuracy_strict"], 1, pct=True))
        if "mdape" in r and not np.isnan(r.get("mdape", np.nan)):
            macro(f"mdape{kk}", fmt(r["mdape"], 1)); macro(f"medianRatio{kk}", fmt(r["median_ratio"], 3))


# ------------------------------------------------------------------ T5 channels
def t_channels():
    df = pd.read_csv(T / "exp2_channels.csv")
    main = "sonnet" if "sonnet" in df.tier.values else df.tier.iloc[0]
    d = df[df.tier == main].copy()
    for c in ["visual_coverage", "text_coverage", "either_coverage", "agreement_when_both", "visual_only_share", "text_only_share"]:
        d[c] = d[c].map(lambda v: fmt(v, 1, pct=True))
    write("channels", tabular(d[["attribute", "visual_coverage", "text_coverage", "either_coverage", "visual_only_share", "text_only_share", "both", "agreement_when_both"]], "lrrrrrrr",
                              ["Attribute", "Images (\\%)", "Text (\\%)", "Either (\\%)", "Images only (\\%)", "Text only (\\%)", "Both n", "Agreement (\\%)"]))
    for _, r in df[df.tier == main].iterrows():
        k = key(r["attribute"].title())
        macro(f"chanVis{k}", fmt(r["visual_coverage"], 0, pct=True)); macro(f"chanTxt{k}", fmt(r["text_coverage"], 0, pct=True))
        macro(f"chanAgree{k}", fmt(r["agreement_when_both"], 0, pct=True)); macro(f"chanEither{k}", fmt(r["either_coverage"], 0, pct=True))


# ------------------------------------------------------------------ T6 image count
def t_image_count():
    df = pd.read_csv(T / "exp3_image_count.csv")
    d = df.copy()
    for c in ["share_with_kitchen", "share_with_bathroom", "share_with_basement", "share_with_exterior", "grade_available", "grade_stable_vs_full", "cladding_stable_vs_full"]:
        d[c] = d[c].map(lambda v: fmt(v, 0, pct=True))
    d["condition_mad_vs_full"] = d["condition_mad_vs_full"].map(lambda v: fmt(v, 2))
    write("image_count", tabular(d[["tier", "K", "share_with_exterior", "share_with_kitchen", "share_with_bathroom", "share_with_basement", "grade_available", "grade_stable_vs_full", "condition_mad_vs_full"]],
                                 "lrrrrrrrr", ["Extractor", "K", "Exterior (\\%)", "Kitchen (\\%)", "Bathroom (\\%)", "Basement (\\%)", "Grade avail. (\\%)", "Grade = full (\\%)", "Cond. MAD"]))
    for _, r in df.iterrows():
        macro(f"k{r['tier']}{int(r['K'])}Basement", fmt(r["share_with_basement"], 1, pct=True)); macro(f"k{r['tier']}{int(r['K'])}Kitchen", fmt(r["share_with_kitchen"], 1, pct=True))
        macro(f"k{r['tier']}{int(r['K'])}Bathroom", fmt(r["share_with_bathroom"], 1, pct=True)); macro(f"k{r['tier']}{int(r['K'])}Grade", fmt(r["grade_available"], 1, pct=True))


# ------------------------------------------------------------------ T7 cross-model
def t_cross_model():
    f = T / "exp4_cross_model.csv"
    if not f.exists():
        return
    df = pd.read_csv(f)
    cols = [c for c in df.columns if c.startswith(("agree_", "kappa_", "corr_"))]
    d = df[["attribute", "n_listings", "alpha_nominal"] + cols].copy()
    for c in ["alpha_nominal"] + cols:
        d[c] = d[c].map(lambda v: fmt(v, 2)) if not c.startswith("agree_") else d[c].map(lambda v: fmt(v, 0, pct=True))
    d["attribute"] = d["attribute"].str.replace("_", " ")
    hdr = ["Attribute", "n", "$\\alpha$"] + [c.replace("agree_", "agree. (\\%) ").replace("kappa_", "$\\kappa$ ").replace("corr_", "r ").replace("_", "–") for c in cols]
    write("cross_model", tabular(d, "lr" + "r" * (len(cols) + 1), hdr))
    R = json.load(open(ROOT / "results" / "raw" / "exp_agreement.json"))
    sa = R.get("exp4_scene_agreement", {})
    if sa:
        macro("sceneAgreement", fmt(sa["agreement"], 1, pct=True)); macro("sceneKappa", fmt(sa["kappa"], 2)); macro("nSceneImages", f"{sa['n_images']:,}")
    for _, r in df.iterrows():
        k = key(r["attribute"].title())
        if not np.isnan(r["alpha_nominal"]):
            macro(f"alpha{k}", fmt(r["alpha_nominal"], 2))
        for c in cols:
            if not np.isnan(r[c]):
                macro(key(c.title()) + k, fmt(r[c], 1, pct=True) if c.startswith("agree_") else fmt(r[c], 2))


# ------------------------------------------------------------------ T8 scene presence (MNAR)
def t_scene_presence():
    df = pd.read_csv(T / "exp9_scene_presence.csv")
    main = "sonnet" if "sonnet" in df.tier.values else df.tier.iloc[0]
    d = df[df.tier == main].copy()
    out = pd.DataFrame({"Room": d["scene"], "Shown (\\%)": d["share_shown"].map(lambda v: fmt(v, 1, pct=True)),
                        "Median price, shown": d["shown_median_price"].map(lambda v: f"{v:,.0f}"), "Median price, not shown": d["notshown_median_price"].map(lambda v: f"{v:,.0f}"),
                        "Mean age, shown": d["shown_mean_age"].map(lambda v: fmt(v, 0)), "Mean age, not shown": d["notshown_mean_age"].map(lambda v: fmt(v, 0)),
                        "Condition, shown": d["shown_cond"].map(lambda v: fmt(v, 2)), "Condition, not shown": d["notshown_cond"].map(lambda v: fmt(v, 2))})
    write("scene_presence", tabular(out, "lrrrrrrr"))
    for _, r in d.iterrows():
        macro(f"shown{r['scene'].title()}", fmt(r["share_shown"], 1, pct=True))
        if r["scene"] == "basement":
            for lab in ["finished", "partially_finished", "unfinished"]:
                if not np.isnan(r.get(f"shown_given_{lab}", np.nan)):
                    macro("basementShownGiven" + lab.title().replace("_", ""), fmt(r[f"shown_given_{lab}"], 1, pct=True)); macro("nBasementRef" + lab.title().replace("_", ""), f"{int(r[f'n_{lab}'])}")
    pos = pd.read_csv(T / "exp9_first_position.csv", index_col=0)
    for sc in ["kitchen", "bathroom", "basement", "exterior_front", "living_or_dining"]:
        if sc in pos.index:
            macro("firstPos" + sc.title().replace("_", ""), fmt(pos.loc[sc, "50%"] + 1, 0)); macro("firstPosMean" + sc.title().replace("_", ""), fmt(pos.loc[sc, "mean"] + 1, 1))


# ------------------------------------------------------------------ T8b MNAR simulation + error taxonomy
def t_mnar_sim():
    f = T / "exp9_simulation.csv"
    if f.exists():
        r = pd.read_csv(f).iloc[0]
        macro("mnarSimDraws", str(int(r["n_draws"]))); macro("mnarSimDropped", fmt(r["share_images_dropped"], 0, pct=True))
        macro("mnarSimCondShift", f"{r['cond_shift']:+.2f}"); macro("mnarSimGradeUp", fmt(r["share_grade_up"], 1, pct=True)); macro("mnarSimGradeDown", fmt(r["share_grade_down"], 1, pct=True))
        macro("mnarSimGradeAvail", fmt(r["grade_available"], 0, pct=True)); macro("mnarSimGradeAvailFull", fmt(r["grade_available_full"], 0, pct=True))
        macro("mnarSimVRatio", fmt(r["median_V_ratio"], 3)); macro("mnarSimVRatioPct", f"{(r['median_V_ratio'] - 1) * 100:+.1f}")
    f = T / "exp1_error_taxonomy.csv"
    if f.exists():
        df = pd.read_csv(f)
        out = pd.DataFrame({"Attribute": df["attribute"], "Category of disagreement": df["category"], "n": df["n"], "Share (\\%)": df["share_of_disagreements"].map(lambda v: fmt(v, 0, pct=True))})
        write("error_taxonomy", tabular(out, "llrr"))
        R = json.load(open(ROOT / "results" / "raw" / "exp_agreement.json")).get("exp1_taxonomy", {})
        for k, v in R.items():
            macro("tax" + key(k.title()), f"{v:,}")


# ------------------------------------------------------------------ T9 calibration
def t_calibration():
    df = pd.read_csv(T / "exp12_calibration.csv")
    d = df.copy()
    for c in ["conf_mean", "roof_acc", "clad_acc"]:
        d[c] = d[c].map(lambda v: fmt(v, 2))
    write("confidence_calibration", tabular(d[["tier", "conf_bin", "n", "conf_mean", "roof_n", "roof_acc", "clad_n", "clad_acc"]], "llrrrrrr",
                                            ["Extractor", "Confidence bin", "Images", "Mean conf.", "Roof n", "Roof agreement", "Cladding n", "Cladding agreement"]))
    sel = pd.read_csv(T / "exp12_selective.csv")
    for _, r in sel.iterrows():
        macro(f"sel{r['tier']}{str(r['threshold']).replace('.', '')}Kept", fmt(r["kept_share"], 0, pct=True)); macro(f"sel{r['tier']}{str(r['threshold']).replace('.', '')}Acc", fmt(r["roof_accuracy"], 1, pct=True))


# ------------------------------------------------------------------ T10 levels summary
def t_levels():
    df = pd.read_csv(T / "exp5_levels_summary.csv")
    main = "sonnet" if "sonnet" in df.tier.values else df.tier.iloc[0]
    d = df[df.tier == main].copy()
    out = pd.DataFrame({"Level": d["level"], "Anchor": d["anchor"], "Median $V$ (\\$)": d["V_median"].map(lambda v: f"{v:,.0f}"),
                        "Land share": d["L_share_median"].map(lambda v: fmt(v, 2)), "Median $D$": d["D_median"].map(lambda v: fmt(v, 2)),
                        "Assessor ratio": d["assessor_median_ratio"].map(lambda v: fmt(v, 2)), "Assessor MdAPE": d["assessor_mdape"].map(lambda v: fmt(v, 1)),
                        "Price ratio": d["price_median_ratio"].map(lambda v: fmt(v, 2)),
                        "Price MdAPE [95\\% CI]": [f"{m:.1f} [{lo:.1f}, {hi:.1f}]" for m, lo, hi in zip(d["price_mdape"], d["price_mdape_lo"], d["price_mdape_hi"])],
                        "Within 20\\%": d["price_within_20pct"].map(lambda v: fmt(v, 0, pct=True))})
    write("levels_summary", tabular(out, "llrrrrrrrr"))
    for _, r in df.iterrows():
        k = f"{r['tier']}{r['level']}{r['anchor']}"
        macro(f"V{k}", f"{r['V_median']:,.0f}"); macro(f"Dmed{k}", fmt(r["D_median"], 2)); macro(f"landShare{k}", fmt(r["L_share_median"], 2))
        macro(f"priceRatio{k}", fmt(r["price_median_ratio"], 2)); macro(f"priceMdape{k}", fmt(r["price_mdape"], 1)); macro(f"priceWithinTwenty{k}", fmt(r["price_within_20pct"], 0, pct=True))
        macro(f"priceMdapeLo{k}", fmt(r["price_mdape_lo"], 1)); macro(f"priceMdapeHi{k}", fmt(r["price_mdape_hi"], 1))
        macro(f"assRatio{k}", fmt(r["assessor_median_ratio"], 2)); macro(f"assMdape{k}", fmt(r["assessor_mdape"], 1)); macro(f"assCod{k}", fmt(r["assessor_cod"], 1))
        macro(f"effAge{k}", fmt(r["eff_age_median"], 0)); macro(f"RCN{k}", f"{r['RCN_median']:,.0f}")
    # all tiers compact (anchor B)
    d2 = df[df.anchor == "B"]
    out2 = pd.DataFrame({"Extractor": d2["tier"], "Level": d2["level"], "Median $V$ (\\$)": d2["V_median"].map(lambda v: f"{v:,.0f}"), "Median $D$": d2["D_median"].map(lambda v: fmt(v, 2)),
                         "Price ratio": d2["price_median_ratio"].map(lambda v: fmt(v, 2)), "Price MdAPE": d2["price_mdape"].map(lambda v: fmt(v, 1)), "Within 20\\%": d2["price_within_20pct"].map(lambda v: fmt(v, 0, pct=True)),
                         "Assessor ratio": d2["assessor_median_ratio"].map(lambda v: fmt(v, 2)), "Assessor MdAPE": d2["assessor_mdape"].map(lambda v: fmt(v, 1))})
    write("levels_by_tier", tabular(out2, "llrrrrrrr"))


# ------------------------------------------------------------------ T11 sensitivity
def t_sensitivity():
    df = pd.read_csv(T / "exp7_8_sensitivity.csv")
    base = df.iloc[0]
    out = pd.DataFrame({"Scenario": df["scenario"], "Median $V$ (\\$)": df["V_median"].map(lambda v: f"{v:,.0f}"),
                        "$\\Delta$ vs baseline (\\%)": ((df["V_median"] / base["V_median"] - 1) * 100).map(lambda v: f"{v:+.1f}"),
                        "Median $D$": df["D_median"].map(lambda v: fmt(v, 2)), "Assessor ratio": df["assessor_median_ratio"].map(lambda v: fmt(v, 2)),
                        "Price ratio": df["price_median_ratio"].map(lambda v: fmt(v, 2)), "Price MdAPE": df["price_mdape"].map(lambda v: fmt(v, 1))})
    write("sensitivity", tabular(out, "lrrrrrr"))
    for _, r in df.iterrows():
        k = key(r["scenario"].title())[:40]
        macro(f"sens{k}Delta", f"{(r['V_median'] / base['V_median'] - 1) * 100:+.1f}"); macro(f"sens{k}PriceRatio", fmt(r["price_median_ratio"], 2)); macro(f"sens{k}AssRatio", fmt(r["assessor_median_ratio"], 2))
        macro(f"sens{k}PriceMdape", fmt(r["price_mdape"], 1))


# ------------------------------------------------------------------ T12 calibration/holdout
def t_holdout():
    df = pd.read_csv(T / "exp10_calibration.csv")
    params = (df["parameters"].str.replace("→", "to", regex=False).str.replace("×", "x", regex=False).str.replace("≈", "approx.", regex=False)
              .str.replace("$", "", regex=False).str.replace("RCN level-only calibration", "level-only", regex=False).str.replace("life-only calibration", "life-only", regex=False)
              .str.replace("joint calibration", "joint", regex=False).str.replace("public parameters", "public", regex=False))
    out = pd.DataFrame({"Anchor": df["anchor"], "Level": df["level"], "Set": df["set"], "Parameters": params,
                        "n": df["n"], "Median $D$": df["D_median"].map(lambda v: fmt(v, 2)), "Assessor ratio": df["assessor_median_ratio"].map(lambda v: fmt(v, 2)),
                        "Assessor MdAPE": df["assessor_mdape"].map(lambda v: fmt(v, 1)), "Price ratio": df["price_median_ratio"].map(lambda v: fmt(v, 2)), "Price MdAPE": df["price_mdape"].map(lambda v: fmt(v, 1))})
    txt = tabular(out, "lllp{5.2cm}rrrrrr")
    write("calibration_holdout", txt)
    for _, r in df.iterrows():
        k = f"{r['anchor']}{r['level']}{r['set'].title()}" + ("Pub" if r["parameters"].startswith("public") else "Life" if r["parameters"].startswith("life") else "Level" if r["parameters"].startswith("RCN") else "Joint")
        macro(f"cal{k}AssRatio", fmt(r["assessor_median_ratio"], 2)); macro(f"cal{k}PriceRatio", fmt(r["price_median_ratio"], 2)); macro(f"cal{k}PriceMdape", fmt(r["price_mdape"], 1))
        macro(f"cal{k}LevelFactor", fmt(r["level_factor"], 2)); macro(f"cal{k}Life", fmt(r["economic_life"], 0)); macro(f"cal{k}N", f"{int(r['n'])}")
        macro(f"cal{k}ImpliedSqft", f"{r['implied_standard_per_sqft']:.0f}"); macro(f"cal{k}AssMdape", fmt(r["assessor_mdape"], 1))


# ------------------------------------------------------------------ T13 subgroups
def t_subgroups():
    df = pd.read_csv(T / "exp10_11_subgroups.csv")
    main = "sonnet" if "sonnet" in df.tier.values else df.tier.iloc[0]
    d = df[(df.tier == main) & (df.level == "B3") & (df.group.isin(["cma", "age_band", "price_tier", "roll_vintage"]))].copy()
    d0 = df[(df.tier == main) & (df.level == "B0")].set_index(["group", "value"])
    d["price_mdape_B0"] = [d0.loc[(g, v), "price_mdape"] if (g, v) in d0.index else np.nan for g, v in zip(d.group, d.value)]
    out = pd.DataFrame({"Group": d["group"].str.replace("_", " "), "Value": d["value"], "n": d["n"], "Assessor ratio": d["assessor_median_ratio"].map(lambda v: fmt(v, 2)),
                        "Assessor COD": d["assessor_cod"].map(lambda v: fmt(v, 1)), "Price ratio": d["price_median_ratio"].map(lambda v: fmt(v, 2)),
                        "Price MdAPE B3": d["price_mdape"].map(lambda v: fmt(v, 1)), "Price MdAPE B0": d["price_mdape_B0"].map(lambda v: fmt(v, 1))})
    write("subgroups", tabular(out, "llrrrrrr"))
    for _, r in d.iterrows():
        k = key(f"{r['group']}{str(r['value']).title()}")
        macro(f"sub{k}PriceMdape", fmt(r["price_mdape"], 1)); macro(f"sub{k}PriceRatio", fmt(r["price_median_ratio"], 2)); macro(f"sub{k}AssRatio", fmt(r["assessor_median_ratio"], 2)); macro(f"sub{k}N", f"{int(r['n'])}")


# ------------------------------------------------------------------ T14 scenarios
def t_scenarios():
    df = pd.read_csv(T / "exp13_scenarios.csv")
    out = pd.DataFrame({"Scenario (counterfactual, labelled)": df["scenario"], "Listings affected (\\%)": df["share_affected"].map(lambda v: fmt(v, 0, pct=True)),
                        "Median $\\Delta V$, affected (\\%)": df["median_delta_V_pct_affected"].map(lambda v: f"{v:+.1f}"),
                        "Median $\\Delta$ improvements, affected (\\%)": df["median_delta_improvements_pct_affected"].map(lambda v: f"{v:+.1f}"),
                        "Median $\\Delta V$, affected (\\$)": df["median_delta_V_cad_affected"].map(lambda v: f"{v:+,.0f}")})
    write("scenarios", tabular(out, "lrrrr"))
    for _, r in df.iterrows():
        k = key(r["scenario"].title())
        macro(f"scen{k}V", f"{r['median_delta_V_pct_affected']:+.1f}"); macro(f"scen{k}Impr", f"{r['median_delta_improvements_pct_affected']:+.1f}")
        macro(f"scen{k}Cad", f"{r['median_delta_V_cad_affected']:+,.0f}"); macro(f"scen{k}Share", fmt(r["share_affected"], 0, pct=True))


# ------------------------------------------------------------------ T15 Monte Carlo
def t_mc():
    R = json.load(open(ROOT / "results" / "raw" / "exp_cost.json"))["exp14"]
    oat = pd.read_csv(T / "exp14_oat.csv", index_col=0)
    out = pd.DataFrame({"Factor (one at a time)": oat.index, "Median $|\\Delta V|$ (\\%)": oat.iloc[:, 0].map(lambda v: fmt(v, 1))})
    write("mc_oat", tabular(out, "lr"))
    macro("mcDraws", str(R["n_draws"])); macro("mcRelWidth", fmt(R["rel_width_median"], 0, pct=True)); macro("mcRelWidthLo", fmt(R["rel_width_p25_p75"][0], 0, pct=True)); macro("mcRelWidthHi", fmt(R["rel_width_p25_p75"][1], 0, pct=True))
    macro("mcPriceCoverage", fmt(R["price_coverage_80"], 0, pct=True)); macro("mcAssessorCoverage", fmt(R["assessor_total_coverage_80"], 0, pct=True))
    for k, v in R["one_at_a_time_median_abs_pct"].items():
        macro("oat" + key(k.title()), fmt(v, 1))


# ------------------------------------------------------------------ T16 market alignment & sales
def t_market():
    df = pd.read_csv(T / "exp15_market_alignment.csv")
    out = pd.DataFrame({"Indication": df["level"].str.replace("benchmark:", "Benchmark: ").str.replace("_", "\\_"), "n": df["n"],
                        "Median ratio to price": df["price_median_ratio"].map(lambda v: fmt(v, 2)), "MdAPE": df["price_mdape"].map(lambda v: fmt(v, 1)),
                        "P10": df["price_p10"].map(lambda v: fmt(v, 2)), "P90": df["price_p90"].map(lambda v: fmt(v, 2)), "Within 20\\%": df["price_within_20pct"].map(lambda v: fmt(v, 0, pct=True)),
                        "Ratio to VP cost est.": df["vp_cost_est_median_ratio"].map(lambda v: fmt(v, 2)) if "vp_cost_est_median_ratio" in df else "--"})
    write("market_alignment", tabular(out, "lrrrrrrr"))
    for _, r in df.iterrows():
        k = key(r["level"].title())
        macro(f"mkt{k}Ratio", fmt(r["price_median_ratio"], 2)); macro(f"mkt{k}Mdape", fmt(r["price_mdape"], 1)); macro(f"mkt{k}Within", fmt(r["price_within_20pct"], 0, pct=True))
        if "vp_cost_est_median_ratio" in r and not np.isnan(r.get("vp_cost_est_median_ratio", np.nan)):
            macro(f"mkt{k}VpCostRatio", fmt(r["vp_cost_est_median_ratio"], 2))
    ds = pd.read_csv(T / "exp15_recorded_sales.csv")
    out = pd.DataFrame({"Set": ds["set"].str.replace("_", " "), "Level": ds["level"], "n": ds["n"], "Median ratio to sale": ds["median_ratio"].map(lambda v: fmt(v, 2)),
                        "MdAPE": ds["mdape"].map(lambda v: fmt(v, 1)), "Within 20\\%": ds["within_20pct"].map(lambda v: fmt(v, 0, pct=True)), "Asking / sale (median)": ds["ask_vs_sale_median"].map(lambda v: fmt(v, 3))})
    write("recorded_sales", tabular(out, "llrrrrr"))
    for _, r in ds.iterrows():
        k = key(r['set'].title() + r['level'])
        macro(f"sale{k}Ratio", fmt(r["median_ratio"], 2)); macro(f"sale{k}Mdape", fmt(r["mdape"], 1)); macro(f"sale{k}N", f"{int(r['n'])}"); macro(f"sale{k}AskSale", fmt(r["ask_vs_sale_median"], 3))


# ------------------------------------------------------------------ T17 review load
def t_review():
    df = pd.read_csv(T / "exp16_review_load.csv")
    out = pd.DataFrame({"Extractor": df["tier"], "Rule": df["rule"], "Flagged (\\%)": df["share_flagged"].map(lambda v: fmt(v, 1, pct=True)),
                        "No grade (\\%)": df["share_no_grade"].map(lambda v: fmt(v, 1, pct=True)), "Defects (\\%)": df["share_defects"].map(lambda v: fmt(v, 1, pct=True))})
    write("review_load", tabular(out, "llrrr"))
    for _, r in df.iterrows():
        macro(f"review{r['tier']}{re.sub(r'[^0-9]', '', r['rule'])}", fmt(r["share_flagged"], 0, pct=True))


# ------------------------------------------------------------------ T18 cost schedule
def t_schedule():
    cfg = yaml.safe_load(open(ROOT / "configs" / "cost_schedule.yaml"))
    rows = []
    for anc in ["B", "A"]:
        a = cfg["rcn"]["anchors"][anc]
        for g in ["basic", "standard", "upgraded", "luxury"]:
            lo, hi = a["grade_sensitivity_per_sqft"][g]
            rows.append({"Anchor": anc, "Grade": g, "\\$/sq ft (point)": a["grade_per_sqft"][g], "Band": f"{lo}–{hi}", "Reference storeys": a["reference_storeys"]})
    write("schedule_grades", tabular(pd.DataFrame(rows), "llrrr"))
    rows = []
    for anc in ["B", "A"]:
        a = cfg["rcn"]["anchors"][anc]
        rows.append({"Parameter": f"Storey multiplier, anchor {anc}", "Value": ", ".join(f"{k}: {v}" for k, v in a["storey_multiplier"].items()), "Source / status": "industry bungalow premium (B); assumption (A)"})
    r = cfg["rcn"]
    rows += [{"Parameter": "Attached-house multiplier", "Value": ", ".join(f"{k}: {v}" for k, v in r["attached_multiplier"].items()), "Source / status": "assumption"},
             {"Parameter": "Basement finish (\\$/sq ft of footprint)", "Value": ", ".join(f"{k}: {v}" for k, v in r["basement_finish_per_sqft"].items()), "Source / status": "assumption, $\\pm$30\\%"},
             {"Parameter": "Garage (\\$)", "Value": ", ".join(f"{k}: {v:,}" for k, v in r["garage"].items() if v), "Source / status": "assumption, $\\pm$30\\% (Dessins Drummond: single 25–50k)"},
             {"Parameter": "Soft costs / entrepreneurial incentive", "Value": f"{r['soft_cost_share']:.0%} / {r['entrepreneurial_incentive']:.0%}", "Source / status": "assumption; ranges 5–15\\% / 0–15\\%"},
             {"Parameter": "Site improvements", "Value": f"{cfg['site_improvements']['baseline_share_of_rcn']:.0%} of RCN + pool", "Source / status": "assumption"},
             {"Parameter": "Economic life / max physical depreciation", "Value": f"{cfg['depreciation']['economic_life_years']} y / {cfg['depreciation']['max_physical']:.0%}", "Source / status": "assumption; 50–75 y"},
             {"Parameter": "Condition factor on effective age", "Value": ", ".join(f"{k}: {v}" for k, v in cfg["depreciation"]["condition_factor"].items()), "Source / status": "assumption; strength 0.5–1.5"},
             {"Parameter": "Renovation (years removed)", "Value": f"extensive {cfg['depreciation']['renovation_extensive_years_removed']}, partial {cfg['depreciation']['renovation_partial_years_removed']}", "Source / status": "assumption"},
             {"Parameter": "Land", "Value": "roll land value $\\times$ market index (roll month $\\rightarrow$ 2026-07)", "Source / status": "MAMH roll; Vrai Prix index; share 0–1"},
             {"Parameter": "Temporal / regional index", "Value": "StatCan 18-10-0289 single-detached, by CMA", "Source / status": "official statistics"}]
    write("schedule_parameters", tabular(pd.DataFrame(rows), "p{4.2cm}p{6cm}p{4.3cm}").replace("\\\\$", "\\$"))
    b = cfg["rcn"]["anchors"]["B"]["grade_per_sqft"]; a_ = cfg["rcn"]["anchors"]["A"]["grade_per_sqft"]
    for g in b:
        macro(f"costB{g.title()}", str(b[g])); macro(f"costA{g.title()}", str(a_[g]))
    macro("econLife", str(cfg["depreciation"]["economic_life_years"])); macro("softShare", f"{cfg['rcn']['soft_cost_share'] * 100:.0f}"); macro("eiShare", f"{cfg['rcn']['entrepreneurial_incentive'] * 100:.0f}")


# ------------------------------------------------------------------ literature counts
def t_literature():
    v = pd.read_csv(ROOT / "docs" / "verification" / "verified.csv")
    macro("nRefsVerified", str(len(v))); macro("nRefsDoi", str(int(v["doi"].fillna("").astype(bool).sum())))


def all_tables():
    for fn in [t_exclusions, t_sample, t_run, t_agreement, t_channels, t_image_count, t_cross_model, t_scene_presence, t_mnar_sim, t_calibration,
               t_levels, t_sensitivity, t_holdout, t_subgroups, t_scenarios, t_mc, t_market, t_review, t_schedule, t_literature]:
        try:
            fn(); print("ok ", fn.__name__)
        except Exception as e:  # noqa: BLE001
            print("FAIL", fn.__name__, repr(e))
    lines = [f"\\newcommand{{\\{k}}}{{{v}}}" for k, v in sorted(NUM.items())]
    (OUT / "numbers.tex").write_text("% generated by src/visualization/tables.py — do not edit\n" + "\n".join(lines) + "\n", encoding="utf-8")
    print(len(NUM), "macros")


if __name__ == "__main__":
    all_tables()