"""Phase F — LaTeX table fragments and number macros generated from pipeline outputs. Writes paper/tables/*.tex and numbers.tex."""
import json, re
from pathlib import Path
import numpy as np, pandas as pd, yaml

ROOT = Path(__file__).resolve().parents[2]; META = ROOT / "data" / "metadata"; FINAL = ROOT / "data" / "final"; ECO = ROOT / "results" / "econometrics"; TAB = ROOT / "paper" / "tables"; TAB.mkdir(parents=True, exist_ok=True)
tax = yaml.safe_load((ROOT / "scripts" / "harmonization" / "taxonomy.yaml").read_text()); LAB = {leaf: v["label"] for g in tax.values() for leaf, v in g.items()}; GRP = {leaf: g for g, grp in tax.items() for leaf in grp}
GRPLAB = {"A_structural": "A. Structural", "B_interior": "B. Interior", "C_exterior": "C. Exterior", "D_location": "D. Location", "E_environment": "E. Environment", "F_neighbourhood": "F. Neighbourhood", "G_market": "G. Market", "H_ai_derived": "H. AI-derived"}
FAM = {"pct_premium": "\\% premium", "semi_elasticity": "\\% per unit", "elasticity": "elasticity"}
MACROS = {}


def esc(s): return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")
def fmt(x, d=2):
    if x is None or (isinstance(x, float) and np.isnan(x)): return "--"
    return f"{x:.{d}f}" if isinstance(x, (float, np.floating)) else str(x)
def macro(name, val, d=2):
    assert name.isalpha(), name; MACROS[name] = val if isinstance(val, str) else (f"{val:,.{d}f}".replace(",", "{,}") if isinstance(val, (float, np.floating)) else f"{val:,}".replace(",", "{,}"))
def longtab(cols, rows, align, name, caption, label):
    out = [f"\\begin{{longtable}}{{{align}}}", f"\\caption{{{caption}}}\\label{{{label}}}\\\\", "\\toprule", " & ".join(cols) + " \\\\", "\\midrule", "\\endfirsthead",
           f"\\multicolumn{{{len(cols)}}}{{l}}{{\\textit{{(continued)}}}}\\\\", "\\toprule", " & ".join(cols) + " \\\\", "\\midrule", "\\endhead", "\\bottomrule", "\\endfoot"]
    out += [" & ".join(r) + " \\\\" for r in rows] + ["\\end{longtable}"]
    (TAB / f"{name}.tex").write_text("\n".join(out) + "\n"); print("longtable", name)


def tabular(cols, rows, align=None, name=None):
    align = align or "l" + "r" * (len(cols) - 1); out = [f"\\begin{{tabular}}{{{align}}}", "\\toprule", " & ".join(cols) + " \\\\", "\\midrule"] + [" & ".join(r) + " \\\\" for r in rows] + ["\\bottomrule", "\\end{tabular}"]
    (TAB / f"{name}.tex").write_text("\n".join(out) + "\n"); print("table", name)


def t01_coverage():
    c = json.load(open(META / "prisma_counts.json")); log = pd.read_csv(META / "search_log.csv")
    rows = [["Records identified (all sources)", f"{c['records_identified_raw']:,}"]] + [[f"\\quad {esc(k)}", f"{v:,}"] for k, v in c["by_source"].items()]
    rows += [["Unique records after deduplication", f"{c['records_after_dedup']:,}"], ["\\quad with an abstract", f"{c['records_with_abstract']:,}"], ["Stage-1 candidates (rule-based)", f"{c['stage1_candidates']:,}"],
             ["Screened by independent agents (title/abstract)", f"{c['screened_by_llm']:,}"], ["\\quad excluded with a coded reason", f"{c['llm_excluded']:,}"], ["\\quad unclear, flagged for review", f"{c['llm_unclear']:,}"],
             ["\\quad included: hedonic coefficient studies", f"{c['included_hedonic_stage1']:,}"], ["\\quad included: ML/AVM predictive studies", f"{c['included_ml_predictive_stage1']:,}"],
             ["Open-access full texts sought (hedonic + unclear)", f"{c['fulltext_sought_hedonic_or_unclear']:,}"], ["\\quad retrieved", f"{c['fulltext_retrieved']:,}"], ["Full texts assessed and extracted", f"{c['fulltext_assessed_extracted']:,}"],
             ["\\quad eligible for coefficient synthesis", f"{c['fulltext_eligible']:,}"], ["Search queries executed", f"{len(log):,}"]]
    tabular(["Stage", "n"], rows, "lr", "tab01_coverage")
    for k in ("records_identified_raw", "records_after_dedup", "records_with_abstract", "stage1_candidates", "screened_by_llm", "llm_excluded", "llm_unclear", "included_hedonic_stage1", "included_ml_predictive_stage1", "fulltext_sought_hedonic_or_unclear", "fulltext_retrieved", "fulltext_assessed_extracted", "fulltext_eligible", "fulltext_ineligible"):
        macro("n" + "".join(w.capitalize() for w in k.replace("1", "_one").split("_")), int(c[k]), 0)
    macro("nQueries", len(log), 0); macro("retrievalRate", 100 * c["fulltext_retrieved"] / max(c["fulltext_sought_hedonic_or_unclear"], 1), 1)


def t02_geography():
    S = pd.read_csv(FINAL / "studies.csv"); S = S[S.eligible == True]; E = pd.read_csv(FINAL / "effects.csv")
    llm = pd.read_csv(META / "screening_stage1_llm.csv"); g1 = llm[llm.reason_code == "ELIGIBLE_HEDONIC"].country_guess.fillna("unknown").map(lambda c: str(c).split(",")[0].strip()).value_counts()
    g2 = S.country.value_counts(); g3 = E.groupby("country").size()
    rows = []
    for ctry in g1.index[:25]:
        rows.append([esc(ctry), f"{int(g1[ctry]):,}", f"{int(g2.get(ctry, 0)):,}", f"{int(g3.get(ctry, 0)):,}"])
    tabular(["Country", "stage-1 included records", "extracted studies", "usable estimates"], rows, "lrrr", "tab02_geography")
    cont = S.continent.fillna("unknown").value_counts(); rows = [[esc(k), f"{int(v):,}", f"{int(E[E.continent == k].study_id.nunique()):,}", f"{int((E.continent == k).sum()):,}"] for k, v in cont.items()]
    tabular(["Continent", "extracted studies", "studies with usable estimates", "usable estimates"], rows, "lrrr", "tab02b_continents")
    macro("nCountriesExtracted", int(S.country.nunique()), 0); macro("nCountriesStageOne", int(len(g1[g1.index != "unknown"])), 0)
    macro("shareUSstageOne", 100 * g1.get("United States", 0) / g1.sum(), 1); macro("shareUSextracted", 100 * g2.get("United States", 0) / max(len(S), 1), 1)


def t03_study_characteristics():
    S = pd.read_csv(FINAL / "studies.csv"); S = S[S.eligible == True]
    def dist(col):
        v = S[col].fillna("unknown").astype(str).value_counts(); return [[esc(k), f"{int(n):,}", f"{100 * n / len(S):.1f}"] for k, n in v.items()]
    rows = [["\\textit{Property type}", "", ""]] + dist("property_type") + [["\\textit{Price measure}", "", ""]] + dist("price_type") + [["\\textit{Document type}", "", ""]] + dist("document_type")
    rows += [["\\textit{Time fixed effects}", "", ""]] + dist("time_fe") + [["\\textit{Location fixed effects}", "", ""]] + dist("location_fe")
    rows += [["Median sample size (observations)", f"{S.n_obs_total.median():,.0f}", ""], ["Median publication year", f"{S.year.median():.0f}", ""], ["Median data-period midpoint", f"{S[['sample_start','sample_end']].apply(pd.to_numeric, errors='coerce').where(lambda d: (d > 1900) & (d < 2027)).mean(axis=1).median():.0f}", ""],
             ["Models per study (median / max)", f"{S.n_models.median():.0f} / {S.n_models.max():.0f}", ""], ["Coefficients per study (median / max)", f"{S.n_coefficients.median():.0f} / {S.n_coefficients.max():.0f}", ""]]
    tabular(["Characteristic", "studies", "\\%"], rows, "lrr", "tab03_study_characteristics")
    macro("nStudiesExtracted", int(len(S)), 0); E = pd.read_csv(FINAL / "effects.csv"); macro("nUsableEstimates", int(len(E)), 0); macro("nUsableStudies", int(E.study_id.nunique()), 0); macro("nModelsExtracted", int(S.n_models.sum()), 0); macro("nCoefExtracted", int(S.n_coefficients.sum()), 0); macro("medianNobs", float(S.n_obs_total.median()), 0)


def t04_taxonomy():
    C = pd.read_csv(FINAL / "coefficients_raw.csv")
    rows = []
    for g, leaves in tax.items():
        rows.append([f"\\textbf{{{GRPLAB[g]}}}", "", ""])
        for leaf, v in leaves.items():
            n = int((C.taxonomy_id == leaf).sum()); k = int(C[C.taxonomy_id == leaf].record_id.nunique())
            rows.append([f"\\quad {esc(v['label'])}", esc(v["family"].replace("_", " ")), f"{k} / {n}"])
    tabular(["Characteristic", "primary effect-size family", "studies / coefficients"], rows, "llr", "tab04_taxonomy")
    longtab(["Characteristic", "primary effect-size family", "studies / coefficients"], rows, "p{6cm}p{4cm}r", "tab04_taxonomy_long", "Global hedonic variable taxonomy and extraction counts (studies / coefficient rows)", "tab:taxonomy")


def t05_most_studied():
    C = pd.read_csv(FINAL / "coefficients_raw.csv"); C = C[C.taxonomy_id != "other"]
    st = C.groupby("taxonomy_id").agg(studies=("record_id", "nunique"), coefs=("record_id", "size"), pos=("beta", lambda b: (b > 0).mean())).sort_values("studies", ascending=False).head(30)
    rows = [[esc(LAB.get(t, t)), esc(GRPLAB.get(GRP.get(t, ""), "")), f"{int(r.studies)}", f"{int(r.coefs)}", f"{100 * r.pos:.0f}"] for t, r in st.iterrows()]
    tabular(["Characteristic", "group", "studies", "coefficients", "\\% positive"], rows, "llrrr", "tab05_most_studied")
    macro("nTaxonomyLeavesObserved", int(C.taxonomy_id.nunique()), 0)


def t06_pooled():
    p = pd.read_csv(ECO / "pooled_effects.csv"); py = pd.read_csv(ECO / "python_pooled.csv") if (ECO / "python_pooled.csv").exists() else None
    for fam, name in (("pct_premium", "tab06_pooled_premiums"), ("semi_elasticity", "tab07_pooled_semi_elasticities"), ("elasticity", "tab07b_pooled_elasticities")):
        s = p[p.family == fam].sort_values("k_studies", ascending=False); rows = []
        for r in s.itertuples():
            rows.append([esc(LAB.get(r.taxonomy_id, r.taxonomy_id)), f"{int(r.k_studies)}", f"{int(r.n_estimates)}", fmt(r.est_re), f"[{fmt(r.ci_lo)}, {fmt(r.ci_hi)}]", f"[{fmt(r.pi_lo)}, {fmt(r.pi_hi)}]", fmt(r.I2, 0), fmt(r.est_ml), fmt(r.se_rve, 3), fmt(r.median_es)])
        tabular(["Characteristic", "$k$", "$n$", "RE estimate", "95\\% CI", "95\\% PI", "$I^2$", "3-level", "RVE SE", "median"], rows, "lrrrrrrrrr", name)
    for r in p.itertuples():
        key = "".join(w.capitalize() for w in r.taxonomy_id.split("_")) + "".join(w.capitalize() for w in r.family.split("_"))
        macro("est" + key, float(r.est_re), 2); macro("lo" + key, float(r.ci_lo), 2); macro("hi" + key, float(r.ci_hi), 2); macro("k" + key, int(r.k_studies), 0); macro("n" + key, int(r.n_estimates), 0); macro("isq" + key, float(r.I2), 0)
    macro("nPooledFamilies", int(len(p)), 0); macro("medianIsq", float(p.I2.median()), 0); macro("minIsq", float(p.I2.min()), 0)


def t08_metaregression():
    f = ECO / "meta_regression.csv"
    if not f.exists(): return
    m = pd.read_csv(f); keys = m.groupby(["family", "taxonomy_id"]).size().index
    terms = [t for t in m.term.unique() if t != "intrcpt"]; rows = []
    for t in terms:
        row = [esc(t)]
        for fam, tx in keys:
            s = m[(m.family == fam) & (m.taxonomy_id == tx) & (m.term == t)]
            row.append(f"{fmt(s.est.iloc[0])} ({fmt(s.se_rve.iloc[0] if pd.notna(s.se_rve.iloc[0]) else s.se.iloc[0], 3)})" + ("$^{*}$" if (s.p_rve.iloc[0] if pd.notna(s.p_rve.iloc[0]) else s.p.iloc[0]) < 0.05 else "") if len(s) else "--")
        rows.append(row)
    rows.append(["$k$ / $n$"] + [f"{int(m[(m.family == fam) & (m.taxonomy_id == tx)].k_studies.iloc[0])} / {int(m[(m.family == fam) & (m.taxonomy_id == tx)].n_estimates.iloc[0])}" for fam, tx in keys])
    tabular(["Moderator"] + [f"{esc(LAB.get(tx, tx))} ({FAM[fam]})" for fam, tx in keys], rows, "l" + "r" * len(keys), "tab08_meta_regression")
    for part, fams in (("a", ("pct_premium",)), ("b", ("semi_elasticity", "elasticity"))):
        kk = [k for k in keys if k[0] in fams]; rows2 = []
        for t in terms:
            row = [esc(t)]
            for fam, tx in kk:
                s_ = m[(m.family == fam) & (m.taxonomy_id == tx) & (m.term == t)]
                row.append(f"{fmt(s_.est.iloc[0])} ({fmt(s_.se_rve.iloc[0] if pd.notna(s_.se_rve.iloc[0]) else s_.se.iloc[0], 3)})" + ("$^{*}$" if (s_.p_rve.iloc[0] if pd.notna(s_.p_rve.iloc[0]) else s_.p.iloc[0]) < 0.05 else "") if len(s_) else "--")
            rows2.append(row)
        rows2.append(["$k$ / $n$"] + [f"{int(m[(m.family == fam) & (m.taxonomy_id == tx)].k_studies.iloc[0])} / {int(m[(m.family == fam) & (m.taxonomy_id == tx)].n_estimates.iloc[0])}" for fam, tx in kk])
        tabular(["Moderator"] + [f"{esc(LAB.get(tx, tx))}" + ("" if part == "a" else f" ({FAM[fam]})") for fam, tx in kk], rows2, "l" + "r" * len(kk), f"tab08{part}_meta_regression")
    macro("nMetaRegModels", int(len(keys)), 0)
    for r in m.itertuples():
        key = "mr" + "".join(w.capitalize() for w in r.taxonomy_id.split("_")) + "".join(w.capitalize() for w in r.family.split("_")) + "".join(w.capitalize() for w in r.term.replace("True", "").split("_"))
        macro(key, float(r.est), 3); macro(key + "P", float(r.p_rve if pd.notna(r.p_rve) else r.p), 3)
    sig = m[(m.term != "intrcpt")].assign(pp=lambda d: d.p_rve.fillna(d.p)); macro("nModTerms", int(len(sig)), 0); macro("nModTermsSig", int((sig.pp < 0.05).sum()), 0)
    for t_ in ("spatial", "ml", "pub_wp", "period", "apt", "rent", "nonUS", "lnN", "time_feTrue", "location_feTrue"):
        sub = sig[sig.term == t_]
        if len(sub): macro("nSig" + "".join(w.capitalize() for w in t_.replace("True", "").split("_")), int((sub.pp < 0.05).sum()), 0); macro("nTested" + "".join(w.capitalize() for w in t_.replace("True", "").split("_")), int(len(sub)), 0)


def t14_sirmans():
    p = pd.read_csv(ECO / "pooled_effects.csv")
    sir = [("living_area", "square footage", "positive; varies by location"), ("lot_size", "lot size", "positive; varies by location; smaller when sq ft controlled"), ("age", "age", "negative; varies by location"),
           ("bedrooms", "bedrooms", "mixed sign"), ("bathrooms", "bathrooms", "positive; varies by location and data type"), ("garage", "garage", "positive"), ("pool", "pool", "positive; varies by location"),
           ("fireplace", "fireplace", "positive"), ("air_conditioning", "air conditioning", "positive; varies by location")]
    rows = []
    for tx, name, finding in sir:
        for fam in ("pct_premium", "semi_elasticity", "elasticity"):
            r = p[(p.family == fam) & (p.taxonomy_id == tx)]
            if len(r):
                r = r.iloc[0]; rows.append([esc(name), FAM[fam], f"{int(r.k_studies)}", f"{int(r.n_estimates)}", fmt(r.est_re), f"[{fmt(r.ci_lo)}, {fmt(r.ci_hi)}]", f"[{fmt(r.pi_lo)}, {fmt(r.pi_hi)}]", esc(finding)]); finding = "\textquotedbl"
    tabular(["Characteristic", "family", "$k$", "$n$", "pooled", "95\% CI", "95\% PI", "Sirmans et al.\ (2006) finding"], rows, "llrrrrrl", "tab14_sirmans_comparison")


def t15_gaps():
    C = pd.read_csv(FINAL / "coefficients_raw.csv"); S = pd.read_csv(FINAL / "studies.csv"); S = S[S.eligible == True]; p = pd.read_csv(ECO / "pooled_effects.csv")
    st = C[C.taxonomy_id != "other"].groupby("taxonomy_id").record_id.nunique(); pooled = set(p.taxonomy_id)
    rows = []
    for g, leaves in tax.items():
        for leaf, v in leaves.items():
            k = int(st.get(leaf, 0)); status = "pooled" if leaf in pooled else ("descriptive only" if k > 0 else "no study")
            if k < 10: rows.append([esc(GRPLAB[g]), esc(v["label"]), str(k), status])
    tabular(["Group", "Characteristic", "studies", "evidence status"], rows, "llrl", "tab15_gaps")
    longtab(["Group", "Characteristic", "studies", "evidence status"], rows, "llrl", "tab15_gaps_long", "Research gaps: characteristics with fewer than ten extracted studies", "tab:gaps")
    cont = S.continent.fillna("unknown").value_counts(); macro("nStudiesAfrica", int(cont.get("Africa", 0)), 0); macro("nStudiesSouthAmerica", int(cont.get("South America", 0)), 0); macro("nStudiesOceania", int(cont.get("Oceania", 0)), 0)
    macro("nStudiesEurope", int(cont.get("Europe", 0)), 0); macro("nStudiesAsia", int(cont.get("Asia", 0)), 0); macro("nStudiesNorthAmerica", int(cont.get("North America", 0)), 0)
    macro("nLeavesNoStudy", int(sum(1 for g, leaves in tax.items() for leaf in leaves if st.get(leaf, 0) == 0)), 0); macro("nLeavesTotal", int(sum(len(l) for l in tax.values())), 0)
    macro("nLeavesPooled", int(len(pooled)), 0)
    pt = S.property_type.value_counts(); macro("nStudiesCondo", int(pt.get("condo_apartment", 0)), 0); macro("nStudiesSingleFamily", int(pt.get("single_family", 0)), 0); macro("nStudiesRental", int(pt.get("rental", 0)), 0)
    pr = S.price_type.value_counts(); macro("nStudiesTransaction", int(pr.get("transaction", 0)), 0); macro("nStudiesListing", int(pr.get("listing", 0)), 0); macro("nStudiesRent", int(pr.get("rent", 0)), 0); macro("nStudiesAppraisal", int(pr.get("appraisal", 0)), 0)
    dt = S.document_type.value_counts(); macro("nStudiesJournal", int(dt.get("journal", 0)), 0)
    E = pd.read_csv(FINAL / "effects.csv"); ec = E.estimator_class.value_counts(); macro("shareSpatialEst", 100 * ec.get("spatial", 0) / len(E), 1); macro("shareOlsEst", 100 * ec.get("ols", 0) / len(E), 1); macro("shareQuantileEst", 100 * ec.get("quantile", 0) / len(E), 1)



def t09_publication():
    f = ECO / "publication_selection.csv"
    if not f.exists(): return
    p = pd.read_csv(f).sort_values("n_estimates", ascending=False); rows = []
    for r in p.itertuples():
        rows.append([esc(LAB.get(r.taxonomy_id, r.taxonomy_id)), FAM.get(r.family, r.family), f"{int(r.n_estimates)}", fmt(r.egger_z), fmt(r.egger_p, 3), f"{fmt(r.pet_intercept)} ({fmt(r.pet_se, 3)})", f"{fmt(r.peese_intercept)} ({fmt(r.peese_se, 3)})", f"{fmt(r.selmodel_est)} ({fmt(r.selmodel_se, 3)})" if pd.notna(r.selmodel_est) else "--", fmt(r.selmodel_lrt_p, 3)])
    tabular(["Characteristic", "family", "$n$", "Egger $z$", "$p$", "PET intercept (SE)", "PEESE intercept (SE)", "selection model (SE)", "LRT $p$"], rows, "llrrrrrrr", "tab09_publication_selection")
    longtab(["Characteristic", "family", "$n$", "Egger $z$", "$p$", "PET (SE)", "PEESE (SE)", "selection model (SE)", "LRT $p$"], rows, "p{2.6cm}p{1.6cm}rrrrrrr", "tab09_publication_selection_long", "Publication-selection diagnostics by characteristic (families with at least ten estimates)", "tab:pubsel")
    macro("nEggerSig", int((p.egger_p < 0.05).sum()), 0); macro("nPubTested", int(len(p)), 0)


def t10_robustness():
    f = ECO / "python_pooled.csv"; r = ECO / "pooled_effects.csv"
    if not f.exists(): return
    py = pd.read_csv(f); rr = pd.read_csv(r); m = rr.merge(py, on=["family", "taxonomy_id"], suffixes=("_r", "_py"))
    loo = pd.read_csv(ECO / "python_leave_one_out.csv") if (ECO / "python_leave_one_out.csv").exists() else None
    bay = pd.read_csv(ECO / "python_bayes.csv") if (ECO / "python_bayes.csv").exists() else None
    rows = []
    for x in m.itertuples():
        lo = loo[(loo.family == x.family) & (loo.taxonomy_id == x.taxonomy_id) & loo.study_id_left_out.notna()] if loo is not None else None
        b = bay[(bay.family == x.family) & (bay.taxonomy_id == x.taxonomy_id)] if bay is not None else None
        rows.append([esc(LAB.get(x.taxonomy_id, x.taxonomy_id)), FAM.get(x.family, x.family), fmt(x.est_re), fmt(x.mu), fmt(x.est_ml), f"[{fmt(lo.mu.min())}, {fmt(lo.mu.max())}]" if lo is not None and len(lo) else "--",
                     f"{fmt(b.mu_mean.iloc[0])} [{fmt(b.mu_q025.iloc[0])}, {fmt(b.mu_q975.iloc[0])}]" if b is not None and len(b) else "--", fmt(getattr(x, "pet_peese", np.nan)) if hasattr(x, "pet_peese") else "--"])
    tabular(["Characteristic", "family", "REML (R)", "REML (Python)", "3-level", "leave-one-study-out range", "Bayesian posterior mean [95\\% CrI]", "PET-PEESE"], rows, "llrrrrrr", "tab10_robustness")
    longtab(["Characteristic", "family", "REML (R)", "REML (Py)", "3-level", "LOSO range", "Bayes mean [95\\% CrI]", "PET-PEESE"], rows, "p{2.4cm}p{1.5cm}rrrrrr", "tab10_robustness_long", "Robustness of the pooled estimates across estimators and implementations", "tab:robustness")
    if bay is not None and len(bay): macro("maxRhat", float(bay[["rhat_mu", "rhat_tau"]].max().max()), 3); macro("minEss", float(bay[["ess_mu", "ess_tau"]].min().min()), 0)


def t11_competing():
    c = pd.read_csv(META / "competing_reviews.csv"); rows = []
    for r in c.sort_values("year").itertuples():
        rows.append([esc(f"{r.authors.split(' and ')[0].split(',')[0]} et al. ({r.year})" if " and " in str(r.authors) else f"{str(r.authors).split(',')[0]} ({r.year})"), esc(str(r.scope)[:40]), esc(str(r.n_studies)[:22] if pd.notna(r.n_studies) else "--"), esc(str(r.n_estimates)[:22] if pd.notna(r.n_estimates) else "--"), esc(str(r.countries)[:30]), esc(str(r.method)[:30])])
    tabular(["Review", "scope", "studies", "estimates", "coverage", "method"], rows, "p{3.1cm}p{3.4cm}rrp{2.6cm}p{3.2cm}", "tab11_competing_reviews")
    longtab(["Review", "scope", "studies", "estimates", "coverage", "method"], rows, "p{2.8cm}p{3.0cm}p{1.9cm}p{1.9cm}p{2.3cm}p{2.9cm}", "tab11_competing_reviews_long", "Competing reviews and meta-analyses of hedonic housing studies (verified; counts as stated in the abstracts or full texts that could be read; blank when not stated)", "tab:competing")
    macro("nCompetingReviews", int(len(c)), 0)


def t12_validation():
    f = ROOT / "data" / "validated" / "validation_summary.json"
    if not f.exists(): return
    v = json.load(open(f)); rows = [[esc(k.replace("_", " ")), fmt(val, 3) if isinstance(val, float) else str(val)] for k, val in v.items()]
    tabular(["Validation statistic", "value"], rows, "lr", "tab12_extraction_validation")
    for k, val in v.items(): macro("val" + "".join(w.capitalize() for w in k.replace("1pct", "onepct").split("_")), 100 * float(val) if isinstance(val, float) and val <= 1 else (float(val) if isinstance(val, float) else int(val)), 1 if isinstance(val, float) else 0)


def t13_ai():
    f = FINAL / "ai_studies.csv"
    if not f.exists(): return
    a = pd.read_csv(f); rows = []
    for r in a.itertuples():
        rows.append([esc(str(r.short_ref)[:28]), esc(str(r.country)[:14]), esc(str(r.best_method)[:22]), esc(str(r.metric)), fmt(r.best_value, 3), fmt(r.hedonic_value, 3), esc(str(r.features_ai)[:26]), esc(str(r.marginal_effects))])
    tabular(["Study", "country", "best model", "metric", "best", "hedonic benchmark", "AI-derived features", "marginal effects?"], rows, "llllrrll", "tab13_ai_evidence")
    macro("nAiStudies", int(len(a)), 0); macro("nAiBenchmark", int(a.hedonic_value.notna().sum()), 0); macro("nAiIncremental", int(a.incremental_ai_gain.notna().sum()), 0)
    macro("nAiShap", int((a.marginal_effects == "SHAP").sum()), 0); macro("nAiCoef", int((a.marginal_effects == "coefficients").sum()), 0); macro("nAiNoMarg", int((a.marginal_effects == "none").sum()), 0)
    bm = a.best_method.fillna("").str.lower(); macro("nAiBoosting", int(bm.str.contains("xgb|boost|lightgbm|gbdt").sum()), 0); macro("nAiRf", int(bm.str.contains("random forest").sum()), 0); macro("nAiNn", int(bm.str.contains("neural|dnn|ann|cnn").sum()), 0)
    macro("nAiHedonicBest", int(bm.str.contains("ols|hedonic|mgwr|regression").sum()), 0); macro("nAiFeatures", int((a.features_ai.fillna("none").str.lower() != "none").sum()), 0)
    cmp_ = a[a.hedonic_value.notna() & (a.metric == "MAPE")]; macro("nAiMapeCmp", int(len(cmp_)), 0)
    if len(cmp_): macro("aiMapeBestMedian", float(cmp_.best_value.median()), 2); macro("aiMapeHedMedian", float(cmp_.hedonic_value.median()), 2)
    cmp2 = a[a.hedonic_value.notna() & (a.metric == "R2")]; macro("nAiRsqCmp", int(len(cmp2)), 0)
    if len(cmp2): macro("aiRsqBestMedian", float(cmp2.best_value.median()), 3); macro("aiRsqHedMedian", float(cmp2.hedonic_value.median()), 3)
    ts = a.test_set.fillna("unknown").str.lower(); macro("nAiHoldout", int(ts.str.contains("holdout").sum()), 0); macro("nAiKfold", int(ts.str.contains("fold").sum()), 0); macro("nAiNoTest", int(ts.str.contains("none|in-sample").sum()), 0)


def write_numbers():
    (TAB / "numbers.tex").write_text("% generated by scripts/visualization/make_tables.py\n" + "\n".join(f"\\newcommand{{\\{k}}}{{{v}}}" for k, v in sorted(MACROS.items())) + "\n"); print("numbers", len(MACROS))


if __name__ == "__main__":
    import traceback
    for f in (t01_coverage, t02_geography, t03_study_characteristics, t04_taxonomy, t05_most_studied, t06_pooled, t08_metaregression, t09_publication, t10_robustness, t11_competing, t12_validation, t13_ai, t14_sirmans, t15_gaps):
        try: f()
        except Exception as e: print("FAILED", f.__name__, repr(e)); traceback.print_exc()
    write_numbers()