"""LaTeX tables from experiments/*.json and data/processed/*.csv -> paper/tables/*.tex (booktabs; notes in minipage, no threeparttable)."""
from __future__ import annotations
import json
import numpy as np, pandas as pd
from src.common import ROOT, get_logger, ledger, PROC_DIR, EXP_DIR, P, BASE_PROC, BASE_EXP

log = get_logger("reports.tables")
PROC = PROC_DIR; TAB = P["tables"]; TAB.mkdir(exist_ok=True); EXP = EXP_DIR
SHORT = {"Saguenay–Lac-Saint-Jean": "Saguenay--Lac-Saint-Jean", "Gaspésie–Îles-de-la-Madeleine": "Gaspésie--Îles-de-la-Madeleine"}
CAT = {"erable": "Maple \\& honey", "epicerie": "Fine grocery", "cafe_the": "Coffee \\& tea", "chocolat": "Chocolate", "boulangerie": "Bakery", "viande_poisson": "Meat \\& fish", "fromage_laitier": "Cheese \\& dairy", "boissons": "Beverages", "alcool": "Alcohol", "sante_beaute": "Beauty \\& care", "maison": "Home \\& decor", "mode": "Fashion", "bijoux": "Jewellery", "art": "Art \\& craft", "enfants": "Children", "animaux": "Pets", "plein_air": "Outdoor \\& sport", "jardin": "Garden", "industriel": "Industrial", "autre": "Other"}


def esc(s): return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("–", "--")
def n(x, d=0): return f"{x:,.{d}f}".replace(",", "\\,")
def J(name): return json.load(open(EXP / name))


def write(name, header, rows, caption, label, notes="", align=None, size="\\small"):
    align = align or "l" + "r" * (len(header) - 1)
    body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in rows)
    tex = (f"\\begin{{table}}[htbp]\\centering{size}\n\\caption{{{caption}}}\\label{{tab:{label}}}\n\\begin{{tabular}}{{{align}}}\\toprule\n" + " & ".join(header) + " \\\\\\midrule\n" + body +
           "\n\\bottomrule\\end{tabular}\n" + (f"\\begin{{minipage}}{{0.95\\textwidth}}\\vspace{{2pt}}\\footnotesize\\textit{{Notes:}} {notes}\\end{{minipage}}\n" if notes else "") + "\\end{table}\n")
    (TAB / f"{name}.tex").write_text(tex); log.info("table %s", name)


def t_sample_flow():
    f = json.load(open(BASE_EXP / "sample_flow.json"))["flow"]
    write("sample_flow", ["Step", "Listings"], [[esc(k.replace("…", "\\ldots").replace("[", "\\textbf{[").replace("]", "]}")), n(v)] for k, v in f],
          "Sample flow from database rows to the analysis samples", "sample_flow", "Snapshot of 2026-11-01T17:38Z. Nothing is deleted: each step is a flag in the cleaned listing table. The assortment sample is used for H1--H4 and H7; the price sample for H5.")


def t_region_summary():
    sp = pd.read_parquet(PROC / "store_panel.parquet"); d = pd.read_csv(PROC / "diversity_region.csv").set_index("region")
    g = sp.groupby("region_final").agg(stores=("id", "size"), listings=("n_listings", "sum"), median_cat=("n_listings", "median"), certified=("certified", "mean"))
    g = g.sort_values("stores", ascending=False); rows = []
    for r, x in g.iterrows():
        dd = d.loc[r] if r in d.index else None
        rows.append([esc(r), n(x.stores), n(x.listings), n(x.median_cat), f"{x.certified:.2f}", f"{dd.H_norm_M:.3f}" if dd is not None else "--",
                     (f"{dd.H_rarefied15:.3f} [{dd.H_rarefied15_lo:.3f}; {dd.H_rarefied15_hi:.3f}]" if dd is not None and not np.isnan(dd.H_rarefied15) else "--")])
    write("region_summary", ["Region", "Stores", "Listings", "Median catalogue", "Certified share", "$\\tilde H$ (M)", "$\\tilde H$ rarefied (15 stores) [2.5; 97.5]"], rows,
          "Regional sample: stores, listings, certification and assortment diversity", "region_summary",
          "Assortment sample. $\\tilde H$ = normalised Shannon entropy of category shares (20 categories); M = merchant-equal weights. Rarefied entropy = mean over 2\\,000 draws of 15 stores without replacement; brackets give the 2.5th and 97.5th percentiles of the draws. `Inconnue' = stores whose location could not be established (excluded from regional inference).", size="\\footnotesize")


def t_h1():
    r = J("h1_raa.json")["results"]; rows = []
    for x in r:
        rows.append([x["sample"].replace("_", "\\_"), x["scheme"], n(x["stores"]), x["regions"], f"{x['cramers_v']:.3f}", f"{x['perm_null_mean']:.3f}", f"{x['perm_null_p95']:.3f}", f"{x['p_perm']:.4f}", f"{x['cells_sig_gt1']}/{x['cells_sig_lt1']}"])
    rb = J("robustness.json")
    for k, v in rb.items():
        if isinstance(v, dict) and "h1_M" in v and k != "baseline_S_R":
            rows.append([esc(k), "M", n(v["h1_M"]["stores"]), v["h1_M"]["regions"], f"{v['h1_M']['V']:.3f}", f"{v['h1_M']['null_mean']:.3f}", "", f"{v['h1_M']['p']:.3f}", ""])
    write("h1_raa", ["Sample / variant", "Weights", "Stores", "Regions", "Cramér's $V$", "Null mean", "Null p95", "$p$ (perm.)", "Cells RAA$>1$/$<1$"], rows,
          "H1 --- Region $\\times$ category dependence against the store-reshuffle null", "h1",
          "$V$ = Cramér's $V$ of the region $\\times$ category table of listing weights (L = one per listing; M = merchant-equal, each store sums to one; T = listings after dropping the 1\\% largest stores; `stores' = count of stores present in the category). Null = 2\\,000 permutations of region labels across stores (each region keeps its number of stores, each store keeps its catalogue); $p$ is one-sided. Last column: number of the $R\\times C$ cells whose store-bootstrap 95\\% interval for RAA lies entirely above / below 1. Variants from the preregistered robustness list (1\\,000 permutations).", size="\\footnotesize")


def t_raa_top():
    m = pd.read_csv(PROC / "raa_matrix.csv"); m = m[(m["sample"] == "S_R") & (m.scheme == "M") & m.sig_gt1].sort_values("raa", ascending=False).head(20)
    rows = [[esc(x.region), CAT.get(x.category, x.category), f"{x.raa:.2f}", f"[{x.raa_lo:.2f}; {x.raa_hi:.2f}]", n(x.n, 1)] for _, x in m.iterrows()]
    write("raa_top", ["Region", "Category", "RAA", "95\\% CI", "Merchant-equal weight"], rows, "Largest significant revealed assortment advantages (merchant-equal weights)", "raa_top",
          "Cells with a store-bootstrap 95\\% interval above 1, sorted by RAA. The weight column is the merchant-equal count $N_{rc}$ (sum of store shares).", size="\\footnotesize")


def t_h3():
    h = J("h3_longtail.json"); pl = h["powerlaw"]; lv = h["lognormal_vs_powerlaw"]
    rows = [["Stores / listings", f"{n(h['stores'])} / {n(h['listings'])}"], ["Gini of listings across stores [95\\% CI]", f"{h['gini']:.3f} [{h['gini_ci'][0]:.3f}; {h['gini_ci'][1]:.3f}]"],
            ["Share of listings, top 1\\% of stores [95\\% CI]", f"{h['top1pct_share']:.3f} [{h['top1_ci'][0]:.3f}; {h['top1_ci'][1]:.3f}]"], ["Share of listings, top 10\\% of stores [95\\% CI]", f"{h['top10pct_share']:.3f} [{h['top10_ci'][0]:.3f}; {h['top10_ci'][1]:.3f}]"],
            ["Median / p90 / p99 / max catalogue size", f"{n(h['median_catalogue'])} / {n(h['p90'])} / {n(h['p99'])} / {n(h['max'])}"], ["Rank--size slope (log--log)", f"{h['rank_size_slope_loglog']:.3f}"],
            ["Discrete power law: $x_{\\min}$, $\\alpha$, KS, tail $n$", f"{pl['xmin']}, {pl['alpha']:.3f}, {pl['ks']:.3f}, {pl['n_tail']}"], ["Vuong LR (power law vs log-normal on the tail): $z$, $p$", f"{lv['vuong_z']:.2f}, {lv['vuong_p']:.2g} ({lv['favoured']})"],
            ["Preregistered criterion (Gini $\\ge$ 0.70 and top-10\\% $\\ge$ 0.50)", "met" if h["decision_prereg"] else "not met"]]
    wr = h["within_region"]; rows2 = [[esc(r), v["stores"], f"{v['gini']:.3f} [{v['gini_ci'][0]:.3f}; {v['gini_ci'][1]:.3f}]" if v.get("gini") is not None else "--", f"{v.get('top10pct_share', float('nan')):.3f}" if v.get("gini") is not None else "--"] for r, v in sorted(wr.items(), key=lambda kv: -(kv[1].get("gini") or 0))]
    write("h3_concentration", ["Statistic", "Value"], rows, "H3 --- Concentration of listings across merchants (assortment sample)", "h3", "Bootstrap CIs from 2\\,000 resamples of stores. Power-law fit by maximum likelihood with $x_{\\min}$ chosen by KS minimisation (Clauset et al.\\ 2009); the log-normal is fitted on the same tail. Inconnue-region stores included (all stores).", align="lr")
    write("h3_within_region", ["Region", "Stores", "Gini [95\\% CI]", "Top-10\\% share"], rows2, "Within-region concentration of catalogue sizes", "h3_region", "500 store-bootstrap resamples per region; regions with fewer than five stores omitted.", size="\\footnotesize")


def t_h5():
    h = J("h5_hedonic.json"); rows = []
    for m in h["models"]:
        if m.get("status") == "insufficient": continue
        c = m["coef"]
        rows.append([m["label"].replace("_", "\\_"), n(m["obs"]), n(m["groups"]), m["stores"], m["regions"], f"{m['wald_region']:.1f} ({m['df_region']})", f"{m['p_region_wildboot']:.4f}",
                     f"{c['reseller']['b']:.3f} ({c['reseller']['se_cluster']:.3f})", f"{m['p_reseller_wildboot']:.4f}", f"{c['certified']['b']:.3f} ({c['certified']['se_cluster']:.3f})", f"{c['log_n']['b']:.3f} ({c['log_n']['se_cluster']:.3f})",
                     f"{m['within_R2']:.3f}", f"{m['gap_median']:.3f} / {m['gap_p90']:.3f}", f"{m['share_gap_gt5pct']:.2f}"])
    write("h5_hedonic", ["Spec.", "Obs.", "Groups", "Stores", "Regions", "Wald region (df)", "$p$ wild-boot", "Reseller (SE)", "$p$ wild-boot", "Certified (SE)", "log catalogue (SE)", "Within $R^2$", "Gap median / p90", "Share gap $>$5\\%"], rows,
          "H5 --- Matched-product price dispersion across merchants", "h5",
          "Dependent variable: log posted price at the snapshot. Match-group fixed effects absorbed; region and platform dummies, log catalogue size, reseller/collective and certification indicators. SE clustered by store; $p$-values from a wild cluster bootstrap (Rademacher, 9\\,999 draws, restricted residuals). primary\\_AB = tiers A (shared SKU) and B (same brand, identical title); sens\\_ABC adds fuzzy tier C. Gap = within-group max--min log price.", size="\\scriptsize", align="l" + "r" * 13)
    cc = h["certification_contrast"]
    write("cert_contrast", ["Statistic", "Value"], [["Observations / stores / certified stores", f"{n(cc['obs'])} / {n(cc['stores'])} / {n(cc['certified_stores'])}"], ["Certified store, log price difference within category (SE, store-clustered)", f"{cc['b_log_price']:.3f} ({cc['se']:.3f})"]],
          "Descriptive store-certification price contrast (not a test of H6)", "cert", esc(cc["note"]), align="lr")


def t_h7():
    h = J("h7_complexity.json"); v1 = h["V1_bootstrap_rank_stability"]; v2 = h["V2_reshuffle_null"]; v4 = h["V4_community_stability"]
    rows = [["Matrix density (share of cells with RAA $\\ge$ 1)", f"{h['M_density']:.3f}"], ["V1 ECI rank stability: median (p10) Spearman $\\rho$, 500 store bootstraps", f"{v1['eci_rho_median']:.2f} ({v1['eci_rho_p10']:.2f})"],
            ["V1 Fitness / Diversity rank stability (median $\\rho$)", f"{v1['fitness_rho_median']:.2f} / {v1['diversity_rho_median']:.2f}"], ["V1 criterion ($\\rho \\ge 0.8$) for ECI", "met" if v1["pass_eci"] else "not met"],
            ["V2 $\\lambda_2$ of $\\tilde M$: observed, null p95, $p$", f"{v2['obs_lambda2']:.3f}, {v2['null_lambda2_p95']:.3f}, {v2['p_lambda2']:.3f}"], ["V2 spectral radius of the bipartite graph: observed, null p95, $p$", f"{v2['obs_spectral_radius']:.2f}, {v2['null_spectral_radius_p95']:.2f}, {v2['p_spectral_radius']:.3f}"],
            ["V2 corr(ECI, diversity): observed, null [p5; p95]", f"{v2['obs_corr_eci_diversity']:.2f}, [{v2['null_corr_eci_diversity_p5_p95'][0]:.2f}; {v2['null_corr_eci_diversity_p5_p95'][1]:.2f}]"],
            ["V4 Louvain communities: mode of count, median NMI across 200 seeds", f"{v4.get('n_communities_mode')}, {v4.get('nmi_median_vs_run0')}"]]
    ind = pd.read_csv(PROC / "h7_region_indicators.csv").sort_values("eci", ascending=False)
    rows2 = [[esc(r.region), int(r.n_stores), int(r.diversity), f"{r.eci:.2f}", f"{r.fitness:.2f}"] for _, r in ind.iterrows()]
    write("h7_validation", ["Validation statistic", "Value"], rows, "H7 --- Validation of the region--category complexity indicator", "h7", "$M_{rc} = 1[\\mathrm{RAA}_{rc} \\ge 1]$ on merchant-equal weights, regional sample. Null = 2\\,000 store-level region reshuffles. Per the preregistration, the ECI is reported as unstable when V1 fails and is omitted from the QDACI.", align="lr")
    write("h7_indicators", ["Region", "Stores", "Diversity", "ECI", "Fitness"], rows2, "Region-level indicators (reported for transparency; unstable under bootstrap)", "h7ind", "Diversity = number of categories with RAA $\\ge 1$. ECI standardised; sign fixed so that corr(ECI, diversity) $\\ge 0$.", size="\\footnotesize")


def t_h4():
    h = J("h4_dynamics.json")
    rows = [["Baseline / outcome windows", f"{h['baseline_window'][0]} to {h['baseline_window'][1]} / {h['outcome_window'][0]} to {h['outcome_window'][1]}"], ["Stores; (store, category) cells absent at baseline", f"{n(h['stores'])}; {n(h['store_category_cells_absent_at_baseline'])}"],
            ["Entries (cells); entering stores", f"{h['entries']}; {h['entering_stores']}"], ["Mean density, cells with / without entry", f"{h['density_mean_entry']:.3f} / {h['density_mean_no_entry']:.3f}"],
            ["LPM coefficient on density (store-clustered SE); $t$", f"{h['lpm_density_coef']:.3f} ({h['se_cluster_store']:.3f}); {h['t']:.2f}"], ["Placebo (category-relabelled proximity, 200 draws): mean, p95, $p$", f"{h['placebo_coef_mean']:.3f}, {h['placebo_p95']:.3f}, {h['placebo_p']:.3f}"],
            ["Region-level category entries / exits", f"{len(h['region_level_entries'])} / {len(h['region_level_exits'])}"], ["Share of entering stores with an `added' burst $>$50\\% of baseline catalogue", f"{h['share_entering_stores_with_added_burst']:.2f}"]]
    write("h4_dynamics", ["Statistic", "Value"], rows, "H4 (exploratory) --- Entry into related categories within stores", "h4", "Linear probability model with store and category fixed effects; density = share of a category's proximity mass covered by the store's baseline portfolio (Hidalgo-type proximity from co-occurrence across stores). Exploratory by preregistration: no causal claim.", align="lr")


def t_audits():
    p = BASE_EXP / "category_audit.json"
    if p.exists():
        a = json.load(open(p)); pc = a["per_category_precision_vs_consensus"]
        rows = [[CAT.get(k, k), v["n_consensus"], f"{v['precision']:.2f}" if v["precision"] is not None else "--"] for k, v in pc.items()]
        rows.append(["\\textbf{All}", int(a["n"] * a["consensus_share"]), f"\\textbf{{{a['rule_precision_vs_consensus']:.2f}}}"])
        write("category_audit", ["Rule label", "Listings with annotator consensus", "Precision of the rule label"], rows, "Audit of the rule-based category labels (500-listing stratified sample)", "cat_audit",
              f"25 random listings per category annotated independently by two LLMs (claude-opus-5-5, gpt-5.5) with the platform's taxonomy; agreement rule vs A {a['agree_category_llm_a']:.2f}, rule vs B {a['agree_category_llm_b']:.2f}, A vs B {a['agree_llm_a_llm_b']:.2f}; consensus exists for {100*a['consensus_share']:.0f}\\% of listings. Precision = share of consensus labels equal to the rule label. Disagreements listed in the replication package.", size="\\footnotesize")
    p = BASE_EXP / "match_validation.json"
    if p.exists():
        m = json.load(open(p)); pt = m["precision_by_tier_consensus"]
        rows = [[t, v["n"], f"{v['yes']:.2f}", f"{v['no']:.2f}", f"{m['precision_by_tier_comparable_only'].get(t, float('nan')):.2f}"] for t, v in pt.items()]
        write("match_validation", ["Tier", "Pairs with consensus", "Same product", "Different", "Same product, comparable-variant pairs"], rows, "Validation of the cross-merchant matching tiers (60 pairs per tier)", "match_val",
              f"Two independent LLM annotators (agreement {m['agree_llm']:.2f}; consensus on {100*m['consensus_share']:.0f}\\% of pairs). Tier A = shared merchant SKU; B = same brand and identical normalised title; C = same brand, fuzzy title; D = identical title without brand (never used for inference).", size="\\footnotesize")


def t_h2():
    p = EXP / "h2_scale.json"
    if not p.exists(): return
    h = J("h2_scale.json"); rows = []
    for k, v in h["correlations"].items():
        rows.append([esc(k), v["n"], f"{v['rho']:.2f}", f"[{v['ci'][0]:.2f}; {v['ci'][1]:.2f}]", f"{v['ols_b']:.3f} ({v['ols_se_hc3']:.3f})" if v.get("ols_b") is not None else "--"])
    write("h2_scale", ["Outcome $\\sim$ covariate", "$n$", "Spearman $\\rho$", "95\\% CI", "OLS slope (HC3 SE)"], rows, "H2 --- Regional assortment diversity and regional scale", "h2", esc(h.get("note", "")), size="\\footnotesize")


def t_qdaci():
    p = EXP / "qdaci.json"
    if not p.exists(): return
    q = J("qdaci.json"); comp = pd.read_csv(PROC / "qdaci_components.csv")
    cols = [c for c in comp.columns if c not in ("region",)]
    rows = [[esc(r.region)] + [f"{r[c]:.2f}" if isinstance(r[c], float) and not np.isnan(r[c]) else ("--" if isinstance(r[c], float) else str(r[c])) for c in cols] for _, r in comp.iterrows()]
    write("qdaci", ["Region"] + [esc(c) for c in cols], rows, "QDACI components by region", "qdaci", esc(q.get("note", "")), size="\\scriptsize", align="l" + "r" * len(cols))


def t_external():
    p = EXP / "external_validation.json"
    if not p.exists(): return
    e = J("external_validation.json"); rows = [[esc(r["region"]), n(r["stores"]), n(r["businesses"]) if r.get("businesses") else "--", f"{r['stores_per_1000_businesses']:.2f}" if r.get("stores_per_1000_businesses") else "--", n(r["population"]) if r.get("population") else "--", f"{r['stores_per_100k_pop']:.1f}" if r.get("stores_per_100k_pop") else "--"] for r in e["rows"]]
    write("external_coverage", ["Region", "Fabri-Ka stores", "Business locations", "Stores per 1\\,000 businesses", "Population", "Stores per 100\\,000 pop."], rows, "Coverage of the regional sample against external business and population counts", "external", esc(e.get("note", "")), size="\\footnotesize")


def main():
    for f in (t_sample_flow, t_region_summary, t_h1, t_raa_top, t_h3, t_h5, t_h7, t_h4, t_audits, t_h2, t_qdaci, t_external):
        try: f()
        except Exception as ex:  # noqa: BLE001
            log.error("%s failed: %s", f.__name__, ex)
    ledger("reports.make_tables", tables=sorted(p.name for p in TAB.glob("*.tex")))


if __name__ == "__main__":
    main()