"""Phase 7 — LaTeX tables (booktabs, threeparttable notes) from processed outputs → paper/tables/*.tex."""
import json, numpy as np, pandas as pd
from src.common import ROOT, path, SETTINGS, get_logger
log = get_logger("tables"); P = path("processed"); T = ROOT / "paper" / "tables"; E = ROOT / "experiments"; T.mkdir(exist_ok=True)

def tex_escape(s):
    return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")

def table(df, fname, caption, label, note, colfmt=None, float_fmt="{:.3f}", index=False, wide=False):
    df = df.copy()
    for c in df.columns:
        if df[c].dtype.kind == "f" and df[c].dropna().apply(lambda x: float(x).is_integer()).all() and df[c].notna().any() and df[c].abs().max() >= 10:
            df[c] = df[c].map(lambda x: "" if pd.isna(x) else f"{int(x):,}".replace(",", "\\,"))
        elif df[c].dtype.kind == "f": df[c] = df[c].map(lambda x: "" if pd.isna(x) else float_fmt.format(x))
        elif df[c].dtype.kind in "iu": df[c] = df[c].map(lambda x: f"{x:,}".replace(",", "\\,"))
        else: df[c] = df[c].map(tex_escape)
    cols = [tex_escape(c) for c in df.columns]; colfmt = colfmt or ("l" + "r" * (len(cols) - 1))
    body = " \\\\\n".join(" & ".join(map(str, r)) for r in df.values)
    tab = f"""\\begin{{tabular}}{{{colfmt}}}
\\toprule
{' & '.join(cols)} \\\\
\\midrule
{body} \\\\
\\bottomrule
\\end{{tabular}}"""
    if wide:   # threeparttable conflicts with resized boxes ("Missing \endgroup"): caption + adjustbox + notes in a minipage
        tex = f"""\\begin{{table}}[htbp]\\centering
\\caption{{{caption}}}\\label{{{label}}}
\\small
\\begin{{adjustbox}}{{max width=\\textwidth}}
{tab}
\\end{{adjustbox}}
\\begin{{minipage}}{{\\textwidth}}\\footnotesize\\vspace{{2pt}}
{note}
\\end{{minipage}}
\\end{{table}}
"""
    else:
        tex = f"""\\begin{{table}}[htbp]\\centering
\\begin{{threeparttable}}
\\caption{{{caption}}}\\label{{{label}}}
\\small
{tab}
\\begin{{tablenotes}}[flushleft]\\footnotesize
\\item {note}
\\end{{tablenotes}}
\\end{{threeparttable}}
\\end{{table}}
"""
    (T / fname).write_text(tex); log.info("wrote %s", fname)

def main():
    A = json.load(open(E / "modelA_summary.json")); B = json.load(open(E / "modelB_summary.json")); C = json.load(open(E / "modelC_summary.json"))
    D = json.load(open(E / "modelD_summary.json")); Ee = json.load(open(E / "modelE_summary.json")); R = json.load(open(E / "robustness_summary.json"))
    # T1 coverage by feed (top 20 + rest)
    f = pd.read_csv(P / "coverage_by_feed.csv"); f = f[f.active > 0]
    t1 = f[["company", "banner", "channel", "listings", "active", "median_days_obs", "price_events", "unit_price_cov", "brand_cov"]].head(24).copy()
    t1.columns = ["Company", "Banner", "Channel", "Listings", "Active", "Median days", "Price events", "Unit price", "Brand"]
    table(t1, "tab_coverage_feeds.tex", "Coverage of the 24 largest source feeds (snapshot of 1 November 2026)", "tab:coverage",
          "Listings = all uids ever observed in the feed (active and deactivated); Active = listed on 1 Nov 2026; Median days = median observation span; Price events = rows in the price log; Unit price / Brand = share of active listings with a parsable package size (re-parsed) / a brand field. Channel: online storefront vs weekly circular (Flipp, postal code H2X 1Y4). Full table of 54 feeds in the replication package.", float_fmt="{:.2f}", wide=True)
    # T2 attrition
    att = pd.read_csv(P / "attrition_listings.csv"); att.columns = ["Step", "Listings", "Share of initial"]
    table(att, "tab_attrition.tex", "Listing-level sample attrition", "tab:attrition", "Each row applies the stated filter to the previous row. Matching tiers A--C shown; the primary analysis uses A+B.", colfmt="p{9cm}rr")
    # T3 matching validation
    v = pd.read_csv(P / "match_validation_summary.csv")
    t3 = v[["tier", "n", "same", "different", "unsure", "precision_strict", "ci_lo", "ci_hi", "upc_precision", "upc_recall"]].copy()
    t3.columns = ["Tier", "n", "Same", "Different", "Unsure", "Precision", "CI low", "CI high", "UPC precision", "UPC recall"]
    table(t3, "tab_match_validation.tex", "Matching validation: stratified sample (60 pairs per tier) and UPC ground truth", "tab:validation",
          "Precision = share labelled SAME (Wilson 95\\% CI); labels by Claude (claude-opus-5-5) as second annotator with full operator review of every pair; PI adjudication pending. UPC columns: precision/recall of the tier's text rule on all Metro\\,$\\times$\\,Super~C candidate pairs within brand\\,$\\times$\\,package blocks, where the shared GS1 code is the truth (B: identical token sets; C: token-sort ratio $\\ge 0.90$). Tier D has no UPC benchmark.", wide=True)
    # T4 dispersion summary
    rows = []
    for k, lab in [("gap_all", "All group-weeks"), ("gap_same_company", "Same company (two banners)"), ("gap_cross_company", "Different companies")]:
        g = A[k]; rows.append(dict(Sample=lab, **{"Group-weeks": g["n"], "Median": g["median"], "P25": g["p25"], "P75": g["p75"], "Mean": g["mean"], "Share $\\geq$ 5%": g["share_ge_5pct"], "Share $\\geq$ 10%": g["share_ge_10pct"]}))
    t4 = pd.DataFrame(rows)
    table(t4, "tab_dispersion.tex", "Within-product cross-banner log price gaps (max$-$min), tiers A+B, storefront channel", "tab:dispersion",
          f"Weekly posted price = duration-weighted modal price. Residual s.d. of the product-week/banner two-way fixed-effects model: {A['resid_sd']:.3f} (N = {A['n_obs_fe']:,}). AR(1) of banner-pair gaps: {A['ar1_rho']:.3f} (s.e. {A['ar1_se']:.3f}); placebo with banner labels permuted within group-week: {A['placebo_ar1_mean']:.3f} (s.d. {A['placebo_ar1_sd']:.3f}). Share of banner pairs ($\\ge$ 4 weeks) whose sign never flips: {A["share_pairs_sign_stable_ge4w"]:.2f}.", wide=True)
    # T5 banner effects
    fe = pd.read_csv(P / "modelA_banner_effects.csv").sort_values("coef")
    t5 = fe[["banner", "coef", "se", "se_twoway", "n_listing_weeks"]].copy(); t5.columns = ["Banner", "Coefficient", "s.e. (group)", "s.e. (group+banner)", "Listing-weeks"]
    table(t5, "tab_banner_effects.tex", f"Banner fixed effects on log posted price within product-week (reference: {A['banner_fe_reference']})", "tab:bannerfe",
          "Model A: $\\log p_{irw} = \\alpha_{iw} + \\gamma_r + \\varepsilon_{irw}$ on matched groups (tiers A+B), storefront channel, non-ambiguous group-weeks. Standard errors clustered by match group, and two-way by group and banner. Coefficients are relative posted-price levels for the same product in the same week; banners with few listing-weeks are imprecisely estimated.", float_fmt="{:.3f}", wide=False)
    # T6 promotions
    rows = [("Listing-weeks promoted (inferred, $p \\le 0.9\\times$ trailing 8-week mode)", B["promo_share_listing_weeks"]), ("Median depth (log ref/p), promoted weeks", B["promo_depth_median"]),
            ("Agreement with current on-sale flag (last week)", B["promo_vs_on_sale_now_agreement"]), ("S.d. weekly $\\Delta\\log p$, raw", B["sd_raw_all"]), ("S.d. weekly $\\Delta\\log$ reference", B["sd_ref_all"]),
            ("Ratio raw / reference (bootstrap 95\\% CI)", f"{B['ratio_all']:.2f} [{B['ratio_ci95'][0]:.2f}, {B['ratio_ci95'][1]:.2f}]"), ("Share of weekly price changes that are promotion transitions", B["share_weekly_changes_promo_transition"]),
            ("Episodes (uncensored)", f"{B['n_episodes']:,} ({B['n_uncensored']:,})"), ("Episode kinds: return / reset down / reset up", " / ".join(f"{B['episode_kind_shares'].get(k, 0):.2f}" for k in ("return", "reset_down", "reset_up"))),
            ("Return episodes: median duration (weeks), share 1 week", f"{B['return_episodes']['duration_median']:.0f}, {B['return_episodes']['share_duration_1w']:.2f}"), ("Return episodes: median depth", B["return_episodes"]["depth_median"]),
            ("Median recurrence interval (weeks), share of listings with $\\ge$ 2 episodes", f"{B['recurrence_median_weeks']:.0f}, {B['share_listings_with_2plus_episodes']:.2f}"),
            ("H3 (return episodes): mean $\\Delta\\log$ reference after vs before (s.e.)", f"{B['H3_return_dlog_ref_mean']:.4f} ({B['H3_return_dlog_ref_se']:.4f}), n = {B['H3_return_n']:,}"),
            ("H3 (all low-price episodes): mean $\\Delta\\log$ reference (s.e.)", f"{B['H3_dlog_ref_mean']:.4f} ({B['H3_dlog_ref_se']:.4f})"), ("Placebo (random weeks, non-promoted listings)", f"{B['placebo_dlog_ref_mean']:.4f} ({B['placebo_dlog_ref_se']:.4f})"),
            ("Event-study pretrend test (weeks $-4..-2$), $p$", f"{B.get('pretrend_p', float('nan')):.3f}"), ("Circular items with observed regular price; median observed depth", f"{B['circular_share_with_regular']:.2f}; {B['circular_depth_median']:.3f}"),
            ("Multi-price weeks: single-store shops vs anti-bot storefronts", f"{B['multi_price_week_share_single_store']:.3f} vs {B['multi_price_week_share_antibot']:.3f}")]
    t6 = pd.DataFrame(rows, columns=["Statistic", "Value"]); t6["Value"] = t6.Value.map(lambda v: f"{v:.3f}" if isinstance(v, float) else v)
    table(t6, "tab_promotions.tex", "Promotion dynamics in storefront feeds (Model B)", "tab:promotions",
          "Sample: grocery storefront listings observed in $\\ge$ 8 of 11 complete weeks. Promotion state is inferred from the weekly modal price; regular prices are not historised in the source. Return episodes end with a price within $\\pm$2\\% of the pre-episode reference; reset episodes are regular-price changes that the 10\\% rule also flags.", colfmt="p{10.5cm}r")
    # T7 spillovers
    rows = []
    for col, lab in [("rival_all", "All rivals"), ("rival_same", "Same-company rivals"), ("rival_cross", "Other-company rivals")]:
        if col not in C: continue
        c = C[col]; r = {"Rival set": lab, "N": c["n"]}
        for t, nm in [(col, "$k=0$"), (col + "_l1", "$k=1$"), (col + "_l2", "$k=2$"), (col + "_l3", "$k=3$"), (col + "_f1", "lead 1"), (col + "_f2", "lead 2")]:
            r[nm] = f"{c['coefs'][t]:.3f} ({c['ses'][t]:.3f})"
        rows.append(r)
    table(pd.DataFrame(rows), "tab_spillovers.tex", "Descriptive lead--lag associations between a banner's weekly log price change and rivals' changes (Model C)", "tab:spillovers",
          "Dependent variable: $\\Delta\\log p$ of banner $r$ for product $i$ in week $w$; regressors: mean $\\Delta\\log p$ of the other banners in the match group at $w-k$ (lags) and $w+k$ (leads, placebo). Banner and week fixed effects; s.e. clustered by match group. Contemporaneous terms absorb common shocks and are not causal; a non-zero lead coefficient indicates timing noise or anticipation and is reported as a failed placebo.", colfmt="lr" + "r" * 6, wide=True)
    # T8 indices
    rows = []
    for k, lab in [("storefront_posted", "Storefront, posted price"), ("storefront_unit", "Storefront, unit price"), ("all_channels_posted", "All channels, posted price")]:
        s = D[k]; rows.append({"Sample": lab, "Naive mean": s["naive_mean_change"], "Naive median": s["naive_median_change"], "Chained Jevons": s["jevons_chained_change"], "Fixed Jevons": s["fixed_jevons_change"], "Fixed Dutot": s["fixed_dutot_change"], "Fixed Carli": s["fixed_carli_change"], "Balanced n": s["n_balanced"]})
    table(pd.DataFrame(rows), "tab_indices.tex", "Cumulative price change over 11 weeks (13 Aug $\\to$ 22 Oct 2026) under alternative unweighted index formulas", "tab:indices",
          f"No expenditure weights exist: none of these is a consumer price index. Bootstrap (999 listing resamples) 95\\% CI for the storefront chained Jevons: [{D['bootstrap']['jevons_ci95'][0]:.3f}, {D['bootstrap']['jevons_ci95'][1]:.3f}]; naive-mean minus Jevons gap: {D['bootstrap']['gap_mean']:.3f} [{D['bootstrap']['gap_ci95'][0]:.3f}, {D['bootstrap']['gap_ci95'][1]:.3f}]. Statistics Canada CPI, Quebec, food purchased from stores: Aug$\\to$Sep 2026 {D['statcan']['cpi_change_aug_sep']:+.3f} (our chained Jevons, monthly means, {D['statcan']['ours_jevons_aug_sep']:+.3f}); 12-month CPI change to Sep 2026 {D['statcan']['cpi_yoy_sep']:+.3f}.", wide=True)
    # T9 basket
    bs = pd.read_csv(P / "modelE_block_sensitivity.csv"); bs.columns = ["Banners (B)", "Items", "Cost K=1", "Cost K=2", "Cost unconstrained", "Savings share", "Share captured at K=2"]
    table(bs, "tab_basket.tex", "Fixed-basket cost under banner-count constraints (Model E, weekly means over 11 weeks, CAD)", "tab:basket",
          f"Complete-block design (amendment A-001): the $B$ widest-coverage storefront banners and the basket items all of them carry; item cost = cheapest unit price $\\times$ reference quantity. K = 1: best single banner. Travel costs are unobserved; with an illustrative fixed cost per banner of 5 / 10 / 20 CAD the optimum uses {Ee['travel_cost_sensitivity'][0]['n_used']:.2f} / {Ee['travel_cost_sensitivity'][1]['n_used']:.2f} / {Ee['travel_cost_sensitivity'][2]['n_used']:.2f} banners on average. Spread between the cheapest and most expensive single banner for the same items: {100*Ee['most_expensive_vs_cheapest_single_share']:.0f}\\% of the cheapest.", float_fmt="{:.2f}")
    # T10 forecasts
    fs = pd.read_csv(P / "modelF_summary.csv"); fp = fs[fs.target == "log_price"][["model", "origins", "n", "mae", "rmse"]]; fq = fs[fs.target == "promotion"][["model", "origins", "n", "brier", "auc"]]
    fp.columns = ["Model", "Origins", "N", "MAE", "RMSE"]; fq.columns = ["Model", "Origins", "N", "Brier", "AUC"]
    table(fp, "tab_forecast_price.tex", "Rolling-origin forecasts of next-week log posted price (storefront listings)", "tab:fprice", "Strict time-based holdouts: train on weeks $\\le t-1$, test on week $t$, origins $t = 6,\\dots,9$; features are lags only. Lower is better.", float_fmt="{:.4f}")
    table(fq, "tab_forecast_promo.tex", "Rolling-origin forecasts of next-week promotion state", "tab:fpromo", "Promotion = modal price $\\le 0.9\\times$ trailing 8-week mode. Brier: lower is better; AUC: higher is better. Base rate $\\approx$ 5\\%.", float_fmt="{:.4f}")
    # T11 robustness
    rows = [("Baseline median gap (A+B, storefront)", R["baseline_median_gap"]), ("Balanced groups present in all 11 weeks (n groups %d)" % R["balanced"]["groups"], R["balanced"]["median_gap"]), ("Unit-price normalisation", R["unit_price"]["median_gap"]),
            ("Trimmed 1/99\\% (mean, vs untrimmed mean)", f"{R['trimmed_1_99']['mean_gap']:.3f} vs {R['trimmed_1_99']['mean_untrimmed']:.3f}"), ("Cross-company group-weeks only", R["cross_company_only"]["median_gap"]), ("Non-promoted weeks only", R["non_promo_weeks_only"]["median_gap"]), ("Group-weeks with a promoted listing", R["group_weeks_with_a_promo"]["median_gap"])]
    sens = pd.read_csv(P / "modelA_sensitivity.csv")
    for _, r in sens.iterrows(): rows.append((f"Tiers {r.tiers}, price = {r.price_def}", r.median_gap))
    t11 = pd.DataFrame(rows, columns=["Specification", "Median log gap"]); t11["Median log gap"] = t11["Median log gap"].map(lambda v: f"{v:.3f}" if isinstance(v, float) else v)
    table(t11, "tab_robustness.tex", "Robustness of the dispersion statistic (H1)", "tab:robust", "All rows use storefront listings in non-ambiguous group-weeks with $\\ge$ 2 banners.", colfmt="p{10cm}r")
    holm = pd.read_csv(P / "robustness_holm.csv"); holm.columns = ["Preregistered test", "p", "Holm-adjusted p"]
    table(holm, "tab_holm.tex", "Preregistered primary tests with Holm correction", "tab:holm", "CI-based entries take p = 0 when the bootstrap 95\\% CI excludes the null value and 1 otherwise (conservative coding).", float_fmt="{:.4f}", colfmt="p{8cm}rr")
    # T12 QGPAI
    q = pd.read_csv(P / "qgpai_components.csv"); tq = q[["week", "comparable_price_index", "cpi_lo", "cpi_hi", "promo_share", "dispersion_median_gap", "basket_K1", "basket_K2", "basket_unconstrained", "circular_depth_median"]].copy()
    tq["week"] = pd.to_datetime(tq.week).dt.strftime("%d %b"); tq.columns = ["Week", "Comparable index", "CI low", "CI high", "Promo share", "Dispersion", "Basket K=1", "Basket K=2", "Basket uncon.", "Circular depth"]
    table(tq, "tab_qgpai.tex", "QGPAI components by circular week (Thursday start), 2026", "tab:qgpai", "Component 5 (access and affordability) is not computed: no store geography or income data are joined. Comparable index: chained Jevons over matched groups (A+B), base 13 Aug = 100, bootstrap band over groups; Dispersion: median within-group max$-$min log gap; Basket: Model E (B = 5). Composites are illustrative only and reported in the replication package.", float_fmt="{:.3f}", wide=True)
    log.info("tables done")

if __name__ == "__main__":
    main()