1"""LaTeX tables from experiments/*.json and data/processed/*.csv -> paper/tables/*.tex (booktabs; notes in minipage, no threeparttable)."""2from __future__ import annotations3import json4import numpy as np, pandas as pd5from src.common import ROOT, get_logger, ledger, PROC_DIR, EXP_DIR, P, BASE_PROC, BASE_EXP67log = get_logger("reports.tables")8PROC = PROC_DIR; TAB = P["tables"]; TAB.mkdir(exist_ok=True); EXP = EXP_DIR9SHORT = {"Saguenay–Lac-Saint-Jean": "Saguenay--Lac-Saint-Jean", "Gaspésie–Îles-de-la-Madeleine": "Gaspésie--Îles-de-la-Madeleine"}10CAT = {"erable": "Maple \\& honey", "epicerie": "Fine grocery", "cafe_the": "Coffee \\& tea", "chocolat": "Chocolate", "boulangerie": "Bakery", "viande_poisson": "Meat \\& fish", "fromage_laitier": "Cheese \\& dairy", "boissons": "Beverages", "alcool": "Alcohol", "sante_beaute": "Beauty \\& care", "maison": "Home \\& decor", "mode": "Fashion", "bijoux": "Jewellery", "art": "Art \\& craft", "enfants": "Children", "animaux": "Pets", "plein_air": "Outdoor \\& sport", "jardin": "Garden", "industriel": "Industrial", "autre": "Other"}111213def esc(s): return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("–", "--")14def n(x, d=0): return f"{x:,.{d}f}".replace(",", "\\,")15def J(name): return json.load(open(EXP / name))161718def write(name, header, rows, caption, label, notes="", align=None, size="\\small"):19 align = align or "l" + "r" * (len(header) - 1)20 body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in rows)21 tex = (f"\\begin{{table}}[htbp]\\centering{size}\n\\caption{{{caption}}}\\label{{tab:{label}}}\n\\begin{{adjustbox}}{{max width=\\textwidth}}\n\\begin{{tabular}}{{{align}}}\\toprule\n" + " & ".join(header) + " \\\\\\midrule\n" + body +22 "\n\\bottomrule\\end{tabular}\n\\end{adjustbox}\n" + (f"\\begin{{minipage}}{{0.95\\textwidth}}\\vspace{{2pt}}\\footnotesize\\textit{{Notes:}} {notes}\\end{{minipage}}\n" if notes else "") + "\\end{table}\n")23 (TAB / f"{name}.tex").write_text(tex); log.info("table %s", name)242526def t_sample_flow():27 f = json.load(open(BASE_EXP / "sample_flow.json"))["flow"]28 write("sample_flow", ["Step", "Listings"], [[esc(k.replace("…", "\\ldots").replace("[", "\\textbf{[").replace("]", "]}")), n(v)] for k, v in f],29 "Sample flow from database rows to the analysis samples", "sample_flow", "Snapshot of 2026-11-01T17:38Z. Nothing is deleted: each step is a flag in the cleaned listing table. The assortment sample is used for H1--H4 and H7; the price sample for H5.")303132def t_region_summary():33 sp = pd.read_parquet(PROC / "store_panel.parquet"); d = pd.read_csv(PROC / "diversity_region.csv").set_index("region")34 g = sp.groupby("region_final").agg(stores=("id", "size"), listings=("n_listings", "sum"), median_cat=("n_listings", "median"), certified=("certified", "mean"))35 g = g.sort_values("stores", ascending=False); rows = []36 for r, x in g.iterrows():37 dd = d.loc[r] if r in d.index else None38 rows.append([esc(r), n(x.stores), n(x.listings), n(x.median_cat), f"{x.certified:.2f}", f"{dd.H_norm_M:.3f}" if dd is not None else "--",39 (f"{dd.H_rarefied15:.3f} [{dd.H_rarefied15_lo:.3f}; {dd.H_rarefied15_hi:.3f}]" if dd is not None and not np.isnan(dd.H_rarefied15) else "--")])40 write("region_summary", ["Region", "Stores", "Listings", "Median catalogue", "Certified share", "$\\tilde H$ (M)", "$\\tilde H$ rarefied (15 stores) [2.5; 97.5]"], rows,41 "Regional sample: stores, listings, certification and assortment diversity", "region_summary",42 "Assortment sample. $\\tilde H$ = normalised Shannon entropy of category shares (20 categories); M = merchant-equal weights. Rarefied entropy = mean over 2\\,000 draws of 15 stores without replacement; brackets give the 2.5th and 97.5th percentiles of the draws. `Inconnue' = stores whose location could not be established (excluded from regional inference).", size="\\footnotesize")434445def t_h1():46 r = J("h1_raa.json")["results"]; rows = []47 for x in r:48 rows.append([x["sample"].replace("_", "\\_"), x["scheme"], n(x["stores"]), x["regions"], f"{x['cramers_v']:.3f}", f"{x['perm_null_mean']:.3f}", f"{x['perm_null_p95']:.3f}", f"{x['p_perm']:.4f}", f"{x['cells_sig_gt1']}/{x['cells_sig_lt1']}"])49 rb = J("robustness.json")50 for k, v in rb.items():51 if isinstance(v, dict) and "h1_M" in v and k != "baseline_S_R":52 rows.append([esc(k), "M", n(v["h1_M"]["stores"]), v["h1_M"]["regions"], f"{v['h1_M']['V']:.3f}", f"{v['h1_M']['null_mean']:.3f}", "", f"{v['h1_M']['p']:.3f}", ""])53 write("h1_raa", ["Sample / variant", "Weights", "Stores", "Regions", "Cramér's $V$", "Null mean", "Null p95", "$p$ (perm.)", "Cells RAA$>1$/$<1$"], rows,54 "H1 --- Region $\\times$ category dependence against the store-reshuffle null", "h1",55 "$V$ = Cramér's $V$ of the region $\\times$ category table of listing weights (L = one per listing; M = merchant-equal, each store sums to one; T = listings after dropping the 1\\% largest stores; `stores' = count of stores present in the category). Null = 2\\,000 permutations of region labels across stores (each region keeps its number of stores, each store keeps its catalogue); $p$ is one-sided. Last column: number of the $R\\times C$ cells whose store-bootstrap 95\\% interval for RAA lies entirely above / below 1. Variants from the preregistered robustness list (1\\,000 permutations).", size="\\footnotesize")565758def t_raa_top():59 m = pd.read_csv(PROC / "raa_matrix.csv"); m = m[(m["sample"] == "S_R") & (m.scheme == "M") & m.sig_gt1].sort_values("raa", ascending=False).head(20)60 rows = [[esc(x.region), CAT.get(x.category, x.category), f"{x.raa:.2f}", f"[{x.raa_lo:.2f}; {x.raa_hi:.2f}]", n(x.n, 1)] for _, x in m.iterrows()]61 write("raa_top", ["Region", "Category", "RAA", "95\\% CI", "Merchant-equal weight"], rows, "Largest significant revealed assortment advantages (merchant-equal weights)", "raa_top",62 "Cells with a store-bootstrap 95\\% interval above 1, sorted by RAA. The weight column is the merchant-equal count $N_{rc}$ (sum of store shares).", size="\\footnotesize")636465def t_h3():66 h = J("h3_longtail.json"); pl = h["powerlaw"]; lv = h["lognormal_vs_powerlaw"]67 rows = [["Stores / listings", f"{n(h['stores'])} / {n(h['listings'])}"], ["Gini of listings across stores [95\\% CI]", f"{h['gini']:.3f} [{h['gini_ci'][0]:.3f}; {h['gini_ci'][1]:.3f}]"],68 ["Share of listings, top 1\\% of stores [95\\% CI]", f"{h['top1pct_share']:.3f} [{h['top1_ci'][0]:.3f}; {h['top1_ci'][1]:.3f}]"], ["Share of listings, top 10\\% of stores [95\\% CI]", f"{h['top10pct_share']:.3f} [{h['top10_ci'][0]:.3f}; {h['top10_ci'][1]:.3f}]"],69 ["Median / p90 / p99 / max catalogue size", f"{n(h['median_catalogue'])} / {n(h['p90'])} / {n(h['p99'])} / {n(h['max'])}"], ["Rank--size slope (log--log)", f"{h['rank_size_slope_loglog']:.3f}"],70 ["Discrete power law: $x_{\\min}$, $\\alpha$, KS, tail $n$", f"{pl['xmin']}, {pl['alpha']:.3f}, {pl['ks']:.3f}, {pl['n_tail']}"], ["Vuong LR (power law vs log-normal on the tail): $z$, $p$", f"{lv['vuong_z']:.2f}, {lv['vuong_p']:.2g} ({lv['favoured']})"],71 ["Preregistered criterion (Gini $\\ge$ 0.70 and top-10\\% $\\ge$ 0.50)", "met" if h["decision_prereg"] else "not met"]]72 wr = h["within_region"]; rows2 = [[esc(r), v["stores"], f"{v['gini']:.3f} [{v['gini_ci'][0]:.3f}; {v['gini_ci'][1]:.3f}]" if v.get("gini") is not None else "--", f"{v.get('top10pct_share', float('nan')):.3f}" if v.get("gini") is not None else "--"] for r, v in sorted(wr.items(), key=lambda kv: -(kv[1].get("gini") or 0))]73 write("h3_concentration", ["Statistic", "Value"], rows, "H3 --- Concentration of listings across merchants (assortment sample)", "h3", "Bootstrap CIs from 2\\,000 resamples of stores. Power-law fit by maximum likelihood with $x_{\\min}$ chosen by KS minimisation (Clauset et al.\\ 2009); the log-normal is fitted on the same tail. Inconnue-region stores included (all stores).", align="lr")74 write("h3_within_region", ["Region", "Stores", "Gini [95\\% CI]", "Top-10\\% share"], rows2, "Within-region concentration of catalogue sizes", "h3_region", "500 store-bootstrap resamples per region; regions with fewer than five stores omitted.", size="\\footnotesize")757677def t_h5():78 h = J("h5_hedonic.json"); ms = [m for m in h["models"] if m.get("status") != "insufficient"]79 def row(lbl, f): return [lbl] + [f(m) for m in ms]80 c = lambda m, k: f"{m['coef'][k]['b']:.3f} ({m['coef'][k]['se_cluster']:.3f})"81 rows = [row("Observations", lambda m: n(m["obs"])), row("Match groups", lambda m: n(m["groups"])), row("Stores", lambda m: str(m["stores"])), row("Regions (with $\\ge$ 5 matched stores)", lambda m: f"{m['regions']} ({m['regions_with_ge5_stores']})"),82 row("Wald statistic, region effects (df)", lambda m: f"{m['wald_region']:.1f} ({m['df_region']})"), row("\\quad $p$, wild cluster bootstrap", lambda m: f"{m['p_region_wildboot']:.4f}"),83 row("Reseller / collective store (SE)", lambda m: c(m, "reseller")), row("\\quad $p$, wild cluster bootstrap", lambda m: f"{m['p_reseller_wildboot']:.4f}"), row("Certified store (SE)", lambda m: c(m, "certified")), row("log catalogue size (SE)", lambda m: c(m, "log_n")),84 row("Within-group $R^2$", lambda m: f"{m['within_R2']:.3f}"), row("Within-group gap: median / p90", lambda m: f"{m['gap_median']:.3f} / {m['gap_p90']:.3f}"), row("Share of groups with gap $>$ 5\\%", lambda m: f"{m['share_gap_gt5pct']:.2f}"), row("Share of groups with identical prices", lambda m: f"{m['share_gap_zero']:.2f}")]85 write("h5_hedonic", ["Statistic"] + [m["label"].replace("_", "\\_") for m in ms], rows, "H5 --- Matched-product price dispersion across merchants", "h5",86 "Dependent variable: log posted price at the snapshot. Match-group fixed effects absorbed; region and platform dummies, log catalogue size, reseller/collective and certification indicators. SE clustered by store; $p$-values from a wild cluster bootstrap (Rademacher, 9\\,999 draws, restricted residuals). primary\\_AB = tiers A (shared SKU) and B (same brand, identical title); sens\\_ABC adds fuzzy tier C. Gap = within-group max--min log price.", align="l" + "r" * len(ms))87 cc = h["certification_contrast"]88 write("cert_contrast", ["Statistic", "Value"], [["Observations / stores / certified stores", f"{n(cc['obs'])} / {n(cc['stores'])} / {n(cc['certified_stores'])}"], ["Certified store, log price difference within category (SE, store-clustered)", f"{cc['b_log_price']:.3f} ({cc['se']:.3f})"]],89 "Descriptive store-certification price contrast (not a test of H6)", "cert", esc(cc["note"]), align="lr")909192def t_h7():93 h = J("h7_complexity.json"); v1 = h["V1_bootstrap_rank_stability"]; v2 = h["V2_reshuffle_null"]; v4 = h["V4_community_stability"]94 rows = [["Matrix density (share of cells with RAA $\\ge$ 1)", f"{h['M_density']:.3f}"], ["V1 ECI rank stability: median (p10) Spearman $\\rho$, 500 store bootstraps", f"{v1['eci_rho_median']:.2f} ({v1['eci_rho_p10']:.2f})"],95 ["V1 Fitness / Diversity rank stability (median $\\rho$)", f"{v1['fitness_rho_median']:.2f} / {v1['diversity_rho_median']:.2f}"], ["V1 criterion ($\\rho \\ge 0.8$) for ECI", "met" if v1["pass_eci"] else "not met"],96 ["V2 $\\lambda_2$ of $\\tilde M$: observed, null p95, $p$", f"{v2['obs_lambda2']:.3f}, {v2['null_lambda2_p95']:.3f}, {v2['p_lambda2']:.3f}"], ["V2 spectral radius of the bipartite graph: observed, null p95, $p$", f"{v2['obs_spectral_radius']:.2f}, {v2['null_spectral_radius_p95']:.2f}, {v2['p_spectral_radius']:.3f}"],97 ["V2 corr(ECI, diversity): observed, null [p5; p95]", f"{v2['obs_corr_eci_diversity']:.2f}, [{v2['null_corr_eci_diversity_p5_p95'][0]:.2f}; {v2['null_corr_eci_diversity_p5_p95'][1]:.2f}]"],98 ["V4 Louvain communities: mode of count, median NMI across 200 seeds", f"{v4.get('n_communities_mode')}, {v4.get('nmi_median_vs_run0')}"]]99 ind = pd.read_csv(PROC / "h7_region_indicators.csv").sort_values("eci", ascending=False)100 rows2 = [[esc(r.region), int(r.n_stores), int(r.diversity), f"{r.eci:.2f}", f"{r.fitness:.2f}"] for _, r in ind.iterrows()]101 write("h7_validation", ["Validation statistic", "Value"], rows, "H7 --- Validation of the region--category complexity indicator", "h7", "$M_{rc} = 1[\\mathrm{RAA}_{rc} \\ge 1]$ on merchant-equal weights, regional sample. Null = 2\\,000 store-level region reshuffles. Per the preregistration, the ECI is reported as unstable when V1 fails and is omitted from the QDACI.", align="lr")102 write("h7_indicators", ["Region", "Stores", "Diversity", "ECI", "Fitness"], rows2, "Region-level indicators (reported for transparency; unstable under bootstrap)", "h7ind", "Diversity = number of categories with RAA $\\ge 1$. ECI standardised; sign fixed so that corr(ECI, diversity) $\\ge 0$.", size="\\footnotesize")103104105def t_h4():106 h = J("h4_dynamics.json")107 rows = [["Baseline / outcome windows", f"{h['baseline_window'][0]} to {h['baseline_window'][1]} / {h['outcome_window'][0]} to {h['outcome_window'][1]}"], ["Stores; (store, category) cells absent at baseline", f"{n(h['stores'])}; {n(h['store_category_cells_absent_at_baseline'])}"],108 ["Entries (cells); entering stores", f"{h['entries']}; {h['entering_stores']}"], ["Mean density, cells with / without entry", f"{h['density_mean_entry']:.3f} / {h['density_mean_no_entry']:.3f}"],109 ["LPM coefficient on density (store-clustered SE); $t$", f"{h['lpm_density_coef']:.3f} ({h['se_cluster_store']:.3f}); {h['t']:.2f}"], ["Placebo (category-relabelled proximity, 200 draws): mean, p95, $p$", f"{h['placebo_coef_mean']:.3f}, {h['placebo_p95']:.3f}, {h['placebo_p']:.3f}"],110 ["Region-level category entries / exits", f"{len(h['region_level_entries'])} / {len(h['region_level_exits'])}"], ["Share of entering stores with an `added' burst $>$50\\% of baseline catalogue", f"{h['share_entering_stores_with_added_burst']:.2f}"]]111 write("h4_dynamics", ["Statistic", "Value"], rows, "H4 (exploratory) --- Entry into related categories within stores", "h4", "Linear probability model with store and category fixed effects; density = share of a category's proximity mass covered by the store's baseline portfolio (Hidalgo-type proximity from co-occurrence across stores). Exploratory by preregistration: no causal claim.", align="lr")112113114def t_audits():115 p = BASE_EXP / "category_audit.json"116 if p.exists():117 a = json.load(open(p)); pc = a["per_category_precision_vs_consensus"]118 rows = [[CAT.get(k, k), v["n_consensus"], f"{v['precision']:.2f}" if v["precision"] is not None else "--"] for k, v in pc.items()]119 rows.append(["\\textbf{All}", int(a["n"] * a["consensus_share"]), f"\\textbf{{{a['rule_precision_vs_consensus']:.2f}}}"])120 write("category_audit", ["Rule label", "Listings with annotator consensus", "Precision of the rule label"], rows, "Audit of the rule-based category labels (500-listing stratified sample)", "cat_audit",121 f"25 random listings per category annotated independently by two LLMs (claude-opus-5-5, gpt-5.5) with the platform's taxonomy; agreement rule vs A {a['agree_category_llm_a']:.2f}, rule vs B {a['agree_category_llm_b']:.2f}, A vs B {a['agree_llm_a_llm_b']:.2f}; consensus exists for {100*a['consensus_share']:.0f}\\% of listings. Precision = share of consensus labels equal to the rule label. Disagreements listed in the replication package.", size="\\footnotesize")122 p = BASE_EXP / "match_validation.json"123 if p.exists():124 m = json.load(open(p)); pt = m["precision_by_tier_consensus"]125 rows = [[t, v["n"], f"{v['yes']:.2f}", f"{v['no']:.2f}", f"{m['precision_by_tier_comparable_only'].get(t, float('nan')):.2f}"] for t, v in pt.items()]126 write("match_validation", ["Tier", "Pairs with consensus", "Same product", "Different", "Same product, comparable-variant pairs"], rows, "Validation of the cross-merchant matching tiers (60 pairs per tier)", "match_val",127 f"Two independent LLM annotators (agreement {m['agree_llm']:.2f}; consensus on {100*m['consensus_share']:.0f}\\% of pairs). Tier A = shared merchant SKU; B = same brand and identical normalised title; C = same brand, fuzzy title; D = identical title without brand (never used for inference).", size="\\footnotesize")128129130def t_h2():131 p = EXP / "h2_scale.json"132 if not p.exists(): return133 h = J("h2_scale.json"); rows = []134 for k, v in h["correlations"].items():135 rows.append([esc(k), v["n"], f"{v['rho']:.2f}", f"[{v['ci'][0]:.2f}; {v['ci'][1]:.2f}]", f"{v['ols_b']:.3f} ({v['ols_se_hc3']:.3f})" if v.get("ols_b") is not None else "--"])136 write("h2_scale", ["Outcome $\\sim$ covariate", "$n$", "Spearman $\\rho$", "95\\% CI", "OLS slope (HC3 SE)"], rows, "H2 --- Regional assortment diversity and regional scale", "h2", esc(h.get("note", "")), size="\\footnotesize")137138139def t_qdaci():140 p = EXP / "qdaci.json"141 if not p.exists(): return142 q = J("qdaci.json"); comp = pd.read_csv(PROC / "qdaci_components.csv")143 cols = [c for c in comp.columns if c not in ("region",)]144 rows = [[esc(r.region)] + [f"{r[c]:.2f}" if isinstance(r[c], float) and not np.isnan(r[c]) else ("--" if isinstance(r[c], float) else str(r[c])) for c in cols] for _, r in comp.iterrows()]145 write("qdaci", ["Region"] + [esc(c) for c in cols], rows, "QDACI components by region", "qdaci", esc(q.get("note", "")), size="\\scriptsize", align="l" + "r" * len(cols))146147148def t_external():149 p = EXP / "external_validation.json"150 if not p.exists(): return151 e = J("external_validation.json"); rows = [[esc(r["region"]), n(r["stores"]), n(r["businesses"]) if r.get("businesses") else "--", f"{r['stores_per_1000_businesses']:.2f}" if r.get("stores_per_1000_businesses") else "--", n(r["population"]) if r.get("population") else "--", f"{r['stores_per_100k_pop']:.1f}" if r.get("stores_per_100k_pop") else "--"] for r in e["rows"]]152 write("external_coverage", ["Region", "Fabri-Ka stores", "Business locations", "Stores per 1\\,000 businesses", "Population", "Stores per 100\\,000 pop."], rows, "Coverage of the regional sample against external business and population counts", "external", esc(e.get("note", "")), size="\\footnotesize")153154155def main():156 for f in (t_sample_flow, t_region_summary, t_h1, t_raa_top, t_h3, t_h5, t_h7, t_h4, t_audits, t_h2, t_qdaci, t_external):157 try: f()158 except Exception as ex: # noqa: BLE001159 log.error("%s failed: %s", f.__name__, ex)160 ledger("reports.make_tables", tables=sorted(p.name for p in TAB.glob("*.tex")))161162163if __name__ == "__main__":164 main()165