1"""Diversity and concentration by region (S_R): normalised Shannon entropy (schemes L and M), effective number of categories,2HHI of category shares, rarefied entropy at m = 15 stores (2 000 draws), store-bootstrap 95 % CIs. Output data/processed/diversity_region.csv"""3from __future__ import annotations4import json5import numpy as np, pandas as pd6from src.common import ROOT, SEED, get_logger, ledger, register_output, write_json, PROC_DIR, EXP_DIR, P78log = get_logger("features.diversity")9PROC = PROC_DIR10M_RAREFY, NB = 15, 2000111213def entropy(shares):14 s = shares[shares > 0]; return float(-(s * np.log(s)).sum())151617def region_stats(d, scheme):18 w = d.n.astype(float) if scheme == "L" else d.n / d.groupby("store_id").n.transform("sum")19 t = w.groupby(d.category).sum(); p = (t / t.sum()).values; K = len(p)20 H = entropy(p); return {"H": H, "H_norm": H / np.log(20), "eff_cat": float(np.exp(H)), "hhi": float((p ** 2).sum()), "n_cat": int((p > 0).sum())}212223def main():24 rng = np.random.default_rng(SEED)25 sc = pd.read_parquet(PROC / "store_category.parquet"); d = sc[sc.region_final != "Inconnue"]26 rows = []27 for r, g in d.groupby("region_final"):28 stores = g.store_id.unique(); ns = len(stores); groups = {s: h for s, h in g.groupby("store_id")}29 rec = {"region": r, "n_stores": ns, "n_listings": int(g.n.sum())}30 for scheme in ("L", "M"):31 st = region_stats(g, scheme); rec.update({f"{k}_{scheme}": v for k, v in st.items()})32 boots = np.array([region_stats(pd.concat([groups[s] for s in rng.choice(stores, ns, replace=True)]), scheme)["H_norm"] for _ in range(500)])33 rec[f"H_norm_{scheme}_lo"], rec[f"H_norm_{scheme}_hi"] = float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5))34 if ns >= M_RAREFY:35 rar = np.array([region_stats(pd.concat([groups[s] for s in rng.choice(stores, M_RAREFY, replace=False)]), "M")["H_norm"] for _ in range(NB)])36 rec["H_rarefied15"], rec["H_rarefied15_lo"], rec["H_rarefied15_hi"] = float(rar.mean()), float(np.percentile(rar, 2.5)), float(np.percentile(rar, 97.5))37 else:38 rec["H_rarefied15"] = rec["H_rarefied15_lo"] = rec["H_rarefied15_hi"] = np.nan39 rows.append(rec)40 out = pd.DataFrame(rows).sort_values("H_rarefied15", ascending=False)41 out.to_csv(PROC / "diversity_region.csv", index=False); register_output(PROC / "diversity_region.csv", "diversity")42 # Quebec-wide reference and store-level diversity distribution43 allq = region_stats(d, "M"); store_cats = sc.groupby("store_id").category.nunique()44 summ = {"quebec_H_norm_M": round(allq["H_norm"], 4), "quebec_eff_cat_M": round(allq["eff_cat"], 3), "regions_rarefied": int(out.H_rarefied15.notna().sum()),45 "store_n_categories_dist": store_cats.value_counts().sort_index().to_dict(), "range_H_rarefied15": [round(float(out.H_rarefied15.min()), 3), round(float(out.H_rarefied15.max()), 3)]}46 write_json(summ, EXP_DIR / "diversity.json", "diversity"); ledger("features.diversity", **summ); log.info(json.dumps(summ))474849if __name__ == "__main__":50 main()51