1"""Model E — fixed-basket shopping optimisation (H6). Preregistration §2 H6 and config/basket.yaml (frozen).23For each complete week: map each basket item to candidate listings per banner (category + name regex, parsable size,4unit price), take the cheapest unit price per banner, and solve min-cost assignment with at most K banners (K=1..5) as a5MIP (PuLP/CBC). Travel costs are unobserved: an illustrative fixed per-banner cost grid is added as a labelled assumption.6"""7import json, re, numpy as np, pandas as pd, pulp, yaml8from src.common import path, ROOT, get_logger, ledger9log = get_logger("modelE"); P = path("processed"); E = ROOT / "experiments"1011def candidates(pw, items):12 rows = []13 for it in items:14 d = pw[(pw.category == it["category"]) & pw.name.fillna("").str.contains(it["name_regex"], case=False, regex=True)]15 if it.get("exclude_regex"): d = d[~d.name.fillna("").str.contains(it["exclude_regex"], case=False, regex=True)]16 d = d[(d.unit_base == ("un" if it["base"] == "un" else f"100 {it['base']}")) & d.unit_p.notna() & (d.unit_p > 0)]17 if len(d) == 0: continue18 d = d.assign(item=it["id"], item_cost=lambda x: x.unit_p * (it["qty"] if it["base"] == "un" else it["qty"] / 100.0))19 rows.append(d[["item", "week", "banner", "company", "uid", "name", "size_label", "p_mode", "unit_p", "item_cost"]])20 return pd.concat(rows, ignore_index=True)2122def solve(cost_tbl, K, fixed_cost=0.0):23 """cost_tbl: item x banner matrix of min item cost (NaN = unavailable). Returns (total cost, banners used, items covered)."""24 items = list(cost_tbl.index); banners = list(cost_tbl.columns)25 prob = pulp.LpProblem("basket", pulp.LpMinimize)26 x = {(i, b): pulp.LpVariable(f"x_{n}_{m}", cat="Binary") for n, i in enumerate(items) for m, b in enumerate(banners) if not np.isnan(cost_tbl.at[i, b])}27 y = {b: pulp.LpVariable(f"y_{m}", cat="Binary") for m, b in enumerate(banners)}28 prob += pulp.lpSum(cost_tbl.at[i, b] * v for (i, b), v in x.items()) + fixed_cost * pulp.lpSum(y.values())29 for i in items:30 prob += pulp.lpSum(v for (ii, b), v in x.items() if ii == i) == 131 for (i, b), v in x.items():32 prob += v <= y[b]33 prob += pulp.lpSum(y.values()) <= K34 prob.solve(pulp.PULP_CBC_CMD(msg=False))35 if pulp.LpStatus[prob.status] != "Optimal": return np.nan, [], 036 used = [b for b, v in y.items() if v.value() > 0.5]37 tot = sum(cost_tbl.at[i, b] * v.value() for (i, b), v in x.items())38 return float(tot), used, len(items)3940def main():41 out = {}42 items = yaml.safe_load((ROOT / "config" / "basket.yaml").read_text())["items"]43 pw = pd.read_parquet(P / "panel_weekly.parquet"); sf = pw[pw.channel == "online_storefront"]44 cand = candidates(sf, items); cand.to_parquet(P / "modelE_candidates.parquet", index=False)45 cov = cand.groupby("item").agg(listings=("uid", "nunique"), banners=("banner", "nunique")).reset_index(); cov.to_csv(P / "modelE_item_coverage.csv", index=False)46 log.info("basket items with candidates: %d / %d ; median banners per item %.0f", cov.item.nunique(), len(items), cov.banners.median())47 results = []48 for w, d in cand.groupby("week"):49 tbl = d.groupby(["item", "banner"]).item_cost.min().unstack()50 # keep banners carrying >= 60 % of covered items, and items available at >= 2 banners (so that K=1 is feasible on a common set)51 tbl = tbl.loc[:, tbl.notna().mean() >= 0.6]52 tbl = tbl[tbl.notna().sum(axis=1) >= 2]53 # for K=1 feasibility, restrict to items available at every retained banner54 full = tbl.dropna()55 for K in (1, 2, 3, 4, 5, len(tbl.columns)):56 tot, used, n = solve(full, K)57 results.append(dict(week=w, K=K if K < len(tbl.columns) else 99, cost=tot, banners=";".join(used), n_items=n, n_banners_avail=len(tbl.columns)))58 # single-banner cost for each banner (full item set)59 for b in full.columns:60 results.append(dict(week=w, K=1, cost=float(full[b].sum()), banners=b, n_items=len(full), n_banners_avail=len(tbl.columns), single=True))61 for fc in (5.0, 10.0, 20.0):62 tot, used, n = solve(full, len(full.columns), fixed_cost=fc)63 results.append(dict(week=w, K=99, cost=tot - fc * len(used), banners=";".join(used), n_items=n, n_banners_avail=len(tbl.columns), fixed_cost=fc, n_used=len(used), cost_incl_travel=tot))64 r = pd.DataFrame(results); r.to_csv(P / "modelE_results.csv", index=False)65 base = r[(r.get("single").isna()) & (r.fixed_cost.isna())] if "fixed_cost" in r else r[r.get("single").isna()]66 front = base.groupby("K").cost.mean(); singles = r[r.get("single") == True].groupby("week").cost.agg(["min", "max", "mean"])67 out["items_in_common_set_mean"] = float(base.n_items.mean()); out["banners_available_mean"] = float(base.n_banners_avail.mean())68 out["frontier_mean_cost_by_K"] = {int(k): float(v) for k, v in front.items()}69 c1, c2, cinf = front.get(1), front.get(2), front.get(99)70 out["savings_unconstrained_vs_best_single"] = float(c1 - cinf); out["savings_share_unconstrained"] = float((c1 - cinf) / c1)71 out["share_of_unconstrained_savings_at_K2"] = float((c1 - c2) / (c1 - cinf)) if c1 - cinf > 0 else None72 out["single_banner_cost_spread_mean"] = float((singles["max"] - singles["min"]).mean()); out["single_banner_mean_cost"] = float(singles["mean"].mean())73 out["most_expensive_vs_cheapest_single_share"] = float(((singles["max"] - singles["min"]) / singles["min"]).mean())74 if "fixed_cost" in r:75 tc = r[r.fixed_cost.notna()].groupby("fixed_cost").agg(n_used=("n_used", "mean"), cost_goods=("cost", "mean"), cost_incl=("cost_incl_travel", "mean")).reset_index()76 tc.to_csv(P / "modelE_travel_sensitivity.csv", index=False); out["travel_cost_sensitivity"] = tc.to_dict(orient="records")77 (E / "modelE_summary.json").write_text(json.dumps(out, indent=2, default=float))78 log.info("E: common items %.1f, banners %.1f ; cost K=1 %.2f, K=2 %.2f, unconstrained %.2f ; unconstrained savings %.1f%% ; share captured at K=2 %.2f ; single-banner spread %.2f (%.1f%%)",79 out["items_in_common_set_mean"], out["banners_available_mean"], c1, c2, cinf, 100 * out["savings_share_unconstrained"], out["share_of_unconstrained_savings_at_K2"] or float("nan"), out["single_banner_cost_spread_mean"], 100 * out["most_expensive_vs_cheapest_single_share"])80 ledger("4", "model_e", f"K1 {c1:.2f} K2 {c2:.2f} Kinf {cinf:.2f}")8182if __name__ == "__main__":83 main()84