phase4: models A (dispersion), C (spillovers), D (indices), E (basket MIP), F (forecasts); pulp pinned <3

Simon-Pierre Boucher committed 2 days ago (Nov 1, 2026) parent 8931742 Browse files

17 changed files +930 −2

added experiments/modelA_summary.json +59 −0

++{
++  "gap_all": {
++    "n": 30042,
++    "median": 0.15415067982725827,
++    "p25": 0.05901417143705534,
++    "p75": 0.27866231037551104,
++    "mean": 0.18936608290604118,
++    "share_ge_5pct": 0.7851341455295919,
++    "share_ge_10pct": 0.6325144797283803
++  },
++  "gap_same_company": {
++    "n": 27939,
++    "median": 0.15128944754622609,
++    "p25": 0.057746822212323856,
++    "p75": 0.26844510500133234,
++    "mean": 0.18576393643510483,
++    "share_ge_5pct": 0.7821682952145746,
++    "share_ge_10pct": 0.6273667633057733
++  },
++  "gap_cross_company": {
++    "n": 2103,
++    "median": 0.18254405095440074,
++    "p25": 0.08017111858864046,
++    "p75": 0.3370446464822916,
++    "mean": 0.23722170356771002,
++    "share_ge_5pct": 0.8245363766048502,
++    "share_ge_10pct": 0.7009034712315739
++  },
++  "cv_all": {
++    "n": 30042,
++    "median": 0.07341658510817389,
++    "p25": 0.029429075927015894,
++    "p75": 0.13034226381318856,
++    "mean": 0.08951038693130896,
++    "share_ge_5pct": 0.6284867851674323,
++    "share_ge_10pct": 0.3644231409360229
++  },
++  "banner_fe_reference": "Metro",
++  "resid_sd": 0.11456460507159695,
++  "n_obs_fe": 67732,
++  "ar1_rho": 0.663411776499718,
++  "ar1_se": 0.013989188479587735,
++  "share_pairs_sign_stable_ge4w": 0.457002457002457,
++  "n_pairs_ge4w": 4884,
++  "ar1_same_company": [
++    0.6675381113692006,
++    0.013865058825916333,
++    0.45115007970849463,
++    4391
++  ],
++  "ar1_cross_company": [
++    0.6302714331760938,
++    0.03769326769881746,
++    0.5091277890466531,
++    493
++  ],
++  "placebo_ar1_mean": 0.0035750286222568906,
++  "placebo_ar1_sd": 0.011048954753771625
++}
+ \ No newline at end of file

added experiments/modelC_summary.json +71 −0

++{
++  "rival_all": {
++    "n": 22213,
++    "coefs": {
++      "rival_all": 0.036983611990011225,
++      "rival_all_l1": 0.03674370006018662,
++      "rival_all_l2": 0.014867128258090253,
++      "rival_all_l3": 0.04341238029758149,
++      "rival_all_f1": 0.008899934370312498,
++      "rival_all_f2": 0.013343162704560697
++    },
++    "ses": {
++      "rival_all": 0.018207550537640733,
++      "rival_all_l1": 0.012276621362321993,
++      "rival_all_l2": 0.013107075427690282,
++      "rival_all_l3": 0.014656351032885128,
++      "rival_all_f1": 0.012048599868225932,
++      "rival_all_f2": 0.014684867882272816
++    },
++    "P_own_change_given_rival_change_l1": 0.2527131782945736,
++    "P_own_change_given_no_rival_change_l1": 0.1711969770843491
++  },
++  "rival_same": {
++    "n": 21192,
++    "coefs": {
++      "rival_same": 0.027632240932606675,
++      "rival_same_l1": 0.04166870039441454,
++      "rival_same_l2": 0.01145626003174879,
++      "rival_same_l3": 0.034260009805555276,
++      "rival_same_f1": 0.0132625155508785,
++      "rival_same_f2": 0.014714728442878371
++    },
++    "ses": {
++      "rival_same": 0.01831796970903035,
++      "rival_same_l1": 0.01234955835151791,
++      "rival_same_l2": 0.013246905947066992,
++      "rival_same_l3": 0.015028588991760062,
++      "rival_same_f1": 0.012251592712678097,
++      "rival_same_f2": 0.014337631789336497
++    },
++    "P_own_change_given_rival_change_l1": 0.25367852486496556,
++    "P_own_change_given_no_rival_change_l1": 0.17588320798837137
++  },
++  "rival_cross": {
++    "n": 1784,
++    "coefs": {
++      "rival_cross": 0.08084154607798574,
++      "rival_cross_l1": -0.0012395703633851167,
++      "rival_cross_l2": -0.056863578932101456,
++      "rival_cross_l3": 0.046743672971606504,
++      "rival_cross_f1": -0.10799246348174829,
++      "rival_cross_f2": -0.064020894931646
++    },
++    "ses": {
++      "rival_cross": 0.06613440280922223,
++      "rival_cross_l1": 0.04779603312485358,
++      "rival_cross_l2": 0.028928467120645663,
++      "rival_cross_l3": 0.06013080127915635,
++      "rival_cross_f1": 0.048390891586986506,
++      "rival_cross_f2": 0.04882941084495431
++    },
++    "P_own_change_given_rival_change_l1": 0.2744360902255639,
++    "P_own_change_given_no_rival_change_l1": 0.18370607028753994
++  },
++  "promo_comovement": {
++    "group_weeks": 30042,
++    "share_any_low": 0.16027561414020372,
++    "share_two_plus_low": 0.017209240396777844,
++    "expected_two_plus_if_independent": 0.006207116594008007
++  }
++}
+ \ No newline at end of file

added experiments/modelD_summary.json +59 −0

++{
++  "storefront_posted": {
++    "naive_mean_change": 0.0036769234266957884,
++    "naive_median_change": 0.0,
++    "jevons_chained_change": 0.0028498796180160824,
++    "fixed_jevons_change": 0.000961438796386993,
++    "fixed_dutot_change": -0.0005377093740115413,
++    "fixed_carli_change": 0.026867158956678905,
++    "n_balanced": 44423,
++    "n_total_first_week": 51434
++  },
++  "storefront_unit": {
++    "naive_mean_change": -0.012446313570385947,
++    "naive_median_change": -0.006544530339934296,
++    "jevons_chained_change": 0.0027664755512584183,
++    "fixed_jevons_change": 0.0007395269633583279,
++    "fixed_dutot_change": -0.004284566992962091,
++    "fixed_carli_change": 0.03566130323976746,
++    "n_balanced": 29671,
++    "n_total_first_week": 36003
++  },
++  "all_channels_posted": {
++    "naive_mean_change": -0.02927666201945356,
++    "naive_median_change": 0.0,
++    "jevons_chained_change": 0.0023785412285990404,
++    "fixed_jevons_change": 0.0005360460992416272,
++    "fixed_dutot_change": -0.0008775599602732775,
++    "fixed_carli_change": 0.027756326994421432,
++    "n_balanced": 45214,
++    "n_total_first_week": 57942
++  },
++  "bootstrap": {
++    "gap_mean": 0.0006271123459269828,
++    "gap_ci95": [
++      -0.007830383632211136,
++      0.008431239690139423
++    ],
++    "jevons_ci95": [
++      0.001509196870153784,
++      0.004221513313892511
++    ]
++  },
++  "assortment": {
++    "listings": 71694,
++    "balanced": 44515,
++    "entrants": 20260,
++    "exits": 17295,
++    "weekly_entry_rate": 0.03595173117340451
++  },
++  "size_label_history_available": false,
++  "statcan": {
++    "cpi_qc_food_stores_aug": 194.4,
++    "sep": 193.1,
++    "cpi_change_aug_sep": -0.006687242798353976,
++    "ours_jevons_aug_sep": 0.000991433026574784,
++    "cpi_latest": "2026-09-01",
++    "cpi_yoy_sep": 0.016315789473684283
++  }
++}
+ \ No newline at end of file

added experiments/modelE_summary.json +38 −0

++{
++  "items_in_common_set_mean": 7.181818181818182,
++  "banners_available_mean": 7.7272727272727275,
++  "frontier_mean_cost_by_K": {
++    "1": 23.12194937122785,
++    "2": 20.888101262563136,
++    "3": 20.204123130937315,
++    "4": 20.054139457868775,
++    "5": 20.034210164939484,
++    "99": 20.028755619484937
++  },
++  "savings_unconstrained_vs_best_single": 3.0931937517429127,
++  "savings_share_unconstrained": 0.1337773775939487,
++  "share_of_unconstrained_savings_at_K2": 0.7221817603265276,
++  "single_banner_cost_spread_mean": 39.69805844961497,
++  "single_banner_mean_cost": 32.88295457993973,
++  "most_expensive_vs_cheapest_single_share": 1.856238474704022,
++  "travel_cost_sensitivity": [
++    {
++      "fixed_cost": 5.0,
++      "n_used": 1.0,
++      "cost_goods": 18.12194937122785,
++      "cost_incl": 23.12194937122785
++    },
++    {
++      "fixed_cost": 10.0,
++      "n_used": 1.0,
++      "cost_goods": 13.12194937122785,
++      "cost_incl": 23.12194937122785
++    },
++    {
++      "fixed_cost": 20.0,
++      "n_used": 1.0,
++      "cost_goods": 3.121949371227848,
++      "cost_incl": 23.12194937122785
++    }
++  ]
++}
+ \ No newline at end of file

added experiments/modelF_summary.json +104 −0

++{
++  "summary": [
++    {
++      "target": "log_price",
++      "model": "banner_cat_drift",
++      "origins": 4,
++      "n": 215264,
++      "mae": 0.024807352788469867,
++      "rmse": 0.10842948605770154,
++      "brier": NaN,
++      "auc": NaN
++    },
++    {
++      "target": "log_price",
++      "model": "lightgbm",
++      "origins": 4,
++      "n": 215264,
++      "mae": 0.030072224891589998,
++      "rmse": 0.10001695748926878,
++      "brier": NaN,
++      "auc": NaN
++    },
++    {
++      "target": "log_price",
++      "model": "mean_reversion_ref",
++      "origins": 4,
++      "n": 215264,
++      "mae": 0.02837850853015459,
++      "rmse": 0.10249483142934793,
++      "brier": NaN,
++      "auc": NaN
++    },
++    {
++      "target": "log_price",
++      "model": "naive_last",
++      "origins": 4,
++      "n": 215264,
++      "mae": 0.022605082941485766,
++      "rmse": 0.10825508687878052,
++      "brier": NaN,
++      "auc": NaN
++    },
++    {
++      "target": "log_price",
++      "model": "ridge",
++      "origins": 4,
++      "n": 215264,
++      "mae": 0.029885922259458478,
++      "rmse": 0.09917790589117646,
++      "brier": NaN,
++      "auc": NaN
++    },
++    {
++      "target": "promotion",
++      "model": "base_rate",
++      "origins": 4,
++      "n": 215264,
++      "mae": NaN,
++      "rmse": NaN,
++      "brier": 0.0568232667303276,
++      "auc": 0.5
++    },
++    {
++      "target": "promotion",
++      "model": "hist_rate",
++      "origins": 4,
++      "n": 215264,
++      "mae": NaN,
++      "rmse": NaN,
++      "brier": 0.057192760775280654,
++      "auc": 0.6778171962496753
++    },
++    {
++      "target": "promotion",
++      "model": "lightgbm",
++      "origins": 4,
++      "n": 215264,
++      "mae": NaN,
++      "rmse": NaN,
++      "brier": 0.03127230289344953,
++      "auc": 0.9359091370790923
++    },
++    {
++      "target": "promotion",
++      "model": "logit",
++      "origins": 4,
++      "n": 215264,
++      "mae": NaN,
++      "rmse": NaN,
++      "brier": 0.04392957451269847,
++      "auc": 0.8939986946425332
++    },
++    {
++      "target": "promotion",
++      "model": "persist",
++      "origins": 4,
++      "n": 215264,
++      "mae": NaN,
++      "rmse": NaN,
++      "brier": 0.050992947231181564,
++      "auc": 0.7518564786841642
++    }
++  ]
++}
+ \ No newline at end of file

modified pyproject.toml +1 −1

				`@@ -14,7 +14,7 @@ dependencies = [`
14	"lightgbm>=4.3",	14	"lightgbm>=4.3",
15	"matplotlib>=3.9",	15	"matplotlib>=3.9",
16	"duckdb>=1.0",	16	"duckdb>=1.0",
17	− "pulp>=2.8",	17	+ "pulp>=2.8,<3",
18	"pyyaml>=6.0",	18	"pyyaml>=6.0",
19	"pytest>=8.0",	19	"pytest>=8.0",
20	"requests>=2.32",	20	"requests>=2.32",

modified requirements-lock.txt +1 −1

				`@@ -57,7 +57,7 @@ pluggy==1.7.0`
57	prompt-toolkit==3.0.53	57	prompt-toolkit==3.0.53
58	psutil==7.2.2	58	psutil==7.2.2
59	ptyprocess==0.7.0	59	ptyprocess==0.7.0
60	−pulp==4.2.0	60	+pulp==2.9.0
61	pure-eval==0.2.4	61	pure-eval==0.2.4
62	pyarrow==26.0.0	62	pyarrow==26.0.0
63	pycparser==3.12	63	pycparser==3.12

added src/econometrics/__init__.py +0 −0

added src/econometrics/build_matched_panel.py +58 −0

++"""Phase 4.0 — analysis-ready weekly panels (preregistration §1).
++
++Outputs (data/processed):
++  panel_weekly.parquet            — grocery-universe listing × complete circular week, posted price = weekly mode, unit prices,
++                                    banner/company/channel/category, size, flags
++  panel_matched_<TIERS>.parquet   — panel_weekly restricted to listings in cross-banner match groups (A, AB, ABC, ABCD),
++                                    with group_id and `ambiguous` flag (group has >1 simultaneously posted listing from the
++                                    same banner-channel in that week)
++"""
++import numpy as np, pandas as pd
++from src.common import path, get_logger, ledger, SETTINGS, write_manifest
++log = get_logger("panel4")
++P = path("processed")
++
++def main():
++    wk = pd.read_parquet(P / "offers_weekly.parquet")
++    prod = pd.read_parquet(P / "products.parquet")
++    ret = pd.read_parquet(P / "retailers.parquet")
++    prod = prod.merge(ret[["feed_id", "banner", "company", "channel", "store_format", "is_grocery"]], left_on="source", right_on="feed_id", how="left")
++    sc = SETTINGS["scope"]
++    uni = prod[(~prod.source.isin(sc["exclude_sources"])) & prod.is_grocery & (~prod.category.isin(sc["non_grocery_categories"])) & (~prod.flag_unrealistic)]
++    log.info("grocery universe listings: %d (of %d)", len(uni), len(prod))
++    weeks = sorted(wk.week.unique())
++    complete = [w for w in weeks if w >= pd.Timestamp("2026-08-13").date() and w <= pd.Timestamp("2026-10-22").date()]
++    pw = wk[wk.week.isin(complete) & wk.uid.isin(uni.uid)].copy()
++    cols = ["uid", "source", "banner", "company", "channel", "store_format", "category", "name", "brand", "size_label", "size_qty", "size_base", "size_pack",
++            "per_weight_label", "flag_weighted", "flag_multibuy", "flag_loyalty", "flag_online_only", "gtin_valid", "active", "on_sale", "regular_price", "price"]
++    pw = pw.merge(uni[cols].rename(columns={"source": "source_p", "price": "price_now", "on_sale": "on_sale_now", "regular_price": "regular_now"}), on="uid", how="left").drop(columns=["source_p"])
++    pw["week"] = pd.to_datetime(pw.week); pw["week_idx"] = pw.week.rank(method="dense").astype(int)
++    pw["log_p"] = np.log(pw.p_mode)
++    qty = pw.size_qty.where(pw.size_base != "un", pw.size_qty)
++    pw["unit_p"] = np.where(pw.size_base == "un", pw.p_mode / pw.size_qty, pw.p_mode / pw.size_qty * 100.0)
++    pw.loc[pw.per_weight_label.eq("kg"), "unit_p"] = pw.p_mode / 10.0
++    pw.loc[pw.per_weight_label.eq("lb"), "unit_p"] = pw.p_mode / 4.53592
++    pw["unit_base"] = np.where(pw.size_base == "un", "un", np.where(pw.size_base == "ml", "100 ml", "100 g"))
++    pw.loc[pw.per_weight_label.isin(["kg", "lb"]), "unit_base"] = "100 g"
++    pw["log_unit_p"] = np.log(pw.unit_p)
++    pw["multi_price_week"] = pw.n_distinct > 1
++    pw.to_parquet(P / "panel_weekly.parquet", index=False)
++    log.info("panel_weekly: %d listing-weeks, %d listings, %d weeks (%s → %s), %d banners", len(pw), pw.uid.nunique(), pw.week.nunique(), pw.week.min().date(), pw.week.max().date(), pw.banner.nunique())
++    files = [P / "panel_weekly.parquet"]
++    for tiers in ["A", "AB", "ABC", "ABCD"]:
++        g = pd.read_parquet(P / f"match_groups_{tiers}.parquet")[["group_id", "uid"]]
++        pm = pw.merge(g, on="uid", how="inner")
++        # keep group-weeks with >= 2 banners posting
++        nb = pm.groupby(["group_id", "week"]).banner.transform("nunique"); pm = pm[nb >= 2].copy()
++        dup = pm.groupby(["group_id", "week", "banner", "channel"]).uid.transform("size") > 1
++        amb = pm.assign(d=dup).groupby(["group_id", "week"]).d.transform("max"); pm["ambiguous"] = amb.astype(bool)
++        pm["same_company_group"] = pm.groupby(["group_id", "week"]).company.transform("nunique") == 1
++        pm.to_parquet(P / f"panel_matched_{tiers}.parquet", index=False); files.append(P / f"panel_matched_{tiers}.parquet")
++        log.info("panel_matched_%-4s: %d listing-weeks, %d groups, %d group-weeks, ambiguous group-weeks %.1f%%, same-company-only group-weeks %.1f%%",
++                 tiers, len(pm), pm.group_id.nunique(), pm.groupby(["group_id", "week"]).ngroups,
++                 100 * pm.drop_duplicates(["group_id", "week"]).ambiguous.mean(), 100 * pm.drop_duplicates(["group_id", "week"]).same_company_group.mean())
++    write_manifest("panel4", files)
++    ledger("4", "build_matched_panel", f"{len(pw)} listing-weeks; weeks {pw.week.min().date()}..{pw.week.max().date()}")
++
++if __name__ == "__main__":
++    main()

added src/econometrics/model_a_dispersion.py +108 −0

++"""Model A — matched-product cross-banner price dispersion (H1). Preregistration §2 H1.
++
++Outputs: data/processed/modelA_*.csv and experiments/modelA_summary.json
++"""
++import json, re, numpy as np, pandas as pd, pyfixest as pf
++from src.common import path, ROOT, get_logger, ledger, seed_everything, SEED
++log = get_logger("modelA"); P = path("processed"); E = ROOT / "experiments"; E.mkdir(exist_ok=True)
++
++def group_week_stats(pm):
++    g = pm.groupby(["group_id", "week"])
++    st = g.agg(n_banners=("banner", "nunique"), n_listings=("uid", "size"), gap=("log_p", lambda s: s.max() - s.min()),
++               cv=("p_mode", lambda s: s.std(ddof=0) / s.mean()), iqr=("log_p", lambda s: s.quantile(.75) - s.quantile(.25)),
++               same_company=("company", lambda s: s.nunique() == 1), category=("category", "first"), ambiguous=("ambiguous", "max")).reset_index()
++    return st
++
++def pair_gaps(pm):
++    """Banner-pair log gaps per group-week (one listing per banner-channel; ambiguous group-weeks dropped upstream)."""
++    rows = []
++    for (gid, w), g in pm.groupby(["group_id", "week"]):
++        g = g.drop_duplicates("banner")
++        b = g.set_index("banner")
++        bs = sorted(b.index)
++        for i in range(len(bs)):
++            for j in range(i + 1, len(bs)):
++                rows.append((gid, w, bs[i], bs[j], b.at[bs[i], "log_p"] - b.at[bs[j], "log_p"], b.at[bs[i], "company"] == b.at[bs[j], "company"], g.category.iloc[0]))
++    return pd.DataFrame(rows, columns=["group_id", "week", "banner_r", "banner_s", "gap", "same_company", "category"])
++
++def persistence(pg):
++    pg = pg.sort_values(["group_id", "banner_r", "banner_s", "week"])
++    pg["gap_lag"] = pg.groupby(["group_id", "banner_r", "banner_s"]).gap.shift(1)
++    pg["week_lag"] = pg.groupby(["group_id", "banner_r", "banner_s"]).week.shift(1)
++    ok = pg[(pg.week - pg.week_lag).dt.days == 7].dropna(subset=["gap_lag"])
++    fit = pf.feols("gap ~ gap_lag", data=ok, vcov={"CRV1": "group_id"})
++    rho = float(fit.coef()["gap_lag"]); se = float(fit.se()["gap_lag"])
++    sign = pg.dropna(subset=["gap"]).groupby(["group_id", "banner_r", "banner_s"]).gap.agg(lambda s: (np.sign(s).nunique() == 1) and (s.abs() > 1e-9).all() and len(s) >= 4)
++    nflip = pg.groupby(["group_id", "banner_r", "banner_s"]).gap.agg(lambda s: ((np.sign(s) != np.sign(s.shift(1))) & s.shift(1).notna()).sum())
++    nweeks = pg.groupby(["group_id", "banner_r", "banner_s"]).size()
++    stable = sign[nweeks >= 4].mean()
++    return rho, se, float(stable), int((nweeks >= 4).sum()), ok
++
++def main():
++    seed_everything()
++    out = {}
++    pm_all = pd.read_parquet(P / "panel_matched_AB.parquet")
++    pm = pm_all[(pm_all.channel == "online_storefront") & (~pm_all.ambiguous)].copy()
++    nb = pm.groupby(["group_id", "week"]).banner.transform("nunique"); pm = pm[nb >= 2]
++    log.info("H1 primary sample: %d listing-weeks, %d groups, %d group-weeks", len(pm), pm.group_id.nunique(), pm.groupby(["group_id", "week"]).ngroups)
++    st = group_week_stats(pm); st.to_csv(P / "modelA_groupweek_stats.csv", index=False)
++    def q(s): return dict(n=int(len(s)), median=float(s.median()), p25=float(s.quantile(.25)), p75=float(s.quantile(.75)), mean=float(s.mean()), share_ge_5pct=float((s >= 0.05).mean()), share_ge_10pct=float((s >= 0.10).mean()))
++    out["gap_all"] = q(st.gap); out["gap_same_company"] = q(st[st.same_company].gap); out["gap_cross_company"] = q(st[~st.same_company].gap)
++    out["cv_all"] = q(st.cv)
++    bycat = st.groupby("category").gap.agg(["size", "median", "mean"]).reset_index().sort_values("median", ascending=False); bycat.to_csv(P / "modelA_gap_by_category.csv", index=False)
++    # Banner fixed effects: log_p ~ 1 | group_week + banner (reference banner = most frequent)
++    pm["gw"] = pm.group_id + "_" + pm.week.dt.strftime("%Y%m%d")
++    ref = pm.banner.value_counts().index[0]
++    pm["banner_f"] = pd.Categorical(pm.banner, categories=[ref] + sorted(set(pm.banner) - {ref}))
++    fit = pf.feols("log_p ~ i(banner_f, ref=" + repr(ref) + ") | gw", data=pm, vcov={"CRV1": "group_id"})
++    co = fit.coef(); se = fit.se()
++    fe = pd.DataFrame({"term": co.index, "coef": co.values, "se": se.values}); fe["banner"] = fe.term.map(lambda t: re.sub(r"^.*?::|^.*\[T\.|\]$", "", t))
++    fe["n_listing_weeks"] = fe.banner.map(pm.banner.value_counts()); fe = fe.sort_values("coef"); fe.to_csv(P / "modelA_banner_effects.csv", index=False)
++    out["banner_fe_reference"] = ref; out["resid_sd"] = float(np.sqrt(np.mean(fit.resid() ** 2))); out["n_obs_fe"] = int(fit._N)
++    # two-way cluster sensitivity (group + banner)
++    try:
++        fit2 = pf.feols("log_p ~ i(banner_f, ref=" + repr(ref) + ") | gw", data=pm, vcov={"CRV1": "group_id + banner"})
++        fe["se_twoway"] = fe.term.map(fit2.se()); fe.to_csv(P / "modelA_banner_effects.csv", index=False)
++    except Exception as e:
++        log.warning("two-way clustering failed: %s", e)
++    # Persistence
++    pg = pair_gaps(pm); pg.to_parquet(P / "modelA_pair_gaps.parquet", index=False)
++    rho, se_rho, stable, npairs, ok = persistence(pg)
++    out["ar1_rho"] = rho; out["ar1_se"] = se_rho; out["share_pairs_sign_stable_ge4w"] = stable; out["n_pairs_ge4w"] = npairs
++    rho_same, se_same, st_same, n_same, _ = persistence(pg[pg.same_company]); rho_x, se_x, st_x, n_x, _ = persistence(pg[~pg.same_company])
++    out["ar1_same_company"] = [rho_same, se_same, st_same, n_same]; out["ar1_cross_company"] = [rho_x, se_x, st_x, n_x]
++    # Placebo: permute banner labels within group-week, recompute AR(1)
++    rng = np.random.default_rng(SEED); rhos = []
++    for b in range(20):
++        pp = pm.copy(); pp["banner"] = pp.groupby(["group_id", "week"]).banner.transform(lambda s: rng.permutation(s.values))
++        pgp = pair_gaps(pp); r, _, _, _, _ = persistence(pgp); rhos.append(r)
++    out["placebo_ar1_mean"] = float(np.mean(rhos)); out["placebo_ar1_sd"] = float(np.std(rhos))
++    # Banner-pair table (median gap and persistence) for the heatmap
++    pairtab = pg.groupby(["banner_r", "banner_s"]).agg(n=("gap", "size"), median_gap=("gap", "median"), mean_gap=("gap", "mean"), share_r_cheaper=("gap", lambda s: (s < 0).mean())).reset_index()
++    pairtab.to_csv(P / "modelA_banner_pairs.csv", index=False)
++    # Within-banner store-mixture bound (storefront, grocery universe)
++    pw = pd.read_parquet(P / "panel_weekly.parquet", columns=["banner", "channel", "n_distinct", "rel_range", "p_min", "p_max", "p_mode"])
++    sf = pw[pw.channel == "online_storefront"]
++    sm = sf.groupby("banner").agg(listing_weeks=("rel_range", "size"), share_multi=("n_distinct", lambda s: (s > 1).mean()),
++                                  median_range_when_multi=("rel_range", lambda s: s[s > 0].median()), median_log_range_multi=("p_max", lambda s: np.nan)).reset_index()
++    sf2 = sf[sf.n_distinct > 1].assign(lr=lambda d: np.log(d.p_max / d.p_min)); sm["median_log_range_multi"] = sm.banner.map(sf2.groupby("banner").lr.median())
++    sm.sort_values("share_multi", ascending=False).to_csv(P / "modelA_within_banner_mixture.csv", index=False)
++    # Sensitivity: tiers and price definitions
++    sens = []
++    for tiers in ["A", "AB", "ABC", "ABCD"]:
++        for pdef in ["p_mode", "p_min", "p_max", "p_twmean"]:
++            x = pd.read_parquet(P / f"panel_matched_{tiers}.parquet"); x = x[(x.channel == "online_storefront") & (~x.ambiguous)]
++            x = x[x.groupby(["group_id", "week"]).banner.transform("nunique") >= 2]
++            lg = np.log(x[pdef]); gap = lg.groupby([x.group_id, x.week]).agg(lambda s: s.max() - s.min())
++            sens.append(dict(tiers=tiers, price_def=pdef, group_weeks=len(gap), median_gap=gap.median(), share_ge_5pct=(gap >= .05).mean()))
++    for inc in ["with_circulars"]:
++        x = pm_all[~pm_all.ambiguous]; x = x[x.groupby(["group_id", "week"]).banner.transform("nunique") >= 2]
++        gap = x.groupby(["group_id", "week"]).log_p.agg(lambda s: s.max() - s.min()); sens.append(dict(tiers="AB", price_def="p_mode+circulars", group_weeks=len(gap), median_gap=gap.median(), share_ge_5pct=(gap >= .05).mean()))
++    pd.DataFrame(sens).to_csv(P / "modelA_sensitivity.csv", index=False)
++    (E / "modelA_summary.json").write_text(json.dumps(out, indent=2, default=float))
++    log.info("H1: median gap %.3f (same-co %.3f, cross-co %.3f); share>=5%% %.2f; AR1 %.3f (se %.3f), placebo %.3f; sign-stable pairs %.2f (n=%d)",
++             out["gap_all"]["median"], out["gap_same_company"]["median"], out["gap_cross_company"]["median"], out["gap_all"]["share_ge_5pct"], rho, se_rho, out["placebo_ar1_mean"], stable, npairs)
++    ledger("4", "model_a", f"median gap {out['gap_all']['median']:.3f}; AR1 {rho:.3f}")
++
++if __name__ == "__main__":
++    main()

added src/econometrics/model_b_promotions.py +134 −0

++"""Model B — promotion dynamics (H2, H3). Preregistration §2 H2–H3.
++
++Storefront promotion state is inferred: week promoted if p_mode <= 0.90 x reference, reference = modal p_mode over the
++trailing 8 weeks (incl. current). Circular items carry observed regular prices (original_price) and are analysed separately.
++"""
++import json, numpy as np, pandas as pd, pyfixest as pf
++from scipy import stats
++from src.common import path, ROOT, get_logger, ledger, seed_everything, SEED
++log = get_logger("modelB"); P = path("processed"); E = ROOT / "experiments"
++
++def rolling_mode(vals, window=8):
++    out = []
++    for i in range(len(vals)):
++        w = vals[max(0, i - window + 1): i + 1]
++        vc = pd.Series(w).value_counts()
++        top = vc[vc == vc.max()].index
++        out.append(max(top))  # ties -> highest price (reference = regular)
++    return out
++
++def main():
++    seed_everything(); out = {}
++    pw = pd.read_parquet(P / "panel_weekly.parquet")
++    sf = pw[pw.channel == "online_storefront"].sort_values(["uid", "week"]).copy()
++    nw = sf.groupby("uid").week.transform("nunique"); sf = sf[nw >= 8].copy()
++    log.info("H2/H3 storefront sample: %d listing-weeks, %d listings, %d banners", len(sf), sf.uid.nunique(), sf.banner.nunique())
++    sf["ref"] = sf.groupby("uid").p_mode.transform(lambda s: pd.Series(rolling_mode(s.tolist()), index=s.index))
++    sf["promo"] = sf.p_mode <= 0.90 * sf.ref
++    sf["depth"] = np.where(sf.promo, np.log(sf.ref / sf.p_mode), 0.0)
++    sf["log_ref"] = np.log(sf.ref)
++    sf["dlog_p"] = sf.groupby("uid").log_p.diff(); sf["dlog_ref"] = sf.groupby("uid").log_ref.diff()
++    sf["promo_lag"] = sf.groupby("uid").promo.shift(1)
++    # validation vs current on_sale flag (listings active now; last complete week)
++    last = sf[sf.week == sf.week.max()]; last = last[last.active == 1]
++    agree = (last.promo == (last.on_sale_now == 1)).mean()
++    out["promo_vs_on_sale_now_agreement"] = float(agree); out["promo_rate_last_week"] = float(last.promo.mean()); out["on_sale_now_rate"] = float((last.on_sale_now == 1).mean())
++    out["promo_share_listing_weeks"] = float(sf.promo.mean()); out["promo_depth_median"] = float(sf.loc[sf.promo, "depth"].median()); out["promo_depth_mean"] = float(sf.loc[sf.promo, "depth"].mean())
++    # H2: volatility raw vs reference
++    chg = sf.dropna(subset=["dlog_p"])
++    vol = chg.groupby("banner").agg(n=("dlog_p", "size"), sd_raw=("dlog_p", "std"), sd_ref=("dlog_ref", "std"), freq_change=("dlog_p", lambda s: (s.abs() > 1e-9).mean()),
++                                    freq_ref_change=("dlog_ref", lambda s: (s.abs() > 1e-9).mean()),
++                                    share_changes_promo=("promo", lambda s: np.nan)).reset_index()
++    chg["is_change"] = chg.dlog_p.abs() > 1e-9; chg["promo_transition"] = chg.promo != chg.promo_lag
++    vol["share_changes_promo"] = vol.banner.map(chg[chg.is_change].groupby("banner").promo_transition.mean())
++    vol["ratio_raw_ref"] = vol.sd_raw / vol.sd_ref; vol.sort_values("n", ascending=False).to_csv(P / "modelB_volatility_by_banner.csv", index=False)
++    volc = chg.groupby("category").agg(n=("dlog_p", "size"), sd_raw=("dlog_p", "std"), sd_ref=("dlog_ref", "std")).reset_index(); volc["ratio"] = volc.sd_raw / volc.sd_ref
++    volc.to_csv(P / "modelB_volatility_by_category.csv", index=False)
++    out["sd_raw_all"] = float(chg.dlog_p.std()); out["sd_ref_all"] = float(chg.dlog_ref.std()); out["ratio_all"] = out["sd_raw_all"] / out["sd_ref_all"]
++    out["share_weekly_changes_promo_transition"] = float(chg[chg.is_change].promo_transition.mean()); out["freq_weekly_change"] = float(chg.is_change.mean())
++    # bootstrap over listings for the ratio
++    rng = np.random.default_rng(SEED); uids = chg.uid.unique(); byuid = {u: g for u, g in chg.groupby("uid")}
++    rat = []
++    for b in range(300):
++        samp = rng.choice(uids, size=len(uids), replace=True); d = pd.concat([byuid[u] for u in samp])
++        rat.append(d.dlog_p.std() / d.dlog_ref.std())
++    out["ratio_ci95"] = [float(np.percentile(rat, 2.5)), float(np.percentile(rat, 97.5))]
++    # Falsification: single-store online shops (Shopify/WooCommerce) vs anti-bot storefronts — share of multi-price weeks
++    single = ["giant_tiger", "pa", "epipresto", "nuvo", "boite_a_grains", "bocoboco", "aliments_merci", "akhavan", "mayrand", "aubut", "maturin", "tau", "loco", "avril"]
++    out["multi_price_week_share_single_store"] = float(sf[sf.source.isin(single)].multi_price_week.mean()); out["multi_price_week_share_antibot"] = float(sf[~sf.source.isin(single)].multi_price_week.mean())
++    # Transition matrix
++    tm = pd.crosstab(sf.promo_lag.dropna().astype(bool), sf.loc[sf.promo_lag.notna(), "promo"], normalize="index"); tm.to_csv(P / "modelB_transition_matrix.csv")
++    out["P_stay_promo"] = float(tm.loc[True, True]) if True in tm.index else None; out["P_enter_promo"] = float(tm.loc[False, True]) if False in tm.index else None
++    # Episodes
++    eps = []
++    for uid, g in sf.groupby("uid"):
++        g = g.reset_index(drop=True); pr = g.promo.values; i = 0
++        while i < len(pr):
++            if pr[i]:
++                j = i
++                while j + 1 < len(pr) and pr[j + 1]: j += 1
++                censored_start = i == 0; censored_end = j == len(pr) - 1
++                pre_ref = g.ref.iloc[i - 1] if i > 0 else np.nan; post_ref = g.ref.iloc[j + 1] if j + 1 < len(pr) else np.nan
++                post_p = g.p_mode.iloc[j + 1] if j + 1 < len(pr) else np.nan
++                eps.append(dict(uid=uid, banner=g.banner.iloc[0], category=g.category.iloc[0], start=g.week.iloc[i], end=g.week.iloc[j], duration=j - i + 1,
++                                depth=g.depth.iloc[i:j + 1].mean(), censored=censored_start or censored_end, pre_ref=pre_ref, post_ref=post_ref, post_p=post_p,
++                                dlog_ref=np.log(post_ref / pre_ref) if pre_ref and post_ref and not (censored_start or censored_end) else np.nan,
++                                dlog_post_price_vs_pre_ref=np.log(post_p / pre_ref) if pre_ref and post_p and not (censored_start or censored_end) else np.nan))
++                i = j + 1
++            else: i += 1
++    ep = pd.DataFrame(eps); ep.to_parquet(P / "modelB_episodes.parquet", index=False)
++    unc = ep[~ep.censored]
++    out["n_episodes"] = int(len(ep)); out["n_uncensored"] = int(len(unc)); out["duration_median_unc"] = float(unc.duration.median()); out["duration_mean_unc"] = float(unc.duration.mean())
++    out["share_duration_1w"] = float((unc.duration == 1).mean()); out["depth_median_unc"] = float(unc.depth.median())
++    rec = ep.sort_values(["uid", "start"]); rec["prev_end"] = rec.groupby("uid").end.shift(1); rec["recur_weeks"] = (rec.start - rec.prev_end).dt.days / 7
++    out["recurrence_median_weeks"] = float(rec.recur_weeks.dropna().median()) if rec.recur_weeks.notna().any() else None; out["share_listings_with_2plus_episodes"] = float((ep.groupby("uid").size() >= 2).mean())
++    # H3: reference change after episode
++    d = unc.dlog_ref.dropna(); out["H3_dlog_ref_mean"] = float(d.mean()); out["H3_dlog_ref_se"] = float(d.std() / np.sqrt(len(d))); out["H3_n"] = int(len(d))
++    out["H3_share_ref_up"] = float((d > 1e-9).mean()); out["H3_share_ref_down"] = float((d < -1e-9).mean()); out["H3_share_ref_same"] = float((d.abs() <= 1e-9).mean())
++    d2 = unc.dlog_post_price_vs_pre_ref.dropna(); out["H3_post_price_vs_pre_ref_mean"] = float(d2.mean()); out["H3_post_price_vs_pre_ref_share_up"] = float((d2 > 1e-9).mean())
++    # Event study around episode end (weeks -4..+4), listing FE + week FE; pretrend joint test
++    ev = []
++    for _, e in unc.iterrows():
++        g = sf[sf.uid == e.uid]
++        for k in range(-4, 5):
++            wk = e.end + pd.Timedelta(weeks=k)
++            r = g[g.week == wk]
++            if len(r): ev.append(dict(uid=e.uid, week=wk, k=k, log_p=float(r.log_p.iloc[0]), log_ref=float(r.log_ref.iloc[0])))
++    ev = pd.DataFrame(ev)
++    if len(ev):
++        ev["k_f"] = ev.k.astype(int)
++        fit = pf.feols("log_p ~ i(k_f, ref=-1) | uid + week", data=ev, vcov={"CRV1": "uid"})
++        es = pd.DataFrame({"term": fit.coef().index, "coef": fit.coef().values, "se": fit.se().values}); es.to_csv(P / "modelB_event_study.csv", index=False)
++        pre = [t for t in fit.coef().index if any(t.endswith(f"[T.{k}]") for k in (-4, -3, -2))]
++        try:
++            w = fit.wald_test(R=None, q=None) if False else None
++        except Exception: w = None
++        b = fit.coef()[pre].values; V = fit._vcov; idx = [list(fit.coef().index).index(t) for t in pre]; Vs = V[np.ix_(idx, idx)]
++        W = float(b @ np.linalg.solve(Vs, b)); pval = float(1 - stats.chi2.cdf(W, len(pre)))
++        out["pretrend_wald"] = W; out["pretrend_p"] = pval; out["event_study_n"] = int(fit._N)
++        fitr = pf.feols("log_ref ~ i(k_f, ref=-1) | uid + week", data=ev, vcov={"CRV1": "uid"})
++        pd.DataFrame({"term": fitr.coef().index, "coef": fitr.coef().values, "se": fitr.se().values}).to_csv(P / "modelB_event_study_ref.csv", index=False)
++    # Placebo: random pseudo-episode ends for non-promoted listings
++    rng = np.random.default_rng(SEED); pl = []
++    nonp = sf[~sf.promo].groupby("uid").filter(lambda g: len(g) >= 9)
++    for uid, g in list(nonp.groupby("uid"))[:3000]:
++        g = g.sort_values("week"); i = rng.integers(2, len(g) - 2); pre = g.ref.iloc[i - 1]; post = g.ref.iloc[i + 1]
++        pl.append(np.log(post / pre))
++    pl = np.array(pl); out["placebo_dlog_ref_mean"] = float(pl.mean()); out["placebo_dlog_ref_se"] = float(pl.std() / np.sqrt(len(pl))); out["placebo_n"] = int(len(pl))
++    # Circular channel: observed regular prices
++    circ = pd.read_parquet(P / "products.parquet"); ret = pd.read_parquet(P / "retailers.parquet")
++    circ = circ.merge(ret[["feed_id", "channel", "banner", "is_grocery"]], left_on="source", right_on="feed_id")
++    circ = circ[circ.channel.str.startswith("circular") & circ.is_grocery & circ.price.notna()]
++    circ["depth_obs"] = np.log(circ.regular_price / circ.price)
++    out["circular_items"] = int(len(circ)); out["circular_share_with_regular"] = float(circ.regular_price.notna().mean())
++    out["circular_depth_median"] = float(circ.depth_obs.dropna().median()); out["circular_depth_mean"] = float(circ.depth_obs.dropna().mean())
++    out["circular_share_recurring_items"] = float((circ.span_days > 8).mean())
++    circ.groupby("banner").agg(items=("uid", "size"), with_regular=("regular_price", lambda s: s.notna().mean()), depth_median=("depth_obs", "median"), recurring=("span_days", lambda s: (s > 8).mean())).reset_index().to_csv(P / "modelB_circulars_by_banner.csv", index=False)
++    (E / "modelB_summary.json").write_text(json.dumps(out, indent=2, default=float))
++    log.info("H2: sd raw %.4f vs ref %.4f (ratio %.2f, CI %s); promo share %.3f depth med %.3f; agreement w/ on_sale %.2f", out["sd_raw_all"], out["sd_ref_all"], out["ratio_all"], np.round(out["ratio_ci95"], 2), out["promo_share_listing_weeks"], out["promo_depth_median"], agree)
++    log.info("H3: episodes %d (uncensored %d), duration median %.1f, dlog_ref mean %.4f (se %.4f) up %.2f down %.2f same %.2f; placebo %.4f; pretrend p %.3f",
++             out["n_episodes"], out["n_uncensored"], out["duration_median_unc"], out["H3_dlog_ref_mean"], out["H3_dlog_ref_se"], out["H3_share_ref_up"], out["H3_share_ref_down"], out["H3_share_ref_same"], out["placebo_dlog_ref_mean"], out.get("pretrend_p", float("nan")))
++    ledger("4", "model_b", f"ratio {out['ratio_all']:.2f}; episodes {out['n_episodes']}; H3 mean {out['H3_dlog_ref_mean']:.4f}")
++
++if __name__ == "__main__":
++    main()

added src/econometrics/model_c_spillovers.py +63 −0

++"""Model C — competitive spillovers (H4), descriptive lead–lag only. Preregistration §2 H4.
++
++Within matched groups (A+B, storefront), regress a banner's weekly log-price change on rivals' current and lagged
++changes (k = 0..3), with group-week and banner fixed effects; leads (k = -1, -2) as placebo. Rival change = mean change of
++the other banners in the group that week. Same-company vs cross-company rivals estimated separately. No causal claim:
++contemporaneous terms absorb common shocks; lagged terms are lead–lag associations.
++"""
++import json, numpy as np, pandas as pd, pyfixest as pf
++from src.common import path, ROOT, get_logger, ledger, seed_everything
++log = get_logger("modelC"); P = path("processed"); E = ROOT / "experiments"
++
++def main():
++    seed_everything(); out = {}
++    pm = pd.read_parquet(P / "panel_matched_AB.parquet"); pm = pm[(pm.channel == "online_storefront") & (~pm.ambiguous)].copy()
++    pm = pm.drop_duplicates(["group_id", "week", "banner"]).sort_values(["group_id", "banner", "week"])
++    pm["dlp"] = pm.groupby(["group_id", "banner"]).log_p.diff()
++    pm["gap_w"] = (pm.week - pm.groupby(["group_id", "banner"]).week.shift(1)).dt.days
++    pm.loc[pm.gap_w != 7, "dlp"] = np.nan
++    # rival mean change, same-company vs cross-company
++    tot = pm.groupby(["group_id", "week"]).dlp.transform(lambda s: s.sum(min_count=1)); cnt = pm.groupby(["group_id", "week"]).dlp.transform("count")
++    pm["rival_all"] = (tot - pm.dlp.fillna(0)) / (cnt - pm.dlp.notna().astype(int)).replace(0, np.nan)
++    def rival_by(same):
++        rows = []
++        for (g, w), d in pm.groupby(["group_id", "week"]):
++            for i, r in d.iterrows():
++                oth = d[(d.index != i) & ((d.company == r.company) == same)].dlp.dropna()
++                rows.append((i, oth.mean() if len(oth) else np.nan))
++        return pd.Series(dict(rows))
++    pm["rival_same"] = rival_by(True); pm["rival_cross"] = rival_by(False)
++    for col in ["rival_all", "rival_same", "rival_cross"]:
++        for k in (1, 2, 3):
++            pm[f"{col}_l{k}"] = pm.groupby(["group_id", "banner"])[col].shift(k)
++        for k in (1, 2):
++            pm[f"{col}_f{k}"] = pm.groupby(["group_id", "banner"])[col].shift(-k)
++    pm["gw"] = pm.group_id + "_" + pm.week.dt.strftime("%Y%m%d")
++    res = []
++    for col in ["rival_all", "rival_same", "rival_cross"]:
++        terms = [col] + [f"{col}_l{k}" for k in (1, 2, 3)] + [f"{col}_f{k}" for k in (1, 2)]
++        d = pm.dropna(subset=["dlp"] + terms)
++        if len(d) < 500: log.warning("%s: only %d obs", col, len(d)); continue
++        fit = pf.feols("dlp ~ " + " + ".join(terms) + " | banner + week", data=d, vcov={"CRV1": "group_id"})
++        for t in terms:
++            res.append(dict(rival=col, term=t, coef=float(fit.coef()[t]), se=float(fit.se()[t]), n=int(fit._N)))
++        out[col] = dict(n=int(fit._N), coefs={t: float(fit.coef()[t]) for t in terms}, ses={t: float(fit.se()[t]) for t in terms})
++        # descriptive: probability a banner changes price in w given rival changed in w-1
++        d["rival_changed_l1"] = d[f"{col}_l1"].abs() > 1e-9; d["own_changed"] = d.dlp.abs() > 1e-9
++        ct = d.groupby("rival_changed_l1").own_changed.mean()
++        out[col]["P_own_change_given_rival_change_l1"] = float(ct.get(True, np.nan)); out[col]["P_own_change_given_no_rival_change_l1"] = float(ct.get(False, np.nan))
++    pd.DataFrame(res).to_csv(P / "modelC_distributed_lags.csv", index=False)
++    # Promotion co-movement: share of group-weeks in which >=2 banners are simultaneously at a low (p <= 0.9 x own 8-week mode)
++    pm["ref"] = pm.groupby(["group_id", "banner"]).p_mode.transform(lambda s: s.rolling(8, min_periods=1).apply(lambda x: pd.Series(x).mode().max()))
++    pm["low"] = pm.p_mode <= 0.9 * pm.ref
++    gl = pm.groupby(["group_id", "week"]).agg(n=("banner", "size"), nlow=("low", "sum"))
++    gl = gl[gl.n >= 2]; out["promo_comovement"] = dict(group_weeks=int(len(gl)), share_any_low=float((gl.nlow >= 1).mean()), share_two_plus_low=float((gl.nlow >= 2).mean()),
++                                                      expected_two_plus_if_independent=float(((gl.nlow / gl.n).mean()) ** 2))
++    (E / "modelC_summary.json").write_text(json.dumps(out, indent=2, default=float))
++    for col in out:
++        if col.startswith("rival"): log.info("%s: n=%d, k0 %.3f (%.3f), l1 %.3f (%.3f), l2 %.3f (%.3f), f1 %.3f (%.3f)", col, out[col]["n"], out[col]["coefs"][col], out[col]["ses"][col], out[col]["coefs"][col + "_l1"], out[col]["ses"][col + "_l1"], out[col]["coefs"][col + "_l2"], out[col]["ses"][col + "_l2"], out[col]["coefs"][col + "_f1"], out[col]["ses"][col + "_f1"])
++    log.info("promo co-movement: %s", out["promo_comovement"])
++    ledger("4", "model_c", "distributed lags estimated (descriptive)")
++
++if __name__ == "__main__":
++    main()

added src/forecasting/__init__.py +0 −0

added src/forecasting/rolling_forecast.py +65 −0

++"""Model F — rolling-origin forecasts of next-week posted log price and promotion indicator (preregistration §2 F).
++
++Features use only information up to week t (strict lags). Origins t = 6..10 (train weeks <= t, test week t+1).
++Models: seasonal naive (last value), banner×category mean reversion (shrink toward 8-week listing mode), ridge, LightGBM.
++Promotion target: next week p <= 0.9 x trailing 8-week mode. Metrics: MAE/RMSE (log price), Brier/AUC (promotion).
++"""
++import json, numpy as np, pandas as pd, lightgbm as lgb
++from sklearn.linear_model import Ridge, LogisticRegression
++from sklearn.metrics import roc_auc_score, brier_score_loss
++from src.common import path, ROOT, get_logger, ledger, seed_everything, SEED
++log = get_logger("modelF"); P = path("processed"); E = ROOT / "experiments"
++
++def rolling_mode_series(s, window=8):
++    out = []
++    v = s.tolist()
++    for i in range(len(v)):
++        w = pd.Series(v[max(0, i - window + 1): i + 1]).value_counts(); out.append(max(w[w == w.max()].index))
++    return pd.Series(out, index=s.index)
++
++def main():
++    seed_everything(); out = {}
++    pw = pd.read_parquet(P / "panel_weekly.parquet"); sf = pw[pw.channel == "online_storefront"].sort_values(["uid", "week"]).copy()
++    sf = sf[sf.groupby("uid").week.transform("nunique") >= 6]
++    sf["t"] = sf.week.rank(method="dense").astype(int)
++    g = sf.groupby("uid")
++    sf["ref"] = g.p_mode.transform(rolling_mode_series); sf["promo"] = (sf.p_mode <= 0.9 * sf.ref).astype(int)
++    for k in (1, 2, 3): sf[f"lp_l{k}"] = g.log_p.shift(k)
++    sf["lp_ref"] = np.log(sf.ref); sf["promo_l1"] = g.promo.shift(1); sf["promo_l2"] = g.promo.shift(2)
++    sf["n_changes_l0"] = sf.n_changes; sf["rel_range_l0"] = sf.rel_range; sf["multi_l0"] = sf.multi_price_week.astype(int)
++    sf["promo_rate_hist"] = g.promo.transform(lambda s: s.shift(1).expanding().mean())
++    sf["y_lp"] = g.log_p.shift(-1); sf["y_promo"] = g.promo.shift(-1); sf["t_next"] = g.t.shift(-1)
++    sf = sf[(sf.t_next == sf.t + 1)]
++    sf["banner_c"] = sf.banner.astype("category").cat.codes; sf["cat_c"] = sf.category.astype("category").cat.codes
++    feats = ["log_p", "lp_l1", "lp_l2", "lp_l3", "lp_ref", "promo", "promo_l1", "promo_l2", "n_changes_l0", "rel_range_l0", "multi_l0", "promo_rate_hist", "banner_c", "cat_c", "t"]
++    res = []
++    for t in range(6, int(sf.t.max())):
++        tr = sf[(sf.t <= t - 1)].dropna(subset=feats + ["y_lp", "y_promo"]); te = sf[sf.t == t].dropna(subset=feats + ["y_lp", "y_promo"])
++        if len(te) == 0 or len(tr) == 0: continue
++        # price models
++        preds = {"naive_last": te.log_p.values, "mean_reversion_ref": 0.5 * te.log_p.values + 0.5 * te.lp_ref.values}
++        bc = tr.groupby(["banner", "category"]).apply(lambda d: (d.y_lp - d.log_p).mean()).rename("drift").reset_index()
++        preds["banner_cat_drift"] = te.log_p.values + te.merge(bc, on=["banner", "category"], how="left").drift.fillna(0).values
++        X = tr[feats].values; Xt = te[feats].values
++        rd = Ridge(alpha=1.0).fit(X, tr.y_lp.values); preds["ridge"] = rd.predict(Xt)
++        gb = lgb.LGBMRegressor(n_estimators=400, learning_rate=0.05, num_leaves=63, min_child_samples=50, subsample=0.8, colsample_bytree=0.8, random_state=SEED, verbose=-1).fit(X, tr.y_lp.values)
++        preds["lightgbm"] = gb.predict(Xt)
++        for m, p in preds.items():
++            e = p - te.y_lp.values; res.append(dict(origin_t=t, target="log_price", model=m, n=len(te), mae=float(np.mean(np.abs(e))), rmse=float(np.sqrt(np.mean(e ** 2)))))
++        # promotion models
++        pp = {"persist": te.promo.values.astype(float), "base_rate": np.full(len(te), tr.y_promo.mean()), "hist_rate": te.promo_rate_hist.fillna(tr.y_promo.mean()).values}
++        lr = LogisticRegression(max_iter=2000).fit(X, tr.y_promo.values); pp["logit"] = lr.predict_proba(Xt)[:, 1]
++        gc = lgb.LGBMClassifier(n_estimators=400, learning_rate=0.05, num_leaves=63, min_child_samples=50, subsample=0.8, colsample_bytree=0.8, random_state=SEED, verbose=-1).fit(X, tr.y_promo.values)
++        pp["lightgbm"] = gc.predict_proba(Xt)[:, 1]
++        for m, p in pp.items():
++            y = te.y_promo.values
++            res.append(dict(origin_t=t, target="promotion", model=m, n=len(te), brier=float(brier_score_loss(y, np.clip(p, 0, 1))), auc=float(roc_auc_score(y, p)) if len(set(y)) > 1 else np.nan, base_rate=float(y.mean())))
++    r = pd.DataFrame(res); r.to_csv(P / "modelF_rolling_results.csv", index=False)
++    summ = r.groupby(["target", "model"]).agg(origins=("origin_t", "nunique"), n=("n", "sum"), mae=("mae", "mean"), rmse=("rmse", "mean"), brier=("brier", "mean"), auc=("auc", "mean")).reset_index()
++    summ.to_csv(P / "modelF_summary.csv", index=False); out["summary"] = summ.to_dict(orient="records")
++    (E / "modelF_summary.json").write_text(json.dumps(out, indent=2, default=float))
++    log.info("\n%s", summ.round(4).to_string(index=False))
++    ledger("4", "model_f", "rolling-origin forecasts done")
++
++if __name__ == "__main__":
++    main()

added src/indices/__init__.py +0 −0

added src/indices/basket_optimization.py +83 −0

++"""Model E — fixed-basket shopping optimisation (H6). Preregistration §2 H6 and config/basket.yaml (frozen).
++
++For each complete week: map each basket item to candidate listings per banner (category + name regex, parsable size,
++unit price), take the cheapest unit price per banner, and solve min-cost assignment with at most K banners (K=1..5) as a
++MIP (PuLP/CBC). Travel costs are unobserved: an illustrative fixed per-banner cost grid is added as a labelled assumption.
++"""
++import json, re, numpy as np, pandas as pd, pulp, yaml
++from src.common import path, ROOT, get_logger, ledger
++log = get_logger("modelE"); P = path("processed"); E = ROOT / "experiments"
++
++def candidates(pw, items):
++    rows = []
++    for it in items:
++        d = pw[(pw.category == it["category"]) & pw.name.fillna("").str.contains(it["name_regex"], case=False, regex=True)]
++        if it.get("exclude_regex"): d = d[~d.name.fillna("").str.contains(it["exclude_regex"], case=False, regex=True)]
++        d = d[(d.unit_base == ("un" if it["base"] == "un" else f"100 {it['base']}")) & d.unit_p.notna() & (d.unit_p > 0)]
++        if len(d) == 0: continue
++        d = d.assign(item=it["id"], item_cost=lambda x: x.unit_p * (it["qty"] if it["base"] == "un" else it["qty"] / 100.0))
++        rows.append(d[["item", "week", "banner", "company", "uid", "name", "size_label", "p_mode", "unit_p", "item_cost"]])
++    return pd.concat(rows, ignore_index=True)
++
++def solve(cost_tbl, K, fixed_cost=0.0):
++    """cost_tbl: item x banner matrix of min item cost (NaN = unavailable). Returns (total cost, banners used, items covered)."""
++    items = list(cost_tbl.index); banners = list(cost_tbl.columns)
++    prob = pulp.LpProblem("basket", pulp.LpMinimize)
++    x = {(i, b): pulp.LpVariable(f"x_{n}_{m}", cat="Binary") for n, i in enumerate(items) for m, b in enumerate(banners) if not np.isnan(cost_tbl.at[i, b])}
++    y = {b: pulp.LpVariable(f"y_{m}", cat="Binary") for m, b in enumerate(banners)}
++    prob += pulp.lpSum(cost_tbl.at[i, b] * v for (i, b), v in x.items()) + fixed_cost * pulp.lpSum(y.values())
++    for i in items:
++        prob += pulp.lpSum(v for (ii, b), v in x.items() if ii == i) == 1
++    for (i, b), v in x.items():
++        prob += v <= y[b]
++    prob += pulp.lpSum(y.values()) <= K
++    prob.solve(pulp.PULP_CBC_CMD(msg=False))
++    if pulp.LpStatus[prob.status] != "Optimal": return np.nan, [], 0
++    used = [b for b, v in y.items() if v.value() > 0.5]
++    tot = sum(cost_tbl.at[i, b] * v.value() for (i, b), v in x.items())
++    return float(tot), used, len(items)
++
++def main():
++    out = {}
++    items = yaml.safe_load((ROOT / "config" / "basket.yaml").read_text())["items"]
++    pw = pd.read_parquet(P / "panel_weekly.parquet"); sf = pw[pw.channel == "online_storefront"]
++    cand = candidates(sf, items); cand.to_parquet(P / "modelE_candidates.parquet", index=False)
++    cov = cand.groupby("item").agg(listings=("uid", "nunique"), banners=("banner", "nunique")).reset_index(); cov.to_csv(P / "modelE_item_coverage.csv", index=False)
++    log.info("basket items with candidates: %d / %d ; median banners per item %.0f", cov.item.nunique(), len(items), cov.banners.median())
++    results = []
++    for w, d in cand.groupby("week"):
++        tbl = d.groupby(["item", "banner"]).item_cost.min().unstack()
++        # keep banners carrying >= 60 % of covered items, and items available at >= 2 banners (so that K=1 is feasible on a common set)
++        tbl = tbl.loc[:, tbl.notna().mean() >= 0.6]
++        tbl = tbl[tbl.notna().sum(axis=1) >= 2]
++        # for K=1 feasibility, restrict to items available at every retained banner
++        full = tbl.dropna()
++        for K in (1, 2, 3, 4, 5, len(tbl.columns)):
++            tot, used, n = solve(full, K)
++            results.append(dict(week=w, K=K if K < len(tbl.columns) else 99, cost=tot, banners=";".join(used), n_items=n, n_banners_avail=len(tbl.columns)))
++        # single-banner cost for each banner (full item set)
++        for b in full.columns:
++            results.append(dict(week=w, K=1, cost=float(full[b].sum()), banners=b, n_items=len(full), n_banners_avail=len(tbl.columns), single=True))
++        for fc in (5.0, 10.0, 20.0):
++            tot, used, n = solve(full, len(full.columns), fixed_cost=fc)
++            results.append(dict(week=w, K=99, cost=tot - fc * len(used), banners=";".join(used), n_items=n, n_banners_avail=len(tbl.columns), fixed_cost=fc, n_used=len(used), cost_incl_travel=tot))
++    r = pd.DataFrame(results); r.to_csv(P / "modelE_results.csv", index=False)
++    base = r[(r.get("single").isna()) & (r.fixed_cost.isna())] if "fixed_cost" in r else r[r.get("single").isna()]
++    front = base.groupby("K").cost.mean(); singles = r[r.get("single") == True].groupby("week").cost.agg(["min", "max", "mean"])
++    out["items_in_common_set_mean"] = float(base.n_items.mean()); out["banners_available_mean"] = float(base.n_banners_avail.mean())
++    out["frontier_mean_cost_by_K"] = {int(k): float(v) for k, v in front.items()}
++    c1, c2, cinf = front.get(1), front.get(2), front.get(99)
++    out["savings_unconstrained_vs_best_single"] = float(c1 - cinf); out["savings_share_unconstrained"] = float((c1 - cinf) / c1)
++    out["share_of_unconstrained_savings_at_K2"] = float((c1 - c2) / (c1 - cinf)) if c1 - cinf > 0 else None
++    out["single_banner_cost_spread_mean"] = float((singles["max"] - singles["min"]).mean()); out["single_banner_mean_cost"] = float(singles["mean"].mean())
++    out["most_expensive_vs_cheapest_single_share"] = float(((singles["max"] - singles["min"]) / singles["min"]).mean())
++    if "fixed_cost" in r:
++        tc = r[r.fixed_cost.notna()].groupby("fixed_cost").agg(n_used=("n_used", "mean"), cost_goods=("cost", "mean"), cost_incl=("cost_incl_travel", "mean")).reset_index()
++        tc.to_csv(P / "modelE_travel_sensitivity.csv", index=False); out["travel_cost_sensitivity"] = tc.to_dict(orient="records")
++    (E / "modelE_summary.json").write_text(json.dumps(out, indent=2, default=float))
++    log.info("E: common items %.1f, banners %.1f ; cost K=1 %.2f, K=2 %.2f, unconstrained %.2f ; unconstrained savings %.1f%% ; share captured at K=2 %.2f ; single-banner spread %.2f (%.1f%%)",
++             out["items_in_common_set_mean"], out["banners_available_mean"], c1, c2, cinf, 100 * out["savings_share_unconstrained"], out["share_of_unconstrained_savings_at_K2"] or float("nan"), out["single_banner_cost_spread_mean"], 100 * out["most_expensive_vs_cheapest_single_share"])
++    ledger("4", "model_e", f"K1 {c1:.2f} K2 {c2:.2f} Kinf {cinf:.2f}")
++
++if __name__ == "__main__":
++    main()

added src/indices/price_indices.py +86 −0

++"""Model D — price indices (H5). Preregistration §2 H5. No expenditure weights exist: all indices are unweighted and are
++NOT consumer price indices. Size-label history is not observed (only current state) → within-listing package changes are
++unobservable; this is reported as a limitation, not estimated.
++"""
++import json, numpy as np, pandas as pd
++from src.common import path, ROOT, get_logger, ledger, seed_everything, SEED
++log = get_logger("modelD"); P = path("processed"); E = ROOT / "experiments"
++
++def jevons_chained(df, col):
++    """Chained Jevons (matched-model): geometric mean of week-on-week ratios over listings present in both weeks."""
++    weeks = sorted(df.week.unique()); idx = [100.0]; n = [np.nan]
++    piv = df.pivot_table(index="uid", columns="week", values=col)
++    for a, b in zip(weeks, weeks[1:]):
++        both = piv[[a, b]].dropna(); r = np.exp((np.log(both[b]) - np.log(both[a])).mean()) if len(both) else np.nan
++        idx.append(idx[-1] * r); n.append(len(both))
++    return pd.DataFrame({"week": weeks, "index": idx, "n_matched": n})
++
++def fixed_basket(df, col):
++    """Balanced listings (present in every week), equal weights: Dutot (ratio of means), Carli (mean of ratios), Jevons (geo mean of ratios)."""
++    weeks = sorted(df.week.unique()); piv = df.pivot_table(index="uid", columns="week", values=col).dropna()
++    base = piv[weeks[0]]
++    rows = []
++    for w in weeks:
++        rows.append(dict(week=w, n=len(piv), dutot=100 * piv[w].mean() / base.mean(), carli=100 * (piv[w] / base).mean(), jevons=100 * np.exp(np.log(piv[w] / base).mean())))
++    return pd.DataFrame(rows)
++
++def naive_mean(df, col):
++    g = df.groupby("week")[col].agg(["mean", "median", "size"]).reset_index(); g["index_mean"] = 100 * g["mean"] / g["mean"].iloc[0]; g["index_median"] = 100 * g["median"] / g["median"].iloc[0]
++    return g
++
++def main():
++    seed_everything(); out = {}
++    pw = pd.read_parquet(P / "panel_weekly.parquet"); sf = pw[pw.channel == "online_storefront"].copy()
++    res = {}
++    for name, d, col in [("storefront_posted", sf, "p_mode"), ("storefront_unit", sf[sf.unit_p.notna()], "unit_p"), ("all_channels_posted", pw, "p_mode")]:
++        nm = naive_mean(d, col); jc = jevons_chained(d, col); fb = fixed_basket(d, col)
++        m = nm.merge(jc, on="week").merge(fb, on="week", suffixes=("", "_fb")); m["sample"] = name; res[name] = m
++        out[name] = dict(naive_mean_change=float(nm.index_mean.iloc[-1] / 100 - 1), naive_median_change=float(nm.index_median.iloc[-1] / 100 - 1), jevons_chained_change=float(jc["index"].iloc[-1] / 100 - 1),
++                         fixed_jevons_change=float(fb.jevons.iloc[-1] / 100 - 1), fixed_dutot_change=float(fb.dutot.iloc[-1] / 100 - 1), fixed_carli_change=float(fb.carli.iloc[-1] / 100 - 1), n_balanced=int(fb.n.iloc[0]), n_total_first_week=int(nm["size"].iloc[0]))
++    pd.concat(res.values()).to_csv(P / "modelD_indices.csv", index=False)
++    # by category (storefront posted)
++    bc = []
++    for c, d in sf.groupby("category"):
++        if d.uid.nunique() < 100: continue
++        nm = naive_mean(d, "p_mode"); jc = jevons_chained(d, "p_mode"); fb = fixed_basket(d, "p_mode")
++        bc.append(dict(category=c, listings=d.uid.nunique(), naive_mean_change=nm.index_mean.iloc[-1] / 100 - 1, jevons_chained_change=jc["index"].iloc[-1] / 100 - 1, fixed_jevons_change=fb.jevons.iloc[-1] / 100 - 1, n_balanced=fb.n.iloc[0]))
++    pd.DataFrame(bc).to_csv(P / "modelD_indices_by_category.csv", index=False)
++    # by banner (storefront posted): Jevons chained
++    bb = []
++    for b, d in sf.groupby("banner"):
++        if d.uid.nunique() < 100: continue
++        jc = jevons_chained(d, "p_mode"); fb = fixed_basket(d, "p_mode"); nm = naive_mean(d, "p_mode")
++        bb.append(dict(banner=b, listings=d.uid.nunique(), jevons_chained_change=jc["index"].iloc[-1] / 100 - 1, fixed_jevons_change=fb.jevons.iloc[-1] / 100 - 1, naive_mean_change=nm.index_mean.iloc[-1] / 100 - 1, n_balanced=fb.n.iloc[0]))
++    pd.DataFrame(bb).to_csv(P / "modelD_indices_by_banner.csv", index=False)
++    # Bootstrap (listing-level, B=999) of the naive-vs-Jevons gap, storefront posted
++    rng = np.random.default_rng(SEED); uids = sf.uid.unique(); piv = sf.pivot_table(index="uid", columns="week", values="p_mode"); weeks = list(piv.columns)
++    gaps = []; jev = []
++    for b in range(999):
++        s = rng.choice(uids, size=len(uids), replace=True); pb = piv.loc[s]
++        naive = pb[weeks[-1]].mean() / pb[weeks[0]].mean() - 1
++        lj = 0.0
++        for a, c in zip(weeks, weeks[1:]):
++            both = pb[[a, c]].dropna(); lj += (np.log(both[c]) - np.log(both[a])).mean()
++        jev.append(np.exp(lj) - 1); gaps.append(naive - (np.exp(lj) - 1))
++    out["bootstrap"] = dict(gap_mean=float(np.mean(gaps)), gap_ci95=[float(np.percentile(gaps, 2.5)), float(np.percentile(gaps, 97.5))], jevons_ci95=[float(np.percentile(jev, 2.5)), float(np.percentile(jev, 97.5))])
++    # Assortment turnover (entry/exit) — the observable part of H5
++    first = sf.groupby("uid").week.min(); last = sf.groupby("uid").week.max(); weeks_s = sorted(sf.week.unique())
++    out["assortment"] = dict(listings=int(sf.uid.nunique()), balanced=int(((first == weeks_s[0]) & (last == weeks_s[-1])).sum()), entrants=int((first > weeks_s[0]).sum()), exits=int((last < weeks_s[-1]).sum()),
++                             weekly_entry_rate=float(sf.groupby("week").uid.nunique().pipe(lambda s: (first.value_counts().reindex(weeks_s).fillna(0)[1:] / s.values[:-1]).mean())))
++    # Unit-price vs posted: cross-banner unit price gap for matched groups is covered in Model A; here, size-missing share
++    out["size_label_history_available"] = False
++    # StatCan benchmark: Quebec CPI 'Food purchased from stores' Aug->Sep 2026 ; our storefront Jevons Aug (weeks 13,20,27 Aug) -> Sep (3,10,17,24 Sep)
++    cpi = pd.read_csv(P / "statcan_cpi_food.csv"); q = cpi[(cpi.geo == "Quebec") & (cpi["product"] == "Food purchased from stores")].sort_values("ref_date")
++    aug = q[q.ref_date.str.startswith("2026-08")].value.iloc[0]; sep = q[q.ref_date.str.startswith("2026-09")].value.iloc[0]
++    jc = jevons_chained(sf, "p_mode"); jc["month"] = pd.to_datetime(jc.week).dt.month
++    ours = jc.groupby("month")["index"].mean(); out["statcan"] = dict(cpi_qc_food_stores_aug=float(aug), sep=float(sep), cpi_change_aug_sep=float(sep / aug - 1), ours_jevons_aug_sep=float(ours.loc[9] / ours.loc[8] - 1),
++                                                                     cpi_latest=str(q.ref_date.iloc[-1]), cpi_yoy_sep=float(sep / q[q.ref_date.str.startswith("2025-09")].value.iloc[0] - 1))
++    (E / "modelD_summary.json").write_text(json.dumps(out, indent=2, default=float))
++    s = out["storefront_posted"]
++    log.info("D storefront: naive mean %.3f, naive median %.3f, Jevons chained %.3f (CI %s), fixed Jevons %.3f (n=%d), Dutot %.3f, Carli %.3f; gap naive−Jevons %.3f CI %s",
++             s["naive_mean_change"], s["naive_median_change"], s["jevons_chained_change"], np.round(out["bootstrap"]["jevons_ci95"], 3), s["fixed_jevons_change"], s["n_balanced"], s["fixed_dutot_change"], s["fixed_carli_change"], out["bootstrap"]["gap_mean"], np.round(out["bootstrap"]["gap_ci95"], 3))
++    log.info("StatCan QC food-from-stores Aug→Sep %.3f vs ours %.3f ; CPI yoy Sep %.3f", out["statcan"]["cpi_change_aug_sep"], out["statcan"]["ours_jevons_aug_sep"], out["statcan"]["cpi_yoy_sep"])
++    ledger("4", "model_d", f"Jevons {s['jevons_chained_change']:.3f} vs naive {s['naive_mean_change']:.3f}")
++
++if __name__ == "__main__":
++    main()

spb/wp37_uqo

phase4: models A (dispersion), C (spillovers), D (indices), E (basket MIP), F (forecasts); pulp pinned <3

Theme