"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""
import json, os
import numpy as np
import pandas as pd

D, O = "data", "out"
OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")

old = pd.read_csv(os.path.join(D, OLD), parse_dates=["date"])
new = pd.read_csv(os.path.join(D, NEW), parse_dates=["date"])
asm = pd.read_csv(os.path.join(D, ASM), parse_dates=["date"])  # read for audit only
old["vintage_used"], new["vintage_used"] = "2016-07-15", "2026-10-28"

# --- audit of the hand-assembled file against the rule-based splice (not used for computation)
issues = []
ledger = []
def step(name, rule, nb, na, why):
    ledger.append(dict(step=name, rule=rule, n_before=nb, n_after=na, n_removed=nb - na, rationale=why))

# --- splice: latest vintage wins for every month it covers (row present, even if value empty)
n_read = len(old) + len(new)
superseded = old["date"].isin(new["date"])
series = pd.concat([old[~superseded], new], ignore_index=True).sort_values("date").reset_index(drop=True)
step("1_splice", "drop 2016-07-15 vintage rows for months covered by the 2026-10-28 vintage",
     n_read, len(series), "information rule: use most recent vintage in which the month is available")
assert series["date"].is_unique
full = pd.date_range(series["date"].min(), series["date"].max(), freq="MS")
assert len(full) == len(series), "gaps in month sequence"

s = series.set_index("date")
lag = s["cpi"].shift(12)  # contiguous monthly index verified above -> shift(12) = t-12 months
lag_v = s["vintage_used"].shift(12)
s["cpi_lag12"] = lag
s["growth_12m"] = 100.0 * (s["cpi"] / s["cpi_lag12"] - 1.0)  # NaN if either level missing: no imputation

win = s.loc[W0:W1].copy()
step("2_window", "keep months 2001-01 .. 2026-09", len(series), len(win), "estimation window defined in TASK.md")
missing_months = [d.strftime("%Y-%m") for d in win.index[win["growth_12m"].isna()]]
ana = win[win["growth_12m"].notna()].copy()
step("3_computable", "drop months where P_t or P_{t-12} is missing in the sources",
     len(win), len(ana), "no interpolation/imputation allowed; missing months: " + ";".join(missing_months))

# --- data issues
miss_lvl = new.loc[new["cpi"].isna(), "date"].dt.strftime("%Y-%m").tolist()
if miss_lvl:
    issues.append({"issue": f"2026-10-28 vintage reports no value (empty cpi) for {', '.join(miss_lvl)}",
                   "handling": "not imputed; g_t for affected window months (" + ", ".join(missing_months)
                               + ") reported missing and excluded from means; the latest vintage still governs that month, so the older vintage is not used as a fallback (it does not cover it anyway)"})
ov = old.merge(new, on="date", suffixes=("_old", "_new"))
d = (ov["cpi_old"] - ov["cpi_new"]).abs()
issues.append({"issue": f"vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m}..{ov.date.max():%Y-%m}) with different values (mean abs diff {d.mean():.3f}, max {d.max():.3f} index points; seasonal-factor revisions)",
               "handling": "2026-10-28 vintage values used for overlap months per information rule"})
mixed = ana[ana["vintage_used"] != lag_v.reindex(ana.index)]
if len(mixed):
    issues.append({"issue": f"for {len(mixed)} months ({mixed.index.min():%Y-%m}..{mixed.index.max():%Y-%m}) P_t comes from the 2026-10-28 vintage but P_(t-12) only exists in the 2016-07-15 vintage (new extract starts 2014-01), so the ratio mixes vintages",
                   "handling": "computed as prescribed by the latest-vintage rule; flagged as a limitation (revision noise in those 12 growth rates)"})
# assembled-file checks
ref = series[["date", "cpi", "vintage_used"]].merge(asm, on="date", how="outer", indicator=True)
only_asm = ref.loc[ref["_merge"] == "right_only", "date"].dt.strftime("%Y-%m").tolist()
only_ref = ref.loc[ref["_merge"] == "left_only", "date"].dt.strftime("%Y-%m").tolist()
both = ref[ref["_merge"] == "both"]
val_bad = both[~((both["cpi_x"] == both["cpi_y"]) | (both["cpi_x"].isna() & both["cpi_y"].isna()))]
if only_asm or only_ref or len(val_bad):
    issues.append({"issue": f"cpi_assembled.csv contains assembly errors: months missing {only_ref}, extra months {only_asm}, "
                            f"{len(val_bad)} months whose value differs from the rule-based splice (2014-01..2016-06 taken from the old vintage; "
                            "from 2016-08 on, each new-vintage value is labelled one month late)",
                   "handling": "assembled file not used for computation; series rebuilt from the two authoritative vintage files"})

def g(month):
    t = pd.Timestamp(month + "-01")
    v = s["growth_12m"].get(t, np.nan)
    return None if pd.isna(v) else round(float(v), 10)

gr = ana["growth_12m"]
last_level = s.loc[:W1, "cpi"].dropna().index.max()
feas = "complete" if not missing_months else "partial"
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": ana.index.min().strftime("%Y-%m"),
    "last_month": last_level.strftime("%Y-%m"),
    "n_months_growth": int(len(ana)),
    "n_months_missing_growth": int(len(missing_months)),
    "n_analysed": int(len(ana)),
    "mean_growth_12m": round(float(gr.mean()), 10),
    "mean_growth_12m_2001_2013": round(float(gr.loc["2001-01-01":"2013-12-01"].mean()), 10),
    "mean_growth_12m_2014_2026": round(float(gr.loc["2014-01-01":"2026-09-01"].mean()), 10),
    "growth_2014_06": g("2014-06"), "growth_2016_07": g("2016-07"), "growth_2020_05": g("2020-05"),
    "growth_2022_06": g("2022-06"), "growth_2026_09": g("2026-09"),
    "max_growth_12m": round(float(gr.max()), 10), "max_growth_month": gr.idxmax().strftime("%Y-%m"),
    "min_growth_12m": round(float(gr.min()), 10), "min_growth_month": gr.idxmin().strftime("%Y-%m"),
    "feasibility": feas,
    "feasibility_reason": None if feas == "complete" else
        f"g_t not computable for {len(missing_months)} window month(s) ({', '.join(missing_months)}) because the source publishes no index level for {', '.join(miss_lvl)}; all summary estimands computed over the remaining {len(ana)} months",
    "data_issues": issues,
    "n_input_rows": {OLD: int(len(old)), NEW: int(len(new)), ASM: int(len(asm))},
    "random_seed": None,
    # auxiliary diagnostics
    "n_months_window": int(len(win)),
    "n_months_2001_2013": int(gr.loc["2001-01-01":"2013-12-01"].size),
    "n_months_2014_2026": int(gr.loc["2014-01-01":"2026-09-01"].size),
    "missing_growth_months": missing_months,
    "sd_growth_12m": round(float(gr.std(ddof=1)), 10),
}
os.makedirs(O, exist_ok=True)
out = ana.reset_index()[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]
out["date"] = out["date"].dt.strftime("%Y-%m-%d")
out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10f")
pd.DataFrame(ledger).to_csv(os.path.join(O, "exclusions.csv"), index=False)
with open(os.path.join(O, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=1))