"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""
import json
import numpy as np
import pandas as pd

OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")


def load(f):
    d = pd.read_csv("data/" + f, dtype={"date": str})
    d["date"] = pd.to_datetime(d["date"])
    return d


old, new, asm = load(OLD), load(NEW), load(ASM)
for d in (old, new):
    assert not d["date"].duplicated().any()
n_input_rows = {OLD: len(old), NEW: len(new), ASM: len(asm)}

# Latest-vintage rule: a month is "covered" by the 2026 vintage if it appears in that extract
# (rows 2014-01..2026-09). An empty cell there means the source reports no value; we do NOT
# fall back to the older vintage (it does not cover 2025-10 anyway) and do not impute.
o = old.set_index("date")["cpi"]
nw = new.set_index("date")["cpi"]
months = pd.date_range(min(o.index.min(), nw.index.min()), max(o.index.max(), nw.index.max()), freq="MS")
level = pd.Series(np.nan, index=months)
vint = pd.Series(None, index=months, dtype=object)
level.loc[o.index] = o.values
vint.loc[o.index] = "2016-07-15"
level.loc[nw.index] = nw.values
vint.loc[nw.index] = "2026-10-28"

window = pd.date_range(W0, W1, freq="MS")
lag = level.shift(12)
g = 100.0 * (level / lag - 1.0)
df = pd.DataFrame({"cpi": level, "cpi_lag12": lag, "vintage_used": vint, "growth_12m": g}).loc[window]
ok = df["growth_12m"].notna()
analytic = df[ok].copy()
missing_months = [d.strftime("%Y-%m") for d in df.index[~ok]]

# Exclusion ledger
excl = [
    dict(step=1, rule="calendar months 2001-01..2026-09 (window)", n_before=len(window), n_after=len(window),
         n_removed=0, rationale="starting set: every month of the estimation window"),
]
n_lvl = int(df["cpi"].notna().sum())
excl.append(dict(step=2, rule="drop months with no index level P_t in latest covering vintage",
                 n_before=len(window), n_after=n_lvl, n_removed=len(window) - n_lvl,
                 rationale="no interpolation/imputation allowed (e.g. 2025-10 empty in 2026-10-28 vintage)"))
n_ok = int(ok.sum())
excl.append(dict(step=3, rule="drop months with no lagged level P_{t-12}", n_before=n_lvl, n_after=n_ok,
                 n_removed=n_lvl - n_ok, rationale="g_t requires both P_t and P_{t-12}"))
pd.DataFrame(excl).to_csv("out/exclusions.csv", index=False)

out_csv = analytic.reset_index().rename(columns={"index": "date"})
out_csv["date"] = out_csv["date"].dt.strftime("%Y-%m-01")
out_csv.to_csv("out/analytic.csv", index=False, float_format="%.10g")

G = analytic["growth_12m"]


def gm(m):
    t = pd.Timestamp(m + "-01")
    v = G.get(t, np.nan)
    return None if pd.isna(v) else round(float(v), 10)


def r(x):
    return round(float(x), 10)


# Data issues detected (computed, not hard-coded where possible)
issues = []
empty_new = [d.strftime("%Y-%m") for d in nw.index[nw.isna()]]
if empty_new:
    issues.append({"issue": f"{NEW}: empty cpi for {', '.join(empty_new)} (source reports no value)",
                   "handling": "level treated as missing, not imputed; g_t for that month reported missing "
                               "(and any month t whose t-12 is that month; inside the window: none)"})
ov = o.index.intersection(nw.index)
dif = (nw[ov] - o[ov]).dropna()
if len(dif) and (dif.abs() > 1e-9).any():
    issues.append({"issue": f"vintages overlap for {len(ov)} months ({ov.min():%Y-%m}..{ov.max():%Y-%m}) with revised values "
                            f"(max abs diff {dif.abs().max():.3f} index points)",
                   "handling": "2026-10-28 vintage used for overlap months per latest-vintage rule"})
a = asm.set_index("date")
a_old = a[a["vintage"].astype(str) == "2016-07-15"]
a_new = a[a["vintage"].astype(str) == "2026-10-28"]
asm_missing = [d.strftime("%Y-%m") for d in months if d not in a.index]
if (a_old.index >= nw.index.min()).any():
    issues.append({"issue": f"{ASM}: months {nw.index.min():%Y-%m}..{a_old.index.max():%Y-%m} taken from the older 2016-07-15 vintage although the 2026-10-28 vintage covers them",
                   "handling": "assembled file not used; levels taken directly from the vintage files"})
if asm_missing:
    issues.append({"issue": f"{ASM}: month(s) {', '.join(asm_missing)} absent",
                   "handling": "assembled file not used; levels taken directly from the vintage files"})
nw_shift = nw.shift(1, freq="MS")
common = a_new.index.intersection(nw_shift.index)
same_shift = np.allclose(a_new.loc[common, "cpi"].fillna(-1), nw_shift.loc[common].fillna(-1))
if same_shift:
    issues.append({"issue": f"{ASM}: rows labelled 2026-10-28 ({a_new.index.min():%Y-%m}..{a_new.index.max():%Y-%m}) are shifted one month late (value at month m equals the 2026-10-28 vintage value for m-1); file extends to {a.index.max():%Y-%m}, beyond the source",
                   "handling": "assembled file not used; levels taken directly from the vintage files"})

last_month = level.loc[window].dropna().index.max().strftime("%Y-%m")
imax, imin = G.idxmax(), G.idxmin()
n_miss = len(window) - n_ok
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": G.index.min().strftime("%Y-%m"),
    "last_month": last_month,
    "n_months_growth": n_ok,
    "n_months_missing_growth": n_miss,
    "n_analysed": n_ok,
    "mean_growth_12m": r(G.mean()),
    "mean_growth_12m_2001_2013": r(G[:"2013-12-01"].mean()),
    "mean_growth_12m_2014_2026": r(G["2014-01-01":].mean()),
    "growth_2014_06": gm("2014-06"),
    "growth_2016_07": gm("2016-07"),
    "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"),
    "growth_2026_09": gm("2026-09"),
    "max_growth_12m": r(G.max()),
    "max_growth_month": imax.strftime("%Y-%m"),
    "min_growth_12m": r(G.min()),
    "min_growth_month": imin.strftime("%Y-%m"),
    "feasibility": "complete" if n_miss == 0 else "partial",
    "feasibility_reason": None if n_miss == 0 else
        f"g_t not computable for {n_miss} month(s) of the window ({', '.join(missing_months)}): index level absent in the source; not imputed per the task rules",
    "missing_growth_months": missing_months,
    "n_months_window": len(window),
    "n_months_growth_2001_2013": int(G[:"2013-12-01"].notna().sum()),
    "n_months_growth_2014_2026": int(G["2014-01-01":].notna().sum()),
    "n_months_mixed_vintage": int((vint.shift(12).loc[analytic.index] != analytic["vintage_used"]).sum()),
    "sd_growth_12m": r(G.std(ddof=1)),
    "data_issues": issues,
    "n_input_rows": n_input_rows,
    "random_seed": None,
}
with open("out/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: res[k] for k in ["n_analysed", "n_months_missing_growth", "mean_growth_12m", "feasibility"]}))