"""F1-conf-01: 12-month CPIAUCSL growth from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
"""
import json, os
import numpy as np
import pandas as pd

D, O = "data", "out"
V16, V26, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = "2001-01-01", "2026-09-01"

def load(f):
    d = pd.read_csv(os.path.join(D, f), dtype={"date": str})
    return d

a, b, asm = load(V16), load(V26), load(ASM)
n_rows = {V16: len(a), V26: len(b), ASM: len(asm)}
issues = []

# --- validation of vintage files
for name, d in ((V16, a), (V26, b)):
    assert not d.date.duplicated().any(), name
    dt = pd.to_datetime(d.date)
    assert (dt.diff().dropna().dt.days <= 31).all(), f"gap in {name}"
missing_src = b[b.cpi.isna()].date.tolist()
for m in missing_src:
    issues.append({"issue": f"Vintage 2026-10-28 reports no CPI value for {m[:7]} (empty cell; the source publishes no observation for that month).",
                   "handling": "Not interpolated or imputed (rule). Not back-filled from the older vintage, which does not cover that month anyway. g_t for that month is reported missing; g_{t+12} (2026-10) lies outside the window."})

# --- splice: most recent vintage covering the month
a["vintage"] = "2016-07-15"; b["vintage"] = "2026-10-28"
b_cov = b.dropna(subset=["cpi"])  # month 'available' in vintage only if it has a value
lvl = pd.concat([a[~a.date.isin(b.date)], b]).sort_values("date").reset_index(drop=True)
# for a month listed but empty in the newest vintage, the older vintage has no value either (checked):
for m in missing_src:
    assert m not in set(a.date)
lvl = lvl.set_index(pd.to_datetime(lvl.date))

overlap = a.merge(b, on="date", suffixes=("_16", "_26"))
diff = (overlap.cpi_16 - overlap.cpi_26).abs()
issues.append({"issue": f"The two vintages overlap for {len(overlap)} months ({overlap.date.min()[:7]}..{overlap.date.max()[:7]}) and disagree (revised seasonal factors): max abs level difference {diff.max():.3f} index points.",
               "handling": "Per the availability rule the 2026-10-28 vintage is used for every month it covers (2014-01 onward); the 2016-07-15 vintage only for months up to 2013-12."})
issues.append({"issue": "For 2014-01..2014-12 the numerator P_t comes from the 2026 vintage while the lag P_{t-12} (2013) comes from the 2016 vintage, so these 12 growth rates mix two vintages (seasonal-factor vintage mismatch).",
               "handling": "Computed as specified by the rule; magnitude quantified as a sensitivity (2016-vintage-only growth for 2014 months) in the report and in the 'sensitivity' block of results.json."})

# --- audit of the hand-assembled file (not used for estimation)
m = asm.merge(lvl.reset_index(drop=True)[["date", "cpi", "vintage"]], on="date", how="outer", suffixes=("_asm", "_ref"), indicator=True)
only_asm = m[m._merge == "left_only"].date.tolist()
only_ref = m[m._merge == "right_only"].date.tolist()
both = m[m._merge == "both"]
mism = both[~((both.cpi_asm == both.cpi_ref) | (both.cpi_asm.isna() & both.cpi_ref.isna()))]
wrongv = both[both.vintage_asm != both.vintage_ref]
if len(wrongv):
    issues.append({"issue": f"cpi_assembled.csv uses the 2016-07-15 vintage for {len(wrongv)} months ({wrongv.date.min()[:7]}..{wrongv.date.max()[:7]}) that are covered by the more recent 2026-10-28 vintage.",
                   "handling": "Assembled file not used for estimation; levels taken directly from the two vintage files."})
if only_ref or only_asm:
    shifted = both[(both.date > "2016-06-01")]
    issues.append({"issue": f"cpi_assembled.csv is missing month(s) {[d[:7] for d in only_ref]} and contains month(s) {[d[:7] for d in only_asm]} absent from the sources; from 2016-07 onward the 2026-vintage values are dated one month late (e.g. 2026-09 value 336.298 appears as 2026-10). {len(mism)} common months have values differing from the authoritative splice.",
                   "handling": "Assembled file treated as unreliable and used only for this audit; all estimates use the vintage files."})

# --- growth
full = pd.date_range("1990-01-01", W1, freq="MS")
P = lvl.cpi.reindex(full)
V = lvl.vintage.reindex(full)
lag = P.shift(12)
g = 100 * (P / lag - 1)
win = pd.date_range(W0, W1, freq="MS")
df = pd.DataFrame({"date": win.strftime("%Y-%m-%d"), "cpi": P.reindex(win).values,
                   "cpi_lag12": lag.reindex(win).values, "vintage_used": V.reindex(win).values,
                   "growth_12m": g.reindex(win).values})
n0 = len(df)
s1 = df[df.cpi.notna()]
s2 = s1[s1.cpi_lag12.notna()]
an = s2.reset_index(drop=True)
ex = pd.DataFrame([
    {"step": 1, "rule": "P_t missing in all sources", "n_before": n0, "n_after": len(s1), "n_removed": n0 - len(s1),
     "rationale": "No index level for month t in the most recent vintage covering it (" + ", ".join(d[:7] for d in df[df.cpi.isna()].date) + "); imputation not allowed."},
    {"step": 2, "rule": "P_{t-12} missing in all sources", "n_before": len(s1), "n_after": len(s2), "n_removed": len(s1) - len(s2),
     "rationale": "Lagged index level required for g_t; imputation not allowed."},
])
an_out = an.copy()
an_out["vintage_used"] = an_out.vintage_used
an_out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")
ex.to_csv(os.path.join(O, "exclusions.csv"), index=False)

gs = an.set_index("date").growth_12m
def gm(ym):
    k = ym + "-01"
    return float(round(gs[k], 6)) if k in gs.index else None
mean = lambda s: float(round(s.mean(), 6))
missing_months = df[df.growth_12m.isna()].date.str[:7].tolist()

# sensitivity: 2014 growth from 2016 vintage alone (same-vintage numerator and lag)
Pa = a.set_index(pd.to_datetime(a.date)).cpi
ga = 100 * (Pa / Pa.shift(12) - 1)
m14 = pd.date_range("2014-01-01", "2016-06-01", freq="MS")
sens = pd.DataFrame({"spliced": g.reindex(m14).values, "v2016_only": ga.reindex(m14).values}, index=m14)
sens["diff"] = sens.spliced - sens.v2016_only
alt = gs.copy(); alt.index = pd.to_datetime(alt.index)
alt.loc[m14[:12]] = ga.reindex(m14[:12]).values

res = {
    "task_id": "F1-conf-01",
    "first_growth_month": an.date.iloc[0][:7],
    "last_month": lvl.cpi.dropna().index.max().strftime("%Y-%m"),
    "n_months_growth": int(len(an)),
    "n_months_missing_growth": int(n0 - len(an)),
    "n_analysed": int(len(an)),
    "mean_growth_12m": mean(gs),
    "mean_growth_12m_2001_2013": mean(gs[gs.index <= "2013-12-01"]),
    "mean_growth_12m_2014_2026": mean(gs[gs.index >= "2014-01-01"]),
    "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),
    "max_growth_12m": float(round(gs.max(), 6)), "max_growth_month": gs.idxmax()[:7],
    "min_growth_12m": float(round(gs.min(), 6)), "min_growth_month": gs.idxmin()[:7],
    "feasibility": "partial" if n0 - len(an) else "complete",
    "feasibility_reason": (f"g_t not computable for {missing_months}: the source (vintage 2026-10-28) publishes no CPI value for 2025-10; "
                           "imputation is prohibited, so all window statistics use the remaining months.") if n0 - len(an) else None,
    "data_issues": issues,
    "n_input_rows": n_rows,
    "random_seed": None,
    "missing_growth_months": missing_months,
    "sensitivity": {
        "growth_2014_mixed_minus_2016vintage_mean_abs_pp": float(round(sens["diff"].iloc[:12].abs().mean(), 6)),
        "growth_2014_mixed_minus_2016vintage_max_abs_pp": float(round(sens["diff"].iloc[:12].abs().max(), 6)),
        "growth_2015_2016H1_vintage_revision_max_abs_pp": float(round(sens["diff"].iloc[12:].abs().max(), 6)),
        "mean_growth_12m_2014_2026_using_2016vintage_for_2014": mean(alt[alt.index >= "2014-01-01"]),
        "growth_2014_06_2016vintage_only": float(round(ga["2014-06-01"], 6)),
    },
}
with open(os.path.join(O, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps(res, indent=2))