1"""F1-conf-02: balanced-panel mean life expectancy 2000-2022.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json5from pathlib import Path6import pandas as pd78DATA = Path("data/life_expectancy_panel.csv")9OUT = Path("out")10YEARS = list(range(2000, 2023))1112raw = pd.read_csv(DATA, dtype={"country_name": str, "iso3": str})13n_raw = len(raw)14ledger, issues = [], []1516def step(name, rule, before, after, why):17 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,18 n_removed=before - after, rationale=why))1920df = raw[raw["year"].between(2000, 2022)].copy()21step("1_year_window", "keep rows with 2000<=year<=2022", n_raw, len(df),22 "Study window; no rows outside it in this file.")2324# Duplicate (iso3, year) rows25dupmask = df.duplicated(["iso3", "year"], keep=False)26if dupmask.any():27 dups = df[dupmask]28 nval = dups.groupby(["iso3", "year"])["life_expectancy"].nunique(dropna=False)29 if (nval > 1).any():30 raise SystemExit("Conflicting duplicate values; cannot resolve without outside information.")31 for iso, g in dups.groupby("iso3"):32 names = sorted(g["country_name"].unique())33 issues.append({"issue": f"{iso}: {g[['iso3','year']].drop_duplicates().shape[0]} country-years appear twice "34 f"under names {names} with identical values (assembly duplicate).",35 "handling": "Kept one row per (iso3, year); values identical so no number depends on the choice; "36 "label 'Turkiye' (current World Bank name) retained where applicable."})37 # prefer 'Turkiye' label deterministically, otherwise the last occurrence38 df["_pref"] = (df["country_name"] == "Turkiye").astype(int)39 df = df.sort_values(["iso3", "year", "_pref"]).drop_duplicates(["iso3", "year"], keep="last").drop(columns="_pref")40before = len(raw[raw["year"].between(2000, 2022)])41step("2_dedupe_iso3_year", "one row per (iso3, year); drop exact duplicates", before, len(df),42 "Each country contributes at most one observation per year (TUR listed as both Turkey and Turkiye).")4344# Missing values45miss = df[df["life_expectancy"].isna()]46for _, r in miss.iterrows():47 issues.append({"issue": f"{r.iso3} {r.year}: empty life_expectancy cell.",48 "handling": "Treated as missing (no imputation); country excluded from balanced set."})49before = len(df)50df = df[df["life_expectancy"].notna()]51step("3_drop_missing_value", "drop country-years with empty life_expectancy", before, len(df),52 "No imputation allowed; missing country-year has no value.")5354# Coverage55all_iso = sorted(raw["iso3"].unique())56cov = df.groupby("iso3")["year"].nunique().reindex(all_iso, fill_value=0)57balanced = sorted(cov[cov == len(YEARS)].index)58incomplete = sorted(set(all_iso) - set(balanced))59for iso in incomplete:60 yrs = sorted(df.loc[df.iso3 == iso, "year"])61 absent = sorted(set(YEARS) - set(raw.loc[raw.iso3 == iso, "year"]))62 if absent:63 issues.append({"issue": f"{iso}: no rows for years {absent[0]}-{absent[-1]} ({len(absent)} years absent; "64 f"recorded {yrs[0]}-{yrs[-1]}).",65 "handling": "Not imputed or looked up; country excluded from balanced set (still in 2022 baseline)."})66before = len(df)67analytic = df[df["iso3"].isin(balanced)].sort_values(["iso3", "year"])[["iso3", "country_name", "year", "life_expectancy"]]68step("4_balanced_panel", "keep countries with a recorded value in all 23 years 2000-2022", before, len(analytic),69 f"Balanced-panel estimand; excluded incomplete countries: {incomplete}.")7071assert len(analytic) == len(balanced) * len(YEARS)72assert not analytic.duplicated(["iso3", "year"]).any()7374m2000 = analytic.loc[analytic.year == 2000, "life_expectancy"].mean()75m2022 = analytic.loc[analytic.year == 2022, "life_expectancy"].mean()76avail22 = df[df.year == 2022] # deduped, non-missing: one row per iso377assert avail22["iso3"].is_unique78m2022_all = avail22["life_expectancy"].mean()7980results = {81 "task_id": "F1-conf-02",82 "n_countries_total": len(all_iso),83 "n_countries_balanced": len(balanced),84 "n_countries_incomplete": len(incomplete),85 "balanced_countries": balanced,86 "incomplete_countries": incomplete,87 "n_analysed": int(len(analytic)),88 "mean_2000_balanced": round(float(m2000), 6),89 "mean_2022_balanced": round(float(m2022), 6),90 "change_2000_2022_balanced": round(float(m2022 - m2000), 6),91 "mean_2022_all_available": round(float(m2022_all), 6),92 "n_countries_2022_available": int(avail22["iso3"].nunique()),93 "baseline_minus_balanced_2022": round(float(m2022_all - m2022), 6),94 "feasibility": "complete",95 "feasibility_reason": None,96 "data_issues": issues,97 "n_input_rows": {"life_expectancy_panel.csv": int(n_raw)},98 "random_seed": None,99}100OUT.mkdir(exist_ok=True)101analytic.to_csv(OUT / "analytic.csv", index=False)102pd.DataFrame(ledger).to_csv(OUT / "exclusions.csv", index=False)103with open(OUT / "results.json", "w") as f:104 json.dump(results, f, indent=2, ensure_ascii=False)105print(json.dumps({k: v for k, v in results.items() if k != "data_issues"}, indent=1))106