1"""F1-conf-02: balanced-panel mean life expectancy 2000-2022. Reads data/ only."""2import json, hashlib3from pathlib import Path4import pandas as pd56ROOT = Path(__file__).resolve().parent.parent7SRC = ROOT / "data" / "life_expectancy_panel.csv"8OUT = ROOT / "out"9YEARS = list(range(2000, 2023))1011raw = pd.read_csv(SRC, dtype={"iso3": str, "country_name": str})12raw["iso3"] = raw["iso3"].str.strip()13raw["country_name"] = raw["country_name"].str.strip()14n_raw = len(raw)15issues, excl = [], []1617def step(name, rule, before, after, why):18 excl.append(dict(step=name, rule=rule, n_before=before, n_after=after,19 n_removed=before - after, rationale=why))2021# --- 1. duplicate (iso3, year) rows ---22dup_mask = raw.duplicated(["iso3", "year"], keep=False)23dups = raw[dup_mask]24for iso, g in dups.groupby("iso3"):25 # verify duplicates carry identical values (otherwise conflict -> cannot choose)26 conflict = g.groupby("year")["life_expectancy"].nunique(dropna=False).gt(1).any()27 if conflict:28 raise SystemExit(f"Conflicting duplicate values for {iso}; not resolvable without outside info")29 names = sorted(g["country_name"].unique())30 issues.append({"issue": f"iso3 {iso} appears under {len(names)} country_name spellings {names} "31 f"with duplicated (iso3, year) rows for {g['year'].nunique()} years "32 f"({len(g)} rows); values are identical across duplicates",33 "handling": "kept one row per (iso3, year) (country identified by iso3); "34 "duplicate rows removed; country counted once"})35# canonical name: last spelling alphabetically ('Turkiye' over 'Turkey'), current WDI name36canon = raw.groupby("iso3")["country_name"].agg(lambda s: sorted(s.unique())[-1])37df = raw.drop_duplicates(["iso3", "year"], keep="first").copy()38df["country_name"] = df["iso3"].map(canon)39step("1_dedup_iso3_year", "keep one row per (iso3, year)", n_raw, len(df),40 "same country (iso3) assembled twice under two name spellings with identical values")4142# --- 2. incomplete countries ---43n_total = df["iso3"].nunique()44obs = df.dropna(subset=["life_expectancy"])45obs = obs[obs["year"].isin(YEARS)]46cnt = obs.groupby("iso3")["year"].nunique().reindex(sorted(df["iso3"].unique()), fill_value=0)47balanced = sorted(cnt[cnt == len(YEARS)].index)48incomplete = sorted(set(df["iso3"]) - set(balanced))49for iso in incomplete:50 yrs = sorted(set(obs.loc[obs.iso3 == iso, "year"]))51 missing = [y for y in YEARS if y not in yrs]52 blank = sorted(df.loc[(df.iso3 == iso) & df.life_expectancy.isna(), "year"].tolist())53 desc = (f"{iso}: no recorded value for {len(missing)} of 23 years "54 f"({missing[0]}-{missing[-1]}" + (")" if len(missing) == missing[-1]-missing[0]+1 else f"; years {missing})"))55 if blank:56 desc += f"; empty life_expectancy cell in year(s) {blank}"57 issues.append({"issue": desc, "handling": "treated as missing (no imputation); country excluded from balanced set"})5859before = len(df)60part_missing = df[df.iso3.isin(incomplete) & (df.groupby("iso3")["year"].transform("size") < len(YEARS))]61# split exclusion into two transparent steps62short = [i for i in incomplete if (df.iso3 == i).sum() < len(YEARS)]63gaps = [i for i in incomplete if i not in short]64df2 = df[~df.iso3.isin(short)]65step("2_drop_countries_missing_rows", f"drop countries without a row for every year 2000-2022 ({', '.join(short)})",66 before, len(df2), "not observed in every year; balanced-panel estimand requires all 23 years")67df3 = df2[~df2.iso3.isin(gaps)]68step("3_drop_countries_with_empty_value", f"drop countries with an empty life_expectancy cell in some year ({', '.join(gaps)})",69 len(df2), len(df3), "missing value may not be imputed; country not in balanced set")7071ana = df3[df3.year.isin(YEARS)].sort_values(["iso3", "year"])[["iso3", "country_name", "year", "life_expectancy"]]72assert ana.life_expectancy.notna().all() and len(ana) == len(balanced) * len(YEARS)73assert not ana.duplicated(["iso3", "year"]).any()74if len(ana) != len(df3):75 step("4_restrict_years", "keep years 2000-2022", len(df3), len(ana), "outside study window")7677m2000 = float(ana.loc[ana.year == 2000, "life_expectancy"].mean())78m2022 = float(ana.loc[ana.year == 2022, "life_expectancy"].mean())79avail22 = df[(df.year == 2022) & df.life_expectancy.notna()]80assert not avail22.iso3.duplicated().any()81m22all = float(avail22["life_expectancy"].mean())82# descriptive dispersion of per-country changes (fixed set of countries, not a sample)83w = ana.pivot(index="iso3", columns="year", values="life_expectancy")84chg = (w[2022] - w[2000])85# sensitivity: naive 2022 mean if the duplicated TUR rows were NOT removed (wrong; shows impact)86m22_raw_dups = float(raw.loc[raw.year == 2022, "life_expectancy"].dropna().mean())8788res = {89 "task_id": "F1-conf-02",90 "n_countries_total": int(n_total),91 "n_countries_balanced": len(balanced),92 "n_countries_incomplete": len(incomplete),93 "balanced_countries": balanced,94 "n_analysed": int(len(ana)),95 "mean_2000_balanced": round(m2000, 6),96 "mean_2022_balanced": round(m2022, 6),97 "change_2000_2022_balanced": round(m2022 - m2000, 6),98 "mean_2022_all_available": round(m22all, 6),99 "n_countries_2022_available": int(avail22.iso3.nunique()),100 "change_per_country_sd": round(float(chg.std(ddof=1)), 6),101 "change_per_country_min": round(float(chg.min()), 6),102 "change_per_country_min_iso3": str(chg.idxmin()),103 "change_per_country_max": round(float(chg.max()), 6),104 "change_per_country_max_iso3": str(chg.idxmax()),105 "sensitivity_mean_2022_naive_with_duplicate_rows": round(m22_raw_dups, 6),106 "feasibility": "complete",107 "feasibility_reason": None,108 "data_issues": issues,109 "n_input_rows": {"life_expectancy_panel.csv": int(n_raw)},110 "random_seed": None,111 "input_sha256": hashlib.sha256(SRC.read_bytes()).hexdigest(),112}113OUT.mkdir(exist_ok=True)114ana.to_csv(OUT / "analytic.csv", index=False, float_format="%.3f")115pd.DataFrame(excl).to_csv(OUT / "exclusions.csv", index=False)116(OUT / "results.json").write_text(json.dumps(res, indent=2, ensure_ascii=False) + "\n")117print(json.dumps({k: res[k] for k in ["n_countries_balanced", "n_analysed", "mean_2000_balanced",118 "mean_2022_balanced", "change_2000_2022_balanced",119 "mean_2022_all_available"]}))120