1"""F1-conf-02: balanced-panel mean life expectancy 2000-2022. Run from workspace root."""2import json, os3import pandas as pd45IN = "data/life_expectancy_panel.csv"6OUT = "out"7YEARS = list(range(2000, 2023))8os.makedirs(OUT, exist_ok=True)910raw = pd.read_csv(IN, dtype={"iso3": str, "country_name": str})11n_read = len(raw)12issues, ledger = [], []1314def step(name, rule, before, after, why):15 ledger.append(dict(step=len(ledger) + 1, rule=f"{name}: {rule}", n_before=before,16 n_after=after, n_removed=before - after, rationale=why))1718d = raw.copy()19# 1. Rows outside 2000-2022 (none expected)20b = len(d); d = d[d.year.isin(YEARS)]21step("year_window", "keep rows with year in 2000-2022", b, len(d), "Study period.")2223# 2. Duplicate (iso3, year) rows from assembly of two extractions24dup = d[d.duplicated(["iso3", "year"], keep=False)]25if len(dup):26 conflicts = dup.groupby(["iso3", "year"]).life_expectancy.nunique(dropna=False).gt(1).sum()27 names = {k: sorted(v.unique()) for k, v in dup.groupby("iso3").country_name}28 if conflicts:29 raise SystemExit(f"Conflicting duplicate values: {conflicts}")30 issues.append({"issue": f"{len(dup) - dup[['iso3','year']].drop_duplicates().shape[0]} duplicate (iso3, year) rows "31 f"from the two-extraction assembly: {names} (same iso3, different country_name spelling); "32 "life_expectancy values identical in every duplicated pair.",33 "handling": "Identified countries by iso3; kept one row per (iso3, year) (first occurrence), "34 "using the latest-spelling country name as the label for the country."})35# canonical name: last-listed spelling per iso3 (later, wider extraction)36canon = d.groupby("iso3").country_name.last()37b = len(d); d = d.drop_duplicates(["iso3", "year"], keep="first").copy()38d["country_name"] = d.iso3.map(canon)39step("dedupe", "one row per (iso3, year)", b, len(d), "Each country counted once per year; duplicates were identical.")4041# 3. Missing values42miss = d[d.life_expectancy.isna()]43if len(miss):44 issues.append({"issue": "Empty life_expectancy cell(s): " + ", ".join(f"{r.iso3} {r.year}" for r in miss.itertuples()),45 "handling": "Treated as missing (no imputation); country excluded from the balanced set."})46b = len(d); d = d[d.life_expectancy.notna()]47step("drop_missing", "drop country-years with empty life_expectancy", b, len(d), "No imputation allowed.")4849# 4. Incomplete coverage50cov = d.groupby("iso3").year.nunique()51all_iso = sorted(raw.iso3.unique())52balanced = sorted(cov[cov == len(YEARS)].index)53incomplete = [c for c in all_iso if c not in balanced]54for c in incomplete:55 yrs = sorted(d[d.iso3 == c].year)56 absent = [y for y in YEARS if y not in yrs]57 if c not in set(miss.iso3):58 issues.append({"issue": f"{c} has no row at all for {len(absent)} years ({min(absent)}-{max(absent)}); "59 f"only {len(yrs)} of 23 years present (likely only in the narrower extraction).",60 "handling": "Not imputed; country excluded from the balanced set (included in the 2022 all-available baseline)."})61b = len(d); an = d[d.iso3.isin(balanced)].sort_values(["iso3", "year"])62step("balanced_panel", "keep countries with a recorded value in every year 2000-2022", b, len(an),63 f"Balanced panel; removed countries: {', '.join(incomplete)}.")6465an[["iso3", "country_name", "year", "life_expectancy"]].to_csv(f"{OUT}/analytic.csv", index=False)66pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)6768m = an.groupby("year").life_expectancy.mean()69all22 = d[d.year == 2022].drop_duplicates("iso3").life_expectancy.mean()70res = {71 "task_id": "F1-conf-02",72 "n_countries_total": len(all_iso),73 "n_countries_balanced": len(balanced),74 "n_countries_incomplete": len(incomplete),75 "balanced_countries": balanced,76 "n_analysed": int(len(an)),77 "mean_2000_balanced": round(float(m[2000]), 4),78 "mean_2022_balanced": round(float(m[2022]), 4),79 "change_2000_2022_balanced": round(float(m[2022] - m[2000]), 4),80 "mean_2022_all_available": round(float(all22), 4),81 "feasibility": "complete",82 "feasibility_reason": None,83 "data_issues": issues,84 "n_input_rows": {"life_expectancy_panel.csv": int(n_read)},85 "random_seed": None,86}87with open(f"{OUT}/results.json", "w") as f:88 json.dump(res, f, indent=2, ensure_ascii=False)89print(json.dumps(res, indent=2, ensure_ascii=False))90print(m.round(3).to_string())9192# Descriptive sensitivity checks (printed only; not part of results.json)93naive_dup = raw[(raw.year == 2022) & raw.life_expectancy.notna()].life_expectancy.mean()94with_che = d[d.iso3.isin(balanced + ["CHE"])]95print("SENS naive 2022 mean if TUR duplicate rows were not removed:", round(float(naive_dup), 4))96print("SENS 15+CHE mean 2000/2022/change:",97 [round(float(with_che[with_che.year == y].life_expectancy.mean()), 4) for y in (2000, 2022)],98 round(float(with_che[with_che.year == 2022].life_expectancy.mean() - with_che[with_che.year == 2000].life_expectancy.mean()), 4))99print("SENS n countries with 2022 value:", d[d.year == 2022].iso3.nunique())100print("SENS per-country change range (balanced):",101 (an[an.year == 2022].set_index("iso3").life_expectancy - an[an.year == 2000].set_index("iso3").life_expectancy).agg(["min", "idxmin", "max", "idxmax"]).to_dict())102