1"""F1-conf-02: balanced-panel mean life expectancy 2000-2022.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json, os5import pandas as pd67SRC = "data/life_expectancy_panel.csv"8OUT = "out"9YEARS = list(range(2000, 2023))10os.makedirs(OUT, exist_ok=True)1112raw = pd.read_csv(SRC, dtype={"iso3": str, "country_name": str})13n_raw = len(raw)14for c in ["iso3", "country_name"]:15 raw[c] = raw[c].str.strip()16issues, excl = [], []1718def step(name, rule, before, after, why):19 excl.append(dict(step=name, rule=rule, n_before=before, n_after=after,20 n_removed=before - after, rationale=why))2122# Step 1: duplicate (iso3, year) rows23dup_mask = raw.duplicated(["iso3", "year"], keep=False)24dups = raw[dup_mask]25conflict = dups.groupby(["iso3", "year"])["life_expectancy"].nunique(dropna=False)26if (conflict > 1).any():27 raise SystemExit(f"Conflicting duplicate values: {conflict[conflict > 1]}")28for iso, g in dups.groupby("iso3"):29 names = sorted(g.country_name.unique())30 issues.append({"issue": f"{iso} appears twice for {g.year.nunique()} years under names {names} "31 f"(assembly of two extractions); duplicate rows have identical values",32 "handling": "kept one row per (iso3, year); identical values so no choice of value needed; "33 "country_name set to the most recent spelling 'Turkiye' for TUR"})34# canonical name: prefer 'Turkiye' for TUR; otherwise the single name35canon = {iso: sorted(g.country_name.unique())[-1] for iso, g in raw.groupby("iso3")}36if "TUR" in canon:37 canon["TUR"] = "Turkiye" if "Turkiye" in set(raw.loc[raw.iso3 == "TUR", "country_name"]) else canon["TUR"]38raw["country_name"] = raw.iso3.map(canon)39d = raw.drop_duplicates(["iso3", "year"]).copy()40step("1_dedup", "drop duplicate (iso3, year) rows (identical values)", n_raw, len(d),41 "each country contributes at most one observation per year; duplicates came from assembling two extractions")4243# Step 2: rows with no recorded value44miss = d[d.life_expectancy.isna()]45for _, r in miss.iterrows():46 issues.append({"issue": f"{r.iso3} {int(r.year)} has an empty life_expectancy cell",47 "handling": "treated as missing (no imputation); country therefore excluded from balanced set"})48n_b = len(d)49d = d[d.life_expectancy.notna()].copy()50step("2_missing_value", "drop rows with empty life_expectancy", n_b, len(d),51 "an empty cell means no recorded value; no imputation allowed")5253# Step 3: keep only countries with a value in every year 2000-202254d = d[d.year.isin(YEARS)]55cover = d.groupby("iso3").year.nunique()56balanced = sorted(cover[cover == len(YEARS)].index.tolist())57all_iso = sorted(raw.iso3.unique())58incomplete = [i for i in all_iso if i not in balanced]59for iso in incomplete:60 yrs = sorted(d.loc[d.iso3 == iso, "year"].unique())61 missing_years = [y for y in YEARS if y not in yrs]62 if len(missing_years) > 1 and missing_years == list(range(missing_years[0], missing_years[-1] + 1)):63 desc = f"{missing_years[0]}-{missing_years[-1]}"64 else:65 desc = ",".join(map(str, missing_years))66 issues.append({"issue": f"{iso} has no recorded value for year(s) {desc} (no row or empty cell)",67 "handling": "excluded from balanced panel; still counted in n_countries_total and, if it has a 2022 value, in the all-available 2022 baseline"})68n_b = len(d)69ana = d[d.iso3.isin(balanced)].sort_values(["iso3", "year"])[["iso3", "country_name", "year", "life_expectancy"]]70step("3_balanced_panel", "keep countries with a recorded value in every year 2000-2022", n_b, len(ana),71 f"balanced-panel estimand; incomplete countries: {incomplete}")72ana.to_csv(f"{OUT}/analytic.csv", index=False)73pd.DataFrame(excl).to_csv(f"{OUT}/exclusions.csv", index=False)7475m = ana.groupby("year").life_expectancy.mean()76assert ana.groupby("year").iso3.nunique().eq(len(balanced)).all()77m2000, m2022 = float(m.loc[2000]), float(m.loc[2022])78base = d[d.year == 2022].drop_duplicates("iso3")79res = {80 "task_id": "F1-conf-02",81 "n_countries_total": len(all_iso),82 "n_countries_balanced": len(balanced),83 "n_countries_incomplete": len(incomplete),84 "balanced_countries": balanced,85 "n_analysed": int(len(ana)),86 "mean_2000_balanced": round(m2000, 6),87 "mean_2022_balanced": round(m2022, 6),88 "change_2000_2022_balanced": round(m2022 - m2000, 6),89 "mean_2022_all_available": round(float(base.life_expectancy.mean()), 6),90 "n_countries_2022_all_available": int(base.iso3.nunique()),91 "feasibility": "complete",92 "feasibility_reason": None,93 "data_issues": issues,94 "n_input_rows": {"life_expectancy_panel.csv": int(n_raw)},95 "random_seed": None,96}97with open(f"{OUT}/results.json", "w") as f:98 json.dump(res, f, indent=2)99print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=2))100