1"""F1-conf-02: balanced-panel mean life expectancy 2000-2022.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv.4No randomness is used (random_seed = null)."""5import json, os6import pandas as pd78DATA = "data/life_expectancy_panel.csv"9OUT = "out"10YEARS = list(range(2000, 2023))11CANON_NAME = {"TUR": "Turkiye"} # documented choice for the duplicated Turkey/Turkiye rows1213raw = pd.read_csv(DATA, dtype={"country_name": str, "iso3": str})14n_raw = len(raw)15ledger, issues = [], []1617def step(name, rule, before, after, why):18 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,19 n_removed=before - after, rationale=why))2021# --- detect duplicates on (iso3, year)22dup = raw[raw.duplicated(["iso3", "year"], keep=False)]23for iso, g in dup.groupby("iso3"):24 conflicting = (g.groupby("year")["life_expectancy"].nunique(dropna=False) > 1).sum()25 if conflicting:26 raise SystemExit(f"Conflicting duplicate values for {iso}: decision required")27 names = sorted(g.country_name.unique())28 issues.append({"issue": f"{iso} appears twice per year under names {names} "29 f"({len(g)} rows for {g.year.nunique()} years; values identical in every year) - "30 "assembly error from the two extractions",31 "handling": f"deduplicated on (iso3, year), keeping one row per year; "32 f"country_name standardised to '{CANON_NAME.get(iso, names[-1])}'; values unaffected"})33d = raw.copy()34d["country_name"] = d.apply(lambda r: CANON_NAME.get(r.iso3, r.country_name), axis=1)35d = d.sort_values(["iso3", "year"]).drop_duplicates(["iso3", "year"], keep="first")36step("1_dedup", "keep one row per (iso3, year); duplicates had identical values",37 n_raw, len(d), "country identified by iso3; at most one observation per country-year")3839# --- missing values40miss = d[d.life_expectancy.isna()]41for _, r in miss.iterrows():42 issues.append({"issue": f"{r.iso3} {int(r.year)}: empty life_expectancy cell",43 "handling": "treated as missing (no imputation); country excluded from balanced set"})44d_obs = d.dropna(subset=["life_expectancy"])45step("2_drop_missing", "drop country-years with empty life_expectancy", len(d), len(d_obs),46 "empty cell = no recorded value; no imputation allowed")4748# --- countries with absent years49cov = d_obs.groupby("iso3")["year"].apply(lambda s: sorted(set(s)))50for iso, ys in cov.items():51 absent_rows = sorted(set(YEARS) - set(d.loc[d.iso3 == iso, "year"]))52 if absent_rows:53 issues.append({"issue": f"{iso}: no rows at all for years {absent_rows[0]}-{absent_rows[-1]} "54 f"({len(absent_rows)} years; coverage only {min(ys)}-{max(ys)})",55 "handling": "treated as missing (no imputation); country excluded from balanced set, "56 "kept in the 2022 all-available baseline"})5758balanced = sorted(iso for iso, ys in cov.items() if set(YEARS) <= set(ys))59an = d_obs[d_obs.iso3.isin(balanced) & d_obs.year.isin(YEARS)]60step("3_balanced", "keep countries with a recorded value in every year 2000-2022", len(d_obs), len(an),61 "balanced-panel estimand: fixed composition across years")62an = an[["iso3", "country_name", "year", "life_expectancy"]].sort_values(["iso3", "year"]).reset_index(drop=True)63assert len(an) == len(balanced) * len(YEARS)64assert not an.duplicated(["iso3", "year"]).any()6566# --- estimates67ymean = an.groupby("year")["life_expectancy"].mean()68m00, m22 = float(ymean[2000]), float(ymean[2022])69base = d_obs[d_obs.year == 2022]70assert base.iso3.is_unique71m22_all = float(base.life_expectancy.mean())7273n_tot = int(d.iso3.nunique())74res = {75 "task_id": "F1-conf-02",76 "n_countries_total": n_tot,77 "n_countries_balanced": len(balanced),78 "n_countries_incomplete": n_tot - len(balanced),79 "balanced_countries": balanced,80 "n_analysed": int(len(an)),81 "mean_2000_balanced": round(m00, 6),82 "mean_2022_balanced": round(m22, 6),83 "change_2000_2022_balanced": round(m22 - m00, 6),84 "mean_2022_all_available": round(m22_all, 6),85 "n_countries_2022_all_available": int(base.iso3.nunique()),86 "incomplete_countries": sorted(set(d.iso3) - set(balanced)),87 "balanced_mean_by_year": {str(y): round(float(v), 6) for y, v in ymean.items()},88 "feasibility": "complete",89 "feasibility_reason": None,90 "data_issues": issues,91 "n_input_rows": {"life_expectancy_panel.csv": int(n_raw)},92 "random_seed": None,93}94os.makedirs(OUT, exist_ok=True)95an.to_csv(f"{OUT}/analytic.csv", index=False, float_format="%.3f")96pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)97with open(f"{OUT}/results.json", "w") as f:98 json.dump(res, f, indent=2, ensure_ascii=False)99print(f"balanced countries: {len(balanced)} {balanced}")100print(f"mean 2000 = {res['mean_2000_balanced']}, mean 2022 = {res['mean_2022_balanced']}, "101 f"change = {res['change_2000_2022_balanced']}; 2022 all-available = {res['mean_2022_all_available']}")102