1"""Phase C — build the analysis dataset from extraction JSONs: data/final/studies.csv, data/final/coefficients_raw.csv,2data/final/effects.csv (harmonised, with moderators). Primary synthesis uses confidence in {high, medium} and family != other.3Usage: python scripts/harmonization/build_effects.py"""4import json, re, sys5from pathlib import Path6import numpy as np, pandas as pd, yaml78ROOT = Path(__file__).resolve().parents[2]; sys.path.insert(0, str(ROOT / "scripts" / "harmonization"))9from harmonize import harmonise_row # noqa: E4021011JS = ROOT / "data" / "extracted" / "json"; FINAL = ROOT / "data" / "final"; FINAL.mkdir(parents=True, exist_ok=True)12countries = yaml.safe_load((ROOT / "scripts" / "harmonization" / "countries.yaml").read_text())13tax = yaml.safe_load((ROOT / "scripts" / "harmonization" / "taxonomy.yaml").read_text())14TAX_GROUP = {leaf: grp for grp, leaves in tax.items() for leaf in leaves}15ALIAS = {"usa": "United States", "us": "United States", "u.s.": "United States", "united states of america": "United States", "uk": "United Kingdom", "england": "United Kingdom",16 "scotland": "United Kingdom", "wales": "United Kingdom", "great britain": "United Kingdom", "korea": "South Korea", "republic of korea": "South Korea", "prc": "China",17 "people's republic of china": "China", "hong kong sar": "Hong Kong", "the netherlands": "Netherlands", "holland": "Netherlands", "czechia": "Czech Republic", "türkiye": "Turkey", "turkiye": "Turkey"}18EST_CLASS = {"ols": "ols", "wls": "ols", "gls": "ols", "fe_panel": "panel", "panel": "panel", "iv": "iv", "2sls": "iv", "quantile": "quantile", "spatial_lag": "spatial", "spatial_error": "spatial", "sac": "spatial",19 "sdm": "spatial", "gwr": "spatial", "spatial": "spatial", "ml_marginal": "ml", "random_forest": "ml", "xgboost": "ml", "gbm": "ml", "neural": "ml", "shap": "ml", "did": "quasi_experimental", "rdd": "quasi_experimental", "box-cox": "ols", "boxcox": "ols"}202122def canon_country(c):23 if not c: return "unknown"24 c0 = str(c).split(",")[0].split(";")[0].split("/")[0].strip(); key = c0.lower()25 if key in ALIAS: return ALIAS[key]26 for name in countries:27 if name.lower() == key or name.lower() in key: return name28 return c0.title() if c0 else "unknown"293031def est_class(e):32 e = (e or "").lower()33 for k, v in EST_CLASS.items():34 if k in e: return v35 return "other" if e else "unknown"363738def build_ai():39 rows = []40 for f in sorted((ROOT / "data" / "extracted" / "ai").glob("*.json")):41 try: j = json.loads(f.read_text())42 except Exception: continue43 if not j.get("eligible", True): continue44 j["record_id"] = j.get("record_id", f.stem); j["country"] = canon_country(j.get("country")); j["methods_compared"] = ";".join(j.get("methods_compared") or [])45 rows.append({k: v for k, v in j.items() if not isinstance(v, (dict, list))})46 if rows:47 pd.DataFrame(rows).to_csv(FINAL / "ai_studies.csv", index=False); print("ai studies", len(rows))484950def main():51 build_ai()52 studies, coefs = [], []53 for f in sorted(JS.glob("*.json")):54 try: j = json.loads(f.read_text())55 except Exception as e:56 studies.append({"record_id": f.stem, "json_error": repr(e)}); continue57 st = j.get("study") or {}; rid = j.get("record_id", f.stem)58 country = canon_country(st.get("country")); meta = countries.get(country, [None, None, None, None, None])59 srow = {"record_id": rid, "eligible": bool(j.get("eligible", False)), "exclusion_reason": j.get("exclusion_reason"), "title": st.get("title"), "year": st.get("year"), "doi": st.get("doi"),60 "country": country, "continent": meta[0], "income_group": meta[1], "iso3": meta[2], "lat": meta[3], "lon": meta[4], "region_city": st.get("region_city"),61 "sample_start": st.get("sample_start"), "sample_end": st.get("sample_end"), "property_type": st.get("property_type") or "unknown", "price_type": st.get("price_type") or "unknown",62 "document_type": st.get("document_type") or "unknown", "estimators": ";".join(st.get("estimators") or []), "spatial_method": st.get("spatial_method"), "time_fe": st.get("time_fe"),63 "location_fe": st.get("location_fe"), "se_type": st.get("se_type"), "n_obs_total": st.get("n_obs_total"), "ai_methods": ";".join(st.get("ai_methods") or []),64 "n_models": len(j.get("models") or []), "n_coefficients": len(j.get("coefficients") or [])}65 studies.append(srow)66 models = {m.get("model_id"): m for m in (j.get("models") or [])}67 for c in j.get("coefficients") or []:68 m = models.get(c.get("model_id"), {})69 coefs.append({"record_id": rid, "study_id": rid, "model_id": f"{rid}:{c.get('model_id')}", "table": m.get("table"), "column": m.get("column"), "page": c.get("page") or m.get("page"),70 "var_name_printed": c.get("var_name_printed"), "taxonomy_id": c.get("taxonomy_id") or "other", "taxonomy_group": TAX_GROUP.get(c.get("taxonomy_id"), "other"),71 "x_transform": c.get("x_transform"), "x_unit": c.get("x_unit"), "beta": c.get("beta"), "se": c.get("se"), "t": c.get("t"), "p": c.get("p"), "p_stars": c.get("p_stars"),72 "ci_lo": c.get("ci_lo"), "ci_hi": c.get("ci_hi"), "confidence": c.get("confidence") or "low", "notes": c.get("notes"),73 "dep_var": m.get("dep_var"), "dep_transform": m.get("dep_transform"), "estimator": m.get("estimator") or (st.get("estimators") or [""])[0], "n_obs": m.get("n_obs") or st.get("n_obs_total"),74 "r2": m.get("r2"), "paren": m.get("se_or_t_in_parentheses"), "sample_subset": m.get("sample_subset")})75 S = pd.DataFrame(studies); C = pd.DataFrame(coefs)76 S.to_csv(FINAL / "studies.csv", index=False); C.to_csv(FINAL / "coefficients_raw.csv", index=False)77 if len(C) == 0:78 print("no coefficients yet"); return79 # harmonise80 H = []81 for r in C.to_dict("records"):82 for k in ("beta", "se", "t", "p", "ci_lo", "ci_hi", "n_obs"):83 try: r[k] = float(r[k]) if r[k] is not None and r[k] != "" else np.nan84 except Exception: r[k] = np.nan85 if np.isnan(r["beta"]): continue86 h = harmonise_row({**r, "k_params": None}); H.append({**r, **h})87 E = pd.DataFrame(H).merge(S[["record_id", "year", "country", "continent", "income_group", "sample_start", "sample_end", "property_type", "price_type", "document_type", "time_fe", "location_fe", "se_type", "ai_methods"]], on="record_id", how="left")88 for c in ("sample_start", "sample_end", "year"):89 E[c] = pd.to_numeric(E[c], errors="coerce"); E.loc[(E[c] < 1900) | (E[c] > 2027), c] = np.nan90 E["data_mid_year"] = E[["sample_start", "sample_end"]].mean(axis=1); E["data_mid_year"] = E.data_mid_year.fillna(E.year - 3)91 E["estimator_class"] = E.estimator.map(est_class); E["pub_type"] = E.document_type.map(lambda d: "journal" if d == "journal" else "other")92 E["usable"] = (E.es_se.notna() & (E.es_se > 0) & E.confidence.isin(["high", "medium"]) & E.family.isin(["pct_premium", "semi_elasticity", "elasticity"]) & (E.taxonomy_id != "other")93 & (E.harm_flags.fillna("") == ""))94 E["usable_partial_r"] = E.partial_r.notna() & E.confidence.isin(["high", "medium"]) & (E.taxonomy_id != "other") & ~E.harm_flags.fillna("").str.contains("year_built")95 # winsorisation flag for extreme harmonised effects within family (robustness, not applied to primary)96 E["extreme"] = False97 for (fam, tx), g in E[E.usable].groupby(["family", "taxonomy_id"]):98 if len(g) >= 10:99 lo, hi = g.es.quantile([0.01, 0.99]); E.loc[g.index[(g.es < lo) | (g.es > hi)], "extreme"] = True100 E.to_csv(FINAL / "effects_all.csv", index=False); E[E.usable].to_csv(FINAL / "effects.csv", index=False)101 print("studies", len(S), "eligible", int(S.eligible.sum()) if "eligible" in S else 0, "coefficients", len(C), "harmonised usable", int(E.usable.sum()), "families", E[E.usable].family.value_counts().to_dict())102103104if __name__ == "__main__":105 main()106