1"""Parse the D-PLACE Ethnographic Atlas CLDF release (v3.3) into one row per society with the variables fixed in2research/preregistration.md §2.2, plus Glottolog language family for Galton controls.34Ordinal categories of EA031 (mean size of local communities) and EA202 (population size) are converted to5log-midpoints of the category ranges printed in `codes.csv` (parsed from the code descriptions, not typed by hand);6the parsed table is written to results/diagnostics/dplace_code_midpoints.csv for inspection.7"""8from __future__ import annotations9import io, re, zipfile10from pathlib import Path11import numpy as np12import pandas as pd1314ROOT = Path(__file__).resolve().parents[3]15ZIP = ROOT / "data" / "raw" / "dplace_ea_v3_3" / "dplace-dataset-ea-v3.3.zip"16GLOTTO = ROOT / "data" / "raw" / "glottolog_cldf_languages" / "glottolog_v5.3_languages.csv"17DIAG = ROOT / "results" / "diagnostics"1819VARS = ["EA031", "EA202", "EA030", "EA033", "EA032", "EA072", "EA073", "EA090", "EA066", "EA067", "EA068",20 "EA070", "EA005", "EA028", "EA042", "EA034", "EA113"]212223def _read(z: zipfile.ZipFile, name: str) -> pd.DataFrame:24 path = [n for n in z.namelist() if n.endswith("cldf/" + name)][0]25 return pd.read_csv(io.BytesIO(z.read(path)), low_memory=False)262728def _range_midpoint(desc: str):29 """'1,000-5,000' -> geometric midpoint; '50,000 or more' -> 50,000 (lower bound, flagged); '<50' -> 25."""30 s = desc.replace(",", "")31 m = re.search(r"(\d+)\s*[-–]\s*(\d+)", s)32 if m:33 lo, hi = float(m.group(1)), float(m.group(2))34 return np.sqrt(lo * hi), lo, hi, "range"35 m = re.search(r"(\d+)\s*(or more|\+|and over|or over)", s)36 if m:37 lo = float(m.group(1)); return lo, lo, np.nan, "open_upper"38 m = re.search(r"(fewer than|less than|under|<)\s*(\d+)", s, flags=re.I)39 if m:40 hi = float(m.group(2)); return hi / 2, np.nan, hi, "open_lower"41 return np.nan, np.nan, np.nan, "none"424344def build() -> dict[str, pd.DataFrame]:45 z = zipfile.ZipFile(ZIP)46 soc = _read(z, "societies.csv"); var = _read(z, "variables.csv"); codes = _read(z, "codes.csv"); data = _read(z, "data.csv")47 d = data[data.Var_ID.isin(VARS)].copy()48 # numeric code value (the ordinal code number) and its description49 # ordinal code number = `ord` column of codes.csv; the "Missing data" code (ord 99) is NaN, never a value50 codes["code_num"] = pd.to_numeric(codes["ord"], errors="coerce")51 codes.loc[codes.Name.eq("Missing data") | codes.ID.str.endswith("-NA"), "code_num"] = np.nan52 cmap = codes.set_index("ID")[["code_num", "Description"]]53 d = d.join(cmap, on="Code_ID")54 wide = d.pivot_table(index="Soc_ID", columns="Var_ID", values="code_num", aggfunc="first")55 desc = d.pivot_table(index="Soc_ID", columns="Var_ID", values="Description", aggfunc="first")56 # midpoints for EA031 and EA202 from code descriptions57 mids = []58 # EA202 (population size) is a continuous variable in the CLDF release: numeric Value, no codes59 ea202 = d[d.Var_ID == "EA202"].set_index("Soc_ID").Value.map(lambda x: pd.to_numeric(x, errors="coerce"))60 wide["EA202_mid"] = ea202.reindex(wide.index) if len(ea202) else np.nan61 wide["EA202"] = wide["EA202_mid"]62 for v in ["EA031"]:63 cc = codes[codes.Var_ID == v].copy()64 parsed = [(_range_midpoint(n) if _range_midpoint(n)[3] != "none" else _range_midpoint(desc)) for n, desc in zip(cc.Name.astype(str), cc.Description.astype(str))]65 cc["mid"] = [p[0] for p in parsed]; cc["lo"] = [p[1] for p in parsed]; cc["hi"] = [p[2] for p in parsed]; cc["kind"] = [p[3] for p in parsed]66 mids.append(cc[["Var_ID", "ID", "Name", "Description", "mid", "lo", "hi", "kind"]])67 m = cc.set_index("ID")["mid"]68 wide[v + "_mid"] = d[d.Var_ID == v].set_index("Soc_ID").Code_ID.map(m).reindex(wide.index)69 DIAG.mkdir(parents=True, exist_ok=True)70 pd.concat(mids).to_csv(DIAG / "dplace_code_midpoints.csv", index=False)71 wide.columns = [str(c) for c in wide.columns]72 out = soc.set_index("ID")[["Name", "Latitude", "Longitude", "Glottocode", "region", "main_focal_year", "xd_id"]].join(wide, how="left")73 g = pd.read_csv(GLOTTO, low_memory=False)[["Glottocode", "Family_ID", "Macroarea"]].drop_duplicates("Glottocode")74 out = out.join(g.set_index("Glottocode"), on="Glottocode")75 out["Family_ID"] = out.Family_ID.fillna(out.Glottocode) # isolates: own family76 out.index.name = "soc_id"77 out = out.rename(columns={"EA031": "ea031_code", "EA202": "ea202_code"})78 out["log_community_size"] = np.log(out["EA031_mid"])79 out["log_population"] = np.log(out["EA202_mid"])80 out["jurisdictional_hierarchy_beyond"] = out["EA033"] - 1 # ord 1 = acephalous -> 0 levels beyond the community81 out["jurisdictional_hierarchy_local"] = out["EA032"]82 return {"societies": out.reset_index(), "variables": var[var.ID.isin(VARS)][["ID", "Name", "Description", "type"]],83 "codes": codes[codes.Var_ID.isin(VARS)][["Var_ID", "ID", "Name", "Description"]], "desc": desc.reset_index()}84