1"""Phase C — compare primary (data/extracted/json) and independent duplicate (data/validated/json) extractions on the2validation sample. Reports study-level eligibility agreement, coefficient recall/precision (matched on model + taxonomy_id +3printed variable name, fuzzy), numerical agreement of beta and se (relative error ≤ 1 %), and writes reports/extraction_audit.md4tables + data/validated/validation_results.csv."""5import json, re6from difflib import SequenceMatcher7from pathlib import Path8import numpy as np, pandas as pd910ROOT = Path(__file__).resolve().parents[2]; P = ROOT / "data" / "extracted" / "json"; V = ROOT / "data" / "validated" / "json"111213def norm(s): return re.sub(r"[^a-z0-9]", "", (s or "").lower())141516def load(d):17 out = {}18 for f in d.glob("*.json"):19 try: out[f.stem] = json.loads(f.read_text())20 except Exception as e: out[f.stem] = {"error": repr(e)}21 return out222324def match_coefs(a, b):25 """Greedy matching of coefficient rows between two extractions of the same paper."""26 ca = [c for c in a.get("coefficients", []) if c.get("confidence") != "low"]; cb = [c for c in b.get("coefficients", []) if c.get("confidence") != "low"]27 used = set(); pairs = []28 for i, x in enumerate(ca):29 best = None; bs = 030 for j, y in enumerate(cb):31 if j in used: continue32 s = (x.get("taxonomy_id") == y.get("taxonomy_id")) * 2 + SequenceMatcher(None, norm(x.get("var_name_printed")), norm(y.get("var_name_printed"))).ratio() + (norm(x.get("model_id")) == norm(y.get("model_id")))33 if s > bs: bs, best = s, j34 if best is not None and bs >= 2.5:35 used.add(best); pairs.append((x, cb[best]))36 return ca, cb, pairs373839def main():40 prim, dup = load(P), load(V); rows = []41 for rid, b in dup.items():42 a = prim.get(rid)43 if a is None: continue44 ea, eb = a.get("eligible"), b.get("eligible")45 ca, cb, pairs = match_coefs(a, b)46 beta_ok = sum(1 for x, y in pairs if x.get("beta") is not None and y.get("beta") is not None and abs(x["beta"] - y["beta"]) <= 0.01 * max(abs(x["beta"]), 1e-9) + 1e-9)47 se_pairs = [(x, y) for x, y in pairs if x.get("se") is not None and y.get("se") is not None]48 se_ok = sum(1 for x, y in se_pairs if abs(x["se"] - y["se"]) <= 0.01 * max(abs(x["se"]), 1e-9) + 1e-9)49 sign_ok = sum(1 for x, y in pairs if x.get("beta") is not None and y.get("beta") is not None and np.sign(x["beta"]) == np.sign(y["beta"]))50 tax_ok = sum(1 for x, y in pairs if x.get("taxonomy_id") == y.get("taxonomy_id"))51 rows.append({"record_id": rid, "eligible_primary": ea, "eligible_duplicate": eb, "eligibility_agree": ea == eb, "n_coef_primary": len(ca), "n_coef_duplicate": len(cb),52 "n_matched": len(pairs), "recall_vs_duplicate": len(pairs) / len(cb) if cb else np.nan, "precision_vs_duplicate": len(pairs) / len(ca) if ca else np.nan,53 "beta_agree_1pct": beta_ok / len(pairs) if pairs else np.nan, "se_agree_1pct": se_ok / len(se_pairs) if se_pairs else np.nan, "sign_agree": sign_ok / len(pairs) if pairs else np.nan,54 "taxonomy_agree": tax_ok / len(pairs) if pairs else np.nan})55 df = pd.DataFrame(rows); (ROOT / "data" / "validated").mkdir(exist_ok=True); df.to_csv(ROOT / "data" / "validated" / "validation_results.csv", index=False)56 if len(df):57 summ = {"papers_double_extracted": len(df), "eligibility_agreement": df.eligibility_agree.mean(), "coef_recall": df.recall_vs_duplicate.mean(), "coef_precision": df.precision_vs_duplicate.mean(),58 "beta_agreement_1pct": df.beta_agree_1pct.mean(), "se_agreement_1pct": df.se_agree_1pct.mean(), "sign_agreement": df.sign_agree.mean(), "taxonomy_agreement": df.taxonomy_agree.mean(),59 "coefficients_compared": int(df.n_matched.sum())}60 (ROOT / "data" / "validated" / "validation_summary.json").write_text(json.dumps(summ, indent=2, default=float)); print(json.dumps(summ, indent=2, default=float))616263if __name__ == "__main__":64 main()65