1#!/usr/bin/env python2"""F4-conf-01: extract printed treatment effects/uncertainty from two source extracts.34Reads only data/; writes out/results.json deterministically. No randomness (random_seed = null).5Values are parsed from the text (not hard-coded); matched source lines are printed.6"""7import json, math, os, re, hashlib8from statistics import NormalDist910ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))11DATA = os.path.join(ROOT, "data")12DCR_ID, RI_ID = "arxiv-2609.27732v1", "arxiv-2606.14621v1"13DCR_F = os.path.join(DATA, "sources", DCR_ID + ".txt")14RI_F = os.path.join(DATA, "sources", RI_ID + ".txt")15Z = 1.96 # normal critical value for two-sided 95%161718def read_lines(p):19 with open(p, encoding="utf-8") as fh:20 return fh.read().split("\n")212223def wc_l(p):24 with open(p, "rb") as fh:25 return fh.read().count(b"\n")262728def num(s):29 """Parse a printed number: strip $, commas, stars, %, units; keep sign."""30 s = s.strip().replace("$", "").replace(",", "").replace("*", "")31 m = re.search(r"-?\d+(?:\.\d+)?", s)32 if not m:33 raise ValueError(f"no number in {s!r}")34 t = m.group(0)35 return float(t) if "." in t else int(t) # keep integers printed without decimals as integers363738def stars(s):39 return len(re.search(r"\**$", s.strip()).group(0))404142def cells(line):43 return [c.strip() for c in line.split("|")]444546# ---------------- DCR tables ----------------47def dcr_table(lines, title_prefix):48 """Return (header, rows) for a DCR table; rows: label -> {col: (cell, se_cell, lineno, se_lineno)}."""49 start = next(i for i, l in enumerate(lines) if l.startswith(title_prefix))50 body = []51 for i in range(start + 1, len(lines)):52 l = lines[i]53 if l.startswith("Note:"):54 break55 if l.strip():56 body.append((i + 1, l))57 header = cells(body[0][1])58 cols = header[1:]59 rows, last = {}, None60 for ln, l in body[1:]:61 c = cells(l)62 if c[0] == "" and last is not None and l.strip().startswith("|") and "(" in l:63 for j, col in enumerate(cols):64 v = c[j + 1] if j + 1 < len(c) else ""65 if v:66 cell, _, cln, _ = rows[last][col]67 rows[last][col] = (cell, v, cln, ln)68 continue69 last = c[0]70 rows[last] = {col: ((c[j + 1] if j + 1 < len(c) else ""), None, ln, None) for j, col in enumerate(cols)}71 return cols, rows727374dcr = read_lines(DCR_F)75t6_cols, t6 = dcr_table(dcr, "Table 6:")76t7_cols, t7 = dcr_table(dcr, "Table 7:")77ta19_cols, ta19 = dcr_table(dcr, "Table A19:")7879matched = []808182def grab(tab, tname, row, col, src=DCR_ID, lines=dcr):83 cell, se, ln, sln = tab[row][col]84 matched.append(f"{src} L{ln}: {lines[ln-1]}")85 if sln:86 matched.append(f"{src} L{sln}: {lines[sln-1]}")87 return cell, se888990def loc(src, table, row, col, extra=""):91 return f'{src} | Table {table} | row "{row}" | column "{col}"' + (f" | {extra}" if extra else "")929394res, locs = {}, {}95c, se = grab(t6, "6", "ATT", "5 km")96res["dcr_att_5km"], res["dcr_att_5km_se"] = num(c), num(se)97locs["dcr_att_5km"] = loc(DCR_ID, 6, "ATT", "5 km")98locs["dcr_att_5km_se"] = loc(DCR_ID, 6, "ATT", "5 km", "standard error in parentheses on the line below")99c, se = grab(t6, "6", "ATT", "3 km")100res["dcr_att_3km"], res["dcr_att_3km_se"] = num(c), num(se)101locs["dcr_att_3km"] = loc(DCR_ID, 6, "ATT", "3 km")102locs["dcr_att_3km_se"] = loc(DCR_ID, 6, "ATT", "3 km", "standard error in parentheses on the line below")103c, _ = grab(t6, "6", "Observations", "5 km")104res["dcr_obs_5km"] = int(num(c))105locs["dcr_obs_5km"] = loc(DCR_ID, 6, "Observations", "5 km")106for key, row, col in [("dcr_narcotics_att_3km", "Narcotics", "3 km"),107 ("dcr_fatal_att_1km", "Drug-related (fatal)", "1 km"),108 ("dcr_hiv_att_5km", "HIV", "5 km")]:109 c, se = grab(t7, "7", row, col)110 res[key], res[key + "_se"] = num(c), num(se)111 locs[key] = loc(DCR_ID, 7, row, col)112 locs[key + "_se"] = loc(DCR_ID, 7, row, col, "standard error in parentheses on the line below")113 if key == "dcr_hiv_att_5km":114 hiv_cell = c115116# exact p-value: Table 7 prints only significance stars, no p-values117t7_start = next(i for i, l in enumerate(dcr) if l.startswith("Table 7:"))118t7_end = next(i for i in range(t7_start, len(dcr)) if dcr[i].startswith("Figure 7 and Table 7"))119pval_printed = any(re.search(r"\bp\s*=\s*0?\.\d", l) for l in dcr[t7_start:t7_end])120res["dcr_hiv_att_5km_p_value"] = None121locs["dcr_hiv_att_5km_p_value"] = (f"not reported in {DCR_ID}: Table 7, row \"HIV\", column \"5 km\" prints only the "122 "coefficient, SE and significance stars (none); no exact p-value is printed")123assert not pval_printed124res["dcr_hiv_att_5km_significant_10pct"] = stars(hiv_cell) >= 1125126# derived 95% CI from printed Table 6 SE (normal, z = 1.96)127b, s = res["dcr_att_5km"], res["dcr_att_5km_se"]128res["dcr_att_5km_ci_low"] = round(b - Z * s, 4)129res["dcr_att_5km_ci_high"] = round(b + Z * s, 4)130locs["dcr_att_5km_ci_low"] = loc(DCR_ID, 6, "ATT", "5 km", "derived: estimate - 1.96 x printed SE")131locs["dcr_att_5km_ci_high"] = loc(DCR_ID, 6, "ATT", "5 km", "derived: estimate + 1.96 x printed SE")132133# ---------------- RESEA Table 2 ----------------134ri = read_lines(RI_F)135136137def ri_table(lines, marker):138 start = next(i for i, l in enumerate(lines) if l.startswith(marker))139 hdr_i = next(i for i in range(start + 1, len(lines)) if lines[i].startswith("Outcome |"))140 cols = cells(lines[hdr_i])[1:]141 rows = {}142 for i in range(hdr_i + 1, len(lines)):143 if not lines[i].strip():144 break145 c = cells(lines[i])146 rows[c[0]] = {col: (c[j + 1], i + 1) for j, col in enumerate(cols)}147 return cols, rows148149150def parse_effect(cell):151 parts = [p.strip() for p in cell.split(";")]152 est = num(parts[0])153 p = parts[1]154 ci = re.search(r"\[(.+?),\s*(-?[\d,\.]+)\]", cell)155 lo, hi = num(ci.group(1)), num(ci.group(2))156 return est, p, lo, hi157158159t2_cols, t2 = ri_table(ri, "[Table 2")160ATE = "Average treatment effect estimate"161c, ln = t2["Wage"][ATE]162matched.append(f"{RI_ID} L{ln}: {ri[ln-1]}")163w_est, w_p, w_lo, w_hi = parse_effect(c)164res["ri_wage_ate"], res["ri_wage_ate_ci_low"], res["ri_wage_ate_ci_high"] = w_est, w_lo, w_hi165locs["ri_wage_ate"] = loc(RI_ID, 2, "Wage", ATE, "point estimate")166locs["ri_wage_ate_ci_low"] = loc(RI_ID, 2, "Wage", ATE, "95% CI lower bound")167locs["ri_wage_ate_ci_high"] = loc(RI_ID, 2, "Wage", ATE, "95% CI upper bound")168res["ri_wage_ate_se_reported"] = bool(re.search(r"\bSE\b|standard error", c, re.I)) or "(" in c169res["ri_wage_ate_se_derived"] = round((w_hi - w_lo) / (2 * Z), 2)170locs["ri_wage_ate_se_derived"] = loc(RI_ID, 2, "Wage", ATE, "derived: (upper - lower) / (2 x 1.96)")171c, ln = t2["Wage"]["Control"]172res["ri_control_mean_wage"] = num(c.split("(")[0])173locs["ri_control_mean_wage"] = loc(RI_ID, 2, "Wage", "Control", "mean (SD in parentheses not used)")174c, ln = t2["Reemployment"][ATE]175matched.append(f"{RI_ID} L{ln}: {ri[ln-1]}")176res["ri_reemployment_ate_pp"] = parse_effect(c)[0]177locs["ri_reemployment_ate_pp"] = loc(RI_ID, 2, "Reemployment", ATE, "point estimate")178c, ln = t2["Weeks on UI"][ATE]179matched.append(f"{RI_ID} L{ln}: {ri[ln-1]}")180u_est, u_p, u_lo, u_hi = parse_effect(c)181res["ri_weeks_ui_ate"], res["ri_weeks_ui_ate_ci_low"], res["ri_weeks_ui_ate_ci_high"] = u_est, u_lo, u_hi182locs["ri_weeks_ui_ate"] = loc(RI_ID, 2, "Weeks on UI", ATE, "point estimate")183locs["ri_weeks_ui_ate_ci_low"] = loc(RI_ID, 2, "Weeks on UI", ATE, "95% CI lower bound")184locs["ri_weeks_ui_ate_ci_high"] = loc(RI_ID, 2, "Weeks on UI", ATE, "95% CI upper bound")185186# ---------------- consistency / sensitivity (not required keys) ----------------187nd = NormalDist()188def p_from(b, s):189 return 2 * (1 - nd.cdf(abs(b / s)))190def star_from_p(p):191 return 3 if p < 0.01 else 2 if p < 0.05 else 1 if p < 0.10 else 0192193star_checks = []194for tname, tab in [("6", t6), ("7", t7), ("A19", ta19)]:195 for row, d in tab.items():196 for col, (cell, se, _, _) in d.items():197 if se and cell:198 bb, ss = num(cell), num(se)199 pz = p_from(bb, ss)200 star_checks.append({"table": tname, "row": row, "col": col, "printed_stars": stars(cell),201 "z_implied_stars": star_from_p(pz), "z_p": round(pz, 4)})202star_mismatch = [x for x in star_checks if x["printed_stars"] != x["z_implied_stars"]]203204a19_cell, a19_se, _, _ = ta19["ATT"]["CS-DR"]205w_mid = (w_lo + w_hi) / 2206sensitivity = {207 "dcr_att_5km_ci_z2.0": [round(b - 2.0 * s, 4), round(b + 2.0 * s, 4)],208 "dcr_att_5km_ci_using_tableA19_se": [round(b - Z * num(a19_se), 4), round(b + Z * num(a19_se), 4)],209 "dcr_hiv_att_5km_normal_p_approx": round(p_from(res["dcr_hiv_att_5km"], res["dcr_hiv_att_5km_se"]), 4),210 "ri_wage_ate_se_z2.0": round((w_hi - w_lo) / 4.0, 2),211 "ri_wage_ate_se_from_lower_half": round((w_est - w_lo) / Z, 2),212 "ri_wage_ate_se_from_upper_half": round((w_hi - w_est) / Z, 2),213 "ri_wage_ci_midpoint": w_mid,214 "ri_wage_printed_p": w_p,215 "ri_wage_normal_p_from_derived_se": round(p_from(w_est, (w_hi - w_lo) / (2 * Z)), 4),216 "significance_star_checks_n": len(star_checks),217 "significance_star_mismatches": star_mismatch,218}219220data_issues = [221 {"issue":222 f"SE of the 5 km CS-DR ATT (-0.431) is printed as (0.210) in Table 6 but {a19_se} in Table A19 (CS-DR column, 5 km buffer, same +/-5-year window)",223 "handling": "Used Table 6 as required by the key definitions; the derived CI with the A19 SE is shown only as a sensitivity value."},224 {"issue": "Tables 6/7 state 608 never-treated control regions, the Table A19 note states 640 never-treated comparison regions",225 "handling": "Not used by any key; reported only."},226 {"issue": "DCR abstract says fatal hospitalisations 'fall by 35%', Section 6.1 says 'approximately 30% at 5 km'; exp(-0.354)-1 = -29.8%",227 "handling": "Not used by any key; text claims are not extracted; reported only."},228 {"issue": "DCR text calls HIV estimates 'close to zero' although the 5 km HIV ATT is -0.340 (SE 0.294, no star)",229 "handling": "Extracted the printed table value; significance taken from the (absent) star."},230 {"issue": "DCR Table 7 prints only significance stars, no exact p-values (and no Observations row)",231 "handling": "dcr_hiv_att_5km_p_value set to null with a 'not reported' locator; a normal-approximation p is given only as sensitivity, not as the key."},232 {"issue": "RESEA text prints the weeks-on-UI CI as positive magnitudes '[1.75 weeks, 2.24 weeks]' while Table 2 prints [-2.24, -1.75]",233 "handling": "Used Table 2 as printed (lower bound first, negative sign)."},234 {"issue": "RESEA text contains an encoding defect 'p ¡ 0.001' (for 'p < 0.001'); Table 2 cells were re-set by the extract editor",235 "handling": "Irrelevant to the keys; values taken from re-set Table 2 cells, which the extract states are verbatim."},236 {"issue": "RESEA Table 2 prints no standard errors, only p-values and 95% CIs",237 "handling": "ri_wage_ate_se_reported = false; SE derived from the printed CI with z = 1.96."},238]239240results = {}241results.update(res)242results["locators"] = locs243results["derivation_notes"] = {244 "dcr_att_5km_ci": ("Normal (Wald) 95% CI: printed Table 6 ATT -0.431 +/- 1.96 x printed clustered SE 0.210; "245 "assumes asymptotic normality of the CS-DR ATT; rounded to 4 decimals. "246 "With 2.0 instead of 1.96: see sensitivity."),247 "ri_wage_ate_se": ("SE = (upper - lower) / (2 x 1.96) from the printed Table 2 95% CI [166, 2,141]; assumes the CI "248 "is a symmetric normal-approximation interval (N = 23,549, so t ~ z); rounded to 2 decimals."),249}250results["sensitivity"] = sensitivity251results["task_id"] = "F4-conf-01"252results["feasibility"] = "complete"253results["feasibility_reason"] = None254results["data_issues"] = data_issues255results["n_input_rows"] = {256 "README.md": wc_l(os.path.join(DATA, "README.md")),257 DCR_ID + ".txt": wc_l(DCR_F),258 RI_ID + ".txt": wc_l(RI_F),259}260results["input_sha256"] = {k: hashlib.sha256(open(p, "rb").read()).hexdigest() for k, p in261 [("README.md", os.path.join(DATA, "README.md")), (DCR_ID + ".txt", DCR_F), (RI_ID + ".txt", RI_F)]}262REQ = ["dcr_att_5km", "dcr_att_5km_se", "dcr_att_3km", "dcr_att_3km_se", "dcr_obs_5km", "dcr_narcotics_att_3km",263 "dcr_narcotics_att_3km_se", "dcr_fatal_att_1km", "dcr_fatal_att_1km_se", "dcr_hiv_att_5km", "dcr_hiv_att_5km_se",264 "dcr_hiv_att_5km_p_value", "dcr_hiv_att_5km_significant_10pct", "dcr_att_5km_ci_low", "dcr_att_5km_ci_high",265 "ri_wage_ate", "ri_wage_ate_ci_low", "ri_wage_ate_ci_high", "ri_wage_ate_se_reported", "ri_wage_ate_se_derived",266 "ri_control_mean_wage", "ri_reemployment_ate_pp", "ri_weeks_ui_ate", "ri_weeks_ui_ate_ci_low", "ri_weeks_ui_ate_ci_high"]267assert all(k in results for k in REQ)268results["n_analysed"] = len(REQ)269results["random_seed"] = None270271os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)272with open(os.path.join(ROOT, "out", "results.json"), "w") as fh:273 json.dump(results, fh, indent=2, ensure_ascii=False)274275print("Matched source lines:")276for m in matched:277 print(" " + m)278print("Extracted / derived values:")279for k in REQ:280 print(f" {k} = {results[k]}")281print(f"Star-consistency checks: {len(star_checks)} cells, mismatches: {len(star_mismatch)}")282