1"""F4-conf-01: extract printed estimates from two arXiv extracts and derive two quantities.2Reads only data/; writes out/results.json deterministically. No randomness."""3import json, os, re45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6DCR_ID, RI_ID = "arxiv-2609.27732v1", "arxiv-2606.14621v1"7DCR_F = os.path.join(ROOT, "data", "sources", DCR_ID + ".txt")8RI_F = os.path.join(ROOT, "data", "sources", RI_ID + ".txt")9README_F = os.path.join(ROOT, "data", "README.md")10Z = 1.96 # normal critical value for two-sided 95%1112def read(p):13 with open(p, encoding="utf-8") as f:14 return f.read()1516def wc_l(p):17 with open(p, "rb") as f:18 return f.read().count(b"\n")1920def num(s):21 """Parse a printed number string (keep sign; strip $, commas, stars, %, units)."""22 s = s.strip().replace("$", "").replace(",", "")23 m = re.search(r"-?\d+(?:\.\d+)?", s)24 return m.group(0)2526def to_val(s):27 return int(s) if re.fullmatch(r"-?\d+", s) else float(s)2829def table_block(text, title_prefix):30 lines = text.splitlines()31 start = next(i for i, l in enumerate(lines) if l.startswith(title_prefix))32 block = []33 for l in lines[start + 1:]:34 if l.startswith("Note:"):35 break36 if l.strip():37 block.append(l)38 return block3940def parse_reg_table(block):41 """Return header cols, {row_label: {col: (coef_str, se_str or None, raw lines)}}."""42 header = [c.strip() for c in block[0].split("|")]43 cols = [c for c in header if c and c != "Outcome"]44 rows, last = {}, None45 for l in block[1:]:46 cells = [c.strip() for c in l.split("|")]47 if cells[0] == "" and last is not None and all(c.startswith("(") for c in cells[1:] if c):48 for c, v in zip(cols, cells[1:]):49 coef, _, raw = rows[last][c]50 rows[last][c] = (coef, v, raw + [l])51 continue52 label = cells[0]53 rows[label] = {c: (v, None, [l]) for c, v in zip(cols, cells[1:])}54 last = label55 return cols, rows5657dcr = read(DCR_F); ri = read(RI_F)58t6_cols, t6 = parse_reg_table(table_block(dcr, "Table 6:"))59t7_cols, t7 = parse_reg_table(table_block(dcr, "Table 7:"))6061R, LOC, PRINTED, MATCHED = {}, {}, {}, {}6263def loc(src, tab, row, col, extra=""):64 return f'{src} | Table {tab} | row "{row}" | column "{col}"' + extra6566def put_dcr(key, tab, rows, row, col, what):67 coef, se, raw = rows[row][col]68 s = coef if what == "coef" else se69 p = num(s)70 R[key] = to_val(p); PRINTED[key] = s71 LOC[key] = loc(DCR_ID, tab, row, col, "" if what == "coef" else " (standard error in parentheses on the line below)")72 MATCHED[key] = raw73 return coef7475put_dcr("dcr_att_5km", 6, t6, "ATT", "5 km", "coef")76put_dcr("dcr_att_5km_se", 6, t6, "ATT", "5 km", "se")77put_dcr("dcr_att_3km", 6, t6, "ATT", "3 km", "coef")78put_dcr("dcr_att_3km_se", 6, t6, "ATT", "3 km", "se")79put_dcr("dcr_obs_5km", 6, t6, "Observations", "5 km", "coef")80put_dcr("dcr_narcotics_att_3km", 7, t7, "Narcotics", "3 km", "coef")81put_dcr("dcr_narcotics_att_3km_se", 7, t7, "Narcotics", "3 km", "se")82put_dcr("dcr_fatal_att_1km", 7, t7, "Drug-related (fatal)", "1 km", "coef")83put_dcr("dcr_fatal_att_1km_se", 7, t7, "Drug-related (fatal)", "1 km", "se")84hiv_coef = put_dcr("dcr_hiv_att_5km", 7, t7, "HIV", "5 km", "coef")85put_dcr("dcr_hiv_att_5km_se", 7, t7, "HIV", "5 km", "se")8687# HIV exact p-value: search the extract for any printed p-value (p= / p-value) -- none expected88pv_hits = [l for l in dcr.splitlines() if re.search(r"p\s*=\s*0?\.\d|p-value", l, re.I)]89R["dcr_hiv_att_5km_p_value"] = None90LOC["dcr_hiv_att_5km_p_value"] = (f"not reported in {DCR_ID} (Table 7 prints only the coefficient, SE and "91 f"significance stars; {len(pv_hits)} exact p-value strings found in the extract)")92stars = len(hiv_coef) - len(hiv_coef.rstrip("*"))93R["dcr_hiv_att_5km_significant_10pct"] = stars >= 1 # * p<0.10 or stronger9495# Derived 95% CI from Table 6 printed values96b, se = R["dcr_att_5km"], R["dcr_att_5km_se"]97R["dcr_att_5km_ci_low"] = round(b - Z * se, 6)98R["dcr_att_5km_ci_high"] = round(b + Z * se, 6)99for k in ("dcr_att_5km_ci_low", "dcr_att_5km_ci_high"):100 LOC[k] = (f'{DCR_ID} | Table 6 | row "ATT" | column "5 km" | derived from printed ATT and SE')101102# RESEA Table 2103lines = ri.splitlines()104hdr_i = next(i for i, l in enumerate(lines) if l.startswith("Outcome | Treatment | Control"))105t2_cols = [c.strip() for c in lines[hdr_i].split("|")]106t2 = {}107for l in lines[hdr_i + 1:]:108 if not l.strip():109 break110 cells = [c.strip() for c in l.split("|")]111 t2[cells[0]] = dict(zip(t2_cols[1:], cells[1:])) | {"_raw": l}112ATE = "Average treatment effect estimate"113114def parse_effect(cell):115 est = cell.split(";")[0]116 m = re.search(r"95% CI:\s*\[([^\]]+)\]", cell)117 lo, hi = [x.strip() for x in m.group(1).split(", ")]118 return est, lo, hi119120def put_ri(key, row, col, s, part):121 R[key] = to_val(num(s)); PRINTED[key] = s122 LOC[key] = loc(RI_ID, 2, row, col, part); MATCHED[key] = [t2[row]["_raw"]]123124w_est, w_lo, w_hi = parse_effect(t2["Wage"][ATE])125put_ri("ri_wage_ate", "Wage", ATE, w_est, " (point estimate)")126put_ri("ri_wage_ate_ci_low", "Wage", ATE, w_lo, " (95% CI lower bound)")127put_ri("ri_wage_ate_ci_high", "Wage", ATE, w_hi, " (95% CI upper bound)")128# Search the wage row and the paper body (after the editor's header) for a printed standard error.129ri_body = ri.split("=" * 20, 1)[1]130wage_row = t2["Wage"]["_raw"]131R["ri_wage_ate_se_reported"] = bool(re.search(r"\bSE\b|s\.e\.|standard error", wage_row, re.I)132 or re.search(r"standard error|\bSE\b", ri_body, re.I))133R["ri_wage_ate_se_derived"] = round((R["ri_wage_ate_ci_high"] - R["ri_wage_ate_ci_low"]) / (2 * Z), 6)134LOC["ri_wage_ate_se_derived"] = (f'{RI_ID} | Table 2 | row "Wage" | column "{ATE}" | derived from printed 95% CI bounds')135ctrl = t2["Wage"]["Control"].split("(")[0].strip()136put_ri("ri_control_mean_wage", "Wage", "Control", ctrl, " (mean)")137r_est, _, _ = parse_effect(t2["Reemployment"][ATE])138put_ri("ri_reemployment_ate_pp", "Reemployment", ATE, r_est, " (point estimate)")139u_est, u_lo, u_hi = parse_effect(t2["Weeks on UI"][ATE])140put_ri("ri_weeks_ui_ate", "Weeks on UI", ATE, u_est, " (point estimate)")141put_ri("ri_weeks_ui_ate_ci_low", "Weeks on UI", ATE, u_lo, " (95% CI lower bound)")142put_ri("ri_weeks_ui_ate_ci_high", "Weeks on UI", ATE, u_hi, " (95% CI upper bound)")143144# Cross-table check values used only for data_issues145a19_cols, a19 = parse_reg_table(table_block(dcr, "Table A19:"))146a19_se = a19["ATT"]["CS-DR"][1]147148derivation_notes = {149 "dcr_att_5km_ci": (f"Normal approximation, critical value 1.96: CI = ATT +/- 1.96*SE using Table 6 printed "150 f"ATT {PRINTED['dcr_att_5km']} and SE {PRINTED['dcr_att_5km_se']} (clustered at MedStat region level); "151 f"bounds rounded to 6 decimals. Not printed in the source."),152 "ri_wage_ate_se": (f"SE = (upper - lower)/(2*1.96) from Table 2 printed 95% CI [{w_lo}, {w_hi}], assuming a symmetric "153 f"normal-approximation interval (the CI method is 'described in the main text', which is not in the extract); "154 f"rounded to 6 decimals. The source prints no standard error."),155}156157data_issues = [158 {"issue": f"{DCR_ID}: the 5 km CS-DR ATT -0.431 has SE {PRINTED['dcr_att_5km_se']} in Table 6 but {a19_se} in Table A19 (same estimator, buffer and window).",159 "handling": "Keys name Table 6, so the Table 6 SE (0.210) is used for dcr_att_5km_se and the derived CI; discrepancy reported."},160 {"issue": f"{DCR_ID}: Table A19 note states 640 never-treated comparison regions whereas Tables 6/7 and the Figure 6 note state 608.",161 "handling": "No requested key depends on it; reported only."},162 {"issue": f"{DCR_ID}: abstract says fatal drug-related hospitalisations fall by 35%, Section 6.1 says approximately 30% at 5 km; Table 7 5 km fatal ATT is -0.354 (exp(-0.354)-1 = -29.8%).",163 "handling": "No requested key depends on the prose; table values used; reported only."},164 {"issue": f"{DCR_ID}: text calls HIV estimates 'insignificant and close to zero' while the Table 7 5 km HIV ATT is -0.340 (SE 0.294), similar in size to significant estimates.",165 "handling": "Significance flag taken from the table stars (none); exact p-value not printed -> null."},166 {"issue": f"{DCR_ID}: Figure 7 note and Table 7 note describe four sub-categories while Table 7 shows five outcome rows (fatal has no ICD code listed).",167 "handling": "Reported only."},168 {"issue": f"{RI_ID}: prose gives the weeks-on-UI CI as positive magnitudes [1.75 weeks, 2.24 weeks] while Table 2 prints [-2.24, -1.75].",169 "handling": "Keys use Table 2 as printed (negative, lower bound first)."},170 {"issue": f"{RI_ID}: rendering defect 'p ¡ 0.001' in the short-term results paragraph (intended 'p < 0.001').",171 "handling": "No requested key affected; reported only."},172 {"issue": f"{RI_ID}: Tables 2 and 3 were re-set by the extract editor (one row per outcome); Table 3 column 1 duplicates Table 2's ATE column.",173 "handling": "Values taken from Table 2 as the keys require; identical to Table 3 'Difference in means'."},174]175176value_keys = [k for k in R]177results = dict(R)178results["locators"] = {k: LOC[k] for k in R if k not in ("dcr_hiv_att_5km_significant_10pct", "ri_wage_ate_se_reported")}179results["printed_strings"] = PRINTED180results["derivation_notes"] = derivation_notes181results["task_id"] = "F4-conf-01"182results["feasibility"] = "complete"183results["feasibility_reason"] = None184results["data_issues"] = data_issues185results["n_input_rows"] = {"README.md": wc_l(README_F), DCR_ID + ".txt": wc_l(DCR_F), RI_ID + ".txt": wc_l(RI_F)}186results["n_analysed"] = len(value_keys)187results["random_seed"] = None188189os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)190with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:191 json.dump(results, f, indent=2, ensure_ascii=False)192193print("Matched source lines:")194for k, raw in MATCHED.items():195 print(f" {k} = {R[k]!r} <- {PRINTED[k]!r} from: {' // '.join(r.strip() for r in raw)}")196for k in ("dcr_hiv_att_5km_p_value", "dcr_hiv_att_5km_significant_10pct", "dcr_att_5km_ci_low",197 "dcr_att_5km_ci_high", "ri_wage_ate_se_reported", "ri_wage_ate_se_derived"):198 print(f" {k} = {R[k]!r}")199print(f"n_analysed = {len(value_keys)}; wrote out/results.json")200