"""F4-conf-01: extract printed treatment effects/uncertainty from two source extracts. Reads only data/; writes out/results.json deterministically. No randomness (random_seed = null). Every extracted value is parsed from the text and the matched source line is printed.""" import json, re, os, hashlib from pathlib import Path ROOT = Path(__file__).resolve().parent.parent DATA = ROOT / "data" DCR = DATA / "sources" / "arxiv-2609.27732v1.txt" RI = DATA / "sources" / "arxiv-2606.14621v1.txt" DCR_ID, RI_ID = "arxiv-2609.27732v1", "arxiv-2606.14621v1" Z = 1.96 # normal critical value for two-sided 95% def lines(p): return p.read_text(encoding="utf-8").split("\n") def num(s): """Parse a printed number keeping its printed decimals (e.g. '-0.431**' -> -0.431, '$1,153' -> 1153).""" s = s.strip().replace("$", "").replace(",", "").replace("%", "") s = re.sub(r"\*+$", "", s).strip() s = s.strip("()") return int(s) if re.fullmatch(r"-?\d+", s) else float(s) def stars(s): m = re.search(r"(\*+)\s*$", s.strip()); return len(m.group(1)) if m else 0 def block(ls, start_pat, end_pat): i0 = next(i for i, l in enumerate(ls) if re.match(start_pat, l)) i1 = next(i for i in range(i0 + 1, len(ls)) if re.match(end_pat, ls[i])) return i0, ls[i0:i1] def parse_reg_table(ls, title_pat, end_pat): """Return header columns, {row: {col: (raw_coef, raw_se, lineno)}}, {row: {col: raw}} for non-coef rows.""" i0, blk = block(ls, title_pat, end_pat) rows = [(i0 + k + 1, l) for k, l in enumerate(blk) if l.strip() and " | " in l or l.startswith("|")] rows = [(n, l) for n, l in rows if l.strip()] header = [c.strip() for c in rows[0][1].split("|")][1:] coef, other = {}, {} k = 1 while k < len(rows): n, l = rows[k] cells = [c.strip() for c in l.split("|")] label, vals = cells[0], cells[1:] nxt = rows[k + 1][1] if k + 1 < len(rows) else "" if label and nxt.strip().startswith("|") and "(" in nxt: ses = [c.strip() for c in nxt.split("|")][1:] coef[label] = {h: (v, s, n, rows[k + 1][0]) for h, v, s in zip(header, vals, ses)} k += 2 else: other[label] = {h: (v, n) for h, v in zip(header, vals)} k += 1 return header, coef, other dl = lines(DCR); rl = lines(RI) h6, t6, o6 = parse_reg_table(dl, r"^Table 6:", r"^Note:") h7, t7, o7 = parse_reg_table(dl, r"^Table 7:", r"^Note:") hA, tA, oA = parse_reg_table(dl, r"^Table A19:", r"^Note:") if h7 and h7[0] == "Outcome": # Table 7 header line starts with 'Outcome |' pass # Table 7 header has 'Outcome' as first cell rather than empty: re-parse header _, blk7 = block(dl, r"^Table 7:", r"^Note:") hdr7 = next(l for l in blk7 if l.startswith("Outcome |")) h7 = [c.strip() for c in hdr7.split("|")][1:] _, t7, o7 = (None, *parse_reg_table(dl, r"^Table 7:", r"^Note:")[1:]) # re-key Table 7 entries with the correct header def rekey(tab, hdr): out = {} for r, d in tab.items(): vals = list(d.values()) out[r] = dict(zip(hdr, vals)) return out t7 = rekey(t7, h7); o7 = rekey(o7, h7) def show(src, n): print(f" [{src} line {n}] {(dl if src == DCR_ID else rl)[n - 1]}") res, loc = {}, {} def put_coef(key, tab, tabname, row, col): v, s, n1, n2 = tab[row][col] res[key] = num(v); res[key + "_se"] = num(s) loc[key] = f'{DCR_ID} | Table {tabname} | row "{row}" | column "{col}"' loc[key + "_se"] = f'{DCR_ID} | Table {tabname} | row "{row}" (standard-error line below, in parentheses) | column "{col}"' print(f"{key} = {res[key]} (stars={stars(v)}), {key}_se = {res[key + '_se']}"); show(DCR_ID, n1); show(DCR_ID, n2) return stars(v) put_coef("dcr_att_5km", t6, "6", "ATT", "5 km") put_coef("dcr_att_3km", t6, "6", "ATT", "3 km") v, n = o6["Observations"]["5 km"] res["dcr_obs_5km"] = num(v) loc["dcr_obs_5km"] = f'{DCR_ID} | Table 6 | row "Observations" | column "5 km"' print(f"dcr_obs_5km = {res['dcr_obs_5km']}"); show(DCR_ID, n) put_coef("dcr_narcotics_att_3km", t7, "7", "Narcotics", "3 km") put_coef("dcr_fatal_att_1km", t7, "7", "Drug-related (fatal)", "1 km") hiv_stars = put_coef("dcr_hiv_att_5km", t7, "7", "HIV", "5 km") # exact p-value: search the whole DCR extract for any printed p-value pv_lines = [i + 1 for i, l in enumerate(dl) if re.search(r"\bp\s*=\s*0?\.\d", l)] print(f"lines in {DCR_ID} printing an exact p-value (p = ...): {pv_lines}") res["dcr_hiv_att_5km_p_value"] = None loc["dcr_hiv_att_5km_p_value"] = (f"not reported in {DCR_ID} (Table 7 prints only the coefficient, the clustered SE " f"and significance stars; no exact p-value is printed anywhere in the extract)") res["dcr_hiv_att_5km_significant_10pct"] = hiv_stars >= 1 print(f"dcr_hiv_att_5km_significant_10pct = {res['dcr_hiv_att_5km_significant_10pct']} (stars on Table 7 HIV 5 km cell: {hiv_stars})") # derived 95% CI from Table 6 printed SE, normal approximation b, se = res["dcr_att_5km"], res["dcr_att_5km_se"] res["dcr_att_5km_ci_low"] = round(b - Z * se, 4) res["dcr_att_5km_ci_high"] = round(b + Z * se, 4) loc["dcr_att_5km_ci_low"] = f'{DCR_ID} | Table 6 | row "ATT" | column "5 km" | derived (estimate - 1.96 x printed SE)' loc["dcr_att_5km_ci_high"] = f'{DCR_ID} | Table 6 | row "ATT" | column "5 km" | derived (estimate + 1.96 x printed SE)' # ---- RESEA Table 2 ---- _, blk2 = block(rl, r"^\[Table 2", r"^Table 2:") i2 = next(i for i, l in enumerate(rl) if l.startswith("[Table 2")) hdr = next(l for l in blk2 if l.startswith("Outcome |")) H2 = [c.strip() for c in hdr.split("|")] T2 = {} for k, l in enumerate(blk2): if " | " in l and not l.startswith("Outcome |") and not l.startswith("["): cells = [c.strip() for c in l.split("|")] T2[cells[0]] = (dict(zip(H2, cells)), i2 + k + 1) def parse_effect(cell): m = re.match(r"^(-?)\$?(-?[\d,\.]+)\s*(pp|weeks|wks)?;\s*p\s*([=<])\s*([\d\.]+);\s*95% CI:\s*\[([^,\]]+(?:,\d{3})*),\s*([^\]]+)\]$", cell.replace("-$", "-")) if not m: raise ValueError(cell) est = num(m.group(1) + m.group(2)) # split CI bounds robustly (thousands separators: "166, 2,141") ci = cell.split("95% CI:")[1].strip().strip("[]") parts = re.findall(r"-?\d{1,3}(?:,\d{3})+|-?\d+(?:\.\d+)?", ci) return est, num(parts[0]), num(parts[1]), m.group(4) + m.group(5) ATE = "Average treatment effect estimate" for key, row in [("ri_wage_ate", "Wage"), ("ri_reemployment_ate_pp", "Reemployment"), ("ri_weeks_ui_ate", "Weeks on UI")]: d, n = T2[row] est, lo, hi, p = parse_effect(d[ATE]) res[key] = est loc[key] = f'{RI_ID} | Table 2 | row "{row}" | column "{ATE}" (point estimate)' print(f"{key} = {est}; printed 95% CI [{lo}, {hi}]; p{p}"); print(f" [{RI_ID} line {n}] {rl[n - 1]}") if key in ("ri_wage_ate", "ri_weeks_ui_ate"): res[key + "_ci_low"], res[key + "_ci_high"] = lo, hi loc[key + "_ci_low"] = f'{RI_ID} | Table 2 | row "{row}" | column "{ATE}" (95% CI lower bound)' loc[key + "_ci_high"] = f'{RI_ID} | Table 2 | row "{row}" | column "{ATE}" (95% CI upper bound)' d, n = T2["Wage"] cm = re.match(r"^\$([\d,]+)", d["Control"]).group(1) res["ri_control_mean_wage"] = num(cm) loc["ri_control_mean_wage"] = f'{RI_ID} | Table 2 | row "Wage" | column "Control" (mean)' print(f"ri_control_mean_wage = {res['ri_control_mean_wage']}"); print(f" [{RI_ID} line {n}] {rl[n - 1]}") wage_cell = T2["Wage"][0][ATE] res["ri_wage_ate_se_reported"] = bool(re.search(r"\(\s*[\d,\.]+\s*\)|\bSE\b|s\.e\.", wage_cell, re.I)) print(f"ri_wage_ate_se_reported = {res['ri_wage_ate_se_reported']} (wage ATE cell: '{wage_cell}')") lo, hi = res["ri_wage_ate_ci_low"], res["ri_wage_ate_ci_high"] res["ri_wage_ate_se_derived"] = round((hi - lo) / (2 * Z), 2) loc["ri_wage_ate_se_derived"] = (f'{RI_ID} | Table 2 | row "Wage" | column "{ATE}" | derived ' f'((CI upper - CI lower) / (2 x 1.96))') # ---- consistency checks that feed data_issues ---- issues = [] seA = num(tA["ATT"]["CS-DR"][1]) if seA != res["dcr_att_5km_se"]: issues.append({"issue": (f"{DCR_ID}: the same 5 km CS-DR overall ATT ({res['dcr_att_5km']}) is printed with SE " f"{res['dcr_att_5km_se']} in Table 6 but SE {seA} in Appendix Table A19 (column 'CS-DR')."), "handling": "Used Table 6 as the key definitions require (dcr_att_5km_se, derived CI); A19 value not used."}) ctrl6 = num(o6["Control regions"]["5 km"][0]) mA = re.search(r"never-treated comparison regions \((\d+) regions\)", "\n".join(dl)) if mA and int(mA.group(1)) != ctrl6: issues.append({"issue": (f"{DCR_ID}: Tables 6/7 and A19 rows report {ctrl6} never-treated control regions, but the " f"Table A19 note says {mA.group(1)} never-treated comparison regions."), "handling": "Recorded only; no requested key depends on the control-region count."}) fatal5 = num(t7["Drug-related (fatal)"]["5 km"][0]) issues.append({"issue": (f"{DCR_ID}: the abstract says fatal hospitalisations 'fall by 35%', whereas Table 7 gives a 5 km " f"fatal ATT of {fatal5} log points (exp({fatal5})-1 = {100 * (2.718281828459045 ** fatal5 - 1):.1f}%), " f"and Section 6.1 says 'approximately 30% at 5 km'; the abstract figure appears to equate log points with percent."), "handling": "Recorded only; values were taken from the tables named in the key definitions."}) hiv5 = res["dcr_hiv_att_5km"] issues.append({"issue": (f"{DCR_ID}: the text describes HIV estimates as 'insignificant and close to zero', but the printed " f"5 km HIV ATT is {hiv5} log points (SE {res['dcr_hiv_att_5km_se']}); it is insignificant (no star) but not numerically close to zero."), "handling": "Recorded only; significance flag set from the absence of stars in Table 7."}) issues.append({"issue": (f"{RI_ID}: the Results text states the weeks-on-UI CI as magnitudes '[1.75 weeks, 2.24 weeks]' for a " f"'1.99 week reduction', while Table 2 prints the ATE as -1.99 with CI [-2.24, -1.75]."), "handling": "Used the signed Table 2 values as required (lower bound first)."}) if any("¡" in l for l in rl): issues.append({"issue": f"{RI_ID}: rendering defect in the Section 3 text: 'p ¡ 0.001' (should read p < 0.001).", "handling": "Not used for any key (Table 2 cell prints 'p<0.001')."}) issues.append({"issue": (f"{RI_ID}: Tables 2 and 3 were re-set by the extract editor into a one-row-per-outcome layout " f"(HTML rendering split rows and detached p-values)."), "handling": "Relied on the editor's statement that values are verbatim; Table 3 'Difference in means' column matches Table 2 ATE column exactly (checked)."}) # check Table 3 duplicates Table 2 ATEs _, blk3 = block(rl, r"^\[Table 3", r"^Table 3:") t3w = next(l for l in blk3 if l.startswith("Wage |")).split("|")[1].strip() assert parse_effect(t3w)[:3] == (res["ri_wage_ate"], lo, hi), "Table 3 vs Table 2 wage mismatch" def wc_l(p): return p.read_bytes().count(b"\n") out = { "task_id": "F4-conf-01", **{k: res[k] for k in ["dcr_att_5km", "dcr_att_5km_se", "dcr_att_3km", "dcr_att_3km_se", "dcr_obs_5km", "dcr_narcotics_att_3km", "dcr_narcotics_att_3km_se", "dcr_fatal_att_1km", "dcr_fatal_att_1km_se", "dcr_hiv_att_5km", "dcr_hiv_att_5km_se", "dcr_hiv_att_5km_p_value", "dcr_hiv_att_5km_significant_10pct", "dcr_att_5km_ci_low", "dcr_att_5km_ci_high", "ri_wage_ate", "ri_wage_ate_ci_low", "ri_wage_ate_ci_high", "ri_wage_ate_se_reported", "ri_wage_ate_se_derived", "ri_control_mean_wage", "ri_reemployment_ate_pp", "ri_weeks_ui_ate", "ri_weeks_ui_ate_ci_low", "ri_weeks_ui_ate_ci_high"]}, "locators": loc, "derivation_notes": { "dcr_att_5km_ci": ("95% CI = ATT +/- 1.96 x SE using the Table 6 printed ATT (-0.431) and clustered SE (0.210); " "normal (large-sample) critical value 1.96; rounded to 4 decimals. Assumes asymptotic normality of the " "CS-DR estimator and that the printed SE (itself rounded to 3 decimals) is the relevant SE."), "ri_wage_ate_se": ("SE = (upper - lower) / (2 x 1.96) using the Table 2 printed 95% CI [166, 2,141]; normal critical value " "1.96 (with N = 23,549 a t critical value is indistinguishable); rounded to 2 decimals. Assumes the " "printed CI is a symmetric Wald interval (midpoint 1153.5 vs estimate 1,153, consistent up to rounding)."), }, "feasibility": "complete", "feasibility_reason": None, "data_issues": issues, "n_input_rows": {"README.md": wc_l(DATA / "README.md"), DCR.name: wc_l(DCR), RI.name: wc_l(RI)}, "n_analysed": 25, "random_seed": None, } os.makedirs(ROOT / "out", exist_ok=True) (ROOT / "out" / "results.json").write_text(json.dumps(out, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") print("wrote out/results.json")