1"""Replication script for F4-conf-02: verdicts on six assertions.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Parses the printed values from the source extracts, applies the task's conventions4(pct = 100*(exp(b)-1); 95% CI = b +/- 1.96*SE, robustness with 2.0) and writes out/results.json."""5import json, math, re, os67D = "data"8DCR = "arxiv-2609.27732v1"9RES = "arxiv-2606.14621v1"1011def lines(p):12 with open(p, encoding="utf-8") as f:13 return f.read().split("\n")1415assertions = json.load(open(os.path.join(D, "assertions.json"), encoding="utf-8"))16dcr = lines(os.path.join(D, "sources", DCR + ".txt"))17res = lines(os.path.join(D, "sources", RES + ".txt"))1819def wc(p):20 with open(p, "rb") as f:21 return f.read().count(b"\n")2223def find(ls, pat, start=0):24 for i in range(start, len(ls)):25 if re.search(pat, ls[i]):26 return i27 raise ValueError(pat)2829def show(tag, ls, i):30 print(f"[{tag} line {i+1}] {ls[i]}")3132def cells(s):33 return [c.strip() for c in s.split("|")]3435def num(s):36 return float(re.sub(r"[^\d\.\-]", "", s.replace("−", "-")))3738def sig(s):39 return s.count("*")4041print("=== Source lines relied on ===")42# Table 643t6 = find(dcr, r"^Table 6:")44i_att = find(dcr, r"^ATT \|", t6); i_se = i_att + 2; i_pct = find(dcr, r"^% change \|", t6)45for i in (i_att, i_se, i_pct): show(DCR, dcr, i)46att6 = cells(dcr[i_att])[1:]; se6 = cells(dcr[i_se])[1:]; pct6 = cells(dcr[i_pct])[1:]47cols = ["1 km", "3 km", "5 km"]48T6 = {c: dict(b=num(a), stars=sig(a), se=num(s), pct=p) for c, a, s, p in zip(cols, att6, se6, pct6)}49i_txt = find(dcr, r"5 km specification, yields an ATT"); show(DCR, dcr, i_txt); show(DCR, dcr, i_txt + 1)5051# Table 752t7 = find(dcr, r"^Table 7:")53T7 = {}54for row in ["Mental/Behavioral", "Hepatitis", "HIV", "Narcotics", r"Drug-related \(fatal\)"]:55 i = find(dcr, "^" + row + r" \|", t7)56 show(DCR, dcr, i); show(DCR, dcr, i + 2)57 c = cells(dcr[i]); s = cells(dcr[i + 2])[1:]58 T7[c[0]] = {col: dict(b=num(a), stars=sig(a), se=num(se)) for col, a, se in zip(cols, c[1:], s)}5960# Table A1961ta = find(dcr, r"^Table A19:")62ia = find(dcr, r"^ATT \|", ta); show(DCR, dcr, ia); show(DCR, dcr, ia + 2)63ina = find(dcr, r"never-treated comparison regions \(640", ta); show(DCR, dcr, ina)64a19_se_csdr = num(cells(dcr[ia + 2])[1])6566# Sex / gender heterogeneity search in DCR extract67sex_hits = [i for i, l in enumerate(dcr) if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]68print(f"[{DCR}] lines mentioning men/women/male/female/sex/gender: {len(sex_hits)}")6970# RESEA Table 271iw = find(res, r"^Weeks on UI \|"); show(RES, res, iw)72c = cells(res[iw])73ctrl_mean = num(c[2].split("weeks")[0])74m = re.search(r"(-?[\d\.]+) weeks; p<0\.001; 95% CI: \[(-?[\d\.]+), (-?[\d\.]+)\]", c[4])75ate, lo, hi = float(m.group(1)), float(m.group(2)), float(m.group(3))76it = find(res, r"11\.5% decrease"); show(RES, res, it)7778print("\n=== Checks ===")79def pct(b): return 100 * (math.exp(b) - 1)80def ci(b, se, z): return (b - z * se, b + z * se)8182checks = {}83# A184a = T6["5 km"]85checks["A1"] = dict(b=a["b"], se=a["se"], stars=a["stars"], pct_printed=a["pct"], pct_calc=round(pct(a["b"]), 2),86 z=round(abs(a["b"]) / a["se"], 3), z_A19_se=round(abs(a["b"]) / a19_se_csdr, 3))87ok1 = (a["b"] == -0.431 and a["se"] == 0.210 and a["stars"] == 2 and round(pct(a["b"]), 1) == -35.0)88# A289h = T7["HIV"]["1 km"]; checks["A2"] = dict(b=h["b"], se=h["se"])90ok2 = h["b"] < 091# A392b3 = T6["3 km"]; checks["A3"] = dict(b=b3["b"], pct_printed=b3["pct"], pct_calc=round(pct(b3["b"]), 2))93ok3 = abs(pct(b3["b"]) - (-28.3)) < 0.0594# A495hp = T7["Hepatitis"]["5 km"]96ci196 = ci(hp["b"], hp["se"], 1.96); ci200 = ci(hp["b"], hp["se"], 2.0); ci1 = ci(hp["b"], hp["se"], 1.0)97checks["A4"] = dict(b=hp["b"], se=hp["se"], stars=hp["stars"], ci_196=[round(x, 3) for x in ci196],98 ci_200=[round(x, 3) for x in ci200], ci_1se=[round(x, 3) for x in ci1])99ok4 = ci196[1] < 0 and abs(ci196[0] + 0.66) < 0.02 and abs(ci196[1] + 0.16) < 0.02100# A5101checks["A5"] = dict(sex_mentions=len(sex_hits))102# A6103checks["A6"] = dict(ate=ate, ci=[lo, hi], control_mean=ctrl_mean, pct_calc=round(100 * ate / ctrl_mean, 2))104ok6 = ate == -1.99 and (lo, hi) == (-2.24, -1.75) and ctrl_mean == 17.3 and round(100 * ate / ctrl_mean, 1) == -11.5105for k, v in checks.items(): print(k, v)106107V = lambda ok: "supported" if ok else "not_supported"108verdicts = {"A1": V(ok1), "A2": V(ok2), "A3": V(ok3), "A4": V(ok4),109 "A5": "not_assessable" if not sex_hits else None, "A6": V(ok6)}110111locators = {112 "A1": f'{DCR} | Table 6 | row "ATT" | column "5 km"',113 "A2": f'{DCR} | Table 7 | row "HIV" | column "1 km"',114 "A3": f'{DCR} | Table 6 | row "% change" | column "3 km"',115 "A4": f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"',116 "A5": f"not reported in {DCR}",117 "A6": f'{RES} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"',118}119reasons = {120 "A1": f"Table 6 prints ATT -0.431** (SE 0.210) at 5 km with p<0.05 and '% change' -35.0%, and 100*(exp(-0.431)-1) = {checks['A1']['pct_calc']}%, so sign, magnitude, SE, significance and percentage all match.",121 "A2": f"The 1 km HIV ATT in Table 7 is printed as +0.063 (SE 0.801), not -0.063, so the asserted negative sign and the claimed fall in HIV hospitalisations are contradicted.",122 "A3": f"A log-point ATT of -0.283 implies 100*(exp(-0.283)-1) = {checks['A3']['pct_calc']}%, matching the printed '% change' of -24.7%, not 28.3% (the assertion equates log points with percent).",123 "A4": f"With ATT -0.406 and SE 0.250 the 95% CI is [{ci196[0]:.3f}, {ci196[1]:.3f}] (±1.96·SE) and includes zero (estimate is unstarred); the asserted [-0.66, -0.16] is only ±1 SE.",124 "A5": "The extract reports no estimates by sex (no men/women/gender subgroup appears in the abstract, Section 6, Tables 6-7 or Table A19), so the comparison cannot be confirmed or refuted.",125 "A6": f"Table 2 prints the ATE on weeks on UI as -1.99 weeks (p<0.001; 95% CI [-2.24, -1.75]) with control mean 17.3 weeks, and 1.99/17.3 = {abs(checks['A6']['pct_calc'])}%, matching the 11.5% stated in the results text.",126}127128data_issues = [129 {"issue": "arxiv-2609.27732v1: the same CS-DR 5 km ATT (-0.431**) has SE 0.210 in Table 6 but SE 0.218 in Appendix Table A19 (column 'CS-DR').",130 "handling": "A1 judged against Table 6, which the assertion paraphrases; the verdict is unaffected because |t| = 2.052 (SE 0.210) or 1.977 (SE 0.218) both exceed 1.96 and both tables print ** (p<0.05)."},131 {"issue": "arxiv-2609.27732v1: Table A19 note says 640 never-treated comparison regions while its own body and Tables 6-7 report 608 control regions.",132 "handling": "Recorded only; no assertion depends on the control-region count."},133 {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes 'fall by 35%', but Table 7 5 km fatal ATT -0.354 implies -29.8%, which Section 6.1 reports as 'approximately 30%'.",134 "handling": "Recorded only; no assertion concerns fatal hospitalisations. Tables preferred over prose where needed."},135 {"issue": "arxiv-2609.27732v1: Table A19 '% change' row prints -30.3% (Synthetic DiD, -0.362) and -13.9% (Stacked DiD, -0.149) whereas 100*(exp(b)-1) gives -30.37% and -13.84%; differences of 0.1 pp, likely rounding from unrounded coefficients.",136 "handling": "Recorded only; not decisive for any assertion."},137 {"issue": "arxiv-2609.27732v1: the text describes HIV estimates as 'close to zero' although the 5 km HIV ATT is -0.340 (SE 0.294); they are insignificant but not small in magnitude.",138 "handling": "Recorded only; A2 is decided from the printed 1 km coefficient's sign."},139 {"issue": "arxiv-2606.14621v1: results text reports weeks-on-UI CIs as magnitudes ([1.75 weeks, 2.24 weeks], [1.55, 1.99]) while Table 2 prints signed intervals [-2.24, -1.75]; and 'p ¡ 0.001' is an HTML rendering defect for 'p < 0.001'.",140 "handling": "Treated as equivalent descriptions of a reduction; A6 judged against the signed Table 2 values."},141 {"issue": "arxiv-2606.14621v1: Tables 2 and 3 were re-set by the extract editor (one row per outcome) per the extract header.",142 "handling": "Values used as printed; Table 3 'Difference in means' column duplicates Table 2 ATE column, consistent (-1.99; [-2.24, -1.75])."},143]144145counts = {k: sum(1 for v in verdicts.values() if v == k) for k in ["supported", "not_supported", "not_assessable"]}146results = {147 "task_id": "F4-conf-02",148 "verdicts": verdicts,149 "locators": locators,150 "reasons": reasons,151 "n_supported": counts["supported"],152 "n_not_supported": counts["not_supported"],153 "n_not_assessable": counts["not_assessable"],154 "feasibility": "complete",155 "feasibility_reason": None,156 "data_issues": data_issues,157 "n_input_rows": {158 "assertions.json": len(assertions),159 "README.md": wc(os.path.join(D, "README.md")),160 f"sources/{DCR}.txt": wc(os.path.join(D, "sources", DCR + ".txt")),161 f"sources/{RES}.txt": wc(os.path.join(D, "sources", RES + ".txt")),162 },163 "n_analysed": len(assertions),164 "random_seed": None,165 "checks": checks,166}167os.makedirs("out", exist_ok=True)168with open("out/results.json", "w", encoding="utf-8") as f:169 json.dump(results, f, indent=2, ensure_ascii=False)170print("\nVerdicts:", verdicts)171print("Counts:", counts)172