1"""F4-conf-02: verify six assertions against their cited source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json deterministically (no randomness)."""4import json, math, os, re56DATA = "data"7SRC = {"arxiv-2609.27732v1": os.path.join(DATA, "sources", "arxiv-2609.27732v1.txt"),8 "arxiv-2606.14621v1": os.path.join(DATA, "sources", "arxiv-2606.14621v1.txt")}9Z, Z_ALT = 1.96, 2.01011def lines(path):12 with open(path, encoding="utf-8") as f:13 return f.read().split("\n")1415def wc_l(path):16 with open(path, "rb") as f:17 return f.read().count(b"\n")1819assertions = json.load(open(os.path.join(DATA, "assertions.json"), encoding="utf-8"))20L = {k: lines(v) for k, v in SRC.items()}21DCR, RES = L["arxiv-2609.27732v1"], L["arxiv-2606.14621v1"]2223def find(src, pattern, start=0):24 """Return (lineno 1-based, line) of first line matching regex at/after start."""25 for i in range(start, len(src)):26 if re.search(pattern, src[i]):27 return i + 1, src[i]28 raise ValueError(pattern)2930def cells(line):31 return [c.strip() for c in line.split("|")]3233def num(s):34 return float(re.sub(r"[^0-9.\-]", "", s.replace("−", "-")))3536def show(tag, ln, text):37 print(f" [{tag} line {ln}] {text}")3839def pct(b):40 return 100 * (math.exp(b) - 1)4142# ---- parse DCR Table 6 ----43t6, _ = find(DCR, r"^Table 6:")44ln_att, att_line = find(DCR, r"^ATT \|", t6)45ln_se, se_line = ln_att + 2, DCR[ln_att + 1]46ln_pc, pc_line = find(DCR, r"^% change \|", t6)47t6_att = cells(att_line)[1:] # 1 km, 3 km, 5 km48t6_se = [num(c) for c in cells(se_line)[1:]]49t6_pc = cells(pc_line)[1:]50# ---- parse DCR Table 7 ----51t7, _ = find(DCR, r"^Table 7:")52def t7row(name):53 ln, row = find(DCR, r"^" + re.escape(name) + r" \|", t7)54 se = DCR[ln + 1]55 return ln, row, cells(row)[1:], [num(c) for c in cells(se)[1:]], se5657verd, loc, reas, detail = {}, {}, {}, {}5859print("A1 -- DCR Table 6, 5 km column")60show("Table 6", ln_att, att_line); show("Table 6 SE", ln_se, se_line); show("Table 6", ln_pc, pc_line)61b5 = num(t6_att[2]); se5 = t6_se[2]; stars5 = t6_att[2].count("*")62z5 = abs(b5) / se563ok = (t6_att[2].startswith("-0.431") and se5 == 0.210 and stars5 == 2 and t6_pc[2] == "-35.0%"64 and round(pct(b5), 1) == -35.0)65print(f" b={b5}, SE={se5}, stars={stars5} (** = p<0.05), |b|/SE={z5:.3f}, 100(exp(b)-1)={pct(b5):.2f}%")66verd["A1"] = "supported" if ok else "not_supported"67loc["A1"] = 'arxiv-2609.27732v1 | Table 6 | row "ATT" | column "5 km"'68reas["A1"] = ("Table 6 prints the 5 km ATT as -0.431** with SE (0.210), ** denoting p<0.05, and a % change of -35.0% "69 "(100(exp(-0.431)-1) = -35.0%), matching every element of the assertion.")70detail["A1"] = {"b": b5, "se": se5, "z": round(z5, 4), "pct": round(pct(b5), 2)}7172print("A2 -- DCR Table 7, HIV row, 1 km column")73ln, row, c, se, seline = t7row("HIV")74show("Table 7", ln, row); show("Table 7 SE", ln + 2, seline)75hiv1 = num(c[0])76print(f" HIV 1 km ATT = {c[0]} (sign {'positive' if hiv1 > 0 else 'negative'}), SE={se[0]}")77verd["A2"] = "not_supported" if hiv1 > 0 else "supported"78loc["A2"] = 'arxiv-2609.27732v1 | Table 7 | row "HIV" | column "1 km"'79reas["A2"] = ("Table 7 prints the 1 km HIV ATT as +0.063 (SE 0.801), positive and insignificant, so the claimed "80 "negative sign (-0.063) and the claimed fall in HIV hospitalisations are contradicted.")81detail["A2"] = {"b": hiv1, "se": se[0]}8283print("A3 -- DCR Table 6, 3 km column")84show("Table 6", ln_att, att_line); show("Table 6", ln_pc, pc_line)85b3 = num(t6_att[1])86print(f" b={b3}, printed % change={t6_pc[1]}, 100(exp(b)-1)={pct(b3):.2f}% (assertion: -28.3%)")87verd["A3"] = "not_supported" if abs(pct(b3) - (-28.3)) > 0.5 else "supported"88loc["A3"] = 'arxiv-2609.27732v1 | Table 6 | row "% change" | column "3 km"'89reas["A3"] = ("Table 6 converts the 3 km ATT of -0.283 to a -24.7% change (100(exp(-0.283)-1) = -24.7%), not 28.3%, "90 "so the stated magnitude is contradicted (28.3% is the log-point coefficient read directly as a percentage).")91detail["A3"] = {"b": b3, "se": t6_se[1], "pct": round(pct(b3), 2), "printed_pct": t6_pc[1]}9293print("A4 -- DCR Table 7, Hepatitis row, 5 km column")94ln, row, c, se, seline = t7row("Hepatitis")95show("Table 7", ln, row); show("Table 7 SE", ln + 2, seline)96bh, sh = num(c[2]), se[2]97ci = (bh - Z * sh, bh + Z * sh); ci2 = (bh - Z_ALT * sh, bh + Z_ALT * sh)98crit_needed = abs(bh) / sh99print(f" b={bh}, SE={sh}; 95% CI (1.96) = [{ci[0]:.3f}, {ci[1]:.3f}]; (2.0) = [{ci2[0]:.3f}, {ci2[1]:.3f}]; "100 f"asserted [-0.66,-0.16] = b +/- {(-0.16 - -0.66)/2:.2f} (=1.0 SE); CI excludes 0 only if crit < {crit_needed:.3f}")101verd["A4"] = "not_supported" if ci[1] > 0 and ci2[1] > 0 else "supported"102loc["A4"] = 'arxiv-2609.27732v1 | Table 7 | row "Hepatitis" | column "5 km"'103reas["A4"] = ("With ATT -0.406 and SE (0.250), the 95% CI is -0.406 +/- 1.96*0.250 = [-0.896, 0.084], which includes zero "104 "(the asserted [-0.66, -0.16] is only +/- 1 SE), and the estimate carries no significance star.")105detail["A4"] = {"b": bh, "se": sh, "ci_196": [round(ci[0], 3), round(ci[1], 3)],106 "ci_200": [round(ci2[0], 3), round(ci2[1], 3)], "crit_to_exclude_zero": round(crit_needed, 3)}107108print("A5 -- sex heterogeneity in DCR extract")109hits = [(i + 1, l) for i, l in enumerate(DCR) if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]110print(f" lines mentioning men/women/male/female/sex/gender: {len(hits)}")111for h in hits: show("match", *h)112verd["A5"] = "not_assessable" if not hits else "REVIEW"113loc["A5"] = "not reported in arxiv-2609.27732v1"114reas["A5"] = ("The extract reports ATTs only by buffer (Table 6), outcome category (Table 7) and estimator (Table A19) "115 "and never by sex, so a men-versus-women comparison can be neither confirmed nor refuted.")116detail["A5"] = {"n_sex_mentions": len(hits)}117118print("A6 -- RESEA Table 2, Weeks on UI row")119ln, row = find(RES, r"^Weeks on UI \|")120show("Table 2", ln, row)121lt, txt = find(RES, r"11\.5% decrease")122show("Results text", lt, txt[txt.find("a 1.99 week"):txt.find("17.3 weeks.") + 11])123cc = cells(row)124ctrl = num(cc[2].split("weeks")[0]); ate = cc[4]125ate_b = num(ate.split(";")[0].replace("weeks", ""))126ci6 = [num(x) for x in re.search(r"\[([^\]]+)\]", ate).group(1).split(",")]127rel = 100 * abs(ate_b) / ctrl128print(f" control mean={ctrl}, ATE={ate_b}, CI={ci6}, |ATE|/control = {rel:.2f}%")129ok6 = ate_b == -1.99 and ci6 == [-2.24, -1.75] and ctrl == 17.3 and round(rel, 1) == 11.5130verd["A6"] = "supported" if ok6 else "not_supported"131loc["A6"] = ('arxiv-2606.14621v1 | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"')132reas["A6"] = ("Table 2 gives the ATE of RESEA selection on weeks on UI as -1.99 weeks (95% CI [-2.24, -1.75], p<0.001) "133 "with a control mean of 17.3 weeks, and 1.99/17.3 = 11.5% as also stated in the results text.")134detail["A6"] = {"ate": ate_b, "ci": ci6, "control_mean": ctrl, "rel_pct": round(rel, 2)}135136# ---- naive baseline: do all numbers in the assertion appear verbatim in the cited source? ----137baseline = {}138for a in assertions:139 srctext = "\n".join(L[a["cites"]]).replace("−", "-")140 nums = re.findall(r"\d+\.\d+", a["text"])141 baseline[a["id"]] = "all_numbers_found" if all(n in srctext for n in nums) else "some_number_missing"142143ids = [a["id"] for a in assertions]144assert ids == ["A1", "A2", "A3", "A4", "A5", "A6"]145assert all(v in ("supported", "not_supported", "not_assessable") for v in verd.values())146147data_issues = [148 {"issue": "arxiv-2609.27732v1: Table A19 CS-DR column prints SE (0.218) for the same ATT -0.431 for which Table 6 (5 km) prints SE (0.210).",149 "handling": "A1 judged against Table 6 (the table the assertion describes); significance at 5% holds under either SE (|b|/SE = 2.05 or 1.98 > 1.96), noted in report."},150 {"issue": "arxiv-2609.27732v1: Table A19 note says 640 never-treated comparison regions, whereas Tables 6, 7, A19 body and Figure 6 note say 608.",151 "handling": "Recorded; not decisive for any assertion."},152 {"issue": "arxiv-2609.27732v1: abstract says fatal hospitalisations fall by 35%, but Table 7 5 km fatal ATT -0.354 implies -29.8%, matching Section 6.1 ('approximately 30%').",153 "handling": "Recorded; no assertion concerns fatal outcomes; tables preferred over abstract text."},154 {"issue": "arxiv-2609.27732v1: text announces five diagnostic sub-categories, Figure 7 note says four, Table 7 note gives ICD codes for four.",155 "handling": "Recorded; not decisive for any assertion."},156 {"issue": "arxiv-2606.14621v1: rendering defect 'p ¡ 0.001' in the short-term weeks-on-UI sentence (should read p < 0.001).",157 "handling": "Read as p < 0.001 (consistent with Table 2); not decisive."},158 {"issue": "arxiv-2606.14621v1: results text prints the weeks-on-UI CI as reduction magnitudes [1.75, 2.24] while Tables 2 and 3 print signed [-2.24, -1.75].",159 "handling": "Treated as equivalent sign conventions; A6 judged against Table 2."},160]161162res = {163 "task_id": "F4-conf-02",164 "verdicts": verd, "locators": loc, "reasons": reas,165 "n_supported": sum(v == "supported" for v in verd.values()),166 "n_not_supported": sum(v == "not_supported" for v in verd.values()),167 "n_not_assessable": sum(v == "not_assessable" for v in verd.values()),168 "feasibility": "complete", "feasibility_reason": None,169 "data_issues": data_issues,170 "n_input_rows": {"assertions.json": len(assertions), "README.md": wc_l(os.path.join(DATA, "README.md")),171 "sources/arxiv-2609.27732v1.txt": wc_l(SRC["arxiv-2609.27732v1"]),172 "sources/arxiv-2606.14621v1.txt": wc_l(SRC["arxiv-2606.14621v1"])},173 "n_analysed": len(assertions),174 "random_seed": None,175 "baseline_verdicts_number_match": baseline,176 "computation_details": detail,177 "conventions": {"pct_change": "100*(exp(b)-1)", "ci_crit": Z, "ci_crit_alt": Z_ALT},178}179os.makedirs("out", exist_ok=True)180with open(os.path.join("out", "results.json"), "w", encoding="utf-8") as f:181 json.dump(res, f, indent=2, ensure_ascii=False)182print("verdicts:", verd)183print("counts: supported=%d not_supported=%d not_assessable=%d" % (res["n_supported"], res["n_not_supported"], res["n_not_assessable"]))184