1"""F4-conf-02: verify six assertions against cited source extracts.2Reads only data/; writes out/results.json deterministically (no randomness)."""3import json, math, os, re45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7OUT = os.path.join(ROOT, "out")89def lines_of(p):10 with open(p, encoding="utf-8") as f:11 return f.read().split("\n")1213def wc_l(p):14 with open(p, "rb") as f:15 return f.read().count(b"\n")1617assertions = json.load(open(os.path.join(D, "assertions.json"), encoding="utf-8"))18A = {a["id"]: a for a in assertions}19DCR, RES = "arxiv-2609.27732v1", "arxiv-2606.14621v1"20src = {s: lines_of(os.path.join(D, "sources", s + ".txt")) for s in (DCR, RES)}2122def find(sid, pattern, start=0):23 """Return (line_no, text) of first line at/after `start` matching regex."""24 for i, l in enumerate(src[sid][start:], start=start):25 if re.search(pattern, l):26 return i + 1, l27 raise ValueError(f"pattern {pattern!r} not found in {sid}")2829def show(tag, sid, ln, txt):30 print(f"[{tag}] {sid}:{ln}: {txt.strip()}")3132def nums(s):33 return [float(x.replace(",", "")) for x in re.findall(r"-?\d[\d,]*\.?\d*", s)]3435def cells(line):36 return [c.strip() for c in line.split("|")]3738def coef(c): # '-0.431**' -> -0.43139 return float(re.sub(r"[*%]", "", c))4041def pct(b):42 return 100 * (math.exp(b) - 1)4344def ci(b, se, z):45 return (b - z * se, b + z * se)4647# ---------- locate tables in DCR source ----------48t6, _ = find(DCR, r"^Table 6:")49ln_att6, l_att6 = find(DCR, r"^ATT \|", t6)50ln_se6, l_se6 = ln_att6 + 2, src[DCR][ln_att6 + 1]51ln_pc6, l_pc6 = find(DCR, r"^% change \|", t6)52att6 = [coef(c) for c in cells(l_att6)[1:]] # 1,3,5 km53se6 = [float(c.strip("()")) for c in cells(l_se6)[1:]]54pc6 = [coef(c) for c in cells(l_pc6)[1:]]55stars6 = [c.count("*") for c in cells(l_att6)[1:]]5657t7, _ = find(DCR, r"^Table 7:")58def t7row(name):59 ln, l = find(DCR, r"^" + re.escape(name) + r" \|", t7)60 se_l = src[DCR][ln + 1]61 return ln, l, ln + 2, se_l, [coef(c) for c in cells(l)[1:]], [float(c.strip("()")) for c in cells(se_l)[1:]]62hiv = t7row("HIV")63hep = t7row("Hepatitis")6465ta19, _ = find(DCR, r"^Table A19:")66ln_a19, l_a19 = find(DCR, r"^ATT \|", ta19)67l_a19se = src[DCR][ln_a19 + 1]68ln_a19note, l_a19note = find(DCR, r"never-treated comparison regions", ta19)6970print("=== Source lines relied on ===")71for a in assertions:72 print(f"{a['id']} (cites {a['cites']}): {a['text']}")73print()7475verdicts, locators, reasons, details = {}, {}, {}, {}7677# ---------- A1 ----------78show("A1", DCR, ln_att6, l_att6); show("A1", DCR, ln_se6, l_se6); show("A1", DCR, ln_pc6, l_pc6)79b, se, st, p = att6[2], se6[2], stars6[2], pc6[2]80ok = (b == -0.431 and se == 0.210 and st == 2 and p == -35.0 and round(pct(b), 1) == -35.081 and abs(b / se) > 1.96)82se_a19 = float(cells(l_a19se)[1].strip("()"))83show("A1", DCR, ln_a19, l_a19); show("A1", DCR, ln_a19 + 2, l_a19se)84details["A1"] = {"att": b, "se": se, "stars": st, "printed_pct": p, "computed_pct": round(pct(b), 2),85 "z": round(b / se, 3), "se_tableA19_CSDR": se_a19, "z_with_A19_se": round(b / se_a19, 3)}86verdicts["A1"] = "supported" if ok else "not_supported"87locators["A1"] = f'{DCR} | Table 6 | row "ATT" | column "5 km"'88reasons["A1"] = ("Table 6 prints the 5 km ATT as -0.431** with SE (0.210) and % change -35.0% "89 "(100*(exp(-0.431)-1) = -35.0%), so estimate, SE, 5% significance and the 35.0% reduction all match.")9091# ---------- A2 ----------92show("A2", DCR, hiv[0], hiv[1]); show("A2", DCR, hiv[2], hiv[3])93b1 = hiv[4][0]94details["A2"] = {"hiv_att_1km": b1, "hiv_se_1km": hiv[5][0], "asserted": -0.063}95verdicts["A2"] = "supported" if b1 < 0 else "not_supported"96locators["A2"] = f'{DCR} | Table 7 | row "HIV" | column "1 km"'97reasons["A2"] = ("Table 7 prints the 1 km HIV ATT as +0.063 (SE 0.801), so the asserted negative sign "98 "(-0.063) and the claimed fall in HIV hospitalisations are contradicted.")99100# ---------- A3 ----------101show("A3", DCR, ln_att6, l_att6); show("A3", DCR, ln_pc6, l_pc6)102b3, p3 = att6[1], pc6[1]103details["A3"] = {"att_3km": b3, "printed_pct_3km": p3, "computed_pct_3km": round(pct(b3), 2), "asserted_pct": -28.3}104verdicts["A3"] = "supported" if abs(p3 - (-28.3)) < 0.05 else "not_supported"105locators["A3"] = f'{DCR} | Table 6 | row "% change" | column "3 km"'106reasons["A3"] = ("Table 6 converts the 3 km ATT of -0.283 to a -24.7% change (100*(exp(-0.283)-1) = -24.6%), "107 "so the asserted 28.3% reduction wrongly reads log points as percent.")108109# ---------- A4 ----------110show("A4", DCR, hep[0], hep[1]); show("A4", DCR, hep[2], hep[3])111b4, se4 = hep[4][2], hep[5][2]112c196, c200, c1 = ci(b4, se4, 1.96), ci(b4, se4, 2.0), ci(b4, se4, 1.0)113excl196 = c196[0] > 0 or c196[1] < 0114excl200 = c200[0] > 0 or c200[1] < 0115details["A4"] = {"att_5km": b4, "se_5km": se4, "ci95_z1.96": [round(x, 3) for x in c196],116 "ci95_z2.0": [round(x, 3) for x in c200], "pm_1se": [round(x, 3) for x in c1],117 "excludes_zero_z1.96": excl196, "excludes_zero_z2.0": excl200,118 "stars_printed": hep[1].split("|")[3].count("*")}119verdicts["A4"] = "supported" if (excl196 and abs(c196[0] + 0.66) < 0.03 and abs(c196[1] + 0.16) < 0.03) else "not_supported"120locators["A4"] = f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"'121reasons["A4"] = (f"Table 7 prints the 5 km hepatitis ATT as -0.406 (SE 0.250) with no star, giving a 95% CI of "122 f"[{c196[0]:.3f}, {c196[1]:.3f}] that includes zero; the asserted [-0.66, -0.16] is only +/-1 SE.")123124# ---------- A5 ----------125sex_hits = [(i + 1, l) for i, l in enumerate(src[DCR])126 if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]127print(f"[A5] lines in {DCR} mentioning men/women/male/female/sex/gender: {len(sex_hits)}")128for ln, l in sex_hits:129 show("A5", DCR, ln, l)130details["A5"] = {"n_lines_mentioning_sex_or_gender": len(sex_hits)}131verdicts["A5"] = "not_assessable" if not sex_hits else "REVIEW"132locators["A5"] = f"not reported in {DCR}"133reasons["A5"] = ("The extract reports ATTs only by buffer (Table 6) and diagnostic category (Table 7, Table A19) "134 "and contains no estimate by sex, so the men-versus-women comparison can be neither confirmed nor refuted.")135136# ---------- A6 ----------137ln6, l6 = find(RES, r"^Weeks on UI \|")138show("A6", RES, ln6, l6)139ln6t, l6t = find(RES, r"11\.5% decrease")140i0 = l6t.find("a 1.99 week reduction"); i1 = l6t.find("17.3 weeks") + len("17.3 weeks.")141show("A6", RES, ln6t, "..." + l6t[i0:i1])142c = cells(l6)143ate = c[4]; ctrl = c[2]144ate_n = nums(ate)145ctrl_mean = nums(ctrl)[0]146est, lo, hi = ate_n[0], ate_n[-2], ate_n[-1]147rel = 100 * est / ctrl_mean148details["A6"] = {"ate": est, "ci95": [lo, hi], "control_mean": ctrl_mean, "computed_rel_pct": round(rel, 2)}149ok6 = est == -1.99 and lo == -2.24 and hi == -1.75 and ctrl_mean == 17.3 and round(rel, 1) == -11.5150verdicts["A6"] = "supported" if ok6 else "not_supported"151locators["A6"] = f'{RES} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'152reasons["A6"] = ("Table 2 prints the ATE as -1.99 weeks with 95% CI [-2.24, -1.75] and a control mean of 17.3 weeks, "153 "and -1.99/17.3 = -11.5%, matching the text's 11.5% decrease.")154155# ---------- naive baseline: do all numbers in the assertion appear verbatim in the cited source? ----------156baseline = {}157for a in assertions:158 text = src[a["cites"]]159 blob = "\n".join(text).replace("−", "-")160 toks = re.findall(r"\d+(?:\.\d+)?", a["text"].replace("−", "-"))161 toks = [t for t in toks if t not in ("1", "3", "5", "95")]162 baseline[a["id"]] = all(t in blob for t in toks)163print("\nNaive baseline (all numbers verbatim in source):", baseline)164165data_issues = [166 {"issue": f"{DCR}: Table A19 prints the CS-DR 5 km ATT -0.431** with SE (0.218), whereas Table 6 prints the same estimate with SE (0.210).",167 "handling": "Table 6 (the main results table, the one A1 cites) is used as the locator for A1; A1's 5% significance also holds with SE 0.218 at z=1.96 (|t|=1.98), and the source prints ** in both tables."},168 {"issue": f"{DCR}: Table A19 note says 640 never-treated comparison regions, while Table A19 body, Table 6 and the Figure 6 note say 608.",169 "handling": "Recorded only; no assertion depends on the control-region count."},170 {"issue": f"{DCR}: abstract states fatal hospitalisations fall by 35%, but Table 7 5 km fatal ATT -0.354 implies -29.8% and Section 6.1 says approximately 30%.",171 "handling": "Recorded only; no assertion concerns fatal hospitalisations. Tables are treated as primary evidence."},172 {"issue": f"{DCR}: Section 6 text calls HIV estimates 'close to zero' although Table 7 HIV ATTs at 3 km/5 km are -0.239/-0.340 (insignificant).",173 "handling": "A2 is decided by the Table 7 1 km cell (+0.063), not by the narrative."},174 {"issue": f"{DCR}: Table 6 prints the 3 km % change as -24.7%, while 100*(exp(-0.283)-1) = -24.65% rounds to -24.6% (a 0.05 pp rounding difference, presumably from the unrounded coefficient).",175 "handling": "Immaterial for A3: both -24.6% and -24.7% are far from the asserted -28.3%; the printed -24.7% is quoted."},176 {"issue": f"{RES}: results text gives the weeks-on-UI CI as unsigned magnitudes [1.75 weeks, 2.24 weeks] while Table 2 prints the signed CI [-2.24, -1.75].",177 "handling": "Treated as the same interval expressed as a reduction; A6 is decided by Table 2."},178 {"issue": f"{RES}: rendering defect 'p ¡ 0.001' in the short-term weeks-on-UI sentence.",179 "handling": "Read as p < 0.001 (consistent with Table 2); not decisive for any assertion."},180]181182n_input_rows = {"assertions.json": len(assertions), "README.md": wc_l(os.path.join(D, "README.md"))}183for s in (DCR, RES):184 n_input_rows[s + ".txt"] = wc_l(os.path.join(D, "sources", s + ".txt"))185186vals = list(verdicts.values())187results = {188 "task_id": "F4-conf-02",189 "verdicts": verdicts,190 "locators": locators,191 "reasons": reasons,192 "n_supported": vals.count("supported"),193 "n_not_supported": vals.count("not_supported"),194 "n_not_assessable": vals.count("not_assessable"),195 "feasibility": "complete",196 "feasibility_reason": None,197 "data_issues": data_issues,198 "n_input_rows": n_input_rows,199 "n_analysed": len(assertions),200 "random_seed": None,201 "baseline_numeric_verbatim_match": baseline,202 "computation_details": details,203}204os.makedirs(OUT, exist_ok=True)205with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:206 json.dump(results, f, indent=2, ensure_ascii=False)207print("\nVerdicts:", verdicts)208print("Counts: supported=%d not_supported=%d not_assessable=%d" %209 (results["n_supported"], results["n_not_supported"], results["n_not_assessable"]))210