1"""F4-conf-02: verify six assertions against cited source extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import json, math, os, re, hashlib45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7OUT = os.path.join(ROOT, "out")8DCR, RES = "arxiv-2609.27732v1", "arxiv-2606.14621v1"910def lines(p):11 with open(p, encoding="utf-8") as f:12 return f.read().split("\n")1314src = {s: lines(os.path.join(D, "sources", s + ".txt")) for s in (DCR, RES)}15assertions = json.load(open(os.path.join(D, "assertions.json"), encoding="utf-8"))16assert [a["id"] for a in assertions] == ["A1", "A2", "A3", "A4", "A5", "A6"]1718def find(s, pat, start=0):19 for i, l in enumerate(src[s][start:], start):20 if re.search(pat, l):21 print(f" [{s} line {i+1}] {l.strip()[:200]}")22 return i23 raise ValueError(f"pattern {pat!r} not found in {s}")2425def nums(l):26 return [float(x.replace(",", "")) for x in re.findall(r"-?\d+(?:,\d{3})*\.\d+|-?\d+", l)]2728def cells(l):29 return [c.strip() for c in l.split("|")]3031def val(c): # strip stars / % from a table cell32 return float(re.sub(r"[*%()]", "", c))3334pct = lambda b: 100 * (math.exp(b) - 1)35V, L, R = {}, {}, {}3637# ---- Table 6 (DCR) ----38print("Table 6 (overall ATT):")39t6 = find(DCR, r"^Table 6:")40i_att = find(DCR, r"^ATT \|", t6)41i_se = i_att + 2; print(f" [{DCR} line {i_se+1}] {src[DCR][i_se]}")42i_pc = find(DCR, r"^% change", t6)43att6 = cells(src[DCR][i_att])[1:]; se6 = [val(c) for c in cells(src[DCR][i_se])[1:]]44pc6 = [val(c) for c in cells(src[DCR][i_pc])[1:]]45# columns: 1 km, 3 km, 5 km46b5, s5, stars5 = val(att6[2]), se6[2], att6[2].count("*")47b3 = val(att6[1])4849# A150z5 = abs(b5) / s551a1_ok = (b5 == -0.431 and s5 == 0.210 and stars5 == 2 and pc6[2] == -35.052 and round(pct(b5), 1) == -35.0 and z5 > 1.96)53V["A1"] = "supported" if a1_ok else "not_supported"54L["A1"] = f'{DCR} | Table 6 | row "ATT" / "% change" | column "5 km"'55R["A1"] = (f"Table 6 prints ATT {att6[2]} with SE ({s5:.3f}) (** = p<0.05; |b/SE|={z5:.2f}) and % change {pc6[2]}%, "56 f"and 100(exp({b5})-1)={pct(b5):.1f}%, so sign, magnitude, SE, 5% significance and the 35.0% reduction all hold.")5758# A359a3_ok = round(abs(pct(b3)), 1) == 28.3 and abs(pc6[1]) == 28.360V["A3"] = "supported" if a3_ok else "not_supported"61L["A3"] = f'{DCR} | Table 6 | row "% change" | column "3 km"'62R["A3"] = (f"The ATT {b3} is in log points; Table 6 prints the implied change as {pc6[1]}% "63 f"(100(exp({b3})-1)={pct(b3):.1f}%), not a 28.3% reduction, which wrongly equates log points with percent.")6465# ---- Table 7 (DCR) ----66print("Table 7 (ATT by outcome):")67t7 = find(DCR, r"^Table 7:")68i_hiv = find(DCR, r"^HIV \|", t7)69i_hep = find(DCR, r"^Hepatitis \|", t7)70print(f" [{DCR} line {i_hep+3}] {src[DCR][i_hep+2]}")71hiv = cells(src[DCR][i_hiv]); hep = cells(src[DCR][i_hep]); hep_se = cells(src[DCR][i_hep + 2])72hiv1 = val(hiv[1])73V["A2"] = "supported" if hiv1 < 0 else "not_supported"74L["A2"] = f'{DCR} | Table 7 | row "HIV" | column "1 km"'75R["A2"] = (f"Table 7 prints the 1 km HIV ATT as {hiv[1]} (positive, SE {cells(src[DCR][i_hiv+2])[1]}), "76 f"not -0.063, so the sign and the claimed fall in HIV hospitalisations are contradicted.")7778bh, sh = val(hep[3]), val(hep_se[3])79ci196 = (bh - 1.96 * sh, bh + 1.96 * sh); ci2 = (bh - 2 * sh, bh + 2 * sh); ci1 = (bh - sh, bh + sh)80excl = ci196[1] < 0 or ci196[0] > 081claimed = (-0.66, -0.16)82a4_ok = excl and abs(ci196[0] - claimed[0]) < 0.02 and abs(ci196[1] - claimed[1]) < 0.0283V["A4"] = "supported" if a4_ok else "not_supported"84L["A4"] = f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"'85R["A4"] = (f"With ATT {hep[3]} and SE {hep_se[3]} (no stars), the 95% CI is [{ci196[0]:.3f}, {ci196[1]:.3f}] "86 f"(±2·SE: [{ci2[0]:.3f}, {ci2[1]:.3f}]), which includes zero; the claimed [-0.66, -0.16] is only ±1 SE "87 f"([{ci1[0]:.3f}, {ci1[1]:.3f}]).")8889# A5: sex heterogeneity90print("Search for any sex/gender breakdown in the DCR extract:")91hits = [i for i, l in enumerate(src[DCR]) if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]92for i in hits: print(f" [{DCR} line {i+1}] {src[DCR][i][:200]}")93print(f" -> {len(hits)} matching lines")94V["A5"] = "not_assessable" if not hits else "REVIEW"95L["A5"] = f"not reported in {DCR}"96R["A5"] = ("The extract reports ATTs only by buffer (Table 6), diagnostic category (Table 7) and estimator (Table A19); "97 "no estimate by sex appears anywhere, so the men-vs-women comparison can be neither confirmed nor refuted.")9899# ---- Table 2 (RESEA) ----100print("Table 2 (RESEA):")101t2 = find(RES, r"^Outcome \| Treatment \| Control")102i_ui = find(RES, r"^Weeks on UI \|", t2)103ui = cells(src[RES][i_ui])104ctrl = nums(ui[2])[0]; ate = nums(ui[4]); find(RES, r"11\.5% decrease")105ate_b, ate_lo, ate_hi = ate[0], ate[-2], ate[-1]106rel = 100 * abs(ate_b) / ctrl107a6_ok = (ate_b == -1.99 and (ate_lo, ate_hi) == (-2.24, -1.75) and ctrl == 17.3 and round(rel, 1) == 11.5)108V["A6"] = "supported" if a6_ok else "not_supported"109L["A6"] = f'{RES} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'110R["A6"] = (f"Table 2 prints the ATE as {ate_b} weeks (95% CI [{ate_lo}, {ate_hi}]) and a control mean of {ctrl} weeks; "111 f"1.99/17.3 = {rel:.1f}%, matching the 11.5% decrease stated in the results text.")112113data_issues = [114 {"issue": f"{DCR}: Table 6 prints the 3 km % change as -24.7%, whereas 100(exp(-0.283)-1) = -24.65% rounds to -24.6% (a last-digit rounding difference, possibly from an unrounded coefficient).",115 "handling": "Immaterial for A3: both values are far from the asserted 28.3%."},116 {"issue": f"{DCR}: the CS-DR 5 km ATT -0.431 carries SE (0.210) in Table 6 but SE (0.218) in Appendix Table A19, which states the same specification (5 km, 50 treated, 608 control, 17,250 obs).",117 "handling": "A1 cites Table 6 and quotes SE 0.210, so Table 6 is used; the verdict (significant at 5%) is unchanged under SE 0.218 (|z|=1.98>1.96)."},118 {"issue": f"{DCR}: Table A19 note says 640 never-treated comparison regions, while the A19 table body, Table 6 and Table 7 say 608.",119 "handling": "Recorded only; no assertion depends on the control-region count."},120 {"issue": f"{DCR}: the abstract says fatal outcomes 'fall by 35%', but Table 7 gives -0.354 at 5 km, i.e. 100(exp(-0.354)-1) = -29.8%, matching Section 6.1 ('approximately 30%'); the text also calls HIV estimates 'close to zero' although the 5 km HIV ATT is -0.340.",121 "handling": "Recorded only; tables are treated as authoritative over narrative text; no assertion depends on these statements."},122 {"issue": f"{RES}: results text writes the weeks-on-UI CIs as positive magnitudes ('[1.75 weeks, 2.24 weeks]') for a reduction, and contains a rendering defect 'p ¡ 0.001' for p < 0.001.",123 "handling": "Interpreted as the same interval as Table 2 [-2.24, -1.75] expressed as reduction magnitudes; Table 2 used as locator for A6."},124]125126def wcl(p):127 with open(p, "rb") as f:128 return f.read().count(b"\n")129n_input_rows = {130 "assertions.json": len(assertions),131 "README.md": wcl(os.path.join(D, "README.md")),132 f"sources/{DCR}.txt": wcl(os.path.join(D, "sources", DCR + ".txt")),133 f"sources/{RES}.txt": wcl(os.path.join(D, "sources", RES + ".txt")),134}135vals = list(V.values())136assert all(v in ("supported", "not_supported", "not_assessable") for v in vals)137ids = ["A1", "A2", "A3", "A4", "A5", "A6"]138res = {139 "task_id": "F4-conf-02",140 "verdicts": {k: V[k] for k in ids},141 "locators": {k: L[k] for k in ids},142 "reasons": {k: R[k] for k in ids},143 "n_supported": vals.count("supported"),144 "n_not_supported": vals.count("not_supported"),145 "n_not_assessable": vals.count("not_assessable"),146 "feasibility": "complete",147 "feasibility_reason": None,148 "data_issues": data_issues,149 "n_input_rows": n_input_rows,150 "n_analysed": len(assertions),151 "random_seed": None,152 "baseline_naive_all_supported_verdicts": {k: "supported" for k in ids},153 "derived": {154 "A1_pct_from_log": round(pct(b5), 2), "A1_z": round(z5, 3),155 "A1_z_with_A19_se_0218": round(abs(b5) / 0.218, 3),156 "A3_pct_from_log": round(pct(b3), 2),157 "A4_ci95_1p96": [round(ci196[0], 3), round(ci196[1], 3)],158 "A4_ci95_2p0": [round(ci2[0], 3), round(ci2[1], 3)],159 "A4_ci_pm1se": [round(ci1[0], 3), round(ci1[1], 3)],160 "A5_sex_term_hits": len(hits),161 "A6_relative_pct": round(rel, 2),162 },163}164os.makedirs(OUT, exist_ok=True)165with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:166 json.dump(res, f, indent=2, ensure_ascii=False, sort_keys=False)167print("Verdicts:", res["verdicts"])168print("Counts: supported=%d not_supported=%d not_assessable=%d" % (res["n_supported"], res["n_not_supported"], res["n_not_assessable"]))169