1"""F4-conf-02: verify six assertions against the cited source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Deterministic; reads data/ only; prints the source lines it relies on."""4import json, math, os, re56DATA = "data"7DCR = "arxiv-2609.27732v1"8RESEA = "arxiv-2606.14621v1"910def lines(path):11 with open(path, encoding="utf-8") as f:12 return f.read().split("\n")1314def wc_l(path):15 with open(path, "rb") as f:16 return f.read().count(b"\n")1718assertions = json.load(open(os.path.join(DATA, "assertions.json"), encoding="utf-8"))19src = {s: lines(os.path.join(DATA, "sources", s + ".txt")) for s in (DCR, RESEA)}2021def find(sid, pattern, start=0):22 """Return (line_no, text) of the first line matching regex pattern at/after start."""23 for i, l in enumerate(src[sid][start:], start=start):24 if re.search(pattern, l):25 return i, l26 raise ValueError(f"pattern {pattern!r} not found in {sid}")2728def row_with_se(sid, pattern, start=0):29 """Coefficient row and the following non-empty SE row."""30 i, l = find(sid, pattern, start)31 j = i + 132 while not src[sid][j].strip():33 j += 134 cells = [c.strip() for c in l.split("|")]35 ses = [c.strip() for c in src[sid][j].split("|") if c.strip()]36 print(f" [{sid} L{i+1}] {l}")37 print(f" [{sid} L{j+1}] {src[sid][j]}")38 return cells, ses3940def num(cell):41 return float(re.sub(r"[^\d.\-]", "", cell.replace("−", "-")))4243pct = lambda b: 100 * (math.exp(b) - 1)44verdicts, locators, reasons = {}, {}, {}45checks = {}4647for a in assertions:48 print(f"{a['id']} (cites {a['cites']}): {a['text']}")49print()5051# ---------- A1: Table 6, 5 km ----------52print("A1 evidence:")53t6 = find(DCR, r"^Table 6")[0]54cells, ses = row_with_se(DCR, r"^ATT \|", t6)55pc = find(DCR, r"^% change", t6); print(f" [{DCR} L{pc[0]+1}] {pc[1]}")56b, se = num(cells[3]), num(ses[2])57stars = cells[3].count("*")58pct_printed = pc[1].split("|")[3].strip()59ok = (b == -0.431 and se == 0.210 and stars >= 2 and pct_printed == "-35.0%"60 and abs(pct(b) - (-35.0)) < 0.05)61checks["A1"] = dict(att=b, se=se, stars=stars, pct_printed=pct_printed, pct_implied=round(pct(b), 2))62verdicts["A1"] = "supported" if ok else "not_supported"63locators["A1"] = f'{DCR} | Table 6 | row "ATT" | column "5 km"'64reasons["A1"] = ("Table 6 prints ATT -0.431** (SE 0.210) at 5 km (** = p<0.05) and a % change of -35.0%, "65 f"matching 100*(exp(-0.431)-1) = {pct(b):.1f}%.")6667# ---------- A2: Table 7, HIV, 1 km ----------68print("A2 evidence:")69t7 = find(DCR, r"^Table 7")[0]70cells, ses = row_with_se(DCR, r"^HIV \|", t7)71b = num(cells[1])72checks["A2"] = dict(att_1km=b, se=num(ses[0]))73verdicts["A2"] = "supported" if b < 0 else "not_supported"74locators["A2"] = f'{DCR} | Table 7 | row "HIV" | column "1 km"'75reasons["A2"] = (f"Table 7 prints the 1 km HIV ATT as +{cells[1]} (SE {ses[0]}), positive and insignificant, "76 "so the sign claimed (-0.063, a fall) is contradicted.")7778# ---------- A3: Table 6, 3 km ----------79print("A3 evidence:")80cells, ses = row_with_se(DCR, r"^ATT \|", t6)81b = num(cells[2]); p3 = pc[1].split("|")[2].strip()82checks["A3"] = dict(att=b, pct_printed=p3, pct_implied=round(pct(b), 2), claimed=-28.3)83verdicts["A3"] = "supported" if abs(pct(b) - (-28.3)) < 0.05 else "not_supported"84locators["A3"] = f'{DCR} | Table 6 | row "% change" | column "3 km"'85reasons["A3"] = (f"An ATT of -0.283 log points implies 100*(exp(-0.283)-1) = {pct(b):.1f}% (Table 6 prints {p3}), "86 "not 28.3%; the assertion equates log points with percent.")8788# ---------- A4: Table 7, Hepatitis, 5 km ----------89print("A4 evidence:")90cells, ses = row_with_se(DCR, r"^Hepatitis \|", t7)91b, se = num(cells[3]), num(ses[2])92ci196 = (round(b - 1.96 * se, 3), round(b + 1.96 * se, 3))93ci200 = (round(b - 2.0 * se, 3), round(b + 2.0 * se, 3))94ci1se = (round(b - se, 3), round(b + se, 3))95checks["A4"] = dict(att=b, se=se, stars=cells[3].count("*"), ci_1_96=ci196, ci_2_00=ci200, att_pm_1se=ci1se)96excl = ci196[1] < 0 or ci196[0] > 097verdicts["A4"] = "supported" if excl else "not_supported"98locators["A4"] = f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"'99reasons["A4"] = (f"With ATT -0.406 and SE 0.250 (no star), the 95% CI is [{ci196[0]}, {ci196[1]}] (±1.96 SE), which "100 f"includes zero; the claimed [-0.66, -0.16] is only ±1 SE ({ci1se}).")101102# ---------- A5: sex heterogeneity ----------103print("A5 evidence: search for sex/gender breakdown in", DCR)104hits = [(i + 1, l) for i, l in enumerate(src[DCR])105 if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]106for h in hits: print(f" [{DCR} L{h[0]}] {h[1][:160]}")107print(f" -> {len(hits)} matching lines")108checks["A5"] = dict(sex_mentions=len(hits))109verdicts["A5"] = "not_assessable" if not hits else "MANUAL_CHECK"110locators["A5"] = f"not reported in {DCR}"111reasons["A5"] = ("The extract reports ATTs only by buffer, outcome category and estimator; no estimate by sex "112 "(men vs women) appears anywhere, so the comparison can be neither confirmed nor refuted.")113114# ---------- A6: RESEA Table 2, Weeks on UI ----------115print("A6 evidence:")116i, l = find(RESEA, r"^Weeks on UI \|"); print(f" [{RESEA} L{i+1}] {l}")117cells = [c.strip() for c in l.split("|")]118ctrl = num(cells[2].split("weeks")[0])119ate = cells[4]120ate_est = num(ate.split("weeks")[0])121ci = [num(x) for x in re.search(r"\[(.*?)\]", ate).group(1).split(",")]122j, t = find(RESEA, r"11\.5% decrease"); print(f" [{RESEA} L{j+1}] ...{t[t.find('a 1.99 week'):][:260]}")123rel = 100 * ate_est / ctrl124checks["A6"] = dict(ate=ate_est, ci=ci, control_mean=ctrl, rel_pct=round(rel, 2))125ok = ate_est == -1.99 and ci == [-2.24, -1.75] and ctrl == 17.3 and abs(rel - (-11.5)) < 0.05126verdicts["A6"] = "supported" if ok else "not_supported"127locators["A6"] = f'{RESEA} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'128reasons["A6"] = (f"Table 2 prints the ATE -1.99 weeks (95% CI [-2.24, -1.75]) with control mean 17.3 weeks, and "129 f"1.99/17.3 = {abs(rel):.1f}%, as also stated in the Section 3 text.")130131print("\nChecks:", json.dumps(checks, indent=1))132133data_issues = [134 {"issue": f"{DCR}: Table A19 prints the CS-DR 5 km ATT -0.431** with SE (0.218), whereas Table 6 prints the same estimate with SE (0.210).",135 "handling": "A1 assessed against Table 6 (the cited main table); with SE 0.218 the estimate remains significant at 5% (|t|=1.98), so the verdict is unchanged."},136 {"issue": f"{DCR}: Table A19 note states 640 never-treated comparison regions while its 'Control regions' row and Tables 6-7 state 608.",137 "handling": "Recorded; does not bear on any assertion."},138 {"issue": f"{DCR}: abstract says fatal hospitalisations fall by 35%, but Table 7 (5 km, -0.354) implies -29.8% and Section 6.1 says 'approximately 30%'.",139 "handling": "Recorded as an internal inconsistency; no assertion concerns this number."},140 {"issue": f"{DCR}: results text calls HIV estimates 'close to zero', although 3 km and 5 km point estimates are -0.239 and -0.340 (insignificant).",141 "handling": "Recorded; A2 judged on the printed 1 km coefficient (+0.063)."},142 {"issue": f"{DCR}: extract ends mid-sentence in the Table A19 note (truncated).",143 "handling": "No assertion depends on the missing text."},144 {"issue": f"{RESEA}: Section 3 text reports the weeks-on-UI CIs as positive magnitudes ([1.75, 2.24], [1.55, 1.99]) while Table 2 prints signed values; text contains rendering defect 'p ¡ 0.001' (for p < 0.001).",145 "handling": "Treated as sign convention for a 'reduction' and a character-encoding defect; A6 judged on Table 2 values."},146]147148n_input_rows = {149 "assertions.json": len(assertions),150 "README.md": wc_l(os.path.join(DATA, "README.md")),151 f"{DCR}.txt": wc_l(os.path.join(DATA, "sources", DCR + ".txt")),152 f"{RESEA}.txt": wc_l(os.path.join(DATA, "sources", RESEA + ".txt")),153}154vals = list(verdicts.values())155res = {156 "task_id": "F4-conf-02",157 "verdicts": verdicts, "locators": locators, "reasons": reasons,158 "n_supported": vals.count("supported"),159 "n_not_supported": vals.count("not_supported"),160 "n_not_assessable": vals.count("not_assessable"),161 "feasibility": "complete", "feasibility_reason": None,162 "data_issues": data_issues,163 "n_input_rows": n_input_rows,164 "n_analysed": len(assertions),165 "random_seed": None,166}167os.makedirs("out", exist_ok=True)168with open("out/results.json", "w", encoding="utf-8") as f:169 json.dump(res, f, indent=2, ensure_ascii=False)170print("\nVerdicts:", verdicts)171