1"""F4-conf-02: verify six assertions against cited source extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import json, math, re, os45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7DCR, RES = "arxiv-2609.27732v1", "arxiv-2606.14621v1"89def lines(p):10 with open(p, encoding="utf-8") as f:11 return f.read().split("\n")[:-1] if True else None1213def wc_l(p):14 with open(p, "rb") as f:15 return f.read().count(b"\n")1617assertions = json.load(open(os.path.join(D, "assertions.json"), encoding="utf-8"))18src = {s: open(os.path.join(D, "sources", s + ".txt"), encoding="utf-8").read().split("\n")19 for s in (DCR, RES)}2021def find(sid, prefix, after=None):22 """Return (lineno, line) of first line starting with prefix (optionally after a line index)."""23 L = src[sid]24 start = 0 if after is None else after25 for i in range(start, len(L)):26 if L[i].startswith(prefix):27 return i, L[i]28 raise ValueError(f"{prefix!r} not found in {sid}")2930def cells(line):31 return [c.strip() for c in line.split("|")]3233def num(s):34 s = s.replace("−", "-").replace("*", "").replace("%", "").replace("(", "").replace(")", "")35 return float(s)3637def show(sid, i, line):38 print(f" [{sid} L{i+1}] {line}")3940pct = lambda b: 100 * (math.exp(b) - 1)4142# ---- Baseline (P2): naive literal check: are all numbers of the assertion printed in the cited extract?43def baseline(a):44 text = open(os.path.join(D, "sources", a["cites"] + ".txt"), encoding="utf-8").read().replace("−", "-")45 nums = re.findall(r"\d+\.\d+", a["text"].replace("−", "-"))46 return "supported" if all(n in text for n in nums) else "not_supported"47baseline_verdicts = {a["id"]: baseline(a) for a in assertions}4849V, LOC, R = {}, {}, {}50derived = {}5152# ---- Table 6 (DCR)53t6, _ = find(DCR, "Table 6:")54i_att, l_att = find(DCR, "ATT |", t6); i_se = i_att + 2; l_se = src[DCR][i_se]55i_pc, l_pc = find(DCR, "% change |", t6)56att6 = [num(c) for c in cells(l_att)[1:]]; se6 = [num(c) for c in cells(l_se)[1:]]; pc6 = [num(c) for c in cells(l_pc)[1:]]57stars5 = cells(l_att)[3].count("*")5859print("A1:"); [show(DCR, i, src[DCR][i]) for i in (i_att, i_se, i_pc)]60i_txt, l_txt = find(DCR, "Table 6 reports the pooled ATT"); show(DCR, i_txt, l_txt[:200] + " ...")61p5 = pct(att6[2]); derived["A1_pct_change_5km"] = round(p5, 2)62ok = att6[2] == -0.431 and se6[2] == 0.210 and stars5 == 2 and abs(p5 - (-35.0)) < 0.05 and pc6[2] == -35.063V["A1"] = "supported" if ok else "not_supported"64LOC["A1"] = f'{DCR} | Table 6 | row "ATT" | column "5 km"'65R["A1"] = ("Table 6 prints ATT -0.431** (SE 0.210) at 5 km, ** denotes p<0.05, and the % change row prints -35.0% "66 f"(100(exp(-0.431)-1) = {p5:.1f}%), so every element matches.")6768# ---- Table 7 (DCR)69t7, _ = find(DCR, "Table 7:")70def t7row(name):71 i, l = find(DCR, name + " |", t7)72 return i, l, [num(c) for c in cells(l)[1:]], [num(c) for c in cells(src[DCR][i + 2])[1:]]7374i_hiv, l_hiv, hiv, hiv_se = t7row("HIV")75print("A2:"); show(DCR, i_hiv, l_hiv); show(DCR, i_hiv + 2, src[DCR][i_hiv + 2])76V["A2"] = "supported" if hiv[0] < 0 else "not_supported"77LOC["A2"] = f'{DCR} | Table 7 | row "HIV" | column "1 km"'78R["A2"] = (f"Table 7 prints the 1 km HIV ATT as +{hiv[0]:.3f} (SE {hiv_se[0]:.3f}), positive and insignificant, "79 "so the asserted negative sign and the claimed fall in HIV hospitalisations are contradicted.")8081print("A3:"); show(DCR, i_att, l_att); show(DCR, i_pc, l_pc)82p3 = pct(att6[1]); derived["A3_pct_change_3km"] = round(p3, 2)83V["A3"] = "supported" if abs(p3 - (-28.3)) < 0.05 else "not_supported"84LOC["A3"] = f'{DCR} | Table 6 | row "% change" | column "3 km"'85R["A3"] = (f"Table 6 converts the 3 km ATT of -0.283 into a -24.7% change (100(exp(-0.283)-1) = {p3:.2f}%), "86 "not 28.3%, which is the log-point coefficient misread as a percentage.")8788i_hep, l_hep, hep, hep_se = t7row("Hepatitis")89print("A4:"); show(DCR, i_hep, l_hep); show(DCR, i_hep + 2, src[DCR][i_hep + 2])90b, s = hep[2], hep_se[2]91ci196 = (b - 1.96 * s, b + 1.96 * s); ci2 = (b - 2 * s, b + 2 * s); ci1 = (b - s, b + s)92derived["A4_ci95_1.96"] = [round(x, 3) for x in ci196]; derived["A4_ci95_2.0"] = [round(x, 3) for x in ci2]93derived["A4_ci_pm_1se"] = [round(x, 3) for x in ci1]94excl = ci196[1] < 0 and ci2[1] < 095V["A4"] = "supported" if excl and abs(ci196[0] + 0.66) < 0.02 and abs(ci196[1] + 0.16) < 0.02 else "not_supported"96LOC["A4"] = f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"'97R["A4"] = (f"With ATT -0.406 and SE 0.250 (no stars), the 95% CI is [{ci196[0]:.2f}, {ci196[1]:.2f}] (±1.96·SE), which "98 f"includes zero; the asserted [-0.66, -0.16] is only ±1 SE ([{ci1[0]:.3f}, {ci1[1]:.3f}]).")99100print("A5: search for sex-specific results")101hits = [(s_, i, l) for s_ in (DCR,) for i, l in enumerate(src[s_])102 if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]103for h in hits: show(*h)104print(f" {len(hits)} lines mention sex/gender in {DCR}")105V["A5"] = "not_assessable" if not hits else "REVIEW"106LOC["A5"] = f"not reported in {DCR}"107R["A5"] = ("The extract reports ATTs only by buffer, estimator and diagnostic category (Tables 6, 7, A19) and never by sex, "108 "so a men-versus-women comparison can be neither confirmed nor refuted.")109110# ---- RESEA Table 2111i_w, l_w = find(RES, "Weeks on UI |")112print("A6:"); show(RES, i_w, l_w)113i_t, l_t = find(RES, "Overall, we find"); show(RES, i_t, l_t[l_t.find("and a 1.99"):])114c = cells(l_w)115ctrl = float(re.search(r"([\d.]+) weeks", c[2]).group(1))116ate = c[4]117est = float(re.search(r"(-?[\d.]+) weeks", ate).group(1))118lo, hi = [float(x) for x in re.search(r"\[(-?[\d.]+), (-?[\d.]+)\]", ate).groups()]119rel = 100 * est / ctrl; derived["A6_pct_of_control_mean"] = round(rel, 2)120ok6 = est == -1.99 and (lo, hi) == (-2.24, -1.75) and ctrl == 17.3 and abs(rel + 11.5) < 0.05121V["A6"] = "supported" if ok6 else "not_supported"122LOC["A6"] = f'{RES} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'123R["A6"] = (f"Table 2 prints the ATE as -1.99 weeks (95% CI [-2.24, -1.75]) against a control mean of 17.3 weeks, "124 f"and -1.99/17.3 = {rel:.1f}%, matching the 11.5% decrease stated in the results text.")125126assert "REVIEW" not in V.values()127vals = list(V.values())128data_issues = [129 {"issue": "arxiv-2609.27732v1 Table A19 prints the CS-DR 5 km ATT -0.431** with SE (0.218), while Table 6 prints the same specification with SE (0.210).",130 "handling": "A1 cites the main result; judged against Table 6 (SE 0.210). With SE 0.210, |t|=2.05 exceeds both 1.96 and 2.0; with SE 0.218, |t|=1.98 exceeds 1.96 but not 2.0, yet ** is printed in both tables, so the verdict rests on the printed significance stars."},131 {"issue": "arxiv-2609.27732v1 Table A19 note states 640 never-treated comparison regions; table body and Tables 6-7 state 608.",132 "handling": "Recorded only; no assertion depends on the control-region count."},133 {"issue": "arxiv-2609.27732v1 abstract says fatal hospitalisations fall by 35%, but Table 7 5 km fatal ATT -0.354 implies -29.8% (Section 6.1 says ~30%).",134 "handling": "Recorded only; no assertion concerns fatal outcomes."},135 {"issue": "arxiv-2609.27732v1 Figure 7 note mentions four sub-categories while the text and Table 7 report five.",136 "handling": "Recorded only; not decision-relevant."},137 {"issue": "arxiv-2606.14621v1 results text reports CIs of reductions as unsigned magnitudes ([1.75, 2.24]) while Table 2 uses signed values; rendering defect 'p ¡ 0.001'.",138 "handling": "Treated as equivalent; A6 judged against the signed Table 2 values."},139]140n_input_rows = {141 "assertions.json": len(assertions),142 "README.md": wc_l(os.path.join(D, "README.md")),143 f"sources/{DCR}.txt": wc_l(os.path.join(D, "sources", DCR + ".txt")),144 f"sources/{RES}.txt": wc_l(os.path.join(D, "sources", RES + ".txt")),145}146results = {147 "task_id": "F4-conf-02",148 "verdicts": V, "locators": LOC, "reasons": R,149 "n_supported": vals.count("supported"), "n_not_supported": vals.count("not_supported"),150 "n_not_assessable": vals.count("not_assessable"),151 "feasibility": "complete", "feasibility_reason": None,152 "data_issues": data_issues, "n_input_rows": n_input_rows,153 "n_analysed": len(assertions), "random_seed": None,154 "baseline_verdicts_number_match": baseline_verdicts,155 "baseline_agreement_with_final": sum(baseline_verdicts[k] == V[k] for k in V),156 "derived_quantities": derived,157}158os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)159with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:160 json.dump(results, f, indent=2, ensure_ascii=False)161print("verdicts:", V)162print("baseline:", baseline_verdicts)163