1"""F4-conf-02: verdicts on six assertions against cited source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json deterministically. No randomness."""4import json, math, re, os56D = "data"7S1, S2 = "arxiv-2609.27732v1", "arxiv-2606.14621v1"8src = {s: open(f"{D}/sources/{s}.txt", encoding="utf-8").read().split("\n") for s in (S1, S2)}9assertions = json.load(open(f"{D}/assertions.json", encoding="utf-8"))1011def wcl(path):12 with open(path, "rb") as f:13 return f.read().count(b"\n")1415def find(s, pattern):16 """Return (index, line) of first line matching regex; print it; fail loudly if absent."""17 for i, l in enumerate(src[s]):18 if re.search(pattern, l):19 print(f" [{s} L{i+1}] {l.strip()[:170]}")20 return i, l21 raise RuntimeError(f"pattern not found in {s}: {pattern}")2223def cells(line):24 return [c.strip() for c in line.split("|")]2526def num(c):27 return float(re.sub(r"[^0-9.\-]", "", c.replace("−", "-")))2829def coef_row(s, pattern):30 """Coefficient row and the SE row that follows it (next non-blank line)."""31 i, l = find(s, pattern)32 j = i + 133 while src[s][j].strip() == "":34 j += 135 print(f" [{s} L{j+1}] {src[s][j].strip()}")36 return cells(l), cells(src[s][j])3738pct = lambda b: 100 * (math.exp(b) - 1)39V, L, R = {}, {}, {}4041# P2 baseline: do all decimal numbers of the assertion appear verbatim in the cited extract?42baseline = {}43for a in assertions:44 nums = re.findall(r"\d+\.\d+", a["text"])45 txt = "\n".join(src[a["cites"]])46 baseline[a["id"]] = all(n in txt for n in nums)4748print("A1")49c, se = coef_row(S1, r"^ATT \| -0\.399")50_, pc = find(S1, r"^% change \| -32\.9%")51b5, se5, p5 = c[3], se[3], cells(pc)[3]52b = num(b5); s_ = num(se5); stars = b5.count("*")53ok = (b == -0.431 and s_ == 0.210 and stars >= 2 and p5 == "-35.0%" and round(pct(b), 1) == -35.0)54V["A1"] = "supported" if ok else "not_supported"55L["A1"] = f'{S1} | Table 6 | row "ATT" | column "5 km"'56R["A1"] = (f"Table 6 prints the 5 km ATT as {b5} with SE {se5} (** = p<0.05) and a % change of {p5} "57 f"(100(exp({b})-1) = {pct(b):.2f}%), matching every element of the assertion.")5859print("A2")60c, se = coef_row(S1, r"^HIV \|")61hiv1 = num(c[1])62V["A2"] = "supported" if hiv1 < 0 else "not_supported"63L["A2"] = f'{S1} | Table 7 | row "HIV" | column "1 km"'64R["A2"] = (f"Table 7 prints the 1 km HIV ATT as {c[1]} (positive, SE {se[1]}), not -0.063, so the sign "65 f"and the claimed fall in HIV hospitalisations are contradicted.")6667print("A3")68c, se = coef_row(S1, r"^ATT \| -0\.399")69_, pc = find(S1, r"^% change \| -32\.9%")70b3 = num(c[2]); p3 = cells(pc)[2]71V["A3"] = "supported" if abs(pct(b3) - (-28.3)) < 0.05 else "not_supported"72L["A3"] = f'{S1} | Table 6 | row "% change" | column "3 km"'73R["A3"] = (f"The 3 km ATT {c[2]} log points converts to 100(exp({b3})-1) = {pct(b3):.1f}% and Table 6 prints "74 f"{p3}, not -28.3% (the assertion equates log points with percent; the estimate is also insignificant, SE {se[2]}).")7576print("A4")77c, se = coef_row(S1, r"^Hepatitis \|")78bh, sh = num(c[3]), num(se[3])79lo196, hi196 = bh - 1.96 * sh, bh + 1.96 * sh80lo2, hi2 = bh - 2 * sh, bh + 2 * sh81excl = hi196 < 0 or lo196 > 082V["A4"] = "supported" if excl and abs(lo196 + 0.66) < 0.02 else "not_supported"83L["A4"] = f'{S1} | Table 7 | row "Hepatitis" | column "5 km"'84R["A4"] = (f"With ATT {c[3]} and SE {se[3]} the 95% CI is [{lo196:.3f}, {hi196:.3f}] (±1.96·SE), which includes zero "85 f"(and the estimate carries no star); [-0.66, -0.16] is only ±1 SE.")8687print("A5")88hits = [i + 1 for i, l in enumerate(src[S1]) if re.search(r"\b(men|women|male|female|gender|sex)\b", l, re.I)]89print(f" lines in {S1} mentioning men/women/male/female/gender/sex: {hits}")90for t in ("Table 6", "Table 7", "Table A19"):91 find(S1, rf"^{t}:")92V["A5"] = "not_assessable" if not hits else "REVIEW"93L["A5"] = f"not reported in {S1}"94R["A5"] = ("The extract reports ATTs only by buffer (Table 6), outcome category (Table 7) and estimator (Table A19) "95 "and contains no estimate by sex, so the men-versus-women comparison can be neither confirmed nor refuted.")9697print("A6")98i, l = find(S2, r"^Weeks on UI \|")99c = cells(l); ate = c[4]; ctrl = c[2]100find(S2, r"11\.5% decrease")101m = re.search(r"(-?\d+\.\d+) weeks; p<0\.001; 95% CI: \[(-?\d+\.\d+), (-?\d+\.\d+)\]", ate)102est, lo, hi = map(float, m.groups()); cm = num(ctrl.split("weeks")[0])103rel = 100 * abs(est) / cm104ok = (est == -1.99 and lo == -2.24 and hi == -1.75 and cm == 17.3 and round(rel, 1) == 11.5)105V["A6"] = "supported" if ok else "not_supported"106L["A6"] = f'{S2} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'107R["A6"] = (f"Table 2 gives the ATE as {ate.split(';')[0]} with 95% CI [{lo}, {hi}] and a control mean of {cm} weeks; "108 f"1.99/17.3 = {rel:.2f}%, matching the 11.5% stated in the Section 3 text.")109110assert "REVIEW" not in V.values()111vals = list(V.values())112data_issues = [113 {"issue": f"{S1}: Table A19 prints the CS-DR 5 km SE as (0.218) whereas Table 6 prints (0.210) for the same ATT -0.431.",114 "handling": "A1 is judged against Table 6 (the table whose values the assertion quotes); 5% significance holds under either SE with the 1.96 critical value (t=2.05 or 1.98)."},115 {"issue": f"{S1}: Table A19 note states 640 never-treated comparison regions, while its body and Tables 6-7 state 608.",116 "handling": "Recorded only; no assertion depends on the control-region count."},117 {"issue": f"{S1}: abstract says fatal outcomes fall by 35%, but Table 7 5 km fatal ATT -0.354 implies -29.8% and Section 6.1 says approximately 30%.",118 "handling": "Recorded only; no assertion concerns fatal outcomes. Tables treated as authoritative over prose."},119 {"issue": f"{S1}: text describes HIV estimates as close to zero, while Table 7 shows -0.340 (0.294) at 5 km and -0.239 (0.337) at 3 km.",120 "handling": "A2 is decided from the Table 7 1 km cell (0.063), not from the prose."},121 {"issue": f"{S2}: results text renders 'p ¡ 0.001' (character-encoding defect for 'p < 0.001').",122 "handling": "Interpreted as p < 0.001; does not affect A6, which relies on Table 2."},123 {"issue": f"{S2}: text reports the weeks-on-UI CI in magnitude form [1.75, 2.24] while Table 2 reports [-2.24, -1.75].",124 "handling": "Treated as the same interval under a sign convention; A6 uses Table 2."},125]126n_input_rows = {127 "assertions.json": len(assertions),128 "README.md": wcl(f"{D}/README.md"),129 f"{S1}.txt": wcl(f"{D}/sources/{S1}.txt"),130 f"{S2}.txt": wcl(f"{D}/sources/{S2}.txt"),131}132res = {133 "task_id": "F4-conf-02",134 "verdicts": V, "locators": L, "reasons": R,135 "n_supported": vals.count("supported"),136 "n_not_supported": vals.count("not_supported"),137 "n_not_assessable": vals.count("not_assessable"),138 "feasibility": "complete", "feasibility_reason": None,139 "data_issues": data_issues,140 "n_input_rows": n_input_rows,141 "n_analysed": len(assertions),142 "random_seed": None,143 "baseline_literal_number_match": baseline,144 "conversions": {145 "A1_pct_change_5km": round(pct(-0.431), 4),146 "A1_t_stat_table6": round(0.431 / 0.210, 4),147 "A1_t_stat_tableA19": round(0.431 / 0.218, 4),148 "A3_pct_change_3km": round(pct(b3), 4),149 "A4_ci95_196": [round(lo196, 4), round(hi196, 4)],150 "A4_ci95_200": [round(lo2, 4), round(hi2, 4)],151 "A6_relative_reduction_pct": round(rel, 4),152 },153}154os.makedirs("out", exist_ok=True)155with open("out/results.json", "w", encoding="utf-8") as f:156 json.dump(res, f, indent=2, ensure_ascii=False)157print("verdicts:", V)158