1"""F4-conf-02: verdicts on six assertions against two source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Verdict logic is explicit; all numbers are parsed from data/ and checked."""4import json, math, re, os56D = "data"7S1, S2 = "arxiv-2609.27732v1", "arxiv-2606.14621v1"8src = {s: open(f"{D}/sources/{s}.txt", encoding="utf-8").read().splitlines() for s in (S1, S2)}9assertions = json.load(open(f"{D}/assertions.json", encoding="utf-8"))1011def line(s, pattern, start=0):12 for i, l in enumerate(src[s][start:], start):13 if re.search(pattern, l):14 print(f"[{s} L{i+1}] {l.strip()[:300]}")15 return i, l16 raise ValueError(pattern)1718def nums(l):19 return [float(x.replace("−", "-")) for x in re.findall(r"-?\d+\.\d+", l)]2021def pct(b): return 100 * (math.exp(b) - 1)2223print("=== Assertions ===")24for a in assertions: print(a["id"], a["cites"], a["text"])25print("\n=== Source lines relied on ===")2627# Table 628i6, _ = line(S1, r"^Table 6:")29i, att6 = line(S1, r"^ATT \|", i6); att6 = nums(att6)30_, se6 = line(S1, r"^\| \(", i); se6 = nums(se6)31_, pc6 = line(S1, r"^% change", i); pc6 = [float(x) for x in re.findall(r"-?\d+\.\d+", pc6)]32star5 = "**" in src[S1][i].split("|")[3]33line(S1, r"statistically significant at the 5% level")34# Table 735i7, _ = line(S1, r"^Table 7:")36i, hep = line(S1, r"^Hepatitis \|", i7); hep = nums(hep)37_, hepse = line(S1, r"^\| \(", i); hepse = nums(hepse)38i, hiv = line(S1, r"^HIV \|", i7); hiv = nums(hiv)39_, hivse = line(S1, r"^\| \(", i)40# Table A19 (consistency check)41iA, _ = line(S1, r"^Table A19")42i, _ = line(S1, r"^ATT \|", iA)43_, seA = line(S1, r"^\| \(", i); seA = nums(seA)44_, ctlA = line(S1, r"never-treated comparison regions \(\d+ regions\)", iA)45# abstract / mechanisms consistency46line(S1, r"fall by 35%")47line(S1, r"approximately 30% at 5 km")48line(S1, r"HIV estimates are insignificant and close to zero")49# sex / gender search50sex_hits = [l for l in src[S1] if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]51print(f"[{S1}] lines mentioning sex/gender: {len(sex_hits)}")52# RESEA Table 253_, ui = line(S2, r"^Weeks on UI \|")54cells = [c.strip() for c in ui.split("|")]55ctl_mean = float(re.search(r"([\d.]+) weeks", cells[2]).group(1))56ate = cells[4]; ate_b = float(re.search(r"(-?[\d.]+) weeks", ate).group(1))57ate_ci = [float(x) for x in re.search(r"\[(-?[\d.]+), (-?[\d.]+)\]", ate).groups()]58line(S2, r"11\.5% decrease")5960print("\n=== Checks ===")61v, loc, why = {}, {}, {}62# A163c1 = (att6[2] == -0.431 and se6[2] == 0.210 and star5 and pc6[2] == -35.0 and abs(pct(-0.431) + 35.0) < 0.05)64print(f"A1: ATT5={att6[2]} SE={se6[2]} star**={star5} %={pc6[2]} 100(exp(b)-1)={pct(att6[2]):.2f}")65v["A1"] = "supported" if c1 else "not_supported"66loc["A1"] = f'{S1} | Table 6 | row "ATT" | column "5 km"'67why["A1"] = ("Table 6 prints ATT -0.431** with SE (0.210) at 5 km and a -35.0% change (100*(exp(-0.431)-1) = -35.0%), "68 "and the text states significance at the 5% level, so every element matches.")69# A270print(f"A2: HIV 1 km ATT = {hiv[0]}")71v["A2"] = "supported" if hiv[0] < 0 else "not_supported"72loc["A2"] = f'{S1} | Table 7 | row "HIV" | column "1 km"'73why["A2"] = (f"The 1 km HIV ATT is printed as +{hiv[0]:.3f} (SE 0.801), positive and far from significant, "74 "so the claimed negative sign (-0.063) and the claimed fall in HIV hospitalisations are contradicted.")75# A376print(f"A3: 3 km ATT={att6[1]} printed %={pc6[1]} implied={pct(att6[1]):.2f}")77v["A3"] = "supported" if abs(pct(att6[1]) + 28.3) < 0.5 else "not_supported"78loc["A3"] = f'{S1} | Table 6 | row "% change" | column "3 km"'79why["A3"] = (f"The source converts the 3 km ATT of -0.283 to a {pc6[1]}% change (100*(exp(-0.283)-1) = {pct(att6[1]):.2f}%), "80 "not -28.3%, and the estimate is not statistically significant (SE 0.279), so reading log points as percent is wrong.")81# A482lo, hi = hep[2] - 1.96 * hepse[2], hep[2] + 1.96 * hepse[2]83lo2, hi2 = hep[2] - 2 * hepse[2], hep[2] + 2 * hepse[2]84print(f"A4: Hep 5 km ATT={hep[2]} SE={hepse[2]} CI1.96=[{lo:.3f},{hi:.3f}] CI2.0=[{lo2:.3f},{hi2:.3f}]")85v["A4"] = "supported" if (hi < 0 and abs(lo + 0.66) < 0.02 and abs(hi + 0.16) < 0.02) else "not_supported"86loc["A4"] = f'{S1} | Table 7 | row "Hepatitis" | column "5 km"'87why["A4"] = (f"With ATT {hep[2]} and SE ({hepse[2]:.3f}) the 95% CI is [{lo:.3f}, {hi:.3f}], which includes zero (no star; text calls it insignificant); "88 "the claimed [-0.66, -0.16] corresponds to +/-1 SE only.")89# A590print(f"A5: sex-specific estimates found: {len(sex_hits)}")91v["A5"] = "not_assessable" if not sex_hits else "REVIEW"92loc["A5"] = f"not reported in {S1}"93why["A5"] = ("The extract reports no estimates by sex (Tables 6, 7, A19 and Section 6.1 are pooled over the population), "94 "so a men-versus-women comparison can be neither confirmed nor refuted.")95# A696rel = 100 * abs(ate_b) / ctl_mean97print(f"A6: ATE={ate_b} CI={ate_ci} control mean={ctl_mean} relative={rel:.2f}%")98v["A6"] = "supported" if (ate_b == -1.99 and ate_ci == [-2.24, -1.75] and ctl_mean == 17.3 and abs(rel - 11.5) < 0.05) else "not_supported"99loc["A6"] = f'{S2} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'100why["A6"] = (f"Table 2 prints the ATE as -1.99 weeks with 95% CI [-2.24, -1.75] and a control mean of 17.3 weeks; "101 f"1.99/17.3 = {rel:.1f}%, matching the 11.5% stated in the results text.")102103data_issues = [104 {"issue": "Armendariz: the CS-DR 5 km ATT -0.431** has SE (0.210) in Table 6 but (0.218) in Table A19 for the same estimator, buffer and sample (17,250 obs).",105 "handling": "A1 judged against Table 6 (the table the assertion paraphrases); significance at 5% holds under either SE (|t| = 0.431/0.210 = 2.05 or 0.431/0.218 = 1.98, both > 1.96; starred ** in both tables), so the verdict does not change."},106 {"issue": "Armendariz: Table 6/7 notes state 608 never-treated control regions; Table A19 note states 640.",107 "handling": "Recorded; not decisive for any assertion."},108 {"issue": "Armendariz: abstract says fatal hospitalisations fall by 35%, but Table 7 5 km fatal ATT -0.354 implies -29.8%, matching Section 6.1's 'approximately 30%'.",109 "handling": "Recorded; not decisive for any assertion (none concerns fatal outcomes)."},110 {"issue": "Armendariz: text says HIV estimates are 'close to zero', while the 5 km HIV ATT is -0.340 (imprecise, SE 0.294).",111 "handling": "Recorded; A2 is decided by the printed 1 km coefficient (+0.063), not by the prose."},112 {"issue": "Li et al.: results text gives the weeks-on-UI CIs as positive magnitudes ([1.75, 2.24]) while Table 2 prints signed values [-2.24, -1.75]; a rendering artefact 'p ¡ 0.001' appears in the short-term paragraph.",113 "handling": "Treated as equivalent sign conventions for a reduction; Table 2 used as the locator for A6."},114]115116def wc(path): 117 with open(path, "rb") as f: return f.read().count(b"\n")118out = {119 "task_id": "F4-conf-02",120 "verdicts": v, "locators": loc, "reasons": why,121 "n_supported": sum(x == "supported" for x in v.values()),122 "n_not_supported": sum(x == "not_supported" for x in v.values()),123 "n_not_assessable": sum(x == "not_assessable" for x in v.values()),124 "feasibility": "complete", "feasibility_reason": None,125 "data_issues": data_issues,126 "n_input_rows": {"assertions.json": len(assertions), "README.md": wc(f"{D}/README.md"),127 f"{S1}.txt": wc(f"{D}/sources/{S1}.txt"), f"{S2}.txt": wc(f"{D}/sources/{S2}.txt")},128 "n_analysed": len(assertions),129 "random_seed": None,130}131os.makedirs("out", exist_ok=True)132json.dump(out, open("out/results.json", "w", encoding="utf-8"), indent=2, ensure_ascii=False)133print("\nVerdicts:", v)134