1"""F4-conf-02: verdicts on six assertions against cited source extracts.2Reads data/ only; writes out/results.json deterministically. No randomness."""3import json, math, os, re45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7DCR, RES = "arxiv-2609.27732v1", "arxiv-2606.14621v1"89def lines(path):10 with open(path, encoding="utf-8") as f:11 return f.read().split("\n")1213def wc_l(path):14 with open(path, "rb") as f:15 return f.read().count(b"\n")1617assertions = json.load(open(os.path.join(D, "assertions.json"), encoding="utf-8"))18src = {s: lines(os.path.join(D, "sources", s + ".txt")) for s in (DCR, RES)}1920def find(s, pattern, start=0):21 """Return (index, text) of first line at/after `start` matching regex; print it."""22 for i, l in enumerate(src[s][start:], start):23 if re.search(pattern, l):24 print(f" [{s} L{i+1}] {l.strip()}")25 return i, l26 raise ValueError(f"pattern {pattern!r} not found in {s}")2728def cells(l):29 return [c.strip() for c in l.split("|")]3031def num(c):32 return float(re.sub(r"[^\d\.\-]", "", c.replace("−", "-")))3334def stars(c):35 return c.count("*")3637pct = lambda b: 100 * (math.exp(b) - 1)38cols = {"1 km": 1, "3 km": 2, "5 km": 3}3940# ---------- DCR Table 6 ----------41print("Table 6 (DCR):")42t6, _ = find(DCR, r"^Table 6:")43i, att = find(DCR, r"^ATT \|", t6); att = cells(att)44_, se = find(DCR, r"^\| \(", i); se = cells(se)45_, pc = find(DCR, r"^% change \|", i); pc = cells(pc)46find(DCR, r"statistically significant at the 5% level", i)47t6v = {k: dict(b=num(att[c]), st=stars(att[c]), se=num(se[c]), pct=num(pc[c])) for k, c in cols.items()}4849# ---------- DCR Table 7 ----------50print("Table 7 (DCR):")51t7, _ = find(DCR, r"^Table 7:")52t7v = {}53for row in ("Hepatitis", "HIV"):54 i, l = find(DCR, rf"^{row} \|", t7); l = cells(l)55 _, s = find(DCR, r"^\| \(", i); s = cells(s)56 t7v[row] = {k: dict(b=num(l[c]), st=stars(l[c]), se=num(s[c])) for k, c in cols.items()}5758# ---------- DCR A19 (consistency check only) ----------59print("Table A19 (DCR, consistency check):")60a19, _ = find(DCR, r"^Table A19:")61i, l = find(DCR, r"^ATT \|", a19)62_, s = find(DCR, r"^\| \(", i)63a19_se_csdr = num(cells(s)[1])64_, a19note = find(DCR, r"never-treated comparison regions \(\d+ regions\)", a19)65a19_controls = int(re.search(r"\((\d+) regions\)", a19note).group(1))6667# ---------- Sex/gender search (A5) ----------68print("Search for sex/gender breakdown in DCR extract:")69sex_hits = [k + 1 for k, l in enumerate(src[DCR]) if re.search(r"\b(men|women|male|female|males|females|sex|gender)\b", l, re.I)]70print(f" lines mentioning men/women/male/female/sex/gender: {sex_hits}")7172# ---------- RESEA Table 2 ----------73print("Table 2 (RESEA):")74_, w = find(RES, r"^Weeks on UI \|")75w = cells(w)76ctrl_mean = num(w[2].split("weeks")[0])77ate = w[4]78ate_b = float(re.search(r"(-?\d+\.\d+) weeks", ate).group(1))79ate_ci = [float(x) for x in re.search(r"CI: \[(-?[\d\.]+), (-?[\d\.]+)\]", ate).groups()]80find(RES, r"11\.5% decrease")8182# ---------- Verdict logic ----------83V, L, R, calc = {}, {}, {}, {}84a = {x["id"]: x for x in assertions}85assert all(a[k]["cites"] == DCR for k in ("A1", "A2", "A3", "A4", "A5")) and a["A6"]["cites"] == RES8687# A1: 5 km overall ATT -0.431 (SE 0.210), significant at 5%, 35.0% reduction88x = t6v["5 km"]89ok1 = (x["b"] == -0.431 and x["se"] == 0.210 and x["st"] >= 2 and x["pct"] == -35.090 and round(pct(x["b"]), 1) == -35.0)91calc["A1"] = dict(b=x["b"], se=x["se"], stars=x["st"], pct_printed=x["pct"], pct_implied=round(pct(x["b"]), 2),92 z=round(x["b"] / x["se"], 3))93V["A1"] = "supported" if ok1 else "not_supported"94L["A1"] = f'{DCR} | Table 6 | row "ATT" | column "5 km"'95R["A1"] = ("Table 6 prints ATT -0.431** (SE 0.210) and % change -35.0% at 5 km, and the text states significance at the 5% level, "96 "matching every element of the assertion." if ok1 else "Table 6 5 km values do not match the assertion.")9798# A2: HIV 1 km ATT negative (-0.063)99x = t7v["HIV"]["1 km"]100calc["A2"] = dict(b=x["b"], se=x["se"], stars=x["st"])101V["A2"] = "not_supported" if x["b"] > 0 else "supported"102L["A2"] = f'{DCR} | Table 7 | row "HIV" | column "1 km"'103R["A2"] = (f"Table 7 prints the 1 km HIV ATT as +{x['b']:.3f} (SE {x['se']:.3f}), positive and insignificant, so the claimed negative "104 "sign and the claimed fall in HIV hospitalisations are contradicted.")105106# A3: 3 km ATT -0.283 implies 28.3% reduction107x = t6v["3 km"]108calc["A3"] = dict(b=x["b"], pct_printed=x["pct"], pct_implied=round(pct(x["b"]), 2))109ok3 = abs(pct(x["b"]) - (-28.3)) < 0.05110V["A3"] = "supported" if ok3 else "not_supported"111L["A3"] = f'{DCR} | Table 6 | row "% change" | column "3 km"'112R["A3"] = (f"The ATT of {x['b']:.3f} log points corresponds to a {x['pct']:.1f}% change (printed in Table 6; "113 f"100(exp({x['b']:.3f})-1) = {pct(x['b']):.2f}%), not a 28.3% reduction, which wrongly reads log points as percent.")114115# A4: hepatitis 5 km 95% CI approx [-0.66, -0.16], excludes zero116x = t7v["Hepatitis"]["5 km"]117ci196 = [round(x["b"] - 1.96 * x["se"], 3), round(x["b"] + 1.96 * x["se"], 3)]118ci200 = [round(x["b"] - 2.0 * x["se"], 3), round(x["b"] + 2.0 * x["se"], 3)]119ci100 = [round(x["b"] - x["se"], 3), round(x["b"] + x["se"], 3)]120calc["A4"] = dict(b=x["b"], se=x["se"], stars=x["st"], ci95_1p96=ci196, ci95_2p00=ci200, ci_pm_1se=ci100)121excl = ci196[1] < 0 or ci196[0] > 0122V["A4"] = "supported" if excl else "not_supported"123L["A4"] = f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"'124R["A4"] = (f"With ATT {x['b']:.3f} and SE {x['se']:.3f} the 95% CI is [{ci196[0]:.3f}, {ci196[1]:.3f}] (±1.96·SE), which includes zero "125 "(the estimate carries no star); the asserted interval is only ±1 SE.")126127# A5: larger reduction for men than women128calc["A5"] = dict(sex_lines=sex_hits)129V["A5"] = "not_assessable" if not sex_hits else "REVIEW"130L["A5"] = f"not reported in {DCR}"131R["A5"] = ("The extract reports ATTs only by buffer (Table 6), outcome category (Table 7) and estimator (Table A19) and never "132 "by sex, so the men-versus-women comparison can be neither confirmed nor refuted.")133134# A6: ATE -1.99 weeks, CI [-2.24,-1.75], 11.5% of control mean 17.3135rel = 100 * abs(ate_b) / ctrl_mean136calc["A6"] = dict(ate=ate_b, ci=ate_ci, control_mean=ctrl_mean, rel_pct=round(rel, 2))137ok6 = ate_b == -1.99 and ate_ci == [-2.24, -1.75] and ctrl_mean == 17.3 and round(rel, 1) == 11.5138V["A6"] = "supported" if ok6 else "not_supported"139L["A6"] = f'{RES} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'140R["A6"] = (f"Table 2 prints the ATE as -1.99 weeks (95% CI [-2.24, -1.75]) with a control mean of 17.3 weeks, and "141 f"1.99/17.3 = {rel:.2f}% matches the 11.5% decrease stated in the results text.")142143# ---------- Data issues ----------144issues = []145if a19_se_csdr != t6v["5 km"]["se"]:146 issues.append(dict(issue=f"{DCR}: the same CS-DR 5 km ATT (-0.431**) has SE {t6v['5 km']['se']:.3f} in Table 6 but "147 f"{a19_se_csdr:.3f} in Table A19 (internal inconsistency).",148 handling="A1 cites Table 6, whose SE (0.210) matches the assertion; verdict uses Table 6. With SE 0.218, "149 f"|z| = {0.431/a19_se_csdr:.3f}: still significant at 5% under 1.96, borderline under 2.0; A1 verdict unchanged because the printed ** appears in both tables."))150if a19_controls != 608:151 issues.append(dict(issue=f"{DCR}: Table A19 note says {a19_controls} never-treated comparison regions while the Table 6/A19 rows and the Figure 6 note say 608.",152 handling="Not decisive for any assertion; recorded only."))153issues.append(dict(issue=f"{DCR}: narrative percentages for sub-outcomes are inconsistent: abstract says fatal hospitalisations fall by 35% while Section 6.1 says ~30% at 5 km "154 f"(100(exp(-0.354)-1) = {pct(-0.354):.1f}%); Section 6.1 says narcotics +47% at 5 km (implied {pct(0.390):.1f}%); text calls HIV estimates 'close to zero' although the 5 km HIV ATT is -0.340.",155 handling="Verdicts rely on the printed table coefficients, not the narrative; none of these is decisive for A1-A6."))156issues.append(dict(issue=f"{RES}: results text reports the weeks-on-UI CIs as positive magnitudes ([1.75 weeks, 2.24 weeks]) for a 'reduction', while Table 2 prints signed [-2.24, -1.75]; also a rendering defect 'p ¡ 0.001' for p < 0.001.",157 handling="Treated as equivalent sign conventions; A6 judged against the signed Table 2 values."))158issues.append(dict(issue="Both extracts are partial (sections omitted), and RESEA Tables 2-3 were re-set by the extract editor.",159 handling="Verdicts rely only on the extract content as instructed; values in re-set tables are taken as verbatim per the editorial note."))160161ids = [x["id"] for x in assertions]162163# ---------- Naive baseline (P2): verbatim number matching ----------164# 'supported' if every number in the assertion (sign dropped) occurs verbatim in the cited extract, else 'not_supported'.165baseline = {}166for x in assertions:167 nums = re.findall(r"\d+\.\d+|\d+", x["text"].replace("−", "-"))168 nums = [n for n in nums if n not in ("1", "3", "5", "95")] # buffer sizes / CI level are labels169 body = "\n".join(src[x["cites"]])170 baseline[x["id"]] = "supported" if all(n in body for n in nums) else "not_supported"171print("Naive verbatim-number baseline:", json.dumps(baseline))172173res = dict(174 task_id="F4-conf-02",175 verdicts={k: V[k] for k in ids},176 locators={k: L[k] for k in ids},177 reasons={k: R[k] for k in ids},178 n_supported=sum(V[k] == "supported" for k in ids),179 n_not_supported=sum(V[k] == "not_supported" for k in ids),180 n_not_assessable=sum(V[k] == "not_assessable" for k in ids),181 feasibility="complete",182 feasibility_reason=None,183 data_issues=issues,184 n_input_rows={"assertions.json": len(assertions), "README.md": wc_l(os.path.join(D, "README.md")),185 f"{DCR}.txt": wc_l(os.path.join(D, "sources", DCR + ".txt")),186 f"{RES}.txt": wc_l(os.path.join(D, "sources", RES + ".txt"))},187 n_analysed=len(ids),188 random_seed=None,189 supporting_calculations=calc,190 baseline_verdicts_verbatim_number_match=baseline,191 baseline_agreement_with_verdicts=sum(baseline[k] == V[k] for k in ids),192)193os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)194with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:195 json.dump(res, f, indent=2, ensure_ascii=False)196print("\nVerdicts:", json.dumps(res["verdicts"]))197print("Counts: supported", res["n_supported"], "| not_supported", res["n_not_supported"], "| not_assessable", res["n_not_assessable"])198