1"""F4-conf-02: verdicts on six assertions against their cited source extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import json, math, os, re45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7DCR = "arxiv-2609.27732v1"8RES = "arxiv-2606.14621v1"910def lines(p):11 with open(p, encoding="utf-8") as f:12 return f.read().split("\n")1314src = {DCR: lines(os.path.join(D, "sources", DCR + ".txt")),15 RES: lines(os.path.join(D, "sources", RES + ".txt"))}16with open(os.path.join(D, "assertions.json"), encoding="utf-8") as f:17 assertions = json.load(f)18A = {a["id"]: a for a in assertions}19for a in assertions:20 assert a["cites"] in src, a2122def wcl(p):23 with open(p, "rb") as f:24 return f.read().count(b"\n")2526n_input_rows = {27 "assertions.json": len(assertions),28 "README.md": wcl(os.path.join(D, "README.md")),29 DCR + ".txt": wcl(os.path.join(D, "sources", DCR + ".txt")),30 RES + ".txt": wcl(os.path.join(D, "sources", RES + ".txt")),31}3233def find(sid, pattern, start=0):34 """Return (index, line) of first line at/after start matching regex; print it."""35 for i in range(start, len(src[sid])):36 if re.search(pattern, src[sid][i]):37 print(f" [{sid} L{i+1}] {src[sid][i].strip()}")38 return i, src[sid][i]39 raise ValueError(f"pattern not found: {pattern}")4041def cells(line):42 return [c.strip() for c in line.split("|")]4344def num(s):45 return float(re.sub(r"[^0-9.\-]", "", s.replace("−", "-")))4647def pct(b):48 return 100.0 * (math.exp(b) - 1.0)4950def ci(b, se, z):51 return (b - z * se, b + z * se)5253def stars(s):54 return len(re.findall(r"\*", s))5556verdicts, locators, reasons, data_issues = {}, {}, {}, []57cols = {"1 km": 1, "3 km": 2, "5 km": 3}5859# ---------------- Table 6 (DCR) ----------------60print("Table 6 lines relied on:")61i6, _ = find(DCR, r"^Table 6:")62iatt, latt = find(DCR, r"^ATT \|", i6)63_, lse = find(DCR, r"^\| \(", iatt)64_, lpct = find(DCR, r"^% change \|", iatt)65t6 = {k: {"att_s": cells(latt)[v], "att": num(cells(latt)[v]),66 "se": num(cells(lse)[v]), "pct": num(cells(lpct)[v])} for k, v in cols.items()}67find(DCR, r"statistically significant at the 5% level", i6)6869# A170print("A1:", A["A1"]["text"])71r = t6["5 km"]72z = r["att"] / r["se"]73lo, hi = ci(r["att"], r["se"], 1.96)74ok = (r["att"] == -0.431 and r["se"] == 0.210 and stars(r["att_s"]) >= 275 and abs(z) > 1.96 and r["pct"] == -35.0 and round(pct(r["att"]), 1) == -35.0)76print(f" check: ATT={r['att']} SE={r['se']} stars={stars(r['att_s'])} |z|={abs(z):.3f} "77 f"CI95=[{lo:.3f},{hi:.3f}] printed%={r['pct']} 100(exp(b)-1)={pct(r['att']):.2f}")78verdicts["A1"] = "supported" if ok else "not_supported"79locators["A1"] = f'{DCR} | Table 6 | row "ATT" | column "5 km"'80reasons["A1"] = ("Table 6 prints the 5 km ATT as -0.431** (SE 0.210, ** = p<0.05) with a % change of -35.0% "81 "(100*(exp(-0.431)-1) = -35.0%), matching every element of the assertion.")8283# A384print("A3:", A["A3"]["text"])85r = t6["3 km"]86print(f" check: ATT={r['att']} printed%={r['pct']} 100(exp(b)-1)={pct(r['att']):.2f} vs asserted -28.3%")87ok = r["att"] == -0.283 and abs(pct(r["att"]) - (-28.3)) < 0.0588verdicts["A3"] = "supported" if ok else "not_supported"89locators["A3"] = f'{DCR} | Table 6 | row "% change" | column "3 km"'90reasons["A3"] = ("The ATT of -0.283 is in log points and Table 6 converts it to a -24.7% change "91 f"(100*(exp(-0.283)-1) = {pct(r['att']):.2f}%), not a 28.3% reduction; the estimate is also not "92 "significant (SE 0.279).")9394# ---------------- Table 7 (DCR) ----------------95print("Table 7 lines relied on:")96i7, _ = find(DCR, r"^Table 7:")97def t7row(label):98 i, l = find(DCR, r"^" + re.escape(label) + r" \|", i7)99 _, s = find(DCR, r"^\| \(", i)100 return {k: {"att_s": cells(l)[v], "att": num(cells(l)[v]), "se": num(cells(s)[v])} for k, v in cols.items()}101hiv = t7row("HIV")102hep = t7row("Hepatitis")103find(DCR, r"HIV estimates are insignificant and close to zero", i7)104105# A2106print("A2:", A["A2"]["text"])107r = hiv["1 km"]108print(f" check: HIV 1 km ATT={r['att_s']} (SE {r['se']}); asserted -0.063")109verdicts["A2"] = "supported" if r["att"] < 0 else "not_supported"110locators["A2"] = f'{DCR} | Table 7 | row "HIV" | column "1 km"'111reasons["A2"] = ("Table 7 prints the 1 km HIV ATT as 0.063, i.e. positive (SE 0.801), so the asserted negative sign "112 "and the claimed fall in HIV hospitalisations are contradicted.")113114# A4115print("A4:", A["A4"]["text"])116r = hep["5 km"]117c196 = ci(r["att"], r["se"], 1.96); c2 = ci(r["att"], r["se"], 2.0)118c1 = ci(r["att"], r["se"], 1.0); c1645 = ci(r["att"], r["se"], 1.645)119print(f" check: Hepatitis 5 km ATT={r['att_s']} SE={r['se']} CI95(1.96)=[{c196[0]:.3f},{c196[1]:.3f}] "120 f"CI95(2.0)=[{c2[0]:.3f},{c2[1]:.3f}] CI90=[{c1645[0]:.3f},{c1645[1]:.3f}] b±1SE=[{c1[0]:.3f},{c1[1]:.3f}]")121excl = c196[1] < 0 or c196[0] > 0122verdicts["A4"] = "supported" if excl else "not_supported"123locators["A4"] = f'{DCR} | Table 7 | row "Hepatitis" | column "5 km"'124reasons["A4"] = (f"With ATT -0.406 and SE 0.250 (no stars), the 95% CI is about [{c196[0]:.2f}, {c196[1]:.2f}], "125 "which includes zero; the asserted [-0.66, -0.16] is only the estimate ± 1 SE.")126127# A5128print("A5:", A["A5"]["text"])129hits = [i + 1 for i, l in enumerate(src[DCR]) if re.search(r"\b(men|women|male|female|sex|gender)\b", l, re.I)]130print(f" check: lines mentioning men/women/male/female/sex/gender in {DCR}: {hits}")131verdicts["A5"] = "not_assessable" if not hits else "MANUAL_REVIEW"132locators["A5"] = f"not reported in {DCR}"133reasons["A5"] = ("The extract reports no estimates by sex (Tables 6, 7 and A19 report only buffer, outcome-category and estimator breakdowns, none by sex), "134 "so a men-versus-women comparison can be neither confirmed nor refuted.")135136# ---------------- RESEA Table 2 ----------------137print("A6:", A["A6"]["text"])138_, l = find(RES, r"^Weeks on UI \|")139c = cells(l)140ctrl = num(re.match(r"([\d.]+) weeks", c[2]).group(1))141m = re.match(r"(-?[\d.]+) weeks; p<0\.001; 95% CI: \[(-?[\d.]+), (-?[\d.]+)\]", c[4])142ate, lo6, hi6 = (float(m.group(k)) for k in (1, 2, 3))143find(RES, r"11\.5% decrease")144rel = 100 * ate / ctrl145print(f" check: ATE={ate} CI=[{lo6},{hi6}] control mean={ctrl} ATE/control={rel:.2f}%")146ok = ate == -1.99 and (lo6, hi6) == (-2.24, -1.75) and ctrl == 17.3 and round(rel, 1) == -11.5147verdicts["A6"] = "supported" if ok else "not_supported"148locators["A6"] = f'{RES} | Table 2 | row "Weeks on UI" | column "Average treatment effect estimate"'149reasons["A6"] = ("Table 2 prints the ATE for weeks on UI as -1.99 weeks (95% CI [-2.24, -1.75]) with a control mean "150 f"of 17.3 weeks, and -1.99/17.3 = {rel:.1f}%, matching the text's 11.5% decrease.")151152# ---------------- internal-consistency scan (data issues) ----------------153print("Consistency checks on DCR extract:")154iA19, _ = find(DCR, r"^Table A19:")155ia, la = find(DCR, r"^ATT \|", iA19)156_, lsa = find(DCR, r"^\| \(", ia)157_, lpa = find(DCR, r"^% change \|", ia)158find(DCR, r"never-treated comparison regions \(640 regions\)", iA19)159se_a19 = num(cells(lsa)[1])160data_issues.append({161 "issue": (f"{DCR}: the CS-DR 5 km overall ATT -0.431** has SE 0.210 in Table 6 but SE {se_a19:.3f} in Appendix "162 "Table A19 (same estimator, buffer, window and N=17,250)."),163 "handling": ("A1 is judged on Table 6 (the cited main result); the 5% significance claim holds under both SEs "164 f"(|z| = {0.431/0.210:.2f} and {0.431/se_a19:.2f} > 1.96), so the verdict is unaffected.")})165data_issues.append({166 "issue": f"{DCR}: Table A19 lists 608 control regions but its note says 640 never-treated comparison regions.",167 "handling": "Not decisive for any assertion; recorded only."})168fatal5 = num(cells(find(DCR, r"^Drug-related \(fatal\) \|", i7)[1])[3])169data_issues.append({170 "issue": (f"{DCR}: the abstract says fatal outcomes fall by 35%, whereas Table 7 gives a 5 km fatal ATT of "171 f"{fatal5} (100*(exp(b)-1) = {pct(fatal5):.1f}%) and Section 6.1 says approximately 30%."),172 "handling": "Not decisive for any assertion (none concerns fatal outcomes); recorded only."})173for k, lab in [(3, "Synthetic DiD"), (4, "Stacked DiD")]:174 b = num(cells(la)[k]); p = num(cells(lpa)[k])175 print(f" A19 {lab}: b={b} printed%={p} computed={pct(b):.2f}")176data_issues.append({177 "issue": (f"{DCR}: Table A19 printed % changes for Synthetic DiD ({num(cells(lpa)[3])}%) and Stacked DiD "178 f"({num(cells(lpa)[4])}%) differ by 0.1 pp from 100*(exp(b)-1) "179 f"({pct(num(cells(la)[3])):.1f}%, {pct(num(cells(la)[4])):.1f}%), likely rounding from unrounded b."),180 "handling": "Not decisive for any assertion; recorded only."})181data_issues.append({182 "issue": (f"{RES}: the results text reports the weeks-on-UI CI as magnitudes ([1.75 weeks, 2.24 weeks]) while "183 "Table 2 prints signed bounds [-2.24, -1.75]; the text also contains a rendering defect 'p ¡ 0.001'."),184 "handling": "Treated as equivalent sign conventions; A6 judged on the signed Table 2 values."})185186# ---------------- naive baseline: share of asserted numbers found verbatim in the cited extract ----------------187baseline = {}188for a in assertions:189 toks = re.findall(r"\d+(?:\.\d+)?", a["text"].replace("−", "-"))190 text = "\n".join(src[a["cites"]])191 found = sum(1 for t in toks if t in text)192 baseline[a["id"]] = round(found / len(toks), 4) if toks else None193print("Baseline (share of asserted numeric tokens found verbatim in cited extract):", baseline)194195verdicts = {k: verdicts[k] for k in sorted(verdicts)}196locators = {k: locators[k] for k in sorted(locators)}197reasons = {k: reasons[k] for k in sorted(reasons)}198counts = {v: sum(1 for x in verdicts.values() if x == v) for v in ("supported", "not_supported", "not_assessable")}199results = {200 "task_id": "F4-conf-02",201 "verdicts": verdicts,202 "locators": locators,203 "reasons": reasons,204 "n_supported": counts["supported"],205 "n_not_supported": counts["not_supported"],206 "n_not_assessable": counts["not_assessable"],207 "feasibility": "complete",208 "feasibility_reason": None,209 "data_issues": data_issues,210 "n_input_rows": n_input_rows,211 "n_analysed": len(verdicts),212 "random_seed": None,213 "baseline_share_numbers_found_verbatim": baseline,214}215assert sum(counts.values()) == 6216os.makedirs(os.path.join(ROOT, "out"), exist_ok=True)217with open(os.path.join(ROOT, "out", "results.json"), "w", encoding="utf-8") as f:218 json.dump(results, f, indent=2, ensure_ascii=False)219 f.write("\n")220print("Verdicts:", verdicts)221print("Counts:", counts)222