1"""F4-conf-04: verify reference list against source extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import hashlib, json, os, re, unicodedata, math45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7OUT = os.path.join(ROOT, "out", "results.json")89def read(p):10 with open(os.path.join(D, p), encoding="utf-8") as f:11 return f.read()1213def nlines(p):14 with open(os.path.join(D, p), "rb") as f:15 return f.read().count(b"\n")1617refs = json.loads(read("reference_list.json"))18n_input_rows = {"README.md": nlines("README.md"), "reference_list.json": len(refs)}19src = {}20for r in refs:21 sf = r["source_file"]22 txt = read(sf)23 src[r["id"]] = txt24 n_input_rows[sf] = nlines(sf)25for k in sorted(n_input_rows):26 with open(os.path.join(D, k), "rb") as f:27 print("sha256", k, hashlib.sha256(f.read()).hexdigest(), "rows", n_input_rows[k])2829def norm_title(s):30 s = unicodedata.normalize("NFKD", s)31 s = "".join(c for c in s if not unicodedata.combining(c)).lower()32 s = s.replace("&", " and ").replace("’", "'")33 s = re.sub(r"[^a-z0-9 ]", " ", s)34 return " ".join(s.split())3536def entry_title_year(entry):37 m = re.match(r"^.*?\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", entry)38 return int(m.group(1)), m.group(2), m.group(3)3940def header(txt):41 title = re.search(r"^Title: (.*)$", txt, re.M).group(1).strip()42 bib = re.search(r"^Bibliographic record: (.*)$", txt, re.M).group(1).strip()43 body_title = re.search(r"^# (.*)$", txt, re.M).group(1).strip()44 years = [int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", bib)]45 arxiv = re.search(r"arXiv:(\d{4}\.\d{5})v\d", bib).group(1)46 return title, body_title, bib, years, arxiv4748def line_with(txt, start):49 for ln in txt.splitlines():50 if ln.strip().startswith(start):51 return ln.strip()52 raise KeyError(start)5354def cells(txt, start):55 return [c.strip() for c in line_with(txt, start).split("|")]5657def se_line_after(txt, start):58 lines = [l for l in txt.splitlines() if l.strip()]59 for i, l in enumerate(lines):60 if l.strip().startswith(start):61 return [c.strip() for c in lines[i + 1].split("|")]62 raise KeyError(start)6364# ---------------- title / year checks ----------------65meta = {}66for r in refs:67 ey, et, ea = entry_title_year(r["entry"])68 ht, bt, bib, years, ha = header(src[r["id"]])69 print(f"\n[{r['id']}] Title: {ht}\n[{r['id']}] Body title: # {bt}\n[{r['id']}] Bibliographic record: {bib}")70 print(f"[{r['id']}] entry title: {et} | entry year: {ey} | entry arXiv: {ea}")71 tm = norm_title(et) == norm_title(ht) and norm_title(ht) == norm_title(bt)72 # first submission year and year of the reproduced version (last 'submitted' date in the record)73 first_y, version_y = years[0], years[-1]74 ym = ey in (first_y, version_y)75 meta[r["id"]] = dict(title_matches=tm, year_matches=ym, entry_year=ey, first_year=first_y,76 version_year=version_y, arxiv_matches=(ea == ha))77 print(f"[{r['id']}] title_matches={tm} year_matches={ym} (first={first_y}, version={version_y}) arxiv_id_matches={ea == ha}")7879# ---------------- finding checks (cell-level evidence) ----------------80fin = {}8182# R1 Armendariz, Table 6 5 km column83t = src["R1"]84att = cells(t, "ATT | -0.399"); se = se_line_after(t, "ATT | -0.399"); pc = cells(t, "% change | -32.9%")85a19 = cells(t, "ATT | -0.431**"); a19se = se_line_after(t, "ATT | -0.431**")86print("\n[R1] Table 6 ATT row:", att, "SE row:", se, "% row:", pc)87print("[R1] Table A19 ATT row:", a19, "SE row:", a19se)88r1 = (att[3] == "-0.431**" and se[3] == "(0.210)" and pc[3] == "-35.0%"89 and abs(math.exp(-0.431) - 1 + 0.350) < 0.000590 and "staggered introduction of DCRs" in t and "individual-level hospital records" in t91 and "statistically significant at the 5% level" in t and "within 5 km of a facility" in t)92print(f"[R1] exp(-0.431)-1 = {math.exp(-0.431)-1:.4f}; finding_supported={r1}")93fin["R1"] = dict(finding_supported=r1,94 locator="arxiv-2609.27732v1 | Table 6, row ATT (SE row and % change row), column 5 km; abstract; Section 6 text after Table 6",95 note="Title and year match; Table 6 5 km column prints ATT -0.431** (0.210), -35.0%, significant at 5%, staggered DiD on individual-level Swiss hospital records, as cited (Table A19 prints SE 0.218 for the same estimate, which does not change the 5% significance).")9697# R2 RESEA, Table 2 ATE column98t = src["R2"]99w = cells(t, "Wage | $33,073"); re_ = cells(t, "Reemployment | 75.2%"); ui = cells(t, "Weeks on UI | 15.2")100print("\n[R2] Table 2 rows:\n ", w, "\n ", re_, "\n ", ui)101r2 = (w[4] == "$1,153; p=0.022; 95% CI: [166, 2,141]" and re_[4].startswith("1.47 pp")102 and ui[4].startswith("-1.99 weeks") and "(N = 23,549) preregistered randomized controlled trial" in t)103print(f"[R2] finding_supported={r2}")104fin["R2"] = dict(finding_supported=r2,105 locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, Reemployment, Weeks on UI; abstract (N = 23,549, preregistered RCT)",106 note="Title and year match; Table 2 ATE column gives $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549, exactly as cited.")107108# R3 Liu & Zhao, stacked event-study path109t = src["R3"]110co = cells(t, "Clean-onset threshold-2")111path = re.search(r"the stacked coefficient is (.*?)\. Interpreted", t).group(1)112print("\n[R3] Table 3 preferred row:", co)113print("[R3] Section 3.2 stacked path:", path)114tau0 = float(re.search(r"([\d.]+) at \\tau=0", path).group(1))115tau4, tau5 = float(co[4]), float(co[5])116# cited: 3.4-3.7% jump in the entry year, no further appreciation later117r3 = (tau0 >= 0.034) and (tau4 <= tau0 + 1e-9)118print(f"[R3] tau=0 coefficient {tau0} vs cited 0.034-0.037 jump; tau=4 {tau4}, tau=5 {tau5}; finding_supported={r3}")119fin["R3"] = dict(finding_supported=r3,120 locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) and Table 3, row 'Clean-onset threshold-2', columns tau=4 and tau=5",121 note="Title differs entirely and 2024 matches neither 2026 submission; the extract reports ~0.5% at the event year (0.005 at tau=0) rising gradually to 3.4-3.7% by years four-five (0.037, 0.034), contradicting the cited immediate jump with no later appreciation.")122123# R4 Kanayama et al., Appendix Table 2124t = src["R4"]125e1 = cells(t, "Employment elasticity w.r.t MW"); e1se = se_line_after(t, "Employment elasticity w.r.t MW")126e2 = cells(t, "Emp. elasticity w.r.t affected wage"); e2se = se_line_after(t, "Emp. elasticity w.r.t affected wage")127ae = cells(t, "Affected employment"); aese = se_line_after(t, "Affected employment"); aw = cells(t, "Affected wages")128print("\n[R4] Table 2:", e1, e1se, e2, e2se, "| Affected employment", ae, aese, "| Affected wages", aw)129r4 = (e1[1] == "-0.388" and e1se[1] == "(0.132)" and e2[1] == "-0.732" and e2se[1] == "(0.091)"130 and "Japan’s spot labor market" in t)131print(f"[R4] -0.268/0.366 = {-0.268/0.366:.4f}; finding_supported={r4}")132fin["R4"] = dict(finding_supported=r4,133 locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (Estimate (SE) column)",134 note="Title matches and 2025 equals the v1 year; Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market, as cited (the 0.091 SE duplicates the 'Affected employment' SE, flagged as a source issue).")135136# R5 Tossou, Table 3 column (4)137t = src["R5"]138tr = cells(t, "Treatment | -0.0911"); trse = se_line_after(t, "Treatment | -0.0911")139pc = cells(t, "% Change"); mn = cells(t, "Mean Dep. Var.")140print("\n[R5] Table 3:", tr, trse, pc, mn)141ratio = float(tr[4]) / float(mn[4])142r5 = (tr[4] == "-0.0962" and trse[4] == "(0.0129)" and pc[4] == "-0.3685"143 and abs(ratio + 0.3685) < 0.0005 and "most preferred specification (Column (4))" in t144 and "36.85 percent" in t)145print(f"[R5] -0.0962/0.261 = {ratio:.5f}; finding_supported={r5}")146fin["R5"] = dict(finding_supported=r5,147 locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment (SE), % Change, Mean Dep. Var.; Section 4.1 text ('most preferred specification (Column (4))')",148 note="Title matches and 2025 equals the submission year; Table 3 column (4) gives -0.0962 (0.0129) on infant death or stillbirth, -36.85% relative to the mean 0.261 (preferred specification), as cited.")149150# ---------------- assemble ----------------151checks = {}152for r in refs:153 i = r["id"]154 checks[i] = {"title_matches": bool(meta[i]["title_matches"]), "year_matches": bool(meta[i]["year_matches"]),155 "finding_supported": bool(fin[i]["finding_supported"]),156 "locator": fin[i]["locator"], "note": fin[i]["note"]}157unver = [i for i in checks if not all(checks[i][k] for k in ("title_matches", "year_matches", "finding_supported"))]158159# Baseline: naive check = header title/year + every number in cited_for appears verbatim in the extract160def nums(s):161 s = s.replace("−", "-").replace("–", "-")162 return re.findall(r"\d[\d,]*\.?\d*", s)163baseline = {}164for r in refs:165 t = src[r["id"]].replace("−", "-").replace("–", "-")166 ok = all(n.rstrip(".,") in t for n in nums(r["cited_for"]))167 baseline[r["id"]] = bool(meta[r["id"]]["title_matches"] and meta[r["id"]]["year_matches"] and ok)168baseline_unver = [i for i, v in baseline.items() if not v]169print("\nBaseline (header title/year + naive number-presence) unverifiable:", baseline_unver)170171data_issues = [172 {"issue": "arxiv-2609.27732v1 (R1): Table A19 prints SE (0.218) for the CS-DR 5 km ATT -0.431 whose SE is (0.210) in Table 6; Table A19 note says 640 never-treated regions while its body and Tables 6-7 say 608.",173 "handling": "Used Table 6 (main results table, the one the citation reproduces); the 5% significance holds under either SE (0.431/0.218 = 1.98). Finding judged supported; discrepancy noted."},174 {"issue": "arxiv-2609.27732v1 (R1): abstract says fatal outcomes fall by 35% whereas Section 6.1 says approximately 30% (Table 7 fatal 5 km coefficient -0.354).",175 "handling": "Not part of the cited finding; recorded only."},176 {"issue": "arxiv-2505.04555v3 (R4): Table 2 SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment', although the elasticity is the ratio -0.268/0.366; a ratio's SE equal to the numerator's SE is implausible and may be a typesetting duplication.",177 "handling": "The cited value matches the printed cell exactly, so the finding is judged supported per the task rule ('stated in the extract'); the reliability of that SE is flagged as an open issue."},178 {"issue": "arxiv-2510.03658v1 (R5): abstract headline (0.0855, 18.79%, mean 0.455, from omitted Table 2) differs from the Table 3 preferred estimate (0.0962, 36.85%, mean 0.261); Table 3 '% Change' -0.3685 vs -0.0962/0.261 = -0.36858; footnote markers rendered as '^9^9 9'.",179 "handling": "Different outcome definitions (count vs binary indicator) per Section 4.1; citation explicitly refers to the preferred specification of the binary outcome, so Table 3 col (4) decides. Last-digit difference treated as truncation."},180 {"issue": "arxiv-2606.14621v1 (R2): Tables 2-3 re-set by the extract editor; 'p ¡ 0.001' rendering defect. arxiv-2603.03260v2 (R3): Table 3 header row garbled.",181 "handling": "Relied on the editor's declared column order; no numbers affected."},182]183184results = {185 "task_id": "F4-conf-04",186 "reference_checks": checks,187 "unverifiable_references": unver,188 "n_verified": len(refs) - len(unver),189 "feasibility": "complete",190 "feasibility_reason": None,191 "data_issues": data_issues,192 "n_input_rows": n_input_rows,193 "n_analysed": len(refs),194 "random_seed": None,195 "baseline_unverifiable_references": baseline_unver,196 "baseline_n_verified": len(refs) - len(baseline_unver),197}198os.makedirs(os.path.dirname(OUT), exist_ok=True)199with open(OUT, "w", encoding="utf-8") as f:200 json.dump(results, f, indent=2, ensure_ascii=False, sort_keys=True)201print("\nunverifiable_references:", unver, "n_verified:", results["n_verified"])202