1"""F4-conf-04: verify five references against source extracts. Reads data/ only; writes out/results.json.2No randomness (random_seed = null)."""3import json, os, re, unicodedata45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7OUT = os.path.join(ROOT, "out")89def wc_l(path):10 with open(path, "rb") as f:11 return f.read().count(b"\n")1213refs = json.load(open(os.path.join(D, "reference_list.json"), encoding="utf-8"))14n_input_rows = {"README.md": wc_l(os.path.join(D, "README.md")), "reference_list.json": len(refs)}15src = {}16for r in refs:17 p = os.path.join(D, r["source_file"])18 src[r["id"]] = open(p, encoding="utf-8").read().splitlines()19 n_input_rows[os.path.basename(p)] = wc_l(p)2021def norm(s):22 s = unicodedata.normalize("NFKD", s)23 s = "".join(c for c in s if not unicodedata.combining(c)).lower().replace("&", "and")24 return re.sub(r"[^a-z0-9]+", " ", s).strip()2526def header(lines, key):27 for ln in lines:28 if ln.startswith(key + ":"):29 return ln[len(key) + 1:].strip()30 raise KeyError(key)3132def find_line(lines, pattern):33 """Return (lineno, line) of first line matching regex pattern; raises if absent."""34 for i, ln in enumerate(lines, 1):35 if re.search(pattern, ln):36 return i, ln37 raise ValueError("pattern not found: " + pattern)3839def show(rid, lines, pattern, nxt=0):40 i, ln = find_line(lines, pattern)41 print(f" [{rid} L{i}] {ln.strip()[:200]}")42 out = [ln]43 for k in range(1, nxt + 1):44 # skip blank lines to fetch the following non-empty line (SE row)45 j = i46 while True:47 j += 148 if lines[j - 1].strip():49 break50 print(f" [{rid} L{j}] {lines[j-1].strip()[:200]}")51 out.append(lines[j - 1])52 return out5354def cells(line):55 return [c.strip() for c in line.split("|")]5657checks, baseline = {}, []58for r in refs:59 rid, L = r["id"], src[r["id"]]60 print(f"== {rid}: {r['source_file']}")61 title = header(L, "Title"); bib = header(L, "Bibliographic record")62 print(" Title:", title); print(" Bibliographic record:", bib)63 # body title line must agree with header title64 body_title = [ln[2:].strip() for ln in L if ln.startswith("# ")][0]65 assert norm(body_title) == norm(title), rid66 # entry title = text between "(YYYY). " and ". arXiv:"67 m = re.match(r"^(.*?)\((\d{4})\)\.\s+(.*)\.\s+arXiv:(\d{4}\.\d{5})\.?$", r["entry"])68 entry_year, entry_title, entry_arxiv = int(m.group(2)), m.group(3), m.group(4)69 assert entry_arxiv in bib, rid70 years = sorted({int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", bib)})71 title_ok = norm(entry_title) == norm(title)72 year_ok = entry_year in years # years = first submission year and reproduced-version year (and any intermediate)73 print(f" entry title match={title_ok}; entry year {entry_year} vs submission years {years} -> {year_ok}")74 checks[rid] = {"title_matches": title_ok, "year_matches": year_ok}75 if not (title_ok and year_ok):76 baseline.append(rid)7778# ---------------- finding checks (element by element) ----------------79fs = {}80# R1: Table 6, 5 km column81L = src["R1"]82att, se = show("R1", L, r"^ATT \| -0\.399", 1)83pc = show("R1", L, r"^% change \| -32\.9%")[0]84show("R1", L, r"statistically significant at the 5% level")85show("R1", L, r"staggered introduction of DCRs")86a5, s5, p5 = cells(att)[3], cells(se)[3], cells(pc)[3]87e = {"ATT -0.431": a5.startswith("-0.431"),88 "SE 0.210": s5 == "(0.210)",89 "5% significance (** marker)": a5.endswith("**") and not a5.endswith("***"),90 "roughly 35%": p5 == "-35.0%",91 "within 5 km": "within 5 km of a facility" in "\n".join(L),92 "staggered DiD on Swiss hospital records": bool(re.search(r"staggered introduction of DCRs across Swiss cities.*hospital records and a difference-in-differences", "\n".join(L)))}93print(" R1 elements:", e)94fs["R1"] = (all(e.values()), "arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km",95 "Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, and the abstract states a staggered DiD on Swiss hospital records, so every element is supported (Table A19 prints SE 0.218 for the same specification, recorded as a data issue).")9697# R2: Table 2, Average treatment effect column98L = src["R2"]; T = "\n".join(L)99w = cells(show("R2", L, r"^Wage \| \$33,073")[0])[4]100re_ = cells(show("R2", L, r"^Reemployment \| 75\.2%")[0])[4]101ui = cells(show("R2", L, r"^Weeks on UI \| 15\.2")[0])[4]102show("R2", L, r"N = 23,549")103e = {"preregistered RCT N = 23,549": "large (N = 23,549) preregistered randomized controlled trial" in T,104 "wage +$1,153": w.startswith("$1,153"), "95% CI [166, 2,141]": "95% CI: [166, 2,141]" in w,105 "reemployment +1.47 pp": re_.startswith("1.47 pp"), "weeks on UI -1.99": ui.startswith("-1.99 weeks"),106 "annualised wages": "$1,153 higher annually" in T}107print(" R2 elements:", e)108fs["R2"] = (all(e.values()), "arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI",109 "Table 2's ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives the preregistered RCT with N = 23,549.")110111# R3: timing claim vs Section 3.2 coefficients112L = src["R3"]; T = "\n".join(L)113line = show("R3", L, r"stacked coefficient is 0\.005 at")[0]114row = cells(show("R3", L, r"^Clean-onset threshold-2 \|")[0])115coef = {int(k): float(v) for v, k in re.findall(r"(\d\.\d{3}) (?:at|by) \\tau=(\d)", line)}116print(" R3 stacked coefficients by tau:", coef, "| Table 3 preferred tau=4,5:", row[4], row[5])117e = {"3.4-3.7% jump in event year (tau=0 coef >= 0.034)": coef[0] >= 0.034,118 "no further appreciation after event year": max(coef.values()) <= coef[0],119 "3.4-3.7% magnitude exists somewhere (tau=4/5)": float(row[4]) == 0.037 and float(row[5]) == 0.034}120print(" R3 elements:", e)121fs["R3"] = (e["3.4-3.7% jump in event year (tau=0 coef >= 0.034)"] and e["no further appreciation after event year"],122 "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) and Table 3, row Clean-onset threshold-2, columns tau=4 / tau=5",123 "The 3.4-3.7% magnitude is reached only at tau=4-5 (0.037, 0.034) while the event-year coefficient is 0.005, so the claimed immediate jump with no later appreciation contradicts the source; title and year (2024 vs 2026) also disagree.")124125# R4: Appendix A.1 Table 2126L = src["R4"]; T = "\n".join(L)127em = show("R4", L, r"^Employment elasticity w\.r\.t MW", 1)128ow = show("R4", L, r"^Emp\. elasticity w\.r\.t affected wage", 1)129ae = show("R4", L, r"^Affected employment", 1); aw = show("R4", L, r"^Affected wages", 1)130e = {"MW elasticity -0.388": cells(em[0])[1] == "-0.388", "SE 0.132": cells(em[1])[1] == "(0.132)",131 "own-wage elasticity -0.732": cells(ow[0])[1] == "-0.732", "SE 0.091": cells(ow[1])[1] == "(0.091)",132 "Japan's spot labour market": "Japan’s spot labor market" in T}133ratio = float(cells(ae[0])[1]) / float(cells(aw[0])[1])134print(f" R4 elements: {e}; arithmetic check affected employment/affected wages = {ratio:.4f}")135fs["R4"] = (all(e.values()), "arxiv-2505.04555v3 | Appendix A.1 Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",136 "Table 2 prints -0.388 (0.132) and -0.732 (0.091) and the abstract places the study in Japan's spot labor market; the 0.091 SE duplicates the 'Affected employment' SE, recorded as a data issue.")137138# R5: Table 3 column (4)139L = src["R5"]; T = "\n".join(L)140tr = show("R5", L, r"^Treatment \| -0\.0911", 1)141pch = cells(show("R5", L, r"^% Change \|")[0]); mdv = cells(show("R5", L, r"^Mean Dep\. Var\. \|")[0])142show("R5", L, r"most preferred specification \(Column \(4\)\)")143arith = float(cells(tr[0])[4]) / float(mdv[4])144e = {"-0.0962": cells(tr[0])[4] == "-0.0962", "SE 0.0129": cells(tr[1])[4] == "(0.0129)",145 "preferred specification = col (4)": "most preferred specification (Column (4))" in T,146 "36.85% relative to mean": pch[4] == "-0.3685" and "percentage change of 36.85 percent" in T,147 "infant death or stillbirth": "Probability of Infant death or stillbirth" in T}148print(f" R5 elements: {e}; arithmetic -0.0962/0.261 = {arith:.4f}")149fs["R5"] = (all(e.values()), "arxiv-2510.03658v1 | Table 3, row Treatment and row % Change, column (4); Section 4.1",150 "Table 3 column (4), named the preferred specification in Section 4.1, prints -0.0962 (0.0129) and % Change -0.3685, matching the 36.85% stated in the text.")151152for rid, (ok, loc, note) in fs.items():153 checks[rid].update({"finding_supported": bool(ok), "locator": loc})154 checks[rid]["note"] = note155unver = [rid for rid in sorted(checks) if not all(checks[rid][k] for k in ("title_matches", "year_matches", "finding_supported"))]156157data_issues = [158 {"issue": "arxiv-2609.27732v1: Table 6 prints SE (0.210) for the 5 km ATT -0.431 while Table A19 (CS-DR column, same 5 km specification) prints (0.218); Table 6 gives 608 control regions, the A19 note says 640.",159 "handling": "R1's cited SE 0.210 matches the main-results Table 6, which is used as locator; the 5% significance claim holds under either SE (** marker in both tables). Recorded, not treated as a failure."},160 {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes fall by 35%, Section 6.1 says about 30% (Table 7 5 km: -0.354).",161 "handling": "Not relevant to R1's cited finding; recorded only."},162 {"issue": "arxiv-2505.04555v3: SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment' (0.091), despite the elasticity being the ratio -0.268/0.366; note says 'about 4%' MW increase vs 0.047 in the table.",163 "handling": "R4's cited values are printed verbatim in Table 2, so the finding check passes; the possibly duplicated SE is flagged as an open issue."},164 {"issue": "arxiv-2510.03658v1: Table 3 '% Change' col (4) is -0.3685 while -0.0962/0.261 = -0.3686; the abstract's headline (0.0855, 18.79%) refers to a different outcome (Table 2, not in extract).",165 "handling": "Rounding difference ignored (text explicitly states 36.85%); abstract figures not used for R5."},166 {"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering.",167 "handling": "Used the extract editor's column names (tau=4, tau=5 coefficients in the last two columns)."},168 {"issue": "arxiv-2606.14621v1: Tables 2-3 re-set by the extract editor; 'p ¡ 0.001' glyph defect in Section 3 text.",169 "handling": "Used the re-set tables as documented; glyph defect does not affect any checked value."},170]171172results = {173 "reference_checks": checks,174 "unverifiable_references": unver,175 "n_verified": len(checks) - len(unver),176 "task_id": "F4-conf-04",177 "feasibility": "complete",178 "feasibility_reason": None,179 "data_issues": data_issues,180 "n_input_rows": n_input_rows,181 "n_analysed": len(checks),182 "random_seed": None,183 "baseline_unverifiable_title_year_only": baseline,184 "baseline_n_verified_title_year_only": len(checks) - len(baseline),185}186os.makedirs(OUT, exist_ok=True)187with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:188 json.dump(results, f, indent=2, ensure_ascii=False)189print("unverifiable_references:", unver, "| n_verified:", results["n_verified"])190