"""F4-conf-04: verify five references against the provided arXiv extracts. Reads only data/; writes out/results.json deterministically. No randomness.""" import hashlib, json, math, re, unicodedata from pathlib import Path ROOT = Path(__file__).resolve().parent.parent DATA = ROOT / "data" OUT = ROOT / "out" def lines_of(p): return p.read_text(encoding="utf-8").splitlines() def wc_l(p): return p.read_bytes().count(b"\n") def norm_title(t): t = unicodedata.normalize("NFKD", t) t = "".join(c for c in t if not unicodedata.combining(c)).lower() t = t.replace("&", " and ") t = re.sub(r"[^a-z0-9 ]", " ", t) return " ".join(t.split()) def find_line(lines, pattern, show=True): """Return the first line matching regex; print it as evidence.""" for i, l in enumerate(lines): if re.search(pattern, l): if show: print(f" [line {i+1}] {l.strip()[:220]}") return i, l raise AssertionError(f"pattern not found: {pattern}") def row_after(lines, i): """Next non-empty line (SE row in regression tables).""" j = i + 1 while not lines[j].strip(): j += 1 print(f" [line {j+1}] {lines[j].strip()[:220]}") return lines[j] def cells(l): return [c.strip() for c in l.split("|")] refs = json.loads((DATA / "reference_list.json").read_text(encoding="utf-8")) # ---- provenance: verify SHA-256 against README ---- readme = (DATA / "README.md").read_text(encoding="utf-8") for r in refs: p = DATA / r["source_file"] h = hashlib.sha256(p.read_bytes()).hexdigest() assert h in readme, f"hash mismatch for {p}" n_input_rows = {"README.md": wc_l(DATA / "README.md"), "reference_list.json": len(refs)} for r in refs: n_input_rows[Path(r["source_file"]).name] = wc_l(DATA / r["source_file"]) def parse_entry(entry): m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?\s*$", entry) return {"authors": m.group(1).strip(), "year": int(m.group(2)), "title": m.group(3).strip(), "arxiv": m.group(4)} def parse_header(lines): title = next(l for l in lines if l.startswith("Title:"))[len("Title:"):].strip() rec = next(l for l in lines if l.startswith("Bibliographic record:")) body_title = next(l for l in lines if l.startswith("# "))[2:].strip() years = [int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", rec)] first_year = years[0] # year of the version reproduced: the one flagged "(this extract)" or the only version m = re.search(r"\(this extract\) submitted \d{1,2} \w{3} (\d{4})", rec) this_year = int(m.group(1)) if m else years[-1] aid = re.search(r"arXiv:(\d{4}\.\d{5})v\d+", rec).group(1) print(f" Header Title: {title}") print(f" Body title : {body_title}") print(f" {rec.strip()}") return {"title": title, "body_title": body_title, "first_year": first_year, "this_year": this_year, "arxiv": aid} # ---- finding checks (each prints the cells it relies on) ---- def check_R1(L): i, l = find_line(L, r"^ATT \| -0\.399 \| -0\.283 \| -0\.431\*\*") c = cells(l); se = cells(row_after(L, i)) _, pc = find_line(L, r"^% change \| -32\.9%") find_line(L, r"^Table 6:") find_line(L, r"statistically significant at the 5% level") find_line(L, r"staggered introduction of DCRs") find_line(L, r"roughly 35% within 5 km") att, se5, pct = c[3], se[3], cells(pc)[3] implied = 100 * (math.exp(float(att.rstrip("*"))) - 1) print(f" exp(-0.431)-1 = {implied:.1f}%") ok = (att == "-0.431**" and se5 == "(0.210)" and pct == "-35.0%" and abs(implied + 35.0) < 0.1) # internal inconsistency: Table A19 SE for same estimate j, la = find_line(L, r"^ATT \| -0\.431\*\* \| -0\.438") sea = cells(row_after(L, j))[1] return ok, ("arxiv-2609.27732v1 | Table 6, row ATT / column 5 km (SE row; % change row); " "Section 6 text (5% significance); abstract (staggered DiD, hospital records)"), \ f"Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, matching every element (Table A19 prints SE {sea} for the same estimate).", sea def check_R2(L): find_line(L, r"N = 23,549\) preregistered randomized controlled trial") _, w = find_line(L, r"^Wage \| \$33,073") _, r_ = find_line(L, r"^Reemployment \| 75\.2%") _, u = find_line(L, r"^Weeks on UI \| 15\.2") ok = (cells(w)[4] == "$1,153; p=0.022; 95% CI: [166, 2,141]" and cells(r_)[4].startswith("1.47 pp") and cells(u)[4].startswith("-1.99 weeks")) return ok, ("arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, " "Reemployment, Weeks on UI; abstract (N = 23,549, preregistered RCT)"), \ "Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives a preregistered RCT with N = 23,549.", None def check_R3(L): _, l = find_line(L, r"stacked coefficient is 0\.005 at \\tau=0") find_line(L, r"prices do not jump immediately") _, t3 = find_line(L, r"^Clean-onset threshold-2 \| 774") c = cells(t3) tau0 = float(re.search(r"0\.\d+(?= at \\tau=0)", l).group(0)) print(f" tau=0 coefficient {tau0} (~{100*tau0:.1f}%); tau=4 {c[4]}, tau=5 {c[5]}") # cited: 3.4-3.7% jump in event year, no further appreciation -> contradicted ok = tau0 >= 0.034 return ok, ("arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) " "and Table 3, row Clean-onset threshold-2, columns tau=4 / tau=5"), \ "The extract reports ~0.5% (0.005) at the event year rising gradually to 0.037/0.034 at tau=4/5 ('prices do not jump immediately'), contradicting a 3.4-3.7% event-year jump with no later appreciation; title and year (2024 vs 2026) also disagree.", None def check_R4(L): find_line(L, r"Japan.s spot labor market") i, l1 = find_line(L, r"^Employment elasticity w\.r\.t MW \| -0\.388") s1 = cells(row_after(L, i))[1] j, l2 = find_line(L, r"^Emp\. elasticity w\.r\.t affected wage \| -0\.732") s2 = cells(row_after(L, j))[1] ok = s1 == "(0.132)" and s2 == "(0.091)" return ok, ("arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and " "'Emp. elasticity w.r.t affected wage' (Estimate (SE) column); abstract (Japan's spot labor market)"), \ "Table 2 prints -0.388 (0.132) and -0.732 (0.091) for the two elasticities, and the abstract places the study in Japan's spot labor market.", None def check_R5(L): find_line(L, r"^Table 3: Probability of infant death and stillbirth") i, t = find_line(L, r"^Treatment \| -0\.0911") se = cells(row_after(L, i)) _, pc = find_line(L, r"^% Change \|") _, mn = find_line(L, r"^Mean Dep\. Var\. \|") find_line(L, r"most preferred specification \(Column \(4\)\)") b, s, p, m = cells(t)[4], se[4], cells(pc)[4], cells(mn)[4] print(f" -0.0962/0.261 = {float(b)/float(m):.4f} (printed {p})") ok = b == "-0.0962" and s == "(0.0129)" and p == "-0.3685" return ok, ("arxiv-2510.03658v1 | Table 3, column (4), rows Treatment (coef, SE) and % Change; " "Section 4.1 (column (4) preferred, 36.85 percent)"), \ "Table 3 column (4) (stated preferred) prints -0.0962 (0.0129) and % Change -0.3685, i.e. 36.85% as stated in Section 4.1.", None CHECKERS = {"R1": check_R1, "R2": check_R2, "R3": check_R3, "R4": check_R4, "R5": check_R5} reference_checks, extra = {}, {} for r in refs: print(f"\n=== {r['id']} -> {r['source_file']}") L = lines_of(DATA / r["source_file"]) e = parse_entry(r["entry"]) h = parse_header(L) assert e["arxiv"] == h["arxiv"], "arXiv id mismatch" t_ok = norm_title(e["title"]) == norm_title(h["title"]) == norm_title(h["body_title"]) y_ok = e["year"] in (h["first_year"], h["this_year"]) print(f" Entry title : {e['title']} -> title_matches={t_ok}") print(f" Entry year {e['year']} vs first {h['first_year']} / this version {h['this_year']} -> year_matches={y_ok}") print(" Finding evidence:") f_ok, loc, note, x = CHECKERS[r["id"]](L) extra[r["id"]] = x print(f" finding_supported={f_ok}") reference_checks[r["id"]] = {"title_matches": bool(t_ok), "year_matches": bool(y_ok), "finding_supported": bool(f_ok), "locator": loc, "note": note} unver = [k for k, v in reference_checks.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])] baseline_unver = [k for k, v in reference_checks.items() if not (v["title_matches"] and v["year_matches"])] data_issues = [ {"issue": f"arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431** has SE (0.210) in Table 6 but SE {extra['R1']} in Table A19; the A19 note says 640 never-treated regions while Table 6/A19 bodies say 608; abstract says fatal outcomes fall 35% while Section 6.1 says ~30%; extract ends mid-sentence in the A19 note.", "handling": "R1 cites SE 0.210, which is exactly the main-results Table 6 value; it is judged supported, and 5% significance holds under either SE (stars ** printed in both tables). Inconsistency reported, not resolved."}, {"issue": "arxiv-2606.14621v1: rendering defect 'p ยก 0.001' in Section 3 (short-term weeks on UI); Tables 2-3 re-set by the extract editor.", "handling": "Read as p < 0.001 (Table 2 prints p<0.001); does not affect the R2 check, which relies on long-term rows of Table 2."}, {"issue": "arxiv-2603.03260v2: Table 3 header garbled in the HTML rendering.", "handling": "Used the extract editor's column order (Specification | Treated-role LSOAs | Stacked p | Sun p | tau=4 | tau=5), which agrees with Section 3.2 text (0.037, 0.034)."}, {"issue": "arxiv-2505.04555v3: Table 2 prints identical SE (0.091) for 'Affected employment' and for the own-wage elasticity (ratio -0.268/0.366 = -0.732), which is unusual for a ratio whose denominator has SE 0.171.", "handling": "Values taken exactly as printed; R4 cites the printed values, so the finding check passes; the oddity is flagged only."}, {"issue": "arxiv-2510.03658v1: abstract/Section 4.1 headline (0.0855, 18.79%) comes from a Table 2 that is omitted from the extract; Table 3 % Change -0.3685 vs -0.0962/0.261 = -0.3686 (rounding of printed mean).", "handling": "R5 cites the Table 3 column (4) result, which is in the extract; rounding difference is immaterial."}, ] results = { "task_id": "F4-conf-04", "reference_checks": reference_checks, "unverifiable_references": unver, "n_verified": len(refs) - len(unver), "feasibility": "complete", "feasibility_reason": None, "data_issues": data_issues, "n_input_rows": n_input_rows, "n_analysed": len(refs), "random_seed": None, "baseline_unverifiable_bibliographic_only": baseline_unver, "baseline_n_verified_bibliographic_only": len(refs) - len(baseline_unver), } OUT.mkdir(exist_ok=True) (OUT / "results.json").write_text(json.dumps(results, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") print(f"\nUnverifiable references: {unver}; n_verified = {results['n_verified']}") print(f"Baseline (title/year only) unverifiable: {baseline_unver}")