1"""F4-conf-04: verify five references against the provided arXiv extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import hashlib, json, math, re, unicodedata4from pathlib import Path56ROOT = Path(__file__).resolve().parent.parent7DATA = ROOT / "data"8OUT = ROOT / "out"91011def lines_of(p):12 return p.read_text(encoding="utf-8").splitlines()131415def wc_l(p):16 return p.read_bytes().count(b"\n")171819def norm_title(t):20 t = unicodedata.normalize("NFKD", t)21 t = "".join(c for c in t if not unicodedata.combining(c)).lower()22 t = t.replace("&", " and ")23 t = re.sub(r"[^a-z0-9 ]", " ", t)24 return " ".join(t.split())252627def find_line(lines, pattern, show=True):28 """Return the first line matching regex; print it as evidence."""29 for i, l in enumerate(lines):30 if re.search(pattern, l):31 if show:32 print(f" [line {i+1}] {l.strip()[:220]}")33 return i, l34 raise AssertionError(f"pattern not found: {pattern}")353637def row_after(lines, i):38 """Next non-empty line (SE row in regression tables)."""39 j = i + 140 while not lines[j].strip():41 j += 142 print(f" [line {j+1}] {lines[j].strip()[:220]}")43 return lines[j]444546def cells(l):47 return [c.strip() for c in l.split("|")]484950refs = json.loads((DATA / "reference_list.json").read_text(encoding="utf-8"))5152# ---- provenance: verify SHA-256 against README ----53readme = (DATA / "README.md").read_text(encoding="utf-8")54for r in refs:55 p = DATA / r["source_file"]56 h = hashlib.sha256(p.read_bytes()).hexdigest()57 assert h in readme, f"hash mismatch for {p}"5859n_input_rows = {"README.md": wc_l(DATA / "README.md"), "reference_list.json": len(refs)}60for r in refs:61 n_input_rows[Path(r["source_file"]).name] = wc_l(DATA / r["source_file"])626364def parse_entry(entry):65 m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?\s*$", entry)66 return {"authors": m.group(1).strip(), "year": int(m.group(2)), "title": m.group(3).strip(), "arxiv": m.group(4)}676869def parse_header(lines):70 title = next(l for l in lines if l.startswith("Title:"))[len("Title:"):].strip()71 rec = next(l for l in lines if l.startswith("Bibliographic record:"))72 body_title = next(l for l in lines if l.startswith("# "))[2:].strip()73 years = [int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", rec)]74 first_year = years[0]75 # year of the version reproduced: the one flagged "(this extract)" or the only version76 m = re.search(r"\(this extract\) submitted \d{1,2} \w{3} (\d{4})", rec)77 this_year = int(m.group(1)) if m else years[-1]78 aid = re.search(r"arXiv:(\d{4}\.\d{5})v\d+", rec).group(1)79 print(f" Header Title: {title}")80 print(f" Body title : {body_title}")81 print(f" {rec.strip()}")82 return {"title": title, "body_title": body_title, "first_year": first_year, "this_year": this_year, "arxiv": aid}838485# ---- finding checks (each prints the cells it relies on) ----86def check_R1(L):87 i, l = find_line(L, r"^ATT \| -0\.399 \| -0\.283 \| -0\.431\*\*")88 c = cells(l); se = cells(row_after(L, i))89 _, pc = find_line(L, r"^% change \| -32\.9%")90 find_line(L, r"^Table 6:")91 find_line(L, r"statistically significant at the 5% level")92 find_line(L, r"staggered introduction of DCRs")93 find_line(L, r"roughly 35% within 5 km")94 att, se5, pct = c[3], se[3], cells(pc)[3]95 implied = 100 * (math.exp(float(att.rstrip("*"))) - 1)96 print(f" exp(-0.431)-1 = {implied:.1f}%")97 ok = (att == "-0.431**" and se5 == "(0.210)" and pct == "-35.0%" and abs(implied + 35.0) < 0.1)98 # internal inconsistency: Table A19 SE for same estimate99 j, la = find_line(L, r"^ATT \| -0\.431\*\* \| -0\.438")100 sea = cells(row_after(L, j))[1]101 return ok, ("arxiv-2609.27732v1 | Table 6, row ATT / column 5 km (SE row; % change row); "102 "Section 6 text (5% significance); abstract (staggered DiD, hospital records)"), \103 f"Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, matching every element (Table A19 prints SE {sea} for the same estimate).", sea104105106def check_R2(L):107 find_line(L, r"N = 23,549\) preregistered randomized controlled trial")108 _, w = find_line(L, r"^Wage \| \$33,073")109 _, r_ = find_line(L, r"^Reemployment \| 75\.2%")110 _, u = find_line(L, r"^Weeks on UI \| 15\.2")111 ok = (cells(w)[4] == "$1,153; p=0.022; 95% CI: [166, 2,141]"112 and cells(r_)[4].startswith("1.47 pp")113 and cells(u)[4].startswith("-1.99 weeks"))114 return ok, ("arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, "115 "Reemployment, Weeks on UI; abstract (N = 23,549, preregistered RCT)"), \116 "Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives a preregistered RCT with N = 23,549.", None117118119def check_R3(L):120 _, l = find_line(L, r"stacked coefficient is 0\.005 at \\tau=0")121 find_line(L, r"prices do not jump immediately")122 _, t3 = find_line(L, r"^Clean-onset threshold-2 \| 774")123 c = cells(t3)124 tau0 = float(re.search(r"0\.\d+(?= at \\tau=0)", l).group(0))125 print(f" tau=0 coefficient {tau0} (~{100*tau0:.1f}%); tau=4 {c[4]}, tau=5 {c[5]}")126 # cited: 3.4-3.7% jump in event year, no further appreciation -> contradicted127 ok = tau0 >= 0.034128 return ok, ("arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) "129 "and Table 3, row Clean-onset threshold-2, columns tau=4 / tau=5"), \130 "The extract reports ~0.5% (0.005) at the event year rising gradually to 0.037/0.034 at tau=4/5 ('prices do not jump immediately'), contradicting a 3.4-3.7% event-year jump with no later appreciation; title and year (2024 vs 2026) also disagree.", None131132133def check_R4(L):134 find_line(L, r"Japan.s spot labor market")135 i, l1 = find_line(L, r"^Employment elasticity w\.r\.t MW \| -0\.388")136 s1 = cells(row_after(L, i))[1]137 j, l2 = find_line(L, r"^Emp\. elasticity w\.r\.t affected wage \| -0\.732")138 s2 = cells(row_after(L, j))[1]139 ok = s1 == "(0.132)" and s2 == "(0.091)"140 return ok, ("arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and "141 "'Emp. elasticity w.r.t affected wage' (Estimate (SE) column); abstract (Japan's spot labor market)"), \142 "Table 2 prints -0.388 (0.132) and -0.732 (0.091) for the two elasticities, and the abstract places the study in Japan's spot labor market.", None143144145def check_R5(L):146 find_line(L, r"^Table 3: Probability of infant death and stillbirth")147 i, t = find_line(L, r"^Treatment \| -0\.0911")148 se = cells(row_after(L, i))149 _, pc = find_line(L, r"^% Change \|")150 _, mn = find_line(L, r"^Mean Dep\. Var\. \|")151 find_line(L, r"most preferred specification \(Column \(4\)\)")152 b, s, p, m = cells(t)[4], se[4], cells(pc)[4], cells(mn)[4]153 print(f" -0.0962/0.261 = {float(b)/float(m):.4f} (printed {p})")154 ok = b == "-0.0962" and s == "(0.0129)" and p == "-0.3685"155 return ok, ("arxiv-2510.03658v1 | Table 3, column (4), rows Treatment (coef, SE) and % Change; "156 "Section 4.1 (column (4) preferred, 36.85 percent)"), \157 "Table 3 column (4) (stated preferred) prints -0.0962 (0.0129) and % Change -0.3685, i.e. 36.85% as stated in Section 4.1.", None158159160CHECKERS = {"R1": check_R1, "R2": check_R2, "R3": check_R3, "R4": check_R4, "R5": check_R5}161162reference_checks, extra = {}, {}163for r in refs:164 print(f"\n=== {r['id']} -> {r['source_file']}")165 L = lines_of(DATA / r["source_file"])166 e = parse_entry(r["entry"])167 h = parse_header(L)168 assert e["arxiv"] == h["arxiv"], "arXiv id mismatch"169 t_ok = norm_title(e["title"]) == norm_title(h["title"]) == norm_title(h["body_title"])170 y_ok = e["year"] in (h["first_year"], h["this_year"])171 print(f" Entry title : {e['title']} -> title_matches={t_ok}")172 print(f" Entry year {e['year']} vs first {h['first_year']} / this version {h['this_year']} -> year_matches={y_ok}")173 print(" Finding evidence:")174 f_ok, loc, note, x = CHECKERS[r["id"]](L)175 extra[r["id"]] = x176 print(f" finding_supported={f_ok}")177 reference_checks[r["id"]] = {"title_matches": bool(t_ok), "year_matches": bool(y_ok),178 "finding_supported": bool(f_ok), "locator": loc, "note": note}179180unver = [k for k, v in reference_checks.items()181 if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]182baseline_unver = [k for k, v in reference_checks.items()183 if not (v["title_matches"] and v["year_matches"])]184185data_issues = [186 {"issue": f"arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431** has SE (0.210) in Table 6 but SE {extra['R1']} in Table A19; the A19 note says 640 never-treated regions while Table 6/A19 bodies say 608; abstract says fatal outcomes fall 35% while Section 6.1 says ~30%; extract ends mid-sentence in the A19 note.",187 "handling": "R1 cites SE 0.210, which is exactly the main-results Table 6 value; it is judged supported, and 5% significance holds under either SE (stars ** printed in both tables). Inconsistency reported, not resolved."},188 {"issue": "arxiv-2606.14621v1: rendering defect 'p ¡ 0.001' in Section 3 (short-term weeks on UI); Tables 2-3 re-set by the extract editor.",189 "handling": "Read as p < 0.001 (Table 2 prints p<0.001); does not affect the R2 check, which relies on long-term rows of Table 2."},190 {"issue": "arxiv-2603.03260v2: Table 3 header garbled in the HTML rendering.",191 "handling": "Used the extract editor's column order (Specification | Treated-role LSOAs | Stacked p | Sun p | tau=4 | tau=5), which agrees with Section 3.2 text (0.037, 0.034)."},192 {"issue": "arxiv-2505.04555v3: Table 2 prints identical SE (0.091) for 'Affected employment' and for the own-wage elasticity (ratio -0.268/0.366 = -0.732), which is unusual for a ratio whose denominator has SE 0.171.",193 "handling": "Values taken exactly as printed; R4 cites the printed values, so the finding check passes; the oddity is flagged only."},194 {"issue": "arxiv-2510.03658v1: abstract/Section 4.1 headline (0.0855, 18.79%) comes from a Table 2 that is omitted from the extract; Table 3 % Change -0.3685 vs -0.0962/0.261 = -0.3686 (rounding of printed mean).",195 "handling": "R5 cites the Table 3 column (4) result, which is in the extract; rounding difference is immaterial."},196]197198results = {199 "task_id": "F4-conf-04",200 "reference_checks": reference_checks,201 "unverifiable_references": unver,202 "n_verified": len(refs) - len(unver),203 "feasibility": "complete",204 "feasibility_reason": None,205 "data_issues": data_issues,206 "n_input_rows": n_input_rows,207 "n_analysed": len(refs),208 "random_seed": None,209 "baseline_unverifiable_bibliographic_only": baseline_unver,210 "baseline_n_verified_bibliographic_only": len(refs) - len(baseline_unver),211}212OUT.mkdir(exist_ok=True)213(OUT / "results.json").write_text(json.dumps(results, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")214print(f"\nUnverifiable references: {unver}; n_verified = {results['n_verified']}")215print(f"Baseline (title/year only) unverifiable: {baseline_unver}")216