1"""F4-conf-04: verify five references against the provided arXiv extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import json, math, os, re, unicodedata45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7OUT = os.path.join(ROOT, "out", "results.json")89def wc_l(path):10 with open(path, "rb") as f:11 return f.read().count(b"\n")1213refs = json.load(open(os.path.join(D, "reference_list.json")))14src = {}15for r in refs:16 with open(os.path.join(D, r["source_file"]), encoding="utf-8") as f:17 src[r["id"]] = f.read().split("\n")1819def norm_title(s):20 s = unicodedata.normalize("NFKD", s)21 s = "".join(c for c in s if not unicodedata.combining(c)).lower()22 s = s.replace("&", " and ")23 s = re.sub(r"[^a-z0-9 ]", " ", s)24 return " ".join(s.split())2526def find(rid, needle):27 """Return (line_no, line) of first line containing needle; print it; raise if absent."""28 for i, l in enumerate(src[rid], 1):29 if needle in l:30 short = l if len(l) < 220 else "..." + l[max(0, l.index(needle) - 80): l.index(needle) + len(needle) + 80] + "..."31 print(f" [{rid} L{i}] {short}")32 return i, l33 print(f" [{rid}] NOT FOUND: {needle!r}")34 return None, None3536def cells(line):37 return [c.strip() for c in line.split("|")]3839def approx(a, b, tol):40 return abs(a - b) <= tol4142results_checks = {}43baseline_flags = {}44for r in refs:45 rid = r["id"]; L = src[rid]46 print(f"=== {rid}: {r['source_file']}")47 title_hdr = next(l for l in L if l.startswith("Title: "))[7:].strip()48 biblio = next(l for l in L if l.startswith("Bibliographic record: "))49 body_title = next(l for l in L if l.startswith("# "))[2:].strip()50 print(" " + "Title: " + title_hdr); print(" " + biblio); print(" body title: " + body_title)51 # entry title: text after "(YYYY). " up to ". arXiv:"52 m = re.match(r"^.*?\((\d{4})\)\.\s+(.*)\.\s+arXiv:(\S+?)\.?$", r["entry"])53 entry_year, entry_title, entry_arxiv = int(m.group(1)), m.group(2), m.group(3)54 title_ok = norm_title(entry_title) == norm_title(title_hdr) == norm_title(body_title)55 # years: first submission and reproduced version56 sub_dates = re.findall(r"submitted (\d{1,2} \w{3} (\d{4}))", biblio)57 years = [int(y) for _, y in sub_dates]58 first_year = years[0]59 this_m = re.search(r"\(this extract\) submitted \d{1,2} \w{3} (\d{4})", biblio)60 version_year = int(this_m.group(1)) if this_m else years[0] # single-version records61 year_ok = entry_year in (first_year, version_year)62 arxiv_ok = biblio.split()[2].startswith("arXiv:" + entry_arxiv)63 print(f" entry year {entry_year}; first-submission year {first_year}; reproduced-version year {version_year}; arXiv id match {arxiv_ok}")64 print(f" entry title: {entry_title}\n title_matches={title_ok} year_matches={year_ok}")65 results_checks[rid] = {"title_matches": bool(title_ok), "year_matches": bool(year_ok),66 "_entry_year": entry_year, "_years": (first_year, version_year)}67 baseline_flags[rid] = not (title_ok and year_ok)6869# ---------- finding checks (each element of cited_for checked against printed text/cells) ----------70F = {}7172print("=== R1 finding")73_, l = find("R1", "ATT | -0.399 | -0.283 | -0.431**"); c = cells(l)74_, se = find("R1", "| (0.412) | (0.279) | (0.210)"); s = cells(se)75_, pc = find("R1", "% change | -32.9% | -24.7% | -35.0%")76find("R1", "{}^{**}p<0.05")77find("R1", "This estimate is statistically significant at the 5% level")78find("R1", "reduce drug-related hospitalization rates by roughly 35% within 5 km of a facility")79find("R1", "staggered introduction of DCRs across Swiss cities")80find("R1", "using individual-level hospital records and a difference-in-differences estimator")81_, a19 = find("R1", "| (0.218) | (0.208) | (0.205) | (0.038)")82att, se5 = float(c[3].rstrip("*")), float(s[3].strip("()"))83pct = math.expm1(att) * 10084print(f" check: exp(-0.431)-1 = {pct:.2f}% (printed -35.0%); |t| = {abs(att)/se5:.3f} (Table 6 SE), {abs(att)/0.218:.3f} (A19 SE)")85F["R1"] = dict(ok=(att == -0.431 and se5 == 0.210 and c[3].endswith("**") and approx(pct, -35.0, 0.05)),86 locator="arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (with Section 6 text and abstract)",87 note="Title and 2026 year match; Table 6 5 km column prints ATT -0.431** (0.210), -35.0%, significant at 5%, Callaway-Sant'Anna staggered DiD on hospital records, so every element is supported (Table A19 prints SE 0.218 for the same estimate, see data_issues).")8889print("=== R2 finding")90find("R2", "large (N = 23,549) preregistered randomized controlled trial (RCT)")91_, w = find("R2", "Wage | $33,073"); wc = cells(w)92_, re_ = find("R2", "Reemployment | 75.2%"); rc = cells(re_)93_, ui = find("R2", "Weeks on UI | 15.2 weeks"); uc = cells(ui)94find("R2", "wage outcome that was $1,153 higher annually (95% CI: [$166, $2,141]")95F["R2"] = dict(ok=(wc[4].startswith("$1,153") and "95% CI: [166, 2,141]" in wc[4]96 and rc[4].startswith("1.47 pp") and uc[4].startswith("-1.99 weeks")),97 locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI (N and design: abstract)",98 note="Title and 2026 year match; Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549.")99100print("=== R3 finding")101find("R3", "the stacked coefficient is 0.005 at \\tau=0")102find("R3", "prices do not jump immediately, but instead drift upward")103_, t3 = find("R3", "Clean-onset threshold-2 | 774"); t3c = cells(t3)104print(f" Table 3 preferred row: tau=4 {t3c[4]}, tau=5 {t3c[5]} (log points) -> 3.4-3.7% reached at years 4-5, not at entry")105F["R3"] = dict(ok=False,106 locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficient 0.005 at tau=0) and Table 3, row 'Clean-onset threshold-2', columns tau=4 (0.037) / tau=5 (0.034)",107 note="Entry has a different title and year 2024 (record: v1 3 Mar 2026, v2 2 Apr 2026), and the extract reports ~0.5% at entry rising gradually to 3.4-3.7% only by years 4-5, contradicting the cited immediate jump with no later appreciation.")108109print("=== R4 finding")110_, e1 = find("R4", "Employment elasticity w.r.t MW | -0.388"); i1 = src["R4"].index(e1)111_, e2 = find("R4", "Emp. elasticity w.r.t affected wage | -0.732"); i2 = src["R4"].index(e2)112se1 = next(x for x in src["R4"][i1 + 1:] if x.strip()); se2 = next(x for x in src["R4"][i2 + 1:] if x.strip())113print(f" SE line after MW elasticity: {se1.strip()} ; after own-wage elasticity: {se2.strip()}")114find("R4", "Japan’s spot labor market")115print(f" arithmetic: -0.268/0.366 = {-0.268/0.366:.3f}; -0.268*0.068/0.047 = {-0.268*0.068/0.047:.3f}")116F["R4"] = dict(ok=(cells(se1)[-1] == "(0.132)" and cells(se2)[-1] == "(0.091)"),117 locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",118 note="Title matches and 2025 equals the first-submission year (v1 7 May 2025); Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market.")119120print("=== R5 finding")121_, tr = find("R5", "Treatment | -0.0911 | -0.0908 | -0.0961 | -0.0962"); trc = cells(tr)122i = src["R5"].index(tr); sel = next(x for x in src["R5"][i + 1:] if x.strip()); print(" SE line: " + sel.strip())123_, pch = find("R5", "% Change | -0.3490"); _, mdv = find("R5", "Mean Dep. Var. | 0.261")124find("R5", "Using my most preferred specification (Column (4))")125find("R5", "this effect represents a percentage change of 36.85 percent")126print(f" arithmetic: -0.0962/0.261 = {-0.0962/0.261:.4f} (printed % Change col 4: {cells(pch)[4]})")127F["R5"] = dict(ok=(trc[4] == "-0.0962" and cells(sel)[4] == "(0.0129)" and cells(pch)[4] == "-0.3685"),128 locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment and % Change (Section 4.1 names column (4) as preferred)",129 note="Title and 2025 year match; Table 3 column (4) prints Treatment -0.0962 (0.0129) and % Change -0.3685 (36.85%), and Section 4.1 calls column (4) the preferred specification.")130131reference_checks = {}132for r in refs:133 rid = r["id"]; rc_ = results_checks[rid]134 reference_checks[rid] = {"title_matches": rc_["title_matches"], "year_matches": rc_["year_matches"],135 "finding_supported": bool(F[rid]["ok"]), "locator": F[rid]["locator"], "note": F[rid]["note"]}136unver = [k for k, v in reference_checks.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]137138data_issues = [139 {"issue": "arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431 has SE (0.210) in Table 6 but (0.218) in Table A19; the A19 note says 640 never-treated regions whereas Tables 6-7 say 608.",140 "handling": "Internal inconsistency of the source. R1 cites Table 6 (the main table), whose SE 0.210 matches the citation; both SEs give |t|>1.96, so the 5% significance claim holds either way. Not grounds for flagging R1; recorded."},141 {"issue": "arxiv-2609.27732v1: the extract header announces 'the paragraph that discusses' Table A19, but the file ends at the A19 note (no discussion paragraph); abstract says fatal outcomes fall by 35% while Section 6.1 says ~30% (Table 7 5 km: -0.354).",142 "handling": "Not needed for the R1 check (decided by Table 6); recorded only."},143 {"issue": "arxiv-2603.03260v2: Table 3 header row is garbled in the rendering.",144 "handling": "Used the extract editor's note giving the six column names in order; tau=4 / tau=5 columns read as 0.037 / 0.034 for the preferred row, consistent with the Section 3.2 text."},145 {"issue": "arxiv-2606.14621v1: rendering defect 'p ¡ 0.001' in the short-term weeks-on-UI sentence; the text reports the weeks-on-UI CI as positive magnitudes while Table 2 prints negative bounds.",146 "handling": "Read '¡' as '<'; sign difference is a reduction-vs-change convention. Neither affects R2 (decided by Table 2)."},147 {"issue": "arxiv-2505.04555v3: the printed SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment' (0.091), although the point estimate is a ratio (-0.268/0.366 = -0.732).",148 "handling": "Possible source-side copy error that cannot be resolved from the extract; the citation reproduces the printed value exactly, so R4's finding is treated as supported. Recorded as a caveat."},149 {"issue": "arxiv-2510.03658v1: abstract headline effect (0.0855, 18.79%, from Table 2, not in extract) differs from Table 3 column (4) (0.0962, 36.85%); Table 3 % Change -0.3685 vs -0.0962/0.261 = -0.3686.",150 "handling": "Different outcome definitions (count vs binary); the citation explicitly refers to the preferred specification of the binary outcome (Table 3). Last-digit difference is rounding. Not grounds for flagging R5."},151]152153n_input_rows = {"README.md": wc_l(os.path.join(D, "README.md")), "reference_list.json": len(refs)}154for r in refs:155 n_input_rows[os.path.basename(r["source_file"])] = wc_l(os.path.join(D, r["source_file"]))156157baseline_unver = [k for k, v in baseline_flags.items() if v]158results = {159 "task_id": "F4-conf-04",160 "reference_checks": reference_checks,161 "unverifiable_references": unver,162 "n_verified": len(refs) - len(unver),163 "feasibility": "complete",164 "feasibility_reason": None,165 "data_issues": data_issues,166 "n_input_rows": n_input_rows,167 "n_analysed": len(refs),168 "random_seed": None,169 "baseline_unverifiable_bibliographic_only": baseline_unver,170 "baseline_n_verified_bibliographic_only": len(refs) - len(baseline_unver),171}172os.makedirs(os.path.dirname(OUT), exist_ok=True)173with open(OUT, "w", encoding="utf-8") as f:174 json.dump(results, f, indent=2, ensure_ascii=False)175print("=== summary")176for k, v in reference_checks.items():177 print(f" {k}: title={v['title_matches']} year={v['year_matches']} finding={v['finding_supported']}")178print(f" unverifiable_references={unver}; n_verified={results['n_verified']}")179