1"""F4-conf-04: verify five references against the provided arXiv extracts.23Reads only data/. Writes out/results.json deterministically. No randomness.4Run from workspace root: ~/wp29/venv/bin/python out/analysis.py5"""6import json, os, re, unicodedata, math78ROOT = os.getcwd()9DATA = os.path.join(ROOT, "data")10OUT = os.path.join(ROOT, "out")111213def read_lines(path):14 with open(path, encoding="utf-8") as f:15 return f.read().split("\n")161718def wc_l(path):19 with open(path, "rb") as f:20 return f.read().count(b"\n")212223def norm_title(s):24 s = unicodedata.normalize("NFKD", s)25 s = "".join(c for c in s if not unicodedata.combining(c)).lower()26 s = s.replace("&", " and ")27 s = re.sub(r"[^a-z0-9 ]+", " ", s)28 return " ".join(s.split())293031def find_line(lines, needle):32 """Return (1-based line number, line) of the first line containing needle; raise if absent."""33 for i, l in enumerate(lines, 1):34 if needle in l:35 return i, l36 raise AssertionError(f"evidence not found: {needle!r}")373839def cell(lines, row_label, col_index, sep=" | "):40 """Return the cell at col_index (0 = label) of the first table row starting with row_label."""41 i, l = find_line(lines, row_label + sep)42 return i, [c.strip() for c in l.split("|")][col_index]434445refs = json.load(open(os.path.join(DATA, "reference_list.json"), encoding="utf-8"))4647# ---- n_input_rows -------------------------------------------------------48n_input_rows = {"README.md": wc_l(os.path.join(DATA, "README.md")),49 "reference_list.json": len(refs)}50for r in refs:51 n_input_rows[os.path.basename(r["source_file"])] = wc_l(os.path.join(DATA, r["source_file"]))5253src = {r["id"]: read_lines(os.path.join(DATA, r["source_file"])) for r in refs}54sid = {r["id"]: os.path.basename(r["source_file"])[:-4] for r in refs}5556# ---- title and year checks (mechanical) ---------------------------------57MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"58meta = {}59for r in refs:60 L = src[r["id"]]61 header_title = L[1].split("Title:", 1)[1].strip()62 body_title = next(l[2:].strip() for l in L if l.startswith("# "))63 bib = next(l for l in L if l.startswith("Bibliographic record:"))64 years = sorted({int(y) for y in re.findall(r"\b\d{1,2} " + MONTHS + r" (\d{4})", bib)})65 m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{4,5})\.?$", r["entry"])66 entry_year, entry_title, entry_arxiv = int(m.group(2)), m.group(3), m.group(4)67 title_ok = norm_title(entry_title) == norm_title(header_title) == norm_title(body_title)68 # consistent iff equal to the first-submission year or the reproduced-version year (last date on the line)69 first_year, version_year = years[0], int(re.findall(r"\b\d{1,2} " + MONTHS + r" (\d{4})", bib)[-1])70 year_ok = entry_year in (first_year, version_year)71 arxiv_ok = sid[r["id"]].startswith("arxiv-" + entry_arxiv)72 meta[r["id"]] = dict(title_ok=title_ok, year_ok=year_ok, arxiv_ok=arxiv_ok,73 entry_title=entry_title, header_title=header_title,74 entry_year=entry_year, first_year=first_year, version_year=version_year)75 print(f"[{r['id']}] {sid[r['id']]}")76 print(f" header : Title: {header_title}")77 print(f" body : # {body_title}")78 print(f" {bib}")79 print(f" entry : title={entry_title!r} year={entry_year} arXiv={entry_arxiv}")80 print(f" -> title_matches={title_ok} year_matches={year_ok} (first={first_year}, version={version_year}) arxiv_id_matches={arxiv_ok}")8182# ---- finding checks: each element of cited_for tied to printed evidence ----83finding = {}848586def show(rid, label, ln, text):87 print(f" [{rid}] {label}: line {ln}: {text}")888990# R1 — Armendariz, Table 6, 5 km column91L = src["R1"]92cols = [c.strip() for c in find_line(L, "| 1 km | 3 km | 5 km")[1].split("|")]93k5 = cols.index("5 km")94ln, att = cell(L, "ATT", k5); show("R1", "Table 6 ATT 5 km", ln, att)95i_se = ln + 2; se = [c.strip() for c in L[i_se - 1].split("|")][k5]; show("R1", "Table 6 SE 5 km", i_se, se)96ln2, pct = cell(L, "% change", k5); show("R1", "Table 6 % change 5 km", ln2, pct)97ln3, sig = find_line(L, "statistically significant at the 5% level"); show("R1", "Sec 6 text", ln3, sig.strip())98ln4, ab = find_line(L, "staggered introduction of DCRs"); show("R1", "abstract", ln4, ab[:160] + "...")99r1 = dict(att=att == "-0.431**", se=se == "(0.210)", pct=pct == "-35.0%",100 pct_arith=abs((math.exp(-0.431) - 1) * 100 - (-35.0)) < 0.05,101 sig5=("**" in att) and abs(0.431 / 0.210) > 1.96,102 design=("staggered" in ab and "difference-in-differences" in ab and "hospital records" in ab103 and "Switzerland" in ab and "within 5 km" in ab))104print(" R1 element checks:", r1)105finding["R1"] = dict(ok=all(r1.values()),106 locator="arxiv-2609.27732v1 | Table 6, row ATT (and SE row below), column 5 km; row % change; Section 6 text after Table 6; abstract",107 note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, the text states significance at 5%, and the abstract describes the staggered DiD on Swiss individual-level hospital records; all elements supported.")108109# R2 — Li et al., Table 2 ATE column110L = src["R2"]111ln, ab = find_line(L, "N = 23,549"); show("R2", "abstract", ln, ab[:200] + "...")112ln_w, w = cell(L, "Wage", 4); show("R2", "Table 2 Wage ATE", ln_w, w)113ln_r, re_ = cell(L, "Reemployment", 4); show("R2", "Table 2 Reemployment ATE", ln_r, re_)114ln_u, u = cell(L, "Weeks on UI", 4); show("R2", "Table 2 Weeks on UI ATE", ln_u, u)115r2 = dict(rct=("preregistered randomized controlled trial" in ab and "N = 23,549" in ab),116 wage=w.startswith("$1,153;") and "95% CI: [166, 2,141]" in w,117 annualised="annualized wages by $1,153" in ab,118 reemp=re_.startswith("1.47 pp;"),119 weeks=u.startswith("-1.99 weeks;"))120print(" R2 element checks:", r2)121finding["R2"] = dict(ok=all(r2.values()),122 locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, Reemployment, Weeks on UI; abstract (N, preregistered RCT)",123 note="Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549 and annualized wages; all elements supported.")124125# R3 — Liu & Zhao, Section 3.2 / Table 3 (timing claim)126L = src["R3"]127ln, txt = find_line(L, "the stacked coefficient is 0.005 at \\tau=0"); show("R3", "Sec 3.2 headline", ln, txt[:330] + "...")128ln2, nj = find_line(L, "prices do not jump immediately"); show("R3", "Sec 3.2", ln2, "...prices do not jump immediately, but instead drift upward...")129ln_t, t4 = cell(L, "Clean-onset threshold-2", 4); _, t5 = cell(L, "Clean-onset threshold-2", 5)130show("R3", "Table 3 preferred row tau=4 / tau=5", ln_t, f"{t4} / {t5}")131tau0 = 0.005132r3 = dict(jump_in_event_year=(tau0 >= 0.034), # cited: 3.4-3.7% in the entry year; printed tau=0 coefficient is 0.005133 no_later_appreciation=not (float(t4) > tau0 and float(t5) > tau0), # cited: no further appreciation134 explicitly_contradicted="prices do not jump immediately" in nj)135print(" R3 element checks:", r3)136finding["R3"] = dict(ok=r3["jump_in_event_year"] and r3["no_later_appreciation"],137 locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row Clean-onset threshold-2, columns tau=4 / tau=5",138 note="The extract reports about 0.5% (0.005) at the event year rising gradually to 3.4-3.7% (0.037 at tau=4, 0.034 at tau=5) and says prices do not jump immediately, contradicting the cited jump in the entry year with no later appreciation; the entry's title and year (2024) also do not match the document (v1 Mar 2026, v2 Apr 2026).")139140# R4 — Kanayama et al., Appendix A.1 Table 2141L = src["R4"]142ln_e, e1 = cell(L, "Employment elasticity w.r.t MW", 1); se1 = L[ln_e + 1].split("|")[1].strip()143show("R4", "Table 2 Employment elasticity w.r.t MW", ln_e, f"{e1} {se1}")144ln_o, e2 = cell(L, "Emp. elasticity w.r.t affected wage", 1); se2 = L[ln_o + 1].split("|")[1].strip()145show("R4", "Table 2 Emp. elasticity w.r.t affected wage", ln_o, f"{e2} {se2}")146ln_a, ab = find_line(L, "Japan’s spot labor market"); show("R4", "abstract", ln_a, ab[:150] + "...")147_, ae = cell(L, "Affected employment", 1); _, aw = cell(L, "Affected wages", 1)148r4 = dict(mw=(e1 == "-0.388" and se1 == "(0.132)"), own=(e2 == "-0.732" and se2 == "(0.091)"),149 population="Japan’s spot labor market" in ab,150 own_arith=abs(float(ae) / float(aw) - (-0.732)) < 0.0005)151print(" R4 element checks:", r4, f"(affected employment / affected wages = {float(ae)/float(aw):.4f})")152finding["R4"] = dict(ok=r4["mw"] and r4["own"] and r4["population"],153 locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (estimate and SE rows); abstract (Japan's spot labor market)",154 note="Table 2 prints -0.388 (0.132) and -0.732 (0.091) and the abstract places the study in Japan's spot labor market; supported, with the year 2025 matching the v1 submission (7 May 2025).")155156# R5 — Tossou, Table 3 column (4)157L = src["R5"]158ln_t, tr = cell(L, "Treatment", 4); tse = L[ln_t + 1].split("|")[4].strip()159show("R5", "Table 3 Treatment col (4)", ln_t, f"{tr} {tse}")160ln_pc, pc = cell(L, "% Change", 4); ln_m, mdv = cell(L, "Mean Dep. Var.", 4)161show("R5", "Table 3 % Change col (4)", ln_pc, pc); show("R5", "Table 3 Mean Dep. Var. col (4)", ln_m, mdv)162ln_p, pref = find_line(L, "most preferred specification (Column (4))"); show("R5", "Sec 4.1", ln_p, "...Using my most preferred specification (Column (4)) ... by 0.0962 ... 36.85 percent")163r5 = dict(coef=(tr == "-0.0962" and tse == "(0.0129)"), preferred="(Column (4))" in pref and "0.0962" in pref,164 pct=pc == "-0.3685" and "36.85 percent" in pref,165 pct_arith=abs(-0.0962 / float(mdv) - float(pc)) < 0.0005,166 outcome="Probability of infant death and stillbirth" in "\n".join(L))167print(" R5 element checks:", r5, f"(-0.0962/{mdv} = {-0.0962/float(mdv):.4f})")168finding["R5"] = dict(ok=all(r5.values()),169 locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment (estimate and SE), % Change, Mean Dep. Var.; Section 4.1 text on the preferred specification",170 note="Table 3 column (4), named the preferred specification in Section 4.1, prints -0.0962 (0.0129) and % Change -0.3685 (36.85% in text; -0.0962/0.261 = -0.3686); supported, although the abstract's headline (0.0855, 18.79%) refers to a different outcome.")171172# ---- assemble -----------------------------------------------------------173reference_checks = {}174for r in refs:175 rid = r["id"]176 reference_checks[rid] = {"title_matches": meta[rid]["title_ok"], "year_matches": meta[rid]["year_ok"],177 "finding_supported": finding[rid]["ok"], "locator": finding[rid]["locator"],178 "note": finding[rid]["note"]}179unverifiable = [k for k, v in reference_checks.items()180 if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]181baseline_unverifiable = [k for k in meta if not (meta[k]["title_ok"] and meta[k]["year_ok"])]182183data_issues = [184 {"issue": "arxiv-2609.27732v1: the same 5 km CS-DR ATT -0.431 has SE (0.210) in Table 6 but (0.218) in Table A19; Table A19 note says 640 never-treated regions while Tables 6, 7 and A19 rows say 608.",185 "handling": "Main-text Table 6 used as the decisive locator (the citation reproduces it); significance at 5% holds under either SE (0.431/0.218 > 1.96); inconsistency reported, not flagged against R1."},186 {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes fall 35% and narcotics rise 'by a comparable amount', while Section 6.1 says about 30% and 47% (consistent with exp(-0.354)-1 and exp(0.390)-1).",187 "handling": "Not part of R1's cited finding; recorded only."},188 {"issue": "arxiv-2606.14621v1: rendering defect 'p ¡ 0.001' in Section 3; Tables 2-3 were re-set by the extract editor (declared in the header).",189 "handling": "Used the re-set tables as declared verbatim; the defect does not affect any cited value."},190 {"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering.",191 "handling": "Column order taken from the extract editor's note (Specification | Treated-role LSOAs | Stacked p | Sun p | tau=4 | tau=5)."},192 {"issue": "arxiv-2505.04555v3: SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment'; table note says 'about 4%' MW increase while '% MW changes' is 0.047.",193 "handling": "Cannot be resolved from the extract; the cited values are printed exactly as cited, so R4 is not flagged; recorded as a caveat."},194 {"issue": "arxiv-2510.03658v1: abstract headline (0.0855, 18.79%, mean 0.455, from omitted Table 2) differs from Table 3 preferred estimate (0.0962, 36.85%, mean 0.261); footnote 11 is garbled (three percentages for five wealth categories) and footnote markers render as '^9^9 9'.",195 "handling": "Treated as different outcomes (count vs binary indicator); R5 cites the Table 3 binary outcome and is checked against Table 3 column (4)."},196]197198results = {199 "task_id": "F4-conf-04",200 "reference_checks": reference_checks,201 "unverifiable_references": unverifiable,202 "n_verified": len(refs) - len(unverifiable),203 "feasibility": "complete",204 "feasibility_reason": None,205 "data_issues": data_issues,206 "n_input_rows": n_input_rows,207 "n_analysed": len(refs),208 "random_seed": None,209 "baseline_metadata_only_unverifiable": baseline_unverifiable,210}211os.makedirs(OUT, exist_ok=True)212with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:213 json.dump(results, f, indent=2, ensure_ascii=False)214print("\nunverifiable_references:", unverifiable, "| n_verified:", results["n_verified"],215 "| metadata-only baseline:", baseline_unverifiable)216