1"""F4-conf-04: verify five references against the provided source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Uses only files under data/. Deterministic; no randomness."""4import json, re, math, unicodedata5from pathlib import Path67DATA = Path("data"); OUT = Path("out"); OUT.mkdir(exist_ok=True)8refs = json.loads((DATA / "reference_list.json").read_text())910def lines(p):11 return (DATA / p).read_text(encoding="utf-8").splitlines()1213def find(p, pattern, show=True):14 """Return the first line of source p matching regex pattern; print it as evidence."""15 for i, l in enumerate(lines(p), 1):16 if re.search(pattern, l):17 if show:18 print(f" [{p}:{i}] {l.strip()[:220]}")19 return l20 raise ValueError(f"pattern {pattern!r} not found in {p}")2122def norm_title(t):23 t = unicodedata.normalize("NFKD", t)24 t = "".join(c for c in t if not unicodedata.combining(c))25 t = t.lower().replace("&", "and")26 t = re.sub(r"[^a-z0-9 ]", " ", t)27 return " ".join(t.split())2829def entry_title_year(entry):30 m = re.match(r"^.*?\((\d{4})\)\.\s*(.*?)\.\s*arXiv:", entry)31 return int(m.group(1)), m.group(2)3233def header(p):34 title = find(p, r"^Title: ")[len("Title: "):].strip()35 bib = find(p, r"^Bibliographic record: ")36 years = sorted(set(int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", bib)))37 body_title = find(p, r"^# ")[2:].strip()38 return title, body_title, years, bib3940def next_nonempty(p, row_line):41 L = lines(p); i = L.index(row_line) + 142 while not L[i].strip():43 i += 144 return L[i]4546def cell(row_line, idx):47 return [c.strip() for c in row_line.split("|")][idx]4849checks, data_issues = {}, []5051# ---------------- generic title/year checks ----------------52meta = {}53for r in refs:54 p = r["source_file"]55 print(f"\n=== {r['id']} -> {p}")56 print(" entry:", r["entry"])57 yr, t = entry_title_year(r["entry"])58 title, body_title, years, bib = header(p)59 tm = norm_title(t) == norm_title(title) == norm_title(body_title)60 ym = yr in years # years = {first submission year, extract-version year, other versions}61 # restrict to first submission and reproduced version years per TASK rule62 first_year = years[0]63 m_this = re.search(r"submitted \d{1,2} \w{3} (\d{4}) \((?:version \d+, the only|this extract)", bib) \64 or re.search(r"\(this extract\) submitted \d{1,2} \w{3} (\d{4})", bib) \65 or re.search(r"submitted \d{1,2} \w{3} (\d{4}) \(version 1, the only", bib)66 this_year = int(m_this.group(1)) if m_this else first_year67 ym = yr in {first_year, this_year}68 print(f" entry year {yr}; first-submission year {first_year}; extract-version year {this_year} -> year_matches={ym}")69 print(f" entry title: {t!r}\n source title: {title!r} -> title_matches={tm}")70 meta[r["id"]] = dict(title_matches=tm, year_matches=ym, entry_year=yr,71 first_year=first_year, this_year=this_year)7273# ---------------- finding checks (source-specific evidence) ----------------74print("\n=== Finding checks")75# R1 Armendariz76p = "sources/arxiv-2609.27732v1.txt"; print("R1:")77abs_l = find(p, r"roughly 35% within 5 km")78att = find(p, r"^ATT \| -0\.399"); se = next_nonempty(p, att)79print(" SE row:", se.strip())80pct = find(p, r"^% change \| -32\.9%")81find(p, r"statistically significant at the 5% level")82find(p, r"staggered introduction of DCRs")83att5, se5 = float(cell(att, 3).rstrip("*")), float(cell(se, 3).strip("()"))84stars5 = cell(att, 3).count("*")85z = att5 / se586print(f" 5 km ATT={att5}, SE={se5}, stars={stars5}, |t|={abs(z):.3f}, exp(ATT)-1={math.exp(att5)-1:.4f}")87a19 = find(p, r"^ATT \| -0\.431\*\* \| -0\.438")88a19se = next_nonempty(p, a19); print(" A19 SE row:", a19se.strip())89a19_se = float(cell(a19se, 1).strip("()"))90print(f" A19 CS-DR |t| with SE {a19_se}: {abs(att5)/a19_se:.3f}")91r1_ok = (att5 == -0.431 and se5 == 0.210 and stars5 == 2 and cell(pct, 3) == "-35.0%")92checks["R1"] = dict(finding_supported=r1_ok,93 locator="arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (also abstract and Section 6 text)",94 note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, and the abstract/Section 6 describe a staggered DiD on Swiss individual-level hospital records; all cited elements match.")95if abs(a19_se - se5) > 1e-9:96 data_issues.append({"issue": f"Armendariz extract: the CS-DR 5 km ATT -0.431 has SE (0.210) in Table 6 but SE ({a19_se}) in Table A19 for the same estimator/buffer/window; Table A19 note also says 640 never-treated regions while Tables 6/A19 rows say 608 control regions.",97 "handling": f"Finding check for R1 decided on Table 6 (the main table, which the citation quotes); significance at 5% holds under either SE (|t|={abs(z):.2f} vs {abs(att5)/a19_se:.2f}). Discrepancy reported, not resolved."})98fatal = find(p, r"^Drug-related \(fatal\)")99data_issues.append({"issue": f"Armendariz extract: abstract says fatal outcomes 'fall by 35%', but Table 7 (5 km) prints {cell(fatal,3)} (exp-1 = {math.exp(float(cell(fatal,3).rstrip('*')))-1:.3f}) and Section 6.1 says 'approximately 30%'; Table 7 note lists four sub-outcomes while five are reported; the paragraph discussing Table A19 announced in the header is cut off after the table note.",100 "handling": "Not relevant to the R1 attributed finding (overall 5 km ATT); recorded only."})101102# R2 Li et al.103p = "sources/arxiv-2606.14621v1.txt"; print("R2:")104find(p, r"N = 23,549\) preregistered randomized controlled trial")105w = find(p, r"^Wage \| \$33,073"); re_ = find(p, r"^Reemployment \| 75\.2%"); ui = find(p, r"^Weeks on UI \| 15\.2")106ate_w, ate_r, ate_u = cell(w, 4), cell(re_, 4), cell(ui, 4)107print(" ATE cells:", ate_w, "||", ate_r, "||", ate_u)108r2_ok = (ate_w.startswith("$1,153") and "[166, 2,141]" in ate_w and ate_r.startswith("1.47 pp")109 and ate_u.startswith("-1.99 weeks"))110checks["R2"] = dict(finding_supported=r2_ok,111 locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI (N and design from abstract)",112 note="Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549; all elements match.")113data_issues.append({"issue": "Li et al. extract: Tables 2 and 3 were re-set by the extract editor (HTML split rows); Section 3 text contains a rendering artefact 'p ¡ 0.001' for p < 0.001.",114 "handling": "Used the re-set values as verbatim per the editorial note; artefact read as p < 0.001; no effect on checks."})115116# R3 Liu & Zhao117p = "sources/arxiv-2603.03260v2.txt"; print("R3:")118find(p, r"rise gradually from about 0\.5% in the event year")119find(p, r"stacked coefficient is 0\.005 at")120find(p, r"prices do not jump immediately")121co = find(p, r"^Clean-onset threshold-2 \|")122print(f" Table 3 preferred row tau=4 {cell(co,4)}, tau=5 {cell(co,5)}")123checks["R3"] = dict(finding_supported=False,124 locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficient 0.005 at tau=0) and Table 3, row Clean-onset threshold-2, columns tau=4/tau=5",125 note="The source reports only ~0.5% (0.005) in the event year rising gradually to 3.4-3.7% by years four-five and states prices 'do not jump immediately', contradicting the cited jump in the entry year with no later appreciation; the entry's title and year (2024 vs 2026 submissions) also disagree with the source.")126data_issues.append({"issue": "Liu & Zhao extract: Table 3 header row is garbled in the HTML rendering (column names reconstructed by extract editor).",127 "handling": "Used the editor's column order (… | tau=4 | tau=5); the tau=4/tau=5 values 0.037/0.034 agree with the Section 3.2 text."})128129# R4 Kanayama et al.130p = "sources/arxiv-2505.04555v3.txt"; print("R4:")131find(p, r"Japan’s spot labor market")132L = lines(p)133e1 = find(p, r"^Employment elasticity w\.r\.t MW"); e1se = next_nonempty(p, e1)134e2 = find(p, r"^Emp\. elasticity w\.r\.t affected wage"); e2se = next_nonempty(p, e2)135print(" SE lines:", e1se.strip(), e2se.strip())136v1, s1 = float(cell(e1, 1)), float(cell(e1se, 1).strip("()"))137v2, s2 = float(cell(e2, 1)), float(cell(e2se, 1).strip("()"))138aw = find(p, r"^Affected wages \|"); ae = find(p, r"^Affected employment \|")139print(f" arithmetic: affected employment/affected wages = {float(cell(ae,1))/float(cell(aw,1)):.3f}")140r4_ok = (v1, s1, v2, s2) == (-0.388, 0.132, -0.732, 0.091)141checks["R4"] = dict(finding_supported=r4_ok,142 locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (column Estimate (SE))",143 note="Table 2 prints -0.388 (0.132) and -0.732 (0.091), the A.1 text calls the latter the own-wage employment elasticity, and the abstract sets the study in Japan's spot labor market; all elements match.")144data_issues.append({"issue": "Kanayama et al. extract: Table 2 rendered with SEs on separate blank-label lines separated by empty lines; the entry year 2025 matches v1 (7 May 2025) while the extract is v3 (16 Mar 2026).",145 "handling": "Each SE assigned to the coefficient row immediately preceding it; year accepted per the first-submission rule."})146147# R5 Tossou148p = "sources/arxiv-2510.03658v1.txt"; print("R5:")149L = lines(p)150tr = find(p, r"^Treatment \| -0\.0911"); trse = next_nonempty(p, tr)151print(" SE line:", trse.strip())152pc = find(p, r"^% Change \|"); mn = find(p, r"^Mean Dep\. Var\. \|")153find(p, r"most preferred specification \(Column \(4\)\)")154b, s = float(cell(tr, 4)), float(cell(trse, 4).strip("()"))155pct4, mean4 = float(cell(pc, 4)), float(cell(mn, 4))156print(f" col(4): beta={b}, SE={s}, %change={pct4}, mean={mean4}, beta/mean={b/mean4:.4f}")157r5_ok = (b == -0.0962 and s == 0.0129 and pct4 == -0.3685)158checks["R5"] = dict(finding_supported=r5_ok,159 locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment / % Change (preferred specification per Section 4.1)",160 note="Table 3 column (4), the stated preferred specification, prints Treatment -0.0962 (0.0129) and % Change -0.3685 (36.85%), matching the citation; the abstract's headline 0.0855 (18.79%) refers to a different outcome/table (Table 2, not in the extract).")161data_issues.append({"issue": f"Tossou extract: -0.0962/{mean4} = {b/mean4:.4f}, while Table 3 prints % Change -0.3685 (consistent if the mean is rounded to 3 decimals); footnotes are rendered inline as '^9^9 9' etc.; the abstract headline (0.0855, 18.79%) comes from Table 2, which is omitted.",162 "handling": "Treated the 0.0001 gap as rounding of the printed mean; footnote artefacts ignored; R5 checked against Table 3 column (4), the outcome the citation names."})163164# ---------------- assemble ----------------165rc = {}166for r in refs:167 i = r["id"]; m = meta[i]; c = checks[i]168 rc[i] = {"title_matches": bool(m["title_matches"]), "year_matches": bool(m["year_matches"]),169 "finding_supported": bool(c["finding_supported"]), "locator": c["locator"], "note": c["note"]}170unver = [i for i in rc if not all(rc[i][k] for k in ("title_matches", "year_matches", "finding_supported"))]171172def nlines(path):173 return path.read_bytes().count(b"\n")174n_rows = {"reference_list.json": len(refs), "README.md": nlines(DATA / "README.md")}175for r in refs:176 n_rows[Path(r["source_file"]).name] = nlines(DATA / r["source_file"])177178res = {"task_id": "F4-conf-04", "reference_checks": rc, "unverifiable_references": unver,179 "n_verified": len(rc) - len(unver), "feasibility": "complete", "feasibility_reason": None,180 "data_issues": data_issues, "n_input_rows": n_rows, "n_analysed": len(rc), "random_seed": None}181(OUT / "results.json").write_text(json.dumps(res, indent=2, ensure_ascii=False))182print("\nUnverifiable:", unver, "n_verified:", res["n_verified"])183for i, v in rc.items():184 print(i, v["title_matches"], v["year_matches"], v["finding_supported"])185