1"""F4-conf-04: verify five references against the provided source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads data/ only; prints every header line and table cell it relies on; writes out/results.json.4The judgements are encoded as explicit, mechanical checks on strings/numbers found in the extracts."""5import json, math, os, re, unicodedata67D = "data"8refs = json.load(open(os.path.join(D, "reference_list.json")))9src = {}10for r in refs:11 src[r["id"]] = open(os.path.join(D, r["source_file"]), encoding="utf-8").read().splitlines()1213def find(rid, pattern):14 """Return first line (1-based no, text) matching regex; print it; fail loudly if absent."""15 for i, l in enumerate(src[rid]):16 if re.search(pattern, l):17 print(f" [{rid} L{i+1}] {l.strip()[:220]}")18 return l19 raise SystemExit(f"pattern not found in {rid}: {pattern}")2021def norm_title(t):22 t = unicodedata.normalize("NFKD", t).encode("ascii", "ignore").decode().lower()23 t = t.replace("&", "and")24 return re.sub(r"[^a-z0-9]+", " ", t).strip()2526def entry_title(entry):27 # entry format: "Authors (YEAR). Title. arXiv:NNNN."28 m = re.match(r".*?\((\d{4})\)\.\s*(.*)\.\s*arXiv:(\S+?)\.?$", entry)29 return int(m.group(1)), m.group(2), m.group(3)3031def cells(line):32 return [c.strip() for c in line.split("|")]3334checks, issues = {}, []35for r in refs:36 rid = r["id"]37 print(f"\n=== {rid}: {r['entry']}")38 year, title, arx = entry_title(r["entry"])39 t_line = find(rid, r"^Title: ")40 b_line = find(rid, r"^Bibliographic record: ")41 doc_title = t_line.split("Title: ", 1)[1]42 years = sorted({int(y) for y in re.findall(r"\d{1,2} \w{3} (\d{4})", b_line)})43 first_year = years[0]44 this_version_year = int(re.findall(r"\d{1,2} \w{3} (\d{4})", b_line)[-1])45 id_ok = b_line.split("arXiv:")[1].split("v")[0] == arx.split("v")[0]46 title_ok = norm_title(title) == norm_title(doc_title)47 year_ok = year in (first_year, this_version_year)48 print(f" entry year={year}; first submission year={first_year}; reproduced version year={this_version_year}; arXiv id match={id_ok}")49 print(f" title match={title_ok}; year match={year_ok}")50 checks[rid] = dict(title_matches=title_ok, year_matches=year_ok)5152# ---------------- finding checks ----------------53print("\n--- Finding checks ---")54# R155print("R1:")56abs1 = find("R1", r"reduce drug-related hospitalization rates by roughly 35% within 5 km")57find("R1", r"staggered introduction of DCRs")58att = cells(find("R1", r"^ATT \| -0.399"))59se = cells(find("R1", r"^\| \(0.412\)"))60pct = cells(find("R1", r"^% change \| -32.9%"))61sig = find("R1", r"statistically significant at the 5% level")62a5, s5 = float(att[3].rstrip("*")), float(se[3].strip("()"))63print(f" Table 6, 5 km: ATT={att[3]}, SE={se[3]}, %change={pct[3]}; exp(ATT)-1={math.exp(a5)-1:.4f}; |t|={abs(a5)/s5:.3f}")64r1 = (att[3] == "-0.431**" and se[3] == "(0.210)" and pct[3] == "-35.0%" and abs(a5) / s5 > 1.96)65a19 = cells(find("R1", r"^ATT \| -0.431\*\* \| -0.438"))66a19se = cells(find("R1", r"^\| \(0.218\)"))67find("R1", r"640 regions")68print(f" Table A19 CS-DR: ATT={a19[1]}, SE={a19se[1]} (vs Table 6 SE {se[3]})")69checks["R1"].update(finding_supported=bool(r1),70 locator="arxiv-2609.27732v1 | Table 6, row 'ATT' (and SE row), column '5 km'; abstract",71 note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, the text states significance at 5% and the abstract describes staggered DiD on individual-level hospital records; Table A19 prints SE (0.218) for the same CS-DR estimate, which does not change the 5% significance.")72issues.append({"issue": "arxiv-2609.27732v1: the same CS-DR 5 km ATT (-0.431**) has SE (0.210) in Table 6 but (0.218) in Table A19; the A19 note says 640 never-treated regions while Tables 6, A19 and Figure 6 note say 608.",73 "handling": "Treated Table 6 (main results table) as decisive for the cited SE; checked that significance at 5% holds under either SE (|t|=2.05 vs 1.98); kept R1 as supported but flagged."})74issues.append({"issue": "arxiv-2609.27732v1: abstract says fatal outcomes 'fall by 35%', while Table 7 (5 km: -0.354**) and Section 6.1 imply about 30% (exp(-0.354)-1 = -29.8%).",75 "handling": "Not part of the cited finding for R1; documented only."})7677# R278print("R2:")79find("R2", r"N = 23,549\) preregistered randomized controlled trial")80w = cells(find("R2", r"^Wage \| \$33,073"))81re_ = cells(find("R2", r"^Reemployment \| 75.2%"))82ui = cells(find("R2", r"^Weeks on UI \| 15.2"))83print(f" Table 2 ATE column: wage='{w[4]}', reemp='{re_[4]}', UI='{ui[4]}'")84r2 = (w[4].startswith("$1,153") and "[166, 2,141]" in w[4] and re_[4].startswith("1.47 pp")85 and ui[4].startswith("-1.99 weeks"))86checks["R2"].update(finding_supported=bool(r2),87 locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI; abstract (N, preregistered RCT)",88 note="Abstract gives a preregistered RCT with N = 23,549 and Table 2's ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, matching every element of the citation.")8990# R391print("R3:")92find("R3", r"rise gradually from about 0.5% in the event year to roughly 3.4")93coef = find("R3", r"stacked coefficient is 0.005 at")94find("R3", r"prices do not jump immediately")95row = cells(find("R3", r"^Clean-onset threshold-2 \|"))96tau0 = float(re.search(r"coefficient is ([\d.]+) at \\tau=0", coef).group(1))97print(f" event-year (tau=0) coefficient={tau0}; Table 3 preferred tau=4={row[4]}, tau=5={row[5]}")98# Cited: 3.4-3.7% jump in the event year, no further appreciation later. Source: 0.005 at tau=0, rising to 0.037/0.034 by tau=4/5.99r3 = tau0 >= 0.034 and float(row[4]) <= tau0100checks["R3"].update(finding_supported=bool(r3),101 locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficient 0.005 at tau=0); Table 3, row 'Clean-onset threshold-2', columns tau=4 (0.037) and tau=5 (0.034)",102 note="The extract reports a gradual rise from about 0.5% (0.005) in the event year to 3.4-3.7% only by years four and five and says prices do not jump immediately, contradicting the cited immediate jump; the entry's title and year (2024) also do not match the document (submitted 2026).")103104# R4105print("R4:")106find("R4", r"Japan’s spot labor market|Japan's spot labor market")107e1 = find("R4", r"^Employment elasticity w.r.t MW")108i1 = src["R4"].index(e1); s1 = src["R4"][i1 + 2]; print(f" [R4 L{i1+3}] {s1}")109e2 = find("R4", r"^Emp. elasticity w.r.t affected wage")110i2 = src["R4"].index(e2); s2 = src["R4"][i2 + 2]; print(f" [R4 L{i2+3}] {s2}")111aw = float(cells(find("R4", r"^Affected wages \|"))[1]); ae = float(cells(find("R4", r"^Affected employment \|"))[1])112print(f" arithmetic check: affected employment / affected wages = {ae/aw:.3f}")113r4 = (cells(e1)[1] == "-0.388" and "(0.132)" in s1 and cells(e2)[1] == "-0.732" and "(0.091)" in s2)114checks["R4"].update(finding_supported=bool(r4),115 locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (Estimate (SE) column); abstract (Japan's spot labor market)",116 note="Table 2 prints -0.388 (0.132) and -0.732 (0.091), the latter equal to -0.268/0.366, and the abstract places the study in Japan's spot labor market; 2025 equals the v1 submission year.")117118# R5119print("R5:")120tr = cells(find("R5", r"^Treatment \| -0.0911"))121trse = cells(find("R5", r"^\| \(0.0113\) \| \(0.0113\)"))122pc = cells(find("R5", r"^% Change \|"))123mn = cells(find("R5", r"^Mean Dep. Var. \|"))124find("R5", r"most preferred specification \(Column \(4\)\)")125print(f" col (4): beta={tr[4]}, SE={trse[4]}, %change={pc[4]}, mean={mn[4]}; beta/mean={float(tr[4])/float(mn[4]):.4f}")126r5 = tr[4] == "-0.0962" and trse[4] == "(0.0129)" and pc[4] == "-0.3685"127checks["R5"].update(finding_supported=bool(r5),128 locator="arxiv-2510.03658v1 | Table 3, row 'Treatment' (and SE row), '% Change' and 'Mean Dep. Var.', column (4); Section 4.1 (column 4 is preferred)",129 note="Table 3 column (4), named the preferred specification in Section 4.1, prints -0.0962 (0.0129) and % change -0.3685 (= -0.0962/0.261), matching the citation.")130issues.append({"issue": "arxiv-2510.03658v1: Section 4.1 refers to a Table 2 estimate of -0.0855 with dependent-variable mean 0.455 (18.79%), while the abstract reports the same 0.0855 headline; Table 3 (mean 0.261) is a different outcome/specification. Footnote markers are rendered inline ('percent^9^9 9').",131 "handling": "R5 cites the Table 3 column (4) result, which was checked directly; the Table 2 figure is not in the extract and was not used."})132issues.append({"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering (editor's note supplies column order); the event-year coefficient tau=0 is only in the text, not a table; tau=3 coefficient is not reported.",133 "handling": "Used the editor's column order; relied on Section 3.2 text for tau=0 to tau=2."})134135unver = [k for k, v in checks.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]136order = ["title_matches", "year_matches", "finding_supported", "locator", "note"]137checks = {k: {f: v[f] for f in order} for k, v in checks.items()}138139def wc(path):140 with open(path, "rb") as f:141 return f.read().count(b"\n")142nrows = {"README.md": wc(os.path.join(D, "README.md")), "reference_list.json": len(refs)}143for r in refs:144 nrows[r["source_file"]] = wc(os.path.join(D, r["source_file"]))145146res = {"task_id": "F4-conf-04", "reference_checks": checks, "unverifiable_references": unver,147 "n_verified": len(refs) - len(unver), "feasibility": "complete", "feasibility_reason": None,148 "data_issues": issues, "n_input_rows": nrows, "n_analysed": len(refs), "random_seed": None}149json.dump(res, open("out/results.json", "w"), indent=2, ensure_ascii=False)150print("\nunverifiable:", unver, "n_verified:", res["n_verified"])151