"""F4-conf-04: verify five references against the provided source extracts. Run from the workspace root: ~/wp29/venv/bin/python out/analysis.py Uses data/ only; deterministic; prints every header line and table cell relied upon.""" import json, math, re, unicodedata from pathlib import Path ROOT = Path(__file__).resolve().parent.parent DATA = ROOT / "data" refs = json.loads((DATA / "reference_list.json").read_text(encoding="utf-8")) def lines_of(rel): return (DATA / rel).read_text(encoding="utf-8").split("\n") def find(lines, pattern, show=True): """Return first line matching regex pattern (and print it).""" for i, l in enumerate(lines): if re.search(pattern, l): if show: print(f" L{i+1}: {l.strip()[:300]}") return i, l raise RuntimeError(f"pattern not found: {pattern}") def next_nonblank(lines, i): j = i + 1 while not lines[j].strip(): j += 1 print(f" L{j+1}: {lines[j].strip()}") return lines[j] def norm_title(t): t = unicodedata.normalize("NFKD", t) t = "".join(c for c in t if not unicodedata.combining(c)) t = t.lower().replace("&", "and") t = re.sub(r"[^a-z0-9 ]", " ", t) return " ".join(t.split()) def entry_title_year(entry): m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:", entry) return int(m.group(2)), m.group(3) def header_info(lines): title = lines[1].split("Title:", 1)[1].strip() bib = [l for l in lines[:10] if l.startswith("Bibliographic record:")][0] years = sorted({int(y) for y in re.findall(r"\b\d{1,2} [A-Z][a-z]{2} (\d{4})", bib)}) body_title = [l for l in lines if l.startswith("# ")][0][2:].strip() return title, body_title, bib, years def cell(row, k): return [c.strip() for c in row.split("|")][k] checks, data_issues = {}, [] n_rows = {"reference_list.json": len(refs)} for r in refs: rid, src = r["id"], r["source_file"] L = lines_of(src) n_rows[src.split("/")[-1]] = (DATA / src).read_text(encoding="utf-8").count("\n") sid = src.split("/")[-1].replace(".txt", "") title, body_title, bib, years = header_info(L) ey, et = entry_title_year(r["entry"]) print(f"\n=== {rid} vs {sid}") print(f" header Title: {title}") print(f" body title : {body_title}") print(f" {bib}") print(f" entry year {ey}, entry title: {et}") tm = norm_title(et) == norm_title(title) == norm_title(body_title) first_year, version_year = years[0], years[-1] ym = ey in (first_year, version_year) print(f" title_matches={tm}; years in record={years}; year_matches={ym}") print(" finding evidence:") if rid == "R1": i, row = find(L, r"^ATT \| -0\.399") se = next_nonblank(L, i) _, pc = find(L, r"^% change \| -32\.9%") find(L, r"statistically significant at the 5% level") find(L, r"roughly 35% within 5 km") find(L, r"staggered introduction of DCRs") att, se5, pct = cell(row, 3), cell(se, 3), cell(pc, 3) calc = 100 * (math.exp(-0.431) - 1) print(f" 5 km: ATT {att}, SE {se5}, % change {pct}; exp(-0.431)-1 = {calc:.1f}%") ok = att == "-0.431**" and se5 == "(0.210)" and pct == "-35.0%" loc = f"{sid} | Table 6, column '5 km', rows ATT / SE / % change (+ abstract)" note = ("Table 6 5 km column prints ATT -0.431** (SE 0.210), -35.0%, significant at 5%; abstract gives " "roughly 35% within 5 km from staggered DCR openings using Swiss hospital records.") j, a19 = find(L, r"^ATT \| -0\.431\*\* \| -0\.438") se19 = next_nonblank(L, j) find(L, r"never-treated comparison regions \(640 regions\)") data_issues.append({"issue": f"{sid}: the same 5 km CS-DR ATT (-0.431**) has SE (0.210) in Table 6 but {cell(se19,1)} in Table A19; " "Table A19 note says 640 never-treated regions while Tables 6/A19 rows say 608.", "handling": "Cited SE 0.210 matches Table 6 (main results table) exactly, so it is counted as stated; " "the 5% significance conclusion holds under either SE (|t| = 2.05 or 1.98). Inconsistency reported, not resolved."}) find(L, r"which fall by 35%") find(L, r"by approximately 30% at 5 km") data_issues.append({"issue": f"{sid}: abstract says fatal hospitalisations fall by 35%, Section 6.1 says ~30% (Table 7 5 km fatal ATT -0.354, exp-1 = {100*(math.exp(-0.354)-1):.1f}%).", "handling": "Does not concern the cited finding (overall ATT); reported only."}) elif rid == "R2": find(L, r"N = 23,549\) preregistered randomized controlled trial") rows = {} for key, pat in [("Wage", r"^Wage \|"), ("Reemployment", r"^Reemployment \|"), ("Weeks on UI", r"^Weeks on UI \|")]: _, row = find(L, pat) rows[key] = cell(row, 4) print(f" ATT column: {rows}") ok = (rows["Wage"].startswith("$1,153;") and "95% CI: [166, 2,141]" in rows["Wage"] and rows["Reemployment"].startswith("1.47 pp;") and rows["Weeks on UI"].startswith("-1.99 weeks;")) loc = f"{sid} | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI (+ abstract for N, preregistered RCT)" note = ("Table 2 ATT column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks; abstract states N = 23,549 " "preregistered RCT and that the wage effect is annualised.") find(L, r"p ¡ 0\.001") data_issues.append({"issue": f"{sid}: rendering defect 'p ¡ 0.001' (should read p < 0.001) in Section 3 text; Tables 2-3 were re-set by the extract editor.", "handling": "Does not affect cited values, which were read from Table 2 and cross-checked against Table 3 column 1 and the text."}) elif rid == "R3": find(L, r"stacked coefficient is 0\.005 at") find(L, r"prices do not jump immediately") find(L, r"^Clean-onset threshold-2 \|") ok = False loc = f"{sid} | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) and Table 3 row 'Clean-onset threshold-2', tau=4/tau=5 columns" note = ("Title and year (2024) do not match the extract (title 'Does Entry of Food-and-Drink Establishments Raise Local House Prices?...', " "submitted 2026), and the source reports a gradual rise from 0.005 at tau=0 to 0.037/0.034 at tau=4/5 with 'prices do not jump immediately', " "contradicting a 3.4-3.7% jump in the entry year with no later appreciation.") data_issues.append({"issue": f"{sid}: Table 3 header row garbled in the HTML rendering (column names restored by the extract editor's note).", "handling": "Used the editor-supplied column order; tau=4/5 values 0.037/0.034 agree with the Section 3.2 text."}) elif rid == "R4": find(L, r"Japan’s spot labor market") i, row = find(L, r"^Employment elasticity w\.r\.t MW \|") se1 = next_nonblank(L, i) j, row2 = find(L, r"^Emp\. elasticity w\.r\.t affected wage \|") se2 = next_nonblank(L, j) k, ae = find(L, r"^Affected employment \|") se_ae = next_nonblank(L, k) _, aw = find(L, r"^Affected wages \|") ok = cell(row, 1) == "-0.388" and cell(se1, 1) == "(0.132)" and cell(row2, 1) == "-0.732" and cell(se2, 1) == "(0.091)" ratio = float(cell(ae, 1)) / float(cell(aw, 1)) print(f" arithmetic: affected employment / affected wages = {ratio:.3f} (printed own-wage elasticity -0.732)") loc = f"{sid} | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (+ abstract for Japan's spot labour market)" note = ("Table 2 prints -0.388 (0.132) and -0.732 (0.091), and the abstract places the study in Japan's spot labor market; " "year 2025 equals the v1 submission year.") data_issues.append({"issue": f"{sid}: SE printed for the own-wage elasticity (0.091) is identical to the SE of 'Affected employment' (0.091), although the elasticity is the ratio " f"affected employment / affected wages ({ratio:.3f}) and affected wages has SE 0.171; possible duplication in the source table.", "handling": "Cannot be resolved from the extract; the cited SE equals the printed value so the finding is counted as stated, with this caveat."}) elif rid == "R5": i, row = find(L, r"^Treatment \| -0\.0911") se = next_nonblank(L, i) _, pc = find(L, r"^% Change \|") _, mean = find(L, r"^Mean Dep\. Var\. \|") find(L, r"most preferred specification \(Column \(4\)\)") b, s, p, m = cell(row, 4), cell(se, 4), cell(pc, 4), cell(mean, 4) print(f" col (4): beta {b}, SE {s}, % change {p}, mean {m}; beta/mean = {float(b)/float(m):.4f}") ok = b == "-0.0962" and s == "(0.0129)" and p == "-0.3685" loc = f"{sid} | Table 3, column (4) (preferred), rows Treatment / SE / % Change (+ Section 4.1 text)" note = ("Table 3 column (4), identified in Section 4.1 as the preferred specification, prints -0.0962 (0.0129) with % change -0.3685 " "(=-0.0962/0.261), i.e. 36.85%.") find(L, r"result in table 2 suggests") data_issues.append({"issue": f"{sid}: Section 4.1 attributes the 0.0855 / 18.79% estimate (also the abstract headline) to Table 2, which is omitted from the extract; " "abstract and Table 3 thus report different headline magnitudes for different outcome definitions.", "handling": "Cited finding refers explicitly to the Table 3 binary outcome in the preferred column, which is in the extract; the abstract figure was not used."}) checks[rid] = {"title_matches": bool(tm), "year_matches": bool(ym), "finding_supported": bool(ok), "locator": loc, "note": note} print(f" -> {rid}: {checks[rid]['title_matches']}, {checks[rid]['year_matches']}, {checks[rid]['finding_supported']}") n_rows["README.md"] = (DATA / "README.md").read_text(encoding="utf-8").count("\n") unver = [k for k, v in checks.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])] results = { "task_id": "F4-conf-04", "reference_checks": checks, "unverifiable_references": unver, "n_verified": len(checks) - len(unver), "feasibility": "complete", "feasibility_reason": None, "data_issues": data_issues, "n_input_rows": dict(sorted(n_rows.items())), "n_analysed": len(checks), "random_seed": None, } (ROOT / "out" / "results.json").write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8") print("\nunverifiable:", unver, "n_verified:", results["n_verified"])