"""F4-conf-04: verify a reference list against provided source extracts. Run from workspace root: ~/wp29/venv/bin/python out/analysis.py Uses data/ only; deterministic; prints every header line and table cell relied on.""" import json, math, re, unicodedata from pathlib import Path ROOT = Path(__file__).resolve().parent.parent DATA = ROOT / "data" refs = json.loads((DATA / "reference_list.json").read_text(encoding="utf-8")) def lines_of(rel): return (DATA / rel).read_text(encoding="utf-8").split("\n") def find(lines, pattern, label): """Return first line containing pattern (regex); print it with its line number.""" for i, l in enumerate(lines, 1): if re.search(pattern, l): m = re.search(pattern, l); a = max(0, m.start() - 120); b = min(len(l), m.end() + 180) print(f" [{label}] L{i}: ...{l.strip()[a:b]}...") return l raise RuntimeError(f"pattern not found: {pattern}") def norm_title(t): t = unicodedata.normalize("NFKD", t) t = "".join(c for c in t if not unicodedata.combining(c)).lower() t = t.replace("&", " and ") t = re.sub(r"[^a-z0-9 ]", " ", t) # British/American spelling variants treated as typographic t = re.sub(r"isation", "ization", t) return " ".join(t.split()) def entry_parts(entry): m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", entry) return {"authors": m.group(1).strip(), "year": int(m.group(2)), "title": m.group(3).strip(), "arxiv": m.group(4)} def header_info(lines): title = lines[1].split("Title:", 1)[1].strip() rec = lines[3] body_title = next(l[2:].strip() for l in lines if l.startswith("# ")) years = [int(y) for y in re.findall(r"submitted (?:\d{1,2} \w{3} )?(\d{4})", rec)] first_year, extract_year = years[0], years[-1] return title, body_title, rec, first_year, extract_year def cells(line): return [c.strip() for c in line.split("|")] results = {} n_input_rows = {"reference_list.json": len(refs)} n_input_rows["README.md"] = (DATA / "README.md").read_text(encoding="utf-8").count("\n") for r in refs: rid = r["id"]; src = r["source_file"]; sid = Path(src).stem L = lines_of(src) n_input_rows[Path(src).name] = (DATA / src).read_text(encoding="utf-8").count("\n") e = entry_parts(r["entry"]) htitle, btitle, rec, y1, yv = header_info(L) print(f"\n=== {rid} ({src})") print(f" entry: {r['entry']}") print(f" cited_for: {r['cited_for']}") print(f" header Title: {htitle}") print(f" body title: {btitle}") print(f" {rec.strip()}") title_ok = norm_title(e["title"]) == norm_title(htitle) == norm_title(btitle) year_ok = e["year"] in (y1, yv) arxiv_ok = e["arxiv"] in rec print(f" -> title_matches={title_ok}; entry year {e['year']} vs first-submission {y1} / extract version {yv} -> year_matches={year_ok}; arXiv id in record: {arxiv_ok}") # ---------------- finding checks ---------------- if rid == "R1": row = cells(find(L, r"^ATT \| -0\.399", "Table 6 ATT row")) se = cells(find(L, r"^\| \(0\.412\)", "Table 6 SE row")) pct = cells(find(L, r"^% change \| -32\.9%", "Table 6 % change row")) find(L, r"significant at the 5% level", "Sec 6 text") find(L, r"staggered introduction of DCRs", "abstract design") a5 = float(row[3].rstrip("*")); s5 = float(se[3].strip("()")) z = abs(a5) / s5; impl = 100 * (math.exp(a5) - 1) print(f" 5 km: ATT={row[3]}, SE={se[3]}, %change={pct[3]}; |t|={z:.3f}; exp(ATT)-1={impl:.1f}%") a19 = cells(find(L, r"^ATT \| -0\.431\*\* \| -0\.438", "Table A19 ATT row")) a19se = cells(find(L, r"^\| \(0\.218\)", "Table A19 SE row")) find(L, r"640 regions", "Table A19 note") ok = (row[3] == "-0.431**" and se[3] == "(0.210)" and pct[3] == "-35.0%" and z > 1.96) results[rid] = dict(locator=f"{sid} | Table 6, row ATT / % change, column 5 km (with Section 6 text)", note=("Table 6 (5 km column) prints ATT -0.431** with SE (0.210) and -35.0%, and the text states 5% significance " "for a staggered DiD on hospital records; Table A19 prints a different SE (0.218) for the same CS-DR estimate, " "which does not change the conclusion."), finding_supported=ok) elif rid == "R2": find(L, r"N = 23,549\) preregistered randomized", "abstract") t = {} for k in ["Wage", "Reemployment", "Weeks on UI"]: c = cells(find(L, rf"^{k} \| ", f"Table 2 {k}")); t[k] = c[4] ok = (t["Wage"].startswith("$1,153") and "[166, 2,141]" in t["Wage"] and t["Reemployment"].startswith("1.47 pp") and t["Weeks on UI"].startswith("-1.99 weeks")) results[rid] = dict(locator=f"{sid} | Table 2, rows Wage / Reemployment / Weeks on UI, column Average treatment effect estimate (with abstract for N and design)", note="Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract describes a preregistered RCT with N = 23,549.", finding_supported=ok) elif rid == "R3": find(L, r"stacked coefficient is 0\.005", "Sec 3.2 text") find(L, r"prices do not jump immediately", "Sec 3.2 text") clean = cells(find(L, r"^Clean-onset threshold-2 \|", "Table 3 preferred row")) find(L, r"In the preferred clean-onset event-study estimates.*3\.4.3\.7% after four to five years", "Sec 5 Conclusion") # event-year coefficient 0.005 log points (~0.5%), not 3.4-3.7%; 3.4-3.7 corresponds to tau=4,5 (0.037, 0.034) ok = False results[rid] = dict(locator=f"{sid} | Section 3.2 'Headline dynamic pattern' (stacked tau=0 coefficient 0.005) and Table 3, row Clean-onset threshold-2, columns tau=4 (0.037) / tau=5 ({clean[5]})", note="The extract reports a gradual rise from about 0.5% (0.005) in the event year to 3.4-3.7% only by years four and five, explicitly stating that prices do not jump immediately, so the 'jump in the entry year with no further appreciation' attribution contradicts the source.", finding_supported=ok) elif rid == "R4": v = {} for k, pat in [("miss", r"^Missing jobs"), ("exc", r"^Excess jobs"), ("aw", r"^Affected wages"), ("ae", r"^Affected employment"), ("emw", r"^Employment elasticity w\.r\.t MW"), ("eaw", r"^Emp\. elasticity w\.r\.t affected wage"), ("below", r"^Job below new MW"), ("dmw", r"^% MW changes")]: v[k] = float(cells(find(L, pat, "Table 2"))[1]) se_emw = cells(find(L, r"^\| \(0\.132\)", "Table 2 SE emw"))[1] idx = L.index(next(l for l in L if l.startswith("Emp. elasticity w.r.t affected wage"))) se_eaw = next(l for l in L[idx + 1:] if l.strip()).split("|")[1].strip() print(f" SE under own-wage elasticity row: {se_eaw}") find(L, r"spot labor market", "abstract") print(f" arithmetic: (miss+exc)/%MW = {(v['miss']+v['exc'])/v['dmw']:.3f} (printed {v['emw']}); " f"affected emp/affected wage = {v['ae']/v['aw']:.3f} (printed {v['eaw']})") ok = (v["emw"] == -0.388 and se_emw == "(0.132)" and v["eaw"] == -0.732 and se_eaw == "(0.091)") results[rid] = dict(locator=f"{sid} | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'", note="Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market (abstract), and the text calls -0.732 the own-wage employment elasticity.", finding_supported=ok) elif rid == "R5": tr = cells(find(L, r"^Treatment \| -0\.0911", "Table 3 Treatment row")) se = cells(find(L, r"^\| \(0\.0113\)", "Table 3 SE row")) pc = cells(find(L, r"^% Change \|", "Table 3 % change row")) mn = cells(find(L, r"^Mean Dep\. Var\.", "Table 3 mean row")) find(L, r"most preferred specification \(Column \(4\)\)", "Sec 4.1 text") calc = float(tr[4]) / float(mn[4]) print(f" col (4): coef={tr[4]}, SE={se[4]}, %change={pc[4]}, mean={mn[4]}; coef/mean={calc:.4f}") ok = (tr[4] == "-0.0962" and se[4] == "(0.0129)" and pc[4] == "-0.3685") results[rid] = dict(locator=f"{sid} | Table 3, row Treatment / % Change, column (4) (preferred specification per Section 4.1)", note="Table 3 column (4), identified as preferred in Section 4.1, prints -0.0962 (0.0129) and % change -0.3685 (= -0.0962/0.261), i.e. a 36.85% reduction in the probability of infant death or stillbirth.", finding_supported=ok) results[rid] = {"title_matches": title_ok, "year_matches": year_ok, "finding_supported": bool(results[rid]["finding_supported"]), "locator": results[rid]["locator"], "note": results[rid]["note"]} if not title_ok or not year_ok: results[rid]["note"] += (f" Bibliographic mismatch: entry title/year '{e['title']}' ({e['year']}) vs extract " f"'{htitle}' (submitted {y1}" + (f", this version {yv}" if yv != y1 else "") + ").") print(f" -> finding_supported={results[rid]['finding_supported']}") unverifiable = [k for k, v in results.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])] out = { "task_id": "F4-conf-04", "reference_checks": results, "unverifiable_references": unverifiable, "n_verified": len(results) - len(unverifiable), "feasibility": "complete", "feasibility_reason": None, "data_issues": [ {"issue": "arxiv-2609.27732v1: the same 5 km CS-DR ATT (-0.431**) has SE (0.210) in Table 6 but (0.218) in Table A19; control regions are 608 in Tables 6-7 but '640 regions' in the Table A19 note.", "handling": "R1's cited SE (0.210) matches Table 6, the main results table; 5% significance holds under either SE (|t| = " + f"{0.431/0.210:.2f} or {0.431/0.218:.2f}" + "), so the finding check is not affected; the inconsistency is reported."}, {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes 'fall by 35%', whereas Section 6.1 says approximately 30% and Table 7 prints -0.354 (exp(-0.354)-1 = " + f"{100*(math.exp(-0.354)-1):.1f}" + "%).", "handling": "Not part of any cited finding; recorded only."}, {"issue": "arxiv-2505.04555v3: Table 2 prints the same SE (0.091) for 'Affected employment' and for the own-wage elasticity (a ratio of two estimates), which may be a duplication in the source.", "handling": "R4 cites the values as printed, so it is treated as supported; the possible duplication is flagged as a caveat."}, {"issue": "arxiv-2510.03658v1: footnote text is interleaved with body text (footnotes 9-12 rendered inline); the text cites Table 2 (0.0855, mean 0.455), which is not in the extract.", "handling": "The R5 check relies only on Table 3 and the Section 4.1 sentence about column (4); the inline footnotes were read as footnotes."}, {"issue": "arxiv-2606.14621v1: Tables 2-3 were re-set by the extract editor; 'p ยก 0.001' rendering defect in Section 3 text.", "handling": "Values used verbatim from the re-set Table 2; the defect does not affect any checked value."}, {"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering (editor's note gives column order).", "handling": "Column order taken from the editor's note; tau=4/tau=5 values agree with the Section 3.2 text."}, ], "n_input_rows": n_input_rows, "n_analysed": len(results), "random_seed": None, } (ROOT / "out" / "results.json").write_text(json.dumps(out, indent=2, ensure_ascii=False), encoding="utf-8") print("\nunverifiable:", unverifiable, "n_verified:", out["n_verified"])