1"""F4-conf-04: verify a reference list against provided source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Uses data/ only; deterministic; prints every header line and table cell relied on."""4import json, math, re, unicodedata5from pathlib import Path67ROOT = Path(__file__).resolve().parent.parent8DATA = ROOT / "data"9refs = json.loads((DATA / "reference_list.json").read_text(encoding="utf-8"))1011def lines_of(rel):12 return (DATA / rel).read_text(encoding="utf-8").split("\n")1314def find(lines, pattern, label):15 """Return first line containing pattern (regex); print it with its line number."""16 for i, l in enumerate(lines, 1):17 if re.search(pattern, l):18 m = re.search(pattern, l); a = max(0, m.start() - 120); b = min(len(l), m.end() + 180)19 print(f" [{label}] L{i}: ...{l.strip()[a:b]}...")20 return l21 raise RuntimeError(f"pattern not found: {pattern}")2223def norm_title(t):24 t = unicodedata.normalize("NFKD", t)25 t = "".join(c for c in t if not unicodedata.combining(c)).lower()26 t = t.replace("&", " and ")27 t = re.sub(r"[^a-z0-9 ]", " ", t)28 # British/American spelling variants treated as typographic29 t = re.sub(r"isation", "ization", t)30 return " ".join(t.split())3132def entry_parts(entry):33 m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", entry)34 return {"authors": m.group(1).strip(), "year": int(m.group(2)),35 "title": m.group(3).strip(), "arxiv": m.group(4)}3637def header_info(lines):38 title = lines[1].split("Title:", 1)[1].strip()39 rec = lines[3]40 body_title = next(l[2:].strip() for l in lines if l.startswith("# "))41 years = [int(y) for y in re.findall(r"submitted (?:\d{1,2} \w{3} )?(\d{4})", rec)]42 first_year, extract_year = years[0], years[-1]43 return title, body_title, rec, first_year, extract_year4445def cells(line):46 return [c.strip() for c in line.split("|")]4748results = {}49n_input_rows = {"reference_list.json": len(refs)}50n_input_rows["README.md"] = (DATA / "README.md").read_text(encoding="utf-8").count("\n")5152for r in refs:53 rid = r["id"]; src = r["source_file"]; sid = Path(src).stem54 L = lines_of(src)55 n_input_rows[Path(src).name] = (DATA / src).read_text(encoding="utf-8").count("\n")56 e = entry_parts(r["entry"])57 htitle, btitle, rec, y1, yv = header_info(L)58 print(f"\n=== {rid} ({src})")59 print(f" entry: {r['entry']}")60 print(f" cited_for: {r['cited_for']}")61 print(f" header Title: {htitle}")62 print(f" body title: {btitle}")63 print(f" {rec.strip()}")64 title_ok = norm_title(e["title"]) == norm_title(htitle) == norm_title(btitle)65 year_ok = e["year"] in (y1, yv)66 arxiv_ok = e["arxiv"] in rec67 print(f" -> title_matches={title_ok}; entry year {e['year']} vs first-submission {y1} / extract version {yv} -> year_matches={year_ok}; arXiv id in record: {arxiv_ok}")6869 # ---------------- finding checks ----------------70 if rid == "R1":71 row = cells(find(L, r"^ATT \| -0\.399", "Table 6 ATT row"))72 se = cells(find(L, r"^\| \(0\.412\)", "Table 6 SE row"))73 pct = cells(find(L, r"^% change \| -32\.9%", "Table 6 % change row"))74 find(L, r"significant at the 5% level", "Sec 6 text")75 find(L, r"staggered introduction of DCRs", "abstract design")76 a5 = float(row[3].rstrip("*")); s5 = float(se[3].strip("()"))77 z = abs(a5) / s5; impl = 100 * (math.exp(a5) - 1)78 print(f" 5 km: ATT={row[3]}, SE={se[3]}, %change={pct[3]}; |t|={z:.3f}; exp(ATT)-1={impl:.1f}%")79 a19 = cells(find(L, r"^ATT \| -0\.431\*\* \| -0\.438", "Table A19 ATT row"))80 a19se = cells(find(L, r"^\| \(0\.218\)", "Table A19 SE row"))81 find(L, r"640 regions", "Table A19 note")82 ok = (row[3] == "-0.431**" and se[3] == "(0.210)" and pct[3] == "-35.0%" and z > 1.96)83 results[rid] = dict(locator=f"{sid} | Table 6, row ATT / % change, column 5 km (with Section 6 text)",84 note=("Table 6 (5 km column) prints ATT -0.431** with SE (0.210) and -35.0%, and the text states 5% significance "85 "for a staggered DiD on hospital records; Table A19 prints a different SE (0.218) for the same CS-DR estimate, "86 "which does not change the conclusion."), finding_supported=ok)87 elif rid == "R2":88 find(L, r"N = 23,549\) preregistered randomized", "abstract")89 t = {}90 for k in ["Wage", "Reemployment", "Weeks on UI"]:91 c = cells(find(L, rf"^{k} \| ", f"Table 2 {k}")); t[k] = c[4]92 ok = (t["Wage"].startswith("$1,153") and "[166, 2,141]" in t["Wage"]93 and t["Reemployment"].startswith("1.47 pp") and t["Weeks on UI"].startswith("-1.99 weeks"))94 results[rid] = dict(locator=f"{sid} | Table 2, rows Wage / Reemployment / Weeks on UI, column Average treatment effect estimate (with abstract for N and design)",95 note="Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract describes a preregistered RCT with N = 23,549.",96 finding_supported=ok)97 elif rid == "R3":98 find(L, r"stacked coefficient is 0\.005", "Sec 3.2 text")99 find(L, r"prices do not jump immediately", "Sec 3.2 text")100 clean = cells(find(L, r"^Clean-onset threshold-2 \|", "Table 3 preferred row"))101 find(L, r"In the preferred clean-onset event-study estimates.*3\.4.3\.7% after four to five years", "Sec 5 Conclusion")102 # event-year coefficient 0.005 log points (~0.5%), not 3.4-3.7%; 3.4-3.7 corresponds to tau=4,5 (0.037, 0.034)103 ok = False104 results[rid] = dict(locator=f"{sid} | Section 3.2 'Headline dynamic pattern' (stacked tau=0 coefficient 0.005) and Table 3, row Clean-onset threshold-2, columns tau=4 (0.037) / tau=5 ({clean[5]})",105 note="The extract reports a gradual rise from about 0.5% (0.005) in the event year to 3.4-3.7% only by years four and five, explicitly stating that prices do not jump immediately, so the 'jump in the entry year with no further appreciation' attribution contradicts the source.",106 finding_supported=ok)107 elif rid == "R4":108 v = {}109 for k, pat in [("miss", r"^Missing jobs"), ("exc", r"^Excess jobs"), ("aw", r"^Affected wages"),110 ("ae", r"^Affected employment"), ("emw", r"^Employment elasticity w\.r\.t MW"),111 ("eaw", r"^Emp\. elasticity w\.r\.t affected wage"), ("below", r"^Job below new MW"),112 ("dmw", r"^% MW changes")]:113 v[k] = float(cells(find(L, pat, "Table 2"))[1])114 se_emw = cells(find(L, r"^\| \(0\.132\)", "Table 2 SE emw"))[1]115 idx = L.index(next(l for l in L if l.startswith("Emp. elasticity w.r.t affected wage")))116 se_eaw = next(l for l in L[idx + 1:] if l.strip()).split("|")[1].strip()117 print(f" SE under own-wage elasticity row: {se_eaw}")118 find(L, r"spot labor market", "abstract")119 print(f" arithmetic: (miss+exc)/%MW = {(v['miss']+v['exc'])/v['dmw']:.3f} (printed {v['emw']}); "120 f"affected emp/affected wage = {v['ae']/v['aw']:.3f} (printed {v['eaw']})")121 ok = (v["emw"] == -0.388 and se_emw == "(0.132)" and v["eaw"] == -0.732 and se_eaw == "(0.091)")122 results[rid] = dict(locator=f"{sid} | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",123 note="Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market (abstract), and the text calls -0.732 the own-wage employment elasticity.",124 finding_supported=ok)125 elif rid == "R5":126 tr = cells(find(L, r"^Treatment \| -0\.0911", "Table 3 Treatment row"))127 se = cells(find(L, r"^\| \(0\.0113\)", "Table 3 SE row"))128 pc = cells(find(L, r"^% Change \|", "Table 3 % change row"))129 mn = cells(find(L, r"^Mean Dep\. Var\.", "Table 3 mean row"))130 find(L, r"most preferred specification \(Column \(4\)\)", "Sec 4.1 text")131 calc = float(tr[4]) / float(mn[4])132 print(f" col (4): coef={tr[4]}, SE={se[4]}, %change={pc[4]}, mean={mn[4]}; coef/mean={calc:.4f}")133 ok = (tr[4] == "-0.0962" and se[4] == "(0.0129)" and pc[4] == "-0.3685")134 results[rid] = dict(locator=f"{sid} | Table 3, row Treatment / % Change, column (4) (preferred specification per Section 4.1)",135 note="Table 3 column (4), identified as preferred in Section 4.1, prints -0.0962 (0.0129) and % change -0.3685 (= -0.0962/0.261), i.e. a 36.85% reduction in the probability of infant death or stillbirth.",136 finding_supported=ok)137 results[rid] = {"title_matches": title_ok, "year_matches": year_ok,138 "finding_supported": bool(results[rid]["finding_supported"]),139 "locator": results[rid]["locator"], "note": results[rid]["note"]}140 if not title_ok or not year_ok:141 results[rid]["note"] += (f" Bibliographic mismatch: entry title/year '{e['title']}' ({e['year']}) vs extract "142 f"'{htitle}' (submitted {y1}" + (f", this version {yv}" if yv != y1 else "") + ").")143 print(f" -> finding_supported={results[rid]['finding_supported']}")144145unverifiable = [k for k, v in results.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]146out = {147 "task_id": "F4-conf-04",148 "reference_checks": results,149 "unverifiable_references": unverifiable,150 "n_verified": len(results) - len(unverifiable),151 "feasibility": "complete",152 "feasibility_reason": None,153 "data_issues": [154 {"issue": "arxiv-2609.27732v1: the same 5 km CS-DR ATT (-0.431**) has SE (0.210) in Table 6 but (0.218) in Table A19; control regions are 608 in Tables 6-7 but '640 regions' in the Table A19 note.",155 "handling": "R1's cited SE (0.210) matches Table 6, the main results table; 5% significance holds under either SE (|t| = " + f"{0.431/0.210:.2f} or {0.431/0.218:.2f}" + "), so the finding check is not affected; the inconsistency is reported."},156 {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes 'fall by 35%', whereas Section 6.1 says approximately 30% and Table 7 prints -0.354 (exp(-0.354)-1 = " + f"{100*(math.exp(-0.354)-1):.1f}" + "%).",157 "handling": "Not part of any cited finding; recorded only."},158 {"issue": "arxiv-2505.04555v3: Table 2 prints the same SE (0.091) for 'Affected employment' and for the own-wage elasticity (a ratio of two estimates), which may be a duplication in the source.",159 "handling": "R4 cites the values as printed, so it is treated as supported; the possible duplication is flagged as a caveat."},160 {"issue": "arxiv-2510.03658v1: footnote text is interleaved with body text (footnotes 9-12 rendered inline); the text cites Table 2 (0.0855, mean 0.455), which is not in the extract.",161 "handling": "The R5 check relies only on Table 3 and the Section 4.1 sentence about column (4); the inline footnotes were read as footnotes."},162 {"issue": "arxiv-2606.14621v1: Tables 2-3 were re-set by the extract editor; 'p ¡ 0.001' rendering defect in Section 3 text.",163 "handling": "Values used verbatim from the re-set Table 2; the defect does not affect any checked value."},164 {"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering (editor's note gives column order).",165 "handling": "Column order taken from the editor's note; tau=4/tau=5 values agree with the Section 3.2 text."},166 ],167 "n_input_rows": n_input_rows,168 "n_analysed": len(results),169 "random_seed": None,170}171(ROOT / "out" / "results.json").write_text(json.dumps(out, indent=2, ensure_ascii=False), encoding="utf-8")172print("\nunverifiable:", unverifiable, "n_verified:", out["n_verified"])173