1"""F4-conf-04: verify five references against the provided source extracts.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json deterministically (no randomness)."""4import json, math, os, re, unicodedata56DATA = "data"7OUT = "out"8SEP = "-" * 7291011def read(rel):12 with open(os.path.join(DATA, rel), encoding="utf-8") as f:13 return f.read()141516def wc_l(rel):17 with open(os.path.join(DATA, rel), "rb") as f:18 return f.read().count(b"\n")192021def norm_title(s):22 s = unicodedata.normalize("NFKD", s)23 s = "".join(c for c in s if not unicodedata.combining(c))24 s = s.lower().replace("&", " and ")25 s = re.sub(r"[^\w\s]", " ", s)26 return re.sub(r"\s+", " ", s).strip()272829def line_starting(text, prefix, nth=0):30 """Return the nth line starting with prefix and the following non-empty line."""31 lines = text.split("\n")32 hits = [i for i, l in enumerate(lines) if l.startswith(prefix)]33 i = hits[nth]34 nxt = next((lines[j] for j in range(i + 1, len(lines)) if lines[j].strip()), "")35 return lines[i], nxt363738def cells(line):39 return [c.strip() for c in line.split("|")]404142def num(s):43 s = s.replace("−", "-").replace("$", "").replace(",", "").replace("%", "")44 m = re.search(r"-?\d+(?:\.\d+)?", s)45 return float(m.group()) if m else None464748refs = json.loads(read("reference_list.json"))49files = ["README.md", "reference_list.json"] + sorted(50 "sources/" + f for f in os.listdir(os.path.join(DATA, "sources")))51n_input_rows = {}52for f in files:53 n_input_rows[f] = len(refs) if f.endswith(".json") else wc_l(f)5455# ---------------------------------------------------------------- header parse56def parse_source(rel):57 t = read(rel)58 head, body = t.split("=" * 20, 1)59 title = re.search(r"^Title: (.*)$", head, re.M).group(1).strip()60 bib = re.search(r"^Bibliographic record: (.*)$", head, re.M).group(1).strip()61 arxiv = re.search(r"arXiv:(\d{4}\.\d{5})v(\d+)", bib)62 body_title = re.search(r"^# (.*)$", body, re.M).group(1).strip()63 vyears = {}64 for v, y in re.findall(r"version (\d+)(?: \(this extract\))? submitted \d+ \w+ (\d{4})", bib):65 vyears[int(v)] = int(y)66 m = re.search(r"submitted \d+ \w+ (\d{4}) \(version (\d+)", bib)67 if m:68 vyears[int(m.group(2))] = int(m.group(1))69 return dict(text=t, title=title, body_title=body_title, bib=bib, arxiv=arxiv.group(1),70 version=int(arxiv.group(2)), vyears=vyears)717273# ---------------------------------------------------------------- finding checks74def el(label, ok, evidence):75 return {"element": label, "ok": bool(ok), "evidence": evidence}767778def check_R1(t):79 att, se = line_starting(t, "ATT | -0.399")80 pct = line_starting(t, "% change | -32.9%")[0]81 hdr = "| 1 km | 3 km | 5 km"82 c_att, c_se, c_pct = cells(att), cells(se), cells(pct)83 print(" Table 6 header:", hdr); print(" ", att); print(" ", se); print(" ", pct)84 a5, s5, p5 = c_att[3], c_se[3], c_pct[3]85 a19, s19 = line_starting(t, "ATT | -0.431** | -0.438**")86 print(" Table A19:", a19, "/", s19)87 implied = 100 * (math.exp(num(a5)) - 1)88 return [89 el("ATT -0.431 at 5 km", num(a5) == -0.431, "Table 6 ATT row, 5 km col: " + a5),90 el("SE 0.210", s5 == "(0.210)", "Table 6 SE row, 5 km col: " + s5),91 el("significant at 5% level", a5.endswith("**") and not a5.endswith("***"),92 "Table 6 stars '**' = p<0.05; text: 'statistically significant at the 5% level'"),93 el("roughly 35% reduction", p5 == "-35.0%" and abs(implied + 35.0) < 0.05,94 f"Table 6 % change 5 km {p5}; exp(-0.431)-1 = {implied:.2f}%"),95 el("within 5 km of a facility", "within 5 km of a facility" in t, "abstract"),96 el("staggered difference-in-differences", "staggered introduction of DCRs" in t97 and "difference-in-differences estimator" in t, "abstract"),98 el("Swiss hospital records", "individual-level hospital records" in t99 and "Switzerland" in t, "abstract"),100 ], {"tableA19_CS_DR_SE": cells(s19)[1], "table6_5km_SE": s5}101102103def check_R2(t):104 w = line_starting(t, "Wage | $33,073")[0]105 r = line_starting(t, "Reemployment | 75.2%")[0]106 u = line_starting(t, "Weeks on UI | 15.2")[0]107 for l in (w, r, u):108 print(" Table 2:", l)109 cw, cr, cu = cells(w)[4], cells(r)[4], cells(u)[4]110 return [111 el("preregistered RCT", "preregistered randomized controlled trial" in t, "abstract"),112 el("N = 23,549", "N = 23,549" in t, "abstract"),113 el("annualised wages +$1,153", cw.startswith("$1,153"), "Table 2 Wage, ATE col: " + cw),114 el("95% CI [166, 2,141]", "95% CI: [166, 2,141]" in cw, "Table 2 Wage, ATE col"),115 el("reemployment +1.47 pp", cr.startswith("1.47 pp"), "Table 2 Reemployment, ATE col: " + cr),116 el("weeks on UI -1.99", cu.startswith("-1.99 weeks"), "Table 2 Weeks on UI, ATE col: " + cu),117 ], {}118119120def check_R3(t):121 m = re.search(r"the stacked coefficient is ([^:]*?) by \\tau=5", t)122 sent = m.group(0)123 print(" Sec 3.2:", sent)124 coefs = {int(k): float(v) for v, k in re.findall(r"(\d\.\d{3}) (?:at|by) \\tau=(\d)", sent)}125 row = line_starting(t, "Clean-onset threshold-2 |")[0]126 print(" Table 3:", row)127 c = cells(row)128 tau0 = coefs[0]129 later_rise = coefs[4] > coefs[0] and coefs[2] > coefs[1] > coefs[0]130 return [131 el("magnitude 3.4–3.7%", "3.4–3.7%" in t and float(c[4]) == 0.037 and float(c[5]) == 0.034,132 f"Table 3 preferred row tau=4 {c[4]}, tau=5 {c[5]}; text '3.4–3.7% after four to five years'"),133 el("jump in the year of entry", abs(tau0) >= 0.034,134 f"Sec 3.2: stacked coefficient at tau=0 is {tau0} (~0.5%), not 3.4–3.7%"),135 el("no further appreciation later", not later_rise,136 f"Sec 3.2: coefficients rise {coefs} ('prices do not jump immediately, but instead drift upward')"),137 ], {}138139140def check_R4(t):141 e1, s1 = line_starting(t, "Employment elasticity w.r.t MW")142 e2, s2 = line_starting(t, "Emp. elasticity w.r.t affected wage")143 aw, saw = line_starting(t, "Affected wages")144 ae, sae = line_starting(t, "Affected employment")145 jb = line_starting(t, "Job below new MW")[0]146 mw = line_starting(t, "% MW changes")[0]147 for l in (e1, s1, e2, s2, aw, saw, ae, sae, jb, mw):148 print(" Table 2:", l)149 v1, v2 = cells(e1)[1], cells(e2)[1]150 ratio = num(cells(ae)[1]) / num(cells(aw)[1])151 mw_el = num(cells(ae)[1]) * num(cells(jb)[1]) / num(cells(mw)[1])152 print(f" arithmetic: affected emp / affected wage = {ratio:.4f}; affected emp * 0.068/0.047 = {mw_el:.4f}")153 return [154 el("Japan's spot labour market", "Japan’s spot labor market" in t, "abstract"),155 el("elasticity w.r.t. MW -0.388", v1 == "-0.388", "Table 2 row 'Employment elasticity w.r.t MW': " + v1),156 el("SE 0.132", cells(s1)[1] == "(0.132)", "Table 2 SE row: " + cells(s1)[1]),157 el("own-wage elasticity -0.732", v2 == "-0.732", "Table 2 row 'Emp. elasticity w.r.t affected wage': " + v2),158 el("SE 0.091", cells(s2)[1] == "(0.091)", "Table 2 SE row: " + cells(s2)[1]),159 ], {"ratio": ratio, "mw_el": mw_el, "own_wage_se": cells(s2)[1], "aff_emp_se": cells(sae)[1]}160161162def check_R5(t):163 tr, se = line_starting(t, "Treatment | -0.0911")164 pc = line_starting(t, "% Change |")[0]165 mn = line_starting(t, "Mean Dep. Var. |")[0]166 for l in (tr, se, pc, mn):167 print(" Table 3:", l)168 b4, s4, p4, m4 = cells(tr)[4], cells(se)[4], cells(pc)[4], cells(mn)[4]169 implied = num(b4) / num(m4)170 print(f" arithmetic: -0.0962/0.261 = {implied:.4f}")171 return [172 el("infant death or stillbirth outcome", "Probability of infant death and stillbirth" in t, "Table 3 title"),173 el("estimate -0.0962", b4 == "-0.0962", "Table 3 Treatment row, col (4): " + b4),174 el("SE 0.0129", s4 == "(0.0129)", "Table 3 SE row, col (4): " + s4),175 el("preferred specification", "most preferred specification (Column (4))" in t, "Sec 4.1"),176 el("36.85% reduction vs mean", p4 == "-0.3685" and "36.85 percent" in t,177 f"Table 3 % Change col (4) {p4}; Sec 4.1 '36.85 percent'; recomputed {implied:.4f}"),178 ], {}179180181CHECKERS = {"R1": check_R1, "R2": check_R2, "R3": check_R3, "R4": check_R4, "R5": check_R5}182LOCATORS = {183 "R1": "arxiv-2609.27732v1 | Table 6, ATT and % change rows, 5 km column (plus abstract for design/population)",184 "R2": "arxiv-2606.14621v1 | Table 2, rows Wage / Reemployment / Weeks on UI, column 'Average treatment effect estimate' (plus abstract for N and design)",185 "R3": "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row 'Clean-onset threshold-2', tau=4/tau=5 columns",186 "R4": "arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",187 "R5": "arxiv-2510.03658v1 | Table 3, Treatment row and % Change row, column (4); Section 4.1",188}189190reference_checks, details, extra = {}, {}, {}191for r in refs:192 rid = r["id"]193 src = parse_source(r["source_file"])194 print(SEP); print(rid, "->", r["source_file"])195 print(" header Title:", src["title"]); print(" body title :", src["body_title"])196 print(" Bibliographic record:", src["bib"])197 m = re.search(r"\(([\d/]+)\)\.\s*(.*)\.\s*arXiv:(\d{4}\.\d{5})", r["entry"])198 e_year, e_title, e_id = int(m.group(1)[:4]), m.group(2).strip(), m.group(3)199 print(" entry title :", e_title, "| entry year:", e_year, "| entry id:", e_id)200 title_ok = norm_title(e_title) == norm_title(src["title"]) == norm_title(src["body_title"])201 admissible = sorted({src["vyears"][1], src["vyears"][src["version"]]})202 year_ok = e_year in admissible203 print(" admissible years (v1, reproduced version):", admissible)204 assert e_id == src["arxiv"], "arXiv id mismatch"205 elements, ex = CHECKERS[rid](src["text"])206 extra[rid] = ex207 find_ok = all(e["ok"] for e in elements)208 for e in elements:209 print(f" [{'OK' if e['ok'] else 'FAIL'}] {e['element']} :: {e['evidence']}")210 details[rid] = elements211 failed = [e["element"] for e in elements if not e["ok"]]212 parts = []213 if not title_ok:214 parts.append(f"entry title '{e_title}' differs from the document title '{src['title']}'")215 if not year_ok:216 parts.append(f"entry year {e_year} not in admissible years {admissible}")217 if failed:218 parts.append("finding elements not supported: " + "; ".join(failed))219 note = ("All elements of the attributed finding, the title and the year agree with the extract."220 if not parts else ("; ".join(parts) + "."))221 note = note[0].upper() + note[1:]222 reference_checks[rid] = {"title_matches": bool(title_ok), "year_matches": bool(year_ok),223 "finding_supported": bool(find_ok), "locator": LOCATORS[rid],224 "note": note}225226unverifiable = [k for k, v in reference_checks.items()227 if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]228229# ---------------------------------------------------------------- data issues (detected from the extracts)230t1 = read("sources/arxiv-2609.27732v1.txt")231data_issues = []232if extra["R1"]["tableA19_CS_DR_SE"] != extra["R1"]["table6_5km_SE"]:233 data_issues.append({234 "issue": f"arxiv-2609.27732v1: the same CS-DR 5 km ATT (-0.431**) has SE {extra['R1']['table6_5km_SE']} in Table 6 but {extra['R1']['tableA19_CS_DR_SE']} in Table A19 (CS-DR column).",235 "handling": "The R1 citation quotes SE 0.210, which is printed in Table 6, the main results table; the finding is treated as supported, and the discrepancy is reported here and in the report."})236if "(640 regions)" in t1 and "608 never-treated" in t1:237 data_issues.append({238 "issue": "arxiv-2609.27732v1: the Table A19 note gives 640 never-treated comparison regions, while Table 6, Table A19 body and the Figure 6 note give 608.",239 "handling": "Not relevant to any cited element; recorded only."})240if "fall by 35%" in t1 and "approximately 30% at 5 km" in t1:241 data_issues.append({242 "issue": f"arxiv-2609.27732v1: the abstract says fatal outcomes fall by 35%, while Section 6.1 says approximately 30% at 5 km (Table 7 fatal 5 km ATT -0.354; exp(-0.354)-1 = {100 * (math.exp(-0.354) - 1):.1f}%).",243 "handling": "Not part of the R1 citation (which concerns the overall outcome); recorded only."})244r4 = extra["R4"]245if r4["own_wage_se"] == r4["aff_emp_se"]:246 data_issues.append({247 "issue": f"arxiv-2505.04555v3 Table 2: the own-wage elasticity SE {r4['own_wage_se']} is identical to the 'Affected employment' SE, although the point estimate equals affected employment / affected wages ({r4['ratio']:.4f}); the MW-elasticity SE (0.132) matches 0.091*0.068/0.047 = {0.091*0.068/0.047:.4f}, whereas the analogous scaling for the own-wage elasticity (0.091/0.366) gives {0.091/0.366:.3f}, not 0.091. The printed SE may be a transcription error in the source (it cannot be resolved from the extract).",248 "handling": "The R4 citation reproduces the printed values exactly, so the finding is treated as stated in the source; the possible source-level inconsistency is reported, not corrected."})249data_issues.append({250 "issue": "arxiv-2603.03260v2 Table 3: the header row is garbled in the HTML rendering (editor's note supplies the column order).",251 "handling": "Used the editor-supplied column order; the tau=4/tau=5 values (0.037/0.034) agree with the Section 3.2 text."})252253# Baseline (P2): header-only screen using title and year checks alone, no finding check.254baseline_header_only = [k for k, v in reference_checks.items()255 if not (v["title_matches"] and v["year_matches"])]256257results = {258 "task_id": "F4-conf-04",259 "reference_checks": reference_checks,260 "unverifiable_references": unverifiable,261 "n_verified": len(refs) - len(unverifiable),262 "feasibility": "complete",263 "feasibility_reason": None,264 "data_issues": data_issues,265 "n_input_rows": n_input_rows,266 "n_analysed": len(refs),267 "random_seed": None,268 "baseline_header_only_unverifiable": baseline_header_only,269}270os.makedirs(OUT, exist_ok=True)271with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:272 json.dump(results, f, indent=2, ensure_ascii=False)273with open(os.path.join(OUT, "finding_elements.json"), "w", encoding="utf-8") as f:274 json.dump(details, f, indent=2, ensure_ascii=False)275print(SEP)276print("unverifiable_references:", unverifiable, "| n_verified:", results["n_verified"])277