1"""F4-conf-04: verify a reference list against provided arXiv extracts.23Reads only data/, writes out/results.json deterministically. No randomness.4Run from workspace root: ~/wp29/venv/bin/python out/analysis.py5"""6import json, math, os, re, unicodedata78DATA = "data"9OUT = os.path.join("out", "results.json")101112def read_lines(path):13 with open(path, encoding="utf-8") as f:14 return f.read().split("\n")151617def wc_l(path):18 with open(path, "rb") as f:19 return f.read().count(b"\n")202122def norm_title(s):23 s = unicodedata.normalize("NFKD", s)24 s = "".join(c for c in s if not unicodedata.combining(c))25 s = s.lower().replace("&", " and ")26 s = re.sub(r"[^a-z0-9 ]", " ", s)27 return " ".join(s.split())282930def find_line(lines, needle):31 """Return (1-based line number, line) of first line containing needle; raise if absent."""32 for i, l in enumerate(lines, 1):33 if needle in l:34 return i, l35 raise AssertionError(f"evidence not found: {needle!r}")363738def show(src, lines, needle):39 i, l = find_line(lines, needle)40 txt = l.strip()41 if len(txt) > 220: # long prose line: print the matched passage with context42 k = txt.index(needle)43 txt = "..." + txt[max(0, k - 40):k + len(needle) + 40] + "..."44 print(f" [{src} L{i}] {txt}")45 return l464748refs = json.load(open(os.path.join(DATA, "reference_list.json"), encoding="utf-8"))49n_input_rows = {"README.md": wc_l(os.path.join(DATA, "README.md")),50 "reference_list.json": len(refs)}5152# ---- per-reference evidence specification -------------------------------------------53# For each reference: the evidence strings (must exist verbatim in the extract) that decide54# the finding check, arithmetic checks, and the decision with its locator.55spec = {56 "R1": dict(57 evidence=["ATT | -0.399 | -0.283 | -0.431**", "| (0.412) | (0.279) | (0.210)",58 "% change | -32.9% | -24.7% | -35.0%",59 "This estimate is statistically significant at the 5% level",60 "using individual-level hospital records and a difference-in-differences estimator",61 "staggered introduction of DCRs across Swiss cities",62 "roughly 35% within 5 km of a facility"],63 arithmetic=[("exp(-0.431)-1 (5 km % change)", math.exp(-0.431) - 1, -0.350, 0.0005),64 ("|ATT/SE| = 0.431/0.210 > 1.96", 0.431 / 0.210, 2.052, 0.001)],65 supported=True,66 locator="arxiv-2609.27732v1 | Table 6, row ATT, column 5 km (with % change row; Section 6 text; abstract)",67 note="ATT -0.431** (SE 0.210) and -35.0% at 5 km are printed in Table 6, 5% significance is stated in the text, and the staggered DiD on Swiss hospital records is described in the abstract."),68 "R2": dict(69 evidence=["large (N = 23,549) preregistered randomized controlled trial",70 "$1,153; p=0.022; 95% CI: [166, 2,141]",71 "1.47 pp; p=0.013; 95% CI: [0.32, 2.62]",72 "-1.99 weeks; p<0.001; 95% CI: [-2.24, -1.75]",73 "selection into the program increased annualized wages by $1,153"],74 arithmetic=[],75 supported=True,76 locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, Reemployment, Weeks on UI (N from abstract)",77 note="Table 2's ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives the preregistered RCT with N = 23,549."),78 "R3": dict(79 evidence=["the stacked coefficient is 0.005 at \\tau=0 , 0.012 at \\tau=1 , 0.018 at \\tau=2 , 0.037 at \\tau=4 , and 0.034 by \\tau=5",80 "roughly 0.5% in the event year and about 3.4–3.7% after four to five years",81 "prices do not jump immediately, but instead drift upward",82 "Clean-onset threshold-2 | 774 | 0.241 | 0.472 | 0.037 | 0.034"],83 arithmetic=[],84 supported=False,85 locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row Clean-onset threshold-2, columns tau=4/tau=5",86 note="The extract reports a gradual rise from 0.005 (about 0.5%) at the event year to 0.037/0.034 (3.4-3.7%) only by years four and five and states prices do not jump immediately, contradicting a 3.4-3.7% jump in the entry year with no later appreciation; title and year (2024) also disagree with the record."),87 "R4": dict(88 evidence=["Employment elasticity w.r.t MW | -0.388", "Emp. elasticity w.r.t affected wage | -0.732",89 "our own-wage employment elasticity is - 0.732",90 "short-run minimum-wage effects in Japan’s spot labor market"],91 se_after={"Employment elasticity w.r.t MW | -0.388": "| (0.132)",92 "Emp. elasticity w.r.t affected wage | -0.732": "| (0.091)"},93 arithmetic=[("affected employment / affected wages = -0.268/0.366", -0.268 / 0.366, -0.732, 0.0005)],94 supported=True,95 locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (Japan spot market from abstract)",96 note="Table 2 prints -0.388 (0.132) and -0.732 (0.091), the text calls the latter the own-wage employment elasticity, and the abstract places the study in Japan's spot labor market."),97 "R5": dict(98 evidence=["Treatment | -0.0911 | -0.0908 | -0.0961 | -0.0962", "| (0.0113) | (0.0113) | (0.0126) | (0.0129)",99 "% Change | -0.3490 | -0.3478 | -0.3681 | -0.3685", "Mean Dep. Var. | 0.261 | 0.261 | 0.261 | 0.261",100 "Using my most preferred specification (Column (4))",101 "this effect represents a percentage change of 36.85 percent"],102 arithmetic=[("0.0962 / 0.261 (relative to mean)", 0.0962 / 0.261, 0.3685, 0.0005)],103 supported=True,104 locator="arxiv-2510.03658v1 | Table 3, row Treatment, column (4) (with % Change and Mean Dep. Var. rows; Section 4.1 text)",105 note="Table 3 column (4), identified as the preferred specification in Section 4.1, prints -0.0962 (0.0129) and a -0.3685 change relative to the mean 0.261, matching the 36.85% stated in the text."),106}107108data_issues = [109 {"issue": "arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431 has SE 0.210 in Table 6 but SE 0.218 in Table A19; Table 6 reports 608 never-treated control regions while the Table A19 note says 640.",110 "handling": "R1 cites Table 6 (SE 0.210), which is printed verbatim; finding judged supported, and with either SE |t| > 1.96 so the 5% significance claim is unaffected. Inconsistency reported, not resolved."},111 {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes fall by 35%, Section 6.1 says approximately 30%; exp(-0.354)-1 = -29.8% from Table 7.",112 "handling": "Not part of any cited finding; recorded only."},113 {"issue": "arxiv-2505.04555v3: the SE of the own-wage elasticity (0.091) equals the SE of 'Affected employment' (0.091); the point estimate -0.732 = -0.268/0.366 is reproduced but the SE cannot be checked from the extract.",114 "handling": "R4 quotes the printed value (0.091); judged supported as stated in the source, possible source typo flagged. The table note's 'about 4%' MW increase vs the '% MW changes' row 0.047 is also noted."},115 {"issue": "arxiv-2603.03260v2: Table 3 header row is garbled in the HTML rendering (column order reconstructed by the extract editor).",116 "handling": "Used the editor's column order; the tau=4/tau=5 values 0.037/0.034 match the Section 3.2 text, so the reconstruction is corroborated."},117 {"issue": "arxiv-2510.03658v1: abstract headline (0.0855, 18.79%, Table 2 not in extract, mean 0.455) differs from Table 3 (binary outcome, 0.0962, 36.85%, mean 0.261).",118 "handling": "Different outcome definitions per Section 4.1; R5 cites the Table 3 result, which is checked directly. Recorded only."},119 {"issue": "arxiv-2606.14621v1: Tables 2 and 3 were re-set by the extract editor (values stated verbatim).",120 "handling": "Values taken as printed; Table 3 column 1 duplicates Table 2's ATE column, providing a cross-check (identical values)."},121]122123reference_checks, baseline = {}, {}124n_items = 0125for r in refs:126 rid = r["id"]127 src_rel = r["source_file"]128 src_id = os.path.basename(src_rel).replace(".txt", "")129 path = os.path.join(DATA, src_rel)130 lines = read_lines(path)131 n_input_rows[src_rel] = wc_l(path)132 print(f"\n=== {rid}: {r['entry']}")133 print(f" cited_for: {r['cited_for']}")134 # --- header ---135 _, tline = find_line(lines, "Title: ")136 show(src_id, lines, "Title: ")137 bib = show(src_id, lines, "Bibliographic record:")138 doc_title = tline.split("Title: ", 1)[1].strip()139 body_title = next(l[2:].strip() for l in lines if l.startswith("# "))140 assert norm_title(doc_title) == norm_title(body_title), "header/body title mismatch"141 # --- entry parsing ---142 m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", r["entry"])143 entry_year, entry_title, entry_arxiv = int(m.group(2)), m.group(3), m.group(4)144 assert entry_arxiv in src_id, "arXiv number mismatch"145 title_ok = norm_title(entry_title) == norm_title(doc_title)146 # --- year rule: first submission year or year of reproduced version ---147 dates = re.findall(r"submitted (\d{1,2} \w{3} (\d{4}))", bib)148 first_year = int(dates[0][1])149 ver = re.search(r"arXiv:\d{4}\.\d{5}v(\d+)", bib).group(1)150 if ver == "1":151 version_year = first_year152 else:153 version_year = int(re.search(r"version %s \(this extract\) submitted \d{1,2} \w{3} (\d{4})" % ver, bib).group(1))154 year_ok = entry_year in (first_year, version_year)155 print(f" title check: entry='{entry_title}' | doc='{doc_title}' -> {title_ok}")156 print(f" year check: entry={entry_year}, first submission={first_year}, reproduced v{ver}={version_year} -> {year_ok}")157 # --- baseline: every number in cited_for appears somewhere in the extract ---158 txt = "\n".join(lines).replace("−", "-")159 nums = re.findall(r"\d[\d,]*\.?\d*", r["cited_for"].replace("−", "-"))160 nums = [n.rstrip(".,") for n in nums]161 missing = [n for n in nums if n not in txt]162 baseline[rid] = {"title_matches": title_ok, "year_matches": year_ok,163 "all_cited_numbers_found": len(missing) == 0, "numbers_not_found": missing}164 # --- finding check ---165 s = spec[rid]166 print(" finding evidence:")167 for e in s["evidence"]:168 show(src_id, lines, e)169 n_items += 1170 for coef, se in s.get("se_after", {}).items():171 i, _ = find_line(lines, coef)172 nxt = next(l for l in lines[i:] if l.strip())173 assert nxt.strip() == se, (coef, nxt)174 print(f" [{src_id} SE row after '{coef}'] {nxt.strip()}")175 for label, val, target, tol in s["arithmetic"]:176 ok = abs(val - target) <= tol177 assert ok, label178 print(f" arithmetic: {label} = {val:.4f} (printed {target}) ok")179 reference_checks[rid] = {"title_matches": bool(title_ok), "year_matches": bool(year_ok),180 "finding_supported": bool(s["supported"]), "locator": s["locator"],181 "note": s["note"]}182 print(f" -> title {title_ok}, year {year_ok}, finding {s['supported']}")183184unverifiable = [k for k, v in reference_checks.items()185 if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]186results = {187 "task_id": "F4-conf-04",188 "reference_checks": reference_checks,189 "unverifiable_references": unverifiable,190 "n_verified": len(reference_checks) - len(unverifiable),191 "feasibility": "complete",192 "feasibility_reason": None,193 "data_issues": data_issues,194 "n_input_rows": n_input_rows,195 "n_analysed": len(reference_checks),196 "random_seed": None,197 "baseline_naive_number_match": baseline,198 "n_evidence_strings_verified": n_items,199}200os.makedirs("out", exist_ok=True)201with open(OUT, "w", encoding="utf-8") as f:202 json.dump(results, f, indent=2, ensure_ascii=False)203print(f"\nUnverifiable references: {unverifiable}; n_verified = {results['n_verified']}")204