1"""F4-conf-04: verify five references against provided arXiv extracts.2Reads only data/; writes out/results.json deterministically. No randomness."""3import json, re, hashlib, unicodedata, os45D = "data"6ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))7os.chdir(ROOT)89def wc_l(p):10 with open(p, "rb") as f:11 return f.read().count(b"\n")1213def sha(p):14 return hashlib.sha256(open(p, "rb").read()).hexdigest()1516refs = json.load(open(f"{D}/reference_list.json", encoding="utf-8"))17n_input_rows = {"README.md": wc_l(f"{D}/README.md"), "reference_list.json": len(refs)}18src = {}19for r in refs:20 p = f"{D}/{r['source_file']}"21 name = os.path.basename(p)22 n_input_rows[name] = wc_l(p)23 src[r["id"]] = open(p, encoding="utf-8").read().splitlines()24 print(f"[{r['id']}] {name} sha256={sha(p)} lines={n_input_rows[name]}")2526def header(lines, key):27 for l in lines:28 if l.startswith(key + ":"):29 return l[len(key) + 1:].strip()30 raise KeyError(key)3132def norm_title(t):33 t = unicodedata.normalize("NFKD", t)34 t = "".join(c for c in t if not unicodedata.combining(c)).lower()35 t = t.replace("&", " and ")36 t = re.sub(r"[^a-z0-9 ]", " ", t)37 return " ".join(t.split())3839def entry_parts(entry):40 m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", entry)41 return {"year": int(m.group(2)), "title": m.group(3), "arxiv": m.group(4)}4243def find(lines, pattern):44 """Return (line_no, line) of first line matching regex; print it as evidence."""45 for i, l in enumerate(lines, 1):46 if re.search(pattern, l):47 return i, l48 raise ValueError(pattern)4950def show(rid, lines, pattern, nxt=False):51 i, l = find(lines, pattern)52 print(f" [{rid} L{i}] {l.strip()[:220]}")53 if nxt: # SE line follows coefficient row (skip blank line)54 j = i55 while not lines[j].strip():56 j += 157 print(f" [{rid} L{j+1}] {lines[j].strip()}")58 return l, lines[j]59 return l6061def cells(line):62 return [c.strip() for c in line.split("|")]6364checks = {}65header_checks = {}66for r in refs:67 rid, L = r["id"], src[r["id"]]68 e = entry_parts(r["entry"])69 title = header(L, "Title")70 rec = header(L, "Bibliographic record")71 body_title = next(l[2:].strip() for l in L if l.startswith("# "))72 print(f"[{rid}] entry: {r['entry']}")73 print(f" Title: {title}\n Body title: {body_title}\n Bibliographic record: {rec}")74 years = sorted({int(y) for y in re.findall(r"\b\d{1,2} [A-Z][a-z]{2} (\d{4})", rec)})75 first_year = years[0]76 m_ver = re.search(r"arXiv:\d{4}\.\d{5}v(\d+)", rec)77 ver = m_ver.group(1)78 # year of the reproduced version: if multiple versions, the one marked "(this extract)", else the only date79 m_this = re.search(r"submitted (\d{1,2} [A-Z][a-z]{2} (\d{4})) \(this extract\)", rec) or \80 re.search(r"\(this extract\) submitted (\d{1,2} [A-Z][a-z]{2} (\d{4}))", rec)81 this_year = int(m_this.group(2)) if m_this else (years[0] if len(years) == 1 else None)82 allowed = {first_year, this_year} - {None}83 tm = norm_title(e["title"]) == norm_title(title) == norm_title(body_title)84 ym = e["year"] in allowed85 idm = e["arxiv"] in rec86 print(f" -> entry year {e['year']}; allowed {sorted(allowed)} (v{ver}); title_match={tm}; year_match={ym}; arXiv id match={idm}")87 header_checks[rid] = {"title_matches": tm, "year_matches": ym, "allowed_years": sorted(allowed)}8889# ---------------- finding checks (evidence printed) ----------------90f = {}91# R192L = src["R1"]; print("[R1] finding evidence")93att, se = show("R1", L, r"^ATT \| -0\.399", nxt=True)94pct = show("R1", L, r"^% change \| -32\.9%")95hdr = show("R1", L, r"^\| 1 km \| 3 km \| 5 km")96show("R1", L, r"statistically significant at the 5% level")97show("R1", L, r"staggered introduction of DCRs")98a5, s5, p5 = cells(att)[3], cells(se)[3], cells(pct)[3]99ok1 = (a5 == "-0.431**" and s5 == "(0.210)" and p5 == "-35.0%")100show("R1", L, r"^ATT \| -0\.431\*\* \| -0\.438", nxt=True) # Table A19 (inconsistent SE)101f["R1"] = dict(ok=ok1, locator="arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (abstract; Section 6 text for 5% significance and staggered DiD on hospital records)",102 note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, the text states 5% significance and the abstract describes a staggered DiD on individual hospital records, so every element is supported.")103# R2104L = src["R2"]; print("[R2] finding evidence")105show("R2", L, r"N = 23,549\) preregistered randomized")106w = cells(show("R2", L, r"^Wage \| \$33,073"))[4]107re_ = cells(show("R2", L, r"^Reemployment \| 75\.2%"))[4]108ui = cells(show("R2", L, r"^Weeks on UI \| 15\.2"))[4]109ok2 = (w.startswith("$1,153;") and "[166, 2,141]" in w and re_.startswith("1.47 pp") and ui.startswith("-1.99 weeks"))110f["R2"] = dict(ok=ok2, locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI (N and design from abstract)",111 note="Table 2 ATE column gives $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives a preregistered RCT with N = 23,549.")112# R3113L = src["R3"]; print("[R3] finding evidence")114txt = show("R3", L, r"stacked coefficient is 0\.005 at")115row = cells(show("R3", L, r"^Clean-onset threshold-2 \|"))116show("R3", L, r"prices do not jump immediately")117tau0 = float(re.search(r"([0-9.]+) at \\tau=0", txt).group(1))118tau4, tau5 = float(row[4]), float(row[5])119# claim: 3.4-3.7% jump in entry year, no later appreciation. Evidence: 0.005 at tau=0, 0.034-0.037 at tau=4-5.120ok3 = (0.034 <= tau0 <= 0.037) and not (tau4 > tau0)121print(f" -> tau0={tau0}, tau4={tau4}, tau5={tau5}; claim requires 0.034-0.037 at tau=0 and no later rise -> supported={ok3}")122f["R3"] = dict(ok=ok3, locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' and Table 3, row 'Clean-onset threshold-2', columns tau=4 / tau=5",123 note="The extract reports a gradual rise from 0.005 log points at tau=0 to 0.037/0.034 at tau=4/5 ('prices do not jump immediately'), contradicting a 3.4-3.7% jump in the entry year; the entry's title and year (2024) also do not match the record.")124# R4125L = src["R4"]; print("[R4] finding evidence")126e1, s1 = show("R4", L, r"^Employment elasticity w\.r\.t MW", nxt=True)127e2, s2 = show("R4", L, r"^Emp\. elasticity w\.r\.t affected wage", nxt=True)128show("R4", L, r"^Affected employment", nxt=True)129show("R4", L, r"Japan’s spot labor market")130ok4 = (cells(e1)[1] == "-0.388" and cells(s1)[1] == "(0.132)" and cells(e2)[1] == "-0.732" and cells(s2)[1] == "(0.091)")131emp, wage = -0.268, 0.366132print(f" -> arithmetic check: affected employment / affected wages = {emp/wage:.3f} (printed -0.732)")133f["R4"] = dict(ok=ok4, locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",134 note="Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market (abstract), matching the citation; the 2025 year equals the v1 submission year.")135# R5136L = src["R5"]; print("[R5] finding evidence")137t, s = show("R5", L, r"^Treatment \| -0\.0911", nxt=True)138pc = cells(show("R5", L, r"^% Change \|"))139mean = cells(show("R5", L, r"^Mean Dep\. Var\."))140show("R5", L, r"most preferred specification \(Column \(4\)\)")141ok5 = (cells(t)[4] == "-0.0962" and cells(s)[4] == "(0.0129)" and pc[4] == "-0.3685")142print(f" -> arithmetic check: -0.0962/{mean[4]} = {-0.0962/float(mean[4]):.4f} (printed -0.3685 = 36.85%)")143f["R5"] = dict(ok=ok5, locator="arxiv-2510.03658v1 | Table 3, row Treatment / % Change, column (4) (Section 4.1 identifies column 4 as preferred)",144 note="Table 3 column (4), the stated preferred specification, prints -0.0962 (0.0129) and % change -0.3685 (36.85% of the 0.261 mean), matching the citation.")145146reference_checks = {}147for r in refs:148 rid = r["id"]149 reference_checks[rid] = {"title_matches": header_checks[rid]["title_matches"],150 "year_matches": header_checks[rid]["year_matches"],151 "finding_supported": bool(f[rid]["ok"]),152 "locator": f[rid]["locator"], "note": f[rid]["note"]}153unver = [k for k, v in reference_checks.items()154 if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]155baseline_unver = [k for k, v in header_checks.items() if not (v["title_matches"] and v["year_matches"])]156157data_issues = [158 {"issue": "arxiv-2609.27732v1: Table A19 prints SE (0.218) for the CS-DR 5 km ATT -0.431, whereas Table 6 prints (0.210) for the same estimate.",159 "handling": "Table 6 (the main results table, which the citation's SE 0.210 matches) is used as the deciding locator; inconsistency reported, finding judged supported since 0.210 is printed in the extract and significance at 5% holds under either SE."},160 {"issue": "arxiv-2609.27732v1: Table A19 note states 640 never-treated comparison regions, while Table 6, Table A19 body and figure notes state 608.",161 "handling": "Not relevant to the cited finding; reported only."},162 {"issue": "arxiv-2609.27732v1: Section 6.1 gives -30% (fatal) and +47% (narcotics) at 5 km, while the abstract says fatal outcomes fall 35% and narcotics rise by a comparable amount.",163 "handling": "Not part of the cited finding; reported only."},164 {"issue": "arxiv-2505.04555v3: Table 2 prints the same SE (0.091) for 'Affected employment' and for the own-wage elasticity, although the latter is a ratio (-0.268/0.366 = -0.732) whose SE would generally differ.",165 "handling": "Cannot be resolved from the extract; the cited SE equals the printed value, so the finding is judged supported, with the caveat noted."},166 {"issue": "arxiv-2510.03658v1: abstract/Section 4.1 headline (0.0855, 18.79%, mean 0.455, from a Table 2 not reproduced) differs from Table 3 column (4) (-0.0962, 36.85%, mean 0.261); footnote markers are rendered as '^9^9 9'.",167 "handling": "Treated as different outcome definitions/tables, not an internal contradiction; Table 3 decides the citation, which explicitly refers to the preferred specification and 36.85%."},168 {"issue": "arxiv-2603.03260v2: Table 3 header row is garbled in the HTML rendering.",169 "handling": "Column order taken from the extract editor's note (Specification | Treated-role LSOAs | Stacked p | Sun p | tau=4 | tau=5); consistent with the Section 3.2 text (0.037, 0.034)."},170]171172results = {173 "task_id": "F4-conf-04",174 "reference_checks": reference_checks,175 "unverifiable_references": unver,176 "n_verified": len(refs) - len(unver),177 "feasibility": "complete",178 "feasibility_reason": None,179 "data_issues": data_issues,180 "n_input_rows": n_input_rows,181 "n_analysed": len(refs),182 "random_seed": None,183 "baseline_unverifiable_title_year_only": baseline_unver,184}185os.makedirs("out", exist_ok=True)186with open("out/results.json", "w", encoding="utf-8") as fh:187 json.dump(results, fh, indent=2, ensure_ascii=False)188for k, v in reference_checks.items():189 print(k, v["title_matches"], v["year_matches"], v["finding_supported"])190print("unverifiable:", unver, "n_verified:", results["n_verified"])191