1"""F4-conf-04: verify five references against source extracts.2Reads only data/, writes out/results.json deterministically. No randomness."""3import hashlib, json, math, os, re, unicodedata45ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))6D = os.path.join(ROOT, "data")7OUT = os.path.join(ROOT, "out", "results.json")89def wc_l(path):10 with open(path, "rb") as f:11 return f.read().count(b"\n")1213def sha(path):14 return hashlib.sha256(open(path, "rb").read()).hexdigest()1516def norm_title(s):17 s = unicodedata.normalize("NFKD", s)18 s = "".join(c for c in s if not unicodedata.combining(c)).lower()19 s = s.replace("&", " and ")20 s = re.sub(r"[^a-z0-9 ]", " ", s)21 return " ".join(s.split())2223refs = json.load(open(os.path.join(D, "reference_list.json")))24n_input_rows = {"README.md": wc_l(os.path.join(D, "README.md")),25 "reference_list.json": len(refs)}2627def find_line(lines, startswith):28 for i, l in enumerate(lines):29 if l.startswith(startswith):30 return i, l31 raise KeyError(startswith)3233def row_cells(lines, label):34 """Return cells of the table row starting with `label |` and the cells of the SE line (next non-empty)."""35 i, l = find_line(lines, label + " |")36 cells = [c.strip() for c in l.split("|")[1:]]37 se = None38 for j in range(i + 1, min(i + 3, len(lines))):39 if lines[j].strip().startswith("| ("):40 se = [c.strip() for c in lines[j].split("|")[1:]]41 break42 print(f" [line {i+1}] {l.strip()}" + (f"\n [SE line] {lines[j].strip()}" if se else ""))43 return cells, se4445src = {}46for r in refs:47 p = os.path.join(D, r["source_file"])48 lines = open(p, encoding="utf-8").read().split("\n")49 fname = os.path.basename(p)50 n_input_rows[fname] = wc_l(p)51 _, tline = find_line(lines, "Title:")52 _, bline = find_line(lines, "Bibliographic record:")53 _, body = find_line(lines, "# ")54 src[r["id"]] = dict(lines=lines, text="\n".join(lines), sid=fname[:-4],55 title=tline[len("Title:"):].strip(), body_title=body[2:].strip(),56 biblio=bline, sha=sha(p))5758checks = {}59for r in refs:60 s = src[r["id"]]61 print(f"=== {r['id']} — {s['sid']} (sha256 {s['sha']})")62 print(" " + "Title: " + s["title"])63 print(" " + s["biblio"])64 m = re.match(r"^(.*?)\s*\((\d{4})\)\.\s*(.*?)\.\s*arXiv:", r["entry"])65 e_year, e_title = int(m.group(2)), m.group(3).strip()66 title_ok = norm_title(e_title) == norm_title(s["title"]) == norm_title(s["body_title"])67 # years in the Bibliographic record line: all 4-digit years following a day+month date68 yrs = [int(y) for y in re.findall(r"\d{1,2} [A-Z][a-z]{2} (\d{4})", s["biblio"])]69 first_year, version_year = yrs[0], yrs[-1]70 year_ok = e_year in (first_year, version_year)71 print(f" entry title: {e_title!r} -> match={title_ok}; entry year {e_year}, "72 f"first submission {first_year}, reproduced version {version_year} -> match={year_ok}")73 checks[r["id"]] = dict(title_matches=title_ok, year_matches=year_ok,74 _entry_year=e_year, _first=first_year, _ver=version_year)7576def has(txt, *phrases):77 return all(p in txt for p in phrases)7879# ---------------- finding checks (each prints the cells it relies on) -------------80print("\n=== Finding checks")81# R182s = src["R1"]; L = s["lines"]83print("R1: Table 6 (5 km column)")84c, se = row_cells(L, "ATT") # first ATT row = Table 685pc, _ = row_cells(L, "% change")86att5, se5, pct5 = c[2], se[2], pc[2]87r1 = (att5 == "-0.431**" and se5 == "(0.210)" and pct5 == "-35.0%"88 and has(s["text"], "statistically significant at the 5% level",89 "reduce drug-related hospitalization rates by roughly 35% within 5 km",90 "staggered introduction", "hospital records", "difference-in-differences"))91print(f" ATT={att5} SE={se5} %={pct5}; exp(-0.431)-1={math.exp(-0.431)-1:.4f}; supported={r1}")92# A19 cross-check (data issue)93ia, la = [(i, l) for i, l in enumerate(L) if l.startswith("ATT |")][1]94a19_se = [x.strip() for x in L[ia + 2].split("|")[1:]][0]95print(f" Table A19 CS-DR: {la.strip()} / SE {a19_se}")9697# R298s = src["R2"]; L = s["lines"]99print("R2: Table 2 (Average treatment effect column)")100w, _ = row_cells(L, "Wage"); re_, _ = row_cells(L, "Reemployment"); wk, _ = row_cells(L, "Weeks on UI")101r2 = ("$1,153" in w[3] and "95% CI: [166, 2,141]" in w[3] and w[3].startswith("$1,153")102 and re_[3].startswith("1.47 pp") and wk[3].startswith("-1.99 weeks")103 and has(s["text"], "N = 23,549", "preregistered randomized controlled trial", "annualized wages by $1,153"))104print(f" supported={r2}")105106# R3107s = src["R3"]; L = s["lines"]108print("R3: Section 3.2 'Headline dynamic pattern' and Table 3 (Clean-onset threshold-2 row)")109coef_txt = [l for l in L if l.startswith("Figure 2 reports")][0]110stk = dict((int(t), float(v)) for v, t in re.findall(r"([0-9.]+) (?:at|by) \\tau=(\d)", coef_txt))111print(f" stacked coefficients by tau: {stk}")112co, _ = row_cells(L, "Clean-onset threshold-2")113tau0_pct = 100 * stk[0]114jump_in_event_year = 3.4 <= tau0_pct <= 3.7115no_further = stk[4] <= stk[0] and stk[5] <= stk[0]116r3 = jump_in_event_year and no_further117print(f" event-year effect {stk[0]} log pts (~{tau0_pct:.1f}%), tau=4 {stk[4]}, tau=5 {stk[5]} "118 f"-> 3.4-3.7% jump in event year: {jump_in_event_year}; no later appreciation: {no_further}; supported={r3}")119120# R4121s = src["R4"]; L = s["lines"]122print("R4: Appendix A.1, Table 2")123e1, se1 = row_cells(L, "Employment elasticity w.r.t MW")124e2, se2 = row_cells(L, "Emp. elasticity w.r.t affected wage")125ae, aese = row_cells(L, "Affected employment"); aw, awse = row_cells(L, "Affected wages")126r4 = (e1[0] == "-0.388" and se1[0] == "(0.132)" and e2[0] == "-0.732" and se2[0] == "(0.091)"127 and has(s["text"], "Japan’s spot labor market"))128ratio = float(ae[0]) / float(aw[0])129dm = abs(ratio) * math.sqrt((0.091 / 0.268) ** 2 + (0.171 / 0.366) ** 2)130print(f" affected emp / affected wages = {ratio:.4f}; delta-method SE approx (no covariance) = {dm:.3f}; supported={r4}")131132# R5133s = src["R5"]; L = s["lines"]134print("R5: Section 4.1 and Table 3, column (4)")135t, tse = row_cells(L, "Treatment"); pc, _ = row_cells(L, "% Change"); mn, _ = row_cells(L, "Mean Dep. Var.")136r5 = (t[3] == "-0.0962" and tse[3] == "(0.0129)" and pc[3] == "-0.3685"137 and has(s["text"], "most preferred specification (Column (4))", "36.85 percent"))138print(f" -0.0962/{mn[3]} = {-0.0962/float(mn[3]):.4f}; supported={r5}")139140fs = {"R1": r1, "R2": r2, "R3": r3, "R4": r4, "R5": r5}141locators = {142 "R1": "arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (+ Section 6 text: significant at the 5% level)",143 "R2": "arxiv-2606.14621v1 | Table 2, rows Wage / Reemployment / Weeks on UI, column Average treatment effect estimate",144 "R3": "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row Clean-onset threshold-2",145 "R4": "arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",146 "R5": "arxiv-2510.03658v1 | Table 3, column (4), rows Treatment / % Change (+ Section 4.1: preferred specification)",147}148notes = {149 "R1": "Title and 2026 year match; Table 6 5 km column prints ATT -0.431** (0.210), -35.0%, and the text states significance at 5% and the staggered DiD on hospital records, so the finding is supported (Table A19 prints SE 0.218 for the same estimate, still p<0.05).",150 "R2": "Title and 2026 year match; Table 2 ATE column gives $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives N = 23,549 in a preregistered RCT, so every element is supported.",151 "R3": "Fails all three checks: the entry's title is a different title, 2024 is neither the first-submission (2026) nor version (2026) year, and the source reports ~0.5% (0.005) in the event year rising gradually to 3.4-3.7% by years 4-5, contradicting a 3.4-3.7% jump at entry with no further appreciation.",152 "R4": "Title matches and 2025 equals the v1 submission year; Table 2 prints -0.388 (0.132) and -0.732 (0.091) and the abstract places the study in Japan's spot labor market, so the finding is stated (note: the printed SE 0.091 for the own-wage elasticity looks inconsistent with its components).",153 "R5": "Title and 2025 year match; Table 3 column (4), the preferred specification, prints -0.0962 (0.0129) and % change -0.3685 (= -0.0962/0.261), matching the stated 36.85% reduction.",154}155reference_checks = {}156for rid in ["R1", "R2", "R3", "R4", "R5"]:157 c = checks[rid]158 reference_checks[rid] = dict(title_matches=c["title_matches"], year_matches=c["year_matches"],159 finding_supported=fs[rid], locator=locators[rid], note=notes[rid])160unver = [k for k, v in reference_checks.items()161 if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]162baseline_unver = [k for k, v in reference_checks.items() if not (v["title_matches"] and v["year_matches"])]163164data_issues = [165 {"issue": f"arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431 has SE (0.210) in Table 6 but {a19_se} in Table A19; the A19 note says 640 never-treated regions while its body and Tables 6-7 say 608.",166 "handling": "R1 cites Table 6, which is the main-text table and matches the text; the discrepancy does not change significance at 5% (t = 2.05 vs 1.98, both starred **). Reported, not resolved."},167 {"issue": "arxiv-2609.27732v1: the abstract says fatal hospitalisations fall by 35% and narcotics poisonings rise 'by a comparable amount', but Table 7 (5 km: -0.354, +0.390 log points) and Section 6.1 give about 30% and 47%.",168 "handling": "Internal inconsistency of the source; not an element of R1's cited finding, so it does not affect the R1 verdict."},169 {"issue": f"arxiv-2505.04555v3: Table 2 prints SE (0.091) for the own-wage elasticity -0.732, identical to the SE of 'Affected employment'; a delta-method approximation from the printed components (ignoring covariance) gives about {dm:.2f}.",170 "handling": "Possible transcription error in the source; R4 reproduces the value as printed, so the finding counts as stated in the extract, with a caveat in the note and report."},171 {"issue": "arxiv-2603.03260v2: the Table 3 header row is garbled in the HTML rendering.",172 "handling": "Used the column order supplied by the extract editor's note; the cells used agree with the Section 3.2 text (0.037 at tau=4, 0.034 at tau=5)."},173 {"issue": "arxiv-2510.03658v1: footnote markers are rendered inline (e.g. 'percent^9^9 9') and footnote 11 percentages are detached from their sentence.",174 "handling": "Rendering defect only; R5 relies on Table 3 and the main text, which are unaffected."},175 {"issue": "arxiv-2606.14621v1: Tables 2-3 were re-set by the extract editor, and the text has a rendering defect ('p ¡ 0.001').",176 "handling": "Re-setting is declared in the header with values verbatim; the Table 2 values used match the Section 3 text."},177]178179results = {180 "task_id": "F4-conf-04",181 "reference_checks": reference_checks,182 "unverifiable_references": unver,183 "n_verified": 5 - len(unver),184 "feasibility": "complete",185 "feasibility_reason": None,186 "data_issues": data_issues,187 "n_input_rows": n_input_rows,188 "n_analysed": len(reference_checks),189 "random_seed": None,190 "baseline_unverifiable_references_bibliographic_only": baseline_unver,191 "baseline_n_verified_bibliographic_only": 5 - len(baseline_unver),192}193with open(OUT, "w", encoding="utf-8") as f:194 json.dump(results, f, indent=2, ensure_ascii=False)195 f.write("\n")196print("\nunverifiable_references:", unver, "| n_verified:", results["n_verified"])197print("baseline (title+year only) unverifiable:", baseline_unver)198