"""F4-conf-04: verify reference list against source extracts.
Reads only data/; writes out/results.json deterministically. No randomness."""
import hashlib, json, os, re, unicodedata, math

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
OUT = os.path.join(ROOT, "out", "results.json")

def read(p):
    with open(os.path.join(D, p), encoding="utf-8") as f:
        return f.read()

def nlines(p):
    with open(os.path.join(D, p), "rb") as f:
        return f.read().count(b"\n")

refs = json.loads(read("reference_list.json"))
n_input_rows = {"README.md": nlines("README.md"), "reference_list.json": len(refs)}
src = {}
for r in refs:
    sf = r["source_file"]
    txt = read(sf)
    src[r["id"]] = txt
    n_input_rows[sf] = nlines(sf)
for k in sorted(n_input_rows):
    with open(os.path.join(D, k), "rb") as f:
        print("sha256", k, hashlib.sha256(f.read()).hexdigest(), "rows", n_input_rows[k])

def norm_title(s):
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c)).lower()
    s = s.replace("&", " and ").replace("’", "'")
    s = re.sub(r"[^a-z0-9 ]", " ", s)
    return " ".join(s.split())

def entry_title_year(entry):
    m = re.match(r"^.*?\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", entry)
    return int(m.group(1)), m.group(2), m.group(3)

def header(txt):
    title = re.search(r"^Title: (.*)$", txt, re.M).group(1).strip()
    bib = re.search(r"^Bibliographic record: (.*)$", txt, re.M).group(1).strip()
    body_title = re.search(r"^# (.*)$", txt, re.M).group(1).strip()
    years = [int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", bib)]
    arxiv = re.search(r"arXiv:(\d{4}\.\d{5})v\d", bib).group(1)
    return title, body_title, bib, years, arxiv

def line_with(txt, start):
    for ln in txt.splitlines():
        if ln.strip().startswith(start):
            return ln.strip()
    raise KeyError(start)

def cells(txt, start):
    return [c.strip() for c in line_with(txt, start).split("|")]

def se_line_after(txt, start):
    lines = [l for l in txt.splitlines() if l.strip()]
    for i, l in enumerate(lines):
        if l.strip().startswith(start):
            return [c.strip() for c in lines[i + 1].split("|")]
    raise KeyError(start)

# ---------------- title / year checks ----------------
meta = {}
for r in refs:
    ey, et, ea = entry_title_year(r["entry"])
    ht, bt, bib, years, ha = header(src[r["id"]])
    print(f"\n[{r['id']}] Title: {ht}\n[{r['id']}] Body title: # {bt}\n[{r['id']}] Bibliographic record: {bib}")
    print(f"[{r['id']}] entry title: {et} | entry year: {ey} | entry arXiv: {ea}")
    tm = norm_title(et) == norm_title(ht) and norm_title(ht) == norm_title(bt)
    # first submission year and year of the reproduced version (last 'submitted' date in the record)
    first_y, version_y = years[0], years[-1]
    ym = ey in (first_y, version_y)
    meta[r["id"]] = dict(title_matches=tm, year_matches=ym, entry_year=ey, first_year=first_y,
                         version_year=version_y, arxiv_matches=(ea == ha))
    print(f"[{r['id']}] title_matches={tm} year_matches={ym} (first={first_y}, version={version_y}) arxiv_id_matches={ea == ha}")

# ---------------- finding checks (cell-level evidence) ----------------
fin = {}

# R1 Armendariz, Table 6 5 km column
t = src["R1"]
att = cells(t, "ATT | -0.399"); se = se_line_after(t, "ATT | -0.399"); pc = cells(t, "% change | -32.9%")
a19 = cells(t, "ATT | -0.431**"); a19se = se_line_after(t, "ATT | -0.431**")
print("\n[R1] Table 6 ATT row:", att, "SE row:", se, "% row:", pc)
print("[R1] Table A19 ATT row:", a19, "SE row:", a19se)
r1 = (att[3] == "-0.431**" and se[3] == "(0.210)" and pc[3] == "-35.0%"
      and abs(math.exp(-0.431) - 1 + 0.350) < 0.0005
      and "staggered introduction of DCRs" in t and "individual-level hospital records" in t
      and "statistically significant at the 5% level" in t and "within 5 km of a facility" in t)
print(f"[R1] exp(-0.431)-1 = {math.exp(-0.431)-1:.4f}; finding_supported={r1}")
fin["R1"] = dict(finding_supported=r1,
    locator="arxiv-2609.27732v1 | Table 6, row ATT (SE row and % change row), column 5 km; abstract; Section 6 text after Table 6",
    note="Title and year match; Table 6 5 km column prints ATT -0.431** (0.210), -35.0%, significant at 5%, staggered DiD on individual-level Swiss hospital records, as cited (Table A19 prints SE 0.218 for the same estimate, which does not change the 5% significance).")

# R2 RESEA, Table 2 ATE column
t = src["R2"]
w = cells(t, "Wage | $33,073"); re_ = cells(t, "Reemployment | 75.2%"); ui = cells(t, "Weeks on UI | 15.2")
print("\n[R2] Table 2 rows:\n ", w, "\n ", re_, "\n ", ui)
r2 = (w[4] == "$1,153; p=0.022; 95% CI: [166, 2,141]" and re_[4].startswith("1.47 pp")
      and ui[4].startswith("-1.99 weeks") and "(N = 23,549) preregistered randomized controlled trial" in t)
print(f"[R2] finding_supported={r2}")
fin["R2"] = dict(finding_supported=r2,
    locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, Reemployment, Weeks on UI; abstract (N = 23,549, preregistered RCT)",
    note="Title and year match; Table 2 ATE column gives $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549, exactly as cited.")

# R3 Liu & Zhao, stacked event-study path
t = src["R3"]
co = cells(t, "Clean-onset threshold-2")
path = re.search(r"the stacked coefficient is (.*?)\. Interpreted", t).group(1)
print("\n[R3] Table 3 preferred row:", co)
print("[R3] Section 3.2 stacked path:", path)
tau0 = float(re.search(r"([\d.]+) at \\tau=0", path).group(1))
tau4, tau5 = float(co[4]), float(co[5])
# cited: 3.4-3.7% jump in the entry year, no further appreciation later
r3 = (tau0 >= 0.034) and (tau4 <= tau0 + 1e-9)
print(f"[R3] tau=0 coefficient {tau0} vs cited 0.034-0.037 jump; tau=4 {tau4}, tau=5 {tau5}; finding_supported={r3}")
fin["R3"] = dict(finding_supported=r3,
    locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) and Table 3, row 'Clean-onset threshold-2', columns tau=4 and tau=5",
    note="Title differs entirely and 2024 matches neither 2026 submission; the extract reports ~0.5% at the event year (0.005 at tau=0) rising gradually to 3.4-3.7% by years four-five (0.037, 0.034), contradicting the cited immediate jump with no later appreciation.")

# R4 Kanayama et al., Appendix Table 2
t = src["R4"]
e1 = cells(t, "Employment elasticity w.r.t MW"); e1se = se_line_after(t, "Employment elasticity w.r.t MW")
e2 = cells(t, "Emp. elasticity w.r.t affected wage"); e2se = se_line_after(t, "Emp. elasticity w.r.t affected wage")
ae = cells(t, "Affected employment"); aese = se_line_after(t, "Affected employment"); aw = cells(t, "Affected wages")
print("\n[R4] Table 2:", e1, e1se, e2, e2se, "| Affected employment", ae, aese, "| Affected wages", aw)
r4 = (e1[1] == "-0.388" and e1se[1] == "(0.132)" and e2[1] == "-0.732" and e2se[1] == "(0.091)"
      and "Japan’s spot labor market" in t)
print(f"[R4] -0.268/0.366 = {-0.268/0.366:.4f}; finding_supported={r4}")
fin["R4"] = dict(finding_supported=r4,
    locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (Estimate (SE) column)",
    note="Title matches and 2025 equals the v1 year; Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market, as cited (the 0.091 SE duplicates the 'Affected employment' SE, flagged as a source issue).")

# R5 Tossou, Table 3 column (4)
t = src["R5"]
tr = cells(t, "Treatment | -0.0911"); trse = se_line_after(t, "Treatment | -0.0911")
pc = cells(t, "% Change"); mn = cells(t, "Mean Dep. Var.")
print("\n[R5] Table 3:", tr, trse, pc, mn)
ratio = float(tr[4]) / float(mn[4])
r5 = (tr[4] == "-0.0962" and trse[4] == "(0.0129)" and pc[4] == "-0.3685"
      and abs(ratio + 0.3685) < 0.0005 and "most preferred specification (Column (4))" in t
      and "36.85 percent" in t)
print(f"[R5] -0.0962/0.261 = {ratio:.5f}; finding_supported={r5}")
fin["R5"] = dict(finding_supported=r5,
    locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment (SE), % Change, Mean Dep. Var.; Section 4.1 text ('most preferred specification (Column (4))')",
    note="Title matches and 2025 equals the submission year; Table 3 column (4) gives -0.0962 (0.0129) on infant death or stillbirth, -36.85% relative to the mean 0.261 (preferred specification), as cited.")

# ---------------- assemble ----------------
checks = {}
for r in refs:
    i = r["id"]
    checks[i] = {"title_matches": bool(meta[i]["title_matches"]), "year_matches": bool(meta[i]["year_matches"]),
                 "finding_supported": bool(fin[i]["finding_supported"]),
                 "locator": fin[i]["locator"], "note": fin[i]["note"]}
unver = [i for i in checks if not all(checks[i][k] for k in ("title_matches", "year_matches", "finding_supported"))]

# Baseline: naive check = header title/year + every number in cited_for appears verbatim in the extract
def nums(s):
    s = s.replace("−", "-").replace("–", "-")
    return re.findall(r"\d[\d,]*\.?\d*", s)
baseline = {}
for r in refs:
    t = src[r["id"]].replace("−", "-").replace("–", "-")
    ok = all(n.rstrip(".,") in t for n in nums(r["cited_for"]))
    baseline[r["id"]] = bool(meta[r["id"]]["title_matches"] and meta[r["id"]]["year_matches"] and ok)
baseline_unver = [i for i, v in baseline.items() if not v]
print("\nBaseline (header title/year + naive number-presence) unverifiable:", baseline_unver)

data_issues = [
    {"issue": "arxiv-2609.27732v1 (R1): Table A19 prints SE (0.218) for the CS-DR 5 km ATT -0.431 whose SE is (0.210) in Table 6; Table A19 note says 640 never-treated regions while its body and Tables 6-7 say 608.",
     "handling": "Used Table 6 (main results table, the one the citation reproduces); the 5% significance holds under either SE (0.431/0.218 = 1.98). Finding judged supported; discrepancy noted."},
    {"issue": "arxiv-2609.27732v1 (R1): abstract says fatal outcomes fall by 35% whereas Section 6.1 says approximately 30% (Table 7 fatal 5 km coefficient -0.354).",
     "handling": "Not part of the cited finding; recorded only."},
    {"issue": "arxiv-2505.04555v3 (R4): Table 2 SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment', although the elasticity is the ratio -0.268/0.366; a ratio's SE equal to the numerator's SE is implausible and may be a typesetting duplication.",
     "handling": "The cited value matches the printed cell exactly, so the finding is judged supported per the task rule ('stated in the extract'); the reliability of that SE is flagged as an open issue."},
    {"issue": "arxiv-2510.03658v1 (R5): abstract headline (0.0855, 18.79%, mean 0.455, from omitted Table 2) differs from the Table 3 preferred estimate (0.0962, 36.85%, mean 0.261); Table 3 '% Change' -0.3685 vs -0.0962/0.261 = -0.36858; footnote markers rendered as '^9^9 9'.",
     "handling": "Different outcome definitions (count vs binary indicator) per Section 4.1; citation explicitly refers to the preferred specification of the binary outcome, so Table 3 col (4) decides. Last-digit difference treated as truncation."},
    {"issue": "arxiv-2606.14621v1 (R2): Tables 2-3 re-set by the extract editor; 'p ¡ 0.001' rendering defect. arxiv-2603.03260v2 (R3): Table 3 header row garbled.",
     "handling": "Relied on the editor's declared column order; no numbers affected."},
]

results = {
    "task_id": "F4-conf-04",
    "reference_checks": checks,
    "unverifiable_references": unver,
    "n_verified": len(refs) - len(unver),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(refs),
    "random_seed": None,
    "baseline_unverifiable_references": baseline_unver,
    "baseline_n_verified": len(refs) - len(baseline_unver),
}
os.makedirs(os.path.dirname(OUT), exist_ok=True)
with open(OUT, "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False, sort_keys=True)
print("\nunverifiable_references:", unver, "n_verified:", results["n_verified"])