"""F4-conf-04: verify five references against the provided arXiv extracts.
Reads only data/; writes out/results.json deterministically. No randomness."""
import json, math, os, re, unicodedata

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
OUT = os.path.join(ROOT, "out", "results.json")

def wc_l(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")

refs = json.load(open(os.path.join(D, "reference_list.json")))
src = {}
for r in refs:
    with open(os.path.join(D, r["source_file"]), encoding="utf-8") as f:
        src[r["id"]] = f.read().split("\n")

def norm_title(s):
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c)).lower()
    s = s.replace("&", " and ")
    s = re.sub(r"[^a-z0-9 ]", " ", s)
    return " ".join(s.split())

def find(rid, needle):
    """Return (line_no, line) of first line containing needle; print it; raise if absent."""
    for i, l in enumerate(src[rid], 1):
        if needle in l:
            short = l if len(l) < 220 else "..." + l[max(0, l.index(needle) - 80): l.index(needle) + len(needle) + 80] + "..."
            print(f"  [{rid} L{i}] {short}")
            return i, l
    print(f"  [{rid}] NOT FOUND: {needle!r}")
    return None, None

def cells(line):
    return [c.strip() for c in line.split("|")]

def approx(a, b, tol):
    return abs(a - b) <= tol

results_checks = {}
baseline_flags = {}
for r in refs:
    rid = r["id"]; L = src[rid]
    print(f"=== {rid}: {r['source_file']}")
    title_hdr = next(l for l in L if l.startswith("Title: "))[7:].strip()
    biblio = next(l for l in L if l.startswith("Bibliographic record: "))
    body_title = next(l for l in L if l.startswith("# "))[2:].strip()
    print("  " + "Title: " + title_hdr); print("  " + biblio); print("  body title: " + body_title)
    # entry title: text after "(YYYY). " up to ". arXiv:"
    m = re.match(r"^.*?\((\d{4})\)\.\s+(.*)\.\s+arXiv:(\S+?)\.?$", r["entry"])
    entry_year, entry_title, entry_arxiv = int(m.group(1)), m.group(2), m.group(3)
    title_ok = norm_title(entry_title) == norm_title(title_hdr) == norm_title(body_title)
    # years: first submission and reproduced version
    sub_dates = re.findall(r"submitted (\d{1,2} \w{3} (\d{4}))", biblio)
    years = [int(y) for _, y in sub_dates]
    first_year = years[0]
    this_m = re.search(r"\(this extract\) submitted \d{1,2} \w{3} (\d{4})", biblio)
    version_year = int(this_m.group(1)) if this_m else years[0]  # single-version records
    year_ok = entry_year in (first_year, version_year)
    arxiv_ok = biblio.split()[2].startswith("arXiv:" + entry_arxiv)
    print(f"  entry year {entry_year}; first-submission year {first_year}; reproduced-version year {version_year}; arXiv id match {arxiv_ok}")
    print(f"  entry title: {entry_title}\n  title_matches={title_ok} year_matches={year_ok}")
    results_checks[rid] = {"title_matches": bool(title_ok), "year_matches": bool(year_ok),
                           "_entry_year": entry_year, "_years": (first_year, version_year)}
    baseline_flags[rid] = not (title_ok and year_ok)

# ---------- finding checks (each element of cited_for checked against printed text/cells) ----------
F = {}

print("=== R1 finding")
_, l = find("R1", "ATT | -0.399 | -0.283 | -0.431**"); c = cells(l)
_, se = find("R1", "| (0.412) | (0.279) | (0.210)"); s = cells(se)
_, pc = find("R1", "% change | -32.9% | -24.7% | -35.0%")
find("R1", "{}^{**}p<0.05")
find("R1", "This estimate is statistically significant at the 5% level")
find("R1", "reduce drug-related hospitalization rates by roughly 35% within 5 km of a facility")
find("R1", "staggered introduction of DCRs across Swiss cities")
find("R1", "using individual-level hospital records and a difference-in-differences estimator")
_, a19 = find("R1", "| (0.218) | (0.208) | (0.205) | (0.038)")
att, se5 = float(c[3].rstrip("*")), float(s[3].strip("()"))
pct = math.expm1(att) * 100
print(f"  check: exp(-0.431)-1 = {pct:.2f}% (printed -35.0%); |t| = {abs(att)/se5:.3f} (Table 6 SE), {abs(att)/0.218:.3f} (A19 SE)")
F["R1"] = dict(ok=(att == -0.431 and se5 == 0.210 and c[3].endswith("**") and approx(pct, -35.0, 0.05)),
    locator="arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (with Section 6 text and abstract)",
    note="Title and 2026 year match; Table 6 5 km column prints ATT -0.431** (0.210), -35.0%, significant at 5%, Callaway-Sant'Anna staggered DiD on hospital records, so every element is supported (Table A19 prints SE 0.218 for the same estimate, see data_issues).")

print("=== R2 finding")
find("R2", "large (N = 23,549) preregistered randomized controlled trial (RCT)")
_, w = find("R2", "Wage | $33,073"); wc = cells(w)
_, re_ = find("R2", "Reemployment | 75.2%"); rc = cells(re_)
_, ui = find("R2", "Weeks on UI | 15.2 weeks"); uc = cells(ui)
find("R2", "wage outcome that was $1,153 higher annually (95% CI: [$166, $2,141]")
F["R2"] = dict(ok=(wc[4].startswith("$1,153") and "95% CI: [166, 2,141]" in wc[4]
                   and rc[4].startswith("1.47 pp") and uc[4].startswith("-1.99 weeks")),
    locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI (N and design: abstract)",
    note="Title and 2026 year match; Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549.")

print("=== R3 finding")
find("R3", "the stacked coefficient is 0.005 at \\tau=0")
find("R3", "prices do not jump immediately, but instead drift upward")
_, t3 = find("R3", "Clean-onset threshold-2 | 774"); t3c = cells(t3)
print(f"  Table 3 preferred row: tau=4 {t3c[4]}, tau=5 {t3c[5]} (log points) -> 3.4-3.7% reached at years 4-5, not at entry")
F["R3"] = dict(ok=False,
    locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficient 0.005 at tau=0) and Table 3, row 'Clean-onset threshold-2', columns tau=4 (0.037) / tau=5 (0.034)",
    note="Entry has a different title and year 2024 (record: v1 3 Mar 2026, v2 2 Apr 2026), and the extract reports ~0.5% at entry rising gradually to 3.4-3.7% only by years 4-5, contradicting the cited immediate jump with no later appreciation.")

print("=== R4 finding")
_, e1 = find("R4", "Employment elasticity w.r.t MW | -0.388"); i1 = src["R4"].index(e1)
_, e2 = find("R4", "Emp. elasticity w.r.t affected wage | -0.732"); i2 = src["R4"].index(e2)
se1 = next(x for x in src["R4"][i1 + 1:] if x.strip()); se2 = next(x for x in src["R4"][i2 + 1:] if x.strip())
print(f"  SE line after MW elasticity: {se1.strip()} ; after own-wage elasticity: {se2.strip()}")
find("R4", "Japan’s spot labor market")
print(f"  arithmetic: -0.268/0.366 = {-0.268/0.366:.3f}; -0.268*0.068/0.047 = {-0.268*0.068/0.047:.3f}")
F["R4"] = dict(ok=(cells(se1)[-1] == "(0.132)" and cells(se2)[-1] == "(0.091)"),
    locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",
    note="Title matches and 2025 equals the first-submission year (v1 7 May 2025); Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market.")

print("=== R5 finding")
_, tr = find("R5", "Treatment | -0.0911 | -0.0908 | -0.0961 | -0.0962"); trc = cells(tr)
i = src["R5"].index(tr); sel = next(x for x in src["R5"][i + 1:] if x.strip()); print("  SE line: " + sel.strip())
_, pch = find("R5", "% Change | -0.3490"); _, mdv = find("R5", "Mean Dep. Var. | 0.261")
find("R5", "Using my most preferred specification (Column (4))")
find("R5", "this effect represents a percentage change of 36.85 percent")
print(f"  arithmetic: -0.0962/0.261 = {-0.0962/0.261:.4f} (printed % Change col 4: {cells(pch)[4]})")
F["R5"] = dict(ok=(trc[4] == "-0.0962" and cells(sel)[4] == "(0.0129)" and cells(pch)[4] == "-0.3685"),
    locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment and % Change (Section 4.1 names column (4) as preferred)",
    note="Title and 2025 year match; Table 3 column (4) prints Treatment -0.0962 (0.0129) and % Change -0.3685 (36.85%), and Section 4.1 calls column (4) the preferred specification.")

reference_checks = {}
for r in refs:
    rid = r["id"]; rc_ = results_checks[rid]
    reference_checks[rid] = {"title_matches": rc_["title_matches"], "year_matches": rc_["year_matches"],
                             "finding_supported": bool(F[rid]["ok"]), "locator": F[rid]["locator"], "note": F[rid]["note"]}
unver = [k for k, v in reference_checks.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]

data_issues = [
    {"issue": "arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431 has SE (0.210) in Table 6 but (0.218) in Table A19; the A19 note says 640 never-treated regions whereas Tables 6-7 say 608.",
     "handling": "Internal inconsistency of the source. R1 cites Table 6 (the main table), whose SE 0.210 matches the citation; both SEs give |t|>1.96, so the 5% significance claim holds either way. Not grounds for flagging R1; recorded."},
    {"issue": "arxiv-2609.27732v1: the extract header announces 'the paragraph that discusses' Table A19, but the file ends at the A19 note (no discussion paragraph); abstract says fatal outcomes fall by 35% while Section 6.1 says ~30% (Table 7 5 km: -0.354).",
     "handling": "Not needed for the R1 check (decided by Table 6); recorded only."},
    {"issue": "arxiv-2603.03260v2: Table 3 header row is garbled in the rendering.",
     "handling": "Used the extract editor's note giving the six column names in order; tau=4 / tau=5 columns read as 0.037 / 0.034 for the preferred row, consistent with the Section 3.2 text."},
    {"issue": "arxiv-2606.14621v1: rendering defect 'p ¡ 0.001' in the short-term weeks-on-UI sentence; the text reports the weeks-on-UI CI as positive magnitudes while Table 2 prints negative bounds.",
     "handling": "Read '¡' as '<'; sign difference is a reduction-vs-change convention. Neither affects R2 (decided by Table 2)."},
    {"issue": "arxiv-2505.04555v3: the printed SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment' (0.091), although the point estimate is a ratio (-0.268/0.366 = -0.732).",
     "handling": "Possible source-side copy error that cannot be resolved from the extract; the citation reproduces the printed value exactly, so R4's finding is treated as supported. Recorded as a caveat."},
    {"issue": "arxiv-2510.03658v1: abstract headline effect (0.0855, 18.79%, from Table 2, not in extract) differs from Table 3 column (4) (0.0962, 36.85%); Table 3 % Change -0.3685 vs -0.0962/0.261 = -0.3686.",
     "handling": "Different outcome definitions (count vs binary); the citation explicitly refers to the preferred specification of the binary outcome (Table 3). Last-digit difference is rounding. Not grounds for flagging R5."},
]

n_input_rows = {"README.md": wc_l(os.path.join(D, "README.md")), "reference_list.json": len(refs)}
for r in refs:
    n_input_rows[os.path.basename(r["source_file"])] = wc_l(os.path.join(D, r["source_file"]))

baseline_unver = [k for k, v in baseline_flags.items() if v]
results = {
    "task_id": "F4-conf-04",
    "reference_checks": reference_checks,
    "unverifiable_references": unver,
    "n_verified": len(refs) - len(unver),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(refs),
    "random_seed": None,
    "baseline_unverifiable_bibliographic_only": baseline_unver,
    "baseline_n_verified_bibliographic_only": len(refs) - len(baseline_unver),
}
os.makedirs(os.path.dirname(OUT), exist_ok=True)
with open(OUT, "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)
print("=== summary")
for k, v in reference_checks.items():
    print(f"  {k}: title={v['title_matches']} year={v['year_matches']} finding={v['finding_supported']}")
print(f"  unverifiable_references={unver}; n_verified={results['n_verified']}")