"""F4-conf-04: verify five references against source extracts. Reads data/ only; writes out/results.json.
No randomness (random_seed = null)."""
import json, os, re, unicodedata

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
OUT = os.path.join(ROOT, "out")

def wc_l(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")

refs = json.load(open(os.path.join(D, "reference_list.json"), encoding="utf-8"))
n_input_rows = {"README.md": wc_l(os.path.join(D, "README.md")), "reference_list.json": len(refs)}
src = {}
for r in refs:
    p = os.path.join(D, r["source_file"])
    src[r["id"]] = open(p, encoding="utf-8").read().splitlines()
    n_input_rows[os.path.basename(p)] = wc_l(p)

def norm(s):
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c)).lower().replace("&", "and")
    return re.sub(r"[^a-z0-9]+", " ", s).strip()

def header(lines, key):
    for ln in lines:
        if ln.startswith(key + ":"):
            return ln[len(key) + 1:].strip()
    raise KeyError(key)

def find_line(lines, pattern):
    """Return (lineno, line) of first line matching regex pattern; raises if absent."""
    for i, ln in enumerate(lines, 1):
        if re.search(pattern, ln):
            return i, ln
    raise ValueError("pattern not found: " + pattern)

def show(rid, lines, pattern, nxt=0):
    i, ln = find_line(lines, pattern)
    print(f"  [{rid} L{i}] {ln.strip()[:200]}")
    out = [ln]
    for k in range(1, nxt + 1):
        # skip blank lines to fetch the following non-empty line (SE row)
        j = i
        while True:
            j += 1
            if lines[j - 1].strip():
                break
        print(f"  [{rid} L{j}] {lines[j-1].strip()[:200]}")
        out.append(lines[j - 1])
    return out

def cells(line):
    return [c.strip() for c in line.split("|")]

checks, baseline = {}, []
for r in refs:
    rid, L = r["id"], src[r["id"]]
    print(f"== {rid}: {r['source_file']}")
    title = header(L, "Title"); bib = header(L, "Bibliographic record")
    print("  Title:", title); print("  Bibliographic record:", bib)
    # body title line must agree with header title
    body_title = [ln[2:].strip() for ln in L if ln.startswith("# ")][0]
    assert norm(body_title) == norm(title), rid
    # entry title = text between "(YYYY). " and ". arXiv:"
    m = re.match(r"^(.*?)\((\d{4})\)\.\s+(.*)\.\s+arXiv:(\d{4}\.\d{5})\.?$", r["entry"])
    entry_year, entry_title, entry_arxiv = int(m.group(2)), m.group(3), m.group(4)
    assert entry_arxiv in bib, rid
    years = sorted({int(y) for y in re.findall(r"submitted \d{1,2} \w{3} (\d{4})", bib)})
    title_ok = norm(entry_title) == norm(title)
    year_ok = entry_year in years  # years = first submission year and reproduced-version year (and any intermediate)
    print(f"  entry title match={title_ok}; entry year {entry_year} vs submission years {years} -> {year_ok}")
    checks[rid] = {"title_matches": title_ok, "year_matches": year_ok}
    if not (title_ok and year_ok):
        baseline.append(rid)

# ---------------- finding checks (element by element) ----------------
fs = {}
# R1: Table 6, 5 km column
L = src["R1"]
att, se = show("R1", L, r"^ATT \| -0\.399", 1)
pc = show("R1", L, r"^% change \| -32\.9%")[0]
show("R1", L, r"statistically significant at the 5% level")
show("R1", L, r"staggered introduction of DCRs")
a5, s5, p5 = cells(att)[3], cells(se)[3], cells(pc)[3]
e = {"ATT -0.431": a5.startswith("-0.431"),
     "SE 0.210": s5 == "(0.210)",
     "5% significance (** marker)": a5.endswith("**") and not a5.endswith("***"),
     "roughly 35%": p5 == "-35.0%",
     "within 5 km": "within 5 km of a facility" in "\n".join(L),
     "staggered DiD on Swiss hospital records": bool(re.search(r"staggered introduction of DCRs across Swiss cities.*hospital records and a difference-in-differences", "\n".join(L)))}
print("  R1 elements:", e)
fs["R1"] = (all(e.values()), "arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km",
            "Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, and the abstract states a staggered DiD on Swiss hospital records, so every element is supported (Table A19 prints SE 0.218 for the same specification, recorded as a data issue).")

# R2: Table 2, Average treatment effect column
L = src["R2"]; T = "\n".join(L)
w = cells(show("R2", L, r"^Wage \| \$33,073")[0])[4]
re_ = cells(show("R2", L, r"^Reemployment \| 75\.2%")[0])[4]
ui = cells(show("R2", L, r"^Weeks on UI \| 15\.2")[0])[4]
show("R2", L, r"N = 23,549")
e = {"preregistered RCT N = 23,549": "large (N = 23,549) preregistered randomized controlled trial" in T,
     "wage +$1,153": w.startswith("$1,153"), "95% CI [166, 2,141]": "95% CI: [166, 2,141]" in w,
     "reemployment +1.47 pp": re_.startswith("1.47 pp"), "weeks on UI -1.99": ui.startswith("-1.99 weeks"),
     "annualised wages": "$1,153 higher annually" in T}
print("  R2 elements:", e)
fs["R2"] = (all(e.values()), "arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI",
            "Table 2's ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives the preregistered RCT with N = 23,549.")

# R3: timing claim vs Section 3.2 coefficients
L = src["R3"]; T = "\n".join(L)
line = show("R3", L, r"stacked coefficient is 0\.005 at")[0]
row = cells(show("R3", L, r"^Clean-onset threshold-2 \|")[0])
coef = {int(k): float(v) for v, k in re.findall(r"(\d\.\d{3}) (?:at|by) \\tau=(\d)", line)}
print("  R3 stacked coefficients by tau:", coef, "| Table 3 preferred tau=4,5:", row[4], row[5])
e = {"3.4-3.7% jump in event year (tau=0 coef >= 0.034)": coef[0] >= 0.034,
     "no further appreciation after event year": max(coef.values()) <= coef[0],
     "3.4-3.7% magnitude exists somewhere (tau=4/5)": float(row[4]) == 0.037 and float(row[5]) == 0.034}
print("  R3 elements:", e)
fs["R3"] = (e["3.4-3.7% jump in event year (tau=0 coef >= 0.034)"] and e["no further appreciation after event year"],
            "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients by tau) and Table 3, row Clean-onset threshold-2, columns tau=4 / tau=5",
            "The 3.4-3.7% magnitude is reached only at tau=4-5 (0.037, 0.034) while the event-year coefficient is 0.005, so the claimed immediate jump with no later appreciation contradicts the source; title and year (2024 vs 2026) also disagree.")

# R4: Appendix A.1 Table 2
L = src["R4"]; T = "\n".join(L)
em = show("R4", L, r"^Employment elasticity w\.r\.t MW", 1)
ow = show("R4", L, r"^Emp\. elasticity w\.r\.t affected wage", 1)
ae = show("R4", L, r"^Affected employment", 1); aw = show("R4", L, r"^Affected wages", 1)
e = {"MW elasticity -0.388": cells(em[0])[1] == "-0.388", "SE 0.132": cells(em[1])[1] == "(0.132)",
     "own-wage elasticity -0.732": cells(ow[0])[1] == "-0.732", "SE 0.091": cells(ow[1])[1] == "(0.091)",
     "Japan's spot labour market": "Japan’s spot labor market" in T}
ratio = float(cells(ae[0])[1]) / float(cells(aw[0])[1])
print(f"  R4 elements: {e}; arithmetic check affected employment/affected wages = {ratio:.4f}")
fs["R4"] = (all(e.values()), "arxiv-2505.04555v3 | Appendix A.1 Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",
            "Table 2 prints -0.388 (0.132) and -0.732 (0.091) and the abstract places the study in Japan's spot labor market; the 0.091 SE duplicates the 'Affected employment' SE, recorded as a data issue.")

# R5: Table 3 column (4)
L = src["R5"]; T = "\n".join(L)
tr = show("R5", L, r"^Treatment \| -0\.0911", 1)
pch = cells(show("R5", L, r"^% Change \|")[0]); mdv = cells(show("R5", L, r"^Mean Dep\. Var\. \|")[0])
show("R5", L, r"most preferred specification \(Column \(4\)\)")
arith = float(cells(tr[0])[4]) / float(mdv[4])
e = {"-0.0962": cells(tr[0])[4] == "-0.0962", "SE 0.0129": cells(tr[1])[4] == "(0.0129)",
     "preferred specification = col (4)": "most preferred specification (Column (4))" in T,
     "36.85% relative to mean": pch[4] == "-0.3685" and "percentage change of 36.85 percent" in T,
     "infant death or stillbirth": "Probability of Infant death or stillbirth" in T}
print(f"  R5 elements: {e}; arithmetic -0.0962/0.261 = {arith:.4f}")
fs["R5"] = (all(e.values()), "arxiv-2510.03658v1 | Table 3, row Treatment and row % Change, column (4); Section 4.1",
            "Table 3 column (4), named the preferred specification in Section 4.1, prints -0.0962 (0.0129) and % Change -0.3685, matching the 36.85% stated in the text.")

for rid, (ok, loc, note) in fs.items():
    checks[rid].update({"finding_supported": bool(ok), "locator": loc})
    checks[rid]["note"] = note
unver = [rid for rid in sorted(checks) if not all(checks[rid][k] for k in ("title_matches", "year_matches", "finding_supported"))]

data_issues = [
    {"issue": "arxiv-2609.27732v1: Table 6 prints SE (0.210) for the 5 km ATT -0.431 while Table A19 (CS-DR column, same 5 km specification) prints (0.218); Table 6 gives 608 control regions, the A19 note says 640.",
     "handling": "R1's cited SE 0.210 matches the main-results Table 6, which is used as locator; the 5% significance claim holds under either SE (** marker in both tables). Recorded, not treated as a failure."},
    {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes fall by 35%, Section 6.1 says about 30% (Table 7 5 km: -0.354).",
     "handling": "Not relevant to R1's cited finding; recorded only."},
    {"issue": "arxiv-2505.04555v3: SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment' (0.091), despite the elasticity being the ratio -0.268/0.366; note says 'about 4%' MW increase vs 0.047 in the table.",
     "handling": "R4's cited values are printed verbatim in Table 2, so the finding check passes; the possibly duplicated SE is flagged as an open issue."},
    {"issue": "arxiv-2510.03658v1: Table 3 '% Change' col (4) is -0.3685 while -0.0962/0.261 = -0.3686; the abstract's headline (0.0855, 18.79%) refers to a different outcome (Table 2, not in extract).",
     "handling": "Rounding difference ignored (text explicitly states 36.85%); abstract figures not used for R5."},
    {"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering.",
     "handling": "Used the extract editor's column names (tau=4, tau=5 coefficients in the last two columns)."},
    {"issue": "arxiv-2606.14621v1: Tables 2-3 re-set by the extract editor; 'p ¡ 0.001' glyph defect in Section 3 text.",
     "handling": "Used the re-set tables as documented; glyph defect does not affect any checked value."},
]

results = {
    "reference_checks": checks,
    "unverifiable_references": unver,
    "n_verified": len(checks) - len(unver),
    "task_id": "F4-conf-04",
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(checks),
    "random_seed": None,
    "baseline_unverifiable_title_year_only": baseline,
    "baseline_n_verified_title_year_only": len(checks) - len(baseline),
}
os.makedirs(OUT, exist_ok=True)
with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)
print("unverifiable_references:", unver, "| n_verified:", results["n_verified"])