"""F4-conf-04: verify five references against the provided source extracts.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json deterministically (no randomness)."""
import json, math, os, re, unicodedata

DATA = "data"
OUT = "out"
SEP = "-" * 72


def read(rel):
    with open(os.path.join(DATA, rel), encoding="utf-8") as f:
        return f.read()


def wc_l(rel):
    with open(os.path.join(DATA, rel), "rb") as f:
        return f.read().count(b"\n")


def norm_title(s):
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c))
    s = s.lower().replace("&", " and ")
    s = re.sub(r"[^\w\s]", " ", s)
    return re.sub(r"\s+", " ", s).strip()


def line_starting(text, prefix, nth=0):
    """Return the nth line starting with prefix and the following non-empty line."""
    lines = text.split("\n")
    hits = [i for i, l in enumerate(lines) if l.startswith(prefix)]
    i = hits[nth]
    nxt = next((lines[j] for j in range(i + 1, len(lines)) if lines[j].strip()), "")
    return lines[i], nxt


def cells(line):
    return [c.strip() for c in line.split("|")]


def num(s):
    s = s.replace("−", "-").replace("$", "").replace(",", "").replace("%", "")
    m = re.search(r"-?\d+(?:\.\d+)?", s)
    return float(m.group()) if m else None


refs = json.loads(read("reference_list.json"))
files = ["README.md", "reference_list.json"] + sorted(
    "sources/" + f for f in os.listdir(os.path.join(DATA, "sources")))
n_input_rows = {}
for f in files:
    n_input_rows[f] = len(refs) if f.endswith(".json") else wc_l(f)

# ---------------------------------------------------------------- header parse
def parse_source(rel):
    t = read(rel)
    head, body = t.split("=" * 20, 1)
    title = re.search(r"^Title: (.*)$", head, re.M).group(1).strip()
    bib = re.search(r"^Bibliographic record: (.*)$", head, re.M).group(1).strip()
    arxiv = re.search(r"arXiv:(\d{4}\.\d{5})v(\d+)", bib)
    body_title = re.search(r"^# (.*)$", body, re.M).group(1).strip()
    vyears = {}
    for v, y in re.findall(r"version (\d+)(?: \(this extract\))? submitted \d+ \w+ (\d{4})", bib):
        vyears[int(v)] = int(y)
    m = re.search(r"submitted \d+ \w+ (\d{4}) \(version (\d+)", bib)
    if m:
        vyears[int(m.group(2))] = int(m.group(1))
    return dict(text=t, title=title, body_title=body_title, bib=bib, arxiv=arxiv.group(1),
                version=int(arxiv.group(2)), vyears=vyears)


# ---------------------------------------------------------------- finding checks
def el(label, ok, evidence):
    return {"element": label, "ok": bool(ok), "evidence": evidence}


def check_R1(t):
    att, se = line_starting(t, "ATT | -0.399")
    pct = line_starting(t, "% change | -32.9%")[0]
    hdr = "| 1 km | 3 km | 5 km"
    c_att, c_se, c_pct = cells(att), cells(se), cells(pct)
    print("  Table 6 header:", hdr); print("  ", att); print("  ", se); print("  ", pct)
    a5, s5, p5 = c_att[3], c_se[3], c_pct[3]
    a19, s19 = line_starting(t, "ATT | -0.431** | -0.438**")
    print("  Table A19:", a19, "/", s19)
    implied = 100 * (math.exp(num(a5)) - 1)
    return [
        el("ATT -0.431 at 5 km", num(a5) == -0.431, "Table 6 ATT row, 5 km col: " + a5),
        el("SE 0.210", s5 == "(0.210)", "Table 6 SE row, 5 km col: " + s5),
        el("significant at 5% level", a5.endswith("**") and not a5.endswith("***"),
           "Table 6 stars '**' = p<0.05; text: 'statistically significant at the 5% level'"),
        el("roughly 35% reduction", p5 == "-35.0%" and abs(implied + 35.0) < 0.05,
           f"Table 6 % change 5 km {p5}; exp(-0.431)-1 = {implied:.2f}%"),
        el("within 5 km of a facility", "within 5 km of a facility" in t, "abstract"),
        el("staggered difference-in-differences", "staggered introduction of DCRs" in t
           and "difference-in-differences estimator" in t, "abstract"),
        el("Swiss hospital records", "individual-level hospital records" in t
           and "Switzerland" in t, "abstract"),
    ], {"tableA19_CS_DR_SE": cells(s19)[1], "table6_5km_SE": s5}


def check_R2(t):
    w = line_starting(t, "Wage | $33,073")[0]
    r = line_starting(t, "Reemployment | 75.2%")[0]
    u = line_starting(t, "Weeks on UI | 15.2")[0]
    for l in (w, r, u):
        print("  Table 2:", l)
    cw, cr, cu = cells(w)[4], cells(r)[4], cells(u)[4]
    return [
        el("preregistered RCT", "preregistered randomized controlled trial" in t, "abstract"),
        el("N = 23,549", "N = 23,549" in t, "abstract"),
        el("annualised wages +$1,153", cw.startswith("$1,153"), "Table 2 Wage, ATE col: " + cw),
        el("95% CI [166, 2,141]", "95% CI: [166, 2,141]" in cw, "Table 2 Wage, ATE col"),
        el("reemployment +1.47 pp", cr.startswith("1.47 pp"), "Table 2 Reemployment, ATE col: " + cr),
        el("weeks on UI -1.99", cu.startswith("-1.99 weeks"), "Table 2 Weeks on UI, ATE col: " + cu),
    ], {}


def check_R3(t):
    m = re.search(r"the stacked coefficient is ([^:]*?) by \\tau=5", t)
    sent = m.group(0)
    print("  Sec 3.2:", sent)
    coefs = {int(k): float(v) for v, k in re.findall(r"(\d\.\d{3}) (?:at|by) \\tau=(\d)", sent)}
    row = line_starting(t, "Clean-onset threshold-2 |")[0]
    print("  Table 3:", row)
    c = cells(row)
    tau0 = coefs[0]
    later_rise = coefs[4] > coefs[0] and coefs[2] > coefs[1] > coefs[0]
    return [
        el("magnitude 3.4–3.7%", "3.4–3.7%" in t and float(c[4]) == 0.037 and float(c[5]) == 0.034,
           f"Table 3 preferred row tau=4 {c[4]}, tau=5 {c[5]}; text '3.4–3.7% after four to five years'"),
        el("jump in the year of entry", abs(tau0) >= 0.034,
           f"Sec 3.2: stacked coefficient at tau=0 is {tau0} (~0.5%), not 3.4–3.7%"),
        el("no further appreciation later", not later_rise,
           f"Sec 3.2: coefficients rise {coefs} ('prices do not jump immediately, but instead drift upward')"),
    ], {}


def check_R4(t):
    e1, s1 = line_starting(t, "Employment elasticity w.r.t MW")
    e2, s2 = line_starting(t, "Emp. elasticity w.r.t affected wage")
    aw, saw = line_starting(t, "Affected wages")
    ae, sae = line_starting(t, "Affected employment")
    jb = line_starting(t, "Job below new MW")[0]
    mw = line_starting(t, "% MW changes")[0]
    for l in (e1, s1, e2, s2, aw, saw, ae, sae, jb, mw):
        print("  Table 2:", l)
    v1, v2 = cells(e1)[1], cells(e2)[1]
    ratio = num(cells(ae)[1]) / num(cells(aw)[1])
    mw_el = num(cells(ae)[1]) * num(cells(jb)[1]) / num(cells(mw)[1])
    print(f"  arithmetic: affected emp / affected wage = {ratio:.4f}; affected emp * 0.068/0.047 = {mw_el:.4f}")
    return [
        el("Japan's spot labour market", "Japan’s spot labor market" in t, "abstract"),
        el("elasticity w.r.t. MW -0.388", v1 == "-0.388", "Table 2 row 'Employment elasticity w.r.t MW': " + v1),
        el("SE 0.132", cells(s1)[1] == "(0.132)", "Table 2 SE row: " + cells(s1)[1]),
        el("own-wage elasticity -0.732", v2 == "-0.732", "Table 2 row 'Emp. elasticity w.r.t affected wage': " + v2),
        el("SE 0.091", cells(s2)[1] == "(0.091)", "Table 2 SE row: " + cells(s2)[1]),
    ], {"ratio": ratio, "mw_el": mw_el, "own_wage_se": cells(s2)[1], "aff_emp_se": cells(sae)[1]}


def check_R5(t):
    tr, se = line_starting(t, "Treatment | -0.0911")
    pc = line_starting(t, "% Change |")[0]
    mn = line_starting(t, "Mean Dep. Var. |")[0]
    for l in (tr, se, pc, mn):
        print("  Table 3:", l)
    b4, s4, p4, m4 = cells(tr)[4], cells(se)[4], cells(pc)[4], cells(mn)[4]
    implied = num(b4) / num(m4)
    print(f"  arithmetic: -0.0962/0.261 = {implied:.4f}")
    return [
        el("infant death or stillbirth outcome", "Probability of infant death and stillbirth" in t, "Table 3 title"),
        el("estimate -0.0962", b4 == "-0.0962", "Table 3 Treatment row, col (4): " + b4),
        el("SE 0.0129", s4 == "(0.0129)", "Table 3 SE row, col (4): " + s4),
        el("preferred specification", "most preferred specification (Column (4))" in t, "Sec 4.1"),
        el("36.85% reduction vs mean", p4 == "-0.3685" and "36.85 percent" in t,
           f"Table 3 % Change col (4) {p4}; Sec 4.1 '36.85 percent'; recomputed {implied:.4f}"),
    ], {}


CHECKERS = {"R1": check_R1, "R2": check_R2, "R3": check_R3, "R4": check_R4, "R5": check_R5}
LOCATORS = {
    "R1": "arxiv-2609.27732v1 | Table 6, ATT and % change rows, 5 km column (plus abstract for design/population)",
    "R2": "arxiv-2606.14621v1 | Table 2, rows Wage / Reemployment / Weeks on UI, column 'Average treatment effect estimate' (plus abstract for N and design)",
    "R3": "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row 'Clean-onset threshold-2', tau=4/tau=5 columns",
    "R4": "arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",
    "R5": "arxiv-2510.03658v1 | Table 3, Treatment row and % Change row, column (4); Section 4.1",
}

reference_checks, details, extra = {}, {}, {}
for r in refs:
    rid = r["id"]
    src = parse_source(r["source_file"])
    print(SEP); print(rid, "->", r["source_file"])
    print("  header Title:", src["title"]); print("  body title  :", src["body_title"])
    print("  Bibliographic record:", src["bib"])
    m = re.search(r"\(([\d/]+)\)\.\s*(.*)\.\s*arXiv:(\d{4}\.\d{5})", r["entry"])
    e_year, e_title, e_id = int(m.group(1)[:4]), m.group(2).strip(), m.group(3)
    print("  entry title :", e_title, "| entry year:", e_year, "| entry id:", e_id)
    title_ok = norm_title(e_title) == norm_title(src["title"]) == norm_title(src["body_title"])
    admissible = sorted({src["vyears"][1], src["vyears"][src["version"]]})
    year_ok = e_year in admissible
    print("  admissible years (v1, reproduced version):", admissible)
    assert e_id == src["arxiv"], "arXiv id mismatch"
    elements, ex = CHECKERS[rid](src["text"])
    extra[rid] = ex
    find_ok = all(e["ok"] for e in elements)
    for e in elements:
        print(f"    [{'OK' if e['ok'] else 'FAIL'}] {e['element']} :: {e['evidence']}")
    details[rid] = elements
    failed = [e["element"] for e in elements if not e["ok"]]
    parts = []
    if not title_ok:
        parts.append(f"entry title '{e_title}' differs from the document title '{src['title']}'")
    if not year_ok:
        parts.append(f"entry year {e_year} not in admissible years {admissible}")
    if failed:
        parts.append("finding elements not supported: " + "; ".join(failed))
    note = ("All elements of the attributed finding, the title and the year agree with the extract."
            if not parts else ("; ".join(parts) + "."))
    note = note[0].upper() + note[1:]
    reference_checks[rid] = {"title_matches": bool(title_ok), "year_matches": bool(year_ok),
                             "finding_supported": bool(find_ok), "locator": LOCATORS[rid],
                             "note": note}

unverifiable = [k for k, v in reference_checks.items()
                if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]

# ---------------------------------------------------------------- data issues (detected from the extracts)
t1 = read("sources/arxiv-2609.27732v1.txt")
data_issues = []
if extra["R1"]["tableA19_CS_DR_SE"] != extra["R1"]["table6_5km_SE"]:
    data_issues.append({
        "issue": f"arxiv-2609.27732v1: the same CS-DR 5 km ATT (-0.431**) has SE {extra['R1']['table6_5km_SE']} in Table 6 but {extra['R1']['tableA19_CS_DR_SE']} in Table A19 (CS-DR column).",
        "handling": "The R1 citation quotes SE 0.210, which is printed in Table 6, the main results table; the finding is treated as supported, and the discrepancy is reported here and in the report."})
if "(640 regions)" in t1 and "608 never-treated" in t1:
    data_issues.append({
        "issue": "arxiv-2609.27732v1: the Table A19 note gives 640 never-treated comparison regions, while Table 6, Table A19 body and the Figure 6 note give 608.",
        "handling": "Not relevant to any cited element; recorded only."})
if "fall by 35%" in t1 and "approximately 30% at 5 km" in t1:
    data_issues.append({
        "issue": f"arxiv-2609.27732v1: the abstract says fatal outcomes fall by 35%, while Section 6.1 says approximately 30% at 5 km (Table 7 fatal 5 km ATT -0.354; exp(-0.354)-1 = {100 * (math.exp(-0.354) - 1):.1f}%).",
        "handling": "Not part of the R1 citation (which concerns the overall outcome); recorded only."})
r4 = extra["R4"]
if r4["own_wage_se"] == r4["aff_emp_se"]:
    data_issues.append({
        "issue": f"arxiv-2505.04555v3 Table 2: the own-wage elasticity SE {r4['own_wage_se']} is identical to the 'Affected employment' SE, although the point estimate equals affected employment / affected wages ({r4['ratio']:.4f}); the MW-elasticity SE (0.132) matches 0.091*0.068/0.047 = {0.091*0.068/0.047:.4f}, whereas the analogous scaling for the own-wage elasticity (0.091/0.366) gives {0.091/0.366:.3f}, not 0.091. The printed SE may be a transcription error in the source (it cannot be resolved from the extract).",
        "handling": "The R4 citation reproduces the printed values exactly, so the finding is treated as stated in the source; the possible source-level inconsistency is reported, not corrected."})
data_issues.append({
    "issue": "arxiv-2603.03260v2 Table 3: the header row is garbled in the HTML rendering (editor's note supplies the column order).",
    "handling": "Used the editor-supplied column order; the tau=4/tau=5 values (0.037/0.034) agree with the Section 3.2 text."})

# Baseline (P2): header-only screen using title and year checks alone, no finding check.
baseline_header_only = [k for k, v in reference_checks.items()
                        if not (v["title_matches"] and v["year_matches"])]

results = {
    "task_id": "F4-conf-04",
    "reference_checks": reference_checks,
    "unverifiable_references": unverifiable,
    "n_verified": len(refs) - len(unverifiable),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(refs),
    "random_seed": None,
    "baseline_header_only_unverifiable": baseline_header_only,
}
os.makedirs(OUT, exist_ok=True)
with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)
with open(os.path.join(OUT, "finding_elements.json"), "w", encoding="utf-8") as f:
    json.dump(details, f, indent=2, ensure_ascii=False)
print(SEP)
print("unverifiable_references:", unverifiable, "| n_verified:", results["n_verified"])