"""F4-conf-04: verify five references against the provided arXiv extracts.

Reads only data/. Writes out/results.json deterministically. No randomness.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
"""
import json, os, re, unicodedata, math

ROOT = os.getcwd()
DATA = os.path.join(ROOT, "data")
OUT = os.path.join(ROOT, "out")


def read_lines(path):
    with open(path, encoding="utf-8") as f:
        return f.read().split("\n")


def wc_l(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")


def norm_title(s):
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c)).lower()
    s = s.replace("&", " and ")
    s = re.sub(r"[^a-z0-9 ]+", " ", s)
    return " ".join(s.split())


def find_line(lines, needle):
    """Return (1-based line number, line) of the first line containing needle; raise if absent."""
    for i, l in enumerate(lines, 1):
        if needle in l:
            return i, l
    raise AssertionError(f"evidence not found: {needle!r}")


def cell(lines, row_label, col_index, sep=" | "):
    """Return the cell at col_index (0 = label) of the first table row starting with row_label."""
    i, l = find_line(lines, row_label + sep)
    return i, [c.strip() for c in l.split("|")][col_index]


refs = json.load(open(os.path.join(DATA, "reference_list.json"), encoding="utf-8"))

# ---- n_input_rows -------------------------------------------------------
n_input_rows = {"README.md": wc_l(os.path.join(DATA, "README.md")),
                "reference_list.json": len(refs)}
for r in refs:
    n_input_rows[os.path.basename(r["source_file"])] = wc_l(os.path.join(DATA, r["source_file"]))

src = {r["id"]: read_lines(os.path.join(DATA, r["source_file"])) for r in refs}
sid = {r["id"]: os.path.basename(r["source_file"])[:-4] for r in refs}

# ---- title and year checks (mechanical) ---------------------------------
MONTHS = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
meta = {}
for r in refs:
    L = src[r["id"]]
    header_title = L[1].split("Title:", 1)[1].strip()
    body_title = next(l[2:].strip() for l in L if l.startswith("# "))
    bib = next(l for l in L if l.startswith("Bibliographic record:"))
    years = sorted({int(y) for y in re.findall(r"\b\d{1,2} " + MONTHS + r" (\d{4})", bib)})
    m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{4,5})\.?$", r["entry"])
    entry_year, entry_title, entry_arxiv = int(m.group(2)), m.group(3), m.group(4)
    title_ok = norm_title(entry_title) == norm_title(header_title) == norm_title(body_title)
    # consistent iff equal to the first-submission year or the reproduced-version year (last date on the line)
    first_year, version_year = years[0], int(re.findall(r"\b\d{1,2} " + MONTHS + r" (\d{4})", bib)[-1])
    year_ok = entry_year in (first_year, version_year)
    arxiv_ok = sid[r["id"]].startswith("arxiv-" + entry_arxiv)
    meta[r["id"]] = dict(title_ok=title_ok, year_ok=year_ok, arxiv_ok=arxiv_ok,
                         entry_title=entry_title, header_title=header_title,
                         entry_year=entry_year, first_year=first_year, version_year=version_year)
    print(f"[{r['id']}] {sid[r['id']]}")
    print(f"   header : Title: {header_title}")
    print(f"   body   : # {body_title}")
    print(f"   {bib}")
    print(f"   entry  : title={entry_title!r} year={entry_year} arXiv={entry_arxiv}")
    print(f"   -> title_matches={title_ok} year_matches={year_ok} (first={first_year}, version={version_year}) arxiv_id_matches={arxiv_ok}")

# ---- finding checks: each element of cited_for tied to printed evidence ----
finding = {}


def show(rid, label, ln, text):
    print(f"   [{rid}] {label}: line {ln}: {text}")


# R1 — Armendariz, Table 6, 5 km column
L = src["R1"]
cols = [c.strip() for c in find_line(L, "| 1 km | 3 km | 5 km")[1].split("|")]
k5 = cols.index("5 km")
ln, att = cell(L, "ATT", k5); show("R1", "Table 6 ATT 5 km", ln, att)
i_se = ln + 2; se = [c.strip() for c in L[i_se - 1].split("|")][k5]; show("R1", "Table 6 SE 5 km", i_se, se)
ln2, pct = cell(L, "% change", k5); show("R1", "Table 6 % change 5 km", ln2, pct)
ln3, sig = find_line(L, "statistically significant at the 5% level"); show("R1", "Sec 6 text", ln3, sig.strip())
ln4, ab = find_line(L, "staggered introduction of DCRs"); show("R1", "abstract", ln4, ab[:160] + "...")
r1 = dict(att=att == "-0.431**", se=se == "(0.210)", pct=pct == "-35.0%",
          pct_arith=abs((math.exp(-0.431) - 1) * 100 - (-35.0)) < 0.05,
          sig5=("**" in att) and abs(0.431 / 0.210) > 1.96,
          design=("staggered" in ab and "difference-in-differences" in ab and "hospital records" in ab
                  and "Switzerland" in ab and "within 5 km" in ab))
print("   R1 element checks:", r1)
finding["R1"] = dict(ok=all(r1.values()),
                     locator="arxiv-2609.27732v1 | Table 6, row ATT (and SE row below), column 5 km; row % change; Section 6 text after Table 6; abstract",
                     note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, the text states significance at 5%, and the abstract describes the staggered DiD on Swiss individual-level hospital records; all elements supported.")

# R2 — Li et al., Table 2 ATE column
L = src["R2"]
ln, ab = find_line(L, "N = 23,549"); show("R2", "abstract", ln, ab[:200] + "...")
ln_w, w = cell(L, "Wage", 4); show("R2", "Table 2 Wage ATE", ln_w, w)
ln_r, re_ = cell(L, "Reemployment", 4); show("R2", "Table 2 Reemployment ATE", ln_r, re_)
ln_u, u = cell(L, "Weeks on UI", 4); show("R2", "Table 2 Weeks on UI ATE", ln_u, u)
r2 = dict(rct=("preregistered randomized controlled trial" in ab and "N = 23,549" in ab),
          wage=w.startswith("$1,153;") and "95% CI: [166, 2,141]" in w,
          annualised="annualized wages by $1,153" in ab,
          reemp=re_.startswith("1.47 pp;"),
          weeks=u.startswith("-1.99 weeks;"))
print("   R2 element checks:", r2)
finding["R2"] = dict(ok=all(r2.values()),
                     locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage, Reemployment, Weeks on UI; abstract (N, preregistered RCT)",
                     note="Table 2 ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract states a preregistered RCT with N = 23,549 and annualized wages; all elements supported.")

# R3 — Liu & Zhao, Section 3.2 / Table 3 (timing claim)
L = src["R3"]
ln, txt = find_line(L, "the stacked coefficient is 0.005 at \\tau=0"); show("R3", "Sec 3.2 headline", ln, txt[:330] + "...")
ln2, nj = find_line(L, "prices do not jump immediately"); show("R3", "Sec 3.2", ln2, "...prices do not jump immediately, but instead drift upward...")
ln_t, t4 = cell(L, "Clean-onset threshold-2", 4); _, t5 = cell(L, "Clean-onset threshold-2", 5)
show("R3", "Table 3 preferred row tau=4 / tau=5", ln_t, f"{t4} / {t5}")
tau0 = 0.005
r3 = dict(jump_in_event_year=(tau0 >= 0.034),  # cited: 3.4-3.7% in the entry year; printed tau=0 coefficient is 0.005
          no_later_appreciation=not (float(t4) > tau0 and float(t5) > tau0),  # cited: no further appreciation
          explicitly_contradicted="prices do not jump immediately" in nj)
print("   R3 element checks:", r3)
finding["R3"] = dict(ok=r3["jump_in_event_year"] and r3["no_later_appreciation"],
                     locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row Clean-onset threshold-2, columns tau=4 / tau=5",
                     note="The extract reports about 0.5% (0.005) at the event year rising gradually to 3.4-3.7% (0.037 at tau=4, 0.034 at tau=5) and says prices do not jump immediately, contradicting the cited jump in the entry year with no later appreciation; the entry's title and year (2024) also do not match the document (v1 Mar 2026, v2 Apr 2026).")

# R4 — Kanayama et al., Appendix A.1 Table 2
L = src["R4"]
ln_e, e1 = cell(L, "Employment elasticity w.r.t MW", 1); se1 = L[ln_e + 1].split("|")[1].strip()
show("R4", "Table 2 Employment elasticity w.r.t MW", ln_e, f"{e1} {se1}")
ln_o, e2 = cell(L, "Emp. elasticity w.r.t affected wage", 1); se2 = L[ln_o + 1].split("|")[1].strip()
show("R4", "Table 2 Emp. elasticity w.r.t affected wage", ln_o, f"{e2} {se2}")
ln_a, ab = find_line(L, "Japan’s spot labor market"); show("R4", "abstract", ln_a, ab[:150] + "...")
_, ae = cell(L, "Affected employment", 1); _, aw = cell(L, "Affected wages", 1)
r4 = dict(mw=(e1 == "-0.388" and se1 == "(0.132)"), own=(e2 == "-0.732" and se2 == "(0.091)"),
          population="Japan’s spot labor market" in ab,
          own_arith=abs(float(ae) / float(aw) - (-0.732)) < 0.0005)
print("   R4 element checks:", r4, f"(affected employment / affected wages = {float(ae)/float(aw):.4f})")
finding["R4"] = dict(ok=r4["mw"] and r4["own"] and r4["population"],
                     locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (estimate and SE rows); abstract (Japan's spot labor market)",
                     note="Table 2 prints -0.388 (0.132) and -0.732 (0.091) and the abstract places the study in Japan's spot labor market; supported, with the year 2025 matching the v1 submission (7 May 2025).")

# R5 — Tossou, Table 3 column (4)
L = src["R5"]
ln_t, tr = cell(L, "Treatment", 4); tse = L[ln_t + 1].split("|")[4].strip()
show("R5", "Table 3 Treatment col (4)", ln_t, f"{tr} {tse}")
ln_pc, pc = cell(L, "% Change", 4); ln_m, mdv = cell(L, "Mean Dep. Var.", 4)
show("R5", "Table 3 % Change col (4)", ln_pc, pc); show("R5", "Table 3 Mean Dep. Var. col (4)", ln_m, mdv)
ln_p, pref = find_line(L, "most preferred specification (Column (4))"); show("R5", "Sec 4.1", ln_p, "...Using my most preferred specification (Column (4)) ... by 0.0962 ... 36.85 percent")
r5 = dict(coef=(tr == "-0.0962" and tse == "(0.0129)"), preferred="(Column (4))" in pref and "0.0962" in pref,
          pct=pc == "-0.3685" and "36.85 percent" in pref,
          pct_arith=abs(-0.0962 / float(mdv) - float(pc)) < 0.0005,
          outcome="Probability of infant death and stillbirth" in "\n".join(L))
print("   R5 element checks:", r5, f"(-0.0962/{mdv} = {-0.0962/float(mdv):.4f})")
finding["R5"] = dict(ok=all(r5.values()),
                     locator="arxiv-2510.03658v1 | Table 3, column (4), rows Treatment (estimate and SE), % Change, Mean Dep. Var.; Section 4.1 text on the preferred specification",
                     note="Table 3 column (4), named the preferred specification in Section 4.1, prints -0.0962 (0.0129) and % Change -0.3685 (36.85% in text; -0.0962/0.261 = -0.3686); supported, although the abstract's headline (0.0855, 18.79%) refers to a different outcome.")

# ---- assemble -----------------------------------------------------------
reference_checks = {}
for r in refs:
    rid = r["id"]
    reference_checks[rid] = {"title_matches": meta[rid]["title_ok"], "year_matches": meta[rid]["year_ok"],
                             "finding_supported": finding[rid]["ok"], "locator": finding[rid]["locator"],
                             "note": finding[rid]["note"]}
unverifiable = [k for k, v in reference_checks.items()
                if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]
baseline_unverifiable = [k for k in meta if not (meta[k]["title_ok"] and meta[k]["year_ok"])]

data_issues = [
    {"issue": "arxiv-2609.27732v1: the same 5 km CS-DR ATT -0.431 has SE (0.210) in Table 6 but (0.218) in Table A19; Table A19 note says 640 never-treated regions while Tables 6, 7 and A19 rows say 608.",
     "handling": "Main-text Table 6 used as the decisive locator (the citation reproduces it); significance at 5% holds under either SE (0.431/0.218 > 1.96); inconsistency reported, not flagged against R1."},
    {"issue": "arxiv-2609.27732v1: abstract says fatal outcomes fall 35% and narcotics rise 'by a comparable amount', while Section 6.1 says about 30% and 47% (consistent with exp(-0.354)-1 and exp(0.390)-1).",
     "handling": "Not part of R1's cited finding; recorded only."},
    {"issue": "arxiv-2606.14621v1: rendering defect 'p ¡ 0.001' in Section 3; Tables 2-3 were re-set by the extract editor (declared in the header).",
     "handling": "Used the re-set tables as declared verbatim; the defect does not affect any cited value."},
    {"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering.",
     "handling": "Column order taken from the extract editor's note (Specification | Treated-role LSOAs | Stacked p | Sun p | tau=4 | tau=5)."},
    {"issue": "arxiv-2505.04555v3: SE of the own-wage elasticity (0.091) is identical to the SE of 'Affected employment'; table note says 'about 4%' MW increase while '% MW changes' is 0.047.",
     "handling": "Cannot be resolved from the extract; the cited values are printed exactly as cited, so R4 is not flagged; recorded as a caveat."},
    {"issue": "arxiv-2510.03658v1: abstract headline (0.0855, 18.79%, mean 0.455, from omitted Table 2) differs from Table 3 preferred estimate (0.0962, 36.85%, mean 0.261); footnote 11 is garbled (three percentages for five wealth categories) and footnote markers render as '^9^9 9'.",
     "handling": "Treated as different outcomes (count vs binary indicator); R5 cites the Table 3 binary outcome and is checked against Table 3 column (4)."},
]

results = {
    "task_id": "F4-conf-04",
    "reference_checks": reference_checks,
    "unverifiable_references": unverifiable,
    "n_verified": len(refs) - len(unverifiable),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(refs),
    "random_seed": None,
    "baseline_metadata_only_unverifiable": baseline_unverifiable,
}
os.makedirs(OUT, exist_ok=True)
with open(os.path.join(OUT, "results.json"), "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)
print("\nunverifiable_references:", unverifiable, "| n_verified:", results["n_verified"],
      "| metadata-only baseline:", baseline_unverifiable)