"""F4-conf-04: verify five references against the provided source extracts.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads data/ only; prints every header line and table cell it relies on; writes out/results.json.
The judgements are encoded as explicit, mechanical checks on strings/numbers found in the extracts."""
import json, math, os, re, unicodedata

D = "data"
refs = json.load(open(os.path.join(D, "reference_list.json")))
src = {}
for r in refs:
    src[r["id"]] = open(os.path.join(D, r["source_file"]), encoding="utf-8").read().splitlines()

def find(rid, pattern):
    """Return first line (1-based no, text) matching regex; print it; fail loudly if absent."""
    for i, l in enumerate(src[rid]):
        if re.search(pattern, l):
            print(f"  [{rid} L{i+1}] {l.strip()[:220]}")
            return l
    raise SystemExit(f"pattern not found in {rid}: {pattern}")

def norm_title(t):
    t = unicodedata.normalize("NFKD", t).encode("ascii", "ignore").decode().lower()
    t = t.replace("&", "and")
    return re.sub(r"[^a-z0-9]+", " ", t).strip()

def entry_title(entry):
    # entry format: "Authors (YEAR). Title. arXiv:NNNN."
    m = re.match(r".*?\((\d{4})\)\.\s*(.*)\.\s*arXiv:(\S+?)\.?$", entry)
    return int(m.group(1)), m.group(2), m.group(3)

def cells(line):
    return [c.strip() for c in line.split("|")]

checks, issues = {}, []
for r in refs:
    rid = r["id"]
    print(f"\n=== {rid}: {r['entry']}")
    year, title, arx = entry_title(r["entry"])
    t_line = find(rid, r"^Title: ")
    b_line = find(rid, r"^Bibliographic record: ")
    doc_title = t_line.split("Title: ", 1)[1]
    years = sorted({int(y) for y in re.findall(r"\d{1,2} \w{3} (\d{4})", b_line)})
    first_year = years[0]
    this_version_year = int(re.findall(r"\d{1,2} \w{3} (\d{4})", b_line)[-1])
    id_ok = b_line.split("arXiv:")[1].split("v")[0] == arx.split("v")[0]
    title_ok = norm_title(title) == norm_title(doc_title)
    year_ok = year in (first_year, this_version_year)
    print(f"  entry year={year}; first submission year={first_year}; reproduced version year={this_version_year}; arXiv id match={id_ok}")
    print(f"  title match={title_ok}; year match={year_ok}")
    checks[rid] = dict(title_matches=title_ok, year_matches=year_ok)

# ---------------- finding checks ----------------
print("\n--- Finding checks ---")
# R1
print("R1:")
abs1 = find("R1", r"reduce drug-related hospitalization rates by roughly 35% within 5 km")
find("R1", r"staggered introduction of DCRs")
att = cells(find("R1", r"^ATT \| -0.399"))
se = cells(find("R1", r"^\| \(0.412\)"))
pct = cells(find("R1", r"^% change \| -32.9%"))
sig = find("R1", r"statistically significant at the 5% level")
a5, s5 = float(att[3].rstrip("*")), float(se[3].strip("()"))
print(f"  Table 6, 5 km: ATT={att[3]}, SE={se[3]}, %change={pct[3]}; exp(ATT)-1={math.exp(a5)-1:.4f}; |t|={abs(a5)/s5:.3f}")
r1 = (att[3] == "-0.431**" and se[3] == "(0.210)" and pct[3] == "-35.0%" and abs(a5) / s5 > 1.96)
a19 = cells(find("R1", r"^ATT \| -0.431\*\* \| -0.438"))
a19se = cells(find("R1", r"^\| \(0.218\)"))
find("R1", r"640 regions")
print(f"  Table A19 CS-DR: ATT={a19[1]}, SE={a19se[1]} (vs Table 6 SE {se[3]})")
checks["R1"].update(finding_supported=bool(r1),
    locator="arxiv-2609.27732v1 | Table 6, row 'ATT' (and SE row), column '5 km'; abstract",
    note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, the text states significance at 5% and the abstract describes staggered DiD on individual-level hospital records; Table A19 prints SE (0.218) for the same CS-DR estimate, which does not change the 5% significance.")
issues.append({"issue": "arxiv-2609.27732v1: the same CS-DR 5 km ATT (-0.431**) has SE (0.210) in Table 6 but (0.218) in Table A19; the A19 note says 640 never-treated regions while Tables 6, A19 and Figure 6 note say 608.",
               "handling": "Treated Table 6 (main results table) as decisive for the cited SE; checked that significance at 5% holds under either SE (|t|=2.05 vs 1.98); kept R1 as supported but flagged."})
issues.append({"issue": "arxiv-2609.27732v1: abstract says fatal outcomes 'fall by 35%', while Table 7 (5 km: -0.354**) and Section 6.1 imply about 30% (exp(-0.354)-1 = -29.8%).",
               "handling": "Not part of the cited finding for R1; documented only."})

# R2
print("R2:")
find("R2", r"N = 23,549\) preregistered randomized controlled trial")
w = cells(find("R2", r"^Wage \| \$33,073"))
re_ = cells(find("R2", r"^Reemployment \| 75.2%"))
ui = cells(find("R2", r"^Weeks on UI \| 15.2"))
print(f"  Table 2 ATE column: wage='{w[4]}', reemp='{re_[4]}', UI='{ui[4]}'")
r2 = (w[4].startswith("$1,153") and "[166, 2,141]" in w[4] and re_[4].startswith("1.47 pp")
      and ui[4].startswith("-1.99 weeks"))
checks["R2"].update(finding_supported=bool(r2),
    locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI; abstract (N, preregistered RCT)",
    note="Abstract gives a preregistered RCT with N = 23,549 and Table 2's ATE column prints $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, matching every element of the citation.")

# R3
print("R3:")
find("R3", r"rise gradually from about 0.5% in the event year to roughly 3.4")
coef = find("R3", r"stacked coefficient is 0.005 at")
find("R3", r"prices do not jump immediately")
row = cells(find("R3", r"^Clean-onset threshold-2 \|"))
tau0 = float(re.search(r"coefficient is ([\d.]+) at \\tau=0", coef).group(1))
print(f"  event-year (tau=0) coefficient={tau0}; Table 3 preferred tau=4={row[4]}, tau=5={row[5]}")
# Cited: 3.4-3.7% jump in the event year, no further appreciation later. Source: 0.005 at tau=0, rising to 0.037/0.034 by tau=4/5.
r3 = tau0 >= 0.034 and float(row[4]) <= tau0
checks["R3"].update(finding_supported=bool(r3),
    locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficient 0.005 at tau=0); Table 3, row 'Clean-onset threshold-2', columns tau=4 (0.037) and tau=5 (0.034)",
    note="The extract reports a gradual rise from about 0.5% (0.005) in the event year to 3.4-3.7% only by years four and five and says prices do not jump immediately, contradicting the cited immediate jump; the entry's title and year (2024) also do not match the document (submitted 2026).")

# R4
print("R4:")
find("R4", r"Japan’s spot labor market|Japan's spot labor market")
e1 = find("R4", r"^Employment elasticity w.r.t MW")
i1 = src["R4"].index(e1); s1 = src["R4"][i1 + 2]; print(f"  [R4 L{i1+3}] {s1}")
e2 = find("R4", r"^Emp. elasticity w.r.t affected wage")
i2 = src["R4"].index(e2); s2 = src["R4"][i2 + 2]; print(f"  [R4 L{i2+3}] {s2}")
aw = float(cells(find("R4", r"^Affected wages \|"))[1]); ae = float(cells(find("R4", r"^Affected employment \|"))[1])
print(f"  arithmetic check: affected employment / affected wages = {ae/aw:.3f}")
r4 = (cells(e1)[1] == "-0.388" and "(0.132)" in s1 and cells(e2)[1] == "-0.732" and "(0.091)" in s2)
checks["R4"].update(finding_supported=bool(r4),
    locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (Estimate (SE) column); abstract (Japan's spot labor market)",
    note="Table 2 prints -0.388 (0.132) and -0.732 (0.091), the latter equal to -0.268/0.366, and the abstract places the study in Japan's spot labor market; 2025 equals the v1 submission year.")

# R5
print("R5:")
tr = cells(find("R5", r"^Treatment \| -0.0911"))
trse = cells(find("R5", r"^\| \(0.0113\) \| \(0.0113\)"))
pc = cells(find("R5", r"^% Change \|"))
mn = cells(find("R5", r"^Mean Dep. Var. \|"))
find("R5", r"most preferred specification \(Column \(4\)\)")
print(f"  col (4): beta={tr[4]}, SE={trse[4]}, %change={pc[4]}, mean={mn[4]}; beta/mean={float(tr[4])/float(mn[4]):.4f}")
r5 = tr[4] == "-0.0962" and trse[4] == "(0.0129)" and pc[4] == "-0.3685"
checks["R5"].update(finding_supported=bool(r5),
    locator="arxiv-2510.03658v1 | Table 3, row 'Treatment' (and SE row), '% Change' and 'Mean Dep. Var.', column (4); Section 4.1 (column 4 is preferred)",
    note="Table 3 column (4), named the preferred specification in Section 4.1, prints -0.0962 (0.0129) and % change -0.3685 (= -0.0962/0.261), matching the citation.")
issues.append({"issue": "arxiv-2510.03658v1: Section 4.1 refers to a Table 2 estimate of -0.0855 with dependent-variable mean 0.455 (18.79%), while the abstract reports the same 0.0855 headline; Table 3 (mean 0.261) is a different outcome/specification. Footnote markers are rendered inline ('percent^9^9 9').",
               "handling": "R5 cites the Table 3 column (4) result, which was checked directly; the Table 2 figure is not in the extract and was not used."})
issues.append({"issue": "arxiv-2603.03260v2: Table 3 header row garbled in the HTML rendering (editor's note supplies column order); the event-year coefficient tau=0 is only in the text, not a table; tau=3 coefficient is not reported.",
               "handling": "Used the editor's column order; relied on Section 3.2 text for tau=0 to tau=2."})

unver = [k for k, v in checks.items() if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]
order = ["title_matches", "year_matches", "finding_supported", "locator", "note"]
checks = {k: {f: v[f] for f in order} for k, v in checks.items()}

def wc(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")
nrows = {"README.md": wc(os.path.join(D, "README.md")), "reference_list.json": len(refs)}
for r in refs:
    nrows[r["source_file"]] = wc(os.path.join(D, r["source_file"]))

res = {"task_id": "F4-conf-04", "reference_checks": checks, "unverifiable_references": unver,
       "n_verified": len(refs) - len(unver), "feasibility": "complete", "feasibility_reason": None,
       "data_issues": issues, "n_input_rows": nrows, "n_analysed": len(refs), "random_seed": None}
json.dump(res, open("out/results.json", "w"), indent=2, ensure_ascii=False)
print("\nunverifiable:", unver, "n_verified:", res["n_verified"])