"""F4-conf-04: verify five references against provided arXiv extracts.
Reads only data/; writes out/results.json deterministically. No randomness."""
import json, re, hashlib, unicodedata, os

D = "data"
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(ROOT)

def wc_l(p):
    with open(p, "rb") as f:
        return f.read().count(b"\n")

def sha(p):
    return hashlib.sha256(open(p, "rb").read()).hexdigest()

refs = json.load(open(f"{D}/reference_list.json", encoding="utf-8"))
n_input_rows = {"README.md": wc_l(f"{D}/README.md"), "reference_list.json": len(refs)}
src = {}
for r in refs:
    p = f"{D}/{r['source_file']}"
    name = os.path.basename(p)
    n_input_rows[name] = wc_l(p)
    src[r["id"]] = open(p, encoding="utf-8").read().splitlines()
    print(f"[{r['id']}] {name} sha256={sha(p)} lines={n_input_rows[name]}")

def header(lines, key):
    for l in lines:
        if l.startswith(key + ":"):
            return l[len(key) + 1:].strip()
    raise KeyError(key)

def norm_title(t):
    t = unicodedata.normalize("NFKD", t)
    t = "".join(c for c in t if not unicodedata.combining(c)).lower()
    t = t.replace("&", " and ")
    t = re.sub(r"[^a-z0-9 ]", " ", t)
    return " ".join(t.split())

def entry_parts(entry):
    m = re.match(r"^(.*?)\((\d{4})\)\.\s*(.*?)\.\s*arXiv:(\d{4}\.\d{5})\.?$", entry)
    return {"year": int(m.group(2)), "title": m.group(3), "arxiv": m.group(4)}

def find(lines, pattern):
    """Return (line_no, line) of first line matching regex; print it as evidence."""
    for i, l in enumerate(lines, 1):
        if re.search(pattern, l):
            return i, l
    raise ValueError(pattern)

def show(rid, lines, pattern, nxt=False):
    i, l = find(lines, pattern)
    print(f"  [{rid} L{i}] {l.strip()[:220]}")
    if nxt:  # SE line follows coefficient row (skip blank line)
        j = i
        while not lines[j].strip():
            j += 1
        print(f"  [{rid} L{j+1}] {lines[j].strip()}")
        return l, lines[j]
    return l

def cells(line):
    return [c.strip() for c in line.split("|")]

checks = {}
header_checks = {}
for r in refs:
    rid, L = r["id"], src[r["id"]]
    e = entry_parts(r["entry"])
    title = header(L, "Title")
    rec = header(L, "Bibliographic record")
    body_title = next(l[2:].strip() for l in L if l.startswith("# "))
    print(f"[{rid}] entry: {r['entry']}")
    print(f"  Title: {title}\n  Body title: {body_title}\n  Bibliographic record: {rec}")
    years = sorted({int(y) for y in re.findall(r"\b\d{1,2} [A-Z][a-z]{2} (\d{4})", rec)})
    first_year = years[0]
    m_ver = re.search(r"arXiv:\d{4}\.\d{5}v(\d+)", rec)
    ver = m_ver.group(1)
    # year of the reproduced version: if multiple versions, the one marked "(this extract)", else the only date
    m_this = re.search(r"submitted (\d{1,2} [A-Z][a-z]{2} (\d{4})) \(this extract\)", rec) or \
             re.search(r"\(this extract\) submitted (\d{1,2} [A-Z][a-z]{2} (\d{4}))", rec)
    this_year = int(m_this.group(2)) if m_this else (years[0] if len(years) == 1 else None)
    allowed = {first_year, this_year} - {None}
    tm = norm_title(e["title"]) == norm_title(title) == norm_title(body_title)
    ym = e["year"] in allowed
    idm = e["arxiv"] in rec
    print(f"  -> entry year {e['year']}; allowed {sorted(allowed)} (v{ver}); title_match={tm}; year_match={ym}; arXiv id match={idm}")
    header_checks[rid] = {"title_matches": tm, "year_matches": ym, "allowed_years": sorted(allowed)}

# ---------------- finding checks (evidence printed) ----------------
f = {}
# R1
L = src["R1"]; print("[R1] finding evidence")
att, se = show("R1", L, r"^ATT \| -0\.399", nxt=True)
pct = show("R1", L, r"^% change \| -32\.9%")
hdr = show("R1", L, r"^\| 1 km \| 3 km \| 5 km")
show("R1", L, r"statistically significant at the 5% level")
show("R1", L, r"staggered introduction of DCRs")
a5, s5, p5 = cells(att)[3], cells(se)[3], cells(pct)[3]
ok1 = (a5 == "-0.431**" and s5 == "(0.210)" and p5 == "-35.0%")
show("R1", L, r"^ATT \| -0\.431\*\* \| -0\.438", nxt=True)  # Table A19 (inconsistent SE)
f["R1"] = dict(ok=ok1, locator="arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (abstract; Section 6 text for 5% significance and staggered DiD on hospital records)",
               note="Table 6 (5 km) prints ATT -0.431** with SE (0.210) and -35.0%, the text states 5% significance and the abstract describes a staggered DiD on individual hospital records, so every element is supported.")
# R2
L = src["R2"]; print("[R2] finding evidence")
show("R2", L, r"N = 23,549\) preregistered randomized")
w = cells(show("R2", L, r"^Wage \| \$33,073"))[4]
re_ = cells(show("R2", L, r"^Reemployment \| 75\.2%"))[4]
ui = cells(show("R2", L, r"^Weeks on UI \| 15\.2"))[4]
ok2 = (w.startswith("$1,153;") and "[166, 2,141]" in w and re_.startswith("1.47 pp") and ui.startswith("-1.99 weeks"))
f["R2"] = dict(ok=ok2, locator="arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI (N and design from abstract)",
               note="Table 2 ATE column gives $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives a preregistered RCT with N = 23,549.")
# R3
L = src["R3"]; print("[R3] finding evidence")
txt = show("R3", L, r"stacked coefficient is 0\.005 at")
row = cells(show("R3", L, r"^Clean-onset threshold-2 \|"))
show("R3", L, r"prices do not jump immediately")
tau0 = float(re.search(r"([0-9.]+) at \\tau=0", txt).group(1))
tau4, tau5 = float(row[4]), float(row[5])
# claim: 3.4-3.7% jump in entry year, no later appreciation. Evidence: 0.005 at tau=0, 0.034-0.037 at tau=4-5.
ok3 = (0.034 <= tau0 <= 0.037) and not (tau4 > tau0)
print(f"  -> tau0={tau0}, tau4={tau4}, tau5={tau5}; claim requires 0.034-0.037 at tau=0 and no later rise -> supported={ok3}")
f["R3"] = dict(ok=ok3, locator="arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' and Table 3, row 'Clean-onset threshold-2', columns tau=4 / tau=5",
               note="The extract reports a gradual rise from 0.005 log points at tau=0 to 0.037/0.034 at tau=4/5 ('prices do not jump immediately'), contradicting a 3.4-3.7% jump in the entry year; the entry's title and year (2024) also do not match the record.")
# R4
L = src["R4"]; print("[R4] finding evidence")
e1, s1 = show("R4", L, r"^Employment elasticity w\.r\.t MW", nxt=True)
e2, s2 = show("R4", L, r"^Emp\. elasticity w\.r\.t affected wage", nxt=True)
show("R4", L, r"^Affected employment", nxt=True)
show("R4", L, r"Japan’s spot labor market")
ok4 = (cells(e1)[1] == "-0.388" and cells(s1)[1] == "(0.132)" and cells(e2)[1] == "-0.732" and cells(s2)[1] == "(0.091)")
emp, wage = -0.268, 0.366
print(f"  -> arithmetic check: affected employment / affected wages = {emp/wage:.3f} (printed -0.732)")
f["R4"] = dict(ok=ok4, locator="arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",
               note="Table 2 prints -0.388 (0.132) and -0.732 (0.091) for Japan's spot labour market (abstract), matching the citation; the 2025 year equals the v1 submission year.")
# R5
L = src["R5"]; print("[R5] finding evidence")
t, s = show("R5", L, r"^Treatment \| -0\.0911", nxt=True)
pc = cells(show("R5", L, r"^% Change \|"))
mean = cells(show("R5", L, r"^Mean Dep\. Var\."))
show("R5", L, r"most preferred specification \(Column \(4\)\)")
ok5 = (cells(t)[4] == "-0.0962" and cells(s)[4] == "(0.0129)" and pc[4] == "-0.3685")
print(f"  -> arithmetic check: -0.0962/{mean[4]} = {-0.0962/float(mean[4]):.4f} (printed -0.3685 = 36.85%)")
f["R5"] = dict(ok=ok5, locator="arxiv-2510.03658v1 | Table 3, row Treatment / % Change, column (4) (Section 4.1 identifies column 4 as preferred)",
               note="Table 3 column (4), the stated preferred specification, prints -0.0962 (0.0129) and % change -0.3685 (36.85% of the 0.261 mean), matching the citation.")

reference_checks = {}
for r in refs:
    rid = r["id"]
    reference_checks[rid] = {"title_matches": header_checks[rid]["title_matches"],
                             "year_matches": header_checks[rid]["year_matches"],
                             "finding_supported": bool(f[rid]["ok"]),
                             "locator": f[rid]["locator"], "note": f[rid]["note"]}
unver = [k for k, v in reference_checks.items()
         if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]
baseline_unver = [k for k, v in header_checks.items() if not (v["title_matches"] and v["year_matches"])]

data_issues = [
 {"issue": "arxiv-2609.27732v1: Table A19 prints SE (0.218) for the CS-DR 5 km ATT -0.431, whereas Table 6 prints (0.210) for the same estimate.",
  "handling": "Table 6 (the main results table, which the citation's SE 0.210 matches) is used as the deciding locator; inconsistency reported, finding judged supported since 0.210 is printed in the extract and significance at 5% holds under either SE."},
 {"issue": "arxiv-2609.27732v1: Table A19 note states 640 never-treated comparison regions, while Table 6, Table A19 body and figure notes state 608.",
  "handling": "Not relevant to the cited finding; reported only."},
 {"issue": "arxiv-2609.27732v1: Section 6.1 gives -30% (fatal) and +47% (narcotics) at 5 km, while the abstract says fatal outcomes fall 35% and narcotics rise by a comparable amount.",
  "handling": "Not part of the cited finding; reported only."},
 {"issue": "arxiv-2505.04555v3: Table 2 prints the same SE (0.091) for 'Affected employment' and for the own-wage elasticity, although the latter is a ratio (-0.268/0.366 = -0.732) whose SE would generally differ.",
  "handling": "Cannot be resolved from the extract; the cited SE equals the printed value, so the finding is judged supported, with the caveat noted."},
 {"issue": "arxiv-2510.03658v1: abstract/Section 4.1 headline (0.0855, 18.79%, mean 0.455, from a Table 2 not reproduced) differs from Table 3 column (4) (-0.0962, 36.85%, mean 0.261); footnote markers are rendered as '^9^9 9'.",
  "handling": "Treated as different outcome definitions/tables, not an internal contradiction; Table 3 decides the citation, which explicitly refers to the preferred specification and 36.85%."},
 {"issue": "arxiv-2603.03260v2: Table 3 header row is garbled in the HTML rendering.",
  "handling": "Column order taken from the extract editor's note (Specification | Treated-role LSOAs | Stacked p | Sun p | tau=4 | tau=5); consistent with the Section 3.2 text (0.037, 0.034)."},
]

results = {
 "task_id": "F4-conf-04",
 "reference_checks": reference_checks,
 "unverifiable_references": unver,
 "n_verified": len(refs) - len(unver),
 "feasibility": "complete",
 "feasibility_reason": None,
 "data_issues": data_issues,
 "n_input_rows": n_input_rows,
 "n_analysed": len(refs),
 "random_seed": None,
 "baseline_unverifiable_title_year_only": baseline_unver,
}
os.makedirs("out", exist_ok=True)
with open("out/results.json", "w", encoding="utf-8") as fh:
    json.dump(results, fh, indent=2, ensure_ascii=False)
for k, v in reference_checks.items():
    print(k, v["title_matches"], v["year_matches"], v["finding_supported"])
print("unverifiable:", unver, "n_verified:", results["n_verified"])