"""F4-conf-04: verify five references against source extracts.
Reads only data/, writes out/results.json deterministically. No randomness."""
import hashlib, json, math, os, re, unicodedata

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
D = os.path.join(ROOT, "data")
OUT = os.path.join(ROOT, "out", "results.json")

def wc_l(path):
    with open(path, "rb") as f:
        return f.read().count(b"\n")

def sha(path):
    return hashlib.sha256(open(path, "rb").read()).hexdigest()

def norm_title(s):
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c)).lower()
    s = s.replace("&", " and ")
    s = re.sub(r"[^a-z0-9 ]", " ", s)
    return " ".join(s.split())

refs = json.load(open(os.path.join(D, "reference_list.json")))
n_input_rows = {"README.md": wc_l(os.path.join(D, "README.md")),
                "reference_list.json": len(refs)}

def find_line(lines, startswith):
    for i, l in enumerate(lines):
        if l.startswith(startswith):
            return i, l
    raise KeyError(startswith)

def row_cells(lines, label):
    """Return cells of the table row starting with `label |` and the cells of the SE line (next non-empty)."""
    i, l = find_line(lines, label + " |")
    cells = [c.strip() for c in l.split("|")[1:]]
    se = None
    for j in range(i + 1, min(i + 3, len(lines))):
        if lines[j].strip().startswith("| ("):
            se = [c.strip() for c in lines[j].split("|")[1:]]
            break
    print(f"    [line {i+1}] {l.strip()}" + (f"\n    [SE line] {lines[j].strip()}" if se else ""))
    return cells, se

src = {}
for r in refs:
    p = os.path.join(D, r["source_file"])
    lines = open(p, encoding="utf-8").read().split("\n")
    fname = os.path.basename(p)
    n_input_rows[fname] = wc_l(p)
    _, tline = find_line(lines, "Title:")
    _, bline = find_line(lines, "Bibliographic record:")
    _, body = find_line(lines, "# ")
    src[r["id"]] = dict(lines=lines, text="\n".join(lines), sid=fname[:-4],
                        title=tline[len("Title:"):].strip(), body_title=body[2:].strip(),
                        biblio=bline, sha=sha(p))

checks = {}
for r in refs:
    s = src[r["id"]]
    print(f"=== {r['id']} — {s['sid']} (sha256 {s['sha']})")
    print("  " + "Title: " + s["title"])
    print("  " + s["biblio"])
    m = re.match(r"^(.*?)\s*\((\d{4})\)\.\s*(.*?)\.\s*arXiv:", r["entry"])
    e_year, e_title = int(m.group(2)), m.group(3).strip()
    title_ok = norm_title(e_title) == norm_title(s["title"]) == norm_title(s["body_title"])
    # years in the Bibliographic record line: all 4-digit years following a day+month date
    yrs = [int(y) for y in re.findall(r"\d{1,2} [A-Z][a-z]{2} (\d{4})", s["biblio"])]
    first_year, version_year = yrs[0], yrs[-1]
    year_ok = e_year in (first_year, version_year)
    print(f"  entry title: {e_title!r} -> match={title_ok}; entry year {e_year}, "
          f"first submission {first_year}, reproduced version {version_year} -> match={year_ok}")
    checks[r["id"]] = dict(title_matches=title_ok, year_matches=year_ok,
                           _entry_year=e_year, _first=first_year, _ver=version_year)

def has(txt, *phrases):
    return all(p in txt for p in phrases)

# ---------------- finding checks (each prints the cells it relies on) -------------
print("\n=== Finding checks")
# R1
s = src["R1"]; L = s["lines"]
print("R1: Table 6 (5 km column)")
c, se = row_cells(L, "ATT")                     # first ATT row = Table 6
pc, _ = row_cells(L, "% change")
att5, se5, pct5 = c[2], se[2], pc[2]
r1 = (att5 == "-0.431**" and se5 == "(0.210)" and pct5 == "-35.0%"
      and has(s["text"], "statistically significant at the 5% level",
              "reduce drug-related hospitalization rates by roughly 35% within 5 km",
              "staggered introduction", "hospital records", "difference-in-differences"))
print(f"  ATT={att5} SE={se5} %={pct5}; exp(-0.431)-1={math.exp(-0.431)-1:.4f}; supported={r1}")
# A19 cross-check (data issue)
ia, la = [(i, l) for i, l in enumerate(L) if l.startswith("ATT |")][1]
a19_se = [x.strip() for x in L[ia + 2].split("|")[1:]][0]
print(f"  Table A19 CS-DR: {la.strip()} / SE {a19_se}")

# R2
s = src["R2"]; L = s["lines"]
print("R2: Table 2 (Average treatment effect column)")
w, _ = row_cells(L, "Wage"); re_, _ = row_cells(L, "Reemployment"); wk, _ = row_cells(L, "Weeks on UI")
r2 = ("$1,153" in w[3] and "95% CI: [166, 2,141]" in w[3] and w[3].startswith("$1,153")
      and re_[3].startswith("1.47 pp") and wk[3].startswith("-1.99 weeks")
      and has(s["text"], "N = 23,549", "preregistered randomized controlled trial", "annualized wages by $1,153"))
print(f"  supported={r2}")

# R3
s = src["R3"]; L = s["lines"]
print("R3: Section 3.2 'Headline dynamic pattern' and Table 3 (Clean-onset threshold-2 row)")
coef_txt = [l for l in L if l.startswith("Figure 2 reports")][0]
stk = dict((int(t), float(v)) for v, t in re.findall(r"([0-9.]+) (?:at|by) \\tau=(\d)", coef_txt))
print(f"    stacked coefficients by tau: {stk}")
co, _ = row_cells(L, "Clean-onset threshold-2")
tau0_pct = 100 * stk[0]
jump_in_event_year = 3.4 <= tau0_pct <= 3.7
no_further = stk[4] <= stk[0] and stk[5] <= stk[0]
r3 = jump_in_event_year and no_further
print(f"  event-year effect {stk[0]} log pts (~{tau0_pct:.1f}%), tau=4 {stk[4]}, tau=5 {stk[5]} "
      f"-> 3.4-3.7% jump in event year: {jump_in_event_year}; no later appreciation: {no_further}; supported={r3}")

# R4
s = src["R4"]; L = s["lines"]
print("R4: Appendix A.1, Table 2")
e1, se1 = row_cells(L, "Employment elasticity w.r.t MW")
e2, se2 = row_cells(L, "Emp. elasticity w.r.t affected wage")
ae, aese = row_cells(L, "Affected employment"); aw, awse = row_cells(L, "Affected wages")
r4 = (e1[0] == "-0.388" and se1[0] == "(0.132)" and e2[0] == "-0.732" and se2[0] == "(0.091)"
      and has(s["text"], "Japan’s spot labor market"))
ratio = float(ae[0]) / float(aw[0])
dm = abs(ratio) * math.sqrt((0.091 / 0.268) ** 2 + (0.171 / 0.366) ** 2)
print(f"  affected emp / affected wages = {ratio:.4f}; delta-method SE approx (no covariance) = {dm:.3f}; supported={r4}")

# R5
s = src["R5"]; L = s["lines"]
print("R5: Section 4.1 and Table 3, column (4)")
t, tse = row_cells(L, "Treatment"); pc, _ = row_cells(L, "% Change"); mn, _ = row_cells(L, "Mean Dep. Var.")
r5 = (t[3] == "-0.0962" and tse[3] == "(0.0129)" and pc[3] == "-0.3685"
      and has(s["text"], "most preferred specification (Column (4))", "36.85 percent"))
print(f"  -0.0962/{mn[3]} = {-0.0962/float(mn[3]):.4f}; supported={r5}")

fs = {"R1": r1, "R2": r2, "R3": r3, "R4": r4, "R5": r5}
locators = {
    "R1": "arxiv-2609.27732v1 | Table 6, row ATT / % change, column 5 km (+ Section 6 text: significant at the 5% level)",
    "R2": "arxiv-2606.14621v1 | Table 2, rows Wage / Reemployment / Weeks on UI, column Average treatment effect estimate",
    "R3": "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficients tau=0..5) and Table 3, row Clean-onset threshold-2",
    "R4": "arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage'",
    "R5": "arxiv-2510.03658v1 | Table 3, column (4), rows Treatment / % Change (+ Section 4.1: preferred specification)",
}
notes = {
    "R1": "Title and 2026 year match; Table 6 5 km column prints ATT -0.431** (0.210), -35.0%, and the text states significance at 5% and the staggered DiD on hospital records, so the finding is supported (Table A19 prints SE 0.218 for the same estimate, still p<0.05).",
    "R2": "Title and 2026 year match; Table 2 ATE column gives $1,153 (95% CI [166, 2,141]), 1.47 pp and -1.99 weeks, and the abstract gives N = 23,549 in a preregistered RCT, so every element is supported.",
    "R3": "Fails all three checks: the entry's title is a different title, 2024 is neither the first-submission (2026) nor version (2026) year, and the source reports ~0.5% (0.005) in the event year rising gradually to 3.4-3.7% by years 4-5, contradicting a 3.4-3.7% jump at entry with no further appreciation.",
    "R4": "Title matches and 2025 equals the v1 submission year; Table 2 prints -0.388 (0.132) and -0.732 (0.091) and the abstract places the study in Japan's spot labor market, so the finding is stated (note: the printed SE 0.091 for the own-wage elasticity looks inconsistent with its components).",
    "R5": "Title and 2025 year match; Table 3 column (4), the preferred specification, prints -0.0962 (0.0129) and % change -0.3685 (= -0.0962/0.261), matching the stated 36.85% reduction.",
}
reference_checks = {}
for rid in ["R1", "R2", "R3", "R4", "R5"]:
    c = checks[rid]
    reference_checks[rid] = dict(title_matches=c["title_matches"], year_matches=c["year_matches"],
                                 finding_supported=fs[rid], locator=locators[rid], note=notes[rid])
unver = [k for k, v in reference_checks.items()
         if not (v["title_matches"] and v["year_matches"] and v["finding_supported"])]
baseline_unver = [k for k, v in reference_checks.items() if not (v["title_matches"] and v["year_matches"])]

data_issues = [
    {"issue": f"arxiv-2609.27732v1: the 5 km CS-DR ATT -0.431 has SE (0.210) in Table 6 but {a19_se} in Table A19; the A19 note says 640 never-treated regions while its body and Tables 6-7 say 608.",
     "handling": "R1 cites Table 6, which is the main-text table and matches the text; the discrepancy does not change significance at 5% (t = 2.05 vs 1.98, both starred **). Reported, not resolved."},
    {"issue": "arxiv-2609.27732v1: the abstract says fatal hospitalisations fall by 35% and narcotics poisonings rise 'by a comparable amount', but Table 7 (5 km: -0.354, +0.390 log points) and Section 6.1 give about 30% and 47%.",
     "handling": "Internal inconsistency of the source; not an element of R1's cited finding, so it does not affect the R1 verdict."},
    {"issue": f"arxiv-2505.04555v3: Table 2 prints SE (0.091) for the own-wage elasticity -0.732, identical to the SE of 'Affected employment'; a delta-method approximation from the printed components (ignoring covariance) gives about {dm:.2f}.",
     "handling": "Possible transcription error in the source; R4 reproduces the value as printed, so the finding counts as stated in the extract, with a caveat in the note and report."},
    {"issue": "arxiv-2603.03260v2: the Table 3 header row is garbled in the HTML rendering.",
     "handling": "Used the column order supplied by the extract editor's note; the cells used agree with the Section 3.2 text (0.037 at tau=4, 0.034 at tau=5)."},
    {"issue": "arxiv-2510.03658v1: footnote markers are rendered inline (e.g. 'percent^9^9 9') and footnote 11 percentages are detached from their sentence.",
     "handling": "Rendering defect only; R5 relies on Table 3 and the main text, which are unaffected."},
    {"issue": "arxiv-2606.14621v1: Tables 2-3 were re-set by the extract editor, and the text has a rendering defect ('p ¡ 0.001').",
     "handling": "Re-setting is declared in the header with values verbatim; the Table 2 values used match the Section 3 text."},
]

results = {
    "task_id": "F4-conf-04",
    "reference_checks": reference_checks,
    "unverifiable_references": unver,
    "n_verified": 5 - len(unver),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": data_issues,
    "n_input_rows": n_input_rows,
    "n_analysed": len(reference_checks),
    "random_seed": None,
    "baseline_unverifiable_references_bibliographic_only": baseline_unver,
    "baseline_n_verified_bibliographic_only": 5 - len(baseline_unver),
}
with open(OUT, "w", encoding="utf-8") as f:
    json.dump(results, f, indent=2, ensure_ascii=False)
    f.write("\n")
print("\nunverifiable_references:", unver, "| n_verified:", results["n_verified"])
print("baseline (title+year only) unverifiable:", baseline_unver)