#!/usr/bin/env python3
"""Verify every literature candidate against primary bibliographic records.

- Crossref (query.bibliographic) for articles/chapters/proceedings; raw JSON saved per key in
  docs/verification/<key>.crossref.json. A candidate is VERIFIED when the best hit's title matches
  (normalised similarity >= 0.80) and the year is within ±1; the Crossref metadata (authors, container,
  volume, pages, DOI, year) then replaces our own typing.
- arXiv API for preprints (id taken from the container field "arXiv:ID"); raw Atom saved.
- Items that fail are listed in docs/verification/unverified.csv and are NOT written to references.bib.

Outputs: paper/references.bib (verified only), docs/verification/verified.csv, docs/verification/unverified.csv
"""
from __future__ import annotations

import csv
import difflib
import json
import re
import sys
import time
import unicodedata
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
VER = ROOT / "docs" / "verification"
VER.mkdir(parents=True, exist_ok=True)
UA = "WP20-UQO-literature-verifier/1.0 (mailto:simon-pierre.boucher@uqo.ca)"


def norm(s: str) -> str:
    s = unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode()
    s = re.sub(r"[^a-z0-9 ]", " ", s.lower())
    return re.sub(r"\s+", " ", s).strip()


def sim(a: str, b: str) -> float:
    return difflib.SequenceMatcher(None, norm(a), norm(b)).ratio()


def get(url: str, tries: int = 4) -> bytes:
    for t in range(tries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=60) as r:
                return r.read()
        except Exception as e:  # noqa: BLE001
            if t == tries - 1:
                raise
            time.sleep(2 * (t + 1))
    return b""


def crossref(c: dict) -> dict | None:
    key = c["key"]
    out = VER / f"{key}.crossref.json"
    if c["doi_hint"]:
        try:
            data = json.loads(get(f"https://api.crossref.org/works/{urllib.parse.quote(c['doi_hint'])}"))
            out.write_text(json.dumps(data, ensure_ascii=False))
            item = data["message"]
            fam = (item.get("author") or [{}])[0].get("family", "")
            y = year_of(item)
            if sim(item.get("title", [""])[0], c["title"]) >= 0.6 or (
                    norm(fam) == norm(c["first_author"]) and y and abs(int(y) - int(c["year"])) <= 1):
                return item
        except Exception:  # noqa: BLE001
            pass
    q = urllib.parse.quote(f"{c['title']} {c['first_author']} {c['year']}")
    data = json.loads(get(f"https://api.crossref.org/works?query.bibliographic={q}&rows=6"))
    out.write_text(json.dumps(data, ensure_ascii=False))
    best, best_s = None, 0.0
    for item in data["message"]["items"]:
        t = (item.get("title") or [""])[0]
        s = sim(t, c["title"])
        if s > best_s:
            best, best_s = item, s
    if best is None or best_s < 0.80:
        return None
    y = year_of(best)
    if y and abs(int(y) - int(c["year"])) > 1:
        return None
    return best


def year_of(item: dict) -> int | None:
    for k in ("published-print", "published-online", "issued", "created"):
        dp = item.get(k, {}).get("date-parts")
        if dp and dp[0] and dp[0][0]:
            return int(dp[0][0])
    return None


def arxiv(c: dict) -> dict | None:
    m = re.search(r"arXiv:([\d.]+)", c["container"])
    if not m:
        return None
    aid = m.group(1)
    time.sleep(3.5)  # arXiv API rate limit
    raw = get(f"https://export.arxiv.org/api/query?id_list={aid}")
    (VER / f"{c['key']}.arxiv.xml").write_bytes(raw)
    ns = {"a": "http://www.w3.org/2005/Atom"}
    root = ET.fromstring(raw)
    e = root.find("a:entry", ns)
    if e is None:
        return None
    title = " ".join((e.findtext("a:title", default="", namespaces=ns) or "").split())
    if sim(title, c["title"]) < 0.8:
        return None
    authors = [a.findtext("a:name", default="", namespaces=ns) for a in e.findall("a:author", ns)]
    year = (e.findtext("a:published", default="", namespaces=ns) or "")[:4]
    return {"title": title, "authors": authors, "year": year, "id": aid,
            "url": f"https://arxiv.org/abs/{aid}", "doi": e.findtext("{http://arxiv.org/schemas/atom}doi")}


def bib_escape(s: str) -> str:
    return s.replace("&", r"\&").replace("%", r"\%").replace("_", r"\_")


def protect(title: str) -> str:
    # protect acronyms / capitalised words after the first
    return re.sub(r"\b([A-Z][A-Z0-9]+|[A-Z][a-z]*[A-Z][A-Za-z]*)\b", r"{\1}", title)


def crossref_bib(c: dict, it: dict) -> tuple[str, dict]:
    authors = " and ".join(f"{a.get('family','')}, {a.get('given','')}".strip(", ") for a in it.get("author", []) if a.get("family"))
    title = (it.get("title") or [c["title"]])[0]
    cont = (it.get("container-title") or [c["container"]])[0] if it.get("container-title") else c["container"]
    year = year_of(it) or c["year"]
    typ = it.get("type", "")
    doi = it.get("DOI", "")
    vol = it.get("volume", ""); issue = it.get("issue", ""); pages = it.get("page", "")
    publisher = it.get("publisher", "")
    if typ == "journal-article":
        kind = "article"
    elif typ in ("proceedings-article",):
        kind = "inproceedings"
    elif typ in ("book-chapter",):
        kind = "incollection"
    elif typ in ("report", "posted-content", "report-component"):
        kind = "techreport"
    else:
        kind = c["kind"] if c["kind"] in ("article", "inproceedings", "incollection", "techreport") else "misc"
    f = {"author": authors, "title": protect(bib_escape(title)), "year": str(year), "doi": doi}
    if kind == "article":
        f["journal"] = bib_escape(cont)
        if vol: f["volume"] = vol
        if issue: f["number"] = issue
        if pages: f["pages"] = pages.replace("-", "--")
    elif kind == "inproceedings":
        f["booktitle"] = bib_escape(cont)
        if pages: f["pages"] = pages.replace("-", "--")
        if publisher: f["publisher"] = bib_escape(publisher)
    elif kind == "incollection":
        f["booktitle"] = bib_escape(cont)
        if pages: f["pages"] = pages.replace("-", "--")
        if publisher: f["publisher"] = bib_escape(publisher)
    else:
        f["institution"] = bib_escape(publisher or cont)
        if "NBER" in c["container"]:
            f["type"] = "NBER Working Paper"
            f["number"] = re.sub(r"\D", "", c["container"])
    entry = "@%s{%s,\n%s\n}\n" % (kind, c["key"], ",\n".join(f"  {k} = {{{v}}}" for k, v in f.items() if v))
    meta = {"key": c["key"], "authors": authors, "year": year, "title": title, "container": cont, "doi": doi,
            "verified_via": "Crossref works API (DOI lookup or query.bibliographic), JSON in docs/verification/"}
    return entry, meta


def arxiv_bib(c: dict, a: dict) -> tuple[str, dict]:
    authors = " and ".join(a["authors"])
    f = {"author": authors, "title": protect(bib_escape(a["title"])), "year": a["year"],
         "eprint": a["id"], "archivePrefix": "arXiv", "url": a["url"], "note": f"arXiv preprint {a['id']}"}
    if a.get("doi"):
        f["doi"] = a["doi"]
    entry = "@misc{%s,\n%s\n}\n" % (c["key"], ",\n".join(f"  {k} = {{{v}}}" for k, v in f.items() if v))
    meta = {"key": c["key"], "authors": authors, "year": a["year"], "title": a["title"], "container": f"arXiv:{a['id']}",
            "doi": a.get("doi") or "", "verified_via": "arXiv API (export.arxiv.org), Atom in docs/verification/"}
    return entry, meta


def main() -> None:
    cands = list(csv.DictReader(open(ROOT / "docs" / "literature_candidates.csv", encoding="utf-8")))
    verified, unverified, bib = [], [], []
    for c in cands:
        try:
            if c["kind"] == "preprint":
                a = arxiv(c)
                if a:
                    e, m = arxiv_bib(c, a); bib.append(e); m["topic"] = c["topic"]; verified.append(m)
                else:
                    unverified.append({**c, "reason": "arXiv title mismatch or not found"})
            else:
                it = crossref(c)
                if it:
                    e, m = crossref_bib(c, it); bib.append(e); m["topic"] = c["topic"]; verified.append(m)
                else:
                    unverified.append({**c, "reason": "no Crossref record with title similarity >= 0.80 and year ±1"})
        except Exception as ex:  # noqa: BLE001
            unverified.append({**c, "reason": f"error: {ex}"})
        print(f"{c['key']:34s} {'OK' if verified and verified[-1]['key']==c['key'] else '--'}", flush=True)
        time.sleep(0.6)
    (ROOT / "paper" / "references.bib").write_text("\n".join(bib), encoding="utf-8")
    with open(VER / "verified.csv", "w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=list(verified[0].keys())); w.writeheader(); w.writerows(verified)
    with open(VER / "unverified.csv", "w", newline="", encoding="utf-8") as fh:
        if unverified:
            w = csv.DictWriter(fh, fieldnames=list(unverified[0].keys())); w.writeheader(); w.writerows(unverified)
    print(f"\nverified {len(verified)} / {len(cands)}; unverified {len(unverified)}")


if __name__ == "__main__":
    sys.exit(main())