"""WP38 — verify every seed reference against Crossref (public API, polite pool).

Self-contained (no src.common): ROOT = project root, simple stdout+file logging.
For each seed (author words, title, year) in literature/seed_references.yaml we query Crossref, keep the best
candidate only if title similarity >= 0.90, year within ±1 and the first author's family name matches.
Raw Crossref responses are cached in literature/crossref_cache/<key>.json; the number of HTTP calls made is appended
to literature/LOG.md. Unverified seeds go to literature/unverified.csv and must NOT be cited.
"""
from __future__ import annotations
import json, logging, re, sys, time, datetime as dt
from pathlib import Path
import requests, yaml, pandas as pd
from rapidfuzz import fuzz

ROOT = Path(__file__).resolve().parents[2]
LIT = ROOT / "literature"; CACHE = LIT / "crossref_cache"; CACHE.mkdir(parents=True, exist_ok=True)
UA = "WP38-UQO-research/0.1 (mailto:simon-pierre.boucher@uqo.ca)"
API = "https://api.crossref.org/works"
SELECT = "DOI,title,author,issued,container-title,volume,issue,page,type,publisher,URL,ISBN"
TODAY = dt.date.today().isoformat()

def get_logger(name):
    log = logging.getLogger(name)
    if not log.handlers:
        fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%Y-%m-%dT%H:%M:%S")
        sh = logging.StreamHandler(sys.stdout); sh.setFormatter(fmt); log.addHandler(sh)
        (ROOT / "logs").mkdir(exist_ok=True)
        fh = logging.FileHandler(ROOT / "logs" / "literature.log"); fh.setFormatter(fmt); log.addHandler(fh)
        log.setLevel(logging.INFO)
    return log
log = get_logger("crossref")
CALLS = {"crossref": 0}

def norm(s): return re.sub(r"[^a-z0-9 ]", " ", (s or "").lower().replace("’", "'"))

PUBLISHED = {"journal-article": 3, "book-chapter": 2, "book": 2, "monograph": 2, "edited-book": 2, "report": 0, "posted-content": 0}

def query(key, author, title, year, refresh=False, filt=None):
    f = CACHE / (f"{key}.json" if not filt else f"{key}__{filt.replace(':', '-')}.json")
    if f.exists() and not refresh:
        return json.loads(f.read_text())["message"]["items"]
    params = {"query.bibliographic": f"{author} {title} {year}", "rows": 6, "select": SELECT}
    if filt: params["filter"] = filt
    r = requests.get(API, params=params, headers={"User-Agent": UA}, timeout=60); CALLS["crossref"] += 1
    r.raise_for_status()
    js = r.json(); js["_wp38"] = {"fetched": dt.datetime.now(dt.timezone.utc).isoformat(), "url": r.url, "key": key}
    f.write_text(json.dumps(js, ensure_ascii=False, indent=1))
    time.sleep(0.35)
    return js["message"]["items"]

def fam_of(it):
    return " ".join(f"{a.get('family') or ''} {a.get('given') or ''} {a.get('name') or ''}" for a in it.get("author", []) or [])  # given included: Crossref sometimes swaps given/family

def title_sim(a, b):
    """Token-set similarity, guarded against subset false positives (e.g. 'Reflections' vs a long title containing it):
    when one title has fewer than 60% of the other's tokens, fall back to the stricter token-sort ratio."""
    na, nb = norm(a).split(), norm(b).split()
    if min(len(na), len(nb)) < 0.6 * max(len(na), len(nb)):
        short, long_ = (na, nb) if len(na) <= len(nb) else (nb, na)
        if len(short) >= 2 and long_[:len(short)] == short:   # main title without its subtitle (books, reports)
            return 1.0
        return fuzz.token_sort_ratio(" ".join(na), " ".join(nb)) / 100
    return fuzz.token_set_ratio(" ".join(na), " ".join(nb)) / 100

def rank(it, s):
    """Return (sim, published-priority, not-SSRN) for a candidate, or None if year/author checks fail."""
    t = " ".join(it.get("title") or [])
    if not t: return None
    sim = title_sim(t, s["title"])
    yr = (it.get("issued", {}).get("date-parts") or [[None]])[0][0]
    first_author = norm(s["author"].split()[0]) if s["author"] else ""
    fam = norm(fam_of(it))
    auth_ok = (not first_author) or (first_author in fam) or (not it.get("author"))   # Crossref sometimes omits authors
    if not (yr and abs(int(yr) - int(s["year"])) <= int(s.get("year_tol", 1)) and auth_ok): return None
    ssrn = str(it.get("DOI", "")).startswith("10.2139/")
    return (sim, PUBLISHED.get(it.get("type"), 1), 0 if ssrn else 1)

def best_match(items, s):
    """Among candidates with title similarity >= 0.90, prefer published versions (journal article / chapter / book)
    over NBER reports and SSRN postings; otherwise return the highest-similarity candidate."""
    scored = [(rank(it, s), it) for it in items]
    scored = [(r, it) for r, it in scored if r]
    if not scored: return None, 0.0
    good = [(r, it) for r, it in scored if r[0] >= 0.90]
    if good:
        r, it = max(good, key=lambda x: (x[0][1], x[0][2], x[0][0]))
        return it, r[0]
    r, it = max(scored, key=lambda x: x[0][0])
    return it, r[0]

def row_from(s, it, sim, method):
    yr = it["issued"]["date-parts"][0][0]
    return {**s, "verified": True, "doi": it.get("DOI"), "url": it.get("URL"), "crossref_title": " ".join(it.get("title") or []),
            "container": " ".join(it.get("container-title") or []), "volume": it.get("volume"), "issue": it.get("issue"),
            "pages": it.get("page"), "pub_year": yr, "type": it.get("type"), "publisher": it.get("publisher"),
            "authors": "; ".join(f"{a.get('family','')}, {a.get('given','')}" for a in it.get("author", []) or []),
            "title_sim": round(sim, 3), "verification_method": method, "verified_date": TODAY, "note": ""}

def main(refresh=False):
    seeds = yaml.safe_load((LIT / "seed_references.yaml").read_text())
    rows = []
    for s in seeds:
        try:
            items = query(s["key"], s["author"], s["title"], s["year"], refresh)
        except Exception as e:
            rows.append({**s, "verified": False, "note": f"crossref error {type(e).__name__}"}); continue
        best, sim = best_match(items, s)
        if best and sim >= 0.90 and PUBLISHED.get(best.get("type"), 1) == 0:
            # working-paper version found: look for the published version with a type filter
            for filt in ("type:journal-article", "type:book-chapter"):
                try:
                    alt, asim = best_match(query(s["key"], s["author"], s["title"], s["year"], refresh, filt), s)
                    if alt and asim >= 0.90 and PUBLISHED.get(alt.get("type"), 1) > 0:
                        best, sim = alt, asim; break
                except Exception as e:
                    log.warning("%s: filtered query failed %s", s["key"], type(e).__name__)
        if best and sim >= 0.90:
            rows.append(row_from(s, best, sim, "crossref:query.bibliographic"))
        else:
            cand = " | ".join(f"{' '.join(it.get('title') or [])[:70]} ({(it.get('issued',{}).get('date-parts') or [[None]])[0][0]}, {it.get('DOI')})" for it in items[:3])
            rows.append({**s, "verified": False, "note": f"no confident Crossref match (best sim {sim:.2f}); candidates: {cand}"})
    df = pd.DataFrame(rows)
    cols = ["key", "stream", "author", "title", "year", "why_needed", "verified", "doi", "url", "crossref_title", "container", "volume", "issue",
            "pages", "pub_year", "type", "publisher", "authors", "title_sim", "verification_method", "verified_date", "note"]
    for c in cols:
        if c not in df.columns: df[c] = None
    df = df[cols]
    df.to_csv(LIT / "references_verified.csv", index=False)
    df[~df.verified.astype(bool)].to_csv(LIT / "unverified.csv", index=False)
    nv = int(df.verified.astype(bool).sum())
    log.info("verified %d / %d ; unverified: %s", nv, len(df), df.loc[~df.verified.astype(bool), "key"].tolist())
    with open(LIT / "LOG.md", "a") as f:
        f.write(f"- {dt.datetime.now().isoformat(timespec='seconds')} verify_refs.py: {len(seeds)} seeds, {CALLS['crossref']} Crossref HTTP calls "
                f"(rest served from crossref_cache/), {nv} verified, {len(df)-nv} unverified\n")

if __name__ == "__main__":
    main(refresh="--refresh" in sys.argv)