"""WP38 — resolve references that Crossref's bibliographic search cannot verify.

literature/manual_references.yaml holds (a) entries with an explicit DOI, resolved through Crossref /works/{doi}
(raw JSON cached in literature/crossref_cache/<key>__doi.json), and (b) DOI-less items (books, reports, official
web pages) verified by fetching the URL: we record HTTP status, final URL, content type, page <title>, whether the
`must_contain` string is present and, for government pages, the dateModified tag.  Raw fetch metadata is cached in
literature/crossref_cache/<key>__url.json.  Rows are merged into literature/references_verified.csv (replacing any
seed row with the same key); unverified.csv is refreshed; call counts are appended to literature/LOG.md.
"""
from __future__ import annotations
import datetime as dt, html, json, re, sys, time
from pathlib import Path
import requests, yaml, pandas as pd

ROOT = Path(__file__).resolve().parents[2]
LIT = ROOT / "literature"; CACHE = LIT / "crossref_cache"; CACHE.mkdir(parents=True, exist_ok=True)
UA = {"User-Agent": "Mozilla/5.0 (Macintosh) WP38-UQO-research/0.1 (mailto:simon-pierre.boucher@uqo.ca)"}
TODAY = dt.date.today().isoformat()
CALLS = {"crossref": 0, "web": 0}

def log(msg): print(f"{dt.datetime.now().isoformat(timespec='seconds')} INFO resolve: {msg}")

def by_doi(key, doi):
    f = CACHE / f"{key}__doi.json"
    if f.exists(): return json.loads(f.read_text())["message"]
    r = requests.get(f"https://api.crossref.org/works/{doi}", headers={"User-Agent": UA["User-Agent"].split(') ')[-1]}, timeout=60)
    CALLS["crossref"] += 1; r.raise_for_status(); js = r.json(); js["_wp38"] = {"fetched": TODAY, "key": key}
    f.write_text(json.dumps(js, ensure_ascii=False, indent=1)); time.sleep(0.35)
    return js["message"]

def fetch_url(key, url):
    f = CACHE / f"{key}__url.json"
    if f.exists(): return json.loads(f.read_text())
    r = requests.get(url, headers=UA, timeout=60, allow_redirects=True); CALLS["web"] += 1
    ct = r.headers.get("content-type", "")
    rec = {"key": key, "url": url, "final_url": r.url, "status": r.status_code, "content_type": ct, "fetched": TODAY,
           "bytes": len(r.content), "title": None, "date_modified": None, "text_sample": None}
    if "html" in ct or "text" in ct:
        t = re.search(r"<title[^>]*>(.*?)</title>", r.text, re.S | re.I)
        rec["title"] = html.unescape(re.sub(r"\s+", " ", t.group(1))).strip() if t else None
        dm = re.search(r'dateModified[^>]*?(\d{4}-\d{2}-\d{2})', r.text) or re.search(r'(\d{4}-\d{2}-\d{2})[^<]{0,40}dateModified', r.text)
        rec["date_modified"] = dm.group(1) if dm else None
        rec["text_sample"] = html.unescape(re.sub(r"<[^>]+>", " ", r.text))
        rec["text_sample"] = re.sub(r"\s+", " ", rec["text_sample"])[:4000]
    elif "pdf" in ct:
        rec["title"] = "(PDF document)"
    f.write_text(json.dumps(rec, ensure_ascii=False, indent=1)); time.sleep(0.5)
    return rec

def main():
    man = yaml.safe_load((LIT / "manual_references.yaml").read_text())
    rows = []
    for m in man:
        base = dict(key=m["key"], stream=m["stream"], author=m.get("author", ""), title=m["title"], year=m.get("year"),
                    why_needed=m.get("why_needed", ""), doi=None, url=m.get("url"), crossref_title=None, container=m.get("container", ""),
                    volume=m.get("volume"), issue=m.get("issue"), pages=m.get("pages"), pub_year=m.get("year"), type=m.get("type", "report"),
                    publisher=m.get("publisher", ""), authors=m.get("author", ""), title_sim=None, verification_method=None, verified_date=TODAY, note="")
        if m.get("doi"):
            it = by_doi(m["key"], m["doi"]); yr = it["issued"]["date-parts"][0][0]
            base.update(doi=it["DOI"], url=it.get("URL"), crossref_title=" ".join(it.get("title") or []), container=" ".join(it.get("container-title") or []),
                        volume=it.get("volume"), issue=it.get("issue"), pages=it.get("page"), pub_year=yr, type=it.get("type"), publisher=it.get("publisher"),
                        authors="; ".join(f"{a.get('family','')}, {a.get('given','')}" for a in it.get("author", []) or []), title_sim=1.0,
                        verification_method="crossref:works/{doi}", verified=True, note="resolved by explicit DOI")
            log(f"{m['key']}: DOI {it['DOI']} -> {base['crossref_title'][:60]} ({base['container']}, {yr})")
        else:
            try:
                rec = fetch_url(m["key"], m["url"])
            except Exception as e:
                base.update(verified=False, verification_method="url:fetch", note=f"fetch error {type(e).__name__}"); rows.append(base); continue
            body = (rec.get("text_sample") or "") + " " + (rec.get("title") or "")
            ok = rec["status"] == 200 and (not m.get("must_contain") or m["must_contain"].lower() in body.lower()) and ("pdf" in rec["content_type"] or rec.get("title"))
            base.update(verified=bool(ok), url=rec["final_url"], verification_method="url:fetch",
                        note=f"HTTP {rec['status']}, {rec['content_type'].split(';')[0]}, page title: {rec.get('title')!r}" + (f", dateModified {rec['date_modified']}" if rec.get("date_modified") else "") + ("" if ok else " — must_contain string NOT found"))
            log(f"{m['key']}: {'OK' if ok else 'FAIL'} {rec['status']} {rec['final_url'][:70]} | {rec.get('title')}")
        rows.append(base)
    new = pd.DataFrame(rows)
    ver = pd.read_csv(LIT / "references_verified.csv")
    ver = ver[~ver.key.isin(new.key)]
    out = pd.concat([ver, new[ver.columns]], ignore_index=True)
    out.to_csv(LIT / "references_verified.csv", index=False)
    out[out.verified != True].to_csv(LIT / "unverified.csv", index=False)
    nv = int((out.verified == True).sum())
    log(f"references now: {nv} verified / {len(out)} ; unverified: {out.loc[out.verified != True, 'key'].tolist()}")
    with open(LIT / "LOG.md", "a") as f:
        f.write(f"- {dt.datetime.now().isoformat(timespec='seconds')} resolve_dois.py: {len(man)} manual entries, {CALLS['crossref']} Crossref DOI calls, "
                f"{CALLS['web']} web fetches (rest cached); total {nv} verified / {len(out)}\n")

if __name__ == "__main__":
    main()