"""Build paper/references.bib ONLY from records verified against Crossref (DOI) or arXiv (id). Each candidate is
(key, doi-or-arxiv, expected title fragment); a record is written only when the fetched title contains the fragment.
Unverified candidates are listed in results/metrics/references_unverified.json and never cited."""
from __future__ import annotations

import json
import re
import sys
import time
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
UA = {"User-Agent": "wp41-chronos-x/0.1 (mailto:simon-pierre.boucher@uqo.ca)"}

CANDIDATES = [
    ("gneiting2007strictly", "10.1198/016214506000001437", "strictly proper scoring rules"),
    ("gneiting2007probabilistic", "10.1111/j.1467-9868.2007.00587.x", "probabilistic forecasts, calibration and sharpness"),
    ("matheson1976scoring", "10.1287/mnsc.22.10.1087", "scoring rules for continuous probability distributions"),
    ("diebold1995comparing", "10.1080/07350015.1995.10524599", "comparing predictive accuracy"),
    ("harvey1997testing", "10.1016/S0169-2070(96)00719-4", "testing the equality of prediction mean squared errors"),
    ("giacomini2006tests", "10.1111/j.1468-0262.2006.00718.x", "tests of conditional predictive ability"),
    ("corsi2009simple", "10.1093/jjfinec/nbp001", "simple approximate long-memory model of realized volatility"),
    ("bollerslev1986generalized", "10.1016/0304-4076(86)90063-1", "generalized autoregressive conditional heteroskedasticity"),
    ("nelson1991conditional", "10.2307/2938260", "conditional heteroskedasticity in asset returns"),
    ("andersen1997intraday", "10.1016/S0927-5398(97)00004-2", "intraday periodicity and volatility persistence"),
    ("andersen1998answering", "10.2307/2527343", "answering the skeptics"),
    ("hansen2005forecast", "10.1002/jae.800", "does anything beat a garch(1,1)"),
    ("patton2011volatility", "10.1016/j.jeconom.2010.03.034", "volatility forecast comparison using imperfect volatility proxies"),
    ("bracher2021evaluating", "10.1371/journal.pcbi.1008618", "evaluating epidemic forecasts in an interval format"),
    ("scheuerer2015variogram", "10.1175/MWR-D-14-00269.1", "variogram-based proper scoring rules"),
    ("jordan2019evaluating", "10.18637/jss.v090.i12", "evaluating probabilistic forecasts with scoringrules"),
    ("taillardat2016calibrated", "10.1175/MWR-D-15-0260.1", "calibrated ensemble forecasts using quantile regression forests"),
    ("gneiting2011comparing", "10.1198/jbes.2010.08110", "comparing density forecasts using threshold"),
    ("cont2001empirical", "10.1080/713665670", "empirical properties of asset returns"),
    ("sirignano2019universal", "10.1080/14697688.2019.1622295", "universal features of price formation in financial markets"),
    ("zhang2019deeplob", "10.1109/TSP.2019.2907260", "deeplob"),
    ("gu2020empirical", "10.1093/rfs/hhaa009", "empirical asset pricing via machine learning"),
    ("kolm2023deep", "10.1111/mafi.12413", "deep order flow imbalance"),
    ("ke2017lightgbm", None, None),   # NeurIPS (no DOI); handled below as a manual check
    ("zou2005regularization", "10.1111/j.1467-9868.2005.00503.x", "regularization and variable selection via the elastic net"),
    ("vaswani2017attention", "arXiv:1706.03762", "attention is all you need"),
    ("nie2023patchtst", "arXiv:2211.14730", "a time series is worth 64 words"),
    ("liu2024itransformer", "arXiv:2310.06625", "itransformer"),
    ("ansari2024chronos", "arXiv:2403.07815", "chronos: learning the language of time series"),
    ("das2024timesfm", "arXiv:2310.10688", "a decoder-only foundation model for time-series forecasting"),
    ("bai2018tcn", "arXiv:1803.01271", "an empirical evaluation of generic convolutional and recurrent networks"),
    ("loshchilov2019adamw", "arXiv:1711.05101", "decoupled weight decay regularization"),
    ("salinas2020deepar", "10.1016/j.ijforecast.2019.07.001", "deepar"),
    ("oreshkin2020nbeats", "arXiv:1905.10437", "n-beats"),
    ("lim2021time", "10.1098/rsta.2020.0209", "time-series forecasting with deep learning"),
    ("hansen2011model", "10.3982/ECTA5771", "the model confidence set"),
    ("newey1987simple", "10.2307/1913610", "heteroskedasticity and autocorrelation consistent"),
    ("holm1979simple", None, None),   # Scand. J. Stat., no DOI in Crossref -> manual
    ("benjamini1995controlling", "10.1111/j.2517-6161.1995.tb02031.x", "controlling the false discovery rate"),
    ("bailey2014deflated", "10.2139/ssrn.2460551", "deflated sharpe ratio"),
    ("engle2001garch", "10.1257/jep.15.4.157", "garch 101"),
    ("hendrycks2016gelu", "arXiv:1606.08415", "gaussian error linear units"),
    ("ba2016layernorm", "arXiv:1607.06450", "layer normalization"),
    ("dauphin2017glu", "arXiv:1612.08083", "language modeling with gated convolutional networks"),
    ("zeng2023transformers", "arXiv:2205.13504", "are transformers effective for time series forecasting"),
    ("wen2023transformers", "arXiv:2202.07125", "transformers in time series"),
    ("rasul2021autoregressive", "arXiv:2101.12072", "autoregressive denoising diffusion models for multivariate probabilistic time series"),
    ("sklar1959", None, None),   # no DOI -> manual
    ("patton2006modelling", "10.1111/j.1468-2354.2006.00387.x", "modelling asymmetric exchange rate dependence"),
    ("andersen2003modeling", "10.1111/1468-0262.00418", "modeling and forecasting realized volatility"),
    ("barndorff2002econometric", "10.1111/1467-9868.00336", "econometric analysis of realized volatility"),
    ("bollerslev2016exploiting", "10.1016/j.jeconom.2015.10.007", "exploiting the errors"),
    ("lopez2018advances", None, None),  # book -> manual
    ("kelly2023financial", "10.1561/0500000064", "financial machine learning"),
]

MANUAL = {
    "ke2017lightgbm": "@inproceedings{ke2017lightgbm,\n  author = {Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan},\n  title = {{LightGBM}: A Highly Efficient Gradient Boosting Decision Tree},\n  booktitle = {Advances in Neural Information Processing Systems 30},\n  year = {2017},\n  pages = {3146--3154}\n}",
    "holm1979simple": "@article{holm1979simple,\n  author = {Holm, Sture},\n  title = {A Simple Sequentially Rejective Multiple Test Procedure},\n  journal = {Scandinavian Journal of Statistics},\n  volume = {6},\n  number = {2},\n  pages = {65--70},\n  year = {1979}\n}",
    "sklar1959": "@article{sklar1959,\n  author = {Sklar, Abe},\n  title = {Fonctions de r\\'epartition \\`a $n$ dimensions et leurs marges},\n  journal = {Publications de l'Institut de Statistique de l'Universit\\'e de Paris},\n  volume = {8},\n  pages = {229--231},\n  year = {1959}\n}",
    "lopez2018advances": "@book{lopez2018advances,\n  author = {L{\\'o}pez de Prado, Marcos},\n  title = {Advances in Financial Machine Learning},\n  publisher = {Wiley},\n  address = {Hoboken, NJ},\n  year = {2018}\n}",
}


def tex(s: str) -> str:
    return s.replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")


def crossref(doi: str) -> dict | None:
    try:
        with urllib.request.urlopen(urllib.request.Request(f"https://api.crossref.org/works/{doi}", headers=UA), timeout=30) as r:
            return json.load(r)["message"]
    except Exception as exc:  # noqa: BLE001
        return {"_error": str(exc)[:120]}


def arxiv(aid: str) -> dict | None:
    try:
        with urllib.request.urlopen(urllib.request.Request(f"http://export.arxiv.org/api/query?id_list={aid}", headers=UA), timeout=30) as r:
            root = ET.fromstring(r.read())
        ns = {"a": "http://www.w3.org/2005/Atom"}
        e = root.find("a:entry", ns)
        if e is None:
            return {"_error": "no entry"}
        return {"title": " ".join(e.findtext("a:title", "", ns).split()), "authors": [x.findtext("a:name", "", ns) for x in e.findall("a:author", ns)],
                "year": e.findtext("a:published", "", ns)[:4], "id": aid}
    except Exception as exc:  # noqa: BLE001
        return {"_error": str(exc)[:120]}


AUTHOR_FIX = {"hansen2011model": "Hansen, Peter R. and Lunde, Asger and Nason, James M."}   # Crossref record lacks authors (verified title/DOI)
YEAR_FIX = {"corsi2009simple": 2009}   # Crossref 'issued' = online date 2008; print volume 7(2), 2009


def bib_from_crossref(key: str, m: dict) -> str:
    authors = AUTHOR_FIX.get(key) or " and ".join(f"{a.get('family', '')}, {a.get('given', '')}".strip(", ") for a in m.get("author", []))
    title = tex(re.sub(r"<[^>]+>", "", " ".join(m["title"][0].split())))
    year = YEAR_FIX.get(key) or (m.get("published-print", {}).get("date-parts") or [[None]])[0][0] or (m.get("issued", {}).get("date-parts") or [[None]])[0][0]
    journal = tex((m.get("container-title") or [""])[0])
    typ = "article" if m.get("type") == "journal-article" else "misc"
    fields = [f"  author = {{{authors}}}", f"  title = {{{title}}}", f"  year = {{{year}}}"]
    if journal:
        fields.append(f"  journal = {{{journal}}}" if typ == "article" else f"  howpublished = {{{journal}}}")
    for k in ("volume", "issue", "page"):
        if m.get(k):
            fields.append(f"  {'number' if k == 'issue' else 'pages' if k == 'page' else k} = {{{tex(str(m[k])).replace('-', '--') if k == 'page' else m[k]}}}")
    fields.append(f"  doi = {{{m['DOI']}}}")
    return f"@{typ}{{{key},\n" + ",\n".join(fields) + "\n}"


def bib_from_arxiv(key: str, m: dict) -> str:
    authors = " and ".join(m["authors"])
    return f"@misc{{{key},\n  author = {{{authors}}},\n  title = {{{tex(m['title'])}}},\n  year = {{{m['year']}}},\n  howpublished = {{arXiv:{m['id']}}},\n  eprint = {{{m['id']}}},\n  archiveprefix = {{arXiv}}\n}}"


def main() -> None:
    entries, unverified, log = [], [], {}
    for key, ident, frag in CANDIDATES:
        if ident is None:
            entries.append(MANUAL[key]); log[key] = {"status": "manual (no DOI)"}; continue
        if ident.startswith("arXiv:"):
            m = arxiv(ident[6:])
            ok = m and "_error" not in m and frag.lower() in m["title"].lower()
            log[key] = {"status": "verified arXiv" if ok else "UNVERIFIED", "fetched_title": None if not m else m.get("title", m.get("_error"))}
            (entries if ok else unverified).append(bib_from_arxiv(key, m) if ok else key)
        else:
            m = crossref(ident)
            t = re.sub(r"<[^>]+>", "", " ".join(m["title"][0].split())) if m and m.get("title") else None
            ok = bool(t) and frag.lower() in t.lower()
            log[key] = {"status": "verified Crossref" if ok else "UNVERIFIED", "fetched_title": t or (m or {}).get("_error")}
            (entries if ok else unverified).append(bib_from_crossref(key, m) if ok else key)
        time.sleep(0.3)
    (ROOT / "paper").mkdir(exist_ok=True)
    (ROOT / "paper" / "references.bib").write_text("% Generated by scripts/build_references.py from Crossref/arXiv records (verified titles). Do not edit by hand.\n\n" + "\n\n".join(entries) + "\n")
    (ROOT / "results" / "metrics" / "references_verification.json").write_text(json.dumps({"log": log, "unverified": unverified, "n_verified": len(entries)}, indent=1))
    print(f"verified {len(entries)} entries; unverified: {unverified}")


if __name__ == "__main__":
    main()