1"""Build paper/references.bib ONLY from records verified against Crossref (DOI) or arXiv (id). Each candidate is2(key, doi-or-arxiv, expected title fragment); a record is written only when the fetched title contains the fragment.3Unverified candidates are listed in results/metrics/references_unverified.json and never cited."""4from __future__ import annotations56import json7import re8import sys9import time10import urllib.request11import xml.etree.ElementTree as ET12from pathlib import Path1314ROOT = Path(__file__).resolve().parents[1]15UA = {"User-Agent": "wp41-chronos-x/0.1 (mailto:simon-pierre.boucher@uqo.ca)"}1617CANDIDATES = [18 ("gneiting2007strictly", "10.1198/016214506000001437", "strictly proper scoring rules"),19 ("gneiting2007probabilistic", "10.1111/j.1467-9868.2007.00587.x", "probabilistic forecasts, calibration and sharpness"),20 ("matheson1976scoring", "10.1287/mnsc.22.10.1087", "scoring rules for continuous probability distributions"),21 ("diebold1995comparing", "10.1080/07350015.1995.10524599", "comparing predictive accuracy"),22 ("harvey1997testing", "10.1016/S0169-2070(96)00719-4", "testing the equality of prediction mean squared errors"),23 ("giacomini2006tests", "10.1111/j.1468-0262.2006.00718.x", "tests of conditional predictive ability"),24 ("corsi2009simple", "10.1093/jjfinec/nbp001", "simple approximate long-memory model of realized volatility"),25 ("bollerslev1986generalized", "10.1016/0304-4076(86)90063-1", "generalized autoregressive conditional heteroskedasticity"),26 ("nelson1991conditional", "10.2307/2938260", "conditional heteroskedasticity in asset returns"),27 ("andersen1997intraday", "10.1016/S0927-5398(97)00004-2", "intraday periodicity and volatility persistence"),28 ("andersen1998answering", "10.2307/2527343", "answering the skeptics"),29 ("hansen2005forecast", "10.1002/jae.800", "does anything beat a garch(1,1)"),30 ("patton2011volatility", "10.1016/j.jeconom.2010.03.034", "volatility forecast comparison using imperfect volatility proxies"),31 ("bracher2021evaluating", "10.1371/journal.pcbi.1008618", "evaluating epidemic forecasts in an interval format"),32 ("scheuerer2015variogram", "10.1175/MWR-D-14-00269.1", "variogram-based proper scoring rules"),33 ("jordan2019evaluating", "10.18637/jss.v090.i12", "evaluating probabilistic forecasts with scoringrules"),34 ("taillardat2016calibrated", "10.1175/MWR-D-15-0260.1", "calibrated ensemble forecasts using quantile regression forests"),35 ("gneiting2011comparing", "10.1198/jbes.2010.08110", "comparing density forecasts using threshold"),36 ("cont2001empirical", "10.1080/713665670", "empirical properties of asset returns"),37 ("sirignano2019universal", "10.1080/14697688.2019.1622295", "universal features of price formation in financial markets"),38 ("zhang2019deeplob", "10.1109/TSP.2019.2907260", "deeplob"),39 ("gu2020empirical", "10.1093/rfs/hhaa009", "empirical asset pricing via machine learning"),40 ("kolm2023deep", "10.1111/mafi.12413", "deep order flow imbalance"),41 ("ke2017lightgbm", None, None), # NeurIPS (no DOI); handled below as a manual check42 ("zou2005regularization", "10.1111/j.1467-9868.2005.00503.x", "regularization and variable selection via the elastic net"),43 ("vaswani2017attention", "arXiv:1706.03762", "attention is all you need"),44 ("nie2023patchtst", "arXiv:2211.14730", "a time series is worth 64 words"),45 ("liu2024itransformer", "arXiv:2310.06625", "itransformer"),46 ("ansari2024chronos", "arXiv:2403.07815", "chronos: learning the language of time series"),47 ("das2024timesfm", "arXiv:2310.10688", "a decoder-only foundation model for time-series forecasting"),48 ("bai2018tcn", "arXiv:1803.01271", "an empirical evaluation of generic convolutional and recurrent networks"),49 ("loshchilov2019adamw", "arXiv:1711.05101", "decoupled weight decay regularization"),50 ("salinas2020deepar", "10.1016/j.ijforecast.2019.07.001", "deepar"),51 ("oreshkin2020nbeats", "arXiv:1905.10437", "n-beats"),52 ("lim2021time", "10.1098/rsta.2020.0209", "time-series forecasting with deep learning"),53 ("hansen2011model", "10.3982/ECTA5771", "the model confidence set"),54 ("newey1987simple", "10.2307/1913610", "heteroskedasticity and autocorrelation consistent"),55 ("holm1979simple", None, None), # Scand. J. Stat., no DOI in Crossref -> manual56 ("benjamini1995controlling", "10.1111/j.2517-6161.1995.tb02031.x", "controlling the false discovery rate"),57 ("bailey2014deflated", "10.2139/ssrn.2460551", "deflated sharpe ratio"),58 ("engle2001garch", "10.1257/jep.15.4.157", "garch 101"),59 ("hendrycks2016gelu", "arXiv:1606.08415", "gaussian error linear units"),60 ("ba2016layernorm", "arXiv:1607.06450", "layer normalization"),61 ("dauphin2017glu", "arXiv:1612.08083", "language modeling with gated convolutional networks"),62 ("zeng2023transformers", "arXiv:2205.13504", "are transformers effective for time series forecasting"),63 ("wen2023transformers", "arXiv:2202.07125", "transformers in time series"),64 ("rasul2021autoregressive", "arXiv:2101.12072", "autoregressive denoising diffusion models for multivariate probabilistic time series"),65 ("sklar1959", None, None), # no DOI -> manual66 ("patton2006modelling", "10.1111/j.1468-2354.2006.00387.x", "modelling asymmetric exchange rate dependence"),67 ("andersen2003modeling", "10.1111/1468-0262.00418", "modeling and forecasting realized volatility"),68 ("barndorff2002econometric", "10.1111/1467-9868.00336", "econometric analysis of realized volatility"),69 ("bollerslev2016exploiting", "10.1016/j.jeconom.2015.10.007", "exploiting the errors"),70 ("lopez2018advances", None, None), # book -> manual71 ("kelly2023financial", "10.1561/0500000064", "financial machine learning"),72]7374MANUAL = {75 "ke2017lightgbm": "@inproceedings{ke2017lightgbm,\n author = {Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan},\n title = {{LightGBM}: A Highly Efficient Gradient Boosting Decision Tree},\n booktitle = {Advances in Neural Information Processing Systems 30},\n year = {2017},\n pages = {3146--3154}\n}",76 "holm1979simple": "@article{holm1979simple,\n author = {Holm, Sture},\n title = {A Simple Sequentially Rejective Multiple Test Procedure},\n journal = {Scandinavian Journal of Statistics},\n volume = {6},\n number = {2},\n pages = {65--70},\n year = {1979}\n}",77 "sklar1959": "@article{sklar1959,\n author = {Sklar, Abe},\n title = {Fonctions de r\\'epartition \\`a $n$ dimensions et leurs marges},\n journal = {Publications de l'Institut de Statistique de l'Universit\\'e de Paris},\n volume = {8},\n pages = {229--231},\n year = {1959}\n}",78 "lopez2018advances": "@book{lopez2018advances,\n author = {L{\\'o}pez de Prado, Marcos},\n title = {Advances in Financial Machine Learning},\n publisher = {Wiley},\n address = {Hoboken, NJ},\n year = {2018}\n}",79}808182def tex(s: str) -> str:83 return s.replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")848586def crossref(doi: str) -> dict | None:87 try:88 with urllib.request.urlopen(urllib.request.Request(f"https://api.crossref.org/works/{doi}", headers=UA), timeout=30) as r:89 return json.load(r)["message"]90 except Exception as exc: # noqa: BLE00191 return {"_error": str(exc)[:120]}929394def arxiv(aid: str) -> dict | None:95 try:96 with urllib.request.urlopen(urllib.request.Request(f"http://export.arxiv.org/api/query?id_list={aid}", headers=UA), timeout=30) as r:97 root = ET.fromstring(r.read())98 ns = {"a": "http://www.w3.org/2005/Atom"}99 e = root.find("a:entry", ns)100 if e is None:101 return {"_error": "no entry"}102 return {"title": " ".join(e.findtext("a:title", "", ns).split()), "authors": [x.findtext("a:name", "", ns) for x in e.findall("a:author", ns)],103 "year": e.findtext("a:published", "", ns)[:4], "id": aid}104 except Exception as exc: # noqa: BLE001105 return {"_error": str(exc)[:120]}106107108AUTHOR_FIX = {"hansen2011model": "Hansen, Peter R. and Lunde, Asger and Nason, James M."} # Crossref record lacks authors (verified title/DOI)109YEAR_FIX = {"corsi2009simple": 2009} # Crossref 'issued' = online date 2008; print volume 7(2), 2009110111112def bib_from_crossref(key: str, m: dict) -> str:113 authors = AUTHOR_FIX.get(key) or " and ".join(f"{a.get('family', '')}, {a.get('given', '')}".strip(", ") for a in m.get("author", []))114 title = tex(re.sub(r"<[^>]+>", "", " ".join(m["title"][0].split())))115 year = YEAR_FIX.get(key) or (m.get("published-print", {}).get("date-parts") or [[None]])[0][0] or (m.get("issued", {}).get("date-parts") or [[None]])[0][0]116 journal = tex((m.get("container-title") or [""])[0])117 typ = "article" if m.get("type") == "journal-article" else "misc"118 fields = [f" author = {{{authors}}}", f" title = {{{title}}}", f" year = {{{year}}}"]119 if journal:120 fields.append(f" journal = {{{journal}}}" if typ == "article" else f" howpublished = {{{journal}}}")121 for k in ("volume", "issue", "page"):122 if m.get(k):123 fields.append(f" {'number' if k == 'issue' else 'pages' if k == 'page' else k} = {{{tex(str(m[k])).replace('-', '--') if k == 'page' else m[k]}}}")124 fields.append(f" doi = {{{m['DOI']}}}")125 return f"@{typ}{{{key},\n" + ",\n".join(fields) + "\n}"126127128def bib_from_arxiv(key: str, m: dict) -> str:129 authors = " and ".join(m["authors"])130 return f"@misc{{{key},\n author = {{{authors}}},\n title = {{{tex(m['title'])}}},\n year = {{{m['year']}}},\n howpublished = {{arXiv:{m['id']}}},\n eprint = {{{m['id']}}},\n archiveprefix = {{arXiv}}\n}}"131132133def main() -> None:134 entries, unverified, log = [], [], {}135 for key, ident, frag in CANDIDATES:136 if ident is None:137 entries.append(MANUAL[key]); log[key] = {"status": "manual (no DOI)"}; continue138 if ident.startswith("arXiv:"):139 m = arxiv(ident[6:])140 ok = m and "_error" not in m and frag.lower() in m["title"].lower()141 log[key] = {"status": "verified arXiv" if ok else "UNVERIFIED", "fetched_title": None if not m else m.get("title", m.get("_error"))}142 (entries if ok else unverified).append(bib_from_arxiv(key, m) if ok else key)143 else:144 m = crossref(ident)145 t = re.sub(r"<[^>]+>", "", " ".join(m["title"][0].split())) if m and m.get("title") else None146 ok = bool(t) and frag.lower() in t.lower()147 log[key] = {"status": "verified Crossref" if ok else "UNVERIFIED", "fetched_title": t or (m or {}).get("_error")}148 (entries if ok else unverified).append(bib_from_crossref(key, m) if ok else key)149 time.sleep(0.3)150 (ROOT / "paper").mkdir(exist_ok=True)151 (ROOT / "paper" / "references.bib").write_text("% Generated by scripts/build_references.py from Crossref/arXiv records (verified titles). Do not edit by hand.\n\n" + "\n\n".join(entries) + "\n")152 (ROOT / "results" / "metrics" / "references_verification.json").write_text(json.dumps({"log": log, "unverified": unverified, "n_verified": len(entries)}, indent=1))153 print(f"verified {len(entries)} entries; unverified: {unverified}")154155156if __name__ == "__main__":157 main()158