1"""Build paper/references.bib ONLY from bibliographic records verified against a public registry.23Each candidate is (bibkey, identifier, expected title fragment). The identifier is a Crossref DOI, an4``arXiv:<id>`` or an ``openalex:<Wid>``; alternatives can be chained with `` || `` (tried in order). A record5is written to the .bib only when the fetched title contains the expected fragment (case-insensitive). Items6without any registry record (books, technical documents, pre-DOI working papers) may be included through the7MANUAL list, but only with a URL that was actually fetched and confirmed on the recorded date. Everything else8goes to results/metrics/references_unverified.json and is never cited.910Outputs11 paper/references.bib12 results/metrics/references_verification.json (per bibkey: identifier, fetched title, status, date, source)13 results/metrics/references_unverified.json1415Run: .venv/bin/python scripts/build_references.py16Rate: ~2 requests/s against api.crossref.org (polite pool, mailto in User-Agent), export.arxiv.org, api.openalex.org.17Literature cutoff: 2026-11-09.18"""19from __future__ import annotations2021import datetime as dt22import json23import re24import sys25import time26import urllib.parse27import urllib.request28import xml.etree.ElementTree as ET29from pathlib import Path3031ROOT = Path(__file__).resolve().parents[1]32MAILTO = "simon-pierre.boucher@uqo.ca"33UA = {"User-Agent": f"wp46-garch-benchmark/0.1 (mailto:{MAILTO})"}34SLEEP = 0.5 # seconds between requests (~2 req/s)35TODAY = dt.date.today().isoformat()3637# --------------------------------------------------------------------------------------------------------------38# Candidates: (bibkey, identifier, expected title fragment). Bibkey = firstauthorYEARfirstword (lowercase); the39# year is the print-volume year. Grouped by literature-matrix block (see docs/literature_matrix.md).40# --------------------------------------------------------------------------------------------------------------41CANDIDATES: list[tuple[str, str | None, str | None]] = [42 # ---- (a) univariate ARCH/GARCH architectures -------------------------------------------------------------43 ("engle1982autoregressive", "10.2307/1912773", "autoregressive conditional heteroscedasticity"),44 ("bollerslev1986generalized", "10.1016/0304-4076(86)90063-1", "generalized autoregressive conditional heteroskedasticity"),45 ("engle1986modelling", "10.1080/07474938608800095", "modelling the persistence of conditional variances"),46 ("engle1987estimating", "10.2307/1913242", "arch-m"),47 ("nelson1991conditional", "10.2307/2938260", "conditional heteroskedasticity in asset returns"),48 ("glosten1993relation", "10.1111/j.1540-6261.1993.tb05128.x", "volatility of the nominal excess return"),49 ("zakoian1994threshold", "10.1016/0165-1889(94)90039-6", "threshold heteroskedastic models"),50 ("ding1993long", "10.1016/0927-5398(93)90006-D", "long memory property of stock market returns"),51 ("schwert1989why", "10.1111/j.1540-6261.1989.tb02647.x", "why does stock market volatility change over time"),52 ("engle1993measuring", "10.1111/j.1540-6261.1993.tb05127.x", "impact of news on volatility"),53 ("higgins1992class", "10.2307/2526988", "class of nonlinear arch models"),54 ("sentana1995quadratic", "10.2307/2298081", "quadratic arch models"),55 ("baillie1996fractionally", "10.1016/S0304-4076(95)01749-6", "fractionally integrated generalized autoregressive"),56 ("bollerslev1996modeling", "10.1016/0304-4076(95)01736-4", "long memory in stock market volatility"),57 ("tse1998conditional", "10.1002/(SICI)1099-1255(199801/02)13:1<49::AID-JAE459>3.0.CO;2-O", "yen-dollar exchange rate"),58 ("davidson2004moment", "10.1198/073500103288619359", "moment and memory properties"),59 ("engle1999long", "10.1093/oso/9780198296836.003.0020", "component model of stock return volatility"),60 ("hamilton1994autoregressive", "10.1016/0304-4076(94)90067-1", "changes in regime"),61 ("cai1994markov", "10.1080/07350015.1994.10524546", "switching-regime arch"),62 ("gray1996modeling", "10.1016/0304-405X(96)00875-6", "regime-switching process"),63 ("klaassen2002improving", "10.1007/s001810100100", "regime-switching garch"),64 ("haas2004new", "10.1093/jjfinec/nbh020", "markov-switching garch"),65 ("bauwens2010theory", "10.1111/j.1368-423X.2009.00307.x", "markov switching garch"),66 ("augustyniak2014maximum", "10.1016/j.csda.2013.01.026", "markov-switching garch"),67 ("ardia2019markov", "10.18637/jss.v091.i04", "msgarch"),68 ("hamilton1989new", "10.2307/1912559", "nonstationary time series and the business cycle"),69 ("gonzalezrivera1998smooth", "10.2202/1558-3708.1041", "smooth-transition garch"),70 ("hagerud1997smooth", None, None), # SSE/EFI WP 162 (no DOI) -> MANUAL71 ("engle2013stock", "10.1162/REST_a_00300", "macroeconomic fundamentals"),72 ("ghysels2006predicting", "10.1016/j.jeconom.2005.01.004", "sampled at different frequencies"),73 ("ghysels2007midas", "10.1080/07474930600972467", "midas regressions"),74 ("engle2008spline", "10.1093/rfs/hhn004", "spline-garch"),75 ("engle2011forecasting", "10.1093/jjfinec/nbr005", "multiplicative component garch"),76 ("jpmorgan1996riskmetrics", None, None), # technical document (no DOI) -> MANUAL77 ("zumbach2009riskmetrics", "10.2139/ssrn.1420185", "riskmetrics 2006"),78 ("creal2013generalized", "10.1002/jae.1279", "generalized autoregressive score models"),79 ("harvey2013dynamic", "10.1017/CBO9781139540933", "dynamic models for volatility and heavy tails"),80 ("harvey2014egarch", "10.1016/j.csda.2013.09.022", "egarch models with fat tails"),81 ("harvey2008beta", None, None), # Cambridge WP 0840 (no DOI) -> MANUAL82 ("blasques2015information", "10.1093/biomet/asu076", "observation-driven time series models"),83 ("ardia2019generalized", "10.18637/jss.v088.i06", "gas package"),84 ("taylor2008modelling", "10.1142/6578", "modelling financial time series"),85 ("taylor1994modeling", "10.1111/j.1467-9965.1994.tb00057.x", "modeling stochastic volatility"),86 ("kim1998stochastic", "10.1111/1467-937X.00050", "stochastic volatility: likelihood inference"),87 ("jacquier1994bayesian", "10.1080/07350015.1994.10524553", "bayesian analysis of stochastic volatility"),88 ("harvey1994multivariate", "10.2307/2297980", "multivariate stochastic variance"),89 ("pagan1990alternative", "10.1016/0304-4076(90)90101-X", "alternative models for conditional stock volatility"),90 ("ding1996modeling", "10.1016/0304-4076(95)01737-2", "volatility persistence of speculative returns"),91 # innovation distributions92 ("bollerslev1987conditionally", "10.2307/1925546", "conditionally heteroskedastic time series model"),93 ("hansen1994autoregressive", "10.2307/2527081", "autoregressive conditional density estimation"),94 ("fernandez1998bayesian", "10.1080/01621459.1998.10474117", "fat tails and skewness"),95 # QMLE theory, stationarity, moments, constraints96 ("bollerslev1992quasi", "10.1080/07474939208800229", "quasi-maximum likelihood estimation and inference"),97 ("lee1994asymptotic", "10.1017/S0266466600008215", "garch(1,1) quasi-maximum likelihood"),98 ("lumsdaine1996consistency", "10.2307/2171862", "igarch(1,1)"),99 ("jensen2004asymptotic", "10.1111/j.1468-0262.2004.00504.x", "qmle estimator of arch"),100 ("francq2004maximum", "10.3150/bj/1093265632", "pure garch and arma-garch"),101 ("straumann2006quasi", "10.1214/009053606000000803", "stochastic recurrence equations approach"),102 ("nelson1990stationarity", "10.1017/S0266466600005296", "stationarity and persistence"),103 ("bougerol1992stationarity", "10.1016/0304-4076(92)90067-2", "stationarity of garch processes"),104 ("he1999properties", "10.1016/S0304-4076(98)00089-X", "properties of moments"),105 ("ling2002stationarity", "10.1016/S0304-4076(01)00090-2", "existence of moments"),106 ("ling2002necessary", "10.1017/S0266466602183071", "necessary and sufficient moment conditions"),107 ("conrad2006inequality", "10.1093/jjfinec/nbj015", "fractionally integrated garch"),108 ("chung1999estimating", None, None), # NTU working paper: no registry record found -> unverified109 ("nelson1992inequality", "10.1080/07350015.1992.10509902", "inequality constraints in the univariate garch"),110 ("lamoureux1990persistence", "10.1080/07350015.1990.10509794", "persistence in variance"),111 ("mikosch2004nonstationarities", "10.1162/003465304323023886", "igarch effects"),112 ("hillebrand2005neglecting", "10.1016/j.jeconom.2004.09.005", "neglecting parameter changes"),113 ("lundbergh2002evaluating", "10.1016/S0304-4076(02)00096-9", "evaluating garch models"),114 # surveys, textbooks, handbooks115 ("engle2001garch", "10.1257/jep.15.4.157", "garch 101"),116 ("bollerslev1992arch", "10.1016/0304-4076(92)90064-X", "arch modeling in finance"),117 ("bollerslev1994arch", "10.1016/S1573-4412(05)80018-2", "arch models"),118 ("terasvirta2009introduction", "10.1007/978-3-540-71297-8_1", "univariate garch models"),119 ("engle2004risk", "10.1257/0002828041464597", "risk and volatility"),120 ("bollerslev2010glossary", "10.1093/acprof:oso/9780199549498.003.0008", "glossary to arch"),121 ("engle2001what", "10.1088/1469-7688/1/2/305", "what good is a volatility model"),122 ("francq2019garch", "10.1002/9781119313472", "garch models"),123 ("bauwens2012handbook", "10.1002/9781118272039", "handbook of volatility models"),124 # ---- (b) realized measures and realized-measure models ------------------------------------------------------125 ("andersen1998answering", "10.2307/2527343", "answering the skeptics"),126 ("andersen2001distribution", "10.1198/016214501750332965", "realized exchange rate volatility"),127 ("andersen2001distributionstock", "10.1016/S0304-405X(01)00055-1", "realized stock return volatility"),128 ("andersen2003modeling", "10.1111/1468-0262.00418", "modeling and forecasting realized volatility"),129 ("barndorff2002econometric", "10.1111/1467-9868.00336", "econometric analysis of realized volatility"),130 ("barndorff2004power", "10.1093/jjfinec/nbh001", "power and bipower variation"),131 ("barndorff2008designing", "10.3982/ECTA6495", "designing realized kernels"),132 ("barndorff2009realized", "10.1111/j.1368-423X.2008.00275.x", "realized kernels in practice"),133 ("barndorff2010measuring", "10.1093/acprof:oso/9780199549498.003.0007", "realized semivariance"),134 ("hansen2006realized", "10.1198/073500106000000071", "market microstructure noise"),135 ("hansen2005realized", "10.1093/jjfinec/nbi028", "realized variance for the whole day"),136 ("hansen2006consistent", "10.1016/j.jeconom.2005.01.005", "consistent ranking of volatility models"),137 ("zhang2005tale", "10.1198/016214505000000169", "tale of two time scales"),138 ("aitsahalia2005how", "10.1093/rfs/hhi016", "how often to sample"),139 ("jacod2009microstructure", "10.1016/j.spa.2008.11.004", "pre-averaging approach"),140 ("liu2015does", "10.1016/j.jeconom.2015.02.008", "does anything beat 5-minute rv"),141 ("corsi2009simple", "10.1093/jjfinec/nbp001", "simple approximate long-memory model"),142 ("andersen2007roughing", "10.1162/rest.89.4.701", "roughing it up"),143 ("patton2015good", "10.1162/REST_a_00503", "good volatility, bad volatility"),144 ("bollerslev2016exploiting", "10.1016/j.jeconom.2015.10.007", "exploiting the errors"),145 ("engle2002new", "10.1002/jae.683", "new frontiers for arch models"),146 ("engle2006multiple", "10.1016/j.jeconom.2005.01.018", "multiple indicators model for volatility"),147 ("shephard2010realising", "10.1002/jae.1158", "heavy"),148 ("hansen2012realized", "10.1002/jae.1234", "realized garch"),149 ("hansen2016exponential", "10.1080/07350015.2015.1038543", "exponential garch modeling with realized measures"),150 ("noureldin2012multivariate", "10.1002/jae.1260", "heavy"),151 ("andersen2005correcting", "10.1111/j.1468-0262.2005.00572.x", "correcting the errors"),152 ("andersen2011realized", "10.1016/j.jeconom.2010.03.032", "realized volatility forecasting and market microstructure noise"),153 ("patton2009optimal", "10.1016/j.ijforecast.2009.01.011", "optimal combinations of realised volatility estimators"),154 ("andersen2010parametric", "10.1016/B978-0-444-50897-3.50005-5", "parametric and nonparametric volatility measurement"),155 ("andersen1999forecasting", "10.1016/S0927-5398(99)00013-4", "sample frequency"),156 ("fleming2003economic", "10.1016/S0304-405X(02)00259-3", "economic value of volatility timing"),157 ("hansen2014realized", "10.1002/jae.2389", "realized beta garch"),158 ("archakov2026multivariate", "10.1016/j.jeconom.2025.106040", "multivariate realized garch"),159 ("bollerslev2018modeling", "10.1016/j.jeconom.2018.05.004", "realized covariances"),160 # ---- (c) multivariate ----------------------------------------------------------------------------------------161 ("bollerslev1990modelling", "10.2307/2109358", "multivariate generalized arch"),162 ("engle1995multivariate", "10.1017/S0266466600009063", "multivariate simultaneous generalized arch"),163 ("engle2002dynamic", "10.1198/073500102288618487", "dynamic conditional correlation"),164 ("engle2001theoretical", "10.3386/w8554", "dynamic conditional correlation"),165 ("tse2002multivariate", "10.1198/073500102288618496", "time-varying correlations"),166 ("cappiello2006asymmetric", "10.1093/jjfinec/nbl005", "asymmetric dynamics in the correlations"),167 ("vanderweide2002gogarch", "10.1002/jae.688", "generalized orthogonal garch"),168 ("aielli2013dynamic", "10.1080/07350015.2013.771027", "dynamic conditional correlation: on properties"),169 ("bauwens2006multivariate", "10.1002/jae.842", "multivariate garch models: a survey"),170 ("silvennoinen2009multivariate", "10.1007/978-3-540-71297-8_9", "multivariate garch models"),171 ("engle2019large", "10.1080/07350015.2017.1345683", "large dynamic covariance matrices"),172 ("pakel2021fitting", "10.1080/07350015.2020.1713795", "vast dimensional"),173 ("laurent2012forecasting", "10.1002/jae.1248", "forecasting accuracy of multivariate garch"),174 ("laurent2013loss", "10.1016/j.jeconom.2012.08.004", "loss functions and ranking"),175 ("patton2009evaluating", "10.1007/978-3-540-71297-8_36", "evaluating volatility and correlation forecasts"),176 ("engle2006testing", "10.1198/073500106000000017", "dynamic correlations for asset allocation"),177 ("caporin2014robust", "10.1016/j.csda.2012.05.012", "robust ranking of multivariate garch"),178 ("creal2011dynamic", "10.1198/jbes.2011.10070", "dynamic multivariate heavy-tailed model"),179 # ---- (d) forecast evaluation and inference -------------------------------------------------------------------180 ("patton2011volatility", "10.1016/j.jeconom.2010.03.034", "imperfect volatility proxies"),181 ("hansen2005forecast", "10.1002/jae.800", "does anything beat a garch(1,1)"),182 ("hansen2003choosing", "10.1046/j.0305-9049.2003.00086.x", "model confidence set approach"),183 ("diebold1995comparing", "10.1080/07350015.1995.10524599", "comparing predictive accuracy"),184 ("diebold2015comparing", "10.1080/07350015.2014.983236", "twenty years later"),185 ("harvey1997testing", "10.1016/S0169-2070(96)00719-4", "prediction mean squared errors"),186 ("west1996asymptotic", "10.2307/2171956", "asymptotic inference about predictive ability"),187 ("giacomini2006tests", "10.1111/j.1468-0262.2006.00718.x", "conditional predictive ability"),188 ("clark2007approximately", "10.1016/j.jeconom.2006.05.023", "nested models"),189 ("white2000reality", "10.1111/1468-0262.00152", "reality check"),190 ("hansen2005test", "10.1198/073500105000000063", "superior predictive ability"),191 ("hansen2011model", "10.3982/ECTA5771", "model confidence set"),192 ("romano2005stepwise", "10.1111/j.1468-0262.2005.00615.x", "stepwise multiple testing"),193 ("holm1979simple", "10.2307/4615733 || openalex:W2121044470", "sequentially rejective"),194 ("politis1994stationary", "10.1080/01621459.1994.10476870", "stationary bootstrap"),195 ("politis2004automatic", "10.1081/ETC-120028836", "automatic block-length selection"),196 ("newey1987simple", "10.2307/1913610", "heteroskedasticity and autocorrelation consistent"),197 ("newey1994automatic", "10.2307/2297912", "automatic lag selection"),198 ("mincer1969evaluation", None, None), # NBER chapter (no DOI) -> MANUAL199 ("poon2003forecasting", "10.1257/jel.41.2.478", "forecasting volatility in financial markets"),200 ("awartani2005predicting", "10.1016/j.ijforecast.2004.08.003", "role of asymmetries"),201 ("brownlees2011practical", "10.21314/JOR.2012.237", "calm and storm"),202 ("bollerslev2018risk", "10.1093/rfs/hhy041", "risk everywhere"),203 ("patton2020comparing", "10.1080/07350015.2019.1585256", "comparing possibly misspecified forecasts"),204 ("gneiting2011making", "10.1198/jasa.2011.r10138", "making and evaluating point forecasts"),205 ("gneiting2007strictly", "10.1198/016214506000001437", "strictly proper scoring rules"),206 ("amisano2007comparing", "10.1198/073500106000000332", "weighted likelihood ratio tests"),207 ("diks2011likelihood", "10.1016/j.jeconom.2011.04.001", "density forecasts in tails"),208 ("lopez2001evaluating", "10.1002/1099-131X(200103)20:2<87::AID-FOR782>3.0.CO;2-7", "predictive accuracy of volatility models"),209 ("hansen2015equivalence", "10.3982/ECTA10581", "wald statistics"),210 ("christoffersen2000how", "10.1162/003465300558597", "how relevant is volatility forecasting"),211 ("kourtis2016international", "10.1002/fut.21792", "implied, realized, and garch volatility forecasts"),212 # ---- (e) tail risk -------------------------------------------------------------------------------------------213 ("kupiec1995techniques", "10.3905/jod.1995.407942", "risk measurement models"),214 ("christoffersen1998evaluating", "10.2307/2527341", "evaluating interval forecasts"),215 ("christoffersen2004backtesting", "10.1093/jjfinec/nbh004", "duration-based approach"),216 ("engle2004caviar", "10.1198/073500104000000370", "caviar"),217 ("berkowitz2011evaluating", "10.1287/mnsc.1080.0964", "desk-level data"),218 ("fissler2016higher", "10.1214/16-AOS1439", "higher order elicitability"),219 ("fissler2015expected", "arXiv:1507.00244", "jointly elicitable"), # arXiv v1 2015; Risk magazine version 2016220 ("patton2019dynamic", "10.1016/j.jeconom.2018.10.008", "dynamic semiparametric models for expected shortfall"),221 ("acerbi2014backtesting", "openalex:W2292483766", "backtesting expected shortfall"),222 ("nolde2017elicitability", "10.1214/17-AOAS1041", "elicitability and backtesting"),223 ("mcneil2000estimation", "10.1016/S0927-5398(00)00012-8", "extreme value approach"),224 ("bayer2022regression", "10.1093/jjfinec/nbaa013", "regression-based expected shortfall backtesting"),225 ("dimitriadis2019joint", "10.1214/19-EJS1560", "joint quantile and expected shortfall regression"),226 ("taylor2019forecasting", "10.1080/07350015.2017.1281815", "asymmetric laplace"),227 ("ziegel2016coherence", "10.1111/mafi.12080", "coherence and elicitability"),228 ("kuester2006value", "10.1093/jjfinec/nbj002", "value-at-risk prediction"),229 ("giot2003value", "10.1002/jae.710", "long and short trading positions"),230 # ---- (f) large-scale benchmarks, ML extensions, software ---------------------------------------------------231 ("marcucci2005forecasting", "10.2202/1558-3708.1145", "regime-switching garch models"),232 ("andersen2006volatility", "10.1016/S1574-0706(05)01015-3", "volatility and correlation forecasting"),233 ("ardia2018forecasting", "10.1016/j.ijforecast.2018.05.004", "large-scale performance study"),234 ("christensen2023machine", "10.1093/jjfinec/nbac020", "machine learning approach to volatility forecasting"),235 ("bucci2020realized", "10.1093/jjfinec/nbaa008", "neural networks"),236 ("zhang2024volatility", "10.1093/jjfinec/nbad005", "intraday commonality"),237 ("rahimikia2024machine", "10.2139/ssrn.4857569", "machine learning for realised volatility forecasting"),238 ("rahimikia2021realised", "arXiv:2108.00480", "financial word embedding"),239 ("lyocsa2021stock", "10.1016/j.ijforecast.2020.12.001", "do we need high-frequency data"),240 ("clements2021practical", "10.1016/j.jbankfin.2021.106285", "har volatility model"),241 ("gunnarsson2024prediction", "10.1016/j.irfa.2024.103221", "ai and machine learning: a review"),242 ("branco2024forecasting", "10.1016/j.jempfin.2024.101524", "does anything beat linear models"),243 ("kim2018forecasting", "10.1016/j.eswa.2018.03.002", "lstm with multiple garch-type"),244 ("reisenhofer2022harnet", "arXiv:2205.07719", "harnet"),245 ("brini2026forecasting", "arXiv:2607.05291", "time series foundation models"),246 ("brini2024comparison", "arXiv:2404.04962", "cryptocurrency volatility"),247 ("catania2019forecasting", "10.1016/j.ijforecast.2018.09.005", "forecasting cryptocurrencies"),248 ("ardia2019regime", "10.1016/j.frl.2018.08.009", "bitcoin garch volatility dynamics"),249 ("trucios2019forecasting", "10.1016/j.ijforecast.2019.01.003", "bitcoin risk measures"),250 # software (CRAN DOIs are Crossref-registered; the arch package is on Zenodo/DataCite -> MANUAL)251 ("sheppard2025arch", None, None),252 ("galanos2011rugarch", "10.32614/CRAN.package.rugarch", "rugarch"),253 ("galanos2013rmgarch", "10.32614/CRAN.package.rmgarch", "rmgarch"),254 ("boudt2012highfrequency", "10.32614/CRAN.package.highfrequency", "highfrequency"),255 ("ardia2016msgarch", "10.32614/CRAN.package.MSGARCH", "msgarch"),256 ("catania2014mcs", "10.32614/CRAN.package.MCS", "model confidence set"),257 ("bayer2020esback", "10.32614/CRAN.package.esback", "esback"),258]259260# Records whose Crossref author list is truncated or empty (checked 2026-11-09 against the journal/OpenAlex261# record); the DOI and title were verified by the fetch, only the author field is completed here.262AUTHOR_FIX = {263 "barndorff2004power": "Barndorff-Nielsen, Ole E. and Shephard, Neil",264 "barndorff2008designing": "Barndorff-Nielsen, Ole E. and Hansen, Peter Reinhard and Lunde, Asger and Shephard, Neil",265 "hansen2005realized": "Hansen, Peter Reinhard and Lunde, Asger",266 "hansen2011model": "Hansen, Peter R. and Lunde, Asger and Nason, James M.",267 "haas2004new": "Haas, Markus and Mittnik, Stefan and Paolella, Marc S.",268 "kuester2006value": "Kuester, Keith and Mittnik, Stefan and Paolella, Marc S.",269 "christoffersen2004backtesting": "Christoffersen, Peter and Pelletier, Denis",270 "conrad2006inequality": "Conrad, Christian and Haag, Berthold R.",271 "andersen2001distributionstock": "Andersen, Torben G. and Bollerslev, Tim and Diebold, Francis X. and Ebens, Heiko",272 "bollerslev2010glossary": "Bollerslev, Tim",273 "jpmorgan1996riskmetrics": "{J.P. Morgan/Reuters}",274}275EDITOR_FIX = {"bauwens2012handbook": "Bauwens, Luc and Hafner, Christian M. and Laurent, S{\\'e}bastien"}276# Crossref metadata typos corrected from the publisher page fetched 2026-11-09277# (https://academic.oup.com/book/52280/chapter/421087965: authors Robert F. Engle and Gary G. J. Lee, editors R. F. Engle and278# H. White, Oxford, 1999, pp. 475-497).279FIELD_FIX = {"engle1999long": {"author": "Engle, Robert F. and Lee, Gary G. J.", "editor": "Engle, Robert F. and White, Halbert",280 "booktitle": "Cointegration, Causality, and Forecasting: A Festschrift in Honour of Clive W. J. Granger",281 "publisher": "Oxford University Press", "address": "Oxford"}}282283# Print-volume year when Crossref's 'issued' date is the online-first date (volume/issue checked 2026-11-09).284YEAR_FIX = {285 "corsi2009simple": 2009, "creal2013generalized": 2013, "laurent2012forecasting": 2012, "noureldin2012multivariate": 2012,286 "hansen2012realized": 2012, "engle2019large": 2019, "christensen2023machine": 2023, "zhang2024volatility": 2024,287 "bayer2022regression": 2022, "taylor2019forecasting": 2019, "patton2020comparing": 2020, "pakel2021fitting": 2021,288 "kuester2006value": 2006, "ziegel2016coherence": 2016, "taylor2008modelling": 2008, "brownlees2011practical": 2011,289 "engle2011forecasting": 2011, "hansen2016exponential": 2016, "archakov2026multivariate": 2026,290}291292# Items with no registry record. Each URL was fetched on the recorded date and the bibliographic data below was read293# from the fetched page/PDF (nothing is taken from memory).294MANUAL = {295 "hagerud1997smooth": {296 "url": "https://econpapers.repec.org/paper/hhshastef/0162.htm", "checked": "2026-11-09",297 "evidence": "EconPapers record: Hagerud, G. E., 'A Smooth Transition ARCH Model for Asset Returns', SSE/EFI Working Paper Series in Economics and Finance No. 162, March 1997.",298 "bib": "@techreport{hagerud1997smooth,\n author = {Hagerud, Gustaf E.},\n title = {A Smooth Transition {ARCH} Model for Asset Returns},\n institution = {Stockholm School of Economics},\n type = {SSE/EFI Working Paper Series in Economics and Finance},\n number = {162},\n year = {1997},\n url = {https://econpapers.repec.org/paper/hhshastef/0162.htm}\n}",299 },300 "harvey2008beta": {301 "url": "https://econpapers.repec.org/paper/camcamdae/0840.htm", "checked": "2026-11-09",302 "evidence": "EconPapers record: Harvey, A. and Chakravarty, T., 'Beta-t-(E)GARCH', Cambridge Working Papers in Economics 0840, September 2008.",303 "bib": "@techreport{harvey2008beta,\n author = {Harvey, Andrew and Chakravarty, Tirthankar},\n title = {Beta-t-({E}){GARCH}},\n institution = {University of Cambridge, Faculty of Economics},\n type = {Cambridge Working Papers in Economics},\n number = {0840},\n year = {2008},\n url = {https://econpapers.repec.org/paper/camcamdae/0840.htm}\n}",304 },305 "mincer1969evaluation": {306 "url": "https://www.nber.org/books-and-chapters/economic-forecasts-and-expectations-analysis-forecasting-behavior-and-performance/evaluation-economic-forecasts",307 "checked": "2026-11-09",308 "evidence": "NBER chapter page: Mincer, J. A. and Zarnowitz, V., 'The Evaluation of Economic Forecasts', chapter 1 in J. A. Mincer (ed.), Economic Forecasts and Expectations: Analysis of Forecasting Behavior and Performance, NBER, 1969 (page range not shown on the page, so omitted).",309 "bib": "@incollection{mincer1969evaluation,\n author = {Mincer, Jacob A. and Zarnowitz, Victor},\n title = {The Evaluation of Economic Forecasts},\n booktitle = {Economic Forecasts and Expectations: Analysis of Forecasting Behavior and Performance},\n editor = {Mincer, Jacob A.},\n publisher = {National Bureau of Economic Research},\n address = {New York},\n year = {1969},\n url = {https://www.nber.org/books-and-chapters/economic-forecasts-and-expectations-analysis-forecasting-behavior-and-performance/evaluation-economic-forecasts}\n}",310 },311 "jpmorgan1996riskmetrics": {312 "url": "https://www.msci.com/documents/10199/5915b101-4206-4ba0-aee2-3449d5c7e95a", "checked": "2026-11-09",313 "evidence": "PDF fetched from MSCI (HTTP 200, application/pdf); cover page reads 'J.P.Morgan/Reuters, RiskMetrics -- Technical Document, Fourth Edition, 1996, New York, December 17, 1996, Morgan Guaranty Trust Company, Risk Management Advisory'.",314 "bib": "@techreport{jpmorgan1996riskmetrics,\n author = {{J.P. Morgan/Reuters}},\n title = {{RiskMetrics}---Technical Document},\n edition = {Fourth},\n institution = {Morgan Guaranty Trust Company, Risk Management Advisory},\n address = {New York},\n year = {1996},\n note = {17 December 1996},\n url = {https://www.msci.com/documents/10199/5915b101-4206-4ba0-aee2-3449d5c7e95a}\n}",315 },316 "sheppard2025arch": {317 "url": "https://zenodo.org/api/records/17404619", "checked": "2026-11-09",318 "evidence": "Zenodo API record 17404619: 'bashtage/arch: Release v8.0.0', version v8.0.0, publication_date 2025-10-21, DOI 10.5281/zenodo.17404619 (concept DOI 10.5281/zenodo.593254), creators Kevin Sheppard, Stanislav Khrapov, Gabor Liptak, et al. DataCite DOI (not Crossref), hence manual.",319 "bib": "@misc{sheppard2025arch,\n author = {Sheppard, Kevin and Khrapov, Stanislav and Lipt{\\'a}k, G{\\'a}bor and others},\n title = {bashtage/arch: Release v8.0.0},\n howpublished = {Zenodo (software)},\n year = {2025},\n month = oct,\n version = {8.0.0},\n doi = {10.5281/zenodo.17404619},\n url = {https://doi.org/10.5281/zenodo.17404619},\n note = {Concept DOI 10.5281/zenodo.593254}\n}",320 },321}322323# Venue completion for OpenAlex-only records (not stored in the OpenAlex record; read from the fetched page listed).324OPENALEX_NOTE = {325 "acerbi2014backtesting": ("Risk (Risk.net technical paper)", "https://www.risk.net/risk-magazine/technical-paper/2381658/back-testing-expected-shortfall"),326}327328329# --------------------------------------------------------------------------------------------------------------330def tex(s: str) -> str:331 return (s.replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%").replace("_", "\\_")332 .replace("#", "\\#").replace("$", "\\$"))333334335def clean_title(t: str) -> str:336 return " ".join(re.sub(r"<[^>]+>", "", t).replace("‐", "-").replace("‑", "-").split())337338339def get(url: str, retries: int = 2) -> bytes | None:340 for i in range(retries + 1):341 try:342 with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=40) as r:343 return r.read()344 except Exception as exc: # noqa: BLE001345 last = str(exc)[:120]346 time.sleep(1.5 * (i + 1))347 return None if last is None else (b"__ERR__" + last.encode())348349350def crossref(doi: str) -> dict:351 raw = get(f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='/')}")352 if raw is None or raw.startswith(b"__ERR__"):353 return {"_error": (raw or b"no response")[7:].decode(errors="replace")}354 return json.loads(raw)["message"]355356357def arxiv(aid: str) -> dict:358 raw = get(f"https://export.arxiv.org/api/query?id_list={aid}")359 if raw is None or raw.startswith(b"__ERR__") or not raw.strip():360 return {"_error": "arXiv request failed"}361 root = ET.fromstring(raw)362 ns = {"a": "http://www.w3.org/2005/Atom"}363 e = root.find("a:entry", ns)364 if e is None or not e.findtext("a:title", "", ns).strip():365 return {"_error": "no entry"}366 return {"title": clean_title(e.findtext("a:title", "", ns)),367 "authors": [x.findtext("a:name", "", ns) for x in e.findall("a:author", ns)],368 "year": e.findtext("a:published", "", ns)[:4], "id": aid}369370371def openalex(wid: str) -> dict:372 raw = get(f"https://api.openalex.org/works/{wid}?mailto={MAILTO}")373 if raw is None or raw.startswith(b"__ERR__"):374 return {"_error": "OpenAlex request failed"}375 w = json.loads(raw)376 src = (w.get("primary_location") or {}).get("source") or {}377 return {"title": clean_title(w.get("display_name") or ""), "authors": [a["author"]["display_name"] for a in w.get("authorships", [])],378 "year": w.get("publication_year"), "venue": src.get("display_name"), "biblio": w.get("biblio") or {},379 "doi": (w.get("doi") or "").replace("https://doi.org/", ""), "id": wid}380381382# --------------------------------------------------------------------------------------------------------------383def bib_from_crossref(key: str, m: dict) -> str:384 authors = AUTHOR_FIX.get(key) or " and ".join(385 f"{a.get('family', '')}, {a.get('given', '')}".strip(", ") for a in m.get("author", []) if a.get("family"))386 editors = EDITOR_FIX.get(key) or " and ".join(387 f"{a.get('family', '')}, {a.get('given', '')}".strip(", ") for a in m.get("editor", []) if a.get("family"))388 title = tex(clean_title(m["title"][0]))389 year = YEAR_FIX.get(key) or (m.get("published-print", {}).get("date-parts") or [[None]])[0][0] \390 or (m.get("issued", {}).get("date-parts") or [[None]])[0][0]391 container = tex(clean_title((m.get("container-title") or [""])[0]))392 ctype = m.get("type")393 typ = {"journal-article": "article", "book-chapter": "incollection", "monograph": "book", "book": "book",394 "edited-book": "book", "report": "techreport", "posted-content": "misc"}.get(ctype, "misc")395 fields = []396 if authors:397 fields.append(f" author = {{{authors}}}")398 if editors and not authors:399 fields.append(f" editor = {{{editors}}}")400 fields += [f" title = {{{title}}}", f" year = {{{year}}}"]401 if typ == "article" and container:402 fields.append(f" journal = {{{container}}}")403 elif typ == "incollection" and container:404 fields.append(f" booktitle = {{{container}}}")405 if m.get("publisher"):406 fields.append(f" publisher = {{{tex(m['publisher'])}}}")407 elif typ == "book":408 if m.get("publisher"):409 fields.append(f" publisher = {{{tex(m['publisher'])}}}")410 elif typ == "techreport":411 fields.append(f" institution = {{{tex(m.get('publisher', ''))}}}")412 if m["DOI"].lower().startswith("10.3386/"):413 fields.append(f" type = {{NBER Working Paper}}")414 fields.append(f" number = {{{m['DOI'].split('/')[-1][1:]}}}")415 else:416 if m["DOI"].lower().startswith("10.2139/ssrn"):417 fields.append(" howpublished = {SSRN working paper}")418 elif m["DOI"].lower().startswith("10.32614/cran"):419 fields.append(" howpublished = {CRAN: Contributed Packages (R package)}")420 elif container:421 fields.append(f" howpublished = {{{container}}}")422 if typ in ("article", "incollection"):423 if m.get("volume"):424 fields.append(f" volume = {{{m['volume']}}}")425 if m.get("issue"):426 fields.append(f" number = {{{m['issue']}}}")427 if m.get("page"):428 fields.append(f" pages = {{{tex(str(m['page'])).replace('-', '--')}}}")429 for name, value in FIELD_FIX.get(key, {}).items():430 fields = [f for f in fields if not f.strip().startswith(f"{name} = ")] + [f" {name} = {{{value}}}"]431 fields.append(f" doi = {{{m['DOI']}}}")432 fields.append(f" url = {{https://doi.org/{m['DOI']}}}")433 return f"@{typ}{{{key},\n" + ",\n".join(fields) + "\n}"434435436def bib_from_arxiv(key: str, m: dict) -> str:437 return (f"@misc{{{key},\n author = {{{' and '.join(m['authors'])}}},\n title = {{{tex(m['title'])}}},\n year = {{{m['year']}}},\n"438 f" howpublished = {{arXiv preprint arXiv:{m['id']}}},\n eprint = {{{m['id']}}},\n archiveprefix = {{arXiv}},\n"439 f" url = {{https://arxiv.org/abs/{m['id']}}}\n}}")440441442def bib_from_openalex(key: str, m: dict) -> str:443 venue, url = OPENALEX_NOTE.get(key, (m.get("venue"), None))444 fields = [f" author = {{{' and '.join(m['authors'])}}}", f" title = {{{tex(m['title'].title() if m['title'].isupper() else m['title'])}}}",445 f" year = {{{m['year']}}}"]446 typ = "article" if m.get("venue") else "misc"447 if typ == "article":448 fields.append(f" journal = {{{tex(venue)}}}")449 b = m["biblio"]450 if b.get("volume"):451 fields.append(f" volume = {{{b['volume']}}}")452 if b.get("issue"):453 fields.append(f" number = {{{b['issue']}}}")454 if b.get("first_page") and b.get("last_page"):455 fields.append(f" pages = {{{b['first_page']}--{b['last_page']}}}")456 elif venue:457 fields.append(f" howpublished = {{{tex(venue)}}}")458 if m.get("doi"):459 fields.append(f" doi = {{{m['doi']}}}")460 fields.append(f" url = {{{url or ('https://doi.org/' + m['doi'] if m.get('doi') else 'https://openalex.org/' + m['id'])}}}")461 fields.append(f" note = {{Bibliographic record from OpenAlex {m['id']}}}")462 return f"@{typ}{{{key},\n" + ",\n".join(fields) + "\n}"463464465# --------------------------------------------------------------------------------------------------------------466def resolve(key: str, ident: str, frag: str) -> tuple[str | None, dict]:467 """Try each alternative identifier; return (bib entry or None, log record)."""468 attempts = []469 for alt in [a.strip() for a in ident.split("||")]:470 if alt.startswith("arXiv:"):471 m = arxiv(alt[6:]); src = "arXiv"472 elif alt.startswith("openalex:"):473 m = openalex(alt[9:]); src = "OpenAlex"474 else:475 m = crossref(alt); src = "Crossref"476 time.sleep(SLEEP)477 if "_error" in m:478 attempts.append({"identifier": alt, "source": src, "error": m["_error"]}); continue479 title = m["title"] if src != "Crossref" else (clean_title(m["title"][0]) if m.get("title") else "")480 ok = bool(title) and frag.lower() in title.lower()481 attempts.append({"identifier": alt, "source": src, "fetched_title": title, "title_match": ok})482 if ok:483 bib = {"arXiv": bib_from_arxiv, "OpenAlex": bib_from_openalex, "Crossref": bib_from_crossref}[src](key, m)484 return bib, {"status": f"verified {src}", "identifier": alt, "source": src, "fetched_title": title,485 "expected_fragment": frag, "date": TODAY, "attempts": attempts}486 return None, {"status": "UNVERIFIED", "identifier": ident, "expected_fragment": frag, "date": TODAY, "attempts": attempts}487488489def main() -> None:490 entries, unverified, log = [], [], {}491 keys = [k for k, _, _ in CANDIDATES]492 assert len(keys) == len(set(keys)), "duplicate bibkeys"493 for key, ident, frag in CANDIDATES:494 if ident is None:495 if key in MANUAL:496 man = MANUAL[key]497 entries.append(man["bib"])498 log[key] = {"status": "manual (URL fetched)", "identifier": man["url"], "source": "manual", "date": man["checked"],499 "fetched_title": None, "evidence": man["evidence"]}500 else:501 unverified.append(key)502 log[key] = {"status": "UNVERIFIED", "identifier": None, "source": None, "date": TODAY,503 "fetched_title": None, "evidence": "no registry record found (Crossref bibliographic search, OpenAlex, EconPapers)"}504 continue505 bib, rec = resolve(key, ident, frag)506 log[key] = rec507 if bib:508 entries.append(bib)509 else:510 unverified.append(key)511 print(f"{rec['status']:22s} {key}", file=sys.stderr)512 (ROOT / "paper").mkdir(exist_ok=True)513 (ROOT / "results" / "metrics").mkdir(parents=True, exist_ok=True)514 header = ("% Generated by scripts/build_references.py from Crossref / arXiv / OpenAlex records whose titles were verified\n"515 f"% against the expected fragment (run {TODAY}; literature cutoff 2026-11-09). Manual entries carry a URL that was\n"516 "% fetched on the recorded date. Do not edit by hand -- edit CANDIDATES/MANUAL in the script and re-run.\n\n")517 (ROOT / "paper" / "references.bib").write_text(header + "\n\n".join(entries) + "\n", encoding="utf-8")518 n_manual = sum(1 for v in log.values() if v["status"].startswith("manual"))519 summary = {"generated": TODAY, "literature_cutoff": "2026-11-09", "n_candidates": len(CANDIDATES), "n_written": len(entries),520 "n_verified_registry": len(entries) - n_manual, "n_manual_url": n_manual, "n_unverified": len(unverified),521 "unverified": unverified, "records": log}522 (ROOT / "results" / "metrics" / "references_verification.json").write_text(json.dumps(summary, indent=1, ensure_ascii=False), encoding="utf-8")523 (ROOT / "results" / "metrics" / "references_unverified.json").write_text(json.dumps(524 {"generated": TODAY, "unverified": [{"bibkey": k, **{kk: vv for kk, vv in log[k].items() if kk != "status"}} for k in unverified]},525 indent=1, ensure_ascii=False), encoding="utf-8")526 print(f"written {len(entries)} entries ({len(entries) - n_manual} registry-verified + {n_manual} manual/URL); unverified: {unverified}")527528529if __name__ == "__main__":530 main()531