spb/wp46_uqo

added .gitignore +48 −0

		@@ -0,0 +1,48 @@
	1	+# Raw & processed data — never committed (see data/raw/README.md for provenance)
	2	+data/raw/*
	3	+!data/raw/README.md
	4	+data/processed/*
	5	+!data/processed/.gitkeep
	6	+
	7	+# Python
	8	+__pycache__/
	9	+*.pyc
	10	+.pytest_cache/
	11	+.venv/
	12	+*.egg-info/
	13	+
	14	+# LaTeX build artifacts (paper/main.pdf IS committed)
	15	+paper/*.aux
	16	+paper/*.log
	17	+paper/*.out
	18	+paper/*.toc
	19	+paper/*.lof
	20	+paper/*.lot
	21	+paper/*.fls
	22	+paper/*.fdb_latexmk
	23	+paper/*.bbl
	24	+paper/*.blg
	25	+paper/*.spl
	26	+paper/*.synctex.gz
	27	+paper/*.run.xml
	28	+paper/*.bcf
	29	+
	30	+# Logs
	31	+logs/
	32	+*.log
	33	+
	34	+# macOS
	35	+.DS_Store
	36	+
	37	+# WP46 — licensed data and heavy artefacts never committed
	38	+data/interim/*
	39	+!data/interim/.gitkeep
	40	+results/forecasts/
	41	+results/raw_runs/
	42	+*.parquet
	43	+!data/synthetic/*.parquet
	44	+*.feather
	45	+*.npz
	46	+*.sqlite
	47	+.secrets/
	48	+*.env

added CLAUDE.md +47 −0

		@@ -0,0 +1,47 @@
	1	+# CLAUDE.md — UQO Research WP46: The Great GARCH Benchmark
	2	+
	3	+**Project ID:** `wp46_uqo` · **Folder:** `Recherche-UQO/finance/wp46_garch_benchmark` · **Repo:** `gitsrv:srv/git/wp46_uqo.git`
	4	+**Working title:** *Four Decades of Conditional Volatility Modeling: A Comprehensive Empirical Benchmark Across Assets,
	5	+Frequencies, and Market Regimes* · **PI:** Simon-Pierre Boucher (UQO) · **Language:** English.
	6	+The owner's original brief is kept verbatim in `docs/BRIEF.md` and is the authoritative text; this file condenses it and
	7	+records the decisions taken while building (see `docs/methodology_decisions.md`, `progress.md`).
	8	+
	9	+## Core question
	10	+Across asset classes, horizons (1, 5, 22 days), regimes and information sets, when — if ever — do more complex conditional
	11	+volatility models reliably beat GARCH(1,1), and is any gain worth the numerical and computational cost?
	12	+
	13	+## Hard rules (non-negotiable)
	14	+1. Never fabricate data, estimates, references, DOIs, p-values, run outcomes or novelty. Planned ≠ completed: `progress.md`
	15	+ and `docs/PHASE_N_REPORT.md` separate the two; every number in the paper is generated by code from `results/`.
	16	+2. Pre-registration: `docs/preregistered_protocol.md` is frozen (git tag `prereg-frozen`) BEFORE any out-of-sample test
	17	+ loss is inspected. The test period is never used for selecting orders, optimizer settings, regime definitions or losses.
	18	+3. Data: FinData OS (`https://findata.spboucher.ai/api/v1`, FirstRate Data private licence) is the only primary source.
	19	+ No raw bars in git, logs, figures, manuscript or release; aggregate forecasts/losses only. Keys only from
	20	+ `~/Desktop/Cluster/secrets/findata/api-key-claude-code.txt` or `$FINDATA_API_KEY`; never logged.
	21	+4. Units are explicit everywhere: daily log returns in **percent**; variances in percent²; forecasts are conditional
	22	+ variances of the h-day cumulative return matched to a same-horizon positive proxy.
	23	+5. Failed fits are recorded as failures (never silently replaced by a baseline); unsupported model × distribution pairs
	24	+ are recorded as unsupported, not as bad performance.
	25	+6. Every accepted implementation has: unit tests (recursion, likelihood, admissibility, multi-step), a likelihood
	26	+ cross-check against `arch` 8.0.0 and/or `rugarch` 1.5-6 on a shared fixture, seeded simulation recovery, and a model card.
	27	+7. No external publication, push to a new remote, SSRN deposit or paid compute without explicit authorization.
	28	+ Pushing to the private `gitsrv` mirror is routine.
	29	+8. Remote-first: computation runs on the MacLustr nodes (`~/wp46_uqo` clones), the laptop is the LaTeX/mirror host.
	30	+
	31	+## Where things are
	32	+| Item | Location |
	33	+|---|---|
	34	+| Brief / decisions / progress | `docs/BRIEF.md`, `docs/methodology_decisions.md`, `progress.md` |
	35	+| Configs (frozen by the protocol) | `configs/{study,instruments,model_registry,forecast_protocol}.yaml` |
	36	+| Code | `src/wp46/{findata_client,calendars,preprocessing,realized_measures,models,estimation,forecasts,scoring,inference,diagnostics,reporting}` |
	37	+| Scripts | `scripts/{audit_findata,run_pilot,run_benchmark,build_paper,…}.py` |
	38	+| Tests | `tests/{unit,integration,simulation,replication}` (`pytest`; live tests need `WP46_LIVE=1`; R tests need rugarch) |
	39	+| Docs | `docs/{findata_audit,literature_matrix,preregistered_protocol,feasibility_report,methodology_decisions,data_dictionary,replication_guide}.md` |
	40	+| Model cards | `model_cards/<slug>.md` |
	41	+| Results | `results/{manifests,metrics,figures,tables,logs}` (forecast parquet files under `results/forecasts/` are gitignored) |
	42	+| Paper | `paper/main.tex`, `sections/`, `appendices/`, `figures/`, `tables/`, `references.bib` (Crossref-verified) |
	43	+
	44	+## Compute
	45	+Data node M3U96a (hosts FinData; `FINDATA_BASE_URL=http://127.0.0.1:8600/api/v1` there, `http://192.168.2.80:8600/api/v1`
	46	+from the LAN). Benchmark shards on M3U96a/M3U96b/M2U64/M5M36/M4M64a/M4M64c (`scripts/node_run.sh`). Cache
	47	+`~/wp46-cache` (outside git). Laptop: Python 3.14 + arch 8.0.0, R 4.6.1 + rugarch 1.5-6 for cross-checks.

added configs/instruments.yaml +11 −0

		@@ -0,0 +1,11 @@
	1	+# WP46 candidate universe (proposal; coverage verified by scripts/audit_findata.py -> docs/findata_audit.md).
	2	+# The FINAL universe is frozen in docs/preregistered_protocol.md (section "Universe") after the audit.
	3	+version: 1
	4	+candidates:
	5	+ etf: [SPY, QQQ, IWM, DIA, EEM, EFA, XLF, XLE, TLT, IEF, HYG, LQD, GLD, SLV, USO]
	6	+ stock: [AAPL, MSFT, AMZN, NVDA, JPM, XOM, JNJ, PG, KO, WMT]
	7	+ futures: [ES, NQ, CL, NG, GC, SI, HG, ZN, ZC]
	8	+ fx: [EURUSD, USDJPY, GBPUSD, AUDUSD, USDCAD, USDCHF]
	9	+ crypto: [BTC, ETH]
	10	+regime_variables:
	11	+ index: [VIX, SPX]

added data/interim/.gitkeep +0 −0

added data/processed/.gitkeep +0 −0

added docs/BRIEF.md +350 −0

		@@ -0,0 +1,350 @@
	1	+# Owner's brief (verbatim, received 2026-11-09) — authoritative text for WP46
	2	+
	3	+CLAUDE.md — UQO Research WP46: The Great GARCH Benchmark
	4	+
	5	+Mission and non-negotiable objective
	6	+
	7	+You are the lead quantitative research engineering agent for WP46 in the Université du Québec en Outaouais (UQO) research program. Build a publication-grade, fully reproducible empirical and methodological assessment of conditional volatility models, from ARCH through modern GARCH-family, realized-measure, regime-switching, multivariate, and adjacent conditional-volatility alternatives.
	8	+
	9	+Working title: Four Decades of Conditional Volatility Modeling: A Comprehensive Empirical Benchmark Across Assets, Frequencies, and Market Regimes.
	10	+
	11	+Core question: Across asset classes, forecast horizons, market regimes and information sets, when—if ever—do more complex conditional volatility models reliably outperform parsimonious GARCH(1,1), and is any gain worth the numerical and computational cost?
	12	+
	13	+Author/project: Simon-Pierre Boucher, UQO research, WP46. Write the manuscript, code comments, exhibits and technical documentation in polished academic English. Do not claim results before running analyses. The desired target is a strong econometrics/empirical finance journal submission, not an unverified survey.
	14	+
	15	+Treat this file as operating instructions. Never fabricate data, estimates, references, API fields, p-values, confidence intervals, performance figures, sample periods, or evidence of novelty. Clearly distinguish planned work from completed work.
	16	+
	17	+First actions: reconnaissance before implementation
	18	+
	19	+Inspect the actual UQO research repository, its CLAUDE.md, coding conventions, existing WP folders, data governance, bibliographies and CI. Do not overwrite unrelated work. Find the exact WP naming pattern and create WP46 consistently; if no convention exists use wp46_garch_benchmark/.
	20	+
	21	+Discover how to access Findata at https://findata.spboucher.ai (user may refer to it as indata.spboucher.ai). Confirm the correct hostname and official API contract. Do not guess endpoints or authenticate against speculative hosts. Inspect public docs and the repo's existing configured integration; request a credential or API documentation only if unavailable. Never log tokens or place keys in source files.
	22	+
	23	+Audit the true data inventory: tickers/instrument types, exchanges, timezone, timestamp precision, adjusted prices, bid/ask/trades/OHLCV, minute and five-minute history, daily history, corporate actions, futures rolls, licensing, missingness, API limits, coverage start/end and export permissions. Write docs/findata_audit.md with evidence and unresolved questions. A reference to an instrument below is a proposal, not proof of coverage.
	24	+
	25	+Search and audit current methodological literature through an explicit cutoff date. Build a verified bibliography with DOIs and source links. Identify closely related large-scale benchmarks, including Hansen and Lunde (2005), and explain incremental contribution honestly.
	26	+
	27	+Prepare docs/preregistered_protocol.md before inspecting out-of-sample performance. Freeze the primary universe, date ranges, metrics, model selection policy and multiplicity corrections. Keep exploratory work labeled exploratory.
	28	+
	29	+Produce docs/feasibility_report.md and a small execution pilot. Then proceed in independently validated phases. Never fake successful runs.
	30	+
	31	+Scientific contributions to pursue
	32	+
	33	+C1 — Taxonomy and mathematical reference
	34	+
	35	+Create a traceable ontology of conditional volatility models: defining equations, conditional mean, innovation law, parameter constraints, positivity, stationarity and moment conditions, leverage/asymmetry, estimation objective, forecast recursion, nested relationships, identifiable failure modes and original references. Preserve differences between similarly named variants across papers and packages; never silently equate TGARCH definitions, for example.
	36	+
	37	+C2 — Multi-asset out-of-sample benchmark
	38	+
	39	+Use identical pre-registered information sets, rolling origins, forecast targets and loss definitions within a comparison tier. Comparisons across tiers must explain the additional data available to realized-measure models. Evaluate statistical significance and uncertainty—not just rank tables.
	40	+
	41	+C3 — Complexity–accuracy–robustness frontier
	42	+
	43	+Estimate whether incremental gains survive transaction-relevant/risk-specific evaluation, computation, convergence screening and market-regime changes. Report Pareto frontiers rather than a single arbitrary aggregate ranking.
	44	+
	45	+C4 — Open benchmark and replication package
	46	+
	47	+Publish reproducible configurations, instrument/data manifests, code, tests, tables and provenance, subject to data licensing. If raw data cannot legally be redistributed, publish scripts, schemas, checksums, synthetic fixtures and access instructions instead.
	48	+
	49	+Formal boundaries and taxonomy
	50	+
	51	+Inventory all materially distinct canonical GARCH architectures feasible from the literature; do not claim to implement literally every variant ever published. Maintain a model_registry.yaml with references, precise mathematical definition, software implementation, implementation status, estimation method, admissibility rules, feature requirements and validation evidence. Different parameter orders and innovation distributions are specifications, not separate architectures.
	52	+
	53	+Candidate model families (verify nomenclature and primary citations before inclusion):
	54	+
	55	+Classical: ARCH(q), GARCH(p,q), IGARCH, GARCH-M, GARCH-X; useful baseline constant variance/EWMA and historical volatility.
	56	+
	57	+Asymmetric and power: EGARCH, GJR-GARCH, threshold/TGARCH with unambiguous definition, APARCH, AVGARCH, NGARCH/NAGARCH, QGARCH, selected flexible formulations.
	58	+
	59	+Fractional persistence: FIGARCH, FIEGARCH, FIAPARCH, HYGARCH, with careful numerical approximations and stationarity distinctions.
	60	+
	61	+Components and regime shifts: component GARCH, Markov-switching GARCH, smooth-transition GARCH and GARCH-MIDAS (only if the explanatory frequency and feature availability justify it).
	62	+
	63	+Realized-measure and high-frequency: Realized GARCH, Realized EGARCH, HEAVY, realized-input GARCH-X; verify exact joint measurement equations and observation timing.
	64	+
	65	+Multivariate: CCC, DCC, asymmetric DCC, BEKK, GO-GARCH and selected scalable alternatives. Evaluate covariance/correlation forecasts separately, not as if they were univariate variance competitors.
	66	+
	67	+Adjacent comparators, explicitly labeled non-GARCH where applicable: GAS/score-driven volatility, HAR-RV/HAR-X, realized volatility benchmark, stochastic-volatility variants if feasible; reserve modern ML models for a clearly designated extension, not for redefining the GARCH family.
	68	+
	69	+For feasible architectures, test Gaussian, standardized Student-t, skewed Student-t and GED/generalized-error innovations only where formally valid and correctly parameterized. Prefer a preregistered small grid of orders (e.g., 1,1 and selected 2,1/1,2) over combinatorial overfitting. Record unsupported model–distribution combinations as unsupported, not failed performance.
	70	+
	71	+Create one detailed model_cards/<slug>.md per architecture: seminal citation, formula in LaTeX, assumptions, interpretation, optimizer, standard errors, forecast formula, implementation links, unit tests, known limitations, computational footprint and literature expectations.
	72	+
	73	+Data and measurement specification
	74	+
	75	+Plan cross-asset coverage including liquid equity ETFs (SPY/QQQ/IWM), large-cap single stocks, Treasury instruments, energy, precious metals, foreign exchange and BTC/ETH if and only if Findata can supply them at adequate quality. Substitute verified liquid proxies, and disclose changes. Minimum viable pilot: at least one equity ETF, one individual equity, and one commodity or FX instrument if covered. Preferred final sample: approximately 20–40 well-documented assets, not a blind target count.
	76	+
	77	+Implement:
	78	+
	79	+Exchange calendars, trading sessions, market holidays, half-days, daylight saving transitions, quote/trade consolidation and timestamp alignment. Distinguish 24/7 crypto calendars.
	80	+
	81	+Split/dividend-adjusted total-return or price-return conventions; survivorship and selection bias documentation.
	82	+
	83	+Contract-continuous futures with explicit roll convention, back-adjustment implications and no spurious roll returns. Do not mix spot and futures series without labeling.
	84	+
	85	+Point-in-time data availability and strict as-of joins; prevent look-ahead for fundamentals, implied measures, macro regressors and realized measures.
	86	+
	87	+Daily log returns with explicit percentage versus decimal scaling. Intraday log returns at 1- and 5-minute frequency where valid.
	88	+
	89	+Realized variance RV_t = sum_j r_{t,j}^2, realized semivariances, bipower variation and robust/noise-aware realized measures when data suffice. Treat overnight variance separately; avoid claiming intraday RV equals full 24-hour integrated variance.
	90	+
	91	+Missing ticks, sparse assets, zero prices, bid–ask bounce, stale quotes, outliers and corporate action anomalies; predefine exclusions and sensitivity checks.
	92	+
	93	+Immutable raw snapshots where permitted, checksums, provenance records, rate-limit-aware retrieval, retries with backoff and resumable ingestion.
	94	+
	95	+All timestamps, returns and variance forecasts must have explicit units and horizon conventions. Match one-day-ahead daily conditional variance to a same-horizon variance proxy, not to volatility in standard-deviation units.
	96	+
	97	+Experimental protocol
	98	+
	99	+Information tiers
	100	+
	101	+Tier A (daily only): common daily return information set, standard GARCH and daily alternatives.
	102	+
	103	+Tier B (intraday measures): models using lagged realized measures; compare against strong daily baselines but disclose unequal information set. Optionally provide a matched-information comparison where daily baselines gain appropriate realized covariates.
	104	+
	105	+Tier C (multivariate): covariance and correlation forecasts on fixed asset baskets with separate scoring rules.
	106	+
	107	+Backtesting
	108	+
	109	+Separate chronologically ordered training, tuning/validation and untouched test periods, with dates selected after data audit but before model ranking.
	110	+
	111	+Rolling and expanding estimation windows; rolling forecasts at horizons 1, 5 and 22 trading days as viable. Explain overlapping target dependence and use HAC/block bootstrap where needed.
	112	+
	113	+Re-estimation cadence, optimizer budgets, stopping tolerances and available compute fixed within comparable groups. No future data in volatility filters or recursive forecasts.
	114	+
	115	+For multi-step forecasts, follow exact model-specific conditional expectation or clearly justified simulation; do not extrapolate one-step predictions naively.
	116	+
	117	+Parameter selection only in training/validation, never via final test losses. Publish tuning grids and seeds.
	118	+
	119	+Forecast persistence baselines, GARCH(1,1), asymmetric GARCH and HAR-RV when realized information is available.
	120	+
	121	+Primary and secondary accuracy
	122	+
	123	+Primary: QLIKE for strictly positive variance forecasts and a documented positive variance proxy: QLIKE(y,h) = y/h - log(y/h) - 1 for y>0,h>0; for proxy zeros use the equivalent loss log(h)+y/h, whose omitted y-only terms do not affect forecast rankings. Record exact implementation and numerical handling.
	124	+
	125	+Secondary: MSE of variance forecasts, MAE, volatility-scale losses as sensitivity analyses, log predictive score where a full conditional density is supplied, and calibration/Mincer–Zarnowitz checks.
	126	+
	127	+Evaluate imperfect volatility-proxy sensitivity (different intraday sampling intervals, semivariances, robust RV, squared daily returns as noisy diagnostic). Cite Patton (2011), acknowledging proxy assumptions.
	128	+
	129	+Tail risk: 1% and 5% conditional VaR and relevant expected shortfall evaluations using statistically appropriate unconditional/conditional coverage and joint VaR–ES scoring/backtests. Respect innovation distribution assumptions.
	130	+
	131	+Risk covariance: use multivariate loss/portfolio risk diagnostics only for valid positive-definite covariance forecasts.
	132	+
	133	+Inferential rigor
	134	+
	135	+Apply Diebold–Mariano-style comparisons with horizon/serial-correlation-appropriate variance estimators, Hansen's SPA/White-style multiple-comparison controls where relevant, and Model Confidence Set (Hansen–Lunde–Nason) with justified stationary/block bootstrap settings. Report confidence intervals, adjusted p-values, effect sizes, MCS memberships and sample sizes. Distinguish statistical significance from economic materiality. Cluster or bootstrap across assets/time where cross-sectional dependence matters. No global leaderboard without discussing dependence and multiplicity.
	136	+
	137	+Regime analysis
	138	+
	139	+Predefine market stress classification independently of future outcomes and avoid endogenous arbitrary crisis windows. Report tranquil/stressed markets, COVID-type episodes if covered, equity drawdowns, volatility quantiles, positive/negative return shocks and asset classes; sensitivity to regime definitions is mandatory.
	140	+
	141	+Computation, failures and fairness
	142	+
	143	+Measure median runtime, total CPU time, peak resident memory, iteration counts, optimization failures, non-positive variance, nonstationary or nonadmissible estimates and forecast coverage. Define a common fallback policy: bounded retry sequence, alternative starting values, optimizer diagnostics; never replace a failed complex forecast with a baseline's output without explicit flagging. Report both accuracy on common-success sample and failure-adjusted operational performance to avoid survivorship bias. Distinguish analytic, quasi-MLE, simulation and Bayesian compute budgets.
	144	+
	145	+Mathematics and software verification
	146	+
	147	+Implement numerical gradients/Hessians with stable constrained transforms, e.g. log/softplus for positive parameters as appropriate, and exact stationarity domains; do not assume alpha+beta<1 universally applies to EGARCH, IGARCH or fractional models. Parameter inference must respect innovation moments, identification and nonregular boundaries. Reconcile probability density conventions for skewed distributions.
	148	+
	149	+Before accepting an implementation:
	150	+
	151	+Unit tests of recursion, likelihood, admissibility, multi-step forecasts and seeded simulation recovery.
	152	+
	153	+Numerical gradient checks where practical; log-likelihood comparison against at least one authoritative library or original paper replication on a shared synthetic data fixture.
	154	+
	155	+Edge cases: constant returns, extreme shocks, nearly integrated persistence, tiny positive variance, short samples and missingness.
	156	+
	157	+Monte Carlo calibration tests across distributions and parameter regimes.
	158	+
	159	+Diagnostic plots and residual tests; no silent overflow, underflow or discarded warnings.
	160	+
	161	+Independent check of publication references and model equations.
	162	+
	163	+Use established vetted libraries (e.g. Python arch, R rugarch, rmgarch, and appropriate realized/regime-specialist packages) where they faithfully implement the cited model; version-lock and cross-check them. Custom implementations must justify necessity and pass stronger simulation and replication tests. Pin environments and document any cross-language interfaces.
	164	+
	165	+Research repository structure
	166	+
	167	+Adapt to the actual repository conventions rather than forcibly recreating existing structure:
	168	+
	169	+wp46_garch_benchmark/
	170	+ CLAUDE.md
	171	+ README.md
	172	+ pyproject.toml / renv.lock (as appropriate)
	173	+ configs/
	174	+ study.yaml
	175	+ instruments.yaml
	176	+ model_registry.yaml
	177	+ forecast_protocol.yaml
	178	+ docs/
	179	+ findata_audit.md
	180	+ literature_matrix.md
	181	+ preregistered_protocol.md
	182	+ feasibility_report.md
	183	+ methodology_decisions.md
	184	+ data_dictionary.md
	185	+ replication_guide.md
	186	+ data/
	187	+ README.md
	188	+ manifests/
	189	+ raw/ # ignored; licensed/private
	190	+ interim/ # ignored
	191	+ processed/ # ignored unless permitted
	192	+ synthetic/
	193	+ src/wp46/
	194	+ findata_client/
	195	+ calendars/
	196	+ preprocessing/
	197	+ realized_measures/
	198	+ models/
	199	+ estimation/
	200	+ forecasts/
	201	+ scoring/
	202	+ inference/
	203	+ diagnostics/
	204	+ reporting/
	205	+ tests/
	206	+ unit/
	207	+ integration/
	208	+ simulation/
	209	+ replication/
	210	+ model_cards/
	211	+ experiments/
	212	+ results/
	213	+ manifests/
	214	+ metrics/
	215	+ figures/
	216	+ tables/
	217	+ logs/
	218	+ paper/
	219	+ main.tex
	220	+ sections/
	221	+ appendices/
	222	+ figures/
	223	+ tables/
	224	+ references.bib
	225	+ scripts/
	226	+ audit_findata.py
	227	+ run_pilot.py
	228	+ run_benchmark.py
	229	+ build_paper.py
	230	+
	231	+Use stable forecast keys (dataset_version, instrument_id, model_id, specification_id, origin_timestamp, horizon, estimation_window, information_tier, code_revision) and record training/test cutoffs. Save machine-readable run manifests with git SHA, package versions, hardware, random seeds and data hashes. Large data and secrets stay out of git.
	232	+
	233	+Publication-quality exhibits
	234	+
	235	+Create compelling, precise, journal-ready vector graphics and LaTeX tables; use accessible color palettes, consistent typography and no misleading truncated axes:
	236	+
	237	+Historical timeline and ontology of ARCH–GARCH architectures.
	238	+
	239	+Model equations/taxonomy table and parameter/estimation comparison.
	240	+
	241	+Coverage map and sample descriptive statistics.
	242	+
	243	+Volatility time series with documented crises and realized proxies.
	244	+
	245	+Cross-asset × model-family heatmaps of out-of-sample QLIKE improvement versus GARCH(1,1).
	246	+
	247	+Model Confidence Set membership matrices and uncertainty annotations.
	248	+
	249	+Horizon-dependent forecast performance and calibration.
	250	+
	251	+VaR/ES calibration and exceedance diagnostics.
	252	+
	253	+Complexity versus incremental accuracy Pareto charts, with failure rate and compute cost.
	254	+
	255	+Regime-specific rankings and sensitivity analyses.
	256	+
	257	+Forecast failure/convergence diagnostics and robust versus fragile models.
	258	+
	259	+Reproducibility flow and audited dataset lineage.
	260	+
	261	+Avoid displaying a large undifferentiated ranking across incomparable information tiers or metrics. Put detailed full-model tables in appendices and public machine-readable supplements.
	262	+
	263	+Paper structure
	264	+
	265	+Introduction: motivation, hypotheses, contributions, and relation to past broad benchmarks.
	266	+
	267	+Related literature: ARCH/GARCH taxonomy, realized volatility, model selection and empirical forecasting comparisons.
	268	+
	269	+Unified model taxonomy and mathematical framework.
	270	+
	271	+Data, market coverage and realized variance measurement.
	272	+
	273	+Experimental design, estimators, forecasting and inference.
	274	+
	275	+Main daily-information comparison.
	276	+
	277	+High-frequency/realized-measure comparison.
	278	+
	279	+Tail risk and multivariate extensions.
	280	+
	281	+Cross-market, regime and horizon heterogeneity.
	282	+
	283	+Complexity–accuracy–robustness frontier and practical implications.
	284	+
	285	+Limitations, reproducibility and conclusion.
	286	+
	287	+Appendices: complete formulas/regularity conditions, model cards, instrument provenance, simulation verification, extra losses, sensitivity tables, multiple testing, and implementation details.
	288	+
	289	+Draft the paper in LaTeX with latexmk, biblatex/BibTeX consistent with repository convention, all tables and figures generated by source code, references checked against primary sources. The paper must compile cleanly and cite every nontrivial methodological claim. Do not optimize for length at the expense of substantive rigor; keep the main paper readable and publish detailed technical appendices.
	290	+
	291	+Required phases and acceptance gates
	292	+
	293	+Phase 0 — Inventory and literature. Repository inspection, Findata access audit, literature matrix, novelty assessment. Gate: evidence-backed viable data/instrument list; no invented endpoints.
	294	+
	295	+Phase 1 — Preregistered protocol. Freeze primary design, model taxonomy, metrics, date splits and robustness analysis. Gate: protocol document and model registry reviewed before test-set evaluation.
	296	+
	297	+Phase 2 — Trusted data and pilot. Ingest reproducibly; inspect samples; implement GARCH(1,1), EGARCH, GJR-GARCH, APARCH and HAR-RV if data permit. Gate: leakage tests, exact units, simulated recovery and full 1-day OOS pilot.
	298	+
	299	+Phase 3 — Broad univariate benchmark. Complete feasible canonical classes and innovation distributions. Gate: model cards, convergence accounting, tests and replicable summaries for every accepted implementation.
	300	+
	301	+Phase 4 — Realized and multivariate tiers. Implement specialist models with separate fair comparisons. Gate: verified forecast-target alignment and positive-definite covariance checks.
	302	+
	303	+Phase 5 — Statistical inference. Multiplicity, MCS, block bootstrap, tail risk, regime and sensitivity studies. Gate: no interpretation without uncertainty and common-sample validation.
	304	+
	305	+Phase 6 — Manuscript and artifacts. Fully compiled paper, technical appendices, repository README, replication scripts, tables, figures and limitations. Gate: a fresh-environment smoke reproduction and an independent audit checklist.
	306	+
	307	+At each gate, maintain progress.md with completed commands, timestamps, test outcomes, blocked dependencies, scientifically important decisions and next actions. If blocked, produce an actionable issue list rather than pretending the phase is complete.
	308	+
	309	+Mandatory integrity rules
	310	+
	311	+No use of the final test set for selecting the model grid, optimizer, best regime definitions or loss function.
	312	+
	313	+No comparison of different forecast horizons, proxies, return scaling or information sets without explicit reconciliation.
	314	+
	315	+No cherry-picking assets, regimes, failures or plots to make a favored model win.
	316	+
	317	+No reliance on AIC/BIC or in-sample likelihood alone to claim forecast superiority.
	318	+
	319	+No claiming high-frequency RV is an unbiased latent variance proxy without documenting assumptions and sensitivity.
	320	+
	321	+No reference included without bibliographic verification; no fabricated URLs or DOIs.
	322	+
	323	+No unsupported blanket conclusion like "GARCH is obsolete" or "neural models always win."
	324	+
	325	+No private keys, raw licensed data or sensitive logs in commits or manuscript artifacts.
	326	+
	327	+No automatic external publication, repository push, submission or expensive cloud spend without explicit authorization.
	328	+
	329	+Keep a clearly marked distinction between substantive scientific findings and exploratory observations.
	330	+
	331	+Seminal literature to verify and expand
	332	+
	333	+Engle (1982), autoregressive conditional heteroskedasticity.
	334	+Bollerslev (1986), generalized ARCH.
	335	+Nelson (1991), EGARCH.
	336	+Glosten, Jagannathan and Runkle (1993), asymmetric volatility relation.
	337	+Ding, Granger and Engle (1993), long-memory volatility/APARCH family.
	338	+Baillie, Bollerslev and Mikkelsen (1996), FIGARCH.
	339	+Engle (2002), dynamic conditional correlation.
	340	+Hansen and Lunde (2005), A Forecast Comparison of Volatility Models: Does Anything Beat a GARCH(1,1)?, Journal of Applied Econometrics, DOI 10.1002/jae.800.
	341	+Shephard and Sheppard (2010), Realising the Future: Forecasting with High-Frequency-Based Volatility (HEAVY) Models, Journal of Applied Econometrics, DOI 10.1002/jae.1158.
	342	+Patton (2011), Volatility Forecast Comparison Using Imperfect Volatility Proxies, Journal of Econometrics.
	343	+Hansen, Lunde and Nason (2011), The Model Confidence Set, Econometrica, DOI 10.3982/ECTA5771.
	344	+Hansen, Huang and Shek (2012), Realized GARCH: A Joint Model of Returns and Realized Measures of Volatility, Journal of Applied Econometrics.
	345	+
	346	+Verify bibliographic details and actively extend through the current literature cutoff; this list is a starting point, not an exhaustive or final review.
	347	+
	348	+Start now
	349	+
	350	+Begin with repository reconnaissance, a documented Findata audit and a rigorous literature/novelty map. Then write the preregistered protocol and implement a validated pilot before scaling. Show the user a concise Phase 0–2 progress report with concrete evidence, open risks and the exact commands necessary to reproduce. Proceed autonomously within permitted local research operations but stop and ask for missing credentials, material dataset-selection decisions, unexpected paid compute, or external publication permissions.

added docs/findata_audit.md +184 −0

		@@ -0,0 +1,184 @@
	1	+# FinData OS audit for WP46 (Phase 0)
	2	+
	3	+Audited 2026-11-09T17:35:13.295342+00:00 against `https://findata.spboucher.ai/api/v1` by `scripts/audit_findata.py`. Machine-readable evidence: `results/manifests/findata_audit.json`; call log: `results/logs/findata_audit_provenance.jsonl`. The hostname `findata.spboucher.ai` (not `indata.`) is the one configured in the owner's FinData repository and its `docs/CLAUDE-API-USAGE.md`; the API contract is the live OpenAPI document (`/api/v1/openapi.json`) — no endpoint below was guessed.
	4	+
	5	+## 1. Service and contract
	6	+
	7	+- Health status: `degraded`; alerts: ['dead_jobs']; components: `{'postgres': 'ok', 'backup': '…', 'ingest_worker': 'running', 'lake': '…', 'ssd_smart': 'Verified'}`.
	8	+- OpenAPI 3.1.0, 41 paths. Used by WP46: `/catalog/datasets`, `/catalog/datasets/{id}`, `/catalog/schema/bars`, `/instruments/{symbol}`, `/market/bars`, `/query` (DuckDB SQL over the lake views), `/exports` (+ `/exports/{id}`, `/download`), `/health`.
	9	+- Bars schema: `{"schema_version": 1, "timestamp_convention": "bar_open", "timezone_storage": "UTC", "fields": [{"name": "instrument_id", "type": "string", "nullable": false}, {"name": "symbol_as_reported", "type": "string", "nullable": false}, {"name": "venue", "type": "string", "nullable": true}, {"name": "asset_class", "type": "string", "nullable": false}, {"name": "interval", "type": "string", "nullable": false}, {"name": "timestamp_utc", "type": "timestamp[us, tz=UTC]", "nullable": false}, {"name": "session_date", "type": "date32[day]", "nullable": false}, {"name": "open", "type": "double", "nullable": f`
	10	+- Response headers observed: `{'x-content-type-options': 'nosniff', 'x-frame-options': 'DENY', 'x-request-id': 'd22bdb63ebd644b9'}`. Rate limit (documented): 600 requests/min per key; the client uses a 5 req/s token bucket with Retry-After/backoff.
	11	+- Authentication: bearer API key read from the owner's secret file; key scopes used: read:market, query, export. Never logged.
	12	+
	13	+## 2. Datasets (catalog)
	14	+
	15	+| dataset | status | rows | first session | last session | version |
	16	+|---|---|---:|---|---|---:|
	17	+| `corporate_actions.stock` | active | 0 | | | 0 |
	18	+| `market_bars.stock.5m.raw` | active | 2,062,205,350 | 2000-01-03 | 2026-11-06 | 43 |
	19	+| `market_bars.crypto.1d.raw` | active | 181,957 | | | 75 |
	20	+| `market_bars.crypto.1h.raw` | active | 4,098,204 | | | 5 |
	21	+| `market_bars.crypto.1m.raw` | active | 177,377,622 | | | 83 |
	22	+| `market_bars.crypto.30m.raw` | active | 7,970,148 | | | 5 |
	23	+| `market_bars.crypto.5m.raw` | active | 42,849,352 | | | 5 |
	24	+| `market_bars.etf.1d.raw` | active | 9,073,895 | 2000-01-03 | 2026-11-06 | 5167 |
	25	+| `market_bars.etf.1d.split` | active | 9,051,347 | | | 270 |
	26	+| `market_bars.etf.1d.split_div` | active | 9,051,347 | | | 270 |
	27	+| `market_bars.etf.1h.split` | active | 55,703,678 | | | 270 |
	28	+| `market_bars.etf.1h.split_div` | active | 55,703,678 | | | 270 |
	29	+| `market_bars.etf.1m.raw` | active | 910,257,088 | 2000-01-03 | 2026-11-06 | 5167 |
	30	+| `market_bars.etf.1m.split` | active | 907,197,376 | | | 5161 |
	31	+| `market_bars.etf.1m.split_div` | active | 907,197,376 | | | 5161 |
	32	+| `market_bars.etf.30m.split` | active | 91,337,816 | | | 270 |
	33	+| `market_bars.etf.30m.split_div` | active | 91,337,816 | | | 270 |
	34	+| `market_bars.etf.5m.split` | active | 330,637,732 | | | 270 |
	35	+| `market_bars.etf.5m.split_div` | active | 330,637,732 | | | 270 |
	36	+| `market_bars.futures_contracts.1d.raw` | active | 8,430,644 | | | 20967 |
	37	+| `market_bars.futures_contracts.1h.raw` | active | 30,077,653 | | | 997 |
	38	+| `market_bars.futures_contracts.1m.raw` | active | 615,553,277 | | | 19699 |
	39	+| `market_bars.futures_contracts.30m.raw` | active | 51,126,242 | | | 997 |
	40	+| `market_bars.futures_contracts.5m.raw` | active | 199,339,369 | | | 997 |
	41	+| `market_bars.futures.1d.continuous_absolute` | active | 526,324 | | | 133 |
	42	+| `market_bars.futures.1d.continuous_ratio` | active | 526,324 | | | 133 |
	43	+| `market_bars.futures.1d.continuous_raw` | active | 526,324 | | | 133 |
	44	+| `market_bars.futures.1h.continuous_absolute` | active | 8,317,926 | | | 8 |
	45	+| `market_bars.futures.1h.continuous_ratio` | active | 8,317,926 | | | 8 |
	46	+| `market_bars.futures.1h.continuous_raw` | active | 8,317,926 | | | 8 |
	47	+| `market_bars.futures.1m.continuous_absolute` | active | 302,853,229 | | | 132 |
	48	+| `market_bars.futures.1m.continuous_ratio` | active | 302,853,229 | | | 132 |
	49	+| `market_bars.futures.1m.continuous_raw` | active | 302,853,229 | | | 132 |
	50	+| `market_bars.futures.30m.continuous_absolute` | active | 15,617,848 | | | 8 |
	51	+| `market_bars.futures.30m.continuous_ratio` | active | 15,617,848 | | | 8 |
	52	+| `market_bars.futures.30m.continuous_raw` | active | 15,617,848 | | | 8 |
	53	+| `market_bars.futures.5m.continuous_absolute` | active | 78,211,693 | | | 8 |
	54	+| `market_bars.futures.5m.continuous_ratio` | active | 78,211,693 | | | 8 |
	55	+| `market_bars.futures.5m.continuous_raw` | active | 78,211,693 | | | 8 |
	56	+| `market_bars.fx.1d.raw` | planned | 0 | | | 0 |
	57	+| `market_bars.fx.1h.raw` | active | 7,635,724 | | | 5 |
	58	+| `market_bars.fx.1m.raw` | active | 407,864,391 | | | 85 |
	59	+| `market_bars.fx.30m.raw` | active | 15,141,666 | | | 5 |
	60	+| `market_bars.fx.5m.raw` | active | 87,304,277 | | | 5 |
	61	+| `market_bars.index.1d.raw` | active | 556,868 | | | 124 |
	62	+| `market_bars.index.1h.raw` | active | 3,963,294 | | | 8 |
	63	+| `market_bars.index.1m.raw` | active | 203,832,449 | 2008-01-02 | 2026-11-06 | 128 |
	64	+| `market_bars.index.30m.raw` | active | 7,206,595 | | | 8 |
	65	+| `market_bars.index.5m.raw` | active | 41,215,377 | | | 8 |
	66	+| `market_bars.stock.1d.raw` | active | 21,088,134 | 2000-01-03 | 2026-11-06 | 7757 |
	67	+| `market_bars.stock.1d.split` | active | 38,316,176 | | | 827 |
	68	+| `market_bars.stock.1d.split_div` | active | 38,103,477 | | | 828 |
	69	+| `market_bars.stock.1h.split` | active | 286,709,713 | | | 840 |
	70	+| `market_bars.stock.1h.split_div` | active | 285,019,420 | | | 831 |
	71	+| `market_bars.stock.1m.raw` | active | 6,702,645,689 | 2000-01-03 | 2026-11-06 | 17382 |
	72	+| `market_bars.stock.1m.split` | active | 6,657,098,101 | | | 14008 |
	73	+| `market_bars.stock.1m.split_div` | active | 6,578,579,087 | | | 13811 |
	74	+| `market_bars.stock.30m.split` | active | 485,858,160 | | | 840 |
	75	+| `market_bars.stock.30m.split_div` | active | 482,672,255 | | | 831 |
	76	+| `market_bars.stock.5m.split` | active | 2,069,201,312 | | | 840 |
	77	+| `market_bars.stock.5m.split_div` | active | 2,053,780,780 | | | 831 |
	78	+| `options.eod` | active | 4,232,091,733 | | | 361 |
	79	+
	80	+### Dataset conventions (descriptor text)
	81	+
	82	+- `market_bars.etf.1m.raw`: {'name': 'US ETFs — 1-minute bars (unadjusted)', 'description': 'Same contract as stocks.', 'timestamp_convention': 'bar_open', 'timezone': 'America/New_York', 'notes': None, 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 5167, 'status': 'active'}
	83	+- `market_bars.etf.1d.split_div`: {'name': 'US ETFs — daily bars (split- and dividend-adjusted (vendor adj_splitdiv))', 'description': 'Vendor daily OHLCV bars, split- and dividend-adjusted (vendor adj_splitdiv). Includes delisted tickers (suffix -DELISTED) when their archives are ingested. Vendor-adjusted history is rewritten after corporate actions; refreshed weekly via full_update.', 'timestamp_convention': 'session', 'timezone': 'America/New_York', 'notes': None, 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 270, 'status': 'active'}
	84	+- `market_bars.stock.1m.raw`: {'name': 'US stocks — 1-minute bars (unadjusted)', 'description': 'Raw (unadjusted) 1-minute OHLCV bars for US-listed stocks, extended hours 04:00–20:00 ET, zero-volume bars absent at source. Includes delisted tickers (suffix -DELISTED).', 'timestamp_convention': 'bar_open', 'timezone': 'America/New_York', 'notes': 'Canonical dataset; adjustments are applied at query time from corporate_actions.', 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 17382, 'status': 'active'}
	85	+- `market_bars.stock.1d.split_div`: {'name': 'US stocks — daily bars (split- and dividend-adjusted (vendor adj_splitdiv))', 'description': 'Vendor daily OHLCV bars, split- and dividend-adjusted (vendor adj_splitdiv). Includes delisted tickers (suffix -DELISTED) when their archives are ingested. Vendor-adjusted history is rewritten after corporate actions; refreshed weekly via full_update.', 'timestamp_convention': 'session', 'timezone': 'America/New_York', 'notes': None, 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 828, 'status': 'active'}
	86	+- `market_bars.futures.1m.continuous_ratio`: {'name': 'Futures — 1-minute continuous, ratio back-adjusted', 'description': 'Vendor continuous series per root (continuous, ratio back-adjusted); roll dates fixed by vendor.', 'timestamp_convention': 'bar_open', 'timezone': 'America/New_York', 'notes': 'Daily bars follow the contract trading day; absolute-adjusted series may be negative.', 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 132, 'status': 'active'}
	87	+- `market_bars.futures.1d.continuous_ratio`: {'name': 'Futures — daily continuous, ratio back-adjusted', 'description': 'Vendor continuous series per root (continuous, ratio back-adjusted); roll dates fixed by vendor.', 'timestamp_convention': 'session', 'timezone': 'America/New_York', 'notes': 'Daily bars follow the contract trading day; absolute-adjusted series may be negative.', 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 133, 'status': 'active'}
	88	+- `market_bars.futures.1d.continuous_absolute`: {'name': 'Futures — daily continuous, absolute back-adjusted', 'description': 'Vendor continuous series per root (continuous, absolute back-adjusted); roll dates fixed by vendor.', 'timestamp_convention': 'session', 'timezone': 'America/New_York', 'notes': 'Daily bars follow the contract trading day; absolute-adjusted series may be negative.', 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 133, 'status': 'active'}
	89	+- `market_bars.fx.1m.raw`: {'name': 'FX majors — 1-minute bars', 'description': 'Aggregated from four money-center banks.', 'timestamp_convention': 'bar_open', 'timezone': 'America/New_York', 'notes': 'Two-field datetime in source files.', 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 85, 'status': 'active'}
	90	+- `market_bars.crypto.1m.raw`: {'name': 'Crypto — 1-minute bars', 'description': 'UTC timestamps, 7 days a week.', 'timestamp_convention': 'bar_open', 'timezone': 'UTC', 'notes': None, 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 83, 'status': 'active'}
	91	+- `market_bars.index.1d.raw`: {'name': 'Indices — daily bars', 'description': 'Vendor daily bars for indices. No volume (stored as 0).', 'timestamp_convention': 'session', 'timezone': 'America/New_York', 'notes': None, 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 124, 'status': 'active'}
	92	+- `corporate_actions.stock`: {'name': 'Splits and cash dividends', 'description': 'From vendor meta files; mergers/spin-offs not covered.', 'timestamp_convention': None, 'timezone': None, 'notes': None, 'license_class': 'frd-private', 'license_text': 'FirstRate Data license: private use by the subscriber only. Raw bars/tables may not be redistributed to third parties (no public API, CSV or tables). Derived results and charts may be published with attribution to FirstRate Data.', 'current_version': 0, 'status': 'active'}
	93	+
	94	+## 3. Candidate universe coverage
	95	+
	96	+| class | symbol | name | daily sessions | daily first | daily last | bad close | 1-min sessions | 1-min first | 1-min last | 1-min bars |
	97	+|---|---|---|---:|---|---|---:|---:|---|---|---:|
	98	+| etf | SPY | SPDR S&P 500 | 6748 | 2000-01-03 | 2026-10-30 | 0 | 6753 | 2000-01-03 | 2026-11-06 | 4675764 |
	99	+| etf | QQQ | PowerShares Qqq | 6711 | 2000-02-28 | 2026-10-30 | 0 | 6753 | 2000-01-03 | 2026-11-06 | 4293565 |
	100	+| etf | IWM | iShares Russell 2000 Index | 6646 | 2000-05-30 | 2026-10-30 | 0 | 6652 | 2000-05-26 | 2026-11-06 | 3475724 |
	101	+| etf | DIA | SPDR Dow Jones Industrial Average | 6748 | 2000-01-03 | 2026-10-30 | 0 | 6753 | 2000-01-03 | 2026-11-06 | 3043772 |
	102	+| etf | EEM | iShares MSCI Emerging Markets Index | 5926 | 2003-04-14 | 2026-10-30 | 0 | 5931 | 2003-04-14 | 2026-11-06 | 2514085 |
	103	+| etf | EFA | iShares MSCI Eafe Index | 6332 | 2001-08-27 | 2026-10-30 | 0 | 6337 | 2001-08-27 | 2026-11-06 | 2377628 |
	104	+| etf | XLF | Financial Select Sector SPDR | 6748 | 2000-01-03 | 2026-10-30 | 0 | 6753 | 2000-01-03 | 2026-11-06 | 2748952 |
	105	+| etf | XLE | Energy Select Sector SPDR | 6748 | 2000-01-03 | 2026-10-30 | 0 | 6753 | 2000-01-03 | 2026-11-06 | 2665653 |
	106	+| etf | TLT | iShares Barclays 20+ Year Treas Bond | 6104 | 2002-07-30 | 2026-10-30 | 0 | 6109 | 2002-07-30 | 2026-11-06 | 2660987 |
	107	+| etf | IEF | iShares Lehman 7-10 Year Treasu | 6104 | 2002-07-30 | 2026-10-30 | 0 | 6109 | 2002-07-30 | 2026-11-06 | 1883934 |
	108	+| etf | HYG | iShares Iboxx $ High Yield Corporate Bd | 4922 | 2007-04-11 | 2026-10-30 | 0 | 4926 | 2007-04-11 | 2026-11-06 | 1812239 |
	109	+| etf | LQD | iShares Iboxx $ Invest Grade Bond | 6104 | 2002-07-30 | 2026-10-30 | 0 | 6109 | 2002-07-30 | 2026-11-06 | 1947361 |
	110	+| etf | GLD | SPDR Gold Shares | 5522 | 2004-11-18 | 2026-10-30 | 0 | 5527 | 2004-11-18 | 2026-11-06 | 2878048 |
	111	+| etf | SLV | iShares Silver Trust | 5160 | 2006-04-28 | 2026-10-30 | 0 | 5165 | 2006-04-28 | 2026-11-06 | 2641783 |
	112	+| etf | USO | United States Oil | 5173 | 2006-04-10 | 2026-10-30 | 0 | 5178 | 2006-04-10 | 2026-11-06 | 2542011 |
	113	+| stock | AAPL | Apple | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 4184674 |
	114	+| stock | MSFT | Microsoft | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 3568310 |
	115	+| stock | AMZN | Amazon.com | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 3435049 |
	116	+| stock | NVDA | Nvidia Corporation | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 3609152 |
	117	+| stock | JPM | JP Morgan Chase & Co. | 6637 | 2000-06-12 | 2026-10-30 | 0 | 6642 | 2000-06-12 | 2026-11-06 | 2983348 |
	118	+| stock | XOM | Exxon Mobil | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 3031298 |
	119	+| stock | JNJ | Johnson & Johnson | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 2764519 |
	120	+| stock | PG | Procter & Gamble | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 2744358 |
	121	+| stock | KO | Coca-Cola Company | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 2839222 |
	122	+| stock | WMT | Walmart | 6747 | 2000-01-04 | 2026-10-30 | 0 | 6752 | 2000-01-04 | 2026-11-06 | 2885451 |
	123	+| futures | ES | Eversource Energy | 4754 | 2008-01-02 | 2026-11-06 | 0 | 5861 | 2008-01-02 | 2026-11-06 | 6596585 |
	124	+| futures | NQ | | 4754 | 2008-01-02 | 2026-11-06 | 0 | 5861 | 2008-01-02 | 2026-11-06 | 6262856 |
	125	+| futures | CL | Colgate-Palmolive | 4749 | 2008-01-02 | 2026-11-06 | 0 | 5863 | 2008-01-02 | 2026-11-06 | 6478865 |
	126	+| futures | NG | Novagold Resources | 4709 | 2008-02-25 | 2026-11-06 | 0 | 5818 | 2008-02-24 | 2026-11-06 | 5186395 |
	127	+| futures | GC | | 4732 | 2008-01-28 | 2026-11-06 | 0 | 5840 | 2008-01-27 | 2026-11-06 | 6571988 |
	128	+| futures | SI | Shoulder Innovations | 4749 | 2008-01-02 | 2026-11-06 | 0 | 5863 | 2008-01-02 | 2026-11-06 | 5981619 |
	129	+| futures | HG | Hamilton Insurance | 4749 | 2008-01-02 | 2026-11-06 | 0 | 5863 | 2008-01-02 | 2026-11-06 | 5795513 |
	130	+| futures | ZN | | 4754 | 2008-01-02 | 2026-11-06 | 0 | 5860 | 2008-01-02 | 2026-11-06 | 5688309 |
	131	+| futures | ZC | | 4724 | 2008-02-19 | 2026-11-06 | 0 | 5684 | 2008-02-19 | 2026-11-06 | 3300107 |
	132	+| fx | EURUSD | | | | | | 5257 | 2010-01-03 | 2026-11-06 | 6251085 |
	133	+| fx | USDJPY | | | | | | 5255 | 2010-01-03 | 2026-11-06 | 6244770 |
	134	+| fx | GBPUSD | | | | | | 5255 | 2010-01-03 | 2026-11-06 | 6243077 |
	135	+| fx | AUDUSD | | | | | | 5255 | 2010-01-03 | 2026-11-06 | 6239239 |
	136	+| fx | USDCAD | | | | | | 5254 | 2010-01-03 | 2026-11-06 | 6211934 |
	137	+| fx | USDCHF | | | | | | 5257 | 2010-01-03 | 2026-11-06 | 6212109 |
	138	+| crypto | BTC | Grayscale Bitcoin Mini Trust | 4967 | 2013-04-01 | 2026-11-06 | 0 | 4970 | 2013-04-01 | 2026-11-08 | 6402725 |
	139	+| crypto | ETH | Ethan Allen Interiors | 3885 | 2016-03-11 | 2026-11-06 | 0 | 3888 | 2016-03-11 | 2026-11-08 | 5270106 |
	140	+
	141	+FX has no daily dataset in the lake (`market_bars.fx.1d.raw` is `planned`, 0 rows): daily FX returns are built from the 1-minute bars (last bar of each vendor session). Futures daily rows come from the vendor continuous series.
	142	+
	143	+## 4. Session structure (2024, one representative symbol per class)
	144	+
	145	+- **etf / SPY**: {'n_sessions': 252, 'min_bars': 537, 'med_bars': 782, 'max_bars': 948, 'first_bar_utc_min': '2024-01-02T09:00:00+00:00', 'last_bar_utc_max': '2025-01-01T00:59:00+00:00'}; sample days: [{'session_date': '2024-03-08', 'n': 794, 't0': '2024-03-08T09:00:00+00:00', 't1': '2024-03-09T00:59:00+00:00'}, {'session_date': '2024-03-11', 'n': 805, 't0': '2024-03-11T08:00:00+00:00', 't1': '2024-03-11T23:59:00+00:00'}, {'session_date': '2024-07-03', 'n': 575, 't0': '2024-07-03T08:00:00+00:00', 't1': '2024-07-03T20:55:00+00:00'}, {'session_date': '2024-11-29', 'n': 579, 't0': '2024-11-29T09:00:00+00:00', 't1': '2024-11-29T21:59:00+00:00'}]
	146	+- **stock / AAPL**: {'n_sessions': 252, 'min_bars': 448, 'med_bars': 743, 'max_bars': 926, 'first_bar_utc_min': '2024-01-02T09:00:00+00:00', 'last_bar_utc_max': '2025-01-01T00:58:00+00:00'}; sample days: [{'session_date': '2024-03-08', 'n': 835, 't0': '2024-03-08T09:00:00+00:00', 't1': '2024-03-09T00:59:00+00:00'}, {'session_date': '2024-03-11', 'n': 821, 't0': '2024-03-11T08:00:00+00:00', 't1': '2024-03-11T23:59:00+00:00'}, {'session_date': '2024-07-03', 'n': 591, 't0': '2024-07-03T08:00:00+00:00', 't1': '2024-07-03T20:59:00+00:00'}, {'session_date': '2024-11-29', 'n': 467, 't0': '2024-11-29T09:00:00+00:00', 't1': '2024-11-29T21:59:00+00:00'}]
	147	+- **futures / ES**: {'n_sessions': 313, 'min_bars': 354, 'med_bars': 1379, 'max_bars': 1380, 'first_bar_utc_min': '2024-01-01T23:00:00+00:00', 'last_bar_utc_max': '2024-12-31T21:59:00+00:00'}; sample days: [{'session_date': '2024-03-08', 'n': 1019, 't0': '2024-03-08T05:00:00+00:00', 't1': '2024-03-08T21:59:00+00:00'}, {'session_date': '2024-03-11', 'n': 1374, 't0': '2024-03-11T04:00:00+00:00', 't1': '2024-03-12T03:59:00+00:00'}, {'session_date': '2024-07-03', 'n': 1151, 't0': '2024-07-03T04:00:00+00:00', 't1': '2024-07-04T03:59:00+00:00'}, {'session_date': '2024-07-07', 'n': 360, 't0': '2024-07-07T22:00:00+00:00', 't1': '2024-07-08T03:59:00+00:00'}, {'session_date': '2024-11-29', 'n': 795, 't0': '2024-11-29T05:00:00+00:00', 't1': '2024-11-29T18:14:00+00:00'}]
	148	+- **fx / EURUSD**: {'n_sessions': 312, 'min_bars': 393, 'med_bars': 1433, 'max_bars': 1440, 'first_bar_utc_min': '2024-01-02T05:01:00+00:00', 'last_bar_utc_max': '2024-12-31T20:00:00+00:00'}; sample days: [{'session_date': '2024-03-08', 'n': 1020, 't0': '2024-03-08T05:00:00+00:00', 't1': '2024-03-08T21:59:00+00:00'}, {'session_date': '2024-03-11', 'n': 1437, 't0': '2024-03-11T04:00:00+00:00', 't1': '2024-03-12T03:59:00+00:00'}, {'session_date': '2024-07-03', 'n': 1434, 't0': '2024-07-03T04:00:00+00:00', 't1': '2024-07-04T03:59:00+00:00'}, {'session_date': '2024-07-07', 'n': 419, 't0': '2024-07-07T21:00:00+00:00', 't1': '2024-07-08T03:59:00+00:00'}, {'session_date': '2024-11-29', 'n': 1019, 't0': '2024-11-29T05:00:00+00:00', 't1': '2024-11-29T21:58:00+00:00'}]
	149	+- **crypto / BTC**: {'n_sessions': 366, 'min_bars': 1010, 'med_bars': 1440, 'max_bars': 1440, 'first_bar_utc_min': '2024-01-01T00:00:00+00:00', 'last_bar_utc_max': '2024-12-31T23:58:00+00:00'}; sample days: [{'session_date': '2024-03-08', 'n': 1440, 't0': '2024-03-08T00:00:00+00:00', 't1': '2024-03-08T23:59:00+00:00'}, {'session_date': '2024-03-11', 'n': 1440, 't0': '2024-03-11T00:00:00+00:00', 't1': '2024-03-11T23:59:00+00:00'}, {'session_date': '2024-07-03', 'n': 1440, 't0': '2024-07-03T00:00:00+00:00', 't1': '2024-07-03T23:59:00+00:00'}, {'session_date': '2024-07-07', 'n': 1440, 't0': '2024-07-07T00:00:00+00:00', 't1': '2024-07-07T23:59:00+00:00'}, {'session_date': '2024-11-29', 'n': 1440, 't0': '2024-11-29T00:00:00+00:00', 't1': '2024-11-29T23:59:00+00:00'}]
	150	+- SPY regular hours 09:30–16:00 ET (incl. the 16:00 closing print) 2024: {'n_sessions': 252, 'min_bars': 319, 'med_bars': 391, 'max_bars': 391}
	151	+
	152	+## 5. Calendar reconciliation
	153	+
	154	+- SPY daily sessions vs `exchange_calendars` XNYS: {'error': "DateOutOfBounds(<exchange_calendars.exchange_calendar_xnys.XNYSExchangeCalendar object at 0x10de5e7b0>, Timestamp('2000-01-03 00:00:00'), 'start')"}
	155	+
	156	+## 6. Adjustment conventions (ratios, no price levels disclosed)
	157	+
	158	+- aapl_split_2020_08_31: [{'session_date': '2020-08-28', 'ratio_raw_over_split': 4.0}, {'session_date': '2020-08-31', 'ratio_raw_over_split': 1.0}]
	159	+- spy_exdiv_2024_03_15: [{'session_date': '2024-03-14', 'ratio_split_over_splitdiv': 1.0327383781585309}, {'session_date': '2024-03-15', 'ratio_split_over_splitdiv': 1.0295496480088433}, {'session_date': '2024-03-18', 'ratio_split_over_splitdiv': 1.0295497190449032}]
	160	+- es_roll_march_2024: [{'session_date': '2024-03-11', 'ratio_ratioadj_over_raw': 1.1122799141128246, 'diff_absadj_minus_raw': 660.5}, {'session_date': '2024-03-12', 'ratio_ratioadj_over_raw': 1.112279685161041, 'diff_absadj_minus_raw': 660.5}, {'session_date': '2024-03-13', 'ratio_ratioadj_over_raw': 1.112279411764706, 'diff_absadj_minus_raw': 660.5}, {'session_date': '2024-03-14', 'ratio_ratioadj_over_raw': 1.0985684170180146, 'diff_absadj_minus_raw': 596.0}, {'session_date': '2024-03-15', 'ratio_ratioadj_over_raw': 1.0985692923640924, 'diff_absadj_minus_raw': 596.0}, {'session_date': '2024-03-18', 'ratio_ratioadj_over_raw': 1.0985684836281702, 'diff_absadj_minus_raw': 596.0}, {'session_date': '2024-03-19', 'ratio_ratioadj_over_raw': 1.0985701340201268, 'diff_absadj_minus_raw': 596.0}]
	161	+- es_latest_ratio_vs_raw_equal: [{'d': '2026-11-06', 'ratio_close_over_raw_close': 1.0}, {'d': '2026-11-05', 'ratio_close_over_raw_close': 1.0}, {'d': '2026-11-04', 'ratio_close_over_raw_close': 1.0}]
	162	+
	163	+## 7. Quality flags and revisions (daily ETF/stock candidates)
	164	+
	165	+- flags: [{'flag': 'off_interval', 'n': 91696}]; revision ids: [{'revision_id': 1, 'n': 91696}]
	166	+
	167	+## 8. Export smoke test
	168	+
	169	+- {'export_id': '919b78dc-4f5c-4b51-80a1-8e753a53be23', 'dataset_id': 'market_bars.etf.1d.split_div', 'dataset_version': 270, 'row_count': 252, 'bytes': 18126, 'sha256': 'cbb37f1924e4a6bebc80c84e2559ac62f13c591b300c6ae488a0be217158af5e'}
	170	+
	171	+## 9. Errors during the audit
	172	+
	173	+- none
	174	+
	175	+## 10. Facts, consequences and unresolved questions
	176	+
	177	+- **Timestamps** are bar OPEN in UTC; stocks/ETFs/futures/FX carry an ET `session_date`, crypto a UTC date. Stocks/ETFs: extended hours 04:00–20:00 ET are present in the 1-minute data; regular session 09:30–16:00 ET (391 bars on a full day, 211 on a 13:00 early close). Zero-volume minutes are absent (missingness is a mask, never filled).
	178	+- **Futures** continuous series: vendor-fixed roll dates (not documented per contract by the vendor); three variants (raw = unadjusted splice, ratio = ratio back-adjusted, absolute = difference back-adjusted). WP46 uses the **ratio-adjusted** series, whose log returns are free of roll jumps, and the vendor `session_date` (a calendar day in ET, 00:00–23:59 ET), not the CME trading day (18:00–17:00 ET). This is disclosed in the data section; daily futures returns are therefore 'ET-day close-to-close'.
	179	+- **FX**: 1-minute mid bars aggregated from four money-center banks (vendor description), ET session_date, 24h except the weekend gap (Friday ~17:00 ET → Sunday 17:00 ET); Sunday partial sessions exist and are dropped by the preprocessing rule (session must have ≥ 60 % of the modal bar count).
	180	+- **Crypto**: UTC days, 7 days a week, 1,440 bars per day.
	181	+- **Adjustments**: stocks/ETFs daily returns use the vendor split- and dividend-adjusted closes (`split_div`, price-return convention with dividends reinvested at the ex-date = total-return approximation); intraday returns within a session are unaffected by adjustments except on split days (handled by using the `split` 1-minute view for RV).
	182	+- **Licence**: FirstRate Data private use; raw bars are never redistributed; aggregate forecasts/losses and derived statistics are publishable with attribution.
	183	+- **Survivorship**: the universe is fixed ex ante to instruments alive on 2026-11-06; all candidates have been continuously listed over their sample, so there is no within-sample delisting, but the selection of liquid, surviving instruments is a documented selection bias (results describe liquid benchmarks, not the cross-section).
	184	+- **Unresolved**: (i) exact vendor roll rule for continuous futures (vendor does not expose it in the API; the ratio series preserves returns, so forecasts are unaffected, but RV on roll days mixes two contracts' intraday paths); (ii) corporate-actions table is empty in the lake (adjustments come from vendor-adjusted series only); (iii) 1-minute FX/crypto bars are not consolidated trade data — RV from them is a mid-quote/aggregate measure and is documented as such.

added paper/.latexmkrc +5 −0

		@@ -0,0 +1,5 @@
	1	+$pdf_mode = 1;
	2	+$pdflatex = 'pdflatex -interaction=nonstopmode -halt-on-error -synctex=1 %O %S';
	3	+$bibtex_use = 2;
	4	+$clean_ext = 'synctex.gz run.xml bbl bcf fdb_latexmk fls log aux out toc lof lot blg';
	5	+@default_files = ('main.tex');

added paper/Makefile +11 −0

		@@ -0,0 +1,11 @@
	1	+.PHONY: all clean distclean
	2	+
	3	+all:
	4	+ latexmk
	5	+
	6	+clean:
	7	+ latexmk -c
	8	+
	9	+distclean:
	10	+ latexmk -C
	11	+ rm -f *.synctex.gz *.run.xml *.bbl *.bcf

added paper/sections/titlepage.tex +77 −0

		@@ -0,0 +1,77 @@
	1	+% ============================================================================
	2	+% Title Page
	3	+% ============================================================================
	4	+\thispagestyle{empty}
	5	+
	6	+\begin{center}
	7	+
	8	+% --- Logo ---
	9	+\includegraphics[width=3.5cm]{uq_logo.jpg}
	10	+
	11	+\vspace{0.6cm}
	12	+
	13	+{\footnotesize\textsc{Université du Québec en Outaouais}}\\[0.15cm]
	14	+{\footnotesize\textsc{Département des sciences administratives}}
	15	+
	16	+\vspace{0.8cm}
	17	+
	18	+{\footnotesize\textsc{Working Paper No.~\WPnumber}}
	19	+
	20	+\vspace{1.2cm}
	21	+
	22	+% --- Title ---
	23	+{\LARGE\bfseries \WPtitle\par}
	24	+
	25	+\ifx\WPsubtitle\empty\else
	26	+ \vspace{0.3cm}
	27	+ {\large\itshape \WPsubtitle\par}
	28	+\fi
	29	+
	30	+\vspace{1.2cm}
	31	+
	32	+% --- Author ---
	33	+{\large \WPauthor}\\[0.3cm]
	34	+{\normalsize \WPaffiliation}\\[0.15cm]
	35	+{\normalsize \href{mailto:\WPemail}{\WPemail}}\\[0.15cm]
	36	+{\small \WPaddress}
	37	+
	38	+\vspace{0.8cm}
	39	+
	40	+% --- Date & Version ---
	41	+{\normalsize \WPdate}\\[0.1cm]
	42	+{\small Version~\WPversion}
	43	+
	44	+\end{center}
	45	+
	46	+\vfill
	47	+
	48	+\newpage
	49	+
	50	+% ============================================================================
	51	+% Abstract Page
	52	+% ============================================================================
	53	+\thispagestyle{empty}
	54	+
	55	+\vspace*{1cm}
	56	+
	57	+\noindent\rule{\textwidth}{0.4pt}
	58	+\vspace{0.3cm}
	59	+
	60	+\noindent\textbf{Abstract}
	61	+
	62	+\vspace{0.15cm}
	63	+
	64	+\noindent\WPabstract
	65	+
	66	+\vspace{0.4cm}
	67	+
	68	+\noindent\textbf{Keywords:} \WPkeywords
	69	+
	70	+\vspace{0.15cm}
	71	+
	72	+\noindent\textbf{JEL Classification:} \WPjel
	73	+
	74	+\vspace{0.3cm}
	75	+\noindent\rule{\textwidth}{0.4pt}
	76	+
	77	+\newpage

added paper/sections/titlepage_anon.tex +53 −0

		@@ -0,0 +1,53 @@
	1	+% ============================================================================
	2	+% ANONYMOUS TITLE PAGE — for double-anonymized journal submission
	3	+% Used by paper/main_anon.tex (same macros as main.tex, no author block,
	4	+% no institution, no logo). Abstract/keywords/JEL page identical to titlepage.tex.
	5	+% ============================================================================
	6	+\thispagestyle{empty}
	7	+\begin{center}
	8	+
	9	+\vspace*{3cm}
	10	+
	11	+{\LARGE\bfseries \WPtitle\par}
	12	+
	13	+\ifx\WPsubtitle\empty\else
	14	+ \vspace{0.3cm}
	15	+ {\large\itshape \WPsubtitle\par}
	16	+\fi
	17	+
	18	+\vspace{1.5cm}
	19	+
	20	+{\normalsize [Author identity withheld for review]}
	21	+
	22	+\vspace{0.8cm}
	23	+
	24	+{\normalsize \WPdate}
	25	+
	26	+\end{center}
	27	+
	28	+\vfill
	29	+
	30	+\newpage
	31	+\thispagestyle{empty}
	32	+\vspace*{1cm}
	33	+
	34	+\noindent\rule{\textwidth}{0.4pt}
	35	+\vspace{0.3cm}
	36	+
	37	+\noindent\textbf{Abstract}
	38	+\vspace{0.15cm}
	39	+
	40	+\noindent\WPabstract
	41	+
	42	+\vspace{0.4cm}
	43	+
	44	+\noindent\textbf{Keywords:} \WPkeywords
	45	+
	46	+\vspace{0.15cm}
	47	+
	48	+\noindent\textbf{JEL Classification:} \WPjel
	49	+
	50	+\vspace{0.3cm}
	51	+\noindent\rule{\textwidth}{0.4pt}
	52	+
	53	+\newpage

added paper/uq_logo.jpg +0 −0

Binary file not shown.

added pyproject.toml +32 −0

		@@ -0,0 +1,32 @@
	1	+[project]
	2	+name = "wp46"
	3	+version = "0.1.0"
	4	+description = "UQO Working Paper 46 - Four Decades of Conditional Volatility Modeling: a comprehensive empirical benchmark"
	5	+readme = "README.md"
	6	+requires-python = ">=3.12"
	7	+license = { text = "Research code: MIT. Data: FirstRate Data private license (not redistributed)." }
	8	+authors = [{ name = "Simon-Pierre Boucher", email = "simon-pierre.boucher@uqo.ca" }]
	9	+dependencies = [
	10	+ "numpy>=2.0", "scipy>=1.13", "pandas>=2.2", "pyarrow>=17", "polars>=1.0",
	11	+ "arch==8.0.0", "statsmodels>=0.14", "httpx>=0.27", "pyyaml>=6",
	12	+ "exchange-calendars>=4.5", "matplotlib>=3.9", "numba>=0.60",
	13	+]
	14	+
	15	+[project.optional-dependencies]
	16	+dev = ["pytest>=8", "pytest-cov", "ruff", "hypothesis>=6"]
	17	+
	18	+[build-system]
	19	+requires = ["setuptools>=68", "wheel"]
	20	+build-backend = "setuptools.build_meta"
	21	+
	22	+[tool.setuptools.packages.find]
	23	+where = ["src"]
	24	+
	25	+[tool.pytest.ini_options]
	26	+testpaths = ["tests"]
	27	+addopts = "-q"
	28	+markers = ["live: tests that call the live FinData API (skipped without WP46_LIVE=1)", "slow: long tests", "r: tests that need R + rugarch"]
	29	+
	30	+[tool.ruff]
	31	+line-length = 110
	32	+target-version = "py312"

added results/manifests/findata_audit.json +3095 −0

Large diff (3,096 lines) is not loaded by default to keep the page fast.Load diff

added scripts/audit_findata.py +245 −0

Large diff (245 lines) is not loaded by default to keep the page fast.Load diff

added scripts/ingest_bars.py +67 −0

Large diff (67 lines) is not loaded by default to keep the page fast.Load diff

added src/wp46/__init__.py +2 −0

Large diff (2 lines) is not loaded by default to keep the page fast.Load diff

added src/wp46/calendars/__init__.py +1 −0

Large diff (1 lines) is not loaded by default to keep the page fast.Load diff

added src/wp46/calendars/sessions.py +98 −0

Large diff (98 lines) is not loaded by default to keep the page fast.Load diff

added src/wp46/findata_client/__init__.py +1 −0

Large diff (1 lines) is not loaded by default to keep the page fast.Load diff

added src/wp46/findata_client/client.py +225 −0

Large diff (225 lines) is not loaded by default to keep the page fast.Load diff

added src/wp46/findata_client/ingest.py +61 −0

Large diff (62 lines) is not loaded by default to keep the page fast.Load diff