WP46: synthetic fixture, data READMEs
7 changed files +83 −5
added
data/README.md
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +# data/ | |
| 2 | + | |
| 3 | +- `raw/`, `interim/`, `processed/`: ignored by git (FirstRate Data private licence). The live cache lives outside the repository | |
| 4 | + (`~/wp46-cache/{bars,panels,rcov}` on the data node and compute nodes); see `docs/replication_guide.md`. | |
| 5 | +- `manifests/`: nothing here — all manifests are under `results/manifests/` (audit, ingest with dataset versions and SHA-256, panels, jobs). | |
| 6 | +- `synthetic/`: licence-free simulated panels (`scripts/make_synthetic.py`) in the exact panel schema (`docs/data_dictionary.md`), to run the | |
| 7 | + pipeline without credentials: `WP46_PANELS=data/synthetic .venv/bin/python scripts/run_benchmark.py --jobs <jobs for SYN1,SYN2> --out results/forecasts/synthetic`. | |
added
data/raw/README.md
+5 −0
@@ -0,0 +1,5 @@ | ||
| 1 | +# data/raw — provenance (no files committed) | |
| 2 | +Source: FinData OS (`https://findata.spboucher.ai/api/v1`), FirstRate Data one-minute and daily bars. Extraction 2026-11-09 by | |
| 3 | +`scripts/ingest_bars.py` (exports chunked by ≤ 5 years; dataset ids, versions, row counts and SHA-256 per file in | |
| 4 | +`results/manifests/ingest_manifest.json`; ≈ 230 M one-minute rows, ≈ 10 GB Parquet). Access requires the owner's API key; the data may | |
| 5 | +not be redistributed. Audit of the inventory: `docs/findata_audit.md`. | |
added
data/synthetic/SYN1.parquet
+0 −0
Binary file not shown.
added
data/synthetic/SYN2.parquet
+0 −0
Binary file not shown.
added
data/synthetic/manifest.json
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +{ | |
| 2 | + "SYN1": { | |
| 3 | + "asset_class": "synthetic", | |
| 4 | + "symbol": "SYN1", | |
| 5 | + "rows": 5500, | |
| 6 | + "first": "2004-01-02", | |
| 7 | + "last": "2025-01-30", | |
| 8 | + "seed": 100 | |
| 9 | + }, | |
| 10 | + "SYN2": { | |
| 11 | + "asset_class": "synthetic", | |
| 12 | + "symbol": "SYN2", | |
| 13 | + "rows": 5500, | |
| 14 | + "first": "2004-01-02", | |
| 15 | + "last": "2025-01-30", | |
| 16 | + "seed": 101 | |
| 17 | + } | |
| 18 | +} | |
| \ No newline at end of file | ||
added
scripts/make_synthetic.py
+49 −0
@@ -0,0 +1,49 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +"""Synthetic fixture (licence-free): two simulated instruments with GJR-GARCH-t returns and a noisy realized proxy, written | |
| 3 | +in the panel format so that the whole pipeline (forecasts -> losses -> analysis -> exhibits) runs without FinData access. | |
| 4 | +usage: make_synthetic.py [--out data/synthetic] ; then WP46_PANELS=data/synthetic scripts/run_benchmark.py ... | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import argparse | |
| 9 | +import json | |
| 10 | +from pathlib import Path | |
| 11 | + | |
| 12 | +import numpy as np | |
| 13 | +import pandas as pd | |
| 14 | + | |
| 15 | +ROOT = Path(__file__).resolve().parents[1] | |
| 16 | + | |
| 17 | + | |
| 18 | +def simulate(T: int, seed: int, om=0.03, al=0.03, ga=0.08, be=0.90, nu=6.0) -> pd.DataFrame: | |
| 19 | + rng = np.random.default_rng(seed) | |
| 20 | + z = rng.standard_t(nu, T) * np.sqrt((nu - 2) / nu) | |
| 21 | + s = np.empty(T); e = np.empty(T); s[0] = om / (1 - al - be - ga / 2) | |
| 22 | + for t in range(T): | |
| 23 | + if t > 0: | |
| 24 | + s[t] = om + al * e[t - 1] ** 2 + ga * e[t - 1] ** 2 * (e[t - 1] < 0) + be * s[t - 1] | |
| 25 | + e[t] = np.sqrt(s[t]) * z[t] | |
| 26 | + r = 0.02 + e | |
| 27 | + rv = s * rng.lognormal(-0.05, 0.3, T) * 0.7 # intraday part (70 % of the daily variance on average) | |
| 28 | + on = np.sqrt(0.3 * s) * rng.standard_normal(T) # overnight return | |
| 29 | + dates = pd.bdate_range("2004-01-02", periods=T) | |
| 30 | + return pd.DataFrame({"session": dates, "close": 100 * np.exp(np.cumsum(r) / 100), "open": np.nan, "r": r, "r_on": on, "r_oc": r - on, | |
| 31 | + "valid_rm": True, "expected_bars": 391, "rv_1m": rv * 1.1, "rv_5m": rv, "rv_5m_ss": rv, "bv_5m": rv * 0.95, | |
| 32 | + "rsv_pos_5m": rv / 2, "rsv_neg_5m": rv / 2, "medrv_5m": rv * 0.9, "n_bars": 391, "n_5m": 78}) | |
| 33 | + | |
| 34 | + | |
| 35 | +def main() -> None: | |
| 36 | + ap = argparse.ArgumentParser(); ap.add_argument("--out", default=str(ROOT / "data" / "synthetic")); ap.add_argument("--T", type=int, default=5500) | |
| 37 | + a = ap.parse_args() | |
| 38 | + out = Path(a.out); out.mkdir(parents=True, exist_ok=True) | |
| 39 | + man = {} | |
| 40 | + for i, sym in enumerate(["SYN1", "SYN2"]): | |
| 41 | + df = simulate(a.T, 100 + i) | |
| 42 | + df.to_parquet(out / f"{sym}.parquet", index=False) | |
| 43 | + man[sym] = {"asset_class": "synthetic", "symbol": sym, "rows": int(len(df)), "first": str(df.session.min().date()), "last": str(df.session.max().date()), "seed": 100 + i} | |
| 44 | + (out / "manifest.json").write_text(json.dumps(man, indent=1)) | |
| 45 | + print("synthetic panels:", list(man)) | |
| 46 | + | |
| 47 | + | |
| 48 | +if __name__ == "__main__": | |
| 49 | + main() | |
modified
tests/replication/rugarch_crosscheck.R
+4 −5
@@ -37,16 +37,15 @@ if (requireNamespace("GAS", quietly = TRUE)) { | ||
| 37 | 37 | suppressPackageStartupMessages(library(GAS)) |
| 38 | 38 | gs <- UniGASSpec(Dist = "std", ScalingType = "Inv", GASPar = list(location = FALSE, scale = TRUE, shape = FALSE)) |
| 39 | 39 | gf <- tryCatch(UniGASFit(gs, r), error = function(e) NULL) |
| 40 | − if (!is.null(gf)) out2$gas_t <- list(converged = TRUE, loglik = gf@Estimates$lLik, coef = as.list(gf@Estimates$lParList), aic = gf@GOF$AIC) else out2$gas_t <- list(converged = FALSE) | |
| 40 | + out2$gas_t <- tryCatch(if (!is.null(gf)) list(converged = TRUE, loglik = as.numeric(gf@Estimates$lLik), coef = as.list(gf@Estimates$lParList)) else list(converged = FALSE), error = function(e) list(converged = FALSE, error = conditionMessage(e))) | |
| 41 | 41 | } |
| 42 | 42 | if (requireNamespace("MSGARCH", quietly = TRUE)) { |
| 43 | 43 | suppressPackageStartupMessages(library(MSGARCH)) |
| 44 | 44 | ms <- CreateSpec(variance.spec = list(model = c("sGARCH", "sGARCH")), distribution.spec = list(distribution = c("norm", "norm")), switch.spec = list(do.mix = FALSE, K = 2)) |
| 45 | 45 | mf <- tryCatch(FitML(ms, data = r - mean(r)), error = function(e) NULL) |
| 46 | − if (!is.null(mf)) { | |
| 47 | − pr <- predict(mf, nahead = 22, do.return.draw = FALSE) | |
| 48 | − out2$msgarch2_n <- list(converged = TRUE, loglik = mf$loglik, par = as.list(mf$par), vol_forecast = as.numeric(pr$vol), mean_removed = mean(r)) | |
| 49 | − } else out2$msgarch2_n <- list(converged = FALSE) | |
| 46 | + out2$msgarch2_n <- tryCatch(if (!is.null(mf)) { pr <- predict(mf, nahead = 22, do.return.draw = FALSE) | |
| 47 | + list(converged = TRUE, loglik = as.numeric(mf$loglik), par = as.list(mf$par), vol_forecast = as.numeric(pr$vol), mean_removed = mean(r)) } else list(converged = FALSE), | |
| 48 | + error = function(e) list(converged = FALSE, error = conditionMessage(e))) | |
| 50 | 49 | } |
| 51 | 50 | out <- c(out, out2) |
| 52 | 51 | write_json(out, a[2], auto_unbox = TRUE, digits = NA) |
| 53 | 52 | |