WP40 RAID: E7 bundle + node runner, E0 universe/export script, Python VAR/connectedness bindings
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
8 changed files +84 −3
modified
CMakePresets.json
+1 −1
@@ -10,7 +10,7 @@ | ||
| 10 | 10 | { "name": "cpu-only", "inherits": "base", "displayName": "Portable CPU-only build (Linux/macOS)", |
| 11 | 11 | "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "RAID_ENABLE_METAL": "OFF", "RAID_WITH_ARROW": "OFF", "RAID_WITH_ACCELERATE": "OFF", "RAID_NATIVE_ARCH": "OFF" } }, |
| 12 | 12 | { "name": "bundle", "inherits": "base", "displayName": "Self-contained bundle for cluster nodes (E7)", |
| 13 | − "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "RAID_ENABLE_METAL": "ON", "RAID_WITH_ARROW": "OFF", "RAID_NATIVE_ARCH": "OFF", "RAID_BUILD_TESTS": "OFF", "CMAKE_OSX_DEPLOYMENT_TARGET": "26.0", "CMAKE_PREFIX_PATH": "/opt/homebrew", "CMAKE_CXX_FLAGS": "-mcpu=apple-m1" } } | |
| 13 | + "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "RAID_ENABLE_METAL": "ON", "RAID_WITH_ARROW": "OFF", "RAID_NATIVE_ARCH": "OFF", "RAID_BUILD_TESTS": "ON", "CMAKE_OSX_DEPLOYMENT_TARGET": "26.0", "CMAKE_PREFIX_PATH": "/opt/homebrew", "CMAKE_CXX_FLAGS": "-mcpu=apple-m1" } } | |
| 14 | 14 | ], |
| 15 | 15 | "buildPresets": [ |
| 16 | 16 | { "name": "macos-release", "configurePreset": "macos-release" }, |
modified
bindings/python/raid_py.cpp
+14 −0
@@ -13,6 +13,7 @@ | ||
| 13 | 13 | #include "raid/estimators/realized.hpp" |
| 14 | 14 | #include "raid/models/garch.hpp" |
| 15 | 15 | #include "raid/estimators/har.hpp" |
| 16 | +#include "raid/models/var.hpp" | |
| 16 | 17 | #include "raid/statistics/bootstrap.hpp" |
| 17 | 18 | #include "raid/statistics/reductions.hpp" |
| 18 | 19 | #include "raid/data/bar_series.hpp" |
@@ -115,6 +116,19 @@ NB_MODULE(_core, m) { | ||
| 115 | 116 | est::HarBatchResult r; { nb::gil_scoped_release rel; r = est::har_batch(std::span<const f64>(rv.data(), rv.shape(0)), std::span<const i64>(off.data(), off.shape(0)), s, p); } |
| 116 | 117 | nb::list fits; for (auto& e : r.fits) fits.append(json_to_py(e.to_json())); nb::dict d; d["fits"] = fits; d["backend"] = std::string(to_string(r.stats.backend)); d["precision"] = std::string(to_string(r.stats.precision)); d["wall_seconds"] = r.stats.wall_seconds; d["device_seconds"] = r.stats.device_seconds; return d; |
| 117 | 118 | }, nb::arg("rv"), nb::arg("offsets"), nb::arg("lags") = std::vector<usize>{1, 5, 22}, nb::arg("horizon") = 1, nb::arg("log") = false, nb::arg("backend") = "auto"); |
| 119 | + // ---- VAR / connectedness ---- | |
| 120 | + m.def("var_connectedness", [](F64 Y, int p, int H, usize window, usize step) { | |
| 121 | + RAID_REQUIRE(Y.ndim() == 2, invalid_argument, "Y must be (T, N)"); const Eigen::Index T = static_cast<Eigen::Index>(Y.shape(0)), N = static_cast<Eigen::Index>(Y.shape(1)); | |
| 122 | + Eigen::MatrixXd Ym = Eigen::Map<const Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(Y.data(), T, N); | |
| 123 | + nb::dict d; { nb::gil_scoped_release rel; auto f = models::var_fit(Ym, p); auto c = models::connectedness(f, H); | |
| 124 | + std::vector<f64> tbl(static_cast<usize>(N * N)); for (Eigen::Index i = 0; i < N; ++i) for (Eigen::Index j = 0; j < N; ++j) tbl[static_cast<usize>(i * N + j)] = c.table(i, j); | |
| 125 | + std::vector<f64> fr(c.from.data(), c.from.data() + N), to(c.to.data(), c.to.data() + N), net(c.net.data(), c.net.data() + N); | |
| 126 | + nb::gil_scoped_acquire acq; d["total"] = c.total; d["table_flat"] = to_np(tbl); d["from"] = to_np(fr); d["to"] = to_np(to); d["net"] = to_np(net); d["stable"] = f.stable; d["max_root"] = f.max_root; d["loglik"] = f.loglik; d["n_obs"] = f.n_obs; | |
| 127 | + if (window > 0) { auto r = models::rolling_connectedness(Ym, p, H, window, step); std::vector<int64_t> e(r.end_index.begin(), r.end_index.end()); d["rolling_end"] = to_np_i64(std::move(e)); d["rolling_total"] = to_np(r.total); d["rolling_failures"] = r.failures; d["rolling_seconds"] = r.seconds; } } | |
| 128 | + return d; | |
| 129 | + }, nb::arg("Y"), nb::arg("p") = 1, nb::arg("H") = 10, nb::arg("window") = 0, nb::arg("step") = 1); | |
| 130 | + m.def("ledoit_wolf", [](F64 X) { const Eigen::Index n = static_cast<Eigen::Index>(X.shape(0)), p = static_cast<Eigen::Index>(X.shape(1)); Eigen::MatrixXd Xm = Eigen::Map<const Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(X.data(), n, p); auto r = models::ledoit_wolf(Xm); std::vector<f64> c(static_cast<usize>(p * p)); for (Eigen::Index i = 0; i < p; ++i) for (Eigen::Index j = 0; j < p; ++j) c[static_cast<usize>(i * p + j)] = r.cov(i, j); nb::dict d; d["cov_flat"] = to_np(c); d["shrinkage"] = r.shrinkage; d["mu"] = r.mu; return d; }); | |
| 131 | + | |
| 118 | 132 | // ---- bootstrap ---- |
| 119 | 133 | m.def("bootstrap", [](F64 x, const std::string& stat, const std::string& scheme, double mean_block, usize replicas, uint64_t seed, double level) { |
| 120 | 134 | stats::BootstrapOptions o; o.scheme = scheme == "circular" ? stats::BootstrapScheme::circular_block : (scheme == "iid" ? stats::BootstrapScheme::iid : stats::BootstrapScheme::stationary); o.mean_block = mean_block; o.replicas = replicas; o.seed = seed; o.ci_level = level; |
added
experiments/configs/e0_universe.yaml
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# E0 — real-data universe and exports for E1–E4 (FinData OS, FirstRate Data private license; cache is git-ignored) | |
| 2 | +dataset_id: market_bars.stock.1m.raw | |
| 3 | +year: 2024 | |
| 4 | +start: 2024-01-02 | |
| 5 | +end: 2024-12-31 | |
| 6 | +min_sessions_total: 250 # symbols active across the year (coverage endpoint: d0 <= start, d1 >= end) | |
| 7 | +sample_sizes: [1, 10, 100, 1000] # nested samples (seeded): the 1000-symbol universe contains the 100, 10, 1 ones | |
| 8 | +seed: 20261102 | |
| 9 | +symbols_per_export: 50 | |
| 10 | +exclude_delisted_suffix: true | |
added
experiments/scripts/e0_universe_and_exports.py
+27 −0
@@ -0,0 +1,27 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +"""E0: select the real universe (seeded, nested samples) from FinData coverage and export 1-minute bars for 2024. | |
| 3 | +Outputs: experiments/results/e0_universe.json (symbols, dataset version, export ids, sha256), cache Parquet files | |
| 4 | +under $RAID_CACHE_DIR (never committed). Requires FINDATA_API_KEY_FILE and the compiled core on sys.path.""" | |
| 5 | +import os, sys, json, time, yaml, hashlib | |
| 6 | +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "python")) | |
| 7 | +import numpy as np, raid | |
| 8 | +cfg = yaml.safe_load(open(os.path.join(os.path.dirname(__file__), "..", "configs", "e0_universe.yaml"))) | |
| 9 | +out_path = os.path.join(os.path.dirname(__file__), "..", "results", "e0_universe.json") | |
| 10 | +t0 = time.time() | |
| 11 | +cov = raid.findata.coverage(cfg["dataset_id"], cfg["min_sessions_total"]) | |
| 12 | +active = [c for c in cov if c["d0"] <= cfg["start"] and c["d1"] >= cfg["end"] and (not cfg["exclude_delisted_suffix"] or not c["symbol"].endswith("-DELISTED"))] | |
| 13 | +active.sort(key=lambda c: c["symbol"]) | |
| 14 | +rng = np.random.default_rng(cfg["seed"]); perm = rng.permutation(len(active)) | |
| 15 | +nmax = max(cfg["sample_sizes"]); chosen = [active[i]["symbol"] for i in perm[:nmax]] | |
| 16 | +samples = {str(n): chosen[:n] for n in cfg["sample_sizes"]} | |
| 17 | +print(f"coverage symbols {len(cov)}, active through {cfg['year']}: {len(active)}, chosen {nmax}; first {chosen[:5]}") | |
| 18 | +exports = []; versions = set() | |
| 19 | +for i in range(0, nmax, cfg["symbols_per_export"]): | |
| 20 | + syms = chosen[i:i + cfg["symbols_per_export"]]; t1 = time.time() | |
| 21 | + e = raid.findata.export(syms, cfg["start"], cfg["end"], dataset_id=cfg["dataset_id"]) | |
| 22 | + versions.add(e["dataset_version"]); exports.append({"symbols": syms, **{k: e[k] for k in ("export_id", "path", "sha256", "dataset_version", "rows", "bytes", "from_cache")}}) | |
| 23 | + print(f"export {i // cfg['symbols_per_export'] + 1}/{(nmax + cfg['symbols_per_export'] - 1) // cfg['symbols_per_export']}: {e['rows']:,} rows {e['bytes'] / 1e6:.1f} MB cache={e['from_cache']} {time.time() - t1:.1f}s", flush=True) | |
| 24 | +json.dump({"config": cfg, "n_coverage": len(cov), "n_active": len(active), "samples": samples, "exports": exports, "dataset_versions": sorted(versions), | |
| 25 | + "total_rows": sum(e["rows"] for e in exports), "total_bytes": sum(e["bytes"] for e in exports), "seconds": time.time() - t0, | |
| 26 | + "universe_sha256": hashlib.sha256(",".join(chosen).encode()).hexdigest()}, open(out_path, "w"), indent=1) | |
| 27 | +print("wrote", out_path, "rows", sum(e["rows"] for e in exports), "seconds", round(time.time() - t0, 1)) | |
added
experiments/scripts/e7_run_nodes.sh
+12 −0
@@ -0,0 +1,12 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# E7: ship the self-contained bundle to each node (home directory only, nothing installed), run, collect results. | |
| 3 | +# Usage: e7_run_nodes.sh [nodes...] (default = generation sweep; FinData/Ka/gateway hosts excluded by design) | |
| 4 | +set -uo pipefail | |
| 5 | +cd "$(dirname "$0")/../.." | |
| 6 | +NODES="${*:-M2M32c M2U64 M3BA24 M3U96b M4M64c m4mm m4md M5M36 m5ma m6ma m2m8a}" | |
| 7 | +OUT=experiments/results/raw/e7; mkdir -p $OUT | |
| 8 | +for n in $NODES; do | |
| 9 | + ( scp -q -o ConnectTimeout=15 build/raid-e7-bundle.tgz $n:/tmp/raid-e7-bundle.tgz && ssh -o ConnectTimeout=15 $n 'mkdir -p ~/raid-e7 && tar -C ~/raid-e7 -xzf /tmp/raid-e7-bundle.tgz && cd ~/raid-e7/e7-bundle && bash run_e7.sh' > $OUT/$n.log 2>&1; echo "== $n: exit=$? | $(grep -E "exit=" $OUT/$n.log | tr '\n' ' ')"; mkdir -p $OUT/$n && scp -q -r "$n:~/raid-e7/e7-bundle/results_*" $OUT/$n/ 2>/dev/null ) & | |
| 10 | +done | |
| 11 | +wait | |
| 12 | +echo "collected into $OUT" | |
modified
python/raid/__init__.py
+2 −2
@@ -18,8 +18,8 @@ except ImportError: # pragma: no cover | ||
| 18 | 18 | from . import findata # noqa: E402 |
| 19 | 19 | from .api import ( # noqa: E402 |
| 20 | 20 | devices, ols, realized_measures, garch_fit, validate_bars, calendar_sessions, Backend, available_backends, |
| 21 | − har_fit, har_rolling, har_batch, bootstrap, | |
| 21 | + har_fit, har_rolling, har_batch, bootstrap, var_connectedness, ledoit_wolf, | |
| 22 | 22 | ) |
| 23 | 23 | |
| 24 | 24 | __all__ = ["__version__", "HAS_CORE", "devices", "ols", "realized_measures", "garch_fit", "validate_bars", |
| 25 | − "calendar_sessions", "Backend", "available_backends", "findata", "har_fit", "har_rolling", "har_batch", "bootstrap"] | |
| 25 | + "calendar_sessions", "Backend", "available_backends", "findata", "har_fit", "har_rolling", "har_batch", "bootstrap", "var_connectedness", "ledoit_wolf"] | |
modified
python/raid/api.py
+10 −0
@@ -96,3 +96,13 @@ def har_batch(rv_concat, offsets, lags=(1, 5, 22), horizon: int = 1, log: bool = | ||
| 96 | 96 | def bootstrap(x, stat: str = "mean", scheme: str = "stationary", mean_block: float = 0.0, replicas: int = 999, seed: int = 20261102, level: float = 0.95) -> dict[str, Any]: |
| 97 | 97 | """Stationary / circular block / iid bootstrap of mean, realized variance (sum of squares) or variance.""" |
| 98 | 98 | return _core().bootstrap(_f64(x), stat, scheme, float(mean_block), int(replicas), int(seed), float(level)) |
| 99 | + | |
| 100 | + | |
| 101 | +def var_connectedness(Y, p: int = 1, H: int = 10, window: int = 0, step: int = 1) -> dict[str, Any]: | |
| 102 | + """VAR(p) + generalized FEVD + Diebold–Yilmaz total/from/to/net (and rolling totals when window > 0).""" | |
| 103 | + Y = _f64(Y); d = _core().var_connectedness(Y, int(p), int(H), int(window), int(step)); N = Y.shape[1] | |
| 104 | + d["table"] = np.asarray(d.pop("table_flat")).reshape(N, N); return d | |
| 105 | + | |
| 106 | + | |
| 107 | +def ledoit_wolf(X) -> dict[str, Any]: | |
| 108 | + X = _f64(X); d = _core().ledoit_wolf(X); d["cov"] = np.asarray(d.pop("cov_flat")).reshape(X.shape[1], X.shape[1]); return d | |
modified
python/tests/test_api.py
+8 −0
@@ -81,3 +81,11 @@ def test_bootstrap_block_vs_iid(): | ||
| 81 | 81 | for t in range(1, n): x[t] = 0.7 * x[t - 1] + rng.standard_normal() |
| 82 | 82 | blk = raid.bootstrap(x, "mean", "stationary", mean_block=20, replicas=399); iid = raid.bootstrap(x, "mean", "iid", replicas=399) |
| 83 | 83 | assert blk["se"] > 1.5 * iid["se"] and blk["failures"] == 0 and blk["ci_lo"] < blk["point"] < blk["ci_hi"] |
| 84 | + | |
| 85 | + | |
| 86 | +def test_var_connectedness_matches_statsmodels_shape_and_bounds(): | |
| 87 | + rng = np.random.default_rng(4); T, N = 600, 3; Y = np.zeros((T, N)); A = np.array([[0.5, 0.1, 0.0], [0.0, 0.4, 0.2], [0.1, 0.0, 0.3]]) | |
| 88 | + for t in range(1, T): Y[t] = A @ Y[t - 1] + rng.standard_normal(N) | |
| 89 | + c = raid.var_connectedness(Y, p=1, H=10, window=300, step=50) | |
| 90 | + assert c["stable"] and 0 <= c["total"] <= 100 and np.allclose(c["table"].sum(axis=1), 1.0) and c["rolling_failures"] == 0 | |
| 91 | + lw = raid.ledoit_wolf(Y); assert 0 <= lw["shrinkage"] <= 1 and lw["cov"].shape == (N, N) | |
| 84 | 92 | |