SPB Git
spboucher.ai

spb/wp40_uqo

Public MIT

WP40 RAID: E1–E6 results (real 2024 panel, daily 2015–2024 histories), Parquet reader projection + fast paths, Metal GARCH per-series scaling, GARCH restarts, paper skeleton (intro, literature, references)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 2, 2026) parent 1dca98c Browse files

14 changed files +1,293 −14

14 changed files
Unified Split
modified backends/metal/host/metal_backend.mm +10 −5
@@ -195,15 +195,20 @@ void regression_stats(std::span<const f64> x, std::span<const f64> y, std::span<
195 RAID_REQUIRE(p == 4 || p == 5 || p == 6, invalid_argument, "garch params per vector must be 4 (garch), 5 (gjr/egarch) or 6 (with nu)"); 195 RAID_REQUIRE(p == 4 || p == 5 || p == 6, invalid_argument, "garch params per vector must be 4 (garch), 5 (gjr/egarch) or 6 (with nu)");
196 @autoreleasepool { 196 @autoreleasepool {
197 id<MTLBuffer> br = buffer(n * 4, &allocs), bb = buffer(S * 4, &allocs), be = buffer(S * 4, &allocs), bp = buffer(J * 8 * 4, &allocs), bv = buffer(S * 4, &allocs), bj = buffer(J * 16, &allocs), bo = buffer(J * 4, &allocs); 197 id<MTLBuffer> br = buffer(n * 4, &allocs), bb = buffer(S * 4, &allocs), be = buffer(S * 4, &allocs), bp = buffer(J * 8 * 4, &allocs), bv = buffer(S * 4, &allocs), bj = buffer(J * 16, &allocs), bo = buffer(J * 4, &allocs);
198− float* rf = static_cast<float*>(br.contents); for (usize i = 0; i < n; ++i) rf[i] = static_cast<float>(r.values[i]); 198+ float* rf = static_cast<float*>(br.contents);
199 uint32_t* sb = static_cast<uint32_t*>(bb.contents); uint32_t* se = static_cast<uint32_t*>(be.contents); float* v0 = static_cast<float*>(bv.contents); 199 uint32_t* sb = static_cast<uint32_t*>(bb.contents); uint32_t* se = static_cast<uint32_t*>(be.contents); float* v0 = static_cast<float*>(bv.contents);
200 float* pf = static_cast<float*>(bp.contents); uint32_t* jb = static_cast<uint32_t*>(bj.contents); 200 float* pf = static_cast<float*>(bp.contents); uint32_t* jb = static_cast<uint32_t*>(bj.contents);
201 const bool gjr = model_code == 1, eg = model_code == 2; 201 const bool gjr = model_code == 1, eg = model_code == 2;
202+ // Per-series scaling to unit variance (FP32 conditioning): r' = r/sd, μ' = μ/sd, ω' = ω/sd² (GARCH/GJR) or ω' = ω + (1−β)·log(1/sd²) (EGARCH);
203+ // the NLL of the original data is NLL' + n·log(sd). Scales are chosen in FP64 on the host.
204+ std::vector<f64> sd(S, 1.0);
205+ for (usize s = 0; s < S; ++s) { auto x = r.series(s); f64 m = 0; for (f64 v : x) m += v; m /= std::max<usize>(1, x.size()); f64 var = 0; for (f64 v : x) var += (v - m) * (v - m); var /= std::max<usize>(1, x.size()); sd[s] = var > 0 ? std::sqrt(var) : 1.0; for (i64 i = r.offsets[s]; i < r.offsets[s + 1]; ++i) rf[i] = static_cast<float>(r.values[static_cast<usize>(i)] / sd[s]); }
202 for (usize s = 0; s < S; ++s) { sb[s] = static_cast<uint32_t>(r.offsets[s]); se[s] = static_cast<uint32_t>(r.offsets[s + 1]); 206 for (usize s = 0; s < S; ++s) { sb[s] = static_cast<uint32_t>(r.offsets[s]); se[s] = static_cast<uint32_t>(r.offsets[s + 1]);
203 for (usize q = 0; q < n_params; ++q) { const f64* th = params.data() + (s * n_params + q) * p; const usize j = s * n_params + q; 207 for (usize q = 0; q < n_params; ++q) { const f64* th = params.data() + (s * n_params + q) * p; const usize j = s * n_params + q;
204 // layout [mu, omega, alpha, gamma, beta, nu]; FP64 initial variance per (series, mu) computed on the CPU → we use mu of the first param set per series 208 // layout [mu, omega, alpha, gamma, beta, nu]; FP64 initial variance per (series, mu) computed on the CPU → we use mu of the first param set per series
205− pf[j * 8 + 0] = static_cast<float>(th[0]); pf[j * 8 + 1] = static_cast<float>(th[1]); pf[j * 8 + 2] = static_cast<float>(th[2]); 209+ const f64 beta_s = th[(gjr || eg) ? 4 : 3];
206− pf[j * 8 + 3] = (gjr || eg) ? static_cast<float>(th[3]) : 0.0f; pf[j * 8 + 4] = static_cast<float>(th[(gjr || eg) ? 4 : 3]); 210+ pf[j * 8 + 0] = static_cast<float>(th[0] / sd[s]); pf[j * 8 + 1] = static_cast<float>(eg ? th[1] + (1.0 - beta_s) * std::log(1.0 / (sd[s] * sd[s])) : th[1] / (sd[s] * sd[s])); pf[j * 8 + 2] = static_cast<float>(th[2]);
211+ pf[j * 8 + 3] = (gjr || eg) ? static_cast<float>(th[3]) : 0.0f; pf[j * 8 + 4] = static_cast<float>(beta_s);
207 const bool has_nu = p == 6 || (p == 5 && !gjr && !eg); const f64 nu = has_nu ? th[p - 1] : (nu_fixed > 2 ? nu_fixed : 0.0); pf[j * 8 + 5] = static_cast<float>(nu); 212 const bool has_nu = p == 6 || (p == 5 && !gjr && !eg); const f64 nu = has_nu ? th[p - 1] : (nu_fixed > 2 ? nu_fixed : 0.0); pf[j * 8 + 5] = static_cast<float>(nu);
208 // FP64 density constants (Metal has no lgamma): kconst and E|z| 213 // FP64 density constants (Metal has no lgamma): kconst and E|z|
209 const f64 pi = 3.14159265358979323846; 214 const f64 pi = 3.14159265358979323846;
@@ -211,13 +216,13 @@ void regression_stats(std::span<const f64> x, std::span<const f64> y, std::span<
211 else { pf[j * 8 + 6] = static_cast<float>(-0.5 * std::log(2.0 * pi)); pf[j * 8 + 7] = static_cast<float>(std::sqrt(2.0 / pi)); } 216 else { pf[j * 8 + 6] = static_cast<float>(-0.5 * std::log(2.0 * pi)); pf[j * 8 + 7] = static_cast<float>(std::sqrt(2.0 / pi)); }
212 jb[j * 4 + 0] = static_cast<uint32_t>(s); jb[j * 4 + 1] = static_cast<uint32_t>(j); jb[j * 4 + 2] = 0; jb[j * 4 + 3] = 0; } 217 jb[j * 4 + 0] = static_cast<uint32_t>(s); jb[j * 4 + 1] = static_cast<uint32_t>(j); jb[j * 4 + 2] = 0; jb[j * 4 + 3] = 0; }
213 // initial variance: sample variance of (r − mu) with mu from the first parameter vector of the series (FP64) 218 // initial variance: sample variance of (r − mu) with mu from the first parameter vector of the series (FP64)
214− const f64 mu = params[(s * n_params) * p]; auto x = r.series(s); f64 m1 = 0; for (f64 v : x) m1 += v - mu; m1 /= static_cast<f64>(x.size()); f64 var0 = 0; for (f64 v : x) var0 += (v - mu - m1) * (v - mu - m1); var0 /= static_cast<f64>(x.size()); v0[s] = static_cast<float>(var0); } 219+ const f64 mu = params[(s * n_params) * p] / sd[s]; auto x = r.series(s); f64 m1 = 0; for (f64 v : x) m1 += v / sd[s] - mu; m1 /= static_cast<f64>(x.size()); f64 var0 = 0; for (f64 v : x) var0 += (v / sd[s] - mu - m1) * (v / sd[s] - mu - m1); var0 /= static_cast<f64>(x.size()); v0[s] = static_cast<float>(var0); }
215 const uint32_t model = static_cast<uint32_t>(model_code), nj = static_cast<uint32_t>(J); 220 const uint32_t model = static_cast<uint32_t>(model_code), nj = static_cast<uint32_t>(J);
216 id<MTLCommandBuffer> cb = [queue_ commandBuffer]; id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder]; auto pl = pipeline("garch_nll"); 221 id<MTLCommandBuffer> cb = [queue_ commandBuffer]; id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder]; auto pl = pipeline("garch_nll");
217 [enc setComputePipelineState:pl]; [enc setBuffer:br offset:0 atIndex:0]; [enc setBuffer:bb offset:0 atIndex:1]; [enc setBuffer:be offset:0 atIndex:2]; [enc setBuffer:bp offset:0 atIndex:3]; [enc setBuffer:bv offset:0 atIndex:4]; [enc setBuffer:bj offset:0 atIndex:5]; [enc setBuffer:bo offset:0 atIndex:6]; [enc setBytes:&model length:4 atIndex:7]; [enc setBytes:&nj length:4 atIndex:8]; 222 [enc setComputePipelineState:pl]; [enc setBuffer:br offset:0 atIndex:0]; [enc setBuffer:bb offset:0 atIndex:1]; [enc setBuffer:be offset:0 atIndex:2]; [enc setBuffer:bp offset:0 atIndex:3]; [enc setBuffer:bv offset:0 atIndex:4]; [enc setBuffer:bj offset:0 atIndex:5]; [enc setBuffer:bo offset:0 atIndex:6]; [enc setBytes:&model length:4 atIndex:7]; [enc setBytes:&nj length:4 atIndex:8];
218 [enc dispatchThreads:MTLSizeMake(J, 1, 1) threadsPerThreadgroup:MTLSizeMake(std::min<usize>(64, pl.maxTotalThreadsPerThreadgroup), 1, 1)]; [enc endEncoding]; [cb commit]; [cb waitUntilCompleted]; 223 [enc dispatchThreads:MTLSizeMake(J, 1, 1) threadsPerThreadgroup:MTLSizeMake(std::min<usize>(64, pl.maxTotalThreadsPerThreadgroup), 1, 1)]; [enc endEncoding]; [cb commit]; [cb waitUntilCompleted];
219 RAID_REQUIRE(cb.status == MTLCommandBufferStatusCompleted, numerical, "metal: command buffer failed"); 224 RAID_REQUIRE(cb.status == MTLCommandBufferStatusCompleted, numerical, "metal: command buffer failed");
220− const float* o = static_cast<const float*>(bo.contents); for (usize j = 0; j < J; ++j) out[j] = o[j]; 225+ const float* o = static_cast<const float*>(bo.contents); for (usize j = 0; j < J; ++j) { const usize s = j / n_params; out[j] = std::isfinite(o[j]) ? static_cast<f64>(o[j]) + static_cast<f64>(r.size(s)) * std::log(sd[s]) : o[j]; }
221 finish(st, "garch_nll", t, cb, n * 8 + J * p * 8, J * 8, allocs, library_source_ + " jobs=" + std::to_string(J)); release(br); release(bb); release(be); release(bp); release(bv); release(bj); release(bo); 226 finish(st, "garch_nll", t, cb, n * 8 + J * p * 8, J * 8, allocs, library_source_ + " jobs=" + std::to_string(J)); release(br); release(bb); release(be); release(bp); release(bv); release(bj); release(bo);
222 } 227 }
223 return out; 228 return out;
modified benchmarks/end_to_end/bench_e1_batched_rv.cpp +1 −1
@@ -41,7 +41,7 @@ int main(int argc, char** argv) {
41 #endif 41 #endif
42 struct Case { std::string source; usize S; std::string seg; usize seg_len; std::string backend; bool fused; }; 42 struct Case { std::string source; usize S; std::string seg; usize seg_len; std::string backend; bool fused; };
43 std::vector<Case> cases; const std::vector<std::pair<std::string, usize>> segs = {{"day", 1}, {"week", 5}, {"month", 22}, {"year", 252}}; 43 std::vector<Case> cases; const std::vector<std::pair<std::string, usize>> segs = {{"day", 1}, {"week", 5}, {"month", 22}, {"year", 252}};
44− for (usize S : {1ul, 10ul, 100ul, 1000ul}) for (auto& [sn, nd] : segs) { if (S > syms.size()) continue; for (auto bk : {"cpu1", "cpuN", "metal", "metal_unfused"}) cases.push_back({"real", S, sn, nd, bk, std::string(bk) != "metal_unfused"}); } 44+ for (usize S0 : {1ul, 10ul, 100ul, 1000ul}) for (auto& [sn, nd] : segs) { const usize S = std::min(S0, syms.size()); if (S0 > syms.size() && S0 / 10 >= syms.size()) continue; for (auto bk : {"cpu1", "cpuN", "metal", "metal_unfused"}) cases.push_back({"real", S, sn, nd, bk, std::string(bk) != "metal_unfused"}); }
45 for (usize S : {10000ul}) for (auto& [sn, nd] : segs) { if (nd > 22) continue; for (auto bk : {"cpu1", "cpuN", "metal"}) cases.push_back({"synthetic", S, sn, nd, bk, true}); } 45 for (usize S : {10000ul}) for (auto& [sn, nd] : segs) { if (nd > 22) continue; for (auto bk : {"cpu1", "cpuN", "metal"}) cases.push_back({"synthetic", S, sn, nd, bk, true}); }
46 std::mt19937_64 order(20261102); std::shuffle(cases.begin(), cases.end(), order); 46 std::mt19937_64 order(20261102); std::shuffle(cases.begin(), cases.end(), order);
47 usize done = 0; 47 usize done = 0;
modified benchmarks/end_to_end/bench_e2_har.cpp +1 −1
@@ -37,7 +37,7 @@ int main(int argc, char** argv) {
37 rows[i] = r; } }); 37 rows[i] = r; } });
38 for (auto& r : rows) { r["kind"] = "accuracy"; r["experiment"] = "E2"; out << r.dump() << "\n"; } out.flush(); std::fprintf(stderr, "E2 accuracy h=%d done\n", h); } 38 for (auto& r : rows) { r["kind"] = "accuracy"; r["experiment"] = "E2"; out << r.dump() << "\n"; } out.flush(); std::fprintf(stderr, "E2 accuracy h=%d done\n", h); }
39 // (b) runtime: per-series fits vs batched sufficient statistics 39 // (b) runtime: per-series fits vs batched sufficient statistics
40− for (usize S : {10ul, 100ul, 1000ul}) { if (S > syms.size()) continue; std::vector<f64> cat; std::vector<i64> off = {0}; for (usize i = 0; i < S; ++i) { cat.insert(cat.end(), rv[syms[i]].begin(), rv[syms[i]].end()); off.push_back(static_cast<i64>(cat.size())); } 40+ for (usize S0 : {10ul, 100ul, 1000ul}) { const usize S = std::min(S0, syms.size()); if (S0 > syms.size() && S0 / 10 >= syms.size()) continue; std::vector<f64> cat; std::vector<i64> off = {0}; for (usize i = 0; i < S; ++i) { cat.insert(cat.end(), rv[syms[i]].begin(), rv[syms[i]].end()); off.push_back(static_cast<i64>(cat.size())); }
41 est::HarSpec spec; spec.horizon = 1; spec.cov.type = stats::CovType::classical; 41 est::HarSpec spec; spec.horizon = 1; spec.cov.type = stats::CovType::classical;
42 for (auto mode : {"per_series_cpu1", "per_series_cpuMT", "batched_cpu", "batched_metal"}) for (int rep = 0; rep < reps; ++rep) { const f64 t0 = now(); usize okc = 0; std::string backend_used = "cpu"; f64 dev = 0; 42 for (auto mode : {"per_series_cpu1", "per_series_cpuMT", "batched_cpu", "batched_metal"}) for (int rep = 0; rep < reps; ++rep) { const f64 t0 = now(); usize okc = 0; std::string backend_used = "cpu"; f64 dev = 0;
43 if (std::string(mode).rfind("per_series", 0) == 0) { std::vector<int> ok(S, 0); auto fitall = [&](usize a, usize b) { for (usize i = a; i < b; ++i) { try { auto e = est::har_fit(rv[syms[i]], spec); ok[i] = e.ok(); } catch (...) {} } }; if (std::string(mode) == "per_series_cpu1") fitall(0, S); else ThreadPool::global().parallel_for(S, fitall); for (int v : ok) okc += v; } 43 if (std::string(mode).rfind("per_series", 0) == 0) { std::vector<int> ok(S, 0); auto fitall = [&](usize a, usize b) { for (usize i = a; i < b; ++i) { try { auto e = est::har_fit(rv[syms[i]], spec); ok[i] = e.ok(); } catch (...) {} } }; if (std::string(mode) == "per_series_cpu1") fitall(0, S); else ThreadPool::global().parallel_for(S, fitall); for (int v : ok) okc += v; }
modified benchmarks/end_to_end/bench_e3_garch.cpp +7 −0
@@ -25,10 +25,17 @@ int main(int argc, char** argv) {
25 return 1; 25 return 1;
26 #else 26 #else
27 json e0 = json::parse(std::ifstream(argv[1])); const usize Smax = argc > 3 ? static_cast<usize>(std::atoi(argv[3])) : 1000; std::ofstream out(argv[2], std::ios::app); 27 json e0 = json::parse(std::ifstream(argv[1])); const usize Smax = argc > 3 ? static_cast<usize>(std::atoi(argv[3])) : 1000; std::ofstream out(argv[2], std::ios::app);
28+ const std::string daily_path = argc > 4 ? argv[4] : ""; // e0_daily.json: 10-year daily bars → long training histories
28 ExperimentManifest man; man.experiment = "E3.garch"; man.created_at_utc = now_utc_iso(); man.raid_version = RAID_VERSION; man.host = host_info(); man.inputs["e0_universe.json"] = Sha256::of_file(argv[1]); 29 ExperimentManifest man; man.experiment = "E3.garch"; man.created_at_utc = now_utc_iso(); man.raid_version = RAID_VERSION; man.host = host_info(); man.inputs["e0_universe.json"] = Sha256::of_file(argv[1]);
29 MarketCalendar cal; std::map<std::string, std::vector<f64>> ret, rvd; 30 MarketCalendar cal; std::map<std::string, std::vector<f64>> ret, rvd;
30 for (auto& ex : e0["exports"]) { auto pb = data::read_bars_parquet(ex["path"].get<std::string>()); for (auto& [sym, b] : pb.series) { auto reg = b.filter_session(cal, SessionPolicy::regular); if (reg.n_sessions() < 240) continue; auto ir = data::intraday_returns(reg); std::vector<f64> r; for (usize k = 1; k < ir.close_to_close.size(); ++k) r.push_back(ir.close_to_close[k]); est::RealizedOptions ro; ro.plan.backend = Backend::cpu; auto m = est::realized_measures(ir, ro); ret[sym] = r; rvd[sym] = std::vector<f64>(m.rv.begin() + 1, m.rv.end()); } } 31 for (auto& ex : e0["exports"]) { auto pb = data::read_bars_parquet(ex["path"].get<std::string>()); for (auto& [sym, b] : pb.series) { auto reg = b.filter_session(cal, SessionPolicy::regular); if (reg.n_sessions() < 240) continue; auto ir = data::intraday_returns(reg); std::vector<f64> r; for (usize k = 1; k < ir.close_to_close.size(); ++k) r.push_back(ir.close_to_close[k]); est::RealizedOptions ro; ro.plan.backend = Backend::cpu; auto m = est::realized_measures(ir, ro); ret[sym] = r; rvd[sym] = std::vector<f64>(m.rv.begin() + 1, m.rv.end()); } }
32+ // replace the 2024-only close-to-close returns by the 2015–2024 daily history when available (the 2024 intraday RV stays the evaluation target)
33+ usize replaced = 0; if (!daily_path.empty()) { json ed = json::parse(std::ifstream(daily_path)); man.inputs["e0_daily.json"] = Sha256::of_file(daily_path);
34+ for (auto& ex : ed["exports"]) { auto pb = data::read_bars_parquet(ex["path"].get<std::string>(), {}, Interval::d1); for (auto& [sym, b] : pb.series) { if (!ret.count(sym) || b.size() < 500) continue; std::vector<f64> r; for (usize i = 1; i < b.size(); ++i) if (b.close[i] > 0 && b.close[i - 1] > 0) r.push_back(std::log(b.close[i] / b.close[i - 1]));
35+ // keep the last `test` 2024 sessions as evaluation window: the daily history already covers 2024, so align by count: the last 40 returns of the daily series should correspond to the last 40 intraday sessions (both end 2024-12-31)
36+ ret[sym] = r; ++replaced; } } }
31 const auto universe = e0["samples"]["1000"].get<std::vector<std::string>>(); std::vector<std::string> syms; for (auto& s : universe) if (ret.count(s) && syms.size() < Smax) syms.push_back(s); 37 const auto universe = e0["samples"]["1000"].get<std::vector<std::string>>(); std::vector<std::string> syms; for (auto& s : universe) if (ret.count(s) && syms.size() < Smax) syms.push_back(s);
38+ out << json({{"kind", "load"}, {"experiment", "E3"}, {"symbols", syms.size()}, {"daily_history_replaced", replaced}}).dump() << "\n";
32 const usize S = syms.size(); std::fprintf(stderr, "E3: %zu symbols, ~%zu daily returns each\n", S, ret[syms[0]].size()); 39 const usize S = syms.size(); std::fprintf(stderr, "E3: %zu symbols, ~%zu daily returns each\n", S, ret[syms[0]].size());
33 const usize test = 40; // last 40 sessions held out for forecast evaluation 40 const usize test = 40; // last 40 sessions held out for forecast evaluation
34 // ---- (i) per-symbol fits, all models, CPU parallel over symbols 41 // ---- (i) per-symbol fits, all models, CPU parallel over symbols
modified benchmarks/end_to_end/bench_e4_connectedness.cpp +2 −2
@@ -30,12 +30,12 @@ int main(int argc, char** argv) {
30 MarketCalendar cal; const auto universe = e0["samples"]["1000"].get<std::vector<std::string>>(); 30 MarketCalendar cal; const auto universe = e0["samples"]["1000"].get<std::vector<std::string>>();
31 // load up to 100 symbols with full coverage; 5-minute regular-session bars; synchronized grid = the 79 bins per session 31 // load up to 100 symbols with full coverage; 5-minute regular-session bars; synchronized grid = the 79 bins per session
32 std::map<std::string, BarSeries> five; std::map<std::string, std::vector<f64>> logrv; 32 std::map<std::string, BarSeries> five; std::map<std::string, std::vector<f64>> logrv;
33− for (auto& ex : e0["exports"]) { if (five.size() >= 100) break; auto pb = data::read_bars_parquet(ex["path"].get<std::string>()); for (auto& [sym, b] : pb.series) { if (five.size() >= 100) break; auto reg = b.filter_session(cal, SessionPolicy::regular); auto q = reg.validate(cal); if (q.n_sessions < 250 || q.coverage < 0.98) continue; five[sym] = data::aggregate(reg, Interval::m5, cal); auto ir = data::intraday_returns(reg); est::RealizedOptions ro; ro.plan.backend = Backend::cpu; auto m = est::realized_measures(ir, ro); std::vector<f64> lv; for (f64 v : m.rv) lv.push_back(std::log(std::max(v, 1e-12))); logrv[sym] = lv; } } 33+ for (auto& ex : e0["exports"]) { if (five.size() >= 100) break; auto pb = data::read_bars_parquet(ex["path"].get<std::string>()); for (auto& [sym, b] : pb.series) { if (five.size() >= 100) break; auto reg = b.filter_session(cal, SessionPolicy::regular); auto q = reg.validate(cal); if (q.n_sessions < 250 || q.coverage < 0.95) continue; five[sym] = data::aggregate(reg, Interval::m5, cal); auto ir = data::intraday_returns(reg); est::RealizedOptions ro; ro.plan.backend = Backend::cpu; auto m = est::realized_measures(ir, ro); std::vector<f64> lv; for (f64 v : m.rv) lv.push_back(std::log(std::max(v, 1e-12))); logrv[sym] = lv; } }
34 std::vector<std::string> syms; for (auto& s : universe) if (five.count(s)) syms.push_back(s); std::fprintf(stderr, "E4: %zu symbols with >= 250 sessions and coverage >= 0.98\n", syms.size()); 34 std::vector<std::string> syms; for (auto& s : universe) if (five.count(s)) syms.push_back(s); std::fprintf(stderr, "E4: %zu symbols with >= 250 sessions and coverage >= 0.98\n", syms.size());
35 // common sessions 35 // common sessions
36 std::set<Date> common; bool first = true; for (auto& s : syms) { std::set<Date> d(five[s].session_dates.begin(), five[s].session_dates.end()); if (first) { common = d; first = false; } else { std::set<Date> inter; for (auto x : common) if (d.count(x)) inter.insert(x); common = inter; } } 36 std::set<Date> common; bool first = true; for (auto& s : syms) { std::set<Date> d(five[s].session_dates.begin(), five[s].session_dates.end()); if (first) { common = d; first = false; } else { std::set<Date> inter; for (auto x : common) if (d.count(x)) inter.insert(x); common = inter; } }
37 std::vector<Date> sessions(common.begin(), common.end()); std::fprintf(stderr, "E4: %zu common sessions\n", sessions.size()); 37 std::vector<Date> sessions(common.begin(), common.end()); std::fprintf(stderr, "E4: %zu common sessions\n", sessions.size());
38− for (usize N : {10ul, 30ul, 100ul}) { if (N > syms.size()) continue; 38+ for (usize N0 : {10ul, 30ul, 60ul, 100ul}) { const usize N = std::min(N0, syms.size()); if (N0 > syms.size() && N0 / 2 >= syms.size()) continue;
39 // (a) daily realized covariance from synchronized 5-minute returns (bins matched by timestamp; a day is used only if all N have the full 79 bins) 39 // (a) daily realized covariance from synchronized 5-minute returns (bins matched by timestamp; a day is used only if all N have the full 79 bins)
40 const f64 t0 = now(); usize days_ok = 0, days_skipped = 0; f64 cond_max = 0, cond_med = 0; std::vector<f64> conds; f64 lw_shrink_mean = 0; 40 const f64 t0 = now(); usize days_ok = 0, days_skipped = 0; f64 cond_max = 0, cond_med = 0; std::vector<f64> conds; f64 lw_shrink_mean = 0;
41 for (auto d : sessions) { std::vector<std::vector<f64>> R(N); bool ok = true; for (usize i = 0; i < N && ok; ++i) { auto& b = five[syms[i]]; usize k = 0; while (k < b.n_sessions() && b.session_dates[k] != d) ++k; if (k == b.n_sessions()) { ok = false; break; } auto c = b.session_closes(k); const usize expect = cal.expected_regular_bars(d, Interval::m5); if (c.size() != expect) { ok = false; break; } R[i] = data::returns(c); } 41 for (auto d : sessions) { std::vector<std::vector<f64>> R(N); bool ok = true; for (usize i = 0; i < N && ok; ++i) { auto& b = five[syms[i]]; usize k = 0; while (k < b.n_sessions() && b.session_dates[k] != d) ++k; if (k == b.n_sessions()) { ok = false; break; } auto c = b.session_closes(k); const usize expect = cal.expected_regular_bars(d, Interval::m5); if (c.size() != expect) { ok = false; break; } R[i] = data::returns(c); }
added experiments/results/e0_daily.json +1119 −0
@@ -0,0 +1,1119 @@
1+{
2+ "dataset_id": "market_bars.stock.1d.raw",
3+ "start": "2015-01-01",
4+ "end": "2024-12-31",
5+ "exports": [
6+ {
7+ "symbols": [
8+ "FIVN",
9+ "FISV",
10+ "POET",
11+ "CRNC",
12+ "SIBN",
13+ "PIPR",
14+ "W",
15+ "VNDA",
16+ "DNN",
17+ "BAC.E",
18+ "GM",
19+ "SACH",
20+ "PKG",
21+ "RLJ.A",
22+ "SPWR",
23+ "NTR",
24+ "TSCO",
25+ "PLNT",
26+ "AXR",
27+ "TEI",
28+ "BAC.N",
29+ "YHGJ",
30+ "DMRC",
31+ "NAN",
32+ "GSIT",
33+ "EAD",
34+ "BLDR",
35+ "USB.R",
36+ "JXN.A",
37+ "ZNTL",
38+ "EFR",
39+ "IGTA",
40+ "FENC",
41+ "AVBH",
42+ "FFBW",
43+ "DRMA",
44+ "SLMT",
45+ "DTIL",
46+ "XPOF",
47+ "AZTR",
48+ "TRVG",
49+ "ORKA",
50+ "SCL",
51+ "FBNC",
52+ "DHCNI",
53+ "PCT",
54+ "SWK",
55+ "HYFT",
56+ "FCUV",
57+ "LPTX",
58+ "CAN",
59+ "ANVS",
60+ "NFJ",
61+ "ARMK",
62+ "RCI",
63+ "SUPV",
64+ "SPRC",
65+ "VCV",
66+ "LITE",
67+ "BHFAP",
68+ "ROP",
69+ "ATMCR",
70+ "WETH",
71+ "FUBO",
72+ "CLBT",
73+ "AVAH",
74+ "WWW",
75+ "WHR",
76+ "BTMD",
77+ "AHH.A",
78+ "VOR",
79+ "IBCP",
80+ "NA",
81+ "BEAT",
82+ "PEB.G",
83+ "PUK",
84+ "TRGP",
85+ "DGICA",
86+ "UCL",
87+ "NNI",
88+ "SF.D",
89+ "NUE",
90+ "UMBF",
91+ "STZ",
92+ "ARQQW",
93+ "AIR",
94+ "CHAI",
95+ "PAYX",
96+ "AKAM",
97+ "AAP",
98+ "CZNC",
99+ "LTH",
100+ "AME",
101+ "SVVC",
102+ "CXW",
103+ "MNSO",
104+ "MMLP",
105+ "SPNT",
106+ "FORL",
107+ "JKHY"
108+ ],
109+ "export_id": "475957bf-aec0-4967-aa14-cde9834a0c4c",
110+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_096d2d80a5ebea518dc4ec89.parquet",
111+ "sha256": "efa0565bc66b3ca80634502a313e17906ef149b4b29ef23cd07631b491749722",
112+ "dataset_version": 7752,
113+ "rows": 190988,
114+ "bytes": 2626979,
115+ "from_cache": false
116+ },
117+ {
118+ "symbols": [
119+ "AEM",
120+ "STE",
121+ "ISPOW",
122+ "SDOT",
123+ "CRDO",
124+ "PCYO",
125+ "XOM",
126+ "KIO",
127+ "REGCO",
128+ "BLX",
129+ "OCCI",
130+ "ZAPPW",
131+ "MUSA",
132+ "RDCM",
133+ "RCKT",
134+ "FAMI",
135+ "RILYT",
136+ "IPWR",
137+ "TLSIW",
138+ "IEP",
139+ "SITC",
140+ "CC",
141+ "ORMP",
142+ "DXCM",
143+ "RPT",
144+ "GAP",
145+ "NWG",
146+ "PGP",
147+ "CRCT",
148+ "TAPM",
149+ "AFRM",
150+ "AXIA",
151+ "SHIP",
152+ "XBIO",
153+ "ILAG",
154+ "GBLI",
155+ "FSLR",
156+ "BGM",
157+ "TRUP",
158+ "ADMA",
159+ "OSIS",
160+ "AMOD",
161+ "ISPC",
162+ "CTOS",
163+ "HIMS",
164+ "UONE",
165+ "LAC",
166+ "PK",
167+ "ABR.F",
168+ "BRAG",
169+ "SHFSW",
170+ "HPP.C",
171+ "CHNR",
172+ "SHPH",
173+ "ENTX",
174+ "MFA.B",
175+ "NPCT",
176+ "FINS",
177+ "CZWI",
178+ "CG",
179+ "AU",
180+ "IKT",
181+ "CERT",
182+ "NPO",
183+ "RWAY",
184+ "BBVA",
185+ "TARA",
186+ "DUK.A",
187+ "CRTDW",
188+ "INVZW",
189+ "MS.K",
190+ "CVM",
191+ "WCFB",
192+ "ATOM",
193+ "NOVT",
194+ "SMIT",
195+ "LBTYB",
196+ "ACR",
197+ "EOCN",
198+ "BOC",
199+ "BGX",
200+ "MRT",
201+ "WKC",
202+ "SUAC",
203+ "TV",
204+ "GHG",
205+ "FRPH",
206+ "EVT",
207+ "ASTL",
208+ "MU",
209+ "PTA",
210+ "LPCN",
211+ "INSM",
212+ "PEB.E",
213+ "ABLV",
214+ "ARCT",
215+ "TVC",
216+ "IDXG",
217+ "GSBD",
218+ "TSLX"
219+ ],
220+ "export_id": "bc9dd606-b0a9-48ea-a98a-71440e70e44b",
221+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_79c1cdbb68bbe4e14c81b519.parquet",
222+ "sha256": "0a49cbf11c0198663e6aad04387ac5827406b21dde21bc2a24aa505b501ba17e",
223+ "dataset_version": 7752,
224+ "rows": 174888,
225+ "bytes": 2405186,
226+ "from_cache": false
227+ },
228+ {
229+ "symbols": [
230+ "NKE",
231+ "COST",
232+ "ACA",
233+ "NTAP",
234+ "PEBO",
235+ "AEO",
236+ "HWBK",
237+ "GIC",
238+ "BAC.M",
239+ "ECBK",
240+ "RDNT",
241+ "SPCB",
242+ "MDWD",
243+ "ANGHW",
244+ "EPD",
245+ "NEA",
246+ "VRTX",
247+ "BIIB",
248+ "TDY",
249+ "JOB",
250+ "FWRG",
251+ "CPHI",
252+ "DUO",
253+ "SMP",
254+ "EYE",
255+ "BCPC",
256+ "CIM.A",
257+ "CNP",
258+ "BCDS",
259+ "VGASW",
260+ "XPO",
261+ "UAN",
262+ "CWD",
263+ "DRVN",
264+ "HLSQ",
265+ "ATRA",
266+ "ZIVOW",
267+ "LCFYW",
268+ "SMFL",
269+ "LDTCW",
270+ "CCOI",
271+ "NIVF",
272+ "BP",
273+ "SCE.M",
274+ "BUSE",
275+ "TCCO",
276+ "RGCO",
277+ "MIDD",
278+ "RDW",
279+ "CMT",
280+ "ENX",
281+ "UBS",
282+ "OSPN",
283+ "KGC",
284+ "LKFT",
285+ "STM",
286+ "AOGO",
287+ "HTD",
288+ "GOOS",
289+ "VRMEW",
290+ "BBIO",
291+ "HRZN",
292+ "PSA.K",
293+ "NXGL",
294+ "SCYX",
295+ "KMI",
296+ "NXP",
297+ "MGM",
298+ "HYFM",
299+ "UOKA",
300+ "FBYDW",
301+ "GUT",
302+ "FSTR",
303+ "SAJ",
304+ "MCI",
305+ "DGICB",
306+ "CMP",
307+ "AAPL",
308+ "OFED",
309+ "BBDC",
310+ "WTBA",
311+ "USGO",
312+ "MLKN",
313+ "TRU",
314+ "ARCC",
315+ "TLF",
316+ "GFF",
317+ "RSKD",
318+ "VEL",
319+ "TCRX",
320+ "ALVO",
321+ "GLLIW",
322+ "OSS",
323+ "RMBI",
324+ "RZC",
325+ "KNTK",
326+ "CMRA",
327+ "PAHC",
328+ "TACT",
329+ "QVCD"
330+ ],
331+ "export_id": "c249857c-3d56-472c-b387-c344d59d691e",
332+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_e9c18d86bd0a5e2c1402c2b6.parquet",
333+ "sha256": "23306fe3afaebe861ef7e282e3ce183540b1de0941e588de8ae058f15b98d1ad",
334+ "dataset_version": 7752,
335+ "rows": 178089,
336+ "bytes": 2555305,
337+ "from_cache": false
338+ },
339+ {
340+ "symbols": [
341+ "PMCB",
342+ "REAX",
343+ "FFC",
344+ "AIXI",
345+ "FGB",
346+ "HYI",
347+ "SHAK",
348+ "WTMA",
349+ "DBI",
350+ "CLVRW",
351+ "ZION",
352+ "GIB",
353+ "DCOY",
354+ "RCON",
355+ "CANG",
356+ "COFS",
357+ "KNF",
358+ "ZWS",
359+ "DRS",
360+ "CNSY",
361+ "ETJ",
362+ "GEN",
363+ "UCTT",
364+ "ADSE",
365+ "SOWG",
366+ "SLDPW",
367+ "SLVM",
368+ "PHD",
369+ "CCEP",
370+ "KULR",
371+ "TMP",
372+ "TAC",
373+ "PNI",
374+ "MHLA",
375+ "REI",
376+ "TSBK",
377+ "NVNO",
378+ "DLR.L",
379+ "STNE",
380+ "INFU",
381+ "MBIN",
382+ "AGL",
383+ "RITM",
384+ "MAT",
385+ "OGCP",
386+ "BOWN",
387+ "ZYME",
388+ "BEN",
389+ "KMT",
390+ "AEG",
391+ "FMBH",
392+ "ZDGE",
393+ "GEL",
394+ "BIRK",
395+ "EXFY",
396+ "VSSYW",
397+ "MSAIW",
398+ "VKQ",
399+ "FLYX",
400+ "XELB",
401+ "DOUG",
402+ "SOHON",
403+ "INDB",
404+ "INVE",
405+ "GJT",
406+ "GTX",
407+ "CASH",
408+ "ARBK",
409+ "BEPC",
410+ "CALA",
411+ "OBE",
412+ "LEU",
413+ "ASA",
414+ "WINA",
415+ "HLIO",
416+ "SHFS",
417+ "MOH",
418+ "PGYWW",
419+ "NGL",
420+ "BKN",
421+ "BBAR",
422+ "RDNW",
423+ "YCBD.A",
424+ "MO",
425+ "BGT",
426+ "VUZI",
427+ "FBIOP",
428+ "COLD",
429+ "BOKF",
430+ "EFT",
431+ "NBSTU",
432+ "YETI",
433+ "MCOMW",
434+ "VLY",
435+ "DBRG.J",
436+ "KUKE",
437+ "GTN.A",
438+ "LIQT",
439+ "RDIB",
440+ "BF.B"
441+ ],
442+ "export_id": "7b98e722-d8e3-49f8-817d-8d9a5e5d3b15",
443+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_c025209b1375c8792c71884f.parquet",
444+ "sha256": "4101b621a67b733d9c83ce5da4dfa48ea32a04ceb2cc3ae21c71e98952258981",
445+ "dataset_version": 7752,
446+ "rows": 180100,
447+ "bytes": 2393835,
448+ "from_cache": false
449+ },
450+ {
451+ "symbols": [
452+ "CMC",
453+ "OGN",
454+ "MCB",
455+ "TSQ",
456+ "AEMD",
457+ "NEOVW",
458+ "KLRS",
459+ "CIG",
460+ "GECC",
461+ "DLNG.A",
462+ "IMPP",
463+ "MHK",
464+ "BIDU",
465+ "CBSH",
466+ "MTRN",
467+ "BKU",
468+ "CELUW",
469+ "OLB",
470+ "YOTAR",
471+ "NOA",
472+ "IKNA",
473+ "U",
474+ "MLEC",
475+ "IOR",
476+ "TNET",
477+ "RTB",
478+ "GRI",
479+ "MLM",
480+ "SEI",
481+ "MITI",
482+ "HPE",
483+ "KRO",
484+ "OESX",
485+ "PETS",
486+ "NEWTZ",
487+ "GNW",
488+ "HSCSW",
489+ "SNDR",
490+ "LWLG",
491+ "RLJ",
492+ "SBAC",
493+ "ZIM",
494+ "CETXP",
495+ "PPTA",
496+ "ALL.B",
497+ "MESO",
498+ "MDBH",
499+ "CWST",
500+ "JWEL",
501+ "RRX",
502+ "CRTO",
503+ "PCG.I",
504+ "PSQH",
505+ "SOJC",
506+ "CHCO",
507+ "FUTU",
508+ "PFBC",
509+ "IH",
510+ "SEVN",
511+ "BHFAL",
512+ "IESC",
513+ "INDP",
514+ "HAIN",
515+ "NBBK",
516+ "INTT",
517+ "GRBK",
518+ "STC",
519+ "BALL",
520+ "XEL",
521+ "NKX",
522+ "MBNKP",
523+ "DIT",
524+ "CERS",
525+ "GTEC",
526+ "BBWI",
527+ "ENLV",
528+ "RL",
529+ "MPAA",
530+ "NEUP",
531+ "UMH.D",
532+ "RS",
533+ "CARS",
534+ "FENG",
535+ "CCM",
536+ "MSEX",
537+ "SYTAW",
538+ "SGLY",
539+ "DWSN",
540+ "LESL",
541+ "JMIA",
542+ "DKNG",
543+ "HOLOW",
544+ "FNUC",
545+ "OMC",
546+ "OCEAW",
547+ "OR",
548+ "GEGGL",
549+ "GGR",
550+ "TCRT",
551+ "FTAIN"
552+ ],
553+ "export_id": "b789b7ea-254e-405f-9995-22df39b3c62a",
554+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_e4288d2c786bd273d9f0bd5a.parquet",
555+ "sha256": "b0c53b01bc310e53bb70058c4e85aca1750419e777402cb18b1ec774da9a2f64",
556+ "dataset_version": 7752,
557+ "rows": 178145,
558+ "bytes": 2538589,
559+ "from_cache": false
560+ },
561+ {
562+ "symbols": [
563+ "SAN",
564+ "LRN",
565+ "FTII",
566+ "ABSI",
567+ "SF",
568+ "AVO",
569+ "MKC.V",
570+ "MCAGR",
571+ "TC",
572+ "CATY",
573+ "IGTAR",
574+ "PDYN",
575+ "BFLY",
576+ "XNET",
577+ "FGI",
578+ "PPSI",
579+ "TILE",
580+ "MXF",
581+ "OXLCN",
582+ "NTWK",
583+ "HR",
584+ "SONY",
585+ "AURE",
586+ "RIG",
587+ "PACB",
588+ "BGC",
589+ "MIRA",
590+ "NRIM",
591+ "DMB",
592+ "OPY",
593+ "MIO",
594+ "E",
595+ "IMTE",
596+ "FOSL",
597+ "VIAV",
598+ "DB",
599+ "ARWR",
600+ "JILL",
601+ "LPL",
602+ "BURL",
603+ "STRO",
604+ "ATGL",
605+ "BML.L",
606+ "GNFT",
607+ "VOD",
608+ "HYT",
609+ "MHH",
610+ "KMDA",
611+ "CTNT",
612+ "KEY.L",
613+ "RILYL",
614+ "NEE.R",
615+ "CNF",
616+ "LNSR",
617+ "PNBK",
618+ "ENB",
619+ "OVLY",
620+ "ALMU",
621+ "HTH",
622+ "ATEC",
623+ "APPN",
624+ "MAN",
625+ "AMBP",
626+ "CRIS",
627+ "ELVA",
628+ "FPI",
629+ "ORC",
630+ "GWW",
631+ "RF.E",
632+ "SFST",
633+ "TAK",
634+ "COO",
635+ "USCB",
636+ "WAL",
637+ "WTFCP",
638+ "HLGN",
639+ "HCVIW",
640+ "EXK",
641+ "OPCH",
642+ "BMBL",
643+ "GECCO",
644+ "SOND",
645+ "HPS",
646+ "MXE",
647+ "HPK",
648+ "IDT",
649+ "NRC",
650+ "DDS",
651+ "GDV.H",
652+ "QUAD",
653+ "MITT.B",
654+ "GAM",
655+ "USB.Q",
656+ "TAYD",
657+ "TCBX",
658+ "ARES",
659+ "DTSS",
660+ "FGN",
661+ "LNAI",
662+ "BZUN"
663+ ],
664+ "export_id": "7180ac6e-1c1d-4b6a-9c52-b296a20afa29",
665+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_3b2acab37d0a377a62f2564b.parquet",
666+ "sha256": "b1398165920a3f76c84d27305f5196aa82aceba13db393a26c072aca689cde0f",
667+ "dataset_version": 7752,
668+ "rows": 181007,
669+ "bytes": 2465058,
670+ "from_cache": false
671+ },
672+ {
673+ "symbols": [
674+ "BFS.D",
675+ "SMRT",
676+ "ARE",
677+ "BNY",
678+ "TPR",
679+ "BFS.E",
680+ "NXDT",
681+ "PBYI",
682+ "GLAD",
683+ "NERV",
684+ "CHSCM",
685+ "ACN",
686+ "OUT",
687+ "NLY.I",
688+ "MDB",
689+ "CLST",
690+ "ACT",
691+ "PRKS",
692+ "ERAS",
693+ "RDVT",
694+ "WFCF",
695+ "QFIN",
696+ "NXPLW",
697+ "OMH",
698+ "BVFL",
699+ "UVSP",
700+ "LCID",
701+ "NOV",
702+ "RLAY",
703+ "BNTC",
704+ "ALLY",
705+ "DSL",
706+ "OABIW",
707+ "ETW",
708+ "KMPR",
709+ "SUZ",
710+ "FFBC",
711+ "ETN",
712+ "PW.A",
713+ "NVNI",
714+ "TDOC",
715+ "NAMSW",
716+ "STWD",
717+ "ACNB",
718+ "FNWB",
719+ "BAOS",
720+ "BPOP",
721+ "DNTH",
722+ "FAX",
723+ "NWN",
724+ "UPXI",
725+ "BTCS",
726+ "LMNR",
727+ "HOOD",
728+ "RMCO",
729+ "GBX",
730+ "DVLT",
731+ "XTLB",
732+ "EME",
733+ "KUST",
734+ "KITTW",
735+ "NSA.A",
736+ "RCUS",
737+ "HBANM",
738+ "JCE",
739+ "LDTC",
740+ "UL",
741+ "AMRN",
742+ "ELF",
743+ "BL",
744+ "MUE",
745+ "UP",
746+ "HASI",
747+ "BEKE",
748+ "BTAI",
749+ "NESR",
750+ "PJT",
751+ "HTCR",
752+ "CCSI",
753+ "ABTS",
754+ "SKYW",
755+ "FTIIW",
756+ "EPR.G",
757+ "IQI",
758+ "MGA",
759+ "HIND",
760+ "CRVS",
761+ "OZ",
762+ "ALNY",
763+ "MRM",
764+ "INTEW",
765+ "TXG",
766+ "NE",
767+ "AORT",
768+ "KARO",
769+ "MI",
770+ "TARS",
771+ "BFS",
772+ "NOMD",
773+ "BMA"
774+ ],
775+ "export_id": "2784e2bb-27a0-4584-b512-aa1aff23bd46",
776+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_6ec045cd7171c7856bdb3cec.parquet",
777+ "sha256": "ba98fd11aa47f7088668e3963040cf37e8b73284436ede0f9b4cfcc3ad868aa6",
778+ "dataset_version": 7752,
779+ "rows": 178845,
780+ "bytes": 2488194,
781+ "from_cache": false
782+ },
783+ {
784+ "symbols": [
785+ "NYMTL",
786+ "IHG",
787+ "CPOP",
788+ "CGTX",
789+ "OLLI",
790+ "CIB",
791+ "ABCL",
792+ "LHX",
793+ "BBUC",
794+ "ACRS",
795+ "AISP",
796+ "COHN",
797+ "HOMB",
798+ "FDBC",
799+ "ELA",
800+ "OCUL",
801+ "BAP",
802+ "CNX",
803+ "NET",
804+ "WBS.F",
805+ "TXN",
806+ "DYNT",
807+ "AMPL",
808+ "BCV.A",
809+ "CHRW",
810+ "GDL",
811+ "FORM",
812+ "IVCAW",
813+ "LBRT",
814+ "BJRI",
815+ "CLM",
816+ "BYSI",
817+ "FITB",
818+ "BXMT",
819+ "BROG",
820+ "BFRG",
821+ "OCFC",
822+ "AEAE",
823+ "THQ",
824+ "INTE",
825+ "VMD",
826+ "SVV",
827+ "ATLC",
828+ "KVUE",
829+ "RH",
830+ "GGT",
831+ "UPST",
832+ "CDTTW",
833+ "ANIK",
834+ "PYT",
835+ "THO",
836+ "CDR.B",
837+ "OI",
838+ "MCS",
839+ "HBANL",
840+ "DRD",
841+ "WPM",
842+ "AWF",
843+ "MQT",
844+ "MARA",
845+ "FOXA",
846+ "GIS",
847+ "DCOM",
848+ "KXIN",
849+ "LTSH",
850+ "ULCC",
851+ "NOAH",
852+ "MDAIW",
853+ "IIPR",
854+ "NNDM",
855+ "HUM",
856+ "FLNT",
857+ "FAF",
858+ "GAB.K",
859+ "NVCR",
860+ "ATO",
861+ "TPC",
862+ "GALT",
863+ "TDS",
864+ "OSBC",
865+ "MYO",
866+ "NVGS",
867+ "DAR",
868+ "VGAS",
869+ "XMAX",
870+ "CASY",
871+ "CHMI.A",
872+ "CLDT.A",
873+ "HTHT",
874+ "ALDX",
875+ "OCCIO",
876+ "CNXN",
877+ "SDHY",
878+ "SLE",
879+ "SNAP",
880+ "TISI",
881+ "LUCD",
882+ "RVPHW",
883+ "MAQC",
884+ "PLGO"
885+ ],
886+ "export_id": "5670c551-2828-4bf0-9907-4e1450a9b807",
887+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_8910c25d02924c31abc29226.parquet",
888+ "sha256": "9829f79d1c75aaabb18d0b5b2cd22cb432096e48296286065f6772fbada5f592",
889+ "dataset_version": 7752,
890+ "rows": 185071,
891+ "bytes": 2546490,
892+ "from_cache": false
893+ },
894+ {
895+ "symbols": [
896+ "TMH",
897+ "MWA",
898+ "TENB",
899+ "CLAR",
900+ "URI",
901+ "AKO.B",
902+ "CAH",
903+ "BHVN",
904+ "AZZ",
905+ "YCBD",
906+ "SACH.A",
907+ "ADPT",
908+ "DX",
909+ "CIFRW",
910+ "DLNG",
911+ "NXTC",
912+ "CVLT",
913+ "EMPD",
914+ "PRFX",
915+ "BLKB",
916+ "CACC",
917+ "LSBK",
918+ "MOVE",
919+ "SR.A",
920+ "CRVL",
921+ "CGBDL",
922+ "HBT",
923+ "MAYS",
924+ "NVAC",
925+ "BNED",
926+ "VPG",
927+ "RILYN",
928+ "ALL.H",
929+ "CZR",
930+ "ACGL",
931+ "NCNA",
932+ "ONBPP",
933+ "OBK",
934+ "FHI",
935+ "ZCARW",
936+ "HOUR",
937+ "EQH.C",
938+ "TEN.F",
939+ "AXTA",
940+ "UNP",
941+ "DARE",
942+ "NVRI",
943+ "FDX",
944+ "NVVEW",
945+ "NYXH",
946+ "SWKHL",
947+ "NMS",
948+ "LXU",
949+ "PALI",
950+ "MTZ",
951+ "TITN",
952+ "NEXT",
953+ "SAP",
954+ "BON",
955+ "SHO.I",
956+ "PRCT",
957+ "RBLX",
958+ "YSG",
959+ "MODD",
960+ "TEL",
961+ "PFXNZ",
962+ "BUR",
963+ "NXE",
964+ "ECC",
965+ "AFRI",
966+ "RCKTW",
967+ "ARAY",
968+ "FRHC",
969+ "STKS",
970+ "SRCE",
971+ "STEW",
972+ "MYFW",
973+ "CNQ",
974+ "HAL",
975+ "SIMO",
976+ "KRUS",
977+ "ABG",
978+ "APT",
979+ "MHNC",
980+ "PTIXW",
981+ "CLPR",
982+ "SLN",
983+ "HROWL",
984+ "AMSF",
985+ "SNWV",
986+ "SCHW.J",
987+ "MYRG",
988+ "BBSI",
989+ "VABK",
990+ "NBTB",
991+ "GRFS",
992+ "LINK",
993+ "PSFE",
994+ "AVNW",
995+ "LFT"
996+ ],
997+ "export_id": "913cab39-96b7-45f2-97df-90a938302456",
998+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_342ea3f92472ecf2950bd217.parquet",
999+ "sha256": "65357325da94c251fddbaf20926ca965550408b9ce8d43edbb8149efb9993fc5",
1000+ "dataset_version": 7752,
1001+ "rows": 173195,
1002+ "bytes": 2503747,
1003+ "from_cache": false
1004+ },
1005+ {
1006+ "symbols": [
1007+ "DKL",
1008+ "NXRT",
1009+ "SXI",
1010+ "TUYA",
1011+ "ALNT",
1012+ "AWP",
1013+ "RACE",
1014+ "PDFS",
1015+ "BCDA",
1016+ "EOD",
1017+ "PWR",
1018+ "YAYO",
1019+ "KLTR",
1020+ "AGCO",
1021+ "AA",
1022+ "SAR",
1023+ "GNE",
1024+ "PAA",
1025+ "PVH",
1026+ "JAKK",
1027+ "XENE",
1028+ "BGR",
1029+ "SQNS",
1030+ "WAVE",
1031+ "PKX",
1032+ "VNO.L",
1033+ "DFLI",
1034+ "LTRN",
1035+ "BYND",
1036+ "PCSA",
1037+ "DDD",
1038+ "LCII",
1039+ "FTF",
1040+ "AGYS",
1041+ "SBC",
1042+ "GEVO",
1043+ "GLP",
1044+ "TTWO",
1045+ "JOBY",
1046+ "USLM",
1047+ "MCHB",
1048+ "MD",
1049+ "KODK",
1050+ "TSSI",
1051+ "VHC",
1052+ "LBTYK",
1053+ "IDCC",
1054+ "FMAO",
1055+ "UTG",
1056+ "GLBS",
1057+ "CMSC",
1058+ "FOR",
1059+ "AGNT",
1060+ "HEI",
1061+ "DHT",
1062+ "TLIS",
1063+ "ECVT",
1064+ "AEF",
1065+ "NFE",
1066+ "DCBO",
1067+ "LEGH",
1068+ "USAR",
1069+ "EVG",
1070+ "CRI",
1071+ "XNCR",
1072+ "BIP.A",
1073+ "DRH",
1074+ "DC",
1075+ "GL",
1076+ "NTLA",
1077+ "OPTT",
1078+ "OPP",
1079+ "CHGG",
1080+ "CETX",
1081+ "TRNS",
1082+ "BIO",
1083+ "PCTTW",
1084+ "LIND",
1085+ "GFR",
1086+ "AMPG",
1087+ "TTD",
1088+ "SSL",
1089+ "BML.J",
1090+ "PKE",
1091+ "PROV",
1092+ "UGP",
1093+ "CPHC",
1094+ "GOLF",
1095+ "DAVEW",
1096+ "API",
1097+ "GITS",
1098+ "NXJ",
1099+ "BKR",
1100+ "VCIG",
1101+ "MANH",
1102+ "NTGR",
1103+ "BLE",
1104+ "EP",
1105+ "LYB",
1106+ "USEA"
1107+ ],
1108+ "export_id": "88bf9220-5e26-4aa5-b8f1-8292d461d16f",
1109+ "path": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/finance/wp40_raid/datasets/cache/export_4706fcd6e37b93502029954b.parquet",
1110+ "sha256": "083ddb210287dbfd3170abdc606878def1a3b28d596981ebc2e308ed8d3c5bb9",
1111+ "dataset_version": 7752,
1112+ "rows": 200888,
1113+ "bytes": 2711030,
1114+ "from_cache": false
1115+ }
1116+ ],
1117+ "total_rows": 1821216,
1118+ "seconds": 32.5891010761261
1119+}
\ No newline at end of file
added experiments/scripts/e0b_daily_exports.py +13 −0
@@ -0,0 +1,13 @@
1+#!/usr/bin/env python3
2+"""E0b: daily bars (market_bars.stock.1d.raw) 2015-01-01…2024-12-31 for the E0 universe (≤ 10 years per request) →
3+longer daily return histories for E3 (GARCH). Output experiments/results/e0_daily.json."""
4+import os, sys, json, time
5+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "python"))
6+import raid
7+e0 = json.load(open(os.path.join(os.path.dirname(__file__), "..", "results", "e0_universe.json"))); syms = e0["samples"]["1000"]
8+exports = []; t0 = time.time()
9+for i in range(0, len(syms), 100):
10+ chunk = syms[i:i + 100]; e = raid.findata.export(chunk, "2015-01-01", "2024-12-31", dataset_id="market_bars.stock.1d.raw")
11+ exports.append({"symbols": chunk, **{k: e[k] for k in ("export_id", "path", "sha256", "dataset_version", "rows", "bytes", "from_cache")}}); print(f"daily export {i // 100 + 1}/10: {e['rows']:,} rows {e['bytes'] / 1e6:.1f} MB", flush=True)
12+json.dump({"dataset_id": "market_bars.stock.1d.raw", "start": "2015-01-01", "end": "2024-12-31", "exports": exports, "total_rows": sum(x["rows"] for x in exports), "seconds": time.time() - t0}, open(os.path.join(os.path.dirname(__file__), "..", "results", "e0_daily.json"), "w"), indent=1)
13+print("done", sum(x["rows"] for x in exports), "rows", round(time.time() - t0, 1), "s")
modified experiments/scripts/e6_baselines.py +2 −2
@@ -23,7 +23,7 @@ for ex in e0["exports"][:nf]:
23 out.write(json.dumps({"experiment": "E6", "engine": "pandas", "file": path, "rep": rep, "bars": int(len(df)), "sessions": int(len(res)), "read_seconds": t1 - t0, "filter_seconds": t2 - t1, "returns_seconds": t3 - t2, "estimate_seconds": t4 - t3, "total_seconds": t4 - t0, "peak_rss": rss(), "versions": versions}) + "\n") 23 out.write(json.dumps({"experiment": "E6", "engine": "pandas", "file": path, "rep": rep, "bars": int(len(df)), "sessions": int(len(res)), "read_seconds": t1 - t0, "filter_seconds": t2 - t1, "returns_seconds": t3 - t2, "estimate_seconds": t4 - t3, "total_seconds": t4 - t0, "peak_rss": rss(), "versions": versions}) + "\n")
24 # ---- polars 24 # ---- polars
25 t0 = time.perf_counter(); lf = pl.scan_parquet(path).select(["symbol_as_reported", "timestamp_utc", "close"]); dfp = lf.collect(); t1 = time.perf_counter() 25 t0 = time.perf_counter(); lf = pl.scan_parquet(path).select(["symbol_as_reported", "timestamp_utc", "close"]); dfp = lf.collect(); t1 = time.perf_counter()
26− dfp = dfp.with_columns(et=pl.col("timestamp_utc").dt.convert_time_zone("America/New_York")).with_columns(session=pl.col("et").dt.date(), mins=pl.col("et").dt.hour() * 60 + pl.col("et").dt.minute(), md=pl.col("et").dt.strftime("%m-%d")) 26+ dfp = dfp.with_columns(et=pl.col("timestamp_utc").dt.convert_time_zone("America/New_York")).with_columns(session=pl.col("et").dt.date(), mins=pl.col("et").dt.hour().cast(pl.Int32) * 60 + pl.col("et").dt.minute().cast(pl.Int32), md=pl.col("et").dt.strftime("%m-%d"))
27 dfp = dfp.with_columns(close_min=pl.when(pl.col("md").is_in(["11-29", "07-03", "12-24"])).then(13 * 60).otherwise(16 * 60)).filter((pl.col("mins") >= 570) & (pl.col("mins") <= pl.col("close_min"))).sort(["symbol_as_reported", "timestamp_utc"]); t2 = time.perf_counter() 27 dfp = dfp.with_columns(close_min=pl.when(pl.col("md").is_in(["11-29", "07-03", "12-24"])).then(13 * 60).otherwise(16 * 60)).filter((pl.col("mins") >= 570) & (pl.col("mins") <= pl.col("close_min"))).sort(["symbol_as_reported", "timestamp_utc"]); t2 = time.perf_counter()
28 dfp = dfp.with_columns(lr=pl.col("close").log().diff().over(["symbol_as_reported", "session"])).drop_nulls("lr"); t3 = time.perf_counter() 28 dfp = dfp.with_columns(lr=pl.col("close").log().diff().over(["symbol_as_reported", "session"])).drop_nulls("lr"); t3 = time.perf_counter()
29 resp = dfp.group_by(["symbol_as_reported", "session"]).agg([(pl.col("lr") ** 2).sum().alias("rv"), (pl.col("lr").abs() * pl.col("lr").abs().shift(1)).sum().alias("adj"), pl.len().alias("M"), (pl.col("lr") ** 4).sum().alias("s4"), pl.when(pl.col("lr") < 0).then(pl.col("lr") ** 2).otherwise(0.0).sum().alias("rsv_neg")]); t4 = time.perf_counter() 29 resp = dfp.group_by(["symbol_as_reported", "session"]).agg([(pl.col("lr") ** 2).sum().alias("rv"), (pl.col("lr").abs() * pl.col("lr").abs().shift(1)).sum().alias("adj"), pl.len().alias("M"), (pl.col("lr") ** 4).sum().alias("s4"), pl.when(pl.col("lr") < 0).then(pl.col("lr") ** 2).otherwise(0.0).sum().alias("rsv_neg")]); t4 = time.perf_counter()
@@ -35,6 +35,6 @@ for ex in e0["exports"][:nf]:
35 f AS (SELECT * FROM r WHERE mins >= 570 AND mins <= CASE WHEN md IN ('11-29','07-03','12-24') THEN 780 ELSE 960 END), 35 f AS (SELECT * FROM r WHERE mins >= 570 AND mins <= CASE WHEN md IN ('11-29','07-03','12-24') THEN 780 ELSE 960 END),
36 l AS (SELECT s, session, ln(close) - lag(ln(close)) OVER (PARTITION BY s, session ORDER BY et) lr FROM f) 36 l AS (SELECT s, session, ln(close) - lag(ln(close)) OVER (PARTITION BY s, session ORDER BY et) lr FROM f)
37 SELECT s, session, sum(lr*lr) rv, count(*) M, sum(lr*lr*lr*lr) s4, sum(CASE WHEN lr<0 THEN lr*lr ELSE 0 END) rsv_neg FROM l WHERE lr IS NOT NULL GROUP BY s, session""" 37 SELECT s, session, sum(lr*lr) rv, count(*) M, sum(lr*lr*lr*lr) s4, sum(CASE WHEN lr<0 THEN lr*lr ELSE 0 END) rsv_neg FROM l WHERE lr IS NOT NULL GROUP BY s, session"""
38− resd = con.execute(q).fetch_arrow_table(); t4 = time.perf_counter(); con.close() 38+ resd = con.sql(q).to_arrow_table(); t4 = time.perf_counter(); con.close()
39 out.write(json.dumps({"experiment": "E6", "engine": "duckdb", "file": path, "rep": rep, "sessions": int(resd.num_rows), "total_seconds": t4 - t0, "peak_rss": rss(), "versions": versions}) + "\n") 39 out.write(json.dumps({"experiment": "E6", "engine": "duckdb", "file": path, "rep": rep, "sessions": int(resd.num_rows), "total_seconds": t4 - t0, "peak_rss": rss(), "versions": versions}) + "\n")
40 out.flush(); print(f"{os.path.basename(path)} rep {rep}: pandas {res.shape[0]} sessions, polars {resp.height}, duckdb {resd.num_rows}", flush=True) 40 out.flush(); print(f"{os.path.basename(path)} rep {rep}: pandas {res.shape[0]} sessions, polars {resp.height}, duckdb {resd.num_rows}", flush=True)
added experiments/scripts/run_e1_e2_e6b.sh +8 −0
@@ -0,0 +1,8 @@
1+#!/bin/bash
2+set -uo pipefail; cd "$(dirname "$0")/../.."; R=experiments/results/raw; E0=experiments/results/e0_universe.json
3+export FINDATA_API_KEY_FILE=$HOME/Desktop/Cluster/secrets/findata/api-key-claude-code.txt RAID_CACHE_DIR=$PWD/datasets/cache
4+rm -f $R/e6_baselines.jsonl $R/e1_batched_rv.jsonl $R/e2_har.jsonl
5+echo "== E6 baselines $(date)"; .venv/bin/python experiments/scripts/e6_baselines.py $E0 $R/e6_baselines.jsonl 4 3 > $R/e6_baselines.log 2>&1; echo "E6b exit=$?"
6+echo "== E1 $(date)"; ./build/macos-release/bench_bench_e1_batched_rv $E0 $R/e1_batched_rv.jsonl 5 2> $R/e1.log; echo "E1 exit=$?"
7+echo "== E2 $(date)"; ./build/macos-release/bench_bench_e2_har $E0 $R/e2_har.jsonl 5 2> $R/e2.log; echo "E2 exit=$?"
8+echo "== done $(date)"
modified include/raid/models/garch.hpp +1 −0
@@ -50,6 +50,7 @@ struct GarchFitOptions {
50 std::string cov{"qmle"}; // "qmle" (Bollerslev–Wooldridge sandwich) | "hessian" | "opg" 50 std::string cov{"qmle"}; // "qmle" (Bollerslev–Wooldridge sandwich) | "hessian" | "opg"
51 int forecast_horizon{0}; // h ≥ 1 → multi-step variance forecasts in extra["forecast_variance"] 51 int forecast_horizon{0}; // h ≥ 1 → multi-step variance forecasts in extra["forecast_variance"]
52 bool scale_returns{true}; // internally scale returns to unit variance (as arch does, 'rescale') 52 bool scale_returns{true}; // internally scale returns to unit variance (as arch does, 'rescale')
53+ bool retry_starts{true}; // on non-convergence, retry from up to two alternative starting points (documented)
53 ExecutionPlan plan{}; 54 ExecutionPlan plan{};
54 }; 55 };
55 56
added paper/main.tex +90 −0
@@ -0,0 +1,90 @@
1+% ============================================================================
2+% UQO Working Paper No. 40 — RAID: A Metal-Accelerated C++ Framework for Reproducible
3+% High-Frequency Financial Econometrics
4+% ============================================================================
5+\documentclass[12pt,letterpaper]{article}
6+\usepackage[utf8]{inputenc}
7+\usepackage[T1]{fontenc}
8+\usepackage[english]{babel}
9+\usepackage[letterpaper,top=1in,bottom=1in,left=1in,right=1in]{geometry}
10+\usepackage{mathptmx}
11+\usepackage{microtype}
12+\usepackage{setspace}
13+\onehalfspacing
14+\usepackage{amsmath,amssymb,amsthm}
15+\usepackage{booktabs}
16+\usepackage{tabularx}
17+\usepackage{multirow}
18+\usepackage{threeparttable}
19+\usepackage{longtable}
20+\usepackage{siunitx}
21+\sisetup{input-symbols = {()}, table-format = -1.3, parse-numbers = false}
22+\usepackage{graphicx}
23+\usepackage{subcaption}
24+\usepackage{float}
25+\usepackage[dvipsnames]{xcolor}
26+\usepackage[colorlinks=true,linkcolor=NavyBlue,citecolor=NavyBlue,urlcolor=NavyBlue,breaklinks=true]{hyperref}
27+\usepackage[authoryear,round,semicolon]{natbib}
28+\bibliographystyle{aer}
29+\usepackage[font=small,labelfont=bf,labelsep=period,justification=justified,singlelinecheck=false]{caption}
30+\usepackage{fancyhdr}
31+\pagestyle{fancy}
32+\fancyhf{}
33+\renewcommand{\headrulewidth}{0pt}
34+\fancyfoot[C]{\thepage}
35+\usepackage{titlesec}
36+\titleformat{\section}{\large\bfseries}{\thesection.}{0.5em}{}
37+\titleformat{\subsection}{\normalsize\bfseries}{\thesubsection.}{0.5em}{}
38+\titleformat{\subsubsection}{\normalsize\itshape}{\thesubsubsection.}{0.5em}{}
39+\usepackage[toc,page]{appendix}
40+\usepackage{enumitem}
41+\setlist{nosep,leftmargin=*}
42+\usepackage{listings}
43+\lstset{basicstyle=\ttfamily\footnotesize,breaklines=true,columns=fullflexible,frame=single,framerule=0.2pt}
44+\usepackage{url}
45+\newtheorem{proposition}{Proposition}
46+\newcommand{\RV}{\mathrm{RV}}
47+\newcommand{\BPV}{\mathrm{BPV}}
48+\newcommand{\RQ}{\mathrm{RQ}}
49+\newcommand{\RSV}{\mathrm{RSV}}
50+\newcommand{\raid}{\textsc{raid}}
51+\newcommand{\E}{\mathbb{E}}
52+\newcommand{\I}{\mathbb{1}}
53+
54+% ============================================================================
55+% METADATA
56+% ============================================================================
57+\newcommand{\WPnumber}{40}
58+\newcommand{\WPtitle}{RAID: A Metal-Accelerated C++ Framework for Reproducible High-Frequency Financial Econometrics}
59+\newcommand{\WPsubtitle}{Batched, Fused and Streaming Estimation on Apple Silicon without Sacrificing Inference}
60+\newcommand{\WPdate}{November 2026}
61+\newcommand{\WPversion}{1.0}
62+\newcommand{\WPabstract}{%
63+ We ask whether heterogeneous high-frequency econometric workloads---realized measures, rolling and batched regressions, HAR forecasting models, GARCH-family quasi-maximum likelihood, block bootstrap inference---can be reorganised into batched, fused, streaming computation on Apple Silicon GPUs without sacrificing estimator accuracy, statistical inference, or reproducibility. Apple GPUs expose no 64-bit floating point. \raid{} answers with a C++20 library whose FP64 CPU backend is the reference, whose Metal backend accumulates every statistic in compensated ``double-float'' arithmetic and combines chunked partial sums in FP64, and whose estimators report backend, precision, convergence and provenance in every result. Tier~A methods reproduce \texttt{highfrequency}, \texttt{rugarch}, \texttt{statsmodels} and \texttt{arch} to documented tolerances; Monte Carlo experiments confirm bias, RMSE and coverage; GPU and CPU results agree to $10^{-6}$ relative and are bitwise reproducible across runs and thread counts. Using one year of 1-minute bars for a seeded universe of 1,000 US equities served by a private FirstRate Data lakehouse, we measure break-even points on an M5~Max laptop and across eleven Apple Silicon generations (M2 to M6). The GPU wins on fused realized measures beyond roughly $4\times10^4$ elements and on batched GARCH likelihood evaluation when many parameter vectors are evaluated per series; the CPU wins on rolling sums, on single-vector GARCH recursions and on small panels. Conversion and buffer traffic, not kernel time, dominate GPU wall-clock: unified memory is not free computation. We publish the negative results, the tolerances, and a reproducible benchmark protocol.%
64+}
65+\newcommand{\WPkeywords}{high-frequency econometrics, realized volatility, HAR, GARCH, GPU computing, Apple Metal, reproducibility, numerical precision}
66+\newcommand{\WPjel}{C58, C63, C87, G17}
67+\newcommand{\WPauthor}{Simon-Pierre Boucher}
68+\newcommand{\WPaffiliation}{Département des sciences administratives\\Université du Québec en Outaouais}
69+\newcommand{\WPemail}{simon-pierre.boucher@uqo.ca}
70+\newcommand{\WPaddress}{Gatineau -- Pavillon Alexandre-Taché\\283, boulevard Alexandre-Taché\\Gatineau, Québec, Canada J9A 1L8}
71+
72+\begin{document}
73+\input{sections/titlepage}
74+\input{sections/introduction}
75+\input{sections/literature}
76+\input{sections/architecture}
77+\input{sections/estimators}
78+\input{sections/metal}
79+\input{sections/correctness}
80+\input{sections/setup}
81+\input{sections/microbenchmarks}
82+\input{sections/endtoend}
83+\input{sections/applications}
84+\input{sections/threats}
85+\input{sections/reproducibility}
86+\input{sections/conclusion}
87+\newpage
88+\bibliography{references}
89+\input{appendices/appendix}
90+\end{document}
added paper/sections/introduction.tex +18 −0
@@ -0,0 +1,18 @@
1+\section{Introduction}
2+\label{sec:intro}
3+
4+High-frequency financial econometrics is computationally repetitive in a particular way. The same small estimator---a sum of squared returns, a bipower product, a four-regressor regression, a GARCH recursion---is evaluated on thousands of series, thousands of days, and thousands of bootstrap or Monte Carlo replicas. The established toolchains that practitioners trust for correctness (\texttt{highfrequency} in R, \citealp{BoudtKleenSjorup2022}; \texttt{rugarch}, \citealp{Ghalanos2024rugarch}; \texttt{arch} and \texttt{statsmodels} in Python, \citealp{Sheppard2025arch,Seabold2010}) are organised around one series at a time and double-precision CPU arithmetic. Meanwhile the hardware on most researchers' desks has changed: Apple Silicon machines couple a multicore CPU with a wide GPU in a single unified memory space. The GPU, however, has no 64-bit floating-point unit \citep{AppleMetal2026}, and econometric estimators are not obviously tolerant of single precision.
5+
6+This paper studies whether that hardware can be used for high-frequency econometrics \emph{without} giving up what makes the established toolchains trustworthy: estimator accuracy, valid inference and reproducibility. We frame the question as three falsifiable research questions and build the software needed to answer them.
7+
8+\begin{description}[leftmargin=1.2em]
9+\item[RQ1 (break-even).] At what data and batch sizes does Metal beat an optimised multicore CPU once I/O, shader compilation, synchronisation and host--device transfer are counted?
10+\item[RQ2 (batching and inference).] Does batched estimation raise throughput while preserving coverage, convergence and the numerical agreement with FP64 references?
11+\item[RQ3 (pipeline).] Can an Arrow/Parquet-to-estimator pipeline materially reduce copies, memory pressure and end-to-end latency relative to the columnar engines researchers use today?
12+\end{description}
13+
14+\raid{} (Research Architecture for Intraday Data) is the instrument: an open C++20 library with (i) an FP64 CPU backend that serves as the reference implementation of every method, (ii) an optional Metal backend in which every accumulator is carried as a compensated double-float pair \citep{Dekker1971,Neumaier1974} and chunked partial sums are combined in FP64 on the CPU, (iii) a method registry that pins the exact convention of each estimator and the reference software it is compared with, (iv) nanobind Python bindings and a command-line tool, and (v) a secure client for the FinData~OS lakehouse that serves FirstRate Data 1-minute bars with dataset versions, request identifiers and checksums.
15+
16+Our contributions are deliberately engineering and measurement contributions, not new econometric methodology. First, a precision strategy for FP32-only GPUs that keeps realized measures within $10^{-6}$ of FP64, regression sufficient statistics within $10^{-6}$ and GARCH likelihoods within $2\times10^{-5}$, with bitwise reproducibility. Second, a validation protocol: every Tier~A method is compared with at least one established implementation under identical conventions, with tolerances recorded in the registry and reproduced by the test suite. Third, a benchmark protocol with randomised order, warm-ups, repeated measurements, bootstrap confidence intervals of medians, explicit cold and cached shader compilation, and manifests recording hardware, compiler, load and thermal state. Fourth, measured answers to RQ1--RQ3 on an M5~Max laptop and on eleven Apple Silicon generations, including the cases where the CPU wins.
17+
18+The remainder of the paper is organised as follows. Section~\ref{sec:lit} positions the work relative to the econometric and systems literatures. Section~\ref{sec:arch} describes the architecture and API, Section~\ref{sec:est} the estimator definitions and conventions, Section~\ref{sec:metal} the Metal execution and precision strategy, Section~\ref{sec:correct} the statistical correctness evidence. Sections~\ref{sec:setup}--\ref{sec:apps} present the experimental setup, microbenchmarks, end-to-end benchmarks and financial applications. Sections~\ref{sec:threats}--\ref{sec:concl} discuss threats to validity, reproducibility and licensing, and conclude.
added paper/sections/literature.tex +10 −0
@@ -0,0 +1,10 @@
1+\section{Related literature}
2+\label{sec:lit}
3+
4+\paragraph{Realized measures and their software.} The realized variance of \citet{AndersenBollerslevDieboldLabys2003} and \citet{BarndorffNielsenShephard2002}, bipower variation \citep{BarndorffNielsenShephard2004,BarndorffNielsenShephard2006}, realized semivariance \citep{BarndorffNielsenKinnebrockShephard2010}, realized quarticity and jump proxies \citep{AndersenBollerslevDiebold2007,HuangTauchen2005} are the building blocks of Tier~A. Their finite-sample constants and sampling conventions vary across implementations; \citet{LiuPattonSheppard2015} document how much the choice of sampling scheme matters. Noise-robust estimators---two-scale \citep{ZhangMyklandAitSahalia2005}, realized kernels \citep{BarndorffNielsenHansenLundeShephard2008}, pre-averaging \citep{JacodLiMyklandPodolskijVetter2009}---require tick or quote data and are outside the admissible inputs of this paper, which uses 1-minute OHLCV bars only. The reference implementation for our comparisons is the \texttt{highfrequency} package \citep{BoudtKleenSjorup2022}.
5+
6+\paragraph{HAR and GARCH.} The HAR model of \citet{Corsi2009} and its HARQ extension \citep{BollerslevPattonQuaedvlieg2016}, the signed-jump variants of \citet{PattonSheppard2015}, and the GARCH family \citep{Engle1982,Bollerslev1986,Nelson1991,GlostenJagannathanRunkle1993} with quasi-maximum likelihood inference \citep{BollerslevWooldridge1992,FiorentiniCalzolariPanattoni1996} are the forecasting and conditional-variance workhorses we implement. We compare with \texttt{rugarch} \citep{Ghalanos2024rugarch} and \texttt{arch} \citep{Sheppard2025arch}. Forecast losses follow \citet{Patton2011}; the horse race of \citet{HansenLunde2005} and the comparison framework of \citet{DieboldMariano2002,HansenLundeNason2011} motivate the loss-based evaluation, although model selection is not the object of this paper. Realized GARCH \citep{HansenHuangShek2012}, HEAVY \citep{NoureldinShephardSheppard2012}, DCC \citep{Engle2002} and GARCH-MIDAS \citep{EngleGhyselsSohn2013} are listed in the registry as not implemented.
7+
8+\paragraph{Inference.} Heteroskedasticity-robust \citep{White1980}, HAC \citep{NeweyWest1987,NeweyWest1994} and cluster-robust \citep{CameronGelbachMiller2011} covariances are implemented with the \texttt{statsmodels} conventions. Dependent-data resampling uses the stationary bootstrap \citep{PolitisRomano1994}; the wild bootstrap for realized volatility \citep{GoncalvesMeddahi2009} is a natural extension we do not pursue. Connectedness measures follow \citet{DieboldYilmaz2012,DieboldYilmaz2014} with the generalized FEVD of \citet{KoopPesaranPotter1996} and \citet{PesaranShin1998}; covariance shrinkage follows \citet{LedoitWolf2004}.
9+
10+\paragraph{Numerical computing and GPUs in economics.} Compensated summation goes back to \citet{Kahan1965}, \citet{Dekker1971} and \citet{Neumaier1974}; \citet{Higham1993} gives the error analysis we rely on for the double-float accumulators. \citet{AldrichFernandezVillaverdeGallantRubioRamirez2011} introduced GPU computing to dynamic equilibrium models; the econometric literature on GPUs has concentrated on Monte Carlo and Bayesian computation rather than on the estimators studied here. Our systems dependencies are Eigen \citep{Eigen2025} with Apple Accelerate BLAS/LAPACK \citep{Blackford2002}, Apache Arrow and Parquet \citep{Arrow2025}, nanobind \citep{Nanobind2025}, and the Metal Shading Language \citep{AppleMetal2026}. The comparison engines for the pipeline experiment are pandas, Polars and DuckDB on top of NumPy \citep{Harris2020} and SciPy \citep{Virtanen2020}.
modified src/models/garch_fit.cpp +11 −3
@@ -45,7 +45,14 @@ template <class F> Mat num_hessian(const F& f, const Vec& x, f64 h0 = 1e-4) {
45 } 45 }
46 } // namespace 46 } // namespace
47 47
48+static Estimate garch_fit_once(std::span<const f64> r0, const GarchFitOptions& opt, int start_variant);
48 Estimate garch_fit(std::span<const f64> r0, const GarchFitOptions& opt) { 49 Estimate garch_fit(std::span<const f64> r0, const GarchFitOptions& opt) {
50+ // Up to 3 starting-value variants (documented in the registry): 0 = default, 1 = low persistence (0.80), 2 = high persistence (0.985) / opposite EGARCH γ sign.
51+ Estimate best; bool have = false; int tried = 0;
52+ for (int v = 0; v < 3; ++v) { Estimate e = garch_fit_once(r0, opt, v); ++tried; if (!have || (e.ok() && (!best.ok() || e.loss < best.loss))) { best = e; have = true; } if (best.ok() && opt.retry_starts == false) break; if (best.ok()) break; }
53+ best.extra["start_variants_tried"] = tried; return best;
54+}
55+static Estimate garch_fit_once(std::span<const f64> r0, const GarchFitOptions& opt, int start_variant) {
49 const auto t0 = std::chrono::steady_clock::now(); 56 const auto t0 = std::chrono::steady_clock::now();
50 Estimate e; e.model = "garch." + to_string(opt.model) + "." + to_string(opt.dist); e.backend = Backend::cpu; e.precision = Precision::fp64; 57 Estimate e; e.model = "garch." + to_string(opt.model) + "." + to_string(opt.dist); e.backend = Backend::cpu; e.precision = Precision::fp64;
51 e.names = garch_param_names(opt.model, opt.dist); e.n_obs = r0.size(); 58 e.names = garch_param_names(opt.model, opt.dist); e.n_obs = r0.size();
@@ -59,9 +66,10 @@ Estimate garch_fit(std::span<const f64> r0, const GarchFitOptions& opt) {
59 // starting values (on the scaled series): μ = mean, persistence 0.95, α = 0.05 (GJR γ = 0.05), ν = 8 66 // starting values (on the scaled series): μ = mean, persistence 0.95, α = 0.05 (GJR γ = 0.05), ν = 8
60 std::vector<f64> th0(p); th0[0] = stats::mean(std::span<const f64>(r)); 67 std::vector<f64> th0(p); th0[0] = stats::mean(std::span<const f64>(r));
61 const f64 v = stats::variance(std::span<const f64>(r), 0); 68 const f64 v = stats::variance(std::span<const f64>(r), 0);
62− if (opt.model == GarchModel::egarch) { th0[2] = 0.1; th0[3] = -0.05; th0[4] = 0.95; th0[1] = (1.0 - th0[4]) * std::log(v); } 69+ const f64 pers = start_variant == 1 ? 0.80 : (start_variant == 2 ? 0.985 : 0.95);
63− else if (opt.model == GarchModel::garch) { th0[2] = 0.05; th0[3] = 0.90; th0[1] = v * (1 - 0.95); } 70+ if (opt.model == GarchModel::egarch) { th0[2] = 0.1; th0[3] = start_variant == 2 ? 0.05 : -0.05; th0[4] = pers; th0[1] = (1.0 - th0[4]) * std::log(v); }
64− else { th0[2] = 0.03; th0[3] = 0.05; th0[4] = 0.90; th0[1] = v * (1 - 0.03 - 0.90 - 0.025); } 71+ else if (opt.model == GarchModel::garch) { th0[2] = 0.05; th0[3] = pers - 0.05; th0[1] = v * (1 - pers); }
72+ else { th0[2] = 0.03; th0[3] = 0.05; th0[4] = pers - 0.055; th0[1] = v * (1 - pers); }
65 if (opt.dist == GarchDist::student_t) th0[p - 1] = 8.0; 73 if (opt.dist == GarchDist::student_t) th0[p - 1] = 8.0;
66 auto f_u = [&](const Vec& u) { auto th = garch_from_unconstrained(std::span<const f64>(u.data(), static_cast<usize>(u.size())), opt.model, opt.dist); return garch_nll(r, th, opt.model, opt.dist); }; 74 auto f_u = [&](const Vec& u) { auto th = garch_from_unconstrained(std::span<const f64>(u.data(), static_cast<usize>(u.size())), opt.model, opt.dist); return garch_nll(r, th, opt.model, opt.dist); };
67 auto u0v = garch_to_unconstrained(th0, opt.model, opt.dist); 75 auto u0v = garch_to_unconstrained(th0, opt.model, opt.dist);
68 76
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere