1// nanobind bindings for RAID. Arrays are passed as NumPy views (no copy for contiguous float64/int64), the GIL is2// released around every heavy call, and results come back as dicts (JSON-like) plus NumPy arrays for vectors.3#include <nanobind/nanobind.h>4#include <nanobind/ndarray.h>5#include <nanobind/stl/string.h>6#include <nanobind/stl/vector.h>7#include <nanobind/stl/optional.h>8#include <nlohmann/json.hpp>9#include "raid/core/types.hpp"10#include "raid/core/error.hpp"11#include "raid/backend/backend.hpp"12#include "raid/estimators/ols.hpp"13#include "raid/estimators/realized.hpp"14#include "raid/models/garch.hpp"15#include "raid/data/bar_series.hpp"16#include "raid/data/calendar.hpp"17#include "raid/data/returns.hpp"18#include "raid/data/findata.hpp"19#include "raid/data/io.hpp"2021namespace nb = nanobind;22using namespace raid;23using json = nlohmann::json;24using F64 = nb::ndarray<const double, nb::c_contig, nb::device::cpu>;25using I64 = nb::ndarray<const int64_t, nb::c_contig, nb::device::cpu>;2627namespace {28nb::object json_to_py(const json& j) {29 switch (j.type()) {30 case json::value_t::null: return nb::none();31 case json::value_t::boolean: return nb::bool_(j.get<bool>());32 case json::value_t::number_integer: return nb::int_(j.get<long long>());33 case json::value_t::number_unsigned: return nb::int_(j.get<unsigned long long>());34 case json::value_t::number_float: return nb::float_(j.get<double>());35 case json::value_t::string: return nb::str(j.get<std::string>().c_str());36 case json::value_t::array: { nb::list l; for (auto& v : j) l.append(json_to_py(v)); return l; }37 case json::value_t::object: { nb::dict d; for (auto& [k, v] : j.items()) d[k.c_str()] = json_to_py(v); return d; }38 default: return nb::none();39 }40}41nb::ndarray<nb::numpy, double> to_np(std::vector<double> v) {42 auto* data = new std::vector<double>(std::move(v));43 nb::capsule owner(data, [](void* p) noexcept { delete static_cast<std::vector<double>*>(p); });44 return nb::ndarray<nb::numpy, double>(data->data(), {data->size()}, owner);45}46nb::ndarray<nb::numpy, int64_t> to_np_i64(std::vector<int64_t> v) {47 auto* data = new std::vector<int64_t>(std::move(v));48 nb::capsule owner(data, [](void* p) noexcept { delete static_cast<std::vector<int64_t>*>(p); });49 return nb::ndarray<nb::numpy, int64_t>(data->data(), {data->size()}, owner);50}51} // namespace5253NB_MODULE(_core, m) {54 m.doc() = "RAID compiled core (C++20, CPU + optional Metal)";55 m.attr("__version__") = RAID_VERSION;56 nb::exception<RaidError>(m, "RaidError");5758 m.def("devices", []() { json out = json::array(); for (auto& d : backend::list_devices()) out.push_back(d.to_json()); return json_to_py(out); });5960 m.def("ols", [](F64 X, F64 y, const std::string& cov, int hac_lags, std::optional<I64> clusters, double ridge, std::vector<usize> unpen, std::optional<F64> weights, std::vector<std::string> names) {61 RAID_REQUIRE(X.ndim() == 2 && y.ndim() == 1 && X.shape(0) == y.shape(0), invalid_argument, "X (n,k), y (n,)");62 const Eigen::Index n = static_cast<Eigen::Index>(X.shape(0)), k = static_cast<Eigen::Index>(X.shape(1));63 Eigen::Map<const Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>> Xm(X.data(), n, k);64 Eigen::Map<const Eigen::VectorXd> ym(y.data(), n);65 est::OlsOptions o; o.cov.type = stats::covtype_from_string(cov); o.cov.hac_lags = hac_lags; o.ridge_lambda = ridge; o.unpenalized = unpen; o.names = names;66 std::vector<i64> cl; if (clusters) { cl.assign(clusters->data(), clusters->data() + clusters->shape(0)); o.cov.clusters = cl; }67 std::vector<f64> w; if (weights) { w.assign(weights->data(), weights->data() + weights->shape(0)); o.weights = w; }68 Eigen::MatrixXd Xd = Xm; json out; { nb::gil_scoped_release rel; auto r = est::ols(Xd, ym, o); out = r.estimate.to_json(); }69 return json_to_py(out);70 }, nb::arg("X"), nb::arg("y"), nb::arg("cov") = "classical", nb::arg("hac_lags") = -1, nb::arg("clusters") = nb::none(), nb::arg("ridge_lambda") = 0.0, nb::arg("unpenalized") = std::vector<usize>{}, nb::arg("weights") = nb::none(), nb::arg("names") = std::vector<std::string>{});7172 m.def("realized_measures", [](F64 r, I64 off, const std::string& backend_s, bool bpv_fs, bool deterministic) {73 est::RealizedOptions o; o.plan.backend = backend_from_string(backend_s); o.bpv_finite_sample = bpv_fs; o.plan.deterministic = deterministic;74 backend::Batch b{std::span<const f64>(r.data(), r.shape(0)), std::span<const i64>(off.data(), off.shape(0))};75 est::RealizedMeasures mres; { nb::gil_scoped_release rel; mres = est::realized_measures_batch(b, o); }76 nb::dict d; d["rv"] = to_np(mres.rv); d["bpv"] = to_np(mres.bpv); d["rsv_neg"] = to_np(mres.rsv_neg); d["rsv_pos"] = to_np(mres.rsv_pos); d["rq"] = to_np(mres.rq); d["jump"] = to_np(mres.jump); d["tripv"] = to_np(mres.tripv); d["ret"] = to_np(mres.daily_return);77 d["backend"] = std::string(to_string(mres.stats.backend)); d["precision"] = std::string(to_string(mres.stats.precision)); d["wall_seconds"] = mres.stats.wall_seconds; d["device_seconds"] = mres.stats.device_seconds; d["note"] = mres.stats.note; d["warnings"] = mres.warnings;78 return d;79 }, nb::arg("returns"), nb::arg("offsets"), nb::arg("backend") = "auto", nb::arg("bpv_finite_sample") = true, nb::arg("deterministic") = true);8081 m.def("garch_fit", [](F64 r, const std::string& model, const std::string& dist, int horizon, const std::string& cov, int max_iter, double tol) {82 models::GarchFitOptions o; o.model = model == "gjr" ? models::GarchModel::gjr : (model == "egarch" ? models::GarchModel::egarch : models::GarchModel::garch);83 o.dist = dist == "student_t" || dist == "t" ? models::GarchDist::student_t : models::GarchDist::gaussian; o.forecast_horizon = horizon; o.cov = cov; o.max_iter = max_iter; o.tol = tol;84 std::span<const f64> rs(r.data(), r.shape(0)); json out; { nb::gil_scoped_release rel; out = models::garch_fit(rs, o).to_json(); }85 return json_to_py(out);86 }, nb::arg("returns"), nb::arg("model") = "garch", nb::arg("dist") = "gaussian", nb::arg("horizon") = 0, nb::arg("cov") = "qmle", nb::arg("max_iter") = 500, nb::arg("tol") = 1e-8);8788 m.def("validate_bars", [](I64 t, F64 o, F64 h, F64 l, F64 c, F64 v, const std::string& symbol, const std::string& interval, const std::string& session) {89 BarSeries b; b.symbol = symbol; b.interval = interval_from_string(interval); const usize n = t.shape(0); b.reserve(n);90 for (usize i = 0; i < n; ++i) b.push_back(Timestamp(t.data()[i]), o.data()[i], h.data()[i], l.data()[i], c.data()[i], v.data()[i]);91 json out; { nb::gil_scoped_release rel; auto st = b.normalize(); MarketCalendar cal; auto q = b.validate(cal, session == "extended" ? SessionPolicy::extended : SessionPolicy::regular); q.duplicates = st.duplicates_removed; q.unsorted = st.reordered; out = q.to_json(); }92 return json_to_py(out);93 });9495 m.def("calendar_sessions", [](const std::string& a, const std::string& b) { MarketCalendar cal; std::vector<std::string> out; for (auto d : cal.sessions(date_from_iso(a), date_from_iso(b))) out.push_back(d.iso()); return out; });9697 // ---- FinData client (keys from FINDATA_API_KEY / FINDATA_API_KEY_FILE; never passed through Python) ----98 m.def("findata_bars", [](const std::string& symbol, const std::string& interval, const std::string& start, const std::string& end, const std::string& adjustment, const std::string& cache_dir, bool offline) {99 findata::Config cfg = findata::Config::from_env(); if (!cache_dir.empty()) cfg.cache_dir = cache_dir; cfg.offline = offline; findata::Client cl(cfg);100 findata::BarsRequest r; r.symbol = symbol; r.interval = interval_from_string(interval); r.start = date_from_iso(start); r.end = date_from_iso(end); r.adjustment = adjustment;101 findata::BarsResult res; { nb::gil_scoped_release rel; res = cl.bars(r); }102 nb::dict d; std::vector<int64_t> t(res.bars.t_open.size()); for (usize i = 0; i < t.size(); ++i) t[i] = res.bars.t_open[i].us;103 d["timestamp_us"] = to_np_i64(std::move(t)); d["open"] = to_np(res.bars.open); d["high"] = to_np(res.bars.high); d["low"] = to_np(res.bars.low); d["close"] = to_np(res.bars.close); d["volume"] = to_np(res.bars.volume);104 std::vector<int64_t> so(res.bars.session_offsets.begin(), res.bars.session_offsets.end()); d["session_offsets"] = to_np_i64(std::move(so)); std::vector<std::string> sd; for (auto x : res.bars.session_dates) sd.push_back(x.iso()); d["sessions"] = sd;105 d["symbol"] = res.bars.symbol; d["instrument_id"] = res.bars.instrument_id; d["dataset_id"] = res.dataset_id; d["dataset_version"] = res.dataset_version; d["from_cache"] = res.from_cache; d["cache_path"] = res.cache_path; d["duplicates_dropped"] = res.duplicates_dropped;106 std::vector<std::string> rid; for (auto& e : res.exchanges) rid.push_back(e.request_id); d["request_ids"] = rid; d["key_fingerprint"] = cl.key_fingerprint();107 return d;108 }, nb::arg("symbol"), nb::arg("interval") = "1m", nb::arg("start"), nb::arg("end"), nb::arg("adjustment") = "raw", nb::arg("cache_dir") = "", nb::arg("offline") = false);109 m.def("findata_export", [](std::vector<std::string> symbols, const std::string& dataset_id, const std::string& start, const std::string& end, const std::string& adjustment, const std::string& out_path, const std::string& cache_dir) {110 findata::Config cfg = findata::Config::from_env(); if (!cache_dir.empty()) cfg.cache_dir = cache_dir; findata::Client cl(cfg);111 findata::ExportRequest r; r.symbols = symbols; r.dataset_id = dataset_id; r.start = date_from_iso(start); r.end = date_from_iso(end); r.adjustment = adjustment;112 findata::ExportResult res; { nb::gil_scoped_release rel; res = cl.export_bars(r, out_path); }113 nb::dict d; d["export_id"] = res.export_id; d["path"] = res.path; d["sha256"] = res.sha256; d["dataset_version"] = res.dataset_version; d["rows"] = res.rows; d["bytes"] = res.bytes; d["from_cache"] = res.from_cache; d["seconds"] = res.seconds_total; d["key_fingerprint"] = cl.key_fingerprint();114 return d;115 }, nb::arg("symbols"), nb::arg("dataset_id") = "market_bars.stock.1m.raw", nb::arg("start"), nb::arg("end"), nb::arg("adjustment") = "raw", nb::arg("out_path") = "", nb::arg("cache_dir") = "");116 m.def("findata_json", [](const std::string& what, const std::string& a, const std::string& b, const std::string& c, int n) {117 findata::Client cl; json out; { nb::gil_scoped_release rel;118 if (what == "health") out = cl.health(); else if (what == "catalog") out = cl.catalog_datasets(); else if (what == "dataset") out = cl.dataset(a);119 else if (what == "sessions") out = cl.sessions(a, interval_from_string(b.empty() ? "1m" : b), date_from_iso(c.substr(0, 10)), date_from_iso(c.substr(11, 10)));120 else if (what == "coverage") out = cl.coverage_symbols_all(a.empty() ? "market_bars.stock.1m.raw" : a, n, 100000);121 else if (what == "fundamentals") { std::vector<std::string> m; if (!c.empty()) { usize p0 = 0; for (usize i = 0; i <= c.size(); ++i) if (i == c.size() || c[i] == ',') { m.push_back(c.substr(p0, i - p0)); p0 = i + 1; } } out = cl.fundamentals(a, b, m); }122 else if (what == "query") out = cl.query(a, static_cast<usize>(n)); else throw RaidError(ErrorCode::invalid_argument, "unknown findata_json kind " + what); }123 return json_to_py(out);124 }, nb::arg("what"), nb::arg("a") = "", nb::arg("b") = "", nb::arg("c") = "", nb::arg("n") = 1000);125#ifdef RAID_WITH_ARROW126 m.def("read_bars_parquet", [](const std::string& path, std::vector<std::string> symbols) {127 data::ParquetBars pb; { nb::gil_scoped_release rel; pb = data::read_bars_parquet(path, symbols); }128 nb::dict out; for (auto& [sym, b] : pb.series) { nb::dict d; std::vector<int64_t> t(b.t_open.size()); for (usize i = 0; i < t.size(); ++i) t[i] = b.t_open[i].us; d["timestamp_us"] = to_np_i64(std::move(t)); d["open"] = to_np(b.open); d["high"] = to_np(b.high); d["low"] = to_np(b.low); d["close"] = to_np(b.close); d["volume"] = to_np(b.volume); std::vector<int64_t> so(b.session_offsets.begin(), b.session_offsets.end()); d["session_offsets"] = to_np_i64(std::move(so)); std::vector<std::string> sd; for (auto x : b.session_dates) sd.push_back(x.iso()); d["sessions"] = sd; d["instrument_id"] = b.instrument_id; out[sym.c_str()] = d; }129 nb::dict meta; for (auto& [k, v] : pb.metadata) meta[k.c_str()] = v; out["__meta__"] = meta; out["__rows__"] = pb.rows_read; out["__duplicates_dropped__"] = pb.duplicates_dropped; return out;130 }, nb::arg("path"), nb::arg("symbols") = std::vector<std::string>{});131#endif132133 m.def("intraday_returns", [](I64 t, F64 c, const std::string& kind) {134 BarSeries b; b.interval = Interval::m1; const usize n = t.shape(0); b.reserve(n);135 for (usize i = 0; i < n; ++i) b.push_back(Timestamp(t.data()[i]), c.data()[i], c.data()[i], c.data()[i], c.data()[i], 0.0);136 data::IntradayReturns ir; { nb::gil_scoped_release rel; b.normalize(); MarketCalendar cal; auto reg = b.filter_session(cal, SessionPolicy::regular); ir = data::intraday_returns(reg, kind == "arithmetic" ? data::ReturnKind::arithmetic : data::ReturnKind::log); }137 nb::dict d; d["r"] = to_np(ir.r); d["offsets"] = to_np_i64(ir.session_offsets); std::vector<std::string> s; for (auto dd : ir.session_dates) s.push_back(dd.iso()); d["sessions"] = s; d["overnight"] = to_np(ir.overnight); d["close_to_close"] = to_np(ir.close_to_close); d["open_to_close"] = to_np(ir.open_to_close);138 return d;139 });140}141