WP40 RAID: Phase 2 data — FinData client (libcurl, Arrow pages, exports, cache, provenance), raid data CLI, Python findata, offline + live tests, aggregation verified vs FinData 5m
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
13 changed files +260 −16
modified
CMakeLists.txt
+1 −1
@@ -66,7 +66,7 @@ target_include_directories(raid PUBLIC | ||
| 66 | 66 | $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> |
| 67 | 67 | $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) |
| 68 | 68 | target_link_libraries(raid PUBLIC Eigen3::Eigen nlohmann_json::nlohmann_json Threads::Threads CURL::libcurl raid_options) |
| 69 | −target_compile_definitions(raid PUBLIC RAID_VERSION="${PROJECT_VERSION}") | |
| 69 | +target_compile_definitions(raid PUBLIC RAID_VERSION="${PROJECT_VERSION}" RAID_WITH_FINDATA=1) | |
| 70 | 70 | if(RAID_WITH_ARROW) |
| 71 | 71 | target_link_libraries(raid PUBLIC Arrow::arrow_shared Parquet::parquet_shared) |
| 72 | 72 | target_compile_definitions(raid PUBLIC RAID_WITH_ARROW=1) |
modified
RESEARCH_STATUS.md
+5 −5
@@ -1,6 +1,6 @@ | ||
| 1 | 1 | # RESEARCH_STATUS.md — RAID (UQO WP 40) |
| 2 | 2 | |
| 3 | −Last update: 2026-11-02 (Phase 1 passed). Legend: ✅ verified by tests/artifacts · 🧪 implemented, validation pending · | |
| 3 | +Last update: 2026-11-02 (Phase 2 passed). Legend: ✅ verified by tests/artifacts · 🧪 implemented, validation pending · | |
| 4 | 4 | 🧭 experimental · ⛔ blocked · ⏳ planned · ✖ not implemented (deliberately). |
| 5 | 5 | |
| 6 | 6 | ## Phases |
@@ -8,7 +8,7 @@ Last update: 2026-11-02 (Phase 1 passed). Legend: ✅ verified by tests/artifact | ||
| 8 | 8 | |---|---|---|---| |
| 9 | 9 | | 0 Audit | scope & feasibility memo | ✅ passed 2026-11-02 | `docs/audit.md`, `docs/capabilities.md`, `docs/novelty_matrix.md`, `docs/risks.md`, `docs/PROGRESS.md` | |
| 10 | 10 | | 1 Foundation | clean build + tests + docs | ✅ passed 2026-11-02 | `ctest` 26/26 (314 assertions), `pytest` 7/7, `docs/PROGRESS.md` Phase 1 | |
| 11 | −| 2 Data | offline synthetic tests + opt-in live tests | ⏳ | — | | |
| 11 | +| 2 Data | offline synthetic tests + opt-in live tests | ✅ passed 2026-11-02 | offline `[findata]` 4/4, live `[live]` 2/2 (60 assertions), `docs/operations/findata.md` | | |
| 12 | 12 | | 3 Numerical primitives | accuracy + measured break-even | ⏳ | — | |
| 13 | 13 | | 4 Econometrics (Tier A) | references + Monte Carlo | ⏳ | — | |
| 14 | 14 | | 5 Extensions (Tier B) | per-feature specs + tests | ⏳ | — | |
@@ -35,6 +35,6 @@ Last update: 2026-11-02 (Phase 1 passed). Legend: ✅ verified by tests/artifact | ||
| 35 | 35 | not tested unless they come back. |
| 36 | 36 | |
| 37 | 37 | ## Next safe action |
| 38 | −Phase 2: FinData client (C++ libcurl: catalog, bars Arrow pages, exports with polling/download, provenance, cache, | |
| 39 | −token bucket, retries), `raid data` CLI, offline tests with a recorded OpenAPI/fixture server, opt-in live tests | |
| 40 | −(`RAID_LIVE=1`), aggregate-vs-FinData-5m check. | |
| 38 | +Phase 3: Metal backend (device discovery, runtime + prebuilt shader library, buffer pool, batched reductions, | |
| 39 | +rolling sums, regression sufficient statistics, GARCH likelihood batch), FP32-vs-FP64 accuracy tests, first break-even | |
| 40 | +microbenchmarks (bench_reductions) with manifests. | |
modified
bindings/python/raid_py.cpp
+38 −0
@@ -15,6 +15,8 @@ | ||
| 15 | 15 | #include "raid/data/bar_series.hpp" |
| 16 | 16 | #include "raid/data/calendar.hpp" |
| 17 | 17 | #include "raid/data/returns.hpp" |
| 18 | +#include "raid/data/findata.hpp" | |
| 19 | +#include "raid/data/io.hpp" | |
| 18 | 20 | |
| 19 | 21 | namespace nb = nanobind; |
| 20 | 22 | using namespace raid; |
@@ -92,6 +94,42 @@ NB_MODULE(_core, m) { | ||
| 92 | 94 | |
| 93 | 95 | m.def("calendar_sessions", [](const std::string& a, const std::string& b) { MarketCalendar cal; std::vector<std::string> out; for (auto d : cal.sessions(date_from_iso(a), date_from_iso(b))) out.push_back(d.iso()); return out; }); |
| 94 | 96 | |
| 97 | + // ---- FinData client (keys from FINDATA_API_KEY / FINDATA_API_KEY_FILE; never passed through Python) ---- | |
| 98 | + m.def("findata_bars", [](const std::string& symbol, const std::string& interval, const std::string& start, const std::string& end, const std::string& adjustment, const std::string& cache_dir, bool offline) { | |
| 99 | + findata::Config cfg = findata::Config::from_env(); if (!cache_dir.empty()) cfg.cache_dir = cache_dir; cfg.offline = offline; findata::Client cl(cfg); | |
| 100 | + findata::BarsRequest r; r.symbol = symbol; r.interval = interval_from_string(interval); r.start = date_from_iso(start); r.end = date_from_iso(end); r.adjustment = adjustment; | |
| 101 | + findata::BarsResult res; { nb::gil_scoped_release rel; res = cl.bars(r); } | |
| 102 | + nb::dict d; std::vector<int64_t> t(res.bars.t_open.size()); for (usize i = 0; i < t.size(); ++i) t[i] = res.bars.t_open[i].us; | |
| 103 | + d["timestamp_us"] = to_np_i64(std::move(t)); d["open"] = to_np(res.bars.open); d["high"] = to_np(res.bars.high); d["low"] = to_np(res.bars.low); d["close"] = to_np(res.bars.close); d["volume"] = to_np(res.bars.volume); | |
| 104 | + std::vector<int64_t> so(res.bars.session_offsets.begin(), res.bars.session_offsets.end()); d["session_offsets"] = to_np_i64(std::move(so)); std::vector<std::string> sd; for (auto x : res.bars.session_dates) sd.push_back(x.iso()); d["sessions"] = sd; | |
| 105 | + d["symbol"] = res.bars.symbol; d["instrument_id"] = res.bars.instrument_id; d["dataset_id"] = res.dataset_id; d["dataset_version"] = res.dataset_version; d["from_cache"] = res.from_cache; d["cache_path"] = res.cache_path; d["duplicates_dropped"] = res.duplicates_dropped; | |
| 106 | + std::vector<std::string> rid; for (auto& e : res.exchanges) rid.push_back(e.request_id); d["request_ids"] = rid; d["key_fingerprint"] = cl.key_fingerprint(); | |
| 107 | + return d; | |
| 108 | + }, nb::arg("symbol"), nb::arg("interval") = "1m", nb::arg("start"), nb::arg("end"), nb::arg("adjustment") = "raw", nb::arg("cache_dir") = "", nb::arg("offline") = false); | |
| 109 | + m.def("findata_export", [](std::vector<std::string> symbols, const std::string& dataset_id, const std::string& start, const std::string& end, const std::string& adjustment, const std::string& out_path, const std::string& cache_dir) { | |
| 110 | + findata::Config cfg = findata::Config::from_env(); if (!cache_dir.empty()) cfg.cache_dir = cache_dir; findata::Client cl(cfg); | |
| 111 | + findata::ExportRequest r; r.symbols = symbols; r.dataset_id = dataset_id; r.start = date_from_iso(start); r.end = date_from_iso(end); r.adjustment = adjustment; | |
| 112 | + findata::ExportResult res; { nb::gil_scoped_release rel; res = cl.export_bars(r, out_path); } | |
| 113 | + nb::dict d; d["export_id"] = res.export_id; d["path"] = res.path; d["sha256"] = res.sha256; d["dataset_version"] = res.dataset_version; d["rows"] = res.rows; d["bytes"] = res.bytes; d["from_cache"] = res.from_cache; d["seconds"] = res.seconds_total; d["key_fingerprint"] = cl.key_fingerprint(); | |
| 114 | + return d; | |
| 115 | + }, nb::arg("symbols"), nb::arg("dataset_id") = "market_bars.stock.1m.raw", nb::arg("start"), nb::arg("end"), nb::arg("adjustment") = "raw", nb::arg("out_path") = "", nb::arg("cache_dir") = ""); | |
| 116 | + m.def("findata_json", [](const std::string& what, const std::string& a, const std::string& b, const std::string& c, int n) { | |
| 117 | + findata::Client cl; json out; { nb::gil_scoped_release rel; | |
| 118 | + if (what == "health") out = cl.health(); else if (what == "catalog") out = cl.catalog_datasets(); else if (what == "dataset") out = cl.dataset(a); | |
| 119 | + else if (what == "sessions") out = cl.sessions(a, interval_from_string(b.empty() ? "1m" : b), date_from_iso(c.substr(0, 10)), date_from_iso(c.substr(11, 10))); | |
| 120 | + else if (what == "coverage") out = cl.coverage_symbols_all(a.empty() ? "market_bars.stock.1m.raw" : a, n, 100000); | |
| 121 | + else if (what == "fundamentals") { std::vector<std::string> m; if (!c.empty()) { usize p0 = 0; for (usize i = 0; i <= c.size(); ++i) if (i == c.size() || c[i] == ',') { m.push_back(c.substr(p0, i - p0)); p0 = i + 1; } } out = cl.fundamentals(a, b, m); } | |
| 122 | + else if (what == "query") out = cl.query(a, static_cast<usize>(n)); else throw RaidError(ErrorCode::invalid_argument, "unknown findata_json kind " + what); } | |
| 123 | + return json_to_py(out); | |
| 124 | + }, nb::arg("what"), nb::arg("a") = "", nb::arg("b") = "", nb::arg("c") = "", nb::arg("n") = 1000); | |
| 125 | +#ifdef RAID_WITH_ARROW | |
| 126 | + m.def("read_bars_parquet", [](const std::string& path, std::vector<std::string> symbols) { | |
| 127 | + data::ParquetBars pb; { nb::gil_scoped_release rel; pb = data::read_bars_parquet(path, symbols); } | |
| 128 | + nb::dict out; for (auto& [sym, b] : pb.series) { nb::dict d; std::vector<int64_t> t(b.t_open.size()); for (usize i = 0; i < t.size(); ++i) t[i] = b.t_open[i].us; d["timestamp_us"] = to_np_i64(std::move(t)); d["open"] = to_np(b.open); d["high"] = to_np(b.high); d["low"] = to_np(b.low); d["close"] = to_np(b.close); d["volume"] = to_np(b.volume); std::vector<int64_t> so(b.session_offsets.begin(), b.session_offsets.end()); d["session_offsets"] = to_np_i64(std::move(so)); std::vector<std::string> sd; for (auto x : b.session_dates) sd.push_back(x.iso()); d["sessions"] = sd; d["instrument_id"] = b.instrument_id; out[sym.c_str()] = d; } | |
| 129 | + nb::dict meta; for (auto& [k, v] : pb.metadata) meta[k.c_str()] = v; out["__meta__"] = meta; out["__rows__"] = pb.rows_read; out["__duplicates_dropped__"] = pb.duplicates_dropped; return out; | |
| 130 | + }, nb::arg("path"), nb::arg("symbols") = std::vector<std::string>{}); | |
| 131 | +#endif | |
| 132 | + | |
| 95 | 133 | m.def("intraday_returns", [](I64 t, F64 c, const std::string& kind) { |
| 96 | 134 | BarSeries b; b.interval = Interval::m1; const usize n = t.shape(0); b.reserve(n); |
| 97 | 135 | for (usize i = 0; i < n; ++i) b.push_back(Timestamp(t.data()[i]), c.data()[i], c.data()[i], c.data()[i], c.data()[i], 0.0); |
modified
datasets/fixtures/reference_values.json
+8 −8
@@ -168,10 +168,10 @@ | ||
| 168 | 168 | "converged": true, |
| 169 | 169 | "forecast_h5": [ |
| 170 | 170 | 0.9428008577891447, |
| 171 | − 0.9346706334289873, | |
| 172 | − 0.927072499422232, | |
| 173 | − 0.9203215917709793, | |
| 174 | − 0.913180400216695 | |
| 171 | + 0.9349887823049752, | |
| 172 | + 0.9276816634654835, | |
| 173 | + 0.9206952073863107, | |
| 174 | + 0.9139209540830693 | |
| 175 | 175 | ] |
| 176 | 176 | }, |
| 177 | 177 | "egarch.t": { |
@@ -195,10 +195,10 @@ | ||
| 195 | 195 | "converged": true, |
| 196 | 196 | "forecast_h5": [ |
| 197 | 197 | 0.9422973309992873, |
| 198 | − 0.934283584998299, | |
| 199 | − 0.9277022823725718, | |
| 200 | − 0.9209925337261364, | |
| 201 | − 0.9142751954934147 | |
| 198 | + 0.9345334516475253, | |
| 199 | + 0.9275624591259901, | |
| 200 | + 0.9204890251862488, | |
| 201 | + 0.9130849041786623 | |
| 202 | 202 | ] |
| 203 | 203 | } |
| 204 | 204 | } |
added
datasets/fixtures/synthetic_bars_page.parquet
+0 −0
Binary file not shown.
modified
docs/PROGRESS.md
+27 −0
@@ -54,3 +54,30 @@ Executed: | ||
| 54 | 54 | Gate evidence: clean build on the macOS preset, all tests green, docs (`docs/architecture/overview.md`, |
| 55 | 55 | `docs/methods/registry.md`, `docs/api/cpp.md`, README) written. Open items carried to later phases: `bars.aggregate` |
| 56 | 56 | vs FinData derived 5m (Phase 2), GARCH backcast option (Phase 4), HAR/bootstrap (Phase 4), Metal (Phase 3). |
| 57 | + | |
| 58 | +## Phase 2 — Data (2026-11-02, 15:00–15:30 ET) — GATE PASSED | |
| 59 | +Plan: secure FinData adapter (C++/libcurl), `raid data` CLI, Python access, offline + opt-in live tests, provenance, | |
| 60 | +calendar/aggregation checks against the data provider. | |
| 61 | + | |
| 62 | +Executed: | |
| 63 | +- `include/raid/data/findata.hpp`, `src/data/findata.cpp`: bearer auth from env/file only, TLS, redacted logs, | |
| 64 | + token bucket (5 req/s, burst 10), retries with backoff + `Retry-After`, Arrow-page pagination (`X-Next-Cursor`), | |
| 65 | + 31/186/3650-day chunking, async exports (POST → poll → download, `.part` rename), SHA-256 cache with `.meta.json` | |
| 66 | + (dataset version, request ids, key fingerprint), provenance export; `raid data catalog|health|sessions|coverage|bars| | |
| 67 | + export|fund|query|log`. Python: `raid.findata.{bars, export, health, catalog, dataset, sessions, coverage, | |
| 68 | + fundamentals, query, read_parquet}` (GIL released). | |
| 69 | +- Offline tests (`[findata]`): chunking caps, token-bucket timing, offline refusal / missing key, synthetic Arrow IPC + | |
| 70 | + Parquet page decoding (schema v1 metadata) → 4 passed, 2 live cases skipped. Fixture bug found: pyarrow IPC stream | |
| 71 | + truncated when the file object was not closed → explicit `with open(...)` (recorded). | |
| 72 | +- Live tests (`RAID_LIVE=1`, key file, cache in `datasets/cache/`, AAPL 2024-11-25…29): health ok; catalog contains | |
| 73 | + `market_bars.stock.1m.raw`; `expected_bars` 211 on 2024-11-29 == RAID calendar; bars pages (2 634 rows, 4 sessions, | |
| 74 | + dataset version 17375) == export rows (62 933-byte Parquet, sha256 recorded) bar for bar; second call served from | |
| 75 | + cache; RAID calendar `expected_regular_bars` == FinData `expected_bars` for every session; **aggregate 1m→5m: | |
| 76 | + extended policy reproduces FinData's derived 5m exactly (280/280 OHLCV)**; regular policy differs only in the 16:00 | |
| 77 | + bin (FinData adds 16:01–16:04 post-close minutes) → convention documented in the registry (D-008 unchanged: the | |
| 78 | + regular session ends with the closing print). Total 60 assertions, all passed; ≈ 25 requests. | |
| 79 | +- Python smoke: `raid.findata.bars` → realized variance per session from live bars; export + `read_parquet` | |
| 80 | + (`zero_copy_possible = true` for single-chunk files). | |
| 81 | + | |
| 82 | +Gate evidence: offline tests in `ctest`; live run transcript above; no vendor data in git (`scripts/check_no_secrets.sh`). | |
| 83 | +Open: least-privilege key for benchmarks (needs `findata create-key` on M3U96a — owner action, optional). | |
modified
docs/methods/registry.md
+1 −1
@@ -8,7 +8,7 @@ reference comparison pending · 🧭 experimental · ✖ not implemented. "Toler | ||
| 8 | 8 | | `returns.log` / `returns.arith` | r_t = log(p_t/p_{t−1}) / p_t/p_{t−1}−1, within session; overnight = prev close → open; close-to-close daily | BarSeries (regular session) | numpy | exact (1e−15) | ✅ | n/a | ✅ | |
| 9 | 9 | | `calendar.xnys` | NYSE trading days 2000–2026 (rules + special closures), early closes 13:00, DST America/New_York | dates | exchange_calendars 4.13 XNYS | exact set equality (sessions, holidays, early closes 2000–2026) | ✅ | n/a | ✅ | |
| 10 | 10 | | `bars.validate` | OHLC consistency, positivity, grid alignment, session membership, coverage vs 391/211, price jumps > 25 % | BarSeries | FinData `/market/sessions` semantics | counts | ✅ | n/a | ✅ (synthetic) | |
| 11 | −| `bars.aggregate` | bins anchored at 09:30 ET, closing print joins last bin, no empty bins, OHLC/sum/count | BarSeries | FinData derived 5m (`trade_count`, `vwap`) | exact for OHLCV | ✅ | n/a | 🧪 (vs FinData 5m pending Phase 2) | | |
| 11 | +| `bars.aggregate` | bins anchored at the session open (09:30 regular / 04:00 extended), the 16:00 closing print is its own bin, no empty bins, OHLC/sum | BarSeries | FinData derived 5m | exact OHLCV: extended policy 280/280 bars (AAPL 2024-11-25…29); regular policy differs only in the closing bin (FinData adds 16:01–16:04) | ✅ | n/a | ✅ (live 2026-11-02) | | |
| 12 | 12 | | `realized.rv` | Σ r² | returns CSR | highfrequency `rCov` (univariate) / numpy | 1e−12 rel (numpy) | ✅ | Phase 3 | ✅ | |
| 13 | 13 | | `realized.bpv` | μ₁⁻²·M/(M−1)·Σ|r_j||r_{j−1}| (finite-sample factor optional) | returns CSR | highfrequency `rBPCov` (makeReturns=FALSE, scaled) | 1e−12 rel (numpy); R comparison Phase 4 | ✅ | Phase 3 | ✅ | |
| 14 | 14 | | `realized.rsv` | Σ r² 1{r<0}, Σ r² 1{r>0} | returns CSR | highfrequency `rSVar` | 1e−12 rel | ✅ | Phase 3 | ✅ | |
added
docs/operations/findata.md
+24 −0
@@ -0,0 +1,24 @@ | ||
| 1 | +# Operating the FinData client | |
| 2 | + | |
| 3 | +- **Key**: `FINDATA_API_KEY` (value) or `FINDATA_API_KEY_FILE` (path). The owner's Claude Code key file is | |
| 4 | + `~/Desktop/Cluster/secrets/findata/api-key-claude-code.txt`; a least-privilege key (read:market, export) is preferable | |
| 5 | + for benchmarks. The client keeps the key in memory only; logs redact `Bearer …`; manifests record a 12-hex | |
| 6 | + `key_fingerprint` (SHA-256 prefix), never the key. | |
| 7 | +- **Base URL**: `FINDATA_URL` (default `https://findata.spboucher.ai/api/v1`); from M3U96a use `http://127.0.0.1:8600`. | |
| 8 | +- **Cache**: `RAID_CACHE_DIR` (default `datasets/cache`, git-ignored). Bars pages are stored as CSV + `.meta.json` | |
| 9 | + (sha256, dataset version, request ids, fetched_at); exports as Parquet + `.meta.json`. A cache hit is only used when | |
| 10 | + the file's SHA-256 matches its manifest. Cache keys include the dataset version when pinned (`version=`); otherwise | |
| 11 | + "latest" at fetch time is recorded in the manifest — experiments must pin the version they report. | |
| 12 | +- **Limits honoured**: 600 req/min (token bucket 5 req/s, burst 10, `RAID_FINDATA_RPS` to lower it), 1m ≤ 31 days per | |
| 13 | + call (chunked), 5m ≤ 186 d, 1d ≤ 10 y, pages ≤ 10 000 rows via `X-Next-Cursor`, exports ≤ 12 y per request (caller | |
| 14 | + chunks), query ≤ 100 000 rows / 30 s. Retries: 429/502/503/504/network with exponential backoff (1→30 s), honouring | |
| 15 | + `Retry-After`, 5 retries. | |
| 16 | +- **Offline / CI**: `RAID_OFFLINE=1` refuses any network call (cache only); CI uses synthetic fixtures. Live tests are | |
| 17 | + opt-in: `RAID_LIVE=1 FINDATA_API_KEY_FILE=… ./raid_tests "[live]"`. | |
| 18 | +- **Conventions verified live (2026-11-02, AAPL 2024-11-25…29)**: pages == export rows; `expected_bars` == RAID calendar | |
| 19 | + (391 / 211); RAID extended-policy 1m→5m aggregation == FinData derived 5m on the regular window (280/280 bars, | |
| 20 | + OHLCV exact); RAID regular-policy closing bin = the 16:00 print only (FinData's derived 16:00 bin adds 16:01–16:04 | |
| 21 | + post-close minutes: volume and close differ in that single bin per session). | |
| 22 | +- **License**: FirstRate private use — cache directories must not be shared or committed; only aggregates are published. | |
| 23 | +- CLI: `raid data catalog | health | sessions SYM --start --end | coverage [--min-sessions N] | bars SYM --start --end | |
| 24 | + [--interval] [--adjustment] [--out csv] | export SYMS --start --end [--dataset] [--out parquet] | fund ENTITY --as-of ISO | query "SQL"`. | |
modified
python/raid/__init__.py
+2 −1
@@ -15,9 +15,10 @@ except ImportError: # pragma: no cover | ||
| 15 | 15 | _core = None |
| 16 | 16 | HAS_CORE = False |
| 17 | 17 | |
| 18 | +from . import findata # noqa: E402 | |
| 18 | 19 | from .api import ( # noqa: E402 |
| 19 | 20 | devices, ols, realized_measures, garch_fit, validate_bars, calendar_sessions, Backend, available_backends, |
| 20 | 21 | ) |
| 21 | 22 | |
| 22 | 23 | __all__ = ["__version__", "HAS_CORE", "devices", "ols", "realized_measures", "garch_fit", "validate_bars", |
| 23 | − "calendar_sessions", "Backend", "available_backends"] | |
| 24 | + "calendar_sessions", "Backend", "available_backends", "findata"] | |
added
python/raid/findata.py
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +"""FinData OS access from Python through the compiled client (`raid._core`). The API key is read by the C++ client | |
| 2 | +from FINDATA_API_KEY or FINDATA_API_KEY_FILE — never passed through Python, never logged, never cached in files. | |
| 3 | + | |
| 4 | + from raid.findata import bars, export, catalog, sessions | |
| 5 | + b = bars("AAPL", "1m", "2024-11-25", "2024-11-29") # dict of NumPy arrays + provenance (dataset version, request ids) | |
| 6 | + e = export(["AAPL", "MSFT"], "2024-01-01", "2024-12-31") # async Parquet export → cached file + sha256 | |
| 7 | +""" | |
| 8 | +from __future__ import annotations | |
| 9 | +from typing import Any, Sequence | |
| 10 | + | |
| 11 | + | |
| 12 | +def _core(): | |
| 13 | + from . import _core as c | |
| 14 | + if c is None: | |
| 15 | + raise ImportError("raid._core is not built") | |
| 16 | + return c | |
| 17 | + | |
| 18 | + | |
| 19 | +def bars(symbol: str, interval: str = "1m", start: str = "", end: str = "", adjustment: str = "raw", cache_dir: str = "", offline: bool = False) -> dict[str, Any]: | |
| 20 | + """1m/5m/1d bars via Arrow pages (≤ 31/186/3650 days per call, chunked and paginated), cached per dataset version.""" | |
| 21 | + return _core().findata_bars(symbol, interval, start, end, adjustment, cache_dir, offline) | |
| 22 | + | |
| 23 | + | |
| 24 | +def export(symbols: Sequence[str], start: str, end: str, dataset_id: str = "market_bars.stock.1m.raw", adjustment: str = "raw", out_path: str = "", cache_dir: str = "") -> dict[str, Any]: | |
| 25 | + """Asynchronous Parquet export (≤ 12 years per request) with polling, download, SHA-256 and dataset version.""" | |
| 26 | + return _core().findata_export(list(symbols), dataset_id, start, end, adjustment, out_path, cache_dir) | |
| 27 | + | |
| 28 | + | |
| 29 | +def health() -> dict: return _core().findata_json("health") | |
| 30 | +def catalog() -> dict: return _core().findata_json("catalog") | |
| 31 | +def dataset(dataset_id: str) -> dict: return _core().findata_json("dataset", dataset_id) | |
| 32 | +def sessions(symbol: str, start: str, end: str, interval: str = "1m") -> dict: return _core().findata_json("sessions", symbol, interval, f"{start} {end}") | |
| 33 | +def coverage(dataset_id: str = "market_bars.stock.1m.raw", min_sessions: int = 250) -> list: return _core().findata_json("coverage", dataset_id, "", "", min_sessions) | |
| 34 | +def fundamentals(entity: str, as_of: str, metrics: Sequence[str] = ()) -> dict: return _core().findata_json("fundamentals", entity, as_of, ",".join(metrics)) | |
| 35 | +def query(sql: str, max_rows: int = 1000) -> dict: return _core().findata_json("query", sql, "", "", max_rows) | |
| 36 | + | |
| 37 | + | |
| 38 | +def read_parquet(path: str, symbols: Sequence[str] = ()) -> dict[str, Any]: | |
| 39 | + """Read a FinData bars Parquet file (schema v1) into per-symbol NumPy dicts (requires the Arrow build).""" | |
| 40 | + return _core().read_bars_parquet(path, list(symbols)) | |
modified
scripts/make_fixtures.py
+20 −0
@@ -88,3 +88,23 @@ except Exception as e: | ||
| 88 | 88 | ref["garch"] = {"error": repr(e)} |
| 89 | 89 | json.dump(ref, open(os.path.join(out_dir, "reference_values.json"), "w"), indent=1) |
| 90 | 90 | print("reference values written:", list(ref.keys())) |
| 91 | + | |
| 92 | +# --- synthetic Arrow IPC page + Parquet in the FinData bars schema v1 (one regular session, 391 bars) -------------- | |
| 93 | +import pyarrow as pa, pyarrow.ipc as ipc, pyarrow.parquet as pq | |
| 94 | +day = pd.Timestamp("2024-11-27", tz="America/New_York"); n = 391 | |
| 95 | +ts = [(day + pd.Timedelta(minutes=9 * 60 + 30 + i)).tz_convert("UTC").to_pydatetime() for i in range(n)] | |
| 96 | +g = np.random.default_rng(3); c = 100 * np.exp(np.cumsum(g.normal(0, 5e-4, n))); o = np.r_[100.0, c[:-1]] | |
| 97 | +schema = pa.schema([("instrument_id", pa.string()), ("symbol_as_reported", pa.string()), ("venue", pa.string()), ("asset_class", pa.string()), ("interval", pa.string()), | |
| 98 | + ("timestamp_utc", pa.timestamp("us", tz="UTC")), ("session_date", pa.date32()), ("open", pa.float64()), ("high", pa.float64()), ("low", pa.float64()), ("close", pa.float64()), | |
| 99 | + ("volume", pa.float64()), ("currency", pa.string()), ("trade_count", pa.int32()), ("vwap", pa.float64()), ("adjustment_type", pa.string()), ("source_id", pa.string()), | |
| 100 | + ("ingested_at", pa.timestamp("us", tz="UTC")), ("quality_flags", pa.list_(pa.string())), ("revision_id", pa.int32())], | |
| 101 | + metadata={"findata.schema": "bars", "findata.schema_version": "1", "findata.timestamp_convention": "bar_open"}) | |
| 102 | +tbl = pa.table({"instrument_id": ["synthetic:stock:SYN1"] * n, "symbol_as_reported": ["SYN1"] * n, "venue": [None] * n, "asset_class": ["stock"] * n, "interval": ["1m"] * n, | |
| 103 | + "timestamp_utc": pa.array(ts, pa.timestamp("us", tz="UTC")), "session_date": [day.date()] * n, "open": o, "high": np.maximum(o, c) * 1.0001, "low": np.minimum(o, c) * 0.9999, "close": c, | |
| 104 | + "volume": g.integers(100, 1000, n).astype(float), "currency": ["USD"] * n, "trade_count": pa.array([None] * n, pa.int32()), "vwap": pa.array([None] * n, pa.float64()), | |
| 105 | + "adjustment_type": ["raw"] * n, "source_id": ["synthetic"] * n, "ingested_at": pa.array([ts[0]] * n, pa.timestamp("us", tz="UTC")), "quality_flags": [[]] * n, "revision_id": pa.array([1] * n, pa.int32())}, schema=schema) | |
| 106 | +with open(os.path.join(out_dir, "synthetic_bars_page.arrow"), "wb") as fh: | |
| 107 | + with ipc.new_stream(fh, schema) as w: | |
| 108 | + w.write_table(tbl) | |
| 109 | +pq.write_table(tbl, os.path.join(out_dir, "synthetic_bars_page.parquet")) | |
| 110 | +print("wrote synthetic_bars_page.arrow/.parquet", tbl.num_rows, "rows") | |
added
tests/integration/test_findata_live.cpp
+59 −0
@@ -0,0 +1,59 @@ | ||
| 1 | +// Live FinData integration tests — opt-in: RAID_LIVE=1 and FINDATA_API_KEY (or FINDATA_API_KEY_FILE) must be set. | |
| 2 | +// Each test makes a handful of small requests (well under the 600/min limit) and never writes vendor data into git | |
| 3 | +// (cache under $RAID_CACHE_DIR or datasets/cache, which is git-ignored). | |
| 4 | +#include <catch2/catch_test_macros.hpp> | |
| 5 | +#include <catch2/matchers/catch_matchers_floating_point.hpp> | |
| 6 | +#include "raid/data/findata.hpp" | |
| 7 | +#include "raid/data/io.hpp" | |
| 8 | +#include "raid/data/resample.hpp" | |
| 9 | +#include "raid/data/returns.hpp" | |
| 10 | +#include "raid/estimators/realized.hpp" | |
| 11 | +#include <cstdlib> | |
| 12 | +#include <cmath> | |
| 13 | +using namespace raid; using namespace raid::findata; using Catch::Matchers::WithinRel; | |
| 14 | + | |
| 15 | +static bool live() { const char* e = std::getenv("RAID_LIVE"); return e && std::string(e) == "1"; } | |
| 16 | + | |
| 17 | +TEST_CASE("Live: health, catalog and provenance headers", "[live][findata]") { | |
| 18 | + if (!live()) { SKIP("RAID_LIVE != 1"); } | |
| 19 | + Client c; auto h = c.health(); CHECK(h["status"] == "ok"); | |
| 20 | + auto cat = c.catalog_datasets(); bool found = false; for (auto& d : cat["data"]) if (d["id"] == "market_bars.stock.1m.raw") found = true; CHECK(found); | |
| 21 | + auto s = c.sessions("SPY", Interval::m1, date_from_iso("2024-11-25"), date_from_iso("2024-11-29")); | |
| 22 | + usize early = 0; for (auto& r : s["data"]) if (r["session_date"] == "2024-11-29") { early = r["expected_bars"].get<usize>(); } CHECK(early == 211); | |
| 23 | + CHECK_FALSE(c.log().empty()); CHECK_FALSE(c.log().back().request_id.empty()); | |
| 24 | +} | |
| 25 | + | |
| 26 | +TEST_CASE("Live: bars pages equal export rows; calendar agrees with FinData sessions; aggregate matches derived 5m", "[live][findata]") { | |
| 27 | + if (!live()) { SKIP("RAID_LIVE != 1"); } | |
| 28 | + Client c; BarsRequest br; br.symbol = "AAPL"; br.start = date_from_iso("2024-11-25"); br.end = date_from_iso("2024-11-29"); | |
| 29 | + auto b = c.bars(br); CHECK(b.bars.n_sessions() == 4); CHECK(b.dataset_version > 0); CHECK(b.dataset_id == "market_bars.stock.1m.raw"); | |
| 30 | + MarketCalendar cal; auto q = b.bars.validate(cal); CHECK_FALSE(q.has_errors()); CHECK(q.coverage > 0.99); | |
| 31 | + auto sess = c.sessions("AAPL", Interval::m1, br.start, br.end); | |
| 32 | + for (auto& r : sess["data"]) { const Date d = date_from_iso(r["session_date"].get<std::string>()); CHECK(cal.expected_regular_bars(d, Interval::m1) == r["expected_bars"].get<usize>()); | |
| 33 | + const usize k = static_cast<usize>(std::find(b.bars.session_dates.begin(), b.bars.session_dates.end(), d) - b.bars.session_dates.begin()); REQUIRE(k < b.bars.n_sessions()); | |
| 34 | + CHECK(static_cast<usize>(b.bars.session_offsets[k + 1] - b.bars.session_offsets[k]) == r["bar_count"].get<usize>()); } | |
| 35 | + // second call served from cache | |
| 36 | + auto b2 = c.bars(br); CHECK(b2.from_cache); CHECK(b2.bars.size() == b.bars.size()); | |
| 37 | + // export of the same window | |
| 38 | + ExportRequest er; er.dataset_id = "market_bars.stock.1m.raw"; er.symbols = {"AAPL"}; er.start = br.start; er.end = br.end; | |
| 39 | + auto e = c.export_bars(er); CHECK(e.rows == b.bars.size()); CHECK(e.sha256.size() == 64); CHECK(e.dataset_version == b.dataset_version); | |
| 40 | +#ifdef RAID_WITH_ARROW | |
| 41 | + auto pq = data::read_bars_parquet(e.path); REQUIRE(pq.series.count("AAPL")); CHECK(pq.series["AAPL"].size() == b.bars.size()); | |
| 42 | + for (usize i = 0; i < b.bars.size(); i += 97) CHECK(pq.series["AAPL"].close[i] == b.bars.close[i]); | |
| 43 | + // calendar-aware aggregation vs FinData derived 5m (regular session) | |
| 44 | + auto reg = b.bars.filter_session(cal, SessionPolicy::regular); auto agg = data::aggregate(reg, Interval::m5, cal); | |
| 45 | + BarsRequest b5; b5.symbol = "AAPL"; b5.interval = Interval::m5; b5.start = br.start; b5.end = br.end; auto five = c.bars(b5); | |
| 46 | + auto five_reg = five.bars.filter_session(cal, SessionPolicy::regular); | |
| 47 | + INFO("agg bars " << agg.size() << " findata 5m regular bars " << five_reg.size()); | |
| 48 | + usize matched = 0, mism = 0; for (usize i = 0; i < agg.size(); ++i) { auto it = std::lower_bound(five_reg.t_open.begin(), five_reg.t_open.end(), agg.t_open[i]); if (it == five_reg.t_open.end() || *it != agg.t_open[i]) continue; const usize j = static_cast<usize>(it - five_reg.t_open.begin()); | |
| 49 | + const bool ok = agg.open[i] == five_reg.open[j] && agg.high[i] == five_reg.high[j] && agg.low[i] == five_reg.low[j] && agg.close[i] == five_reg.close[j] && std::abs(agg.volume[i] - five_reg.volume[j]) < 1e-6; ok ? ++matched : ++mism; if (!ok && mism <= 4) { std::fprintf(stderr, "[live] mismatch %s o %.4f/%.4f h %.4f/%.4f l %.4f/%.4f c %.4f/%.4f v %.0f/%.0f\n", timestamp_iso(agg.t_open[i]).c_str(), agg.open[i], five_reg.open[j], agg.high[i], five_reg.high[j], agg.low[i], five_reg.low[j], agg.close[i], five_reg.close[j], agg.volume[i], five_reg.volume[j]); UNSCOPED_INFO("mismatch at " << timestamp_iso(agg.t_open[i]) << " o " << agg.open[i] << "/" << five_reg.open[j] << " c " << agg.close[i] << "/" << five_reg.close[j] << " v " << agg.volume[i] << "/" << five_reg.volume[j]); } } | |
| 50 | + std::fprintf(stderr, "[live] aggregate 1m->5m vs FinData derived 5m: matched=%zu mismatched=%zu agg_bars=%zu findata_bars=%zu\n", matched, mism, agg.size(), five_reg.size()); INFO("matched " << matched << " mismatched " << mism); CHECK(matched > 0); | |
| 51 | + CHECK(mism <= 4); // regular policy: RAID's closing bin = the 16:00 print only; FinData's derived 16:00 bin also contains 16:01–16:04 (post-close) — one bin per session differs | |
| 52 | + // Under the extended policy the bins are built on all 04:00–19:59 bars and must reproduce FinData's derived 5m exactly on the regular window. | |
| 53 | + auto agg_ext = data::aggregate(b.bars, Interval::m5, cal, SessionPolicy::extended); | |
| 54 | + usize m2 = 0, mm2 = 0; for (usize j = 0; j < five_reg.size(); ++j) { auto it = std::lower_bound(agg_ext.t_open.begin(), agg_ext.t_open.end(), five_reg.t_open[j]); if (it == agg_ext.t_open.end() || *it != five_reg.t_open[j]) { ++mm2; continue; } const usize i = static_cast<usize>(it - agg_ext.t_open.begin()); | |
| 55 | + const bool ok = agg_ext.open[i] == five_reg.open[j] && agg_ext.high[i] == five_reg.high[j] && agg_ext.low[i] == five_reg.low[j] && agg_ext.close[i] == five_reg.close[j] && std::abs(agg_ext.volume[i] - five_reg.volume[j]) < 1e-6; ok ? ++m2 : ++mm2; } | |
| 56 | + std::fprintf(stderr, "[live] extended-policy aggregate vs FinData 5m on the regular window: matched=%zu mismatched=%zu\n", m2, mm2); | |
| 57 | + CHECK(mm2 == 0); | |
| 58 | +#endif | |
| 59 | +} | |
added
tests/unit/test_findata_offline.cpp
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +// Offline tests of the FinData client logic (no network): chunking, rate limiting, offline refusal, IPC page decoding. | |
| 2 | +#include <catch2/catch_test_macros.hpp> | |
| 3 | +#include <catch2/matchers/catch_matchers_floating_point.hpp> | |
| 4 | +#include "raid/data/findata.hpp" | |
| 5 | +#include "raid/data/io.hpp" | |
| 6 | +#include "raid/core/error.hpp" | |
| 7 | +#include <chrono> | |
| 8 | +using namespace raid; using namespace raid::findata; | |
| 9 | + | |
| 10 | +TEST_CASE("Date chunking respects the API caps", "[findata]") { | |
| 11 | + auto c = chunk_dates(date_from_iso("2024-01-01"), date_from_iso("2024-03-15"), 31); | |
| 12 | + REQUIRE(c.size() == 3); CHECK(c[0].first.iso() == "2024-01-01"); CHECK(c[0].second.iso() == "2024-01-31"); CHECK(c[1].first.iso() == "2024-02-01"); CHECK(c[2].second.iso() == "2024-03-15"); | |
| 13 | + CHECK(chunk_dates(date_from_iso("2024-01-01"), date_from_iso("2024-01-01"), 31).size() == 1); | |
| 14 | + CHECK(max_days_per_call(Interval::m1) == 31); CHECK(max_days_per_call(Interval::m5) == 186); CHECK(max_days_per_call(Interval::d1) == 3650); | |
| 15 | + CHECK_THROWS_AS(chunk_dates(date_from_iso("2024-02-01"), date_from_iso("2024-01-01"), 31), RaidError); | |
| 16 | +} | |
| 17 | +TEST_CASE("Token bucket enforces the rate", "[findata]") { | |
| 18 | + TokenBucket b(50.0, 2.0); const auto t0 = std::chrono::steady_clock::now(); | |
| 19 | + for (int i = 0; i < 12; ++i) b.acquire(); | |
| 20 | + const double s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(); | |
| 21 | + CHECK(s >= 0.18); CHECK(s < 1.0); // 12 requests, burst 2, 50/s → ≥ 0.2 s | |
| 22 | +} | |
| 23 | +TEST_CASE("Offline mode refuses network and missing keys are reported", "[findata]") { | |
| 24 | + Config c; c.offline = true; c.api_key = ""; c.cache_dir = "/tmp/raid-test-cache"; Client cl(c); | |
| 25 | + CHECK(cl.key_fingerprint() == "none"); CHECK_THROWS_AS(cl.health(), RaidError); | |
| 26 | + Config c2; c2.api_key = ""; c2.offline = false; CHECK_THROWS_AS(Client(c2), RaidError); | |
| 27 | +} | |
| 28 | +#ifdef RAID_WITH_ARROW | |
| 29 | +TEST_CASE("Synthetic Arrow IPC page decodes with schema metadata", "[findata][arrow]") { | |
| 30 | + auto pb = data::read_bars_arrow_ipc(std::string(RAID_SOURCE_DIR) + "/datasets/fixtures/synthetic_bars_page.arrow"); | |
| 31 | + REQUIRE(pb.series.count("SYN1")); CHECK(pb.metadata["findata.timestamp_convention"] == "bar_open"); CHECK(pb.rows_read == 391); CHECK(pb.series["SYN1"].n_sessions() == 1); | |
| 32 | + auto pq = data::read_bars_parquet(std::string(RAID_SOURCE_DIR) + "/datasets/fixtures/synthetic_bars_page.parquet"); | |
| 33 | + CHECK(pq.rows_read == 391); CHECK(pq.series["SYN1"].close[0] == pb.series["SYN1"].close[0]); | |
| 34 | +} | |
| 35 | +#endif | |
| 36 | ||