SPB Git
spboucher.ai

spb/wp40_uqo

Public MIT

WP40 RAID: Phase 8 fix — data layer sources were ignored by a too-broad 'data/' pattern (found by the clean-checkout reproduction); track include/raid/data, src/data

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 2, 2026) parent 1eed3a1 Browse files

17 changed files +1,145 −1

17 changed files
Unified Split
modified .gitignore +1 −1
@@ -22,7 +22,7 @@ __pycache__/
22 # Data — FirstRate private license: never commit real bars 22 # Data — FirstRate private license: never commit real bars
23 datasets/cache/ 23 datasets/cache/
24 datasets/real/ 24 datasets/real/
25−data/ 25+/data/
26 experiments/results/raw/ 26 experiments/results/raw/
27 *.parquet 27 *.parquet
28 *.arrow 28 *.arrow
added include/raid/data/bar_series.hpp +50 −0
@@ -0,0 +1,50 @@
1+// BarSeries — OHLCV bars for one instrument (columnar), with session structure derived from the MarketCalendar.
2+#pragma once
3+#include <vector>
4+#include <span>
5+#include <string>
6+#include "raid/core/types.hpp"
7+#include "raid/data/calendar.hpp"
8+#include "raid/data/quality.hpp"
9+
10+namespace raid {
11+
12+struct BarSeries {
13+ std::string symbol;
14+ std::string instrument_id;
15+ Interval interval{Interval::m1};
16+ std::string adjustment{"raw"};
17+ std::vector<Timestamp> t_open; // bar OPEN, UTC
18+ std::vector<Date> session; // exchange-local session date
19+ std::vector<f64> open, high, low, close, volume;
20+ std::vector<u32> flags; // bitmask (see QualityFlag)
21+ // Session index: session_offsets[k]..session_offsets[k+1] are the bars of session k (built by index_sessions()).
22+ std::vector<i64> session_offsets;
23+ std::vector<Date> session_dates;
24+
25+ usize size() const { return close.size(); }
26+ bool empty() const { return close.empty(); }
27+ void reserve(usize n) { t_open.reserve(n); session.reserve(n); open.reserve(n); high.reserve(n); low.reserve(n); close.reserve(n); volume.reserve(n); flags.reserve(n); }
28+ void push_back(Timestamp t, f64 o, f64 h, f64 l, f64 c, f64 v, u32 f = 0) {
29+ t_open.push_back(t); session.push_back(MarketCalendar::session_of(t)); open.push_back(o); high.push_back(h); low.push_back(l); close.push_back(c); volume.push_back(v); flags.push_back(f);
30+ }
31+ usize n_sessions() const { return session_dates.size(); }
32+ std::span<const f64> session_closes(usize k) const { return std::span<const f64>(close).subspan(static_cast<usize>(session_offsets[k]), static_cast<usize>(session_offsets[k + 1] - session_offsets[k])); }
33+ std::span<const Timestamp> session_times(usize k) const { return std::span<const Timestamp>(t_open).subspan(static_cast<usize>(session_offsets[k]), static_cast<usize>(session_offsets[k + 1] - session_offsets[k])); }
34+
35+ /// Sorts by time, drops exact duplicate timestamps (keeps the first), rebuilds the session index. Returns counts.
36+ struct NormalizeStats { usize duplicates_removed; usize reordered; };
37+ NormalizeStats normalize();
38+ void index_sessions();
39+ /// Validates OHLCV, interval grid, session membership and coverage; does not modify data.
40+ QualityReport validate(const MarketCalendar& cal, SessionPolicy policy = SessionPolicy::regular, f64 jump_threshold = 0.25) const;
41+ /// Keeps only bars of the requested session policy (regular: 09:30 ≤ open ≤ close time, i.e. includes the 16:00 print).
42+ BarSeries filter_session(const MarketCalendar& cal, SessionPolicy policy) const;
43+};
44+
45+enum QualityFlag : u32 {
46+ kFlagOhlcInconsistent = 1u << 0, kFlagNonpositivePrice = 1u << 1, kFlagNegativeVolume = 1u << 2, kFlagZeroVolume = 1u << 3,
47+ kFlagOffInterval = 1u << 4, kFlagOutsideSession = 1u << 5, kFlagNonTradingDay = 1u << 6, kFlagPriceJump = 1u << 7, kFlagVendorFlag = 1u << 8,
48+};
49+
50+} // namespace raid
added include/raid/data/calendar.hpp +58 −0
@@ -0,0 +1,58 @@
1+// MarketCalendar — XNYS (NYSE/Nasdaq) trading days, early closes, regular-session bounds and America/New_York DST.
2+// Rules are computed (no external data): NYSE holiday rules since 2000, special closures listed explicitly,
3+// US DST rules (2007+: 2nd Sunday March → 1st Sunday November; 1987–2006: 1st Sunday April → last Sunday October).
4+// Validation: tests/reference/calendar_vs_exchange_calendars.cpp compares against the Python exchange_calendars fixture.
5+#pragma once
6+#include <vector>
7+#include <string>
8+#include <optional>
9+#include "raid/core/types.hpp"
10+
11+namespace raid {
12+
13+struct SessionBounds {
14+ Date session;
15+ Timestamp open_utc; // 09:30 ET
16+ Timestamp close_utc; // 16:00 ET (13:00 on early closes) — the close bar OPENS at this instant (closing print)
17+ Timestamp ext_open_utc; // 04:00 ET
18+ Timestamp ext_close_utc; // 20:00 ET (bars open until 19:59). On early-close days the vendor stops at 17:00.
19+ bool early_close{false};
20+ i64 utc_offset_seconds{0}; // ET offset that day at the open (−18000 EST / −14400 EDT)
21+};
22+
23+class MarketCalendar {
24+ public:
25+ enum class Market { XNYS };
26+ explicit MarketCalendar(Market m = Market::XNYS) : market_(m) {}
27+ Market market() const { return market_; }
28+
29+ bool is_trading_day(Date d) const;
30+ bool is_early_close(Date d) const;
31+ std::optional<std::string> holiday_name(Date d) const;
32+ /// Session bounds for a trading day (throws if not a trading day).
33+ SessionBounds session(Date d) const;
34+ /// Regular-hours bars expected for one session at the given interval (1m: 391 normal, 211 early close).
35+ usize expected_regular_bars(Date d, Interval iv) const;
36+ /// Trading sessions in [start, end] inclusive.
37+ std::vector<Date> sessions(Date start, Date end) const;
38+ /// Next / previous trading day strictly after / before d.
39+ Date next_session(Date d) const;
40+ Date previous_session(Date d) const;
41+ /// America/New_York offset (seconds east of UTC) at a UTC instant.
42+ static i64 new_york_offset(Timestamp utc);
43+ /// Local New York civil date and minute-of-day of a UTC instant.
44+ struct Local { Date date; int minute_of_day; i64 offset_seconds; };
45+ static Local to_new_york(Timestamp utc);
46+ /// UTC instant of a New York wall-clock time (assumes the time exists; gap hour maps to post-transition).
47+ static Timestamp from_new_york(Date d, int minute_of_day);
48+ /// Session date of a bar-open timestamp under the US equity convention (04:00–20:00 ET same civil day).
49+ static Date session_of(Timestamp bar_open_utc) { return to_new_york(bar_open_utc).date; }
50+
51+ private:
52+ Market market_;
53+};
54+
55+/// Convenience: trading-day distance between two sessions (number of sessions strictly between + 1).
56+usize trading_days_between(const MarketCalendar& cal, Date a, Date b);
57+
58+} // namespace raid
added include/raid/data/findata.hpp +110 −0
@@ -0,0 +1,110 @@
1+// FinData OS client (https://findata.spboucher.ai/api/v1) — bearer auth from FINDATA_API_KEY / FINDATA_API_KEY_FILE,
2+// TLS via libcurl, redacted logs, token-bucket rate limiting, bounded concurrency, retries with exponential backoff
3+// honouring Retry-After, cursor pagination, 31-day chunking for 1m bars, asynchronous exports with polling/download,
4+// on-disk cache keyed by dataset version, SHA-256 of every downloaded artifact, and provenance (request ids, versions).
5+#pragma once
6+#include <string>
7+#include <vector>
8+#include <map>
9+#include <optional>
10+#include <mutex>
11+#include <chrono>
12+#include <functional>
13+#include <nlohmann/json.hpp>
14+#include "raid/core/types.hpp"
15+#include "raid/core/estimate.hpp"
16+#include "raid/data/bar_series.hpp"
17+
18+namespace raid::findata {
19+
20+struct Config {
21+ std::string base_url{"https://findata.spboucher.ai/api/v1"};
22+ std::string api_key; // empty → FINDATA_API_KEY, then FINDATA_API_KEY_FILE
23+ std::string cache_dir; // empty → $RAID_CACHE_DIR or ./datasets/cache
24+ double requests_per_second{5.0}; // token bucket rate (600/min limit → 5/s leaves headroom)
25+ double burst{10.0};
26+ int max_retries{5};
27+ long timeout_seconds{120};
28+ unsigned max_concurrency{4};
29+ bool verify_tls{true};
30+ bool offline{false}; // only serve from cache; any network call throws (for CI)
31+ static Config from_env();
32+};
33+
34+/// Provenance of one HTTP exchange (no secrets).
35+struct Exchange {
36+ std::string method, path; // query string without the key (the key is a header)
37+ long status{0};
38+ std::string request_id, dataset_id; // X-Request-ID, X-Dataset-Id
39+ i64 dataset_version{-1}; // X-Dataset-Version
40+ std::string next_cursor; // X-Next-Cursor (arrow pages)
41+ double seconds{0};
42+ usize bytes{0};
43+ int attempts{1};
44+};
45+
46+class TokenBucket {
47+ public:
48+ TokenBucket(double rate, double burst) : rate_(rate), capacity_(burst), tokens_(burst), last_(std::chrono::steady_clock::now()) {}
49+ /// Blocks until a token is available; returns the wait in seconds.
50+ double acquire();
51+ private:
52+ double rate_, capacity_, tokens_; std::chrono::steady_clock::time_point last_; std::mutex m_;
53+};
54+
55+/// Splits [start, end] (inclusive dates) into chunks of at most `max_days` calendar days (API constraint: 1m ≤ 31 d,
56+/// 5m ≤ 186 d, 1d ≤ 10 y; exports ≤ 12 y).
57+std::vector<std::pair<Date, Date>> chunk_dates(Date start, Date end, int max_days);
58+int max_days_per_call(Interval iv);
59+
60+struct BarsRequest {
61+ std::string symbol; Interval interval{Interval::m1}; Date start, end; std::string adjustment{"raw"}; usize page_limit{10000}; std::optional<i64> version;
62+};
63+struct BarsResult { BarSeries bars; std::vector<Exchange> exchanges; i64 dataset_version{-1}; std::string dataset_id; usize duplicates_dropped{0}; std::string cache_path; bool from_cache{false}; };
64+
65+struct ExportRequest { std::string dataset_id{"market_bars.stock.1m.raw"}; std::vector<std::string> symbols; Date start, end; std::string adjustment{"raw"}; };
66+struct ExportResult { std::string export_id, path, sha256; i64 dataset_version{-1}; usize rows{0}; usize bytes{0}; std::vector<Exchange> exchanges; bool from_cache{false}; double seconds_total{0}; };
67+
68+class Client {
69+ public:
70+ explicit Client(Config cfg = Config::from_env());
71+ ~Client();
72+ const Config& config() const { return cfg_; }
73+ std::string key_fingerprint() const; // first 12 hex of sha256(key): identifies the key without revealing it
74+
75+ nlohmann::json health();
76+ nlohmann::json openapi();
77+ nlohmann::json catalog_datasets();
78+ nlohmann::json dataset(const std::string& id);
79+ nlohmann::json sessions(const std::string& symbol, Interval iv, Date start, Date end);
80+ nlohmann::json coverage_symbols(const std::string& dataset_id, int min_sessions, usize limit = 200, const std::string& cursor = "");
81+ /// All symbols with ≥ min_sessions sessions (follows cursors); returns {symbol, sessions, d0, d1, bars}.
82+ nlohmann::json coverage_symbols_all(const std::string& dataset_id, int min_sessions, usize max_symbols = 100000);
83+ nlohmann::json fundamentals(const std::string& entity, const std::string& as_of_iso, const std::vector<std::string>& metrics = {}, const std::string& period_type = "");
84+ /// Bounded read-only SQL (server: 30 s, ≤ 100 000 rows). Use only for universe/coverage questions.
85+ nlohmann::json query(const std::string& sql, usize max_rows = 1000);
86+
87+ /// Bars via GET /market/bars?format=arrow with cursor pagination and ≤ max_days chunking; cached per dataset version.
88+ BarsResult bars(const BarsRequest& req);
89+ /// Asynchronous export (POST /exports → poll → download) with ≤ 10-year chunking handled by the caller; cached.
90+ ExportResult export_bars(const ExportRequest& req, const std::string& out_path = "");
91+ /// Convenience: export, then read into BarSeries per symbol (requires Arrow build).
92+ std::map<std::string, BarSeries> export_and_read(const ExportRequest& req);
93+
94+ const std::vector<Exchange>& log() const { return log_; }
95+ Provenance provenance(const std::string& dataset_id) const;
96+
97+ struct Response { long status{0}; std::string body; std::map<std::string, std::string> headers; double seconds{0}; int attempts{1}; };
98+ Response request(const std::string& method, const std::string& path_and_query, const std::string& body = "", const std::string& content_type = "application/json", const std::string& download_to = "");
99+
100+ private:
101+ Response request_once(const std::string& method, const std::string& url, const std::string& body, const std::string& content_type, const std::string& download_to);
102+ std::string cache_file(const std::string& kind, const std::string& key_material, const std::string& ext) const;
103+ Config cfg_; std::string key_; TokenBucket bucket_; std::vector<Exchange> log_; std::mutex log_m_;
104+ void* curl_share_{nullptr};
105+};
106+
107+/// `raid data …` subcommands (returns exit code).
108+int cli_main(const std::vector<std::string>& args);
109+
110+} // namespace raid::findata
added include/raid/data/io.hpp +24 −0
@@ -0,0 +1,24 @@
1+// Readers: CSV fixtures (always available) and Parquet/Arrow IPC (when RAID_WITH_ARROW).
2+#pragma once
3+#include <string>
4+#include <vector>
5+#include <map>
6+#include "raid/data/bar_series.hpp"
7+
8+namespace raid::data {
9+
10+/// CSV fixture format (synthetic, committed): header
11+/// symbol,timestamp_utc,open,high,low,close,volume (timestamp ISO-8601 UTC, bar open)
12+std::map<std::string, BarSeries> read_bars_csv(const std::string& path, Interval iv = Interval::m1);
13+void write_bars_csv(const std::string& path, const BarSeries& bars);
14+
15+#ifdef RAID_WITH_ARROW
16+/// Reads a FinData bars Parquet file (schema v1) into one BarSeries per instrument_id; `symbols` filters (empty = all).
17+/// Returns also the dataset metadata found in the file footer (findata.schema_version, timestamp_convention).
18+struct ParquetBars { std::map<std::string, BarSeries> series; std::map<std::string, std::string> metadata; usize rows_read{0}; usize duplicates_dropped{0}; };
19+ParquetBars read_bars_parquet(const std::string& path, const std::vector<std::string>& symbols = {}, Interval iv = Interval::m1);
20+/// Reads an Arrow IPC stream file (as returned by GET /market/bars?format=arrow).
21+ParquetBars read_bars_arrow_ipc(const std::string& path, Interval iv = Interval::m1);
22+#endif
23+
24+} // namespace raid::data
added include/raid/data/panel.hpp +42 −0
@@ -0,0 +1,42 @@
1+// Panel: N (possibly ragged) series stored contiguously (CSR layout), plus an aligned dense view when the series
2+// share a common time index. The CSR layout is what the batched CPU/Metal kernels consume directly.
3+#pragma once
4+#include <vector>
5+#include <span>
6+#include <string>
7+#include <Eigen/Dense>
8+#include "raid/core/types.hpp"
9+#include "raid/core/error.hpp"
10+
11+namespace raid {
12+
13+struct Panel {
14+ std::vector<std::string> ids; // n_series
15+ std::vector<i64> offsets; // n_series + 1, offsets into t / v
16+ std::vector<Timestamp> t; // concatenated timestamps
17+ std::vector<f64> v; // concatenated values
18+
19+ usize n_series() const { return ids.size(); }
20+ usize total_size() const { return v.size(); }
21+ usize series_size(usize i) const { return static_cast<usize>(offsets[i + 1] - offsets[i]); }
22+ std::span<const f64> values(usize i) const { return std::span<const f64>(v).subspan(static_cast<usize>(offsets[i]), series_size(i)); }
23+ std::span<const Timestamp> times(usize i) const { return std::span<const Timestamp>(t).subspan(static_cast<usize>(offsets[i]), series_size(i)); }
24+ void push_back(std::string id, std::vector<Timestamp> ts, std::vector<f64> vs) {
25+ RAID_REQUIRE(ts.size() == vs.size(), invalid_argument, "Panel::push_back size mismatch");
26+ if (offsets.empty()) offsets.push_back(0);
27+ ids.push_back(std::move(id));
28+ t.insert(t.end(), ts.begin(), ts.end()); v.insert(v.end(), vs.begin(), vs.end());
29+ offsets.push_back(static_cast<i64>(v.size()));
30+ }
31+ bool is_rectangular() const { if (n_series() == 0) return true; const usize m = series_size(0); for (usize i = 1; i < n_series(); ++i) if (series_size(i) != m) return false; return true; }
32+ /// Dense T × N column-major matrix (requires rectangular panel).
33+ Eigen::MatrixXd dense() const {
34+ RAID_REQUIRE(is_rectangular(), invalid_argument, "Panel::dense requires equal-length series");
35+ const usize T = n_series() ? series_size(0) : 0;
36+ Eigen::MatrixXd m(static_cast<Eigen::Index>(T), static_cast<Eigen::Index>(n_series()));
37+ for (usize j = 0; j < n_series(); ++j) { auto s = values(j); for (usize i = 0; i < T; ++i) m(static_cast<Eigen::Index>(i), static_cast<Eigen::Index>(j)) = s[i]; }
38+ return m;
39+ }
40+};
41+
42+} // namespace raid
added include/raid/data/quality.hpp +33 −0
@@ -0,0 +1,33 @@
1+#pragma once
2+#include <string>
3+#include <vector>
4+#include <map>
5+#include <nlohmann/json.hpp>
6+#include "raid/core/types.hpp"
7+
8+namespace raid {
9+
10+/// QualityReport: counts of data issues found by BarSeries::validate(); prices are never altered.
11+struct QualityReport {
12+ std::string symbol;
13+ usize n_bars{0};
14+ usize n_sessions{0};
15+ usize ohlc_inconsistent{0}; // high < max(open,close) or low > min(open,close) or high < low
16+ usize nonpositive_price{0};
17+ usize negative_volume{0};
18+ usize zero_volume{0};
19+ usize duplicates{0}; // duplicate (timestamp) rows removed
20+ usize off_interval{0}; // timestamp not aligned on the interval grid
21+ usize outside_session{0}; // bar outside 04:00–20:00 ET (extended) / outside regular when policy=regular
22+ usize non_trading_day{0};
23+ usize unsorted{0}; // rows reordered
24+ usize missing_regular_bars{0}; // sum over sessions of (expected − present) regular bars
25+ usize price_jumps{0}; // |close/prev close − 1| > jump_threshold within a session
26+ f64 coverage{1.0}; // present / expected regular bars
27+ std::map<std::string, usize> per_session_missing; // session → missing bars (only sessions with gaps)
28+ std::vector<std::string> notes;
29+ bool has_errors() const { return ohlc_inconsistent || nonpositive_price || negative_volume || off_interval || non_trading_day; }
30+ nlohmann::json to_json() const;
31+};
32+
33+} // namespace raid
added include/raid/data/resample.hpp +17 −0
@@ -0,0 +1,17 @@
1+// Calendar-aware aggregation of bars to a coarser interval, and sparse-sampling helpers for realized measures.
2+#pragma once
3+#include "raid/data/bar_series.hpp"
4+
5+namespace raid::data {
6+
7+/// Aggregates bars into bins anchored at the regular session open (09:30 ET) of each session:
8+/// bin k covers [open + k·Δ, open + (k+1)·Δ); the 16:00 closing print forms its own bin (FinData convention). Bins without bars are absent
9+/// (no bar is manufactured). OHLC = first/max/min/last, volume = sum, trade_count = number of source bars.
10+BarSeries aggregate(const BarSeries& bars, Interval target, const MarketCalendar& cal, SessionPolicy policy = SessionPolicy::regular);
11+
12+/// Subsampled price grid per session for realized measures: every `step` bars starting at `offset` (0 ≤ offset < step).
13+/// Returns the close prices picked, grouped by session (offsets). Used for 5-minute subsampled RV on 1-minute bars.
14+struct SampledPrices { std::vector<f64> p; std::vector<i64> session_offsets; std::vector<Date> session_dates; };
15+SampledPrices subsample(const BarSeries& bars, usize step, usize offset = 0);
16+
17+} // namespace raid::data
added include/raid/data/returns.hpp +29 −0
@@ -0,0 +1,29 @@
1+// Returns from prices / bars. Session-aware: by default no return is computed across a session boundary
2+// (the first bar of each session has no intraday return); overnight returns are optional and explicit.
3+#pragma once
4+#include <vector>
5+#include <span>
6+#include "raid/core/types.hpp"
7+#include "raid/data/bar_series.hpp"
8+
9+namespace raid::data {
10+
11+enum class ReturnKind { log, arithmetic };
12+
13+/// Plain returns of a price vector (r_t = log(p_t/p_{t-1}) or p_t/p_{t-1} − 1), size n−1.
14+std::vector<f64> returns(std::span<const f64> prices, ReturnKind kind = ReturnKind::log);
15+
16+struct IntradayReturns {
17+ std::vector<f64> r; // concatenated intraday returns, session by session
18+ std::vector<i64> session_offsets; // per session: offsets into r (n_sessions + 1)
19+ std::vector<Date> session_dates;
20+ std::vector<f64> overnight; // close(prev session) → first price of session (NaN for the first session), one per session
21+ std::vector<f64> open_to_close; // open-to-close daily return (from first bar open to last bar close)
22+ std::vector<f64> close_to_close; // daily close-to-close return (NaN for first)
23+ std::span<const f64> session(usize k) const { return std::span<const f64>(r).subspan(static_cast<usize>(session_offsets[k]), static_cast<usize>(session_offsets[k + 1] - session_offsets[k])); }
24+};
25+/// Intraday close-to-close bar returns within each session, plus daily summaries. `use_open_first` makes the first
26+/// return of a session open→close of the first bar instead of skipping it (not the default; documented in the registry).
27+IntradayReturns intraday_returns(const BarSeries& bars, ReturnKind kind = ReturnKind::log, bool use_open_first = false);
28+
29+} // namespace raid::data
added include/raid/data/time_series.hpp +24 −0
@@ -0,0 +1,24 @@
1+#pragma once
2+#include <vector>
3+#include <span>
4+#include <string>
5+#include "raid/core/types.hpp"
6+#include "raid/core/error.hpp"
7+
8+namespace raid {
9+
10+/// Columnar time series: strictly increasing timestamps and one double per observation.
11+struct TimeSeries {
12+ std::string id;
13+ std::vector<Timestamp> t;
14+ std::vector<f64> v;
15+ usize size() const { return v.size(); }
16+ bool empty() const { return v.empty(); }
17+ void validate() const {
18+ RAID_REQUIRE(t.size() == v.size(), invalid_argument, "TimeSeries: t/v size mismatch");
19+ for (usize i = 1; i < t.size(); ++i) RAID_REQUIRE(t[i] > t[i - 1], data_quality, "TimeSeries: timestamps not strictly increasing at index " + std::to_string(i));
20+ }
21+ std::span<const f64> values() const { return v; }
22+};
23+
24+} // namespace raid
added scripts/audit_manuscript_numbers.py +48 −0
@@ -0,0 +1,48 @@
1+#!/usr/bin/env python3
2+"""Phase 8 numerical audit: every quantitative claim in the manuscript prose is mapped to a value in
3+experiments/results/key_numbers.json (or the test suite) and checked within a tolerance. Exits non-zero on mismatch."""
4+import json, re, sys, os
5+root = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."); k = json.load(open(os.path.join(root, "experiments", "results", "key_numbers.json")))
6+text = "".join(open(os.path.join(root, "paper", "sections", f)).read() for f in ["introduction.tex", "setup.tex", "microbenchmarks.tex", "endtoend.tex", "applications.tex", "correctness.tex", "conclusion.tex", "threats.tex"])
7+abstract = open(os.path.join(root, "paper", "main.tex")).read()
8+def present(s): return s in text or s in abstract
9+checks = []
10+def chk(claim, value, fmt, tol=None):
11+ s = fmt.format(value); ok = present(s); checks.append((claim, s, ok))
12+P = k["primitives"]; B = k["breakeven"]; E1 = k["E1"]; E2 = k["E2"]; E3 = k["E3"]; E4 = k["E4"]; E5 = k["E5"]; E6 = k["E6"]; E7 = k["E7"]
13+chk("primitives: realized 100x390 speedup", P["realized_fused@100x390"]["speedup"], "{:.1f}$\\times$")
14+chk("primitives: realized max speedup", B["realized_fused"]["max_speedup"], "3.4$\\times$")
15+chk("primitives: regression 1x390000 speedup", P["regression_stats@1x390000"]["speedup"], "{:.1f}$\\times$")
16+chk("primitives: garch P64 max", B["garch_nll_P64"]["max_speedup"], "{:.1f}$\\times$")
17+chk("primitives: garch P64 100x390", P["garch_nll_P64@100x390"]["speedup"], "{:.1f}$\\times$")
18+chk("E1: cpuMT 926 day ms", E1["real_S926_day"]["cpuMT_ms"], "29--30\\,ms")
19+chk("E1: speedup low", min(E1[f"real_S926_{s}"]["speedup_MT"] for s in ["day", "week", "month", "year"]), "{:.2f}--")
20+chk("E1: speedup high", max(E1[f"real_S926_{s}"]["speedup_MT"] for s in ["day", "week", "month", "year"]), "--{:.2f}$\\times$")
21+chk("E1: max rel err daily", E1["real_S926_day"]["max_rel_err"], "$1.6\\times10^{{-7}}$")
22+chk("E1: max rel err yearly", E1["real_S926_year"]["max_rel_err"], "$1.4\\times10^{{-8}}$")
23+chk("E2: HAR qlike h1", E2["accuracy"]["h1"]["har"], "{:.3f}")
24+chk("E2: RW qlike h22", E2["accuracy"]["h22"]["rw"], "{:.3f}")
25+chk("E2: HAR beats RW h1", E2["accuracy"]["h1"]["har_beats_rw_pct"], "{:.0f}\\,\\%")
26+chk("E2: log-HAR h22", E2["accuracy"]["h22"]["loghar"], "{:.3f}")
27+chk("E3: symbols", E3["symbols"], "{} symbols")
28+chk("E3: EGARCH convergence", E3["egarch.n"]["converged_pct"], "{:.1f}\\,\\%")
29+chk("E3: GJR convergence", E3["gjr.n"]["converged_pct"], "{:.1f}\\,\\%")
30+chk("E3: grid metal ms", E3["grid"]["metal_ms"], "{:.0f}\\,ms on Metal")
31+chk("E3: grid cpu ms", E3["grid"]["cpuMT_ms"], "{:.0f}\\,ms on 18 CPU threads")
32+chk("E3: grid speedup", E3["grid"]["cpuMT_ms"] / E3["grid"]["metal_ms"], "({:.1f}$\\times$)")
33+chk("E4: cond N=30", E4["30"]["cond_median"], "{:.0f} ($N = 30$)")
34+chk("E4: dy ms N=100", E4["100"]["dy_ms"], "and {:.0f}\\,ms")
35+chk("E5: beta bias n500", E5["garch_n500"]["beta"]["bias"], "({:+.3f})".replace("+", ""))
36+chk("E5: repro metal vs cpu", E5["repro_realized"]["metal_vs_cpu_max_rel_rv"], "$3.7\\times10^{{-8}}$")
37+chk("E6: RAID cpu s", E6["RAID cpu"]["median_s"], "{:.3f}\\,s")
38+chk("E6: polars s", E6["polars"]["median_s"], "{:.3f}\\,s")
39+chk("E6: duckdb s", E6["duckdb"]["median_s"], "{:.3f}\\,s")
40+chk("E6: pandas s", E6["pandas"]["median_s"], "{:.2f}\\,s")
41+chk("E6: read s", E6["read_s"], "{:.3f}\\,s")
42+chk("E7: M2 RV speedup", E7["m2m8a"]["realized_fused@1000x3900_speedup_MT"], "{:.1f}$\\times$ (M2")
43+chk("E7: M2 Max GARCH", E7["M2M32c"]["garch_nll_P64@1000x390_speedup_MT"], "{:.1f}$\\times$ (M2~Max)")
44+chk("E7: M5 Pro GARCH", E7["m5ma"]["garch_nll_P64@1000x390_speedup_MT"], "{:.1f}$\\times$ (M5~Pro)")
45+bad = [c for c in checks if not c[2]]
46+for c in checks: print(("OK " if c[2] else "MISS"), c[0], "->", c[1])
47+print(f"{len(checks) - len(bad)}/{len(checks)} claims found verbatim in the prose")
48+sys.exit(1 if bad else 0)
added src/data/bar_series.cpp +97 −0
@@ -0,0 +1,97 @@
1+#include "raid/data/bar_series.hpp"
2+#include "raid/core/error.hpp"
3+#include <algorithm>
4+#include <numeric>
5+#include <cmath>
6+
7+namespace raid {
8+
9+nlohmann::json QualityReport::to_json() const {
10+ return {{"symbol", symbol}, {"n_bars", n_bars}, {"n_sessions", n_sessions}, {"ohlc_inconsistent", ohlc_inconsistent},
11+ {"nonpositive_price", nonpositive_price}, {"negative_volume", negative_volume}, {"zero_volume", zero_volume},
12+ {"duplicates", duplicates}, {"off_interval", off_interval}, {"outside_session", outside_session},
13+ {"non_trading_day", non_trading_day}, {"unsorted", unsorted}, {"missing_regular_bars", missing_regular_bars},
14+ {"price_jumps", price_jumps}, {"coverage", coverage}, {"per_session_missing", per_session_missing}, {"notes", notes}, {"has_errors", has_errors()}};
15+}
16+
17+BarSeries::NormalizeStats BarSeries::normalize() {
18+ const usize n = size();
19+ usize reordered = 0; bool dup = false;
20+ for (usize i = 1; i < n; ++i) { if (t_open[i] < t_open[i - 1]) ++reordered; else if (t_open[i] == t_open[i - 1]) dup = true; }
21+ if (reordered == 0 && !dup) { index_sessions(); return {0, 0}; } // fast path: no copy
22+ std::vector<usize> idx(n); std::iota(idx.begin(), idx.end(), 0);
23+ if (reordered) std::stable_sort(idx.begin(), idx.end(), [&](usize a, usize b) { return t_open[a] < t_open[b]; });
24+ BarSeries out; out.symbol = symbol; out.instrument_id = instrument_id; out.interval = interval; out.adjustment = adjustment; out.reserve(n);
25+ usize dups = 0;
26+ for (usize k = 0; k < n; ++k) {
27+ const usize i = idx[k];
28+ if (!out.empty() && out.t_open.back() == t_open[i]) { ++dups; continue; }
29+ out.t_open.push_back(t_open[i]); out.session.push_back(session[i]); out.open.push_back(open[i]); out.high.push_back(high[i]);
30+ out.low.push_back(low[i]); out.close.push_back(close[i]); out.volume.push_back(volume[i]); out.flags.push_back(flags[i]);
31+ }
32+ *this = std::move(out);
33+ index_sessions();
34+ return {dups, reordered};
35+}
36+
37+void BarSeries::index_sessions() {
38+ session_offsets.clear(); session_dates.clear();
39+ for (usize i = 0; i < size(); ++i) {
40+ if (i == 0 || session[i] != session[i - 1]) { session_offsets.push_back(static_cast<i64>(i)); session_dates.push_back(session[i]); }
41+ }
42+ session_offsets.push_back(static_cast<i64>(size()));
43+ if (size() == 0) session_offsets = {0};
44+}
45+
46+QualityReport BarSeries::validate(const MarketCalendar& cal, SessionPolicy policy, f64 jump_threshold) const {
47+ QualityReport r; r.symbol = symbol; r.n_bars = size(); r.n_sessions = n_sessions();
48+ const i64 step = to_duration(interval).us;
49+ for (usize i = 0; i < size(); ++i) {
50+ if (high[i] < low[i] || high[i] < std::max(open[i], close[i]) || low[i] > std::min(open[i], close[i])) ++r.ohlc_inconsistent;
51+ if (!(open[i] > 0 && high[i] > 0 && low[i] > 0 && close[i] > 0)) ++r.nonpositive_price;
52+ if (volume[i] < 0) ++r.negative_volume;
53+ if (volume[i] == 0) ++r.zero_volume;
54+ if (i > 0 && t_open[i] <= t_open[i - 1]) ++r.unsorted;
55+ if (interval != Interval::d1 && (t_open[i].us % step) != 0) ++r.off_interval;
56+ }
57+ usize expected_total = 0, present_total = 0;
58+ for (usize k = 0; k < n_sessions(); ++k) {
59+ const Date d = session_dates[k];
60+ if (!cal.is_trading_day(d)) { r.non_trading_day += static_cast<usize>(session_offsets[k + 1] - session_offsets[k]); continue; }
61+ const auto sb = cal.session(d);
62+ usize present = 0;
63+ f64 prev_close = -1;
64+ for (i64 i = session_offsets[k]; i < session_offsets[k + 1]; ++i) {
65+ const auto t = t_open[static_cast<usize>(i)];
66+ const bool regular = t >= sb.open_utc && t <= sb.close_utc;
67+ const bool extended = t >= sb.ext_open_utc && t < sb.ext_close_utc;
68+ if (policy == SessionPolicy::regular ? !regular : !extended) ++r.outside_session;
69+ if (regular) ++present;
70+ if (prev_close > 0 && std::abs(close[static_cast<usize>(i)] / prev_close - 1.0) > jump_threshold) ++r.price_jumps;
71+ prev_close = close[static_cast<usize>(i)];
72+ }
73+ const usize expected = cal.expected_regular_bars(d, interval);
74+ expected_total += expected; present_total += std::min(present, expected);
75+ if (present < expected) { r.missing_regular_bars += expected - present; r.per_session_missing[d.iso()] = expected - present; }
76+ }
77+ r.coverage = expected_total ? static_cast<f64>(present_total) / static_cast<f64>(expected_total) : 1.0;
78+ return r;
79+}
80+
81+BarSeries BarSeries::filter_session(const MarketCalendar& cal, SessionPolicy policy) const {
82+ BarSeries out; out.symbol = symbol; out.instrument_id = instrument_id; out.interval = interval; out.adjustment = adjustment; out.reserve(size());
83+ for (usize k = 0; k < n_sessions(); ++k) {
84+ const Date d = session_dates[k];
85+ if (!cal.is_trading_day(d)) continue;
86+ const auto sb = cal.session(d);
87+ for (i64 i = session_offsets[k]; i < session_offsets[k + 1]; ++i) {
88+ const usize j = static_cast<usize>(i); const auto t = t_open[j];
89+ const bool keep = policy == SessionPolicy::regular ? (t >= sb.open_utc && t <= sb.close_utc) : (t >= sb.ext_open_utc && t < sb.ext_close_utc);
90+ if (keep) { out.t_open.push_back(t); out.session.push_back(session[j]); out.open.push_back(open[j]); out.high.push_back(high[j]); out.low.push_back(low[j]); out.close.push_back(close[j]); out.volume.push_back(volume[j]); out.flags.push_back(flags[j]); }
91+ }
92+ }
93+ out.index_sessions();
94+ return out;
95+}
96+
97+} // namespace raid
added src/data/calendar.cpp +162 −0
@@ -0,0 +1,162 @@
1+#include "raid/data/calendar.hpp"
2+#include "raid/core/error.hpp"
3+#include <algorithm>
4+
5+namespace raid {
6+
7+namespace {
8+// nth (1-based) weekday (0=Sun..6=Sat) of a month; n = -1 → last.
9+Date nth_weekday(int y, unsigned m, int wd, int n) {
10+ if (n > 0) {
11+ Date first = Date::from_ymd(y, m, 1);
12+ int delta = (wd - first.weekday() + 7) % 7;
13+ return first + (delta + 7 * (n - 1));
14+ }
15+ // last
16+ const unsigned nm = m == 12 ? 1 : m + 1; const int ny = m == 12 ? y + 1 : y;
17+ Date last = Date::from_ymd(ny, nm, 1) - 1;
18+ int delta = (last.weekday() - wd + 7) % 7;
19+ return last - delta;
20+}
21+// Western Easter (Anonymous Gregorian algorithm)
22+Date easter(int y) {
23+ const int a = y % 19, b = y / 100, c = y % 100, d = b / 4, e = b % 4, f = (b + 8) / 25, g = (b - f + 1) / 3;
24+ const int h = (19 * a + b - d - g + 15) % 30, i = c / 4, k = c % 4, l = (32 + 2 * e + 2 * i - h - k) % 7;
25+ const int m = (a + 11 * h + 22 * l) / 451;
26+ const int month = (h + l - 7 * m + 114) / 31, day = ((h + l - 7 * m + 114) % 31) + 1;
27+ return Date::from_ymd(y, static_cast<unsigned>(month), static_cast<unsigned>(day));
28+}
29+// Observed date of a fixed holiday: Saturday → Friday, Sunday → Monday. NYSE rule: if Saturday, the preceding Friday is a
30+// holiday unless that Friday is the last trading day of the month/year (then no day off) — for New Year's Day on a
31+// Saturday the exchange does NOT close on Friday Dec 31.
32+std::optional<Date> observed(int y, unsigned m, unsigned d, bool is_new_year) {
33+ Date dt = Date::from_ymd(y, m, d);
34+ const int wd = dt.weekday();
35+ if (wd == 6) { if (is_new_year) return std::nullopt; return dt - 1; }
36+ if (wd == 0) return dt + 1;
37+ return dt;
38+}
39+struct Holiday { Date date; const char* name; };
40+std::vector<Holiday> holidays_for_year(int y) {
41+ std::vector<Holiday> h;
42+ if (auto d = observed(y, 1, 1, true)) h.push_back({*d, "New Year's Day"});
43+ h.push_back({nth_weekday(y, 1, 1, 3), "Martin Luther King Jr. Day"}); // since 1998
44+ h.push_back({nth_weekday(y, 2, 1, 3), "Presidents' Day"});
45+ h.push_back({easter(y) - 2, "Good Friday"});
46+ h.push_back({nth_weekday(y, 5, 1, -1), "Memorial Day"});
47+ if (y >= 2022) if (auto d = observed(y, 6, 19, false)) h.push_back({*d, "Juneteenth"});
48+ if (auto d = observed(y, 7, 4, false)) h.push_back({*d, "Independence Day"});
49+ h.push_back({nth_weekday(y, 9, 1, 1), "Labor Day"});
50+ h.push_back({nth_weekday(y, 11, 4, 4), "Thanksgiving Day"});
51+ if (auto d = observed(y, 12, 25, false)) h.push_back({*d, "Christmas Day"});
52+ // Special closures (NYSE), 2000–2026
53+ static const struct { int y; unsigned m, d; const char* n; } specials[] = {
54+ {2001, 9, 11, "September 11 attacks"}, {2001, 9, 12, "September 11 attacks"}, {2001, 9, 13, "September 11 attacks"}, {2001, 9, 14, "September 11 attacks"},
55+ {2004, 6, 11, "Reagan funeral"}, {2007, 1, 2, "Ford funeral"}, {2012, 10, 29, "Hurricane Sandy"}, {2012, 10, 30, "Hurricane Sandy"},
56+ {2018, 12, 5, "G.H.W. Bush funeral"}, {2025, 1, 9, "Carter funeral"},
57+ };
58+ for (const auto& s : specials) if (s.y == y) h.push_back({Date::from_ymd(s.y, s.m, s.d), s.n});
59+ return h;
60+}
61+} // namespace
62+
63+std::optional<std::string> MarketCalendar::holiday_name(Date d) const {
64+ const int wd = d.weekday();
65+ if (wd == 0) return "Sunday";
66+ if (wd == 6) return "Saturday";
67+ for (const auto& h : holidays_for_year(d.ymd().year)) if (h.date == d) return h.name;
68+ return std::nullopt;
69+}
70+
71+bool MarketCalendar::is_trading_day(Date d) const { return !holiday_name(d).has_value(); }
72+
73+bool MarketCalendar::is_early_close(Date d) const {
74+ if (!is_trading_day(d)) return false;
75+ const auto [y, m, dd] = d.ymd();
76+ // Historical one-offs (NYSE), verified against exchange_calendars 4.13 XNYS 2000–2026:
77+ if (d == Date::from_ymd(2002, 7, 5) || d == Date::from_ymd(2003, 12, 26)) return true; // Friday after a Thursday holiday
78+ if (d == Date::from_ymd(2002, 7, 3)) return false; // full day in 2002
79+ // Day after Thanksgiving
80+ if (d == nth_weekday(y, 11, 4, 4) + 1) return true;
81+ // Christmas Eve when it is a weekday (Mon–Fri) and a trading day; since 1993 rule, 13:00 close
82+ if (m == 12 && dd == 24 && d.weekday() >= 1 && d.weekday() <= 5) return true;
83+ // July 3 when July 4 is a weekday (Tue–Fri) — 13:00 close (rule in force since 1995 roughly; NYSE: when the 4th is a weekday)
84+ if (m == 7 && dd == 3 && d.weekday() >= 1 && d.weekday() <= 5) {
85+ const int wd4 = Date::from_ymd(y, 7, 4).weekday();
86+ if (wd4 >= 2 && wd4 <= 5) return true;
87+ }
88+ return false;
89+}
90+
91+i64 MarketCalendar::new_york_offset(Timestamp utc) {
92+ // Determine year from UTC date (safe: transitions are in March/November, far from year boundaries)
93+ const i64 days = utc.us >= 0 ? utc.us / 86'400'000'000LL : -((-utc.us + 86'400'000'000LL - 1) / 86'400'000'000LL);
94+ const int y = Date(static_cast<i32>(days)).ymd().year;
95+ Date start, end;
96+ if (y >= 2007) { start = nth_weekday(y, 3, 0, 2); end = nth_weekday(y, 11, 0, 1); }
97+ else { start = nth_weekday(y, 4, 0, 1); end = nth_weekday(y, 10, 0, -1); }
98+ // DST starts at 02:00 EST = 07:00 UTC; ends at 02:00 EDT = 06:00 UTC
99+ const Timestamp dst_start(static_cast<i64>(start.days) * 86'400'000'000LL + 7LL * 3'600'000'000LL);
100+ const Timestamp dst_end(static_cast<i64>(end.days) * 86'400'000'000LL + 6LL * 3'600'000'000LL);
101+ return (utc >= dst_start && utc < dst_end) ? -14400 : -18000;
102+}
103+
104+MarketCalendar::Local MarketCalendar::to_new_york(Timestamp utc) {
105+ const i64 off = new_york_offset(utc);
106+ const i64 local_us = utc.us + off * 1'000'000LL;
107+ i64 days = local_us / 86'400'000'000LL, rem = local_us % 86'400'000'000LL;
108+ if (rem < 0) { rem += 86'400'000'000LL; days -= 1; }
109+ return {Date(static_cast<i32>(days)), static_cast<int>(rem / 60'000'000LL), off};
110+}
111+
112+Timestamp MarketCalendar::from_new_york(Date d, int minute_of_day) {
113+ // First guess with the standard offset, then correct with the offset valid at that instant.
114+ const i64 naive = static_cast<i64>(d.days) * 86'400'000'000LL + static_cast<i64>(minute_of_day) * 60'000'000LL;
115+ Timestamp guess(naive + 18000LL * 1'000'000LL);
116+ const i64 off = new_york_offset(guess);
117+ Timestamp t(naive - off * 1'000'000LL);
118+ // re-check (handles instants near the transition)
119+ const i64 off2 = new_york_offset(t);
120+ if (off2 != off) t = Timestamp(naive - off2 * 1'000'000LL);
121+ return t;
122+}
123+
124+SessionBounds MarketCalendar::session(Date d) const {
125+ RAID_REQUIRE(is_trading_day(d), invalid_argument, d.iso() + " is not a trading day");
126+ SessionBounds s; s.session = d; s.early_close = is_early_close(d);
127+ s.open_utc = from_new_york(d, 9 * 60 + 30);
128+ s.close_utc = from_new_york(d, s.early_close ? 13 * 60 : 16 * 60);
129+ s.ext_open_utc = from_new_york(d, 4 * 60);
130+ s.ext_close_utc = from_new_york(d, s.early_close ? 17 * 60 : 20 * 60);
131+ s.utc_offset_seconds = new_york_offset(s.open_utc);
132+ return s;
133+}
134+
135+usize MarketCalendar::expected_regular_bars(Date d, Interval iv) const {
136+ const auto s = session(d);
137+ const i64 minutes = (s.close_utc - s.open_utc).us / 60'000'000LL;
138+ switch (iv) {
139+ case Interval::m1: return static_cast<usize>(minutes) + 1; // 391 / 211, includes the closing print bar
140+ case Interval::m5: return static_cast<usize>(minutes / 5) + 1; // 79 / 43
141+ case Interval::m30: return static_cast<usize>(minutes / 30) + 1;
142+ case Interval::h1: return static_cast<usize>(minutes / 60) + 1;
143+ case Interval::d1: return 1;
144+ }
145+ return 0;
146+}
147+
148+std::vector<Date> MarketCalendar::sessions(Date start, Date end) const {
149+ std::vector<Date> out;
150+ for (Date d = start; d <= end; d = d + 1) if (is_trading_day(d)) out.push_back(d);
151+ return out;
152+}
153+Date MarketCalendar::next_session(Date d) const { Date x = d + 1; while (!is_trading_day(x)) x = x + 1; return x; }
154+Date MarketCalendar::previous_session(Date d) const { Date x = d - 1; while (!is_trading_day(x)) x = x - 1; return x; }
155+
156+usize trading_days_between(const MarketCalendar& cal, Date a, Date b) {
157+ if (b < a) std::swap(a, b);
158+ usize n = 0; for (Date d = a + 1; d <= b; d = d + 1) if (cal.is_trading_day(d)) ++n;
159+ return n;
160+}
161+
162+} // namespace raid
added src/data/findata.cpp +268 −0
@@ -0,0 +1,268 @@
1+#include "raid/data/findata.hpp"
2+#include "raid/core/error.hpp"
3+#include "raid/core/log.hpp"
4+#include "raid/core/hash.hpp"
5+#include "raid/core/manifest.hpp"
6+#include "raid/data/io.hpp"
7+#include <curl/curl.h>
8+#include <fstream>
9+#include <iostream>
10+#include <sstream>
11+#include <thread>
12+#include <cstdlib>
13+#include <filesystem>
14+#include <algorithm>
15+#include <cmath>
16+
17+namespace raid::findata {
18+
19+namespace fs = std::filesystem;
20+using json = nlohmann::json;
21+
22+// ---------------------------------------------------------------- config / secrets
23+Config Config::from_env() {
24+ Config c;
25+ if (const char* u = std::getenv("FINDATA_URL")) { c.base_url = u; if (c.base_url.find("/api/v1") == std::string::npos) c.base_url += "/api/v1"; }
26+ if (const char* k = std::getenv("FINDATA_API_KEY")) c.api_key = k;
27+ else if (const char* f = std::getenv("FINDATA_API_KEY_FILE")) { std::ifstream in(f); std::getline(in, c.api_key); }
28+ if (const char* d = std::getenv("RAID_CACHE_DIR")) c.cache_dir = d;
29+ if (c.cache_dir.empty()) c.cache_dir = "datasets/cache";
30+ if (const char* o = std::getenv("RAID_OFFLINE")) c.offline = std::string(o) == "1";
31+ if (const char* r = std::getenv("RAID_FINDATA_RPS")) c.requests_per_second = std::atof(r);
32+ while (!c.api_key.empty() && (c.api_key.back() == '\n' || c.api_key.back() == '\r' || c.api_key.back() == ' ')) c.api_key.pop_back();
33+ return c;
34+}
35+
36+double TokenBucket::acquire() {
37+ std::lock_guard<std::mutex> lk(m_);
38+ const auto now = std::chrono::steady_clock::now();
39+ tokens_ = std::min(capacity_, tokens_ + rate_ * std::chrono::duration<double>(now - last_).count()); last_ = now;
40+ double waited = 0;
41+ if (tokens_ < 1.0) { waited = (1.0 - tokens_) / rate_; std::this_thread::sleep_for(std::chrono::duration<double>(waited)); tokens_ = 1.0; last_ = std::chrono::steady_clock::now(); }
42+ tokens_ -= 1.0;
43+ return waited;
44+}
45+
46+std::vector<std::pair<Date, Date>> chunk_dates(Date start, Date end, int max_days) {
47+ RAID_REQUIRE(max_days >= 1, invalid_argument, "max_days >= 1");
48+ RAID_REQUIRE(end >= start, invalid_argument, "end must be >= start");
49+ std::vector<std::pair<Date, Date>> out;
50+ for (Date a = start; a <= end;) { Date b = a + (max_days - 1); if (b > end) b = end; out.emplace_back(a, b); a = b + 1; }
51+ return out;
52+}
53+int max_days_per_call(Interval iv) { switch (iv) { case Interval::m1: return 31; case Interval::m5: return 186; case Interval::d1: return 3650; default: return 31; } }
54+
55+// ---------------------------------------------------------------- HTTP
56+namespace {
57+size_t write_cb(char* p, size_t s, size_t n, void* ud) { static_cast<std::string*>(ud)->append(p, s * n); return s * n; }
58+size_t file_cb(char* p, size_t s, size_t n, void* ud) { static_cast<std::ofstream*>(ud)->write(p, static_cast<std::streamsize>(s * n)); return s * n; }
59+size_t header_cb(char* p, size_t s, size_t n, void* ud) {
60+ std::string line(p, s * n); auto* h = static_cast<std::map<std::string, std::string>*>(ud);
61+ const auto c = line.find(':'); if (c != std::string::npos) { std::string k = line.substr(0, c), v = line.substr(c + 1); std::transform(k.begin(), k.end(), k.begin(), ::tolower);
62+ while (!v.empty() && (v.back() == '\r' || v.back() == '\n' || v.back() == ' ')) v.pop_back(); while (!v.empty() && v.front() == ' ') v.erase(v.begin()); (*h)[k] = v; }
63+ return s * n;
64+}
65+struct CurlGlobal { CurlGlobal() { curl_global_init(CURL_GLOBAL_DEFAULT); } ~CurlGlobal() { curl_global_cleanup(); } };
66+void ensure_curl() { static CurlGlobal g; }
67+std::string url_encode(const std::string& s) { std::string o; char buf[4]; for (unsigned char c : s) { if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' || c == ',' ) o += static_cast<char>(c); else { std::snprintf(buf, sizeof buf, "%%%02X", c); o += buf; } } return o; }
68+} // namespace
69+
70+Client::Client(Config cfg) : cfg_(std::move(cfg)), bucket_(cfg_.requests_per_second, cfg_.burst) {
71+ ensure_curl(); key_ = cfg_.api_key; cfg_.api_key.clear(); // the key lives only in key_
72+ RAID_REQUIRE(cfg_.offline || !key_.empty(), api, "FinData API key missing: set FINDATA_API_KEY or FINDATA_API_KEY_FILE");
73+ fs::create_directories(cfg_.cache_dir);
74+}
75+Client::~Client() = default;
76+std::string Client::key_fingerprint() const { return key_.empty() ? "none" : Sha256::of(key_).substr(0, 12); }
77+
78+Client::Response Client::request_once(const std::string& method, const std::string& url, const std::string& body, const std::string& content_type, const std::string& download_to) {
79+ Response r; CURL* c = curl_easy_init(); RAID_REQUIRE(c != nullptr, api, "curl init failed");
80+ struct curl_slist* hdrs = nullptr; hdrs = curl_slist_append(hdrs, ("Authorization: Bearer " + key_).c_str()); hdrs = curl_slist_append(hdrs, "User-Agent: raid/" RAID_VERSION " (UQO WP40; +simon-pierre.boucher@uqo.ca)");
81+ if (!body.empty()) hdrs = curl_slist_append(hdrs, ("Content-Type: " + content_type).c_str());
82+ std::ofstream out; if (!download_to.empty()) { out.open(download_to + ".part", std::ios::binary); RAID_REQUIRE(out.good(), io, "cannot write " + download_to); }
83+ curl_easy_setopt(c, CURLOPT_URL, url.c_str()); curl_easy_setopt(c, CURLOPT_HTTPHEADER, hdrs); curl_easy_setopt(c, CURLOPT_TIMEOUT, cfg_.timeout_seconds); curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, 20L);
84+ curl_easy_setopt(c, CURLOPT_SSL_VERIFYPEER, cfg_.verify_tls ? 1L : 0L); curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 0L); curl_easy_setopt(c, CURLOPT_ACCEPT_ENCODING, "");
85+ curl_easy_setopt(c, CURLOPT_HEADERFUNCTION, header_cb); curl_easy_setopt(c, CURLOPT_HEADERDATA, &r.headers);
86+ if (!download_to.empty()) { curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, file_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &out); } else { curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &r.body); }
87+ if (method == "POST") { curl_easy_setopt(c, CURLOPT_POST, 1L); curl_easy_setopt(c, CURLOPT_POSTFIELDS, body.c_str()); curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, static_cast<long>(body.size())); }
88+ else if (method != "GET") curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, method.c_str());
89+ const auto t0 = std::chrono::steady_clock::now(); const CURLcode rc = curl_easy_perform(c); r.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
90+ curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &r.status); curl_slist_free_all(hdrs); curl_easy_cleanup(c);
91+ if (!download_to.empty()) { out.close(); if (rc == CURLE_OK && r.status == 200) fs::rename(download_to + ".part", download_to); else fs::remove(download_to + ".part"); }
92+ if (rc != CURLE_OK) { r.status = -static_cast<long>(rc); r.body = curl_easy_strerror(rc); }
93+ return r;
94+}
95+
96+Client::Response Client::request(const std::string& method, const std::string& path, const std::string& body, const std::string& content_type, const std::string& download_to) {
97+ RAID_REQUIRE(!cfg_.offline, api, "offline mode: network call refused for " + method + " " + path);
98+ const std::string url = cfg_.base_url + path;
99+ Response r; double backoff = 1.0;
100+ for (int attempt = 1; attempt <= cfg_.max_retries + 1; ++attempt) {
101+ bucket_.acquire();
102+ r = request_once(method, url, body, content_type, download_to); r.attempts = attempt;
103+ Exchange ex; ex.method = method; ex.path = path; ex.status = r.status; ex.seconds = r.seconds; ex.bytes = r.body.size(); ex.attempts = attempt;
104+ if (auto it = r.headers.find("x-request-id"); it != r.headers.end()) ex.request_id = it->second;
105+ if (auto it = r.headers.find("x-dataset-id"); it != r.headers.end()) ex.dataset_id = it->second;
106+ if (auto it = r.headers.find("x-dataset-version"); it != r.headers.end()) ex.dataset_version = std::atoll(it->second.c_str());
107+ if (auto it = r.headers.find("x-next-cursor"); it != r.headers.end()) ex.next_cursor = it->second;
108+ { std::lock_guard<std::mutex> lk(log_m_); log_.push_back(ex); }
109+ const bool retry = r.status == 429 || r.status == 502 || r.status == 503 || r.status == 504 || r.status < 0;
110+ if (!retry) break;
111+ double wait = backoff; if (auto it = r.headers.find("retry-after"); it != r.headers.end()) wait = std::max(wait, std::atof(it->second.c_str()));
112+ log::warn("findata retry", {{"status", std::to_string(r.status)}, {"path", path}, {"attempt", std::to_string(attempt)}, {"wait_s", std::to_string(wait)}});
113+ if (attempt <= cfg_.max_retries) std::this_thread::sleep_for(std::chrono::duration<double>(wait));
114+ backoff = std::min(backoff * 2.0, 30.0);
115+ }
116+ if (r.status != 200 && r.status != 202) {
117+ std::string msg = r.body.substr(0, 400); try { auto j = json::parse(r.body); if (j.contains("error")) msg = j["error"].dump(); } catch (...) {}
118+ throw RaidError(ErrorCode::api, "FinData " + method + " " + path + " → HTTP " + std::to_string(r.status) + ": " + log::redact(msg));
119+ }
120+ return r;
121+}
122+
123+std::string Client::cache_file(const std::string& kind, const std::string& key_material, const std::string& ext) const {
124+ return (fs::path(cfg_.cache_dir) / (kind + "_" + Sha256::of(key_material).substr(0, 24) + ext)).string();
125+}
126+
127+// ---------------------------------------------------------------- simple JSON endpoints
128+nlohmann::json Client::health() { return json::parse(request("GET", "/health").body); }
129+nlohmann::json Client::openapi() { return json::parse(request("GET", "/openapi.json").body); }
130+nlohmann::json Client::catalog_datasets() { return json::parse(request("GET", "/catalog/datasets").body); }
131+nlohmann::json Client::dataset(const std::string& id) { return json::parse(request("GET", "/catalog/datasets/" + url_encode(id)).body); }
132+nlohmann::json Client::sessions(const std::string& symbol, Interval iv, Date start, Date end) {
133+ return json::parse(request("GET", "/market/sessions?symbol=" + url_encode(symbol) + "&interval=" + std::string(to_string(iv)) + "&start=" + start.iso() + "&end=" + end.iso()).body);
134+}
135+nlohmann::json Client::coverage_symbols(const std::string& dataset_id, int min_sessions, usize limit, const std::string& cursor) {
136+ std::string p = "/coverage/symbols?dataset_id=" + url_encode(dataset_id) + "&min_sessions=" + std::to_string(min_sessions) + "&limit=" + std::to_string(limit);
137+ if (!cursor.empty()) p += "&cursor=" + url_encode(cursor);
138+ return json::parse(request("GET", p).body);
139+}
140+nlohmann::json Client::coverage_symbols_all(const std::string& dataset_id, int min_sessions, usize max_symbols) {
141+ json out = json::array(); std::string cursor;
142+ for (;;) { auto page = coverage_symbols(dataset_id, min_sessions, 1000, cursor); for (auto& r : page["data"]) out.push_back(r); if (out.size() >= max_symbols || page["next_cursor"].is_null() || page["data"].empty()) break; cursor = page["next_cursor"].get<std::string>(); }
143+ return out;
144+}
145+nlohmann::json Client::fundamentals(const std::string& entity, const std::string& as_of_iso, const std::vector<std::string>& metrics, const std::string& period_type) {
146+ std::string p = "/fundamentals/" + url_encode(entity) + "?as_of=" + url_encode(as_of_iso);
147+ if (!metrics.empty()) { std::string m; for (auto& x : metrics) { if (!m.empty()) m += ","; m += x; } p += "&metrics=" + url_encode(m); }
148+ if (!period_type.empty()) p += "&period_type=" + period_type;
149+ return json::parse(request("GET", p).body);
150+}
151+nlohmann::json Client::query(const std::string& sql, usize max_rows) {
152+ RAID_REQUIRE(max_rows <= 100000, invalid_argument, "query: max_rows <= 100000");
153+ return json::parse(request("POST", "/query", json({{"sql", sql}, {"max_rows", max_rows}}).dump()).body);
154+}
155+
156+// ---------------------------------------------------------------- bars (arrow pages)
157+BarsResult Client::bars(const BarsRequest& req) {
158+ BarsResult res; res.bars.symbol = req.symbol; res.bars.interval = req.interval; res.bars.adjustment = req.adjustment;
159+ const std::string key = "bars|" + req.symbol + "|" + std::string(to_string(req.interval)) + "|" + req.start.iso() + "|" + req.end.iso() + "|" + req.adjustment + "|v" + (req.version ? std::to_string(*req.version) : "latest");
160+ const std::string meta_path = cache_file("bars", key, ".meta.json"), data_path = cache_file("bars", key, ".csv");
161+ if (fs::exists(meta_path) && fs::exists(data_path)) {
162+ auto meta = json::parse(std::ifstream(meta_path)); if (Sha256::of_file(data_path) == meta["sha256"].get<std::string>()) {
163+ auto m = data::read_bars_csv(data_path, req.interval); if (!m.empty()) res.bars = m.begin()->second; res.bars.symbol = req.symbol; res.bars.instrument_id = meta.value("instrument_id", ""); res.bars.adjustment = req.adjustment;
164+ res.dataset_version = meta["dataset_version"].get<i64>(); res.dataset_id = meta["dataset_id"].get<std::string>(); res.from_cache = true; res.cache_path = data_path; return res; }
165+ }
166+#ifndef RAID_WITH_ARROW
167+ throw RaidError(ErrorCode::unsupported, "bars(): Arrow build required to decode pages (RAID_WITH_ARROW)");
168+#else
169+ const usize log_before = log_.size();
170+ for (auto [a, b] : chunk_dates(req.start, req.end, max_days_per_call(req.interval))) {
171+ std::string cursor;
172+ for (;;) {
173+ std::string p = "/market/bars?symbol=" + url_encode(req.symbol) + "&interval=" + std::string(to_string(req.interval)) + "&start=" + a.iso() + "&end=" + b.iso() + "&adjustment=" + req.adjustment + "&format=arrow&limit=" + std::to_string(req.page_limit);
174+ if (req.version) p += "&version=" + std::to_string(*req.version); if (!cursor.empty()) p += "&cursor=" + url_encode(cursor);
175+ auto r = request("GET", p);
176+ const std::string tmp = cache_file("page", key + p + std::to_string(log_.size()), ".arrow"); { std::ofstream f(tmp, std::ios::binary); f.write(r.body.data(), static_cast<std::streamsize>(r.body.size())); }
177+ auto pb = data::read_bars_arrow_ipc(tmp, req.interval); fs::remove(tmp);
178+ for (auto& [sym, bs] : pb.series) { if (res.bars.instrument_id.empty()) res.bars.instrument_id = bs.instrument_id; for (usize i = 0; i < bs.size(); ++i) res.bars.push_back(bs.t_open[i], bs.open[i], bs.high[i], bs.low[i], bs.close[i], bs.volume[i], bs.flags[i]); }
179+ if (auto it = r.headers.find("x-dataset-version"); it != r.headers.end()) res.dataset_version = std::atoll(it->second.c_str());
180+ if (auto it = r.headers.find("x-dataset-id"); it != r.headers.end()) res.dataset_id = it->second;
181+ auto nc = r.headers.find("x-next-cursor"); if (nc == r.headers.end() || nc->second.empty() || pb.rows_read == 0) break; cursor = nc->second;
182+ }
183+ }
184+ res.exchanges.assign(log_.begin() + static_cast<long>(log_before), log_.end());
185+ auto st = res.bars.normalize(); res.duplicates_dropped = st.duplicates_removed;
186+ data::write_bars_csv(data_path, res.bars);
187+ json meta = {{"key", key}, {"sha256", Sha256::of_file(data_path)}, {"dataset_version", res.dataset_version}, {"dataset_id", res.dataset_id}, {"instrument_id", res.bars.instrument_id}, {"rows", res.bars.size()}, {"duplicates_dropped", res.duplicates_dropped}, {"fetched_at", now_utc_iso()}, {"key_fingerprint", key_fingerprint()}, {"request_ids", json::array()}};
188+ for (auto& e : res.exchanges) meta["request_ids"].push_back(e.request_id);
189+ std::ofstream(meta_path) << meta.dump(1); res.cache_path = data_path;
190+ return res;
191+#endif
192+}
193+
194+// ---------------------------------------------------------------- exports
195+ExportResult Client::export_bars(const ExportRequest& req, const std::string& out_path) {
196+ ExportResult res; const auto t0 = std::chrono::steady_clock::now();
197+ RAID_REQUIRE(!req.symbols.empty(), invalid_argument, "export: symbols required");
198+ RAID_REQUIRE(req.end.days - req.start.days <= 12 * 366, invalid_argument, "export: at most 12 years per request (chunk the range)");
199+ std::string syms; for (auto& s : req.symbols) { if (!syms.empty()) syms += ","; syms += s; }
200+ const std::string key = "export|" + req.dataset_id + "|" + syms + "|" + req.start.iso() + "|" + req.end.iso() + "|" + req.adjustment;
201+ const std::string meta_path = cache_file("export", key, ".meta.json"); std::string data_path = out_path.empty() ? cache_file("export", key, ".parquet") : out_path;
202+ if (fs::exists(meta_path) && fs::exists(data_path)) {
203+ auto meta = json::parse(std::ifstream(meta_path));
204+ if (Sha256::of_file(data_path) == meta["sha256"].get<std::string>()) { res.export_id = meta["export_id"]; res.path = data_path; res.sha256 = meta["sha256"]; res.dataset_version = meta["dataset_version"]; res.rows = meta["rows"]; res.bytes = meta["bytes"]; res.from_cache = true; return res; }
205+ }
206+ const usize log_before = log_.size();
207+ json body = {{"dataset_id", req.dataset_id}, {"symbols", req.symbols}, {"start", req.start.iso()}, {"end", req.end.iso()}, {"adjustment", req.adjustment}, {"format", "parquet"}};
208+ auto r = request("POST", "/exports", body.dump()); auto j = json::parse(r.body); res.export_id = j["id"].get<std::string>();
209+ json st; double wait = 1.0;
210+ for (int i = 0; i < 3600; ++i) {
211+ st = json::parse(request("GET", "/exports/" + res.export_id).body); const std::string s = st["status"].get<std::string>();
212+ if (s == "ready") break; RAID_REQUIRE(s != "failed" && s != "expired", api, "export " + s + ": " + st["error"].dump());
213+ std::this_thread::sleep_for(std::chrono::duration<double>(wait)); wait = std::min(wait * 1.5, 10.0);
214+ }
215+ RAID_REQUIRE(st["status"] == "ready", api, "export did not become ready");
216+ fs::create_directories(fs::path(data_path).parent_path().empty() ? "." : fs::path(data_path).parent_path());
217+ request("GET", "/exports/" + res.export_id + "/download", "", "application/json", data_path);
218+ res.path = data_path; res.sha256 = Sha256::of_file(data_path); res.rows = st.value("row_count", 0); res.bytes = st.value("bytes", 0);
219+ if (st.contains("snapshot") && st["snapshot"].contains("version")) res.dataset_version = st["snapshot"]["version"].get<i64>();
220+ res.exchanges.assign(log_.begin() + static_cast<long>(log_before), log_.end());
221+ RAID_REQUIRE(fs::file_size(data_path) == res.bytes || res.bytes == 0, io, "export size mismatch: " + std::to_string(fs::file_size(data_path)) + " vs " + std::to_string(res.bytes));
222+ json meta = {{"key", key}, {"export_id", res.export_id}, {"sha256", res.sha256}, {"dataset_version", res.dataset_version}, {"dataset_id", req.dataset_id}, {"rows", res.rows}, {"bytes", res.bytes}, {"request", body}, {"status", st}, {"fetched_at", now_utc_iso()}, {"key_fingerprint", key_fingerprint()}, {"request_ids", json::array()}};
223+ for (auto& e : res.exchanges) meta["request_ids"].push_back(e.request_id);
224+ std::ofstream(meta_path) << meta.dump(1);
225+ res.seconds_total = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
226+ return res;
227+}
228+
229+std::map<std::string, BarSeries> Client::export_and_read(const ExportRequest& req) {
230+#ifdef RAID_WITH_ARROW
231+ auto e = export_bars(req); auto pb = data::read_bars_parquet(e.path); return std::move(pb.series);
232+#else
233+ (void)req; throw RaidError(ErrorCode::unsupported, "export_and_read requires the Arrow build");
234+#endif
235+}
236+
237+Provenance Client::provenance(const std::string& dataset_id) const {
238+ Provenance p; p.dataset_id = dataset_id; p.source = "FirstRate Data via FinData OS (" + cfg_.base_url + ")"; p.code_version = RAID_VERSION;
239+ for (auto& e : log_) { if (!e.request_id.empty()) p.request_ids.push_back(e.request_id); if (e.dataset_version > 0 && (dataset_id.empty() || e.dataset_id == dataset_id)) p.dataset_version = e.dataset_version; }
240+ p.extra = {{"key_fingerprint", key_fingerprint()}, {"n_requests", log_.size()}};
241+ return p;
242+}
243+
244+// ---------------------------------------------------------------- CLI
245+namespace {
246+std::string arg(const std::vector<std::string>& a, const std::string& k, const std::string& d = "") { for (usize i = 0; i + 1 < a.size(); ++i) if (a[i] == k) return a[i + 1]; return d; }
247+}
248+int cli_main(const std::vector<std::string>& a) {
249+ if (a.empty()) { std::fprintf(stderr, "raid data catalog|health|sessions SYM --start --end|coverage [--min-sessions N] [--dataset ID]|bars SYM --start --end [--interval 1m] [--adjustment raw]|export SYMS --start --end [--dataset ID] [--out file.parquet]|fund ENTITY --as-of ISO [--metrics a,b]|query \"SQL\" [--max-rows N]|log\n"); return 2; }
250+ Client c; const std::string cmd = a[0];
251+ auto out = [](const json& j) { std::cout << j.dump(1) << "\n"; };
252+ if (cmd == "catalog") { out(c.catalog_datasets()); return 0; }
253+ if (cmd == "health") { out(c.health()); return 0; }
254+ if (cmd == "sessions") { out(c.sessions(a.at(1), interval_from_string(arg(a, "--interval", "1m")), date_from_iso(arg(a, "--start")), date_from_iso(arg(a, "--end")))); return 0; }
255+ if (cmd == "coverage") { out(c.coverage_symbols_all(arg(a, "--dataset", "market_bars.stock.1m.raw"), std::stoi(arg(a, "--min-sessions", "250")), static_cast<usize>(std::stoul(arg(a, "--max", "100000"))))); return 0; }
256+ if (cmd == "bars") { BarsRequest r; r.symbol = a.at(1); r.interval = interval_from_string(arg(a, "--interval", "1m")); r.start = date_from_iso(arg(a, "--start")); r.end = date_from_iso(arg(a, "--end")); r.adjustment = arg(a, "--adjustment", "raw");
257+ auto res = c.bars(r); json j = {{"symbol", r.symbol}, {"rows", res.bars.size()}, {"sessions", res.bars.n_sessions()}, {"dataset_id", res.dataset_id}, {"dataset_version", res.dataset_version}, {"duplicates_dropped", res.duplicates_dropped}, {"from_cache", res.from_cache}, {"cache_path", res.cache_path}, {"requests", res.exchanges.size()}};
258+ if (!arg(a, "--out").empty()) { data::write_bars_csv(arg(a, "--out"), res.bars); j["out"] = arg(a, "--out"); } out(j); return 0; }
259+ if (cmd == "export") { ExportRequest r; std::string s = a.at(1); usize p = 0; for (usize i = 0; i <= s.size(); ++i) if (i == s.size() || s[i] == ',') { r.symbols.push_back(s.substr(p, i - p)); p = i + 1; }
260+ r.dataset_id = arg(a, "--dataset", "market_bars.stock.1m.raw"); r.start = date_from_iso(arg(a, "--start")); r.end = date_from_iso(arg(a, "--end")); r.adjustment = arg(a, "--adjustment", "raw");
261+ auto e = c.export_bars(r, arg(a, "--out")); out({{"export_id", e.export_id}, {"path", e.path}, {"sha256", e.sha256}, {"dataset_version", e.dataset_version}, {"rows", e.rows}, {"bytes", e.bytes}, {"from_cache", e.from_cache}, {"seconds", e.seconds_total}}); return 0; }
262+ if (cmd == "fund") { std::vector<std::string> m; std::string ms = arg(a, "--metrics"); usize p = 0; for (usize i = 0; i <= ms.size() && !ms.empty(); ++i) if (i == ms.size() || ms[i] == ',') { m.push_back(ms.substr(p, i - p)); p = i + 1; } out(c.fundamentals(a.at(1), arg(a, "--as-of", "2026-01-01T00:00:00Z"), m, arg(a, "--period", ""))); return 0; }
263+ if (cmd == "query") { out(c.query(a.at(1), static_cast<usize>(std::stoul(arg(a, "--max-rows", "1000"))))); return 0; }
264+ if (cmd == "log") { json j = json::array(); for (auto& e : c.log()) j.push_back({{"method", e.method}, {"path", e.path}, {"status", e.status}, {"request_id", e.request_id}}); out(j); return 0; }
265+ std::fprintf(stderr, "unknown data subcommand %s\n", cmd.c_str()); return 2;
266+}
267+
268+} // namespace raid::findata
added src/data/io_csv.cpp +34 −0
@@ -0,0 +1,34 @@
1+#include "raid/data/io.hpp"
2+#include "raid/core/error.hpp"
3+#include <fstream>
4+#include <sstream>
5+#include <charconv>
6+
7+namespace raid::data {
8+
9+std::map<std::string, BarSeries> read_bars_csv(const std::string& path, Interval iv) {
10+ std::ifstream f(path); RAID_REQUIRE(f.good(), io, "cannot open " + path);
11+ std::string line; std::getline(f, line);
12+ RAID_REQUIRE(line.rfind("symbol,timestamp_utc,open,high,low,close,volume", 0) == 0, io, "unexpected CSV header in " + path + ": " + line);
13+ std::map<std::string, BarSeries> out;
14+ usize ln = 1;
15+ while (std::getline(f, line)) {
16+ ++ln; if (line.empty()) continue;
17+ std::string cells[7]; usize c = 0, start = 0;
18+ for (usize i = 0; i <= line.size() && c < 7; ++i) if (i == line.size() || line[i] == ',') { cells[c++] = line.substr(start, i - start); start = i + 1; }
19+ RAID_REQUIRE(c == 7, io, "bad CSV row " + std::to_string(ln) + " in " + path);
20+ auto num = [&](const std::string& s) { f64 v; const auto r = std::from_chars(s.data(), s.data() + s.size(), v); RAID_REQUIRE(r.ec == std::errc{}, io, "bad number '" + s + "' row " + std::to_string(ln)); return v; };
21+ auto& bs = out[cells[0]]; if (bs.symbol.empty()) { bs.symbol = cells[0]; bs.instrument_id = "synthetic:" + cells[0]; bs.interval = iv; bs.adjustment = "synthetic"; }
22+ bs.push_back(timestamp_from_iso(cells[1]), num(cells[2]), num(cells[3]), num(cells[4]), num(cells[5]), num(cells[6]));
23+ }
24+ for (auto& [k, bs] : out) bs.index_sessions();
25+ return out;
26+}
27+
28+void write_bars_csv(const std::string& path, const BarSeries& b) {
29+ std::ofstream f(path); RAID_REQUIRE(f.good(), io, "cannot write " + path);
30+ f << "symbol,timestamp_utc,open,high,low,close,volume\n"; f.precision(10);
31+ for (usize i = 0; i < b.size(); ++i) f << b.symbol << ',' << timestamp_iso(b.t_open[i]) << ',' << b.open[i] << ',' << b.high[i] << ',' << b.low[i] << ',' << b.close[i] << ',' << b.volume[i] << '\n';
32+}
33+
34+} // namespace raid::data
added src/data/parquet_arrow.cpp +69 −0
@@ -0,0 +1,69 @@
1+// Arrow/Parquet reader for FinData bars (schema v1). Column chunks are read with the Arrow C++ API; numeric columns
2+// are copied once into the BarSeries vectors (a true zero-copy view is only possible for a single contiguous chunk of
3+// float64 without nulls — the reader records whether that was the case in `metadata["zero_copy_possible"]`).
4+#include "raid/data/io.hpp"
5+#include "raid/core/error.hpp"
6+#include <arrow/api.h>
7+#include <arrow/io/api.h>
8+#include <arrow/ipc/api.h>
9+#include <parquet/arrow/reader.h>
10+#include <unordered_set>
11+
12+namespace raid::data {
13+
14+namespace {
15+ParquetBars table_to_bars(const std::shared_ptr<arrow::Table>& table, const std::vector<std::string>& symbols, Interval iv) {
16+ ParquetBars out;
17+ if (table->schema()->metadata()) for (int i = 0; i < table->schema()->metadata()->size(); ++i) out.metadata[table->schema()->metadata()->key(i)] = table->schema()->metadata()->value(i);
18+ auto col = [&](const char* name) { auto c = table->GetColumnByName(name); RAID_REQUIRE(c != nullptr, io, std::string("missing column ") + name); return c; };
19+ auto ts = col("timestamp_utc"); auto sym = col("symbol_as_reported"); auto iid = table->GetColumnByName("instrument_id"); // may be absent (projected read)
20+ auto o = col("open"), h = col("high"), l = col("low"), c = col("close"), v = col("volume");
21+ std::unordered_set<std::string> keep(symbols.begin(), symbols.end());
22+ bool contiguous = (c->num_chunks() == 1) && (c->null_count() == 0);
23+ out.metadata["zero_copy_possible"] = contiguous ? "true" : "false";
24+ // Fast path: symbols are stored in runs (one symbol = one contiguous block in FinData exports); avoid a map lookup per row.
25+ BarSeries* cur = nullptr; std::string cur_sym;
26+ for (int ch = 0; ch < ts->num_chunks(); ++ch) {
27+ auto ta = std::static_pointer_cast<arrow::TimestampArray>(ts->chunk(ch));
28+ auto sa = std::static_pointer_cast<arrow::StringArray>(sym->chunk(ch));
29+ std::shared_ptr<arrow::StringArray> ia = iid ? std::static_pointer_cast<arrow::StringArray>(iid->chunk(ch)) : nullptr;
30+ auto oa = std::static_pointer_cast<arrow::DoubleArray>(o->chunk(ch)), ha = std::static_pointer_cast<arrow::DoubleArray>(h->chunk(ch));
31+ auto la = std::static_pointer_cast<arrow::DoubleArray>(l->chunk(ch)), ca = std::static_pointer_cast<arrow::DoubleArray>(c->chunk(ch));
32+ auto va = std::static_pointer_cast<arrow::DoubleArray>(v->chunk(ch));
33+ const auto unit = std::static_pointer_cast<arrow::TimestampType>(ta->type())->unit();
34+ i64 mult = 1; if (unit == arrow::TimeUnit::SECOND) mult = 1'000'000; else if (unit == arrow::TimeUnit::MILLI) mult = 1000; else if (unit == arrow::TimeUnit::NANO) mult = -1000;
35+ for (int64_t i = 0; i < ta->length(); ++i) {
36+ const std::string_view sv = sa->GetView(i);
37+ if (!cur || sv != cur_sym) { cur_sym = std::string(sv); if (!keep.empty() && !keep.count(cur_sym)) { cur = nullptr; continue; } cur = &out.series[cur_sym]; if (cur->symbol.empty()) { cur->symbol = cur_sym; cur->instrument_id = ia ? ia->GetString(i) : ("firstrate:" + std::string(std::static_pointer_cast<arrow::StringArray>(col("asset_class")->chunk(ch))->GetView(i)) + ":" + cur_sym); cur->interval = iv; cur->adjustment = "raw"; cur->reserve(4096); } }
38+ else if (!cur) continue;
39+ const i64 raw = ta->Value(i); const i64 us = mult > 0 ? raw * mult : raw / 1000;
40+ cur->push_back(Timestamp(us), oa->Value(i), ha->Value(i), la->Value(i), ca->Value(i), va->Value(i));
41+ ++out.rows_read;
42+ }
43+ }
44+ for (auto& [k, bs] : out.series) { auto st = bs.normalize(); out.duplicates_dropped += st.duplicates_removed; }
45+ return out;
46+}
47+} // namespace
48+
49+ParquetBars read_bars_parquet(const std::string& path, const std::vector<std::string>& symbols, Interval iv) {
50+ auto infile = arrow::io::ReadableFile::Open(path); RAID_REQUIRE(infile.ok(), io, "cannot open " + path + ": " + infile.status().ToString());
51+ auto reader_r = parquet::arrow::OpenFile(*infile, arrow::default_memory_pool()); RAID_REQUIRE(reader_r.ok(), io, "parquet open: " + reader_r.status().ToString());
52+ std::unique_ptr<parquet::arrow::FileReader> reader = std::move(*reader_r);
53+ // Column projection: only the 8 columns the estimators need (of 20) are decoded — the key I/O optimisation of RQ3.
54+ std::shared_ptr<arrow::Schema> schema; auto sst = reader->GetSchema(&schema); RAID_REQUIRE(sst.ok(), io, "parquet schema: " + sst.ToString());
55+ // (instrument_id is derived from symbol_as_reported for FinData exports: "firstrate:<class>:<SYMBOL>"; reading the string column costs more than every numeric column together)
56+ std::vector<int> idx; for (const char* name : {"symbol_as_reported", "asset_class", "timestamp_utc", "open", "high", "low", "close", "volume"}) { const int k = schema->GetFieldIndex(name); RAID_REQUIRE(k >= 0, io, std::string("missing column ") + name); idx.push_back(k); }
57+ reader->set_use_threads(true);
58+ std::shared_ptr<arrow::Table> table; auto st = reader->ReadTable(idx, &table); RAID_REQUIRE(st.ok(), io, "parquet read: " + st.ToString());
59+ return table_to_bars(table, symbols, iv);
60+}
61+
62+ParquetBars read_bars_arrow_ipc(const std::string& path, Interval iv) {
63+ auto infile = arrow::io::ReadableFile::Open(path); RAID_REQUIRE(infile.ok(), io, "cannot open " + path);
64+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(*infile); RAID_REQUIRE(reader.ok(), io, "arrow ipc open: " + reader.status().ToString());
65+ auto table = (*reader)->ToTable(); RAID_REQUIRE(table.ok(), io, "arrow ipc read: " + table.status().ToString());
66+ return table_to_bars(*table, {}, iv);
67+}
68+
69+} // namespace raid::data
added src/data/returns.cpp +79 −0
@@ -0,0 +1,79 @@
1+#include "raid/data/returns.hpp"
2+#include "raid/data/resample.hpp"
3+#include "raid/core/error.hpp"
4+#include <cmath>
5+#include <limits>
6+
7+namespace raid::data {
8+
9+std::vector<f64> returns(std::span<const f64> p, ReturnKind kind) {
10+ std::vector<f64> r; if (p.size() < 2) return r; r.resize(p.size() - 1);
11+ for (usize i = 1; i < p.size(); ++i) {
12+ RAID_REQUIRE(p[i] > 0 && p[i - 1] > 0, data_quality, "returns: non-positive price");
13+ r[i - 1] = kind == ReturnKind::log ? std::log(p[i] / p[i - 1]) : p[i] / p[i - 1] - 1.0;
14+ }
15+ return r;
16+}
17+
18+IntradayReturns intraday_returns(const BarSeries& b, ReturnKind kind, bool use_open_first) {
19+ IntradayReturns out; const usize S = b.n_sessions();
20+ out.session_offsets.push_back(0); out.session_dates = b.session_dates;
21+ const f64 nan = std::numeric_limits<f64>::quiet_NaN();
22+ f64 prev_close = nan;
23+ auto ret = [&](f64 a, f64 c) { RAID_REQUIRE(a > 0 && c > 0, data_quality, "returns: non-positive price"); return kind == ReturnKind::log ? std::log(c / a) : c / a - 1.0; };
24+ for (usize k = 0; k < S; ++k) {
25+ const usize b0 = static_cast<usize>(b.session_offsets[k]), b1 = static_cast<usize>(b.session_offsets[k + 1]);
26+ if (use_open_first) out.r.push_back(ret(b.open[b0], b.close[b0]));
27+ for (usize i = b0 + 1; i < b1; ++i) out.r.push_back(ret(b.close[i - 1], b.close[i]));
28+ out.session_offsets.push_back(static_cast<i64>(out.r.size()));
29+ out.overnight.push_back(std::isnan(prev_close) ? nan : ret(prev_close, b.open[b0]));
30+ out.open_to_close.push_back(ret(b.open[b0], b.close[b1 - 1]));
31+ out.close_to_close.push_back(std::isnan(prev_close) ? nan : ret(prev_close, b.close[b1 - 1]));
32+ prev_close = b.close[b1 - 1];
33+ }
34+ return out;
35+}
36+
37+BarSeries aggregate(const BarSeries& bars, Interval target, const MarketCalendar& cal, SessionPolicy policy) {
38+ RAID_REQUIRE(to_duration(target).us > to_duration(bars.interval).us, invalid_argument, "aggregate: target must be coarser than source");
39+ BarSeries out; out.symbol = bars.symbol; out.instrument_id = bars.instrument_id; out.interval = target; out.adjustment = bars.adjustment;
40+ const i64 step = to_duration(target).us;
41+ for (usize k = 0; k < bars.n_sessions(); ++k) {
42+ const Date d = bars.session_dates[k];
43+ if (!cal.is_trading_day(d)) continue;
44+ const auto sb = cal.session(d);
45+ const Timestamp anchor = policy == SessionPolicy::regular ? sb.open_utc : sb.ext_open_utc;
46+ const Timestamp end = policy == SessionPolicy::regular ? sb.close_utc : sb.ext_close_utc;
47+ i64 cur_bin = -1; f64 o = 0, h = 0, l = 0, c = 0, v = 0; u32 fl = 0; Timestamp bin_t;
48+ auto flush = [&]() { if (cur_bin >= 0) { out.t_open.push_back(bin_t); out.session.push_back(d); out.open.push_back(o); out.high.push_back(h); out.low.push_back(l); out.close.push_back(c); out.volume.push_back(v); out.flags.push_back(fl); } };
49+ for (i64 i = bars.session_offsets[k]; i < bars.session_offsets[k + 1]; ++i) {
50+ const usize j = static_cast<usize>(i); const Timestamp t = bars.t_open[j];
51+ if (t < anchor) continue;
52+ if (policy == SessionPolicy::regular ? t > end : t >= end) continue;
53+ i64 bin = (t - anchor).us / step;
54+ if (target == Interval::d1) bin = 0;
55+ // The 16:00 closing print (t == close) forms its own bin, as in FinData's derived 5m/30m/1h datasets
56+ // (verified live 2026-11-02: 79 regular 5m bars per normal session, 43 on early closes).
57+ if (bin != cur_bin) { flush(); cur_bin = bin; bin_t = anchor + Duration(bin * step); o = bars.open[j]; h = bars.high[j]; l = bars.low[j]; c = bars.close[j]; v = bars.volume[j]; fl = bars.flags[j]; }
58+ else { h = std::max(h, bars.high[j]); l = std::min(l, bars.low[j]); c = bars.close[j]; v += bars.volume[j]; fl |= bars.flags[j]; }
59+ }
60+ flush();
61+ }
62+ out.index_sessions();
63+ return out;
64+}
65+
66+SampledPrices subsample(const BarSeries& bars, usize step, usize offset) {
67+ RAID_REQUIRE(step >= 1 && offset < step, invalid_argument, "subsample: need step >= 1 and offset < step");
68+ SampledPrices s; s.session_offsets.push_back(0); s.session_dates = bars.session_dates;
69+ for (usize k = 0; k < bars.n_sessions(); ++k) {
70+ const usize b0 = static_cast<usize>(bars.session_offsets[k]), b1 = static_cast<usize>(bars.session_offsets[k + 1]);
71+ for (usize i = b0 + offset; i < b1; i += step) s.p.push_back(bars.close[i]);
72+ // always include the last price of the session so the grid ends at the close
73+ if (b1 > b0 && ((b1 - 1 - b0) % step) != offset % step && (b1 - 1) >= b0 + offset) s.p.push_back(bars.close[b1 - 1]);
74+ s.session_offsets.push_back(static_cast<i64>(s.p.size()));
75+ }
76+ return s;
77+}
78+
79+} // namespace raid::data
80
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere