WP40 RAID: Phase 8 fix — data layer sources were ignored by a too-broad 'data/' pattern (found by the clean-checkout reproduction); track include/raid/data, src/data · 1badb1c · wp40_uqo

WP40 RAID: Phase 8 fix — data layer sources were ignored by a too-broad 'data/' pattern (found by the clean-checkout reproduction); track include/raid/data, src/data

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 18 h ago (Nov 2, 2026) parent 1eed3a1 Browse files

17 changed files +1,145 −1

modified .gitignore +1 −1

		`@@ -22,7 +22,7 @@ __pycache__/`
22	22	# Data — FirstRate private license: never commit real bars
23	23	datasets/cache/
24	24	datasets/real/
25		−data/
	25	+/data/
26	26	experiments/results/raw/
27	27	*.parquet
28	28	*.arrow

added include/raid/data/bar_series.hpp +50 −0

		`@@ -0,0 +1,50 @@`
	1	+// BarSeries — OHLCV bars for one instrument (columnar), with session structure derived from the MarketCalendar.
	2	+#pragma once
	3	+#include <vector>
	4	+#include <span>
	5	+#include <string>
	6	+#include "raid/core/types.hpp"
	7	+#include "raid/data/calendar.hpp"
	8	+#include "raid/data/quality.hpp"
	9	+
	10	+namespace raid {
	11	+
	12	+struct BarSeries {
	13	+ std::string symbol;
	14	+ std::string instrument_id;
	15	+ Interval interval{Interval::m1};
	16	+ std::string adjustment{"raw"};
	17	+ std::vector<Timestamp> t_open; // bar OPEN, UTC
	18	+ std::vector<Date> session; // exchange-local session date
	19	+ std::vector<f64> open, high, low, close, volume;
	20	+ std::vector<u32> flags; // bitmask (see QualityFlag)
	21	+ // Session index: session_offsets[k]..session_offsets[k+1] are the bars of session k (built by index_sessions()).
	22	+ std::vector<i64> session_offsets;
	23	+ std::vector<Date> session_dates;
	24	+
	25	+ usize size() const { return close.size(); }
	26	+ bool empty() const { return close.empty(); }
	27	+ void reserve(usize n) { t_open.reserve(n); session.reserve(n); open.reserve(n); high.reserve(n); low.reserve(n); close.reserve(n); volume.reserve(n); flags.reserve(n); }
	28	+ void push_back(Timestamp t, f64 o, f64 h, f64 l, f64 c, f64 v, u32 f = 0) {
	29	+ t_open.push_back(t); session.push_back(MarketCalendar::session_of(t)); open.push_back(o); high.push_back(h); low.push_back(l); close.push_back(c); volume.push_back(v); flags.push_back(f);
	30	+ }
	31	+ usize n_sessions() const { return session_dates.size(); }
	32	+ std::span<const f64> session_closes(usize k) const { return std::span<const f64>(close).subspan(static_cast<usize>(session_offsets[k]), static_cast<usize>(session_offsets[k + 1] - session_offsets[k])); }
	33	+ std::span<const Timestamp> session_times(usize k) const { return std::span<const Timestamp>(t_open).subspan(static_cast<usize>(session_offsets[k]), static_cast<usize>(session_offsets[k + 1] - session_offsets[k])); }
	34	+
	35	+ /// Sorts by time, drops exact duplicate timestamps (keeps the first), rebuilds the session index. Returns counts.
	36	+ struct NormalizeStats { usize duplicates_removed; usize reordered; };
	37	+ NormalizeStats normalize();
	38	+ void index_sessions();
	39	+ /// Validates OHLCV, interval grid, session membership and coverage; does not modify data.
	40	+ QualityReport validate(const MarketCalendar& cal, SessionPolicy policy = SessionPolicy::regular, f64 jump_threshold = 0.25) const;
	41	+ /// Keeps only bars of the requested session policy (regular: 09:30 ≤ open ≤ close time, i.e. includes the 16:00 print).
	42	+ BarSeries filter_session(const MarketCalendar& cal, SessionPolicy policy) const;
	43	+};
	44	+
	45	+enum QualityFlag : u32 {
	46	+ kFlagOhlcInconsistent = 1u << 0, kFlagNonpositivePrice = 1u << 1, kFlagNegativeVolume = 1u << 2, kFlagZeroVolume = 1u << 3,
	47	+ kFlagOffInterval = 1u << 4, kFlagOutsideSession = 1u << 5, kFlagNonTradingDay = 1u << 6, kFlagPriceJump = 1u << 7, kFlagVendorFlag = 1u << 8,
	48	+};
	49	+
	50	+} // namespace raid

added include/raid/data/calendar.hpp +58 −0

		`@@ -0,0 +1,58 @@`
	1	+// MarketCalendar — XNYS (NYSE/Nasdaq) trading days, early closes, regular-session bounds and America/New_York DST.
	2	+// Rules are computed (no external data): NYSE holiday rules since 2000, special closures listed explicitly,
	3	+// US DST rules (2007+: 2nd Sunday March → 1st Sunday November; 1987–2006: 1st Sunday April → last Sunday October).
	4	+// Validation: tests/reference/calendar_vs_exchange_calendars.cpp compares against the Python exchange_calendars fixture.
	5	+#pragma once
	6	+#include <vector>
	7	+#include <string>
	8	+#include <optional>
	9	+#include "raid/core/types.hpp"
	10	+
	11	+namespace raid {
	12	+
	13	+struct SessionBounds {
	14	+ Date session;
	15	+ Timestamp open_utc; // 09:30 ET
	16	+ Timestamp close_utc; // 16:00 ET (13:00 on early closes) — the close bar OPENS at this instant (closing print)
	17	+ Timestamp ext_open_utc; // 04:00 ET
	18	+ Timestamp ext_close_utc; // 20:00 ET (bars open until 19:59). On early-close days the vendor stops at 17:00.
	19	+ bool early_close{false};
	20	+ i64 utc_offset_seconds{0}; // ET offset that day at the open (−18000 EST / −14400 EDT)
	21	+};
	22	+
	23	+class MarketCalendar {
	24	+ public:
	25	+ enum class Market { XNYS };
	26	+ explicit MarketCalendar(Market m = Market::XNYS) : market_(m) {}
	27	+ Market market() const { return market_; }
	28	+
	29	+ bool is_trading_day(Date d) const;
	30	+ bool is_early_close(Date d) const;
	31	+ std::optional<std::string> holiday_name(Date d) const;
	32	+ /// Session bounds for a trading day (throws if not a trading day).
	33	+ SessionBounds session(Date d) const;
	34	+ /// Regular-hours bars expected for one session at the given interval (1m: 391 normal, 211 early close).
	35	+ usize expected_regular_bars(Date d, Interval iv) const;
	36	+ /// Trading sessions in [start, end] inclusive.
	37	+ std::vector<Date> sessions(Date start, Date end) const;
	38	+ /// Next / previous trading day strictly after / before d.
	39	+ Date next_session(Date d) const;
	40	+ Date previous_session(Date d) const;
	41	+ /// America/New_York offset (seconds east of UTC) at a UTC instant.
	42	+ static i64 new_york_offset(Timestamp utc);
	43	+ /// Local New York civil date and minute-of-day of a UTC instant.
	44	+ struct Local { Date date; int minute_of_day; i64 offset_seconds; };
	45	+ static Local to_new_york(Timestamp utc);
	46	+ /// UTC instant of a New York wall-clock time (assumes the time exists; gap hour maps to post-transition).
	47	+ static Timestamp from_new_york(Date d, int minute_of_day);
	48	+ /// Session date of a bar-open timestamp under the US equity convention (04:00–20:00 ET same civil day).
	49	+ static Date session_of(Timestamp bar_open_utc) { return to_new_york(bar_open_utc).date; }
	50	+
	51	+ private:
	52	+ Market market_;
	53	+};
	54	+
	55	+/// Convenience: trading-day distance between two sessions (number of sessions strictly between + 1).
	56	+usize trading_days_between(const MarketCalendar& cal, Date a, Date b);
	57	+
	58	+} // namespace raid

added include/raid/data/findata.hpp +110 −0

		`@@ -0,0 +1,110 @@`
	1	+// FinData OS client (https://findata.spboucher.ai/api/v1) — bearer auth from FINDATA_API_KEY / FINDATA_API_KEY_FILE,
	2	+// TLS via libcurl, redacted logs, token-bucket rate limiting, bounded concurrency, retries with exponential backoff
	3	+// honouring Retry-After, cursor pagination, 31-day chunking for 1m bars, asynchronous exports with polling/download,
	4	+// on-disk cache keyed by dataset version, SHA-256 of every downloaded artifact, and provenance (request ids, versions).
	5	+#pragma once
	6	+#include <string>
	7	+#include <vector>
	8	+#include <map>
	9	+#include <optional>
	10	+#include <mutex>
	11	+#include <chrono>
	12	+#include <functional>
	13	+#include <nlohmann/json.hpp>
	14	+#include "raid/core/types.hpp"
	15	+#include "raid/core/estimate.hpp"
	16	+#include "raid/data/bar_series.hpp"
	17	+
	18	+namespace raid::findata {
	19	+
	20	+struct Config {
	21	+ std::string base_url{"https://findata.spboucher.ai/api/v1"};
	22	+ std::string api_key; // empty → FINDATA_API_KEY, then FINDATA_API_KEY_FILE
	23	+ std::string cache_dir; // empty → $RAID_CACHE_DIR or ./datasets/cache
	24	+ double requests_per_second{5.0}; // token bucket rate (600/min limit → 5/s leaves headroom)
	25	+ double burst{10.0};
	26	+ int max_retries{5};
	27	+ long timeout_seconds{120};
	28	+ unsigned max_concurrency{4};
	29	+ bool verify_tls{true};
	30	+ bool offline{false}; // only serve from cache; any network call throws (for CI)
	31	+ static Config from_env();
	32	+};
	33	+
	34	+/// Provenance of one HTTP exchange (no secrets).
	35	+struct Exchange {
	36	+ std::string method, path; // query string without the key (the key is a header)
	37	+ long status{0};
	38	+ std::string request_id, dataset_id; // X-Request-ID, X-Dataset-Id
	39	+ i64 dataset_version{-1}; // X-Dataset-Version
	40	+ std::string next_cursor; // X-Next-Cursor (arrow pages)
	41	+ double seconds{0};
	42	+ usize bytes{0};
	43	+ int attempts{1};
	44	+};
	45	+
	46	+class TokenBucket {
	47	+ public:
	48	+ TokenBucket(double rate, double burst) : rate_(rate), capacity_(burst), tokens_(burst), last_(std::chrono::steady_clock::now()) {}
	49	+ /// Blocks until a token is available; returns the wait in seconds.
	50	+ double acquire();
	51	+ private:
	52	+ double rate_, capacity_, tokens_; std::chrono::steady_clock::time_point last_; std::mutex m_;
	53	+};
	54	+
	55	+/// Splits [start, end] (inclusive dates) into chunks of at most `max_days` calendar days (API constraint: 1m ≤ 31 d,
	56	+/// 5m ≤ 186 d, 1d ≤ 10 y; exports ≤ 12 y).
	57	+std::vector<std::pair<Date, Date>> chunk_dates(Date start, Date end, int max_days);
	58	+int max_days_per_call(Interval iv);
	59	+
	60	+struct BarsRequest {
	61	+ std::string symbol; Interval interval{Interval::m1}; Date start, end; std::string adjustment{"raw"}; usize page_limit{10000}; std::optional<i64> version;
	62	+};
	63	+struct BarsResult { BarSeries bars; std::vector<Exchange> exchanges; i64 dataset_version{-1}; std::string dataset_id; usize duplicates_dropped{0}; std::string cache_path; bool from_cache{false}; };
	64	+
	65	+struct ExportRequest { std::string dataset_id{"market_bars.stock.1m.raw"}; std::vector<std::string> symbols; Date start, end; std::string adjustment{"raw"}; };
	66	+struct ExportResult { std::string export_id, path, sha256; i64 dataset_version{-1}; usize rows{0}; usize bytes{0}; std::vector<Exchange> exchanges; bool from_cache{false}; double seconds_total{0}; };
	67	+
	68	+class Client {
	69	+ public:
	70	+ explicit Client(Config cfg = Config::from_env());
	71	+ ~Client();
	72	+ const Config& config() const { return cfg_; }
	73	+ std::string key_fingerprint() const; // first 12 hex of sha256(key): identifies the key without revealing it
	74	+
	75	+ nlohmann::json health();
	76	+ nlohmann::json openapi();
	77	+ nlohmann::json catalog_datasets();
	78	+ nlohmann::json dataset(const std::string& id);
	79	+ nlohmann::json sessions(const std::string& symbol, Interval iv, Date start, Date end);
	80	+ nlohmann::json coverage_symbols(const std::string& dataset_id, int min_sessions, usize limit = 200, const std::string& cursor = "");
	81	+ /// All symbols with ≥ min_sessions sessions (follows cursors); returns {symbol, sessions, d0, d1, bars}.
	82	+ nlohmann::json coverage_symbols_all(const std::string& dataset_id, int min_sessions, usize max_symbols = 100000);
	83	+ nlohmann::json fundamentals(const std::string& entity, const std::string& as_of_iso, const std::vector<std::string>& metrics = {}, const std::string& period_type = "");
	84	+ /// Bounded read-only SQL (server: 30 s, ≤ 100 000 rows). Use only for universe/coverage questions.
	85	+ nlohmann::json query(const std::string& sql, usize max_rows = 1000);
	86	+
	87	+ /// Bars via GET /market/bars?format=arrow with cursor pagination and ≤ max_days chunking; cached per dataset version.
	88	+ BarsResult bars(const BarsRequest& req);
	89	+ /// Asynchronous export (POST /exports → poll → download) with ≤ 10-year chunking handled by the caller; cached.
	90	+ ExportResult export_bars(const ExportRequest& req, const std::string& out_path = "");
	91	+ /// Convenience: export, then read into BarSeries per symbol (requires Arrow build).
	92	+ std::map<std::string, BarSeries> export_and_read(const ExportRequest& req);
	93	+
	94	+ const std::vector<Exchange>& log() const { return log_; }
	95	+ Provenance provenance(const std::string& dataset_id) const;
	96	+
	97	+ struct Response { long status{0}; std::string body; std::map<std::string, std::string> headers; double seconds{0}; int attempts{1}; };
	98	+ Response request(const std::string& method, const std::string& path_and_query, const std::string& body = "", const std::string& content_type = "application/json", const std::string& download_to = "");
	99	+
	100	+ private:
	101	+ Response request_once(const std::string& method, const std::string& url, const std::string& body, const std::string& content_type, const std::string& download_to);
	102	+ std::string cache_file(const std::string& kind, const std::string& key_material, const std::string& ext) const;
	103	+ Config cfg_; std::string key_; TokenBucket bucket_; std::vector<Exchange> log_; std::mutex log_m_;
	104	+ void* curl_share_{nullptr};
	105	+};
	106	+
	107	+/// `raid data …` subcommands (returns exit code).
	108	+int cli_main(const std::vector<std::string>& args);
	109	+
	110	+} // namespace raid::findata

added include/raid/data/io.hpp +24 −0

		`@@ -0,0 +1,24 @@`
	1	+// Readers: CSV fixtures (always available) and Parquet/Arrow IPC (when RAID_WITH_ARROW).
	2	+#pragma once
	3	+#include <string>
	4	+#include <vector>
	5	+#include <map>
	6	+#include "raid/data/bar_series.hpp"
	7	+
	8	+namespace raid::data {
	9	+
	10	+/// CSV fixture format (synthetic, committed): header
	11	+/// symbol,timestamp_utc,open,high,low,close,volume (timestamp ISO-8601 UTC, bar open)
	12	+std::map<std::string, BarSeries> read_bars_csv(const std::string& path, Interval iv = Interval::m1);
	13	+void write_bars_csv(const std::string& path, const BarSeries& bars);
	14	+
	15	+#ifdef RAID_WITH_ARROW
	16	+/// Reads a FinData bars Parquet file (schema v1) into one BarSeries per instrument_id; `symbols` filters (empty = all).
	17	+/// Returns also the dataset metadata found in the file footer (findata.schema_version, timestamp_convention).
	18	+struct ParquetBars { std::map<std::string, BarSeries> series; std::map<std::string, std::string> metadata; usize rows_read{0}; usize duplicates_dropped{0}; };
	19	+ParquetBars read_bars_parquet(const std::string& path, const std::vector<std::string>& symbols = {}, Interval iv = Interval::m1);
	20	+/// Reads an Arrow IPC stream file (as returned by GET /market/bars?format=arrow).
	21	+ParquetBars read_bars_arrow_ipc(const std::string& path, Interval iv = Interval::m1);
	22	+#endif
	23	+
	24	+} // namespace raid::data

added include/raid/data/panel.hpp +42 −0

		`@@ -0,0 +1,42 @@`
	1	+// Panel: N (possibly ragged) series stored contiguously (CSR layout), plus an aligned dense view when the series
	2	+// share a common time index. The CSR layout is what the batched CPU/Metal kernels consume directly.
	3	+#pragma once
	4	+#include <vector>
	5	+#include <span>
	6	+#include <string>
	7	+#include <Eigen/Dense>
	8	+#include "raid/core/types.hpp"
	9	+#include "raid/core/error.hpp"
	10	+
	11	+namespace raid {
	12	+
	13	+struct Panel {
	14	+ std::vector<std::string> ids; // n_series
	15	+ std::vector<i64> offsets; // n_series + 1, offsets into t / v
	16	+ std::vector<Timestamp> t; // concatenated timestamps
	17	+ std::vector<f64> v; // concatenated values
	18	+
	19	+ usize n_series() const { return ids.size(); }
	20	+ usize total_size() const { return v.size(); }
	21	+ usize series_size(usize i) const { return static_cast<usize>(offsets[i + 1] - offsets[i]); }
	22	+ std::span<const f64> values(usize i) const { return std::span<const f64>(v).subspan(static_cast<usize>(offsets[i]), series_size(i)); }
	23	+ std::span<const Timestamp> times(usize i) const { return std::span<const Timestamp>(t).subspan(static_cast<usize>(offsets[i]), series_size(i)); }
	24	+ void push_back(std::string id, std::vector<Timestamp> ts, std::vector<f64> vs) {
	25	+ RAID_REQUIRE(ts.size() == vs.size(), invalid_argument, "Panel::push_back size mismatch");
	26	+ if (offsets.empty()) offsets.push_back(0);
	27	+ ids.push_back(std::move(id));
	28	+ t.insert(t.end(), ts.begin(), ts.end()); v.insert(v.end(), vs.begin(), vs.end());
	29	+ offsets.push_back(static_cast<i64>(v.size()));
	30	+ }
	31	+ bool is_rectangular() const { if (n_series() == 0) return true; const usize m = series_size(0); for (usize i = 1; i < n_series(); ++i) if (series_size(i) != m) return false; return true; }
	32	+ /// Dense T × N column-major matrix (requires rectangular panel).
	33	+ Eigen::MatrixXd dense() const {
	34	+ RAID_REQUIRE(is_rectangular(), invalid_argument, "Panel::dense requires equal-length series");
	35	+ const usize T = n_series() ? series_size(0) : 0;
	36	+ Eigen::MatrixXd m(static_cast<Eigen::Index>(T), static_cast<Eigen::Index>(n_series()));
	37	+ for (usize j = 0; j < n_series(); ++j) { auto s = values(j); for (usize i = 0; i < T; ++i) m(static_cast<Eigen::Index>(i), static_cast<Eigen::Index>(j)) = s[i]; }
	38	+ return m;
	39	+ }
	40	+};
	41	+
	42	+} // namespace raid

added include/raid/data/quality.hpp +33 −0

		`@@ -0,0 +1,33 @@`
	1	+#pragma once
	2	+#include <string>
	3	+#include <vector>
	4	+#include <map>
	5	+#include <nlohmann/json.hpp>
	6	+#include "raid/core/types.hpp"
	7	+
	8	+namespace raid {
	9	+
	10	+/// QualityReport: counts of data issues found by BarSeries::validate(); prices are never altered.
	11	+struct QualityReport {
	12	+ std::string symbol;
	13	+ usize n_bars{0};
	14	+ usize n_sessions{0};
	15	+ usize ohlc_inconsistent{0}; // high < max(open,close) or low > min(open,close) or high < low
	16	+ usize nonpositive_price{0};
	17	+ usize negative_volume{0};
	18	+ usize zero_volume{0};
	19	+ usize duplicates{0}; // duplicate (timestamp) rows removed
	20	+ usize off_interval{0}; // timestamp not aligned on the interval grid
	21	+ usize outside_session{0}; // bar outside 04:00–20:00 ET (extended) / outside regular when policy=regular
	22	+ usize non_trading_day{0};
	23	+ usize unsorted{0}; // rows reordered
	24	+ usize missing_regular_bars{0}; // sum over sessions of (expected − present) regular bars
	25	+ usize price_jumps{0}; // \|close/prev close − 1\| > jump_threshold within a session
	26	+ f64 coverage{1.0}; // present / expected regular bars
	27	+ std::map<std::string, usize> per_session_missing; // session → missing bars (only sessions with gaps)
	28	+ std::vector<std::string> notes;
	29	+ bool has_errors() const { return ohlc_inconsistent \|\| nonpositive_price \|\| negative_volume \|\| off_interval \|\| non_trading_day; }
	30	+ nlohmann::json to_json() const;
	31	+};
	32	+
	33	+} // namespace raid

added include/raid/data/resample.hpp +17 −0

		`@@ -0,0 +1,17 @@`
	1	+// Calendar-aware aggregation of bars to a coarser interval, and sparse-sampling helpers for realized measures.
	2	+#pragma once
	3	+#include "raid/data/bar_series.hpp"
	4	+
	5	+namespace raid::data {
	6	+
	7	+/// Aggregates bars into bins anchored at the regular session open (09:30 ET) of each session:
	8	+/// bin k covers [open + k·Δ, open + (k+1)·Δ); the 16:00 closing print forms its own bin (FinData convention). Bins without bars are absent
	9	+/// (no bar is manufactured). OHLC = first/max/min/last, volume = sum, trade_count = number of source bars.
	10	+BarSeries aggregate(const BarSeries& bars, Interval target, const MarketCalendar& cal, SessionPolicy policy = SessionPolicy::regular);
	11	+
	12	+/// Subsampled price grid per session for realized measures: every `step` bars starting at `offset` (0 ≤ offset < step).
	13	+/// Returns the close prices picked, grouped by session (offsets). Used for 5-minute subsampled RV on 1-minute bars.
	14	+struct SampledPrices { std::vector<f64> p; std::vector<i64> session_offsets; std::vector<Date> session_dates; };
	15	+SampledPrices subsample(const BarSeries& bars, usize step, usize offset = 0);
	16	+
	17	+} // namespace raid::data

added include/raid/data/returns.hpp +29 −0

		`@@ -0,0 +1,29 @@`
	1	+// Returns from prices / bars. Session-aware: by default no return is computed across a session boundary
	2	+// (the first bar of each session has no intraday return); overnight returns are optional and explicit.
	3	+#pragma once
	4	+#include <vector>
	5	+#include <span>
	6	+#include "raid/core/types.hpp"
	7	+#include "raid/data/bar_series.hpp"
	8	+
	9	+namespace raid::data {
	10	+
	11	+enum class ReturnKind { log, arithmetic };
	12	+
	13	+/// Plain returns of a price vector (r_t = log(p_t/p_{t-1}) or p_t/p_{t-1} − 1), size n−1.
	14	+std::vector<f64> returns(std::span<const f64> prices, ReturnKind kind = ReturnKind::log);
	15	+
	16	+struct IntradayReturns {
	17	+ std::vector<f64> r; // concatenated intraday returns, session by session
	18	+ std::vector<i64> session_offsets; // per session: offsets into r (n_sessions + 1)
	19	+ std::vector<Date> session_dates;
	20	+ std::vector<f64> overnight; // close(prev session) → first price of session (NaN for the first session), one per session
	21	+ std::vector<f64> open_to_close; // open-to-close daily return (from first bar open to last bar close)
	22	+ std::vector<f64> close_to_close; // daily close-to-close return (NaN for first)
	23	+ std::span<const f64> session(usize k) const { return std::span<const f64>(r).subspan(static_cast<usize>(session_offsets[k]), static_cast<usize>(session_offsets[k + 1] - session_offsets[k])); }
	24	+};
	25	+/// Intraday close-to-close bar returns within each session, plus daily summaries. `use_open_first` makes the first
	26	+/// return of a session open→close of the first bar instead of skipping it (not the default; documented in the registry).
	27	+IntradayReturns intraday_returns(const BarSeries& bars, ReturnKind kind = ReturnKind::log, bool use_open_first = false);
	28	+
	29	+} // namespace raid::data

added include/raid/data/time_series.hpp +24 −0

		`@@ -0,0 +1,24 @@`
	1	+#pragma once
	2	+#include <vector>
	3	+#include <span>
	4	+#include <string>
	5	+#include "raid/core/types.hpp"
	6	+#include "raid/core/error.hpp"
	7	+
	8	+namespace raid {
	9	+
	10	+/// Columnar time series: strictly increasing timestamps and one double per observation.
	11	+struct TimeSeries {
	12	+ std::string id;
	13	+ std::vector<Timestamp> t;
	14	+ std::vector<f64> v;
	15	+ usize size() const { return v.size(); }
	16	+ bool empty() const { return v.empty(); }
	17	+ void validate() const {
	18	+ RAID_REQUIRE(t.size() == v.size(), invalid_argument, "TimeSeries: t/v size mismatch");
	19	+ for (usize i = 1; i < t.size(); ++i) RAID_REQUIRE(t[i] > t[i - 1], data_quality, "TimeSeries: timestamps not strictly increasing at index " + std::to_string(i));
	20	+ }
	21	+ std::span<const f64> values() const { return v; }
	22	+};
	23	+
	24	+} // namespace raid

added scripts/audit_manuscript_numbers.py +48 −0

		`@@ -0,0 +1,48 @@`
	1	+#!/usr/bin/env python3
	2	+"""Phase 8 numerical audit: every quantitative claim in the manuscript prose is mapped to a value in
	3	+experiments/results/key_numbers.json (or the test suite) and checked within a tolerance. Exits non-zero on mismatch."""
	4	+import json, re, sys, os
	5	+root = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."); k = json.load(open(os.path.join(root, "experiments", "results", "key_numbers.json")))
	6	+text = "".join(open(os.path.join(root, "paper", "sections", f)).read() for f in ["introduction.tex", "setup.tex", "microbenchmarks.tex", "endtoend.tex", "applications.tex", "correctness.tex", "conclusion.tex", "threats.tex"])
	7	+abstract = open(os.path.join(root, "paper", "main.tex")).read()
	8	+def present(s): return s in text or s in abstract
	9	+checks = []
	10	+def chk(claim, value, fmt, tol=None):
	11	+ s = fmt.format(value); ok = present(s); checks.append((claim, s, ok))
	12	+P = k["primitives"]; B = k["breakeven"]; E1 = k["E1"]; E2 = k["E2"]; E3 = k["E3"]; E4 = k["E4"]; E5 = k["E5"]; E6 = k["E6"]; E7 = k["E7"]
	13	+chk("primitives: realized 100x390 speedup", P["realized_fused@100x390"]["speedup"], "{:.1f}$\\times$")
	14	+chk("primitives: realized max speedup", B["realized_fused"]["max_speedup"], "3.4$\\times$")
	15	+chk("primitives: regression 1x390000 speedup", P["regression_stats@1x390000"]["speedup"], "{:.1f}$\\times$")
	16	+chk("primitives: garch P64 max", B["garch_nll_P64"]["max_speedup"], "{:.1f}$\\times$")
	17	+chk("primitives: garch P64 100x390", P["garch_nll_P64@100x390"]["speedup"], "{:.1f}$\\times$")
	18	+chk("E1: cpuMT 926 day ms", E1["real_S926_day"]["cpuMT_ms"], "29--30\\,ms")
	19	+chk("E1: speedup low", min(E1[f"real_S926_{s}"]["speedup_MT"] for s in ["day", "week", "month", "year"]), "{:.2f}--")
	20	+chk("E1: speedup high", max(E1[f"real_S926_{s}"]["speedup_MT"] for s in ["day", "week", "month", "year"]), "--{:.2f}$\\times$")
	21	+chk("E1: max rel err daily", E1["real_S926_day"]["max_rel_err"], "$1.6\\times10^{{-7}}$")
	22	+chk("E1: max rel err yearly", E1["real_S926_year"]["max_rel_err"], "$1.4\\times10^{{-8}}$")
	23	+chk("E2: HAR qlike h1", E2["accuracy"]["h1"]["har"], "{:.3f}")
	24	+chk("E2: RW qlike h22", E2["accuracy"]["h22"]["rw"], "{:.3f}")
	25	+chk("E2: HAR beats RW h1", E2["accuracy"]["h1"]["har_beats_rw_pct"], "{:.0f}\\,\\%")
	26	+chk("E2: log-HAR h22", E2["accuracy"]["h22"]["loghar"], "{:.3f}")
	27	+chk("E3: symbols", E3["symbols"], "{} symbols")
	28	+chk("E3: EGARCH convergence", E3["egarch.n"]["converged_pct"], "{:.1f}\\,\\%")
	29	+chk("E3: GJR convergence", E3["gjr.n"]["converged_pct"], "{:.1f}\\,\\%")
	30	+chk("E3: grid metal ms", E3["grid"]["metal_ms"], "{:.0f}\\,ms on Metal")
	31	+chk("E3: grid cpu ms", E3["grid"]["cpuMT_ms"], "{:.0f}\\,ms on 18 CPU threads")
	32	+chk("E3: grid speedup", E3["grid"]["cpuMT_ms"] / E3["grid"]["metal_ms"], "({:.1f}$\\times$)")
	33	+chk("E4: cond N=30", E4["30"]["cond_median"], "{:.0f} ($N = 30$)")
	34	+chk("E4: dy ms N=100", E4["100"]["dy_ms"], "and {:.0f}\\,ms")
	35	+chk("E5: beta bias n500", E5["garch_n500"]["beta"]["bias"], "({:+.3f})".replace("+", ""))
	36	+chk("E5: repro metal vs cpu", E5["repro_realized"]["metal_vs_cpu_max_rel_rv"], "$3.7\\times10^{{-8}}$")
	37	+chk("E6: RAID cpu s", E6["RAID cpu"]["median_s"], "{:.3f}\\,s")
	38	+chk("E6: polars s", E6["polars"]["median_s"], "{:.3f}\\,s")
	39	+chk("E6: duckdb s", E6["duckdb"]["median_s"], "{:.3f}\\,s")
	40	+chk("E6: pandas s", E6["pandas"]["median_s"], "{:.2f}\\,s")
	41	+chk("E6: read s", E6["read_s"], "{:.3f}\\,s")
	42	+chk("E7: M2 RV speedup", E7["m2m8a"]["realized_fused@1000x3900_speedup_MT"], "{:.1f}$\\times$ (M2")
	43	+chk("E7: M2 Max GARCH", E7["M2M32c"]["garch_nll_P64@1000x390_speedup_MT"], "{:.1f}$\\times$ (M2~Max)")
	44	+chk("E7: M5 Pro GARCH", E7["m5ma"]["garch_nll_P64@1000x390_speedup_MT"], "{:.1f}$\\times$ (M5~Pro)")
	45	+bad = [c for c in checks if not c[2]]
	46	+for c in checks: print(("OK " if c[2] else "MISS"), c[0], "->", c[1])
	47	+print(f"{len(checks) - len(bad)}/{len(checks)} claims found verbatim in the prose")
	48	+sys.exit(1 if bad else 0)

added src/data/bar_series.cpp +97 −0

		`@@ -0,0 +1,97 @@`
	1	+#include "raid/data/bar_series.hpp"
	2	+#include "raid/core/error.hpp"
	3	+#include <algorithm>
	4	+#include <numeric>
	5	+#include <cmath>
	6	+
	7	+namespace raid {
	8	+
	9	+nlohmann::json QualityReport::to_json() const {
	10	+ return {{"symbol", symbol}, {"n_bars", n_bars}, {"n_sessions", n_sessions}, {"ohlc_inconsistent", ohlc_inconsistent},
	11	+ {"nonpositive_price", nonpositive_price}, {"negative_volume", negative_volume}, {"zero_volume", zero_volume},
	12	+ {"duplicates", duplicates}, {"off_interval", off_interval}, {"outside_session", outside_session},
	13	+ {"non_trading_day", non_trading_day}, {"unsorted", unsorted}, {"missing_regular_bars", missing_regular_bars},
	14	+ {"price_jumps", price_jumps}, {"coverage", coverage}, {"per_session_missing", per_session_missing}, {"notes", notes}, {"has_errors", has_errors()}};
	15	+}
	16	+
	17	+BarSeries::NormalizeStats BarSeries::normalize() {
	18	+ const usize n = size();
	19	+ usize reordered = 0; bool dup = false;
	20	+ for (usize i = 1; i < n; ++i) { if (t_open[i] < t_open[i - 1]) ++reordered; else if (t_open[i] == t_open[i - 1]) dup = true; }
	21	+ if (reordered == 0 && !dup) { index_sessions(); return {0, 0}; } // fast path: no copy
	22	+ std::vector<usize> idx(n); std::iota(idx.begin(), idx.end(), 0);
	23	+ if (reordered) std::stable_sort(idx.begin(), idx.end(), [&](usize a, usize b) { return t_open[a] < t_open[b]; });
	24	+ BarSeries out; out.symbol = symbol; out.instrument_id = instrument_id; out.interval = interval; out.adjustment = adjustment; out.reserve(n);
	25	+ usize dups = 0;
	26	+ for (usize k = 0; k < n; ++k) {
	27	+ const usize i = idx[k];
	28	+ if (!out.empty() && out.t_open.back() == t_open[i]) { ++dups; continue; }
	29	+ out.t_open.push_back(t_open[i]); out.session.push_back(session[i]); out.open.push_back(open[i]); out.high.push_back(high[i]);
	30	+ out.low.push_back(low[i]); out.close.push_back(close[i]); out.volume.push_back(volume[i]); out.flags.push_back(flags[i]);
	31	+ }
	32	+ *this = std::move(out);
	33	+ index_sessions();
	34	+ return {dups, reordered};
	35	+}
	36	+
	37	+void BarSeries::index_sessions() {
	38	+ session_offsets.clear(); session_dates.clear();
	39	+ for (usize i = 0; i < size(); ++i) {
	40	+ if (i == 0 \|\| session[i] != session[i - 1]) { session_offsets.push_back(static_cast<i64>(i)); session_dates.push_back(session[i]); }
	41	+ }
	42	+ session_offsets.push_back(static_cast<i64>(size()));
	43	+ if (size() == 0) session_offsets = {0};
	44	+}
	45	+
	46	+QualityReport BarSeries::validate(const MarketCalendar& cal, SessionPolicy policy, f64 jump_threshold) const {
	47	+ QualityReport r; r.symbol = symbol; r.n_bars = size(); r.n_sessions = n_sessions();
	48	+ const i64 step = to_duration(interval).us;
	49	+ for (usize i = 0; i < size(); ++i) {
	50	+ if (high[i] < low[i] \|\| high[i] < std::max(open[i], close[i]) \|\| low[i] > std::min(open[i], close[i])) ++r.ohlc_inconsistent;
	51	+ if (!(open[i] > 0 && high[i] > 0 && low[i] > 0 && close[i] > 0)) ++r.nonpositive_price;
	52	+ if (volume[i] < 0) ++r.negative_volume;
	53	+ if (volume[i] == 0) ++r.zero_volume;
	54	+ if (i > 0 && t_open[i] <= t_open[i - 1]) ++r.unsorted;
	55	+ if (interval != Interval::d1 && (t_open[i].us % step) != 0) ++r.off_interval;
	56	+ }
	57	+ usize expected_total = 0, present_total = 0;
	58	+ for (usize k = 0; k < n_sessions(); ++k) {
	59	+ const Date d = session_dates[k];
	60	+ if (!cal.is_trading_day(d)) { r.non_trading_day += static_cast<usize>(session_offsets[k + 1] - session_offsets[k]); continue; }
	61	+ const auto sb = cal.session(d);
	62	+ usize present = 0;
	63	+ f64 prev_close = -1;
	64	+ for (i64 i = session_offsets[k]; i < session_offsets[k + 1]; ++i) {
	65	+ const auto t = t_open[static_cast<usize>(i)];
	66	+ const bool regular = t >= sb.open_utc && t <= sb.close_utc;
	67	+ const bool extended = t >= sb.ext_open_utc && t < sb.ext_close_utc;
	68	+ if (policy == SessionPolicy::regular ? !regular : !extended) ++r.outside_session;
	69	+ if (regular) ++present;
	70	+ if (prev_close > 0 && std::abs(close[static_cast<usize>(i)] / prev_close - 1.0) > jump_threshold) ++r.price_jumps;
	71	+ prev_close = close[static_cast<usize>(i)];
	72	+ }
	73	+ const usize expected = cal.expected_regular_bars(d, interval);
	74	+ expected_total += expected; present_total += std::min(present, expected);
	75	+ if (present < expected) { r.missing_regular_bars += expected - present; r.per_session_missing[d.iso()] = expected - present; }
	76	+ }
	77	+ r.coverage = expected_total ? static_cast<f64>(present_total) / static_cast<f64>(expected_total) : 1.0;
	78	+ return r;
	79	+}
	80	+
	81	+BarSeries BarSeries::filter_session(const MarketCalendar& cal, SessionPolicy policy) const {
	82	+ BarSeries out; out.symbol = symbol; out.instrument_id = instrument_id; out.interval = interval; out.adjustment = adjustment; out.reserve(size());
	83	+ for (usize k = 0; k < n_sessions(); ++k) {
	84	+ const Date d = session_dates[k];
	85	+ if (!cal.is_trading_day(d)) continue;
	86	+ const auto sb = cal.session(d);
	87	+ for (i64 i = session_offsets[k]; i < session_offsets[k + 1]; ++i) {
	88	+ const usize j = static_cast<usize>(i); const auto t = t_open[j];
	89	+ const bool keep = policy == SessionPolicy::regular ? (t >= sb.open_utc && t <= sb.close_utc) : (t >= sb.ext_open_utc && t < sb.ext_close_utc);
	90	+ if (keep) { out.t_open.push_back(t); out.session.push_back(session[j]); out.open.push_back(open[j]); out.high.push_back(high[j]); out.low.push_back(low[j]); out.close.push_back(close[j]); out.volume.push_back(volume[j]); out.flags.push_back(flags[j]); }
	91	+ }
	92	+ }
	93	+ out.index_sessions();
	94	+ return out;
	95	+}
	96	+
	97	+} // namespace raid

added src/data/calendar.cpp +162 −0

		`@@ -0,0 +1,162 @@`
	1	+#include "raid/data/calendar.hpp"
	2	+#include "raid/core/error.hpp"
	3	+#include <algorithm>
	4	+
	5	+namespace raid {
	6	+
	7	+namespace {
	8	+// nth (1-based) weekday (0=Sun..6=Sat) of a month; n = -1 → last.
	9	+Date nth_weekday(int y, unsigned m, int wd, int n) {
	10	+ if (n > 0) {
	11	+ Date first = Date::from_ymd(y, m, 1);
	12	+ int delta = (wd - first.weekday() + 7) % 7;
	13	+ return first + (delta + 7 * (n - 1));
	14	+ }
	15	+ // last
	16	+ const unsigned nm = m == 12 ? 1 : m + 1; const int ny = m == 12 ? y + 1 : y;
	17	+ Date last = Date::from_ymd(ny, nm, 1) - 1;
	18	+ int delta = (last.weekday() - wd + 7) % 7;
	19	+ return last - delta;
	20	+}
	21	+// Western Easter (Anonymous Gregorian algorithm)
	22	+Date easter(int y) {
	23	+ const int a = y % 19, b = y / 100, c = y % 100, d = b / 4, e = b % 4, f = (b + 8) / 25, g = (b - f + 1) / 3;
	24	+ const int h = (19 * a + b - d - g + 15) % 30, i = c / 4, k = c % 4, l = (32 + 2 * e + 2 * i - h - k) % 7;
	25	+ const int m = (a + 11 * h + 22 * l) / 451;
	26	+ const int month = (h + l - 7 * m + 114) / 31, day = ((h + l - 7 * m + 114) % 31) + 1;
	27	+ return Date::from_ymd(y, static_cast<unsigned>(month), static_cast<unsigned>(day));
	28	+}
	29	+// Observed date of a fixed holiday: Saturday → Friday, Sunday → Monday. NYSE rule: if Saturday, the preceding Friday is a
	30	+// holiday unless that Friday is the last trading day of the month/year (then no day off) — for New Year's Day on a
	31	+// Saturday the exchange does NOT close on Friday Dec 31.
	32	+std::optional<Date> observed(int y, unsigned m, unsigned d, bool is_new_year) {
	33	+ Date dt = Date::from_ymd(y, m, d);
	34	+ const int wd = dt.weekday();
	35	+ if (wd == 6) { if (is_new_year) return std::nullopt; return dt - 1; }
	36	+ if (wd == 0) return dt + 1;
	37	+ return dt;
	38	+}
	39	+struct Holiday { Date date; const char* name; };
	40	+std::vector<Holiday> holidays_for_year(int y) {
	41	+ std::vector<Holiday> h;
	42	+ if (auto d = observed(y, 1, 1, true)) h.push_back({*d, "New Year's Day"});
	43	+ h.push_back({nth_weekday(y, 1, 1, 3), "Martin Luther King Jr. Day"}); // since 1998
	44	+ h.push_back({nth_weekday(y, 2, 1, 3), "Presidents' Day"});
	45	+ h.push_back({easter(y) - 2, "Good Friday"});
	46	+ h.push_back({nth_weekday(y, 5, 1, -1), "Memorial Day"});
	47	+ if (y >= 2022) if (auto d = observed(y, 6, 19, false)) h.push_back({*d, "Juneteenth"});
	48	+ if (auto d = observed(y, 7, 4, false)) h.push_back({*d, "Independence Day"});
	49	+ h.push_back({nth_weekday(y, 9, 1, 1), "Labor Day"});
	50	+ h.push_back({nth_weekday(y, 11, 4, 4), "Thanksgiving Day"});
	51	+ if (auto d = observed(y, 12, 25, false)) h.push_back({*d, "Christmas Day"});
	52	+ // Special closures (NYSE), 2000–2026
	53	+ static const struct { int y; unsigned m, d; const char* n; } specials[] = {
	54	+ {2001, 9, 11, "September 11 attacks"}, {2001, 9, 12, "September 11 attacks"}, {2001, 9, 13, "September 11 attacks"}, {2001, 9, 14, "September 11 attacks"},
	55	+ {2004, 6, 11, "Reagan funeral"}, {2007, 1, 2, "Ford funeral"}, {2012, 10, 29, "Hurricane Sandy"}, {2012, 10, 30, "Hurricane Sandy"},
	56	+ {2018, 12, 5, "G.H.W. Bush funeral"}, {2025, 1, 9, "Carter funeral"},
	57	+ };
	58	+ for (const auto& s : specials) if (s.y == y) h.push_back({Date::from_ymd(s.y, s.m, s.d), s.n});
	59	+ return h;
	60	+}
	61	+} // namespace
	62	+
	63	+std::optional<std::string> MarketCalendar::holiday_name(Date d) const {
	64	+ const int wd = d.weekday();
	65	+ if (wd == 0) return "Sunday";
	66	+ if (wd == 6) return "Saturday";
	67	+ for (const auto& h : holidays_for_year(d.ymd().year)) if (h.date == d) return h.name;
	68	+ return std::nullopt;
	69	+}
	70	+
	71	+bool MarketCalendar::is_trading_day(Date d) const { return !holiday_name(d).has_value(); }
	72	+
	73	+bool MarketCalendar::is_early_close(Date d) const {
	74	+ if (!is_trading_day(d)) return false;
	75	+ const auto [y, m, dd] = d.ymd();
	76	+ // Historical one-offs (NYSE), verified against exchange_calendars 4.13 XNYS 2000–2026:
	77	+ if (d == Date::from_ymd(2002, 7, 5) \|\| d == Date::from_ymd(2003, 12, 26)) return true; // Friday after a Thursday holiday
	78	+ if (d == Date::from_ymd(2002, 7, 3)) return false; // full day in 2002
	79	+ // Day after Thanksgiving
	80	+ if (d == nth_weekday(y, 11, 4, 4) + 1) return true;
	81	+ // Christmas Eve when it is a weekday (Mon–Fri) and a trading day; since 1993 rule, 13:00 close
	82	+ if (m == 12 && dd == 24 && d.weekday() >= 1 && d.weekday() <= 5) return true;
	83	+ // July 3 when July 4 is a weekday (Tue–Fri) — 13:00 close (rule in force since 1995 roughly; NYSE: when the 4th is a weekday)
	84	+ if (m == 7 && dd == 3 && d.weekday() >= 1 && d.weekday() <= 5) {
	85	+ const int wd4 = Date::from_ymd(y, 7, 4).weekday();
	86	+ if (wd4 >= 2 && wd4 <= 5) return true;
	87	+ }
	88	+ return false;
	89	+}
	90	+
	91	+i64 MarketCalendar::new_york_offset(Timestamp utc) {
	92	+ // Determine year from UTC date (safe: transitions are in March/November, far from year boundaries)
	93	+ const i64 days = utc.us >= 0 ? utc.us / 86'400'000'000LL : -((-utc.us + 86'400'000'000LL - 1) / 86'400'000'000LL);
	94	+ const int y = Date(static_cast<i32>(days)).ymd().year;
	95	+ Date start, end;
	96	+ if (y >= 2007) { start = nth_weekday(y, 3, 0, 2); end = nth_weekday(y, 11, 0, 1); }
	97	+ else { start = nth_weekday(y, 4, 0, 1); end = nth_weekday(y, 10, 0, -1); }
	98	+ // DST starts at 02:00 EST = 07:00 UTC; ends at 02:00 EDT = 06:00 UTC
	99	+ const Timestamp dst_start(static_cast<i64>(start.days) * 86'400'000'000LL + 7LL * 3'600'000'000LL);
	100	+ const Timestamp dst_end(static_cast<i64>(end.days) * 86'400'000'000LL + 6LL * 3'600'000'000LL);
	101	+ return (utc >= dst_start && utc < dst_end) ? -14400 : -18000;
	102	+}
	103	+
	104	+MarketCalendar::Local MarketCalendar::to_new_york(Timestamp utc) {
	105	+ const i64 off = new_york_offset(utc);
	106	+ const i64 local_us = utc.us + off * 1'000'000LL;
	107	+ i64 days = local_us / 86'400'000'000LL, rem = local_us % 86'400'000'000LL;
	108	+ if (rem < 0) { rem += 86'400'000'000LL; days -= 1; }
	109	+ return {Date(static_cast<i32>(days)), static_cast<int>(rem / 60'000'000LL), off};
	110	+}
	111	+
	112	+Timestamp MarketCalendar::from_new_york(Date d, int minute_of_day) {
	113	+ // First guess with the standard offset, then correct with the offset valid at that instant.
	114	+ const i64 naive = static_cast<i64>(d.days) * 86'400'000'000LL + static_cast<i64>(minute_of_day) * 60'000'000LL;
	115	+ Timestamp guess(naive + 18000LL * 1'000'000LL);
	116	+ const i64 off = new_york_offset(guess);
	117	+ Timestamp t(naive - off * 1'000'000LL);
	118	+ // re-check (handles instants near the transition)
	119	+ const i64 off2 = new_york_offset(t);
	120	+ if (off2 != off) t = Timestamp(naive - off2 * 1'000'000LL);
	121	+ return t;
	122	+}
	123	+
	124	+SessionBounds MarketCalendar::session(Date d) const {
	125	+ RAID_REQUIRE(is_trading_day(d), invalid_argument, d.iso() + " is not a trading day");
	126	+ SessionBounds s; s.session = d; s.early_close = is_early_close(d);
	127	+ s.open_utc = from_new_york(d, 9 * 60 + 30);
	128	+ s.close_utc = from_new_york(d, s.early_close ? 13 * 60 : 16 * 60);
	129	+ s.ext_open_utc = from_new_york(d, 4 * 60);
	130	+ s.ext_close_utc = from_new_york(d, s.early_close ? 17 * 60 : 20 * 60);
	131	+ s.utc_offset_seconds = new_york_offset(s.open_utc);
	132	+ return s;
	133	+}
	134	+
	135	+usize MarketCalendar::expected_regular_bars(Date d, Interval iv) const {
	136	+ const auto s = session(d);
	137	+ const i64 minutes = (s.close_utc - s.open_utc).us / 60'000'000LL;
	138	+ switch (iv) {
	139	+ case Interval::m1: return static_cast<usize>(minutes) + 1; // 391 / 211, includes the closing print bar
	140	+ case Interval::m5: return static_cast<usize>(minutes / 5) + 1; // 79 / 43
	141	+ case Interval::m30: return static_cast<usize>(minutes / 30) + 1;
	142	+ case Interval::h1: return static_cast<usize>(minutes / 60) + 1;
	143	+ case Interval::d1: return 1;
	144	+ }
	145	+ return 0;
	146	+}
	147	+
	148	+std::vector<Date> MarketCalendar::sessions(Date start, Date end) const {
	149	+ std::vector<Date> out;
	150	+ for (Date d = start; d <= end; d = d + 1) if (is_trading_day(d)) out.push_back(d);
	151	+ return out;
	152	+}
	153	+Date MarketCalendar::next_session(Date d) const { Date x = d + 1; while (!is_trading_day(x)) x = x + 1; return x; }
	154	+Date MarketCalendar::previous_session(Date d) const { Date x = d - 1; while (!is_trading_day(x)) x = x - 1; return x; }
	155	+
	156	+usize trading_days_between(const MarketCalendar& cal, Date a, Date b) {
	157	+ if (b < a) std::swap(a, b);
	158	+ usize n = 0; for (Date d = a + 1; d <= b; d = d + 1) if (cal.is_trading_day(d)) ++n;
	159	+ return n;
	160	+}
	161	+
	162	+} // namespace raid

added src/data/findata.cpp +268 −0

		`@@ -0,0 +1,268 @@`
	1	+#include "raid/data/findata.hpp"
	2	+#include "raid/core/error.hpp"
	3	+#include "raid/core/log.hpp"
	4	+#include "raid/core/hash.hpp"
	5	+#include "raid/core/manifest.hpp"
	6	+#include "raid/data/io.hpp"
	7	+#include <curl/curl.h>
	8	+#include <fstream>
	9	+#include <iostream>
	10	+#include <sstream>
	11	+#include <thread>
	12	+#include <cstdlib>
	13	+#include <filesystem>
	14	+#include <algorithm>
	15	+#include <cmath>
	16	+
	17	+namespace raid::findata {
	18	+
	19	+namespace fs = std::filesystem;
	20	+using json = nlohmann::json;
	21	+
	22	+// ---------------------------------------------------------------- config / secrets
	23	+Config Config::from_env() {
	24	+ Config c;
	25	+ if (const char* u = std::getenv("FINDATA_URL")) { c.base_url = u; if (c.base_url.find("/api/v1") == std::string::npos) c.base_url += "/api/v1"; }
	26	+ if (const char* k = std::getenv("FINDATA_API_KEY")) c.api_key = k;
	27	+ else if (const char* f = std::getenv("FINDATA_API_KEY_FILE")) { std::ifstream in(f); std::getline(in, c.api_key); }
	28	+ if (const char* d = std::getenv("RAID_CACHE_DIR")) c.cache_dir = d;
	29	+ if (c.cache_dir.empty()) c.cache_dir = "datasets/cache";
	30	+ if (const char* o = std::getenv("RAID_OFFLINE")) c.offline = std::string(o) == "1";
	31	+ if (const char* r = std::getenv("RAID_FINDATA_RPS")) c.requests_per_second = std::atof(r);
	32	+ while (!c.api_key.empty() && (c.api_key.back() == '\n' \|\| c.api_key.back() == '\r' \|\| c.api_key.back() == ' ')) c.api_key.pop_back();
	33	+ return c;
	34	+}
	35	+
	36	+double TokenBucket::acquire() {
	37	+ std::lock_guard<std::mutex> lk(m_);
	38	+ const auto now = std::chrono::steady_clock::now();
	39	+ tokens_ = std::min(capacity_, tokens_ + rate_ * std::chrono::duration<double>(now - last_).count()); last_ = now;
	40	+ double waited = 0;
	41	+ if (tokens_ < 1.0) { waited = (1.0 - tokens_) / rate_; std::this_thread::sleep_for(std::chrono::duration<double>(waited)); tokens_ = 1.0; last_ = std::chrono::steady_clock::now(); }
	42	+ tokens_ -= 1.0;
	43	+ return waited;
	44	+}
	45	+
	46	+std::vector<std::pair<Date, Date>> chunk_dates(Date start, Date end, int max_days) {
	47	+ RAID_REQUIRE(max_days >= 1, invalid_argument, "max_days >= 1");
	48	+ RAID_REQUIRE(end >= start, invalid_argument, "end must be >= start");
	49	+ std::vector<std::pair<Date, Date>> out;
	50	+ for (Date a = start; a <= end;) { Date b = a + (max_days - 1); if (b > end) b = end; out.emplace_back(a, b); a = b + 1; }
	51	+ return out;
	52	+}
	53	+int max_days_per_call(Interval iv) { switch (iv) { case Interval::m1: return 31; case Interval::m5: return 186; case Interval::d1: return 3650; default: return 31; } }
	54	+
	55	+// ---------------------------------------------------------------- HTTP
	56	+namespace {
	57	+size_t write_cb(char* p, size_t s, size_t n, void* ud) { static_cast<std::string>(ud)->append(p, s n); return s * n; }
	58	+size_t file_cb(char* p, size_t s, size_t n, void* ud) { static_cast<std::ofstream>(ud)->write(p, static_cast<std::streamsize>(s n)); return s * n; }
	59	+size_t header_cb(char* p, size_t s, size_t n, void* ud) {
	60	+ std::string line(p, s * n); auto* h = static_cast<std::map<std::string, std::string>*>(ud);
	61	+ const auto c = line.find(':'); if (c != std::string::npos) { std::string k = line.substr(0, c), v = line.substr(c + 1); std::transform(k.begin(), k.end(), k.begin(), ::tolower);
	62	+ while (!v.empty() && (v.back() == '\r' \|\| v.back() == '\n' \|\| v.back() == ' ')) v.pop_back(); while (!v.empty() && v.front() == ' ') v.erase(v.begin()); (*h)[k] = v; }
	63	+ return s * n;
	64	+}
	65	+struct CurlGlobal { CurlGlobal() { curl_global_init(CURL_GLOBAL_DEFAULT); } ~CurlGlobal() { curl_global_cleanup(); } };
	66	+void ensure_curl() { static CurlGlobal g; }
	67	+std::string url_encode(const std::string& s) { std::string o; char buf[4]; for (unsigned char c : s) { if (isalnum(c) \|\| c == '-' \|\| c == '_' \|\| c == '.' \|\| c == '~' \|\| c == ',' ) o += static_cast<char>(c); else { std::snprintf(buf, sizeof buf, "%%%02X", c); o += buf; } } return o; }
	68	+} // namespace
	69	+
	70	+Client::Client(Config cfg) : cfg_(std::move(cfg)), bucket_(cfg_.requests_per_second, cfg_.burst) {
	71	+ ensure_curl(); key_ = cfg_.api_key; cfg_.api_key.clear(); // the key lives only in key_
	72	+ RAID_REQUIRE(cfg_.offline \|\| !key_.empty(), api, "FinData API key missing: set FINDATA_API_KEY or FINDATA_API_KEY_FILE");
	73	+ fs::create_directories(cfg_.cache_dir);
	74	+}
	75	+Client::~Client() = default;
	76	+std::string Client::key_fingerprint() const { return key_.empty() ? "none" : Sha256::of(key_).substr(0, 12); }
	77	+
	78	+Client::Response Client::request_once(const std::string& method, const std::string& url, const std::string& body, const std::string& content_type, const std::string& download_to) {
	79	+ Response r; CURL* c = curl_easy_init(); RAID_REQUIRE(c != nullptr, api, "curl init failed");
	80	+ struct curl_slist* hdrs = nullptr; hdrs = curl_slist_append(hdrs, ("Authorization: Bearer " + key_).c_str()); hdrs = curl_slist_append(hdrs, "User-Agent: raid/" RAID_VERSION " (UQO WP40; +simon-pierre.boucher@uqo.ca)");
	81	+ if (!body.empty()) hdrs = curl_slist_append(hdrs, ("Content-Type: " + content_type).c_str());
	82	+ std::ofstream out; if (!download_to.empty()) { out.open(download_to + ".part", std::ios::binary); RAID_REQUIRE(out.good(), io, "cannot write " + download_to); }
	83	+ curl_easy_setopt(c, CURLOPT_URL, url.c_str()); curl_easy_setopt(c, CURLOPT_HTTPHEADER, hdrs); curl_easy_setopt(c, CURLOPT_TIMEOUT, cfg_.timeout_seconds); curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, 20L);
	84	+ curl_easy_setopt(c, CURLOPT_SSL_VERIFYPEER, cfg_.verify_tls ? 1L : 0L); curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 0L); curl_easy_setopt(c, CURLOPT_ACCEPT_ENCODING, "");
	85	+ curl_easy_setopt(c, CURLOPT_HEADERFUNCTION, header_cb); curl_easy_setopt(c, CURLOPT_HEADERDATA, &r.headers);
	86	+ if (!download_to.empty()) { curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, file_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &out); } else { curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &r.body); }
	87	+ if (method == "POST") { curl_easy_setopt(c, CURLOPT_POST, 1L); curl_easy_setopt(c, CURLOPT_POSTFIELDS, body.c_str()); curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, static_cast<long>(body.size())); }
	88	+ else if (method != "GET") curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, method.c_str());
	89	+ const auto t0 = std::chrono::steady_clock::now(); const CURLcode rc = curl_easy_perform(c); r.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
	90	+ curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &r.status); curl_slist_free_all(hdrs); curl_easy_cleanup(c);
	91	+ if (!download_to.empty()) { out.close(); if (rc == CURLE_OK && r.status == 200) fs::rename(download_to + ".part", download_to); else fs::remove(download_to + ".part"); }
	92	+ if (rc != CURLE_OK) { r.status = -static_cast<long>(rc); r.body = curl_easy_strerror(rc); }
	93	+ return r;
	94	+}
	95	+
	96	+Client::Response Client::request(const std::string& method, const std::string& path, const std::string& body, const std::string& content_type, const std::string& download_to) {
	97	+ RAID_REQUIRE(!cfg_.offline, api, "offline mode: network call refused for " + method + " " + path);
	98	+ const std::string url = cfg_.base_url + path;
	99	+ Response r; double backoff = 1.0;
	100	+ for (int attempt = 1; attempt <= cfg_.max_retries + 1; ++attempt) {
	101	+ bucket_.acquire();
	102	+ r = request_once(method, url, body, content_type, download_to); r.attempts = attempt;
	103	+ Exchange ex; ex.method = method; ex.path = path; ex.status = r.status; ex.seconds = r.seconds; ex.bytes = r.body.size(); ex.attempts = attempt;
	104	+ if (auto it = r.headers.find("x-request-id"); it != r.headers.end()) ex.request_id = it->second;
	105	+ if (auto it = r.headers.find("x-dataset-id"); it != r.headers.end()) ex.dataset_id = it->second;
	106	+ if (auto it = r.headers.find("x-dataset-version"); it != r.headers.end()) ex.dataset_version = std::atoll(it->second.c_str());
	107	+ if (auto it = r.headers.find("x-next-cursor"); it != r.headers.end()) ex.next_cursor = it->second;
	108	+ { std::lock_guard<std::mutex> lk(log_m_); log_.push_back(ex); }
	109	+ const bool retry = r.status == 429 \|\| r.status == 502 \|\| r.status == 503 \|\| r.status == 504 \|\| r.status < 0;
	110	+ if (!retry) break;
	111	+ double wait = backoff; if (auto it = r.headers.find("retry-after"); it != r.headers.end()) wait = std::max(wait, std::atof(it->second.c_str()));
	112	+ log::warn("findata retry", {{"status", std::to_string(r.status)}, {"path", path}, {"attempt", std::to_string(attempt)}, {"wait_s", std::to_string(wait)}});
	113	+ if (attempt <= cfg_.max_retries) std::this_thread::sleep_for(std::chrono::duration<double>(wait));
	114	+ backoff = std::min(backoff * 2.0, 30.0);
	115	+ }
	116	+ if (r.status != 200 && r.status != 202) {
	117	+ std::string msg = r.body.substr(0, 400); try { auto j = json::parse(r.body); if (j.contains("error")) msg = j["error"].dump(); } catch (...) {}
	118	+ throw RaidError(ErrorCode::api, "FinData " + method + " " + path + " → HTTP " + std::to_string(r.status) + ": " + log::redact(msg));
	119	+ }
	120	+ return r;
	121	+}
	122	+
	123	+std::string Client::cache_file(const std::string& kind, const std::string& key_material, const std::string& ext) const {
	124	+ return (fs::path(cfg_.cache_dir) / (kind + "_" + Sha256::of(key_material).substr(0, 24) + ext)).string();
	125	+}
	126	+
	127	+// ---------------------------------------------------------------- simple JSON endpoints
	128	+nlohmann::json Client::health() { return json::parse(request("GET", "/health").body); }
	129	+nlohmann::json Client::openapi() { return json::parse(request("GET", "/openapi.json").body); }
	130	+nlohmann::json Client::catalog_datasets() { return json::parse(request("GET", "/catalog/datasets").body); }
	131	+nlohmann::json Client::dataset(const std::string& id) { return json::parse(request("GET", "/catalog/datasets/" + url_encode(id)).body); }
	132	+nlohmann::json Client::sessions(const std::string& symbol, Interval iv, Date start, Date end) {
	133	+ return json::parse(request("GET", "/market/sessions?symbol=" + url_encode(symbol) + "&interval=" + std::string(to_string(iv)) + "&start=" + start.iso() + "&end=" + end.iso()).body);
	134	+}
	135	+nlohmann::json Client::coverage_symbols(const std::string& dataset_id, int min_sessions, usize limit, const std::string& cursor) {
	136	+ std::string p = "/coverage/symbols?dataset_id=" + url_encode(dataset_id) + "&min_sessions=" + std::to_string(min_sessions) + "&limit=" + std::to_string(limit);
	137	+ if (!cursor.empty()) p += "&cursor=" + url_encode(cursor);
	138	+ return json::parse(request("GET", p).body);
	139	+}
	140	+nlohmann::json Client::coverage_symbols_all(const std::string& dataset_id, int min_sessions, usize max_symbols) {
	141	+ json out = json::array(); std::string cursor;
	142	+ for (;;) { auto page = coverage_symbols(dataset_id, min_sessions, 1000, cursor); for (auto& r : page["data"]) out.push_back(r); if (out.size() >= max_symbols \|\| page["next_cursor"].is_null() \|\| page["data"].empty()) break; cursor = page["next_cursor"].get<std::string>(); }
	143	+ return out;
	144	+}
	145	+nlohmann::json Client::fundamentals(const std::string& entity, const std::string& as_of_iso, const std::vector<std::string>& metrics, const std::string& period_type) {
	146	+ std::string p = "/fundamentals/" + url_encode(entity) + "?as_of=" + url_encode(as_of_iso);
	147	+ if (!metrics.empty()) { std::string m; for (auto& x : metrics) { if (!m.empty()) m += ","; m += x; } p += "&metrics=" + url_encode(m); }
	148	+ if (!period_type.empty()) p += "&period_type=" + period_type;
	149	+ return json::parse(request("GET", p).body);
	150	+}
	151	+nlohmann::json Client::query(const std::string& sql, usize max_rows) {
	152	+ RAID_REQUIRE(max_rows <= 100000, invalid_argument, "query: max_rows <= 100000");
	153	+ return json::parse(request("POST", "/query", json({{"sql", sql}, {"max_rows", max_rows}}).dump()).body);
	154	+}
	155	+
	156	+// ---------------------------------------------------------------- bars (arrow pages)
	157	+BarsResult Client::bars(const BarsRequest& req) {
	158	+ BarsResult res; res.bars.symbol = req.symbol; res.bars.interval = req.interval; res.bars.adjustment = req.adjustment;
	159	+ const std::string key = "bars\|" + req.symbol + "\|" + std::string(to_string(req.interval)) + "\|" + req.start.iso() + "\|" + req.end.iso() + "\|" + req.adjustment + "\|v" + (req.version ? std::to_string(*req.version) : "latest");
	160	+ const std::string meta_path = cache_file("bars", key, ".meta.json"), data_path = cache_file("bars", key, ".csv");
	161	+ if (fs::exists(meta_path) && fs::exists(data_path)) {
	162	+ auto meta = json::parse(std::ifstream(meta_path)); if (Sha256::of_file(data_path) == meta["sha256"].get<std::string>()) {
	163	+ auto m = data::read_bars_csv(data_path, req.interval); if (!m.empty()) res.bars = m.begin()->second; res.bars.symbol = req.symbol; res.bars.instrument_id = meta.value("instrument_id", ""); res.bars.adjustment = req.adjustment;
	164	+ res.dataset_version = meta["dataset_version"].get<i64>(); res.dataset_id = meta["dataset_id"].get<std::string>(); res.from_cache = true; res.cache_path = data_path; return res; }
	165	+ }
	166	+#ifndef RAID_WITH_ARROW
	167	+ throw RaidError(ErrorCode::unsupported, "bars(): Arrow build required to decode pages (RAID_WITH_ARROW)");
	168	+#else
	169	+ const usize log_before = log_.size();
	170	+ for (auto [a, b] : chunk_dates(req.start, req.end, max_days_per_call(req.interval))) {
	171	+ std::string cursor;
	172	+ for (;;) {
	173	+ std::string p = "/market/bars?symbol=" + url_encode(req.symbol) + "&interval=" + std::string(to_string(req.interval)) + "&start=" + a.iso() + "&end=" + b.iso() + "&adjustment=" + req.adjustment + "&format=arrow&limit=" + std::to_string(req.page_limit);
	174	+ if (req.version) p += "&version=" + std::to_string(*req.version); if (!cursor.empty()) p += "&cursor=" + url_encode(cursor);
	175	+ auto r = request("GET", p);
	176	+ const std::string tmp = cache_file("page", key + p + std::to_string(log_.size()), ".arrow"); { std::ofstream f(tmp, std::ios::binary); f.write(r.body.data(), static_cast<std::streamsize>(r.body.size())); }
	177	+ auto pb = data::read_bars_arrow_ipc(tmp, req.interval); fs::remove(tmp);
	178	+ for (auto& [sym, bs] : pb.series) { if (res.bars.instrument_id.empty()) res.bars.instrument_id = bs.instrument_id; for (usize i = 0; i < bs.size(); ++i) res.bars.push_back(bs.t_open[i], bs.open[i], bs.high[i], bs.low[i], bs.close[i], bs.volume[i], bs.flags[i]); }
	179	+ if (auto it = r.headers.find("x-dataset-version"); it != r.headers.end()) res.dataset_version = std::atoll(it->second.c_str());
	180	+ if (auto it = r.headers.find("x-dataset-id"); it != r.headers.end()) res.dataset_id = it->second;
	181	+ auto nc = r.headers.find("x-next-cursor"); if (nc == r.headers.end() \|\| nc->second.empty() \|\| pb.rows_read == 0) break; cursor = nc->second;
	182	+ }
	183	+ }
	184	+ res.exchanges.assign(log_.begin() + static_cast<long>(log_before), log_.end());
	185	+ auto st = res.bars.normalize(); res.duplicates_dropped = st.duplicates_removed;
	186	+ data::write_bars_csv(data_path, res.bars);
	187	+ json meta = {{"key", key}, {"sha256", Sha256::of_file(data_path)}, {"dataset_version", res.dataset_version}, {"dataset_id", res.dataset_id}, {"instrument_id", res.bars.instrument_id}, {"rows", res.bars.size()}, {"duplicates_dropped", res.duplicates_dropped}, {"fetched_at", now_utc_iso()}, {"key_fingerprint", key_fingerprint()}, {"request_ids", json::array()}};
	188	+ for (auto& e : res.exchanges) meta["request_ids"].push_back(e.request_id);
	189	+ std::ofstream(meta_path) << meta.dump(1); res.cache_path = data_path;
	190	+ return res;
	191	+#endif
	192	+}
	193	+
	194	+// ---------------------------------------------------------------- exports
	195	+ExportResult Client::export_bars(const ExportRequest& req, const std::string& out_path) {
	196	+ ExportResult res; const auto t0 = std::chrono::steady_clock::now();
	197	+ RAID_REQUIRE(!req.symbols.empty(), invalid_argument, "export: symbols required");
	198	+ RAID_REQUIRE(req.end.days - req.start.days <= 12 * 366, invalid_argument, "export: at most 12 years per request (chunk the range)");
	199	+ std::string syms; for (auto& s : req.symbols) { if (!syms.empty()) syms += ","; syms += s; }
	200	+ const std::string key = "export\|" + req.dataset_id + "\|" + syms + "\|" + req.start.iso() + "\|" + req.end.iso() + "\|" + req.adjustment;
	201	+ const std::string meta_path = cache_file("export", key, ".meta.json"); std::string data_path = out_path.empty() ? cache_file("export", key, ".parquet") : out_path;
	202	+ if (fs::exists(meta_path) && fs::exists(data_path)) {
	203	+ auto meta = json::parse(std::ifstream(meta_path));
	204	+ if (Sha256::of_file(data_path) == meta["sha256"].get<std::string>()) { res.export_id = meta["export_id"]; res.path = data_path; res.sha256 = meta["sha256"]; res.dataset_version = meta["dataset_version"]; res.rows = meta["rows"]; res.bytes = meta["bytes"]; res.from_cache = true; return res; }
	205	+ }
	206	+ const usize log_before = log_.size();
	207	+ json body = {{"dataset_id", req.dataset_id}, {"symbols", req.symbols}, {"start", req.start.iso()}, {"end", req.end.iso()}, {"adjustment", req.adjustment}, {"format", "parquet"}};
	208	+ auto r = request("POST", "/exports", body.dump()); auto j = json::parse(r.body); res.export_id = j["id"].get<std::string>();
	209	+ json st; double wait = 1.0;
	210	+ for (int i = 0; i < 3600; ++i) {
	211	+ st = json::parse(request("GET", "/exports/" + res.export_id).body); const std::string s = st["status"].get<std::string>();
	212	+ if (s == "ready") break; RAID_REQUIRE(s != "failed" && s != "expired", api, "export " + s + ": " + st["error"].dump());
	213	+ std::this_thread::sleep_for(std::chrono::duration<double>(wait)); wait = std::min(wait * 1.5, 10.0);
	214	+ }
	215	+ RAID_REQUIRE(st["status"] == "ready", api, "export did not become ready");
	216	+ fs::create_directories(fs::path(data_path).parent_path().empty() ? "." : fs::path(data_path).parent_path());
	217	+ request("GET", "/exports/" + res.export_id + "/download", "", "application/json", data_path);
	218	+ res.path = data_path; res.sha256 = Sha256::of_file(data_path); res.rows = st.value("row_count", 0); res.bytes = st.value("bytes", 0);
	219	+ if (st.contains("snapshot") && st["snapshot"].contains("version")) res.dataset_version = st["snapshot"]["version"].get<i64>();
	220	+ res.exchanges.assign(log_.begin() + static_cast<long>(log_before), log_.end());
	221	+ RAID_REQUIRE(fs::file_size(data_path) == res.bytes \|\| res.bytes == 0, io, "export size mismatch: " + std::to_string(fs::file_size(data_path)) + " vs " + std::to_string(res.bytes));
	222	+ json meta = {{"key", key}, {"export_id", res.export_id}, {"sha256", res.sha256}, {"dataset_version", res.dataset_version}, {"dataset_id", req.dataset_id}, {"rows", res.rows}, {"bytes", res.bytes}, {"request", body}, {"status", st}, {"fetched_at", now_utc_iso()}, {"key_fingerprint", key_fingerprint()}, {"request_ids", json::array()}};
	223	+ for (auto& e : res.exchanges) meta["request_ids"].push_back(e.request_id);
	224	+ std::ofstream(meta_path) << meta.dump(1);
	225	+ res.seconds_total = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
	226	+ return res;
	227	+}
	228	+
	229	+std::map<std::string, BarSeries> Client::export_and_read(const ExportRequest& req) {
	230	+#ifdef RAID_WITH_ARROW
	231	+ auto e = export_bars(req); auto pb = data::read_bars_parquet(e.path); return std::move(pb.series);
	232	+#else
	233	+ (void)req; throw RaidError(ErrorCode::unsupported, "export_and_read requires the Arrow build");
	234	+#endif
	235	+}
	236	+
	237	+Provenance Client::provenance(const std::string& dataset_id) const {
	238	+ Provenance p; p.dataset_id = dataset_id; p.source = "FirstRate Data via FinData OS (" + cfg_.base_url + ")"; p.code_version = RAID_VERSION;
	239	+ for (auto& e : log_) { if (!e.request_id.empty()) p.request_ids.push_back(e.request_id); if (e.dataset_version > 0 && (dataset_id.empty() \|\| e.dataset_id == dataset_id)) p.dataset_version = e.dataset_version; }
	240	+ p.extra = {{"key_fingerprint", key_fingerprint()}, {"n_requests", log_.size()}};
	241	+ return p;
	242	+}
	243	+
	244	+// ---------------------------------------------------------------- CLI
	245	+namespace {
	246	+std::string arg(const std::vector<std::string>& a, const std::string& k, const std::string& d = "") { for (usize i = 0; i + 1 < a.size(); ++i) if (a[i] == k) return a[i + 1]; return d; }
	247	+}
	248	+int cli_main(const std::vector<std::string>& a) {
	249	+ if (a.empty()) { std::fprintf(stderr, "raid data catalog\|health\|sessions SYM --start --end\|coverage [--min-sessions N] [--dataset ID]\|bars SYM --start --end [--interval 1m] [--adjustment raw]\|export SYMS --start --end [--dataset ID] [--out file.parquet]\|fund ENTITY --as-of ISO [--metrics a,b]\|query \"SQL\" [--max-rows N]\|log\n"); return 2; }
	250	+ Client c; const std::string cmd = a[0];
	251	+ auto out = [](const json& j) { std::cout << j.dump(1) << "\n"; };
	252	+ if (cmd == "catalog") { out(c.catalog_datasets()); return 0; }
	253	+ if (cmd == "health") { out(c.health()); return 0; }
	254	+ if (cmd == "sessions") { out(c.sessions(a.at(1), interval_from_string(arg(a, "--interval", "1m")), date_from_iso(arg(a, "--start")), date_from_iso(arg(a, "--end")))); return 0; }
	255	+ if (cmd == "coverage") { out(c.coverage_symbols_all(arg(a, "--dataset", "market_bars.stock.1m.raw"), std::stoi(arg(a, "--min-sessions", "250")), static_cast<usize>(std::stoul(arg(a, "--max", "100000"))))); return 0; }
	256	+ if (cmd == "bars") { BarsRequest r; r.symbol = a.at(1); r.interval = interval_from_string(arg(a, "--interval", "1m")); r.start = date_from_iso(arg(a, "--start")); r.end = date_from_iso(arg(a, "--end")); r.adjustment = arg(a, "--adjustment", "raw");
	257	+ auto res = c.bars(r); json j = {{"symbol", r.symbol}, {"rows", res.bars.size()}, {"sessions", res.bars.n_sessions()}, {"dataset_id", res.dataset_id}, {"dataset_version", res.dataset_version}, {"duplicates_dropped", res.duplicates_dropped}, {"from_cache", res.from_cache}, {"cache_path", res.cache_path}, {"requests", res.exchanges.size()}};
	258	+ if (!arg(a, "--out").empty()) { data::write_bars_csv(arg(a, "--out"), res.bars); j["out"] = arg(a, "--out"); } out(j); return 0; }
	259	+ if (cmd == "export") { ExportRequest r; std::string s = a.at(1); usize p = 0; for (usize i = 0; i <= s.size(); ++i) if (i == s.size() \|\| s[i] == ',') { r.symbols.push_back(s.substr(p, i - p)); p = i + 1; }
	260	+ r.dataset_id = arg(a, "--dataset", "market_bars.stock.1m.raw"); r.start = date_from_iso(arg(a, "--start")); r.end = date_from_iso(arg(a, "--end")); r.adjustment = arg(a, "--adjustment", "raw");
	261	+ auto e = c.export_bars(r, arg(a, "--out")); out({{"export_id", e.export_id}, {"path", e.path}, {"sha256", e.sha256}, {"dataset_version", e.dataset_version}, {"rows", e.rows}, {"bytes", e.bytes}, {"from_cache", e.from_cache}, {"seconds", e.seconds_total}}); return 0; }
	262	+ if (cmd == "fund") { std::vector<std::string> m; std::string ms = arg(a, "--metrics"); usize p = 0; for (usize i = 0; i <= ms.size() && !ms.empty(); ++i) if (i == ms.size() \|\| ms[i] == ',') { m.push_back(ms.substr(p, i - p)); p = i + 1; } out(c.fundamentals(a.at(1), arg(a, "--as-of", "2026-01-01T00:00:00Z"), m, arg(a, "--period", ""))); return 0; }
	263	+ if (cmd == "query") { out(c.query(a.at(1), static_cast<usize>(std::stoul(arg(a, "--max-rows", "1000"))))); return 0; }
	264	+ if (cmd == "log") { json j = json::array(); for (auto& e : c.log()) j.push_back({{"method", e.method}, {"path", e.path}, {"status", e.status}, {"request_id", e.request_id}}); out(j); return 0; }
	265	+ std::fprintf(stderr, "unknown data subcommand %s\n", cmd.c_str()); return 2;
	266	+}
	267	+
	268	+} // namespace raid::findata

added src/data/io_csv.cpp +34 −0

		`@@ -0,0 +1,34 @@`
	1	+#include "raid/data/io.hpp"
	2	+#include "raid/core/error.hpp"
	3	+#include <fstream>
	4	+#include <sstream>
	5	+#include <charconv>
	6	+
	7	+namespace raid::data {
	8	+
	9	+std::map<std::string, BarSeries> read_bars_csv(const std::string& path, Interval iv) {
	10	+ std::ifstream f(path); RAID_REQUIRE(f.good(), io, "cannot open " + path);
	11	+ std::string line; std::getline(f, line);
	12	+ RAID_REQUIRE(line.rfind("symbol,timestamp_utc,open,high,low,close,volume", 0) == 0, io, "unexpected CSV header in " + path + ": " + line);
	13	+ std::map<std::string, BarSeries> out;
	14	+ usize ln = 1;
	15	+ while (std::getline(f, line)) {
	16	+ ++ln; if (line.empty()) continue;
	17	+ std::string cells[7]; usize c = 0, start = 0;
	18	+ for (usize i = 0; i <= line.size() && c < 7; ++i) if (i == line.size() \|\| line[i] == ',') { cells[c++] = line.substr(start, i - start); start = i + 1; }
	19	+ RAID_REQUIRE(c == 7, io, "bad CSV row " + std::to_string(ln) + " in " + path);
	20	+ auto num = [&](const std::string& s) { f64 v; const auto r = std::from_chars(s.data(), s.data() + s.size(), v); RAID_REQUIRE(r.ec == std::errc{}, io, "bad number '" + s + "' row " + std::to_string(ln)); return v; };
	21	+ auto& bs = out[cells[0]]; if (bs.symbol.empty()) { bs.symbol = cells[0]; bs.instrument_id = "synthetic:" + cells[0]; bs.interval = iv; bs.adjustment = "synthetic"; }
	22	+ bs.push_back(timestamp_from_iso(cells[1]), num(cells[2]), num(cells[3]), num(cells[4]), num(cells[5]), num(cells[6]));
	23	+ }
	24	+ for (auto& [k, bs] : out) bs.index_sessions();
	25	+ return out;
	26	+}
	27	+
	28	+void write_bars_csv(const std::string& path, const BarSeries& b) {
	29	+ std::ofstream f(path); RAID_REQUIRE(f.good(), io, "cannot write " + path);
	30	+ f << "symbol,timestamp_utc,open,high,low,close,volume\n"; f.precision(10);
	31	+ for (usize i = 0; i < b.size(); ++i) f << b.symbol << ',' << timestamp_iso(b.t_open[i]) << ',' << b.open[i] << ',' << b.high[i] << ',' << b.low[i] << ',' << b.close[i] << ',' << b.volume[i] << '\n';
	32	+}
	33	+
	34	+} // namespace raid::data

added src/data/parquet_arrow.cpp +69 −0

		`@@ -0,0 +1,69 @@`
	1	+// Arrow/Parquet reader for FinData bars (schema v1). Column chunks are read with the Arrow C++ API; numeric columns
	2	+// are copied once into the BarSeries vectors (a true zero-copy view is only possible for a single contiguous chunk of
	3	+// float64 without nulls — the reader records whether that was the case in `metadata["zero_copy_possible"]`).
	4	+#include "raid/data/io.hpp"
	5	+#include "raid/core/error.hpp"
	6	+#include <arrow/api.h>
	7	+#include <arrow/io/api.h>
	8	+#include <arrow/ipc/api.h>
	9	+#include <parquet/arrow/reader.h>
	10	+#include <unordered_set>
	11	+
	12	+namespace raid::data {
	13	+
	14	+namespace {
	15	+ParquetBars table_to_bars(const std::shared_ptr<arrow::Table>& table, const std::vector<std::string>& symbols, Interval iv) {
	16	+ ParquetBars out;
	17	+ if (table->schema()->metadata()) for (int i = 0; i < table->schema()->metadata()->size(); ++i) out.metadata[table->schema()->metadata()->key(i)] = table->schema()->metadata()->value(i);
	18	+ auto col = [&](const char* name) { auto c = table->GetColumnByName(name); RAID_REQUIRE(c != nullptr, io, std::string("missing column ") + name); return c; };
	19	+ auto ts = col("timestamp_utc"); auto sym = col("symbol_as_reported"); auto iid = table->GetColumnByName("instrument_id"); // may be absent (projected read)
	20	+ auto o = col("open"), h = col("high"), l = col("low"), c = col("close"), v = col("volume");
	21	+ std::unordered_set<std::string> keep(symbols.begin(), symbols.end());
	22	+ bool contiguous = (c->num_chunks() == 1) && (c->null_count() == 0);
	23	+ out.metadata["zero_copy_possible"] = contiguous ? "true" : "false";
	24	+ // Fast path: symbols are stored in runs (one symbol = one contiguous block in FinData exports); avoid a map lookup per row.
	25	+ BarSeries* cur = nullptr; std::string cur_sym;
	26	+ for (int ch = 0; ch < ts->num_chunks(); ++ch) {
	27	+ auto ta = std::static_pointer_cast<arrow::TimestampArray>(ts->chunk(ch));
	28	+ auto sa = std::static_pointer_cast<arrow::StringArray>(sym->chunk(ch));
	29	+ std::shared_ptr<arrow::StringArray> ia = iid ? std::static_pointer_cast<arrow::StringArray>(iid->chunk(ch)) : nullptr;
	30	+ auto oa = std::static_pointer_cast<arrow::DoubleArray>(o->chunk(ch)), ha = std::static_pointer_cast<arrow::DoubleArray>(h->chunk(ch));
	31	+ auto la = std::static_pointer_cast<arrow::DoubleArray>(l->chunk(ch)), ca = std::static_pointer_cast<arrow::DoubleArray>(c->chunk(ch));
	32	+ auto va = std::static_pointer_cast<arrow::DoubleArray>(v->chunk(ch));
	33	+ const auto unit = std::static_pointer_cast<arrow::TimestampType>(ta->type())->unit();
	34	+ i64 mult = 1; if (unit == arrow::TimeUnit::SECOND) mult = 1'000'000; else if (unit == arrow::TimeUnit::MILLI) mult = 1000; else if (unit == arrow::TimeUnit::NANO) mult = -1000;
	35	+ for (int64_t i = 0; i < ta->length(); ++i) {
	36	+ const std::string_view sv = sa->GetView(i);
	37	+ if (!cur \|\| sv != cur_sym) { cur_sym = std::string(sv); if (!keep.empty() && !keep.count(cur_sym)) { cur = nullptr; continue; } cur = &out.series[cur_sym]; if (cur->symbol.empty()) { cur->symbol = cur_sym; cur->instrument_id = ia ? ia->GetString(i) : ("firstrate:" + std::string(std::static_pointer_cast<arrow::StringArray>(col("asset_class")->chunk(ch))->GetView(i)) + ":" + cur_sym); cur->interval = iv; cur->adjustment = "raw"; cur->reserve(4096); } }
	38	+ else if (!cur) continue;
	39	+ const i64 raw = ta->Value(i); const i64 us = mult > 0 ? raw * mult : raw / 1000;
	40	+ cur->push_back(Timestamp(us), oa->Value(i), ha->Value(i), la->Value(i), ca->Value(i), va->Value(i));
	41	+ ++out.rows_read;
	42	+ }
	43	+ }
	44	+ for (auto& [k, bs] : out.series) { auto st = bs.normalize(); out.duplicates_dropped += st.duplicates_removed; }
	45	+ return out;
	46	+}
	47	+} // namespace
	48	+
	49	+ParquetBars read_bars_parquet(const std::string& path, const std::vector<std::string>& symbols, Interval iv) {
	50	+ auto infile = arrow::io::ReadableFile::Open(path); RAID_REQUIRE(infile.ok(), io, "cannot open " + path + ": " + infile.status().ToString());
	51	+ auto reader_r = parquet::arrow::OpenFile(*infile, arrow::default_memory_pool()); RAID_REQUIRE(reader_r.ok(), io, "parquet open: " + reader_r.status().ToString());
	52	+ std::unique_ptr<parquet::arrow::FileReader> reader = std::move(*reader_r);
	53	+ // Column projection: only the 8 columns the estimators need (of 20) are decoded — the key I/O optimisation of RQ3.
	54	+ std::shared_ptr<arrow::Schema> schema; auto sst = reader->GetSchema(&schema); RAID_REQUIRE(sst.ok(), io, "parquet schema: " + sst.ToString());
	55	+ // (instrument_id is derived from symbol_as_reported for FinData exports: "firstrate:<class>:<SYMBOL>"; reading the string column costs more than every numeric column together)
	56	+ std::vector<int> idx; for (const char* name : {"symbol_as_reported", "asset_class", "timestamp_utc", "open", "high", "low", "close", "volume"}) { const int k = schema->GetFieldIndex(name); RAID_REQUIRE(k >= 0, io, std::string("missing column ") + name); idx.push_back(k); }
	57	+ reader->set_use_threads(true);
	58	+ std::shared_ptr<arrow::Table> table; auto st = reader->ReadTable(idx, &table); RAID_REQUIRE(st.ok(), io, "parquet read: " + st.ToString());
	59	+ return table_to_bars(table, symbols, iv);
	60	+}
	61	+
	62	+ParquetBars read_bars_arrow_ipc(const std::string& path, Interval iv) {
	63	+ auto infile = arrow::io::ReadableFile::Open(path); RAID_REQUIRE(infile.ok(), io, "cannot open " + path);
	64	+ auto reader = arrow::ipc::RecordBatchStreamReader::Open(*infile); RAID_REQUIRE(reader.ok(), io, "arrow ipc open: " + reader.status().ToString());
	65	+ auto table = (*reader)->ToTable(); RAID_REQUIRE(table.ok(), io, "arrow ipc read: " + table.status().ToString());
	66	+ return table_to_bars(*table, {}, iv);
	67	+}
	68	+
	69	+} // namespace raid::data

added src/data/returns.cpp +79 −0

		`@@ -0,0 +1,79 @@`
	1	+#include "raid/data/returns.hpp"
	2	+#include "raid/data/resample.hpp"
	3	+#include "raid/core/error.hpp"
	4	+#include <cmath>
	5	+#include <limits>
	6	+
	7	+namespace raid::data {
	8	+
	9	+std::vector<f64> returns(std::span<const f64> p, ReturnKind kind) {
	10	+ std::vector<f64> r; if (p.size() < 2) return r; r.resize(p.size() - 1);
	11	+ for (usize i = 1; i < p.size(); ++i) {
	12	+ RAID_REQUIRE(p[i] > 0 && p[i - 1] > 0, data_quality, "returns: non-positive price");
	13	+ r[i - 1] = kind == ReturnKind::log ? std::log(p[i] / p[i - 1]) : p[i] / p[i - 1] - 1.0;
	14	+ }
	15	+ return r;
	16	+}
	17	+
	18	+IntradayReturns intraday_returns(const BarSeries& b, ReturnKind kind, bool use_open_first) {
	19	+ IntradayReturns out; const usize S = b.n_sessions();
	20	+ out.session_offsets.push_back(0); out.session_dates = b.session_dates;
	21	+ const f64 nan = std::numeric_limits<f64>::quiet_NaN();
	22	+ f64 prev_close = nan;
	23	+ auto ret = [&](f64 a, f64 c) { RAID_REQUIRE(a > 0 && c > 0, data_quality, "returns: non-positive price"); return kind == ReturnKind::log ? std::log(c / a) : c / a - 1.0; };
	24	+ for (usize k = 0; k < S; ++k) {
	25	+ const usize b0 = static_cast<usize>(b.session_offsets[k]), b1 = static_cast<usize>(b.session_offsets[k + 1]);
	26	+ if (use_open_first) out.r.push_back(ret(b.open[b0], b.close[b0]));
	27	+ for (usize i = b0 + 1; i < b1; ++i) out.r.push_back(ret(b.close[i - 1], b.close[i]));
	28	+ out.session_offsets.push_back(static_cast<i64>(out.r.size()));
	29	+ out.overnight.push_back(std::isnan(prev_close) ? nan : ret(prev_close, b.open[b0]));
	30	+ out.open_to_close.push_back(ret(b.open[b0], b.close[b1 - 1]));
	31	+ out.close_to_close.push_back(std::isnan(prev_close) ? nan : ret(prev_close, b.close[b1 - 1]));
	32	+ prev_close = b.close[b1 - 1];
	33	+ }
	34	+ return out;
	35	+}
	36	+
	37	+BarSeries aggregate(const BarSeries& bars, Interval target, const MarketCalendar& cal, SessionPolicy policy) {
	38	+ RAID_REQUIRE(to_duration(target).us > to_duration(bars.interval).us, invalid_argument, "aggregate: target must be coarser than source");
	39	+ BarSeries out; out.symbol = bars.symbol; out.instrument_id = bars.instrument_id; out.interval = target; out.adjustment = bars.adjustment;
	40	+ const i64 step = to_duration(target).us;
	41	+ for (usize k = 0; k < bars.n_sessions(); ++k) {
	42	+ const Date d = bars.session_dates[k];
	43	+ if (!cal.is_trading_day(d)) continue;
	44	+ const auto sb = cal.session(d);
	45	+ const Timestamp anchor = policy == SessionPolicy::regular ? sb.open_utc : sb.ext_open_utc;
	46	+ const Timestamp end = policy == SessionPolicy::regular ? sb.close_utc : sb.ext_close_utc;
	47	+ i64 cur_bin = -1; f64 o = 0, h = 0, l = 0, c = 0, v = 0; u32 fl = 0; Timestamp bin_t;
	48	+ auto flush = [&]() { if (cur_bin >= 0) { out.t_open.push_back(bin_t); out.session.push_back(d); out.open.push_back(o); out.high.push_back(h); out.low.push_back(l); out.close.push_back(c); out.volume.push_back(v); out.flags.push_back(fl); } };
	49	+ for (i64 i = bars.session_offsets[k]; i < bars.session_offsets[k + 1]; ++i) {
	50	+ const usize j = static_cast<usize>(i); const Timestamp t = bars.t_open[j];
	51	+ if (t < anchor) continue;
	52	+ if (policy == SessionPolicy::regular ? t > end : t >= end) continue;
	53	+ i64 bin = (t - anchor).us / step;
	54	+ if (target == Interval::d1) bin = 0;
	55	+ // The 16:00 closing print (t == close) forms its own bin, as in FinData's derived 5m/30m/1h datasets
	56	+ // (verified live 2026-11-02: 79 regular 5m bars per normal session, 43 on early closes).
	57	+ if (bin != cur_bin) { flush(); cur_bin = bin; bin_t = anchor + Duration(bin * step); o = bars.open[j]; h = bars.high[j]; l = bars.low[j]; c = bars.close[j]; v = bars.volume[j]; fl = bars.flags[j]; }
	58	+ else { h = std::max(h, bars.high[j]); l = std::min(l, bars.low[j]); c = bars.close[j]; v += bars.volume[j]; fl \|= bars.flags[j]; }
	59	+ }
	60	+ flush();
	61	+ }
	62	+ out.index_sessions();
	63	+ return out;
	64	+}
	65	+
	66	+SampledPrices subsample(const BarSeries& bars, usize step, usize offset) {
	67	+ RAID_REQUIRE(step >= 1 && offset < step, invalid_argument, "subsample: need step >= 1 and offset < step");
	68	+ SampledPrices s; s.session_offsets.push_back(0); s.session_dates = bars.session_dates;
	69	+ for (usize k = 0; k < bars.n_sessions(); ++k) {
	70	+ const usize b0 = static_cast<usize>(bars.session_offsets[k]), b1 = static_cast<usize>(bars.session_offsets[k + 1]);
	71	+ for (usize i = b0 + offset; i < b1; i += step) s.p.push_back(bars.close[i]);
	72	+ // always include the last price of the session so the grid ends at the close
	73	+ if (b1 > b0 && ((b1 - 1 - b0) % step) != offset % step && (b1 - 1) >= b0 + offset) s.p.push_back(bars.close[b1 - 1]);
	74	+ s.session_offsets.push_back(static_cast<i64>(s.p.size()));
	75	+ }
	76	+ return s;
	77	+}
	78	+
	79	+} // namespace raid::data
	80

spb/wp40_uqo

WP40 RAID: Phase 8 fix — data layer sources were ignored by a too-broad 'data/' pattern (found by the clean-checkout reproduction); track include/raid/data, src/data

Theme