1"""Dense regular-hours grid per instrument and strictly causal feature construction (docs/DATA_CARD.md §grid).23Grid: sessions x 391 slots (slot k = bar opening at 09:30 + k minutes ET). Slot 390 (16:00 closing print) is kept in4`base` for the overnight-gap computation only; context windows and targets use slots 0..389. Missing minutes are5masked (never filled). Past-only aggregates: 5-minute buckets (78/day, bucket j = slots 5j..5j+4) and 30-minute6buckets (13/day, bucket j = slots 30j..30j+29), each with its own presence mask. Seasonal log-volume profile7`seas[s, k]` = mean over sessions s-20..s-1 of log1p(volume) at slot k (past only, NaN when no history).8"""9from __future__ import annotations1011import json12from dataclasses import dataclass13from pathlib import Path1415import numpy as np16import polars as pl1718from .calendar import REGULAR_SLOTS, sessions, slot_index19from .adjustments import split_factor_by_session20from .quality import dedupe, qa_report2122N_DAY = 390 # context/target slots per day (0..389)23N_B5, N_B30 = 78, 1324BASE_COLS = ["lo", "lh", "ll", "lc", "lv"] # log open/high/low/close, log1p(volume)25SEAS_DAYS = 20262728@dataclass29class GridMeta:30 symbol: str31 asset_class: str32 session_dates: list[str]33 n_slots: list[int]34 early_close: list[bool]35 split_factors: list[float]36 overnight_gap: list[float | None]37 qa: dict38 counts: dict39 dataset_version: int | None40 schema_version: int = 1414243def build_grid(raw: pl.DataFrame, splits: list[dict], start: str, end: str, symbol: str, asset_class: str,44 dataset_version: int | None = None) -> tuple[dict[str, np.ndarray], GridMeta]:45 sess = sessions(start, end)46 S = len(sess)47 date2idx = {s.date.date(): i for i, s in enumerate(sess)}48 open_utc = np.array([s.open_utc.to_datetime64() for s in sess], dtype="datetime64[s]")49 n_slots = np.array([s.n_slots for s in sess], dtype=np.int64)5051 qa = qa_report(raw)52 bars = dedupe(raw)53 bars = bars.with_columns(pl.col("session_date").cast(pl.Date))54 sidx = np.array([date2idx.get(d, -1) for d in bars["session_date"].to_list()], dtype=np.int64)55 bars = bars.with_columns(pl.Series("sidx", sidx)).filter(pl.col("sidx") >= 0)56 ts = bars["timestamp_utc"].dt.replace_time_zone(None).to_numpy().astype("datetime64[s]")57 si = bars["sidx"].to_numpy()58 slot = slot_index(ts, open_utc[si], n_slots[si])59 bars = bars.with_columns(pl.Series("slot", slot))60 n_ext = int((slot < 0).sum())61 reg = bars.filter(pl.col("slot") >= 0)62 bad = ((reg["high"] < reg["low"]) | (reg["open"] > reg["high"]) | (reg["open"] < reg["low"]) |63 (reg["close"] > reg["high"]) | (reg["close"] < reg["low"]) | (reg["low"] <= 0) | (reg["volume"] < 0))64 n_bad = int(bad.sum())65 reg = reg.filter(~bad)66 # split adjustment (explicit, session based)67 f = split_factor_by_session(reg["session_date"].to_numpy(), splits)68 base = np.full((S, REGULAR_SLOTS, 5), np.nan, dtype=np.float32)69 mask = np.zeros((S, REGULAR_SLOTS), dtype=np.uint8)70 si = reg["sidx"].to_numpy(); sl = reg["slot"].to_numpy()71 base[si, sl, 0] = np.log(reg["open"].to_numpy() / f)72 base[si, sl, 1] = np.log(reg["high"].to_numpy() / f)73 base[si, sl, 2] = np.log(reg["low"].to_numpy() / f)74 base[si, sl, 3] = np.log(reg["close"].to_numpy() / f)75 base[si, sl, 4] = np.log1p(reg["volume"].to_numpy() * f)76 mask[si, sl] = 177 in_session = (np.arange(REGULAR_SLOTS)[None, :] < n_slots[:, None]).astype(np.uint8)78 mask &= in_session7980 # overnight gap: first present open of day s vs last present close (incl. closing print) of day s-181 gaps: list[float | None] = []82 for s in range(S):83 g = None84 if s > 0 and mask[s].any() and mask[s - 1].any():85 k0 = int(np.argmax(mask[s] > 0)); k1 = int(np.max(np.flatnonzero(mask[s - 1])))86 g = float(base[s, k0, 0] - base[s - 1, k1, 3])87 gaps.append(g)8889 # seasonal volume profile (past 20 sessions, present bars only)90 lv = np.where(mask > 0, base[:, :, 4], 0.0).astype(np.float64)91 cnt = (mask > 0).astype(np.float64)92 cs_lv = np.concatenate([np.zeros((1, REGULAR_SLOTS)), np.cumsum(lv, axis=0)], axis=0)93 cs_c = np.concatenate([np.zeros((1, REGULAR_SLOTS)), np.cumsum(cnt, axis=0)], axis=0)94 lo_i = np.clip(np.arange(S) - SEAS_DAYS, 0, None)95 num = cs_lv[np.arange(S)] - cs_lv[lo_i]96 den = cs_c[np.arange(S)] - cs_c[lo_i]97 with np.errstate(invalid="ignore", divide="ignore"):98 seas = np.where(den > 0, num / den, np.nan).astype(np.float32)99100 b5, m5 = _buckets(base, mask, 5, N_B5)101 b30, m30 = _buckets(base, mask, 30, N_B30)102 # derived flat arrays (slots 0..389 only): log close with NaN, previous present slot, seasonal squared-return profile103 m390 = mask[:, :N_DAY] > 0104 flat_lc = np.where(m390, base[:, :N_DAY, 3], np.nan).reshape(-1).astype(np.float32)105 idx = np.where(m390.reshape(-1), np.arange(S * N_DAY), -1)106 prev_present = np.maximum.accumulate(idx).astype(np.int32)107 r2 = np.zeros((S, N_DAY)); r2c = np.zeros((S, N_DAY))108 lc2 = np.where(m390, base[:, :N_DAY, 3].astype(np.float64), np.nan)109 pr = np.full_like(lc2, np.nan)110 pr[:, 1:] = lc2[:, :-1]111 # same-session previous present close (forward carry inside the day)112 for k in range(1, N_DAY):113 pr[:, k] = np.where(np.isnan(pr[:, k]), pr[:, k - 1], pr[:, k])114 ret = lc2 - pr115 ok = ~np.isnan(ret)116 r2[ok] = ret[ok] ** 2; r2c[ok] = 1.0117 cs_r2 = np.concatenate([np.zeros((1, N_DAY)), np.cumsum(r2, axis=0)]); cs_c2 = np.concatenate([np.zeros((1, N_DAY)), np.cumsum(r2c, axis=0)])118 num2 = cs_r2[np.arange(S)] - cs_r2[lo_i]; den2 = cs_c2[np.arange(S)] - cs_c2[lo_i]119 with np.errstate(invalid="ignore", divide="ignore"):120 seas_r2 = np.where(den2 > 0, num2 / den2, np.nan).astype(np.float32)121 split_by_session = split_factor_by_session(np.array([s.date.date() for s in sess], dtype="datetime64[D]"), splits)122 meta = GridMeta(symbol, asset_class, [s.date.date().isoformat() for s in sess], [int(x) for x in n_slots],123 [bool(s.early_close) for s in sess], [float(x) for x in split_by_session], gaps, qa,124 {"rows_raw": int(raw.height), "rows_dedup": int(bars.height), "rows_extended_hours": n_ext,125 "rows_regular": int(reg.height) + n_bad, "rows_dropped_integrity": n_bad,126 "regular_slots_expected": int(in_session.sum()), "regular_slots_present": int(mask.sum()),127 "coverage": float(mask.sum() / max(1, in_session.sum())),128 "sessions_with_any_bar": int((mask.sum(1) > 0).sum()), "sessions": S},129 dataset_version)130 arrays = {"base": base, "mask": mask, "in_session": in_session, "seas": seas, "b5": b5, "m5": m5, "b30": b30,131 "m30": m30, "flat_lc": flat_lc, "prev_present": prev_present, "seas_r2": seas_r2}132 return arrays, meta133134135def _buckets(base: np.ndarray, mask: np.ndarray, width: int, nb: int) -> tuple[np.ndarray, np.ndarray]:136 """Past-only OHLCV aggregates of `width` slots over slots 0..389 (closing print excluded)."""137 S = base.shape[0]138 x = base[:, :N_DAY].reshape(S, nb, width, 5)139 m = (mask[:, :N_DAY].reshape(S, nb, width) > 0)140 out = np.full((S, nb, 5), np.nan, dtype=np.float32)141 bm = m.any(axis=2)142 # open = first present, close = last present143 first = np.argmax(m, axis=2); last = width - 1 - np.argmax(m[:, :, ::-1], axis=2)144 ii, jj = np.indices((S, nb))145 out[:, :, 0] = x[ii, jj, first, 0]146 out[:, :, 3] = x[ii, jj, last, 3]147 import warnings148 with np.errstate(invalid="ignore"), warnings.catch_warnings():149 warnings.simplefilter("ignore", RuntimeWarning)150 out[:, :, 1] = np.nanmax(np.where(m, x[:, :, :, 1], np.nan), axis=2)151 out[:, :, 2] = np.nanmin(np.where(m, x[:, :, :, 2], np.nan), axis=2)152 vol = np.where(m, np.expm1(x[:, :, :, 4].astype(np.float64)), 0.0).sum(axis=2)153 out[:, :, 4] = np.log1p(vol)154 out[~bm] = np.nan155 return out, bm.astype(np.uint8)156157158def save_grid(arrays: dict[str, np.ndarray], meta: GridMeta, out_dir: Path) -> None:159 out_dir.mkdir(parents=True, exist_ok=True)160 for k, v in arrays.items():161 np.save(out_dir / f"{k}.npy", v)162 (out_dir / "meta.json").write_text(json.dumps(meta.__dict__))163164165def load_grid(dir_: Path, mmap: bool = True) -> tuple[dict[str, np.ndarray], dict]:166 arrays = {k: np.load(dir_ / f"{k}.npy", mmap_mode="r" if mmap else None)167 for k in ("base", "mask", "in_session", "seas", "b5", "m5", "b30", "m30", "flat_lc", "prev_present", "seas_r2")}168 return arrays, json.loads((dir_ / "meta.json").read_text())169