SPB Git
spboucher.ai

spb/findata

Public

SQL fast path: symbol-scoped statements resolve their file list from the catalog

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 2, 2026) parent 6437fd7 Browse files

3 changed files +94 −6

3 changed files
Unified Split
modified services/api/src/findata_api/routers/query.py +4 −3
@@ -48,15 +48,16 @@ def query(body: QueryIn, request: Request, p: A.Principal = Depends(A.require("q
48 48 A.audit(request, "query.sql", None, rows=len(rows), ms=int((time.monotonic() - t0) * 1000), sql=body.sql[:500])
49 49 return {"columns": [{"name": f.name, "type": str(f.type)} for f in tbl.schema], "rows": rows,
50 50 "meta": {**meta, "rows": len(rows), "elapsed_ms": int((time.monotonic() - t0) * 1000),
51 − "views_used": _used(body.sql, views), "engine": "duckdb (read-only, sandboxed, 1 statement, hive-partition pruning)"}}
51 + "views_used": meta.get("views", _used(body.sql, views)), "engine": "duckdb (read-only, sandboxed, 1 statement; symbol-scoped statements resolve files from the catalog, others scan the directory)"}}
52 52
53 53
54 54 @router.get("/views")
55 55 def views(p: A.Principal = Depends(A.require("query"))):
56 56 v = dataset_views()
57 57 return {"views": [{"name": k, "kind": spec.get("kind"), "layouts": spec.get("layouts"), "root": spec.get("root")} for k, spec in sorted(v.items())],
58 − "note": "Views are Hive-partitioned directory scans: filter on the partition columns (symbol, year, session_date; underlying, period for options) "
59 − "to prune files — e.g. WHERE symbol='AAPL' AND year=2024. Whole-dataset scans over millions of files time out (30 s).",
58 + "note": "Fast path: statements with symbol='X' / symbol IN (...) / underlying='X' (plus optional year=, session_date/trade_date ranges) "
59 + "get their file list from the catalog in milliseconds. Without a symbol predicate the view is a Hive directory scan of the whole "
60 + "dataset (30–60 s to list hundreds of thousands of files; may time out).",
60 61 "examples": ["SELECT session_date, min(low) lo, max(high) hi, sum(volume) vol FROM market_bars_stock_1m_raw WHERE symbol='AAPL' AND year=2024 GROUP BY 1 ORDER BY 1",
61 62 "SELECT trade_date, strike, call_put, bid, ask, open_interest, delta FROM options_eod WHERE underlying='SPY' AND period='2024q1' AND expiry=DATE '2024-03-28' ORDER BY trade_date, strike LIMIT 100",
62 63 "SELECT symbol, count(*) bars FROM market_bars_futures_1m_continuous_ratio WHERE symbol IN ('ES','NQ') GROUP BY 1"]}
modified services/ingest/src/findata_ingest/reader.py +66 −3
@@ -8,6 +8,7 @@ Adjustments are applied at query time from `corporate_actions` (explicit, reprod
8 8 """
9 9 from __future__ import annotations
10 10
11 +import re
11 12 from datetime import date
12 13 from typing import Any
13 14
@@ -126,6 +127,7 @@ def run_sql(sql: str, max_rows: int, timeout_s: int, views: dict[str, Any] | Non
126 127 raise ValueError("statement must start with SELECT / WITH")
127 128 is_explain = bool(re.match(r"^\s*explain\b", sql, re.I))
128 129 con = _con()
130 + scoped_used: list[str] = []
129 131 dirs = []
130 132 for r in Lake().roots():
131 133 rr = str(r.resolve()).replace("'", "''")
@@ -137,7 +139,14 @@ def run_sql(sql: str, max_rows: int, timeout_s: int, views: dict[str, Any] | Non
137 139 if name not in _referenced_views(sql, {name: spec}):
138 140 continue # only materialize views the statement uses (directory globs are not free)
139 141 if isinstance(spec, dict) and spec.get("sql"):
140 − con.execute(f"CREATE VIEW {name} AS {spec['sql']}")
142 + fast = None
143 + if spec.get("dataset_id"):
144 + try:
145 + fast = scoped_view_sql(spec["dataset_id"], spec.get("kind", ""), extract_scope(sql), Lake())
146 + except Exception:
147 + fast = None
148 + con.execute(f"CREATE VIEW {name} AS {fast or spec['sql']}")
149 + scoped_used.append(name if fast else f"{name} (full scan)")
141 150 elif isinstance(spec, list) and spec: # legacy explicit file list
142 151 lst = ", ".join("'" + g.replace("'", "''") + "'" for g in spec)
143 152 con.execute(f"CREATE VIEW {name} AS SELECT * FROM read_parquet([{lst}], union_by_name=true, hive_partitioning=false)")
@@ -154,7 +163,7 @@ def run_sql(sql: str, max_rows: int, timeout_s: int, views: dict[str, Any] | Non
154 163 finally:
155 164 timer.cancel()
156 165 truncated = tbl.num_rows > max_rows
157 − return tbl.slice(0, max_rows), {"truncated": truncated}
166 + return tbl.slice(0, max_rows), {"truncated": truncated, "views": scoped_used}
158 167
159 168
160 169 _views_cache: dict[str, Any] = {"at": 0.0, "value": None}
@@ -191,11 +200,65 @@ def dataset_views() -> dict[str, dict[str, Any]]:
191 200 if "delta" in layouts: # session files: derive the same columns so UNION ALL BY NAME lines up
192 201 parts.append(f"SELECT *, symbol_as_reported AS symbol, year(session_date)::BIGINT AS year "
193 202 f"FROM read_parquet('{layouts['delta']}', hive_partitioning=false)")
194 − out[d["id"].replace(".", "_")] = {"sql": " UNION ALL BY NAME ".join(parts), "layouts": sorted(layouts), "root": str(root), "kind": d["kind"]}
203 + out[d["id"].replace(".", "_")] = {"sql": " UNION ALL BY NAME ".join(parts), "layouts": sorted(layouts), "root": str(root),
204 + "kind": d["kind"], "dataset_id": d["id"]}
195 205 _views_cache.update(at=time.monotonic(), value=out)
196 206 return out
197 207
198 208
209 +_SYM_RE = re.compile(r"\b(?:symbol|symbol_as_reported|underlying)\s*=\s*'([A-Za-z0-9.\-_]+)'", re.I)
210 +_SYM_IN_RE = re.compile(r"\b(?:symbol|symbol_as_reported|underlying)\s+IN\s*\(([^)]+)\)", re.I)
211 +_YEAR_RE = re.compile(r"\byear\s*=\s*(\d{4})\b", re.I)
212 +_YEAR_IN_RE = re.compile(r"\byear\s+IN\s*\(([^)]+)\)", re.I)
213 +_DATE_RE = re.compile(r"\b(?:session_date|trade_date)\s*(>=|<=|=|>|<)\s*(?:DATE\s*)?'(\d{4}-\d{2}-\d{2})'", re.I)
214 +_BETWEEN_RE = re.compile(r"\b(?:session_date|trade_date)\s+BETWEEN\s+(?:DATE\s*)?'(\d{4}-\d{2}-\d{2})'\s+AND\s+(?:DATE\s*)?'(\d{4}-\d{2}-\d{2})'", re.I)
215 +
216 +
217 +def extract_scope(sql: str) -> dict[str, Any]:
218 + """Pull simple equality/IN/range predicates on partition columns out of a statement so that the file list can
219 + be resolved from the catalog (indexed) instead of globbing hundreds of thousands of files."""
220 + from datetime import date as _date
221 + syms: set[str] = {m.group(1).upper() for m in _SYM_RE.finditer(sql)}
222 + for m in _SYM_IN_RE.finditer(sql):
223 + syms |= {x.strip().strip("'\"").upper() for x in m.group(1).split(",") if x.strip()}
224 + years: set[int] = {int(m.group(1)) for m in _YEAR_RE.finditer(sql)}
225 + for m in _YEAR_IN_RE.finditer(sql):
226 + years |= {int(x) for x in re.findall(r"\d{4}", m.group(1))}
227 + start = end = None
228 + for op, d in _DATE_RE.findall(sql):
229 + dd = _date.fromisoformat(d)
230 + if op in (">=", ">", "="):
231 + start = dd if start is None else max(start, dd)
232 + if op in ("<=", "<", "="):
233 + end = dd if end is None else min(end, dd)
234 + for a, b in _BETWEEN_RE.findall(sql):
235 + start = max(start, _date.fromisoformat(a)) if start else _date.fromisoformat(a)
236 + end = min(end, _date.fromisoformat(b)) if end else _date.fromisoformat(b)
237 + if years:
238 + ys, ye = _date(min(years), 1, 1), _date(max(years), 12, 31)
239 + start = ys if start is None else max(start, ys)
240 + end = ye if end is None else min(end, ye)
241 + return {"symbols": sorted(syms), "start": start, "end": end}
242 +
243 +
244 +def scoped_view_sql(dataset_id: str, kind: str, scope: dict[str, Any], lake: Lake) -> str | None:
245 + """Explicit file list from the catalog for symbol-scoped statements (fast path). None → use the glob view."""
246 + if not scope["symbols"]:
247 + return None
248 + with connection() as conn:
249 + files = live_files(conn, dataset_id, scope["symbols"], scope["start"], scope["end"])
250 + if len(files) > 20000:
251 + return None
252 + if not files:
253 + return "SELECT * FROM (VALUES (NULL)) t(empty) WHERE false"
254 + lst = ", ".join("'" + str(lake.resolve(f["path"])).replace("'", "''") + "'" for f in files)
255 + if kind == "options":
256 + return (f"SELECT * EXCLUDE (filename), regexp_extract(filename, 'period=([^/]+)', 1) AS period "
257 + f"FROM read_parquet([{lst}], union_by_name=true, hive_partitioning=false, filename=true)")
258 + return (f"SELECT *, symbol_as_reported AS symbol, year(session_date)::BIGINT AS year "
259 + f"FROM read_parquet([{lst}], union_by_name=true, hive_partitioning=false)")
260 +
261 +
199 262 def _referenced_views(sql: str, views: dict[str, Any]) -> list[str]:
200 263 import re as _re
201 264 low = sql.lower()
modified tests/test_full_catalog.py +24 −0
@@ -143,3 +143,27 @@ def test_deflate64_fallback_uses_7z(tmp_path, monkeypatch):
143 143 out = list(A.iter_archive_files(z))
144 144 assert out == [("X_option_chain.txt", b"2026-10-01,33,2026-10-16,c,1,1,1,0,0,0,0,0,0,0,0,0\n")]
145 145 assert not list(tmp_path.glob("unz-*")) # temp dir cleaned
146 +
147 +
148 +def test_extract_scope_from_sql():
149 + from datetime import date
150 +
151 + from findata_ingest.reader import extract_scope
152 + s = extract_scope("SELECT * FROM market_bars_stock_1m_raw WHERE symbol='AAPL' AND year=2024 AND session_date >= DATE '2024-03-01' AND session_date <= '2024-03-31'")
153 + assert s == {"symbols": ["AAPL"], "start": date(2024, 3, 1), "end": date(2024, 3, 31)}
154 + s = extract_scope("select count(*) from options_eod where underlying in ('SPY', 'QQQ') and trade_date between DATE '2024-03-01' and DATE '2024-03-31'")
155 + assert s["symbols"] == ["QQQ", "SPY"] and s["start"] == date(2024, 3, 1) and s["end"] == date(2024, 3, 31)
156 + assert extract_scope("select count(*) from market_bars_stock_1d_raw where session_date = DATE '2024-03-15'")["symbols"] == []
157 + assert extract_scope("select * from x where year in (2019, 2020)")["start"] == date(2019, 1, 1)
158 +
159 +
160 +def test_scoped_sql_uses_catalog_files(vendor, lake, log):
161 + from findata_core.db import fetch_one
162 + from findata_ingest.reader import dataset_views, run_sql
163 + if fetch_one("SELECT count(*) AS n FROM lake_files WHERE dataset_id='market_bars.stock.1m.raw' AND version_removed IS NULL")["n"] == 0:
164 + pytest.skip("no 1m data in this test database")
165 + views = dataset_views()
166 + t, meta = run_sql("SELECT count(*) AS n FROM market_bars_stock_1m_raw WHERE symbol='AAPL' AND session_date=DATE '2026-10-30'", 10, 30, views)
167 + assert t.to_pylist()[0]["n"] == 900 and meta["views"] == ["market_bars_stock_1m_raw"]
168 + t, meta = run_sql("SELECT count(*) AS n FROM market_bars_stock_1m_raw WHERE session_date=DATE '2026-10-30'", 10, 30, views)
169 + assert t.to_pylist()[0]["n"] >= 900 and meta["views"] == ["market_bars_stock_1m_raw (full scan)"]
146 170
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere