SPB Git
spboucher.ai

spb/findata

Public

lint config, infra (mld manifest, alerts, bootstrap), docs (architecture, dictionary, operations, security)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent d6d35c9 Browse files

51 changed files +204 −143

51 changed files
Unified Split
modified connectors/fundamentals/src/findata_fundamentals/__init__.py +5 −1
@@ -1,2 +1,6 @@
1−from findata_fundamentals.firstrate import FirstRateFundamentalsConnector, facts_from_financials, available_at_policy # noqa: F401 1+from findata_fundamentals.firstrate import ( # noqa: F401
2+ FirstRateFundamentalsConnector,
3+ available_at_policy,
4+ facts_from_financials,
5+)
2 from findata_fundamentals.sec_edgar import SecEdgarConnector, facts_from_companyfacts # noqa: F401 6 from findata_fundamentals.sec_edgar import SecEdgarConnector, facts_from_companyfacts # noqa: F401
modified connectors/fundamentals/src/findata_fundamentals/firstrate.py +2 −2
@@ -13,12 +13,12 @@ reconstructed from this vendor (documented limitation).
13 from __future__ import annotations 13 from __future__ import annotations
14 14
15 import json 15 import json
16+from collections.abc import Iterator
16 from datetime import UTC, date, datetime, time, timedelta 17 from datetime import UTC, date, datetime, time, timedelta
17−from typing import Any, Iterator 18+from typing import Any
18 from zoneinfo import ZoneInfo 19 from zoneinfo import ZoneInfo
19 20
20 import httpx 21 import httpx
21−
22 from findata_core.config import get_settings 22 from findata_core.config import get_settings
23 from findata_core.redact import redact 23 from findata_core.redact import redact
24 24
modified connectors/fundamentals/src/findata_fundamentals/sec_edgar.py +2 −2
@@ -12,12 +12,12 @@ PIT: `companyfacts` provides `filed` (filing date), `accn` (accession), `form`,
12 from __future__ import annotations 12 from __future__ import annotations
13 13
14 import time 14 import time
15+from collections.abc import Iterator
15 from datetime import UTC, date, datetime, timedelta 16 from datetime import UTC, date, datetime, timedelta
16−from typing import Any, Iterator 17+from typing import Any
17 from zoneinfo import ZoneInfo 18 from zoneinfo import ZoneInfo
18 19
19 import httpx 20 import httpx
20−
21 from findata_core.config import get_settings 21 from findata_core.config import get_settings
22 22
23 SOURCE_ID = "sec_edgar" 23 SOURCE_ID = "sec_edgar"
modified connectors/market/src/findata_market_firstrate/__init__.py +1 −1
@@ -1,3 +1,3 @@
1+from findata_market_firstrate.archive import iter_archive_files, parse_bar_file, ticker_from_filename # noqa: F401
1 from findata_market_firstrate.client import FRDClient, FRDError # noqa: F401 2 from findata_market_firstrate.client import FRDClient, FRDError # noqa: F401
2 from findata_market_firstrate.connector import FirstRateMarketConnector # noqa: F401 3 from findata_market_firstrate.connector import FirstRateMarketConnector # noqa: F401
3−from findata_market_firstrate.archive import iter_archive_files, parse_bar_file, ticker_from_filename # noqa: F401
modified connectors/market/src/findata_market_firstrate/archive.py +1 −2
@@ -15,12 +15,11 @@ from pathlib import Path
15 15
16 import polars as pl 16 import polars as pl
17 import pyarrow as pa 17 import pyarrow as pa
18−
19 from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval 18 from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval
20 19
21 VENDOR_TF = {"1min": Interval.m1, "5min": Interval.m5, "30min": Interval.m30, "1hour": Interval.h1, "1day": Interval.d1} 20 VENDOR_TF = {"1min": Interval.m1, "5min": Interval.m5, "30min": Interval.m30, "1hour": Interval.h1, "1day": Interval.d1}
22 VENDOR_ADJ = {"UNADJUSTED": Adjustment.raw, "adj_split": Adjustment.split, "adj_splitdiv": Adjustment.split_div} 21 VENDOR_ADJ = {"UNADJUSTED": Adjustment.raw, "adj_split": Adjustment.split, "adj_splitdiv": Adjustment.split_div}
23−_TICKER_RE = re.compile(r"^([A-Za-z0-9.\-]+?)(?:[_-](?:full|day|week|month|1min|5min|30min|1hour|1day|UNADJUSTED|adj_split|adj_splitdiv|contin.*))*\.(txt|csv)$", re.I) 22+_TICKER_RE = re.compile(r"^([A-Za-z0-9.\-]+?)(?:[_-](?:full|day|week|month|1min|5min|30min|1hour|1day|UNADJUSTED|adj_split|adj_splitdiv|contin.*))*\.(txt|csv)$", re.IGNORECASE)
24 23
25 24
26 def ticker_from_filename(name: str) -> str: 25 def ticker_from_filename(name: str) -> str:
modified connectors/market/src/findata_market_firstrate/client.py +0 −1
@@ -15,7 +15,6 @@ from pathlib import Path
15 from typing import Any 15 from typing import Any
16 16
17 import httpx 17 import httpx
18−
19 from findata_core.redact import redact 18 from findata_core.redact import redact
20 19
21 BASE_URL = "https://firstratedata.com/api" 20 BASE_URL = "https://firstratedata.com/api"
modified connectors/market/src/findata_market_firstrate/connector.py +22 −8
@@ -10,13 +10,27 @@ from pathlib import Path
10 from typing import Any 10 from typing import Any
11 11
12 from findata_core.config import get_settings 12 from findata_core.config import get_settings
13−from findata_market_firstrate.archive import (VENDOR_ADJ, VENDOR_TF, iter_archive_files, parse_bar_file,
14− parse_meta_pairs, ticker_from_filename)
15−from findata_market_firstrate.client import FRDClient
16 from findata_quality import validate_bars 13 from findata_quality import validate_bars
17 from findata_schema import AssetClass, Interval 14 from findata_schema import AssetClass, Interval
18−from findata_schema.connector import (Capabilities, Checkpoint, Connector, FetchRequest, FetchResult, 15+from findata_schema.connector import (
19− NormalizedBatch, ValidationReport) 16+ Capabilities,
17+ Checkpoint,
18+ Connector,
19+ FetchRequest,
20+ FetchResult,
21+ NormalizedBatch,
22+ ValidationReport,
23+)
24+
25+from findata_market_firstrate.archive import (
26+ VENDOR_ADJ,
27+ VENDOR_TF,
28+ iter_archive_files,
29+ parse_bar_file,
30+ parse_meta_pairs,
31+ ticker_from_filename,
32+)
33+from findata_market_firstrate.client import FRDClient
20 34
21 SOURCE_ID = "firstrate" 35 SOURCE_ID = "firstrate"
22 SOURCE_TZ = {"crypto": "UTC"} # everything else is US Eastern 36 SOURCE_TZ = {"crypto": "UTC"} # everything else is US Eastern
@@ -40,7 +54,7 @@ class FirstRateMarketConnector(Connector):
40 for t in ("stock", "etf", "futures", "index", "fx", "crypto", "options"): 54 for t in ("stock", "etf", "futures", "index", "fx", "crypto", "options"):
41 try: 55 try:
42 out["last_update"][t] = self.client.last_update(t).isoformat() 56 out["last_update"][t] = self.client.last_update(t).isoformat()
43− except Exception as e: # noqa: BLE001 57+ except Exception as e:
44 out["last_update"][t] = f"error: {e}" 58 out["last_update"][t] = f"error: {e}"
45 return out 59 return out
46 60
@@ -66,7 +80,7 @@ class FirstRateMarketConnector(Connector):
66 vendor_last = None 80 vendor_last = None
67 try: 81 try:
68 vendor_last = self.client.last_update(asset, full=(p.get("period") == "full")) 82 vendor_last = self.client.last_update(asset, full=(p.get("period") == "full"))
69− except Exception: # noqa: BLE001 83+ except Exception:
70 vendor_last = None 84 vendor_last = None
71 stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") 85 stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
72 label = "_".join(f"{k}-{v}" for k, v in sorted(p.items())) 86 label = "_".join(f"{k}-{v}" for k, v in sorted(p.items()))
@@ -77,7 +91,7 @@ class FirstRateMarketConnector(Connector):
77 import zipfile 91 import zipfile
78 with zipfile.ZipFile(got.path) as zf: 92 with zipfile.ZipFile(got.path) as zf:
79 file_count = sum(1 for i in zf.infolist() if not i.is_dir()) 93 file_count = sum(1 for i in zf.infolist() if not i.is_dir())
80− except Exception: # noqa: BLE001 94+ except Exception:
81 pass 95 pass
82 return FetchResult(request=request, path=got.path, sha256=got.sha256, bytes=got.bytes, fetched_at=got.fetched_at, 96 return FetchResult(request=request, path=got.path, sha256=got.sha256, bytes=got.bytes, fetched_at=got.fetched_at,
83 vendor_last_update=vendor_last, file_count=file_count, 97 vendor_last_update=vendor_last, file_count=file_count,
modified packages/core/src/findata_core/db.py +4 −4
@@ -2,9 +2,10 @@
2 from __future__ import annotations 2 from __future__ import annotations
3 3
4 import hashlib 4 import hashlib
5+from collections.abc import Iterator
5 from contextlib import contextmanager 6 from contextlib import contextmanager
6 from importlib import resources 7 from importlib import resources
7−from typing import Any, Iterator 8+from typing import Any
8 9
9 import psycopg 10 import psycopg
10 from psycopg.rows import dict_row 11 from psycopg.rows import dict_row
@@ -40,9 +41,8 @@ def connection() -> Iterator[psycopg.Connection[Any]]:
40 41
41 @contextmanager 42 @contextmanager
42 def transaction() -> Iterator[psycopg.Connection[Any]]: 43 def transaction() -> Iterator[psycopg.Connection[Any]]:
43− with connection() as conn: 44+ with connection() as conn, conn.transaction():
44− with conn.transaction(): 45+ yield conn
45− yield conn
46 46
47 47
48 def fetch_all(sql: str, params: Any = None) -> list[dict[str, Any]]: 48 def fetch_all(sql: str, params: Any = None) -> list[dict[str, Any]]:
modified packages/python-sdk/src/findata_sdk/client.py +3 −4
@@ -18,7 +18,7 @@ from typing import Any
18 18
19 import httpx 19 import httpx
20 import pyarrow as pa 20 import pyarrow as pa
21−import pyarrow.ipc as ipc 21+from pyarrow import ipc
22 22
23 23
24 class FinDataError(RuntimeError): 24 class FinDataError(RuntimeError):
@@ -117,13 +117,12 @@ class FinData:
117 if r.status_code >= 400: 117 if r.status_code >= 400:
118 raise FinDataError(r.status_code, {"error": {"code": "download_failed"}}) 118 raise FinDataError(r.status_code, {"error": {"code": "download_failed"}})
119 with open(out, "wb") as f: 119 with open(out, "wb") as f:
120− for chunk in r.iter_bytes(): 120+ f.writelines(r.iter_bytes())
121− f.write(chunk)
122 return out 121 return out
123 122
124 123
125 def _json(r: httpx.Response) -> Any: 124 def _json(r: httpx.Response) -> Any:
126 try: 125 try:
127 return r.json() 126 return r.json()
128− except Exception: # noqa: BLE001 127+ except Exception:
129 return {"error": {"code": "http", "message": r.text[:300]}} 128 return {"error": {"code": "http", "message": r.text[:300]}}
modified packages/schema/src/findata_schema/__init__.py +19 −5
@@ -1,8 +1,22 @@
1 """findata_schema — Arrow schemas, data contracts and connector interfaces (single source of truth).""" 1 """findata_schema — Arrow schemas, data contracts and connector interfaces (single source of truth)."""
2−from findata_schema.contracts import ( # noqa: F401
3− BARS_SCHEMA, BARS_SCHEMA_VERSION, FUNDAMENTALS_SCHEMA, RAW_BAR_COLUMNS, AssetClass, Interval,
4− Adjustment, QualityFlag, INTERVAL_SECONDS, bars_schema_json,
5−)
6 from findata_schema.connector import ( # noqa: F401 2 from findata_schema.connector import ( # noqa: F401
7− Capabilities, Checkpoint, Connector, FetchRequest, FetchResult, NormalizedBatch, ValidationReport, 3+ Capabilities,
4+ Checkpoint,
5+ Connector,
6+ FetchRequest,
7+ FetchResult,
8+ NormalizedBatch,
9+ ValidationReport,
10+)
11+from findata_schema.contracts import ( # noqa: F401
12+ BARS_SCHEMA,
13+ BARS_SCHEMA_VERSION,
14+ FUNDAMENTALS_SCHEMA,
15+ INTERVAL_SECONDS,
16+ RAW_BAR_COLUMNS,
17+ Adjustment,
18+ AssetClass,
19+ Interval,
20+ QualityFlag,
21+ bars_schema_json,
8 ) 22 )
modified pyproject.toml +4 −0
@@ -55,3 +55,7 @@ filterwarnings = ["ignore::DeprecationWarning"]
55 [tool.ruff] 55 [tool.ruff]
56 line-length = 110 56 line-length = 110
57 target-version = "py312" 57 target-version = "py312"
58+
59+[tool.ruff.lint]
60+select = ["E", "F", "I", "B", "UP", "SIM", "RUF", "PLW", "C4", "S110"]
61+ignore = ["B008", "RUF001", "RUF002", "RUF003", "PLW0603", "PLW2901", "PLW0108", "E402", "E731", "PLW1510", "E501", "RUF059", "F841", "S110", "SIM102", "SIM115", "FLY002", "RUF015", "C401", "UP007", "UP045", "RUF012", "B904"]
modified services/api/src/findata_api/auth.py +3 −3
@@ -18,9 +18,9 @@ import pyotp
18 from argon2 import PasswordHasher 18 from argon2 import PasswordHasher
19 from argon2.exceptions import VerifyMismatchError 19 from argon2.exceptions import VerifyMismatchError
20 from fastapi import Depends, Request 20 from fastapi import Depends, Request
21−
22 from findata_core.config import get_settings 21 from findata_core.config import get_settings
23 from findata_core.db import connection, transaction 22 from findata_core.db import connection, transaction
23+
24 from findata_api.errors import ApiError 24 from findata_api.errors import ApiError
25 25
26 SESSION_COOKIE = "fd_session" 26 SESSION_COOKIE = "fd_session"
@@ -163,7 +163,7 @@ def get_principal(request: Request) -> Principal:
163 return Principal("anonymous", None, None, None, set()) 163 return Principal("anonymous", None, None, None, set())
164 164
165 165
166−def require(scope: str): # noqa: ANN201 166+def require(scope: str):
167 def dep(request: Request, p: Principal = Depends(get_principal)) -> Principal: 167 def dep(request: Request, p: Principal = Depends(get_principal)) -> Principal:
168 if p.kind == "anonymous": 168 if p.kind == "anonymous":
169 raise ApiError(401, "unauthenticated", "Sign in or provide an API key") 169 raise ApiError(401, "unauthenticated", "Sign in or provide an API key")
@@ -218,7 +218,7 @@ def totp_setup(user_id: str, email: str) -> dict[str, Any]:
218 buf = io.BytesIO() 218 buf = io.BytesIO()
219 img.save(buf, format="PNG") 219 img.save(buf, format="PNG")
220 qr = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() 220 qr = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
221− except Exception: # noqa: BLE001 221+ except Exception:
222 qr = None 222 qr = None
223 return {"otpauth_uri": uri, "qr_png_data_url": qr, "secret": secret} 223 return {"otpauth_uri": uri, "qr_png_data_url": qr, "secret": secret}
224 224
modified services/api/src/findata_api/errors.py +1 −1
@@ -4,5 +4,5 @@ from fastapi import HTTPException
4 4
5 5
6 class ApiError(HTTPException): 6 class ApiError(HTTPException):
7− def __init__(self, status: int, code: str, message: str, **extra): # noqa: ANN003 7+ def __init__(self, status: int, code: str, message: str, **extra):
8 super().__init__(status_code=status, detail={"code": code, "message": message, **extra}) 8 super().__init__(status_code=status, detail={"code": code, "message": message, **extra})
modified services/api/src/findata_api/health.py +3 −2
@@ -7,6 +7,7 @@ from typing import Any
7 from findata_core.config import get_settings 7 from findata_core.config import get_settings
8 from findata_core.db import connection 8 from findata_core.db import connection
9 from findata_ingest.lake import Lake 9 from findata_ingest.lake import Lake
10+
10 from findata_api import metrics as m 11 from findata_api import metrics as m
11 12
12 13
@@ -76,7 +77,7 @@ def evaluate() -> dict[str, Any]:
76 an = conn.execute("SELECT dataset_id, severity, count(*) n FROM quality_anomalies WHERE created_at > now() - interval '7 days' GROUP BY 1,2").fetchall() 77 an = conn.execute("SELECT dataset_id, severity, count(*) n FROM quality_anomalies WHERE created_at > now() - interval '7 days' GROUP BY 1,2").fetchall()
77 for a in an: 78 for a in an:
78 m.ANOMALIES.labels(a["dataset_id"], a["severity"]).set(a["n"]) 79 m.ANOMALIES.labels(a["dataset_id"], a["severity"]).set(a["n"])
79− except Exception as e: # noqa: BLE001 80+ except Exception as e:
80 out["components"]["postgres"] = f"error: {e}" 81 out["components"]["postgres"] = f"error: {e}"
81 out["alerts"].append({"severity": "error", "code": "postgres_down", "message": str(e)[:200]}) 82 out["alerts"].append({"severity": "error", "code": "postgres_down", "message": str(e)[:200]})
82 free = lake.free_gb() 83 free = lake.free_gb()
@@ -98,7 +99,7 @@ def evaluate() -> dict[str, Any]:
98 out["components"]["ssd_smart"] = smart 99 out["components"]["ssd_smart"] = smart
99 if smart not in ("Verified", "Not Supported", "unknown"): 100 if smart not in ("Verified", "Not Supported", "unknown"):
100 out["alerts"].append({"severity": "error", "code": "ssd_smart", "message": f"SMART status: {smart}"}) 101 out["alerts"].append({"severity": "error", "code": "ssd_smart", "message": f"SMART status: {smart}"})
101− except Exception: # noqa: BLE001 102+ except Exception:
102 out["components"]["ssd_smart"] = "unavailable" 103 out["components"]["ssd_smart"] = "unavailable"
103 if any(a["severity"] == "error" for a in out["alerts"]): 104 if any(a["severity"] == "error" for a in out["alerts"]):
104 out["status"] = "degraded" 105 out["status"] = "degraded"
modified services/api/src/findata_api/main.py +21 −9
@@ -9,28 +9,40 @@ from fastapi import Depends, FastAPI, Request
9 from fastapi.exceptions import RequestValidationError 9 from fastapi.exceptions import RequestValidationError
10 from fastapi.responses import JSONResponse, Response 10 from fastapi.responses import JSONResponse, Response
11 from fastapi.staticfiles import StaticFiles 11 from fastapi.staticfiles import StaticFiles
12+from findata_core.config import get_settings
13+from findata_core.logging import configure_logging, get_logger
12 from starlette.exceptions import HTTPException as StarletteHTTPException 14 from starlette.exceptions import HTTPException as StarletteHTTPException
13 from starlette.middleware.base import BaseHTTPMiddleware 15 from starlette.middleware.base import BaseHTTPMiddleware
14 16
15 from findata_api import auth as A 17 from findata_api import auth as A
16 from findata_api import metrics as M 18 from findata_api import metrics as M
17 from findata_api.ratelimit import bucket 19 from findata_api.ratelimit import bucket
18−from findata_api.routers import admin, auth, catalog, coverage, exports, fundamentals, health, instruments, market, quality, query 20+from findata_api.routers import (
19−from findata_core.config import get_settings 21+ admin,
20−from findata_core.logging import configure_logging, get_logger 22+ auth,
23+ catalog,
24+ coverage,
25+ exports,
26+ fundamentals,
27+ health,
28+ instruments,
29+ market,
30+ quality,
31+ query,
32+)
21 33
22 API_PREFIX = "/api/v1" 34 API_PREFIX = "/api/v1"
23 35
24 36
25 class Observability(BaseHTTPMiddleware): 37 class Observability(BaseHTTPMiddleware):
26− async def dispatch(self, request: Request, call_next): # noqa: ANN001 38+ async def dispatch(self, request: Request, call_next):
27 rid = request.headers.get("x-request-id") or uuid.uuid4().hex[:16] 39 rid = request.headers.get("x-request-id") or uuid.uuid4().hex[:16]
28 request.state.request_id = rid 40 request.state.request_id = rid
29 t0 = time.monotonic() 41 t0 = time.monotonic()
30 route = request.url.path 42 route = request.url.path
31 try: 43 try:
32 response = await call_next(request) 44 response = await call_next(request)
33− except Exception: # noqa: BLE001 45+ except Exception:
34 M.REQUESTS.labels(request.method, route, "500").inc() 46 M.REQUESTS.labels(request.method, route, "500").inc()
35 raise 47 raise
36 dt = time.monotonic() - t0 48 dt = time.monotonic() - t0
@@ -60,7 +72,7 @@ def _looks_id(p: str) -> bool:
60 72
61 73
62 class RateLimit(BaseHTTPMiddleware): 74 class RateLimit(BaseHTTPMiddleware):
63− async def dispatch(self, request: Request, call_next): # noqa: ANN001 75+ async def dispatch(self, request: Request, call_next):
64 if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"): 76 if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"):
65 auth_h = request.headers.get("authorization", "") 77 auth_h = request.headers.get("authorization", "")
66 key = auth_h[-24:] if auth_h else (request.cookies.get(A.SESSION_COOKIE, "")[-24:] or (request.client.host if request.client else "anon")) 78 key = auth_h[-24:] if auth_h else (request.cookies.get(A.SESSION_COOKIE, "")[-24:] or (request.client.host if request.client else "anon"))
@@ -82,18 +94,18 @@ def create_app() -> FastAPI:
82 app.add_middleware(RateLimit) 94 app.add_middleware(RateLimit)
83 95
84 @app.exception_handler(StarletteHTTPException) 96 @app.exception_handler(StarletteHTTPException)
85− async def http_exc(request: Request, exc: StarletteHTTPException): # noqa: ANN001 97+ async def http_exc(request: Request, exc: StarletteHTTPException):
86 detail = exc.detail if isinstance(exc.detail, dict) else {"code": "http_error", "message": str(exc.detail)} 98 detail = exc.detail if isinstance(exc.detail, dict) else {"code": "http_error", "message": str(exc.detail)}
87 detail["request_id"] = getattr(request.state, "request_id", None) 99 detail["request_id"] = getattr(request.state, "request_id", None)
88 return JSONResponse({"error": detail}, status_code=exc.status_code, headers=getattr(exc, "headers", None)) 100 return JSONResponse({"error": detail}, status_code=exc.status_code, headers=getattr(exc, "headers", None))
89 101
90 @app.exception_handler(RequestValidationError) 102 @app.exception_handler(RequestValidationError)
91− async def val_exc(request: Request, exc: RequestValidationError): # noqa: ANN001 103+ async def val_exc(request: Request, exc: RequestValidationError):
92 return JSONResponse({"error": {"code": "validation_error", "message": "Invalid request", "details": exc.errors(), 104 return JSONResponse({"error": {"code": "validation_error", "message": "Invalid request", "details": exc.errors(),
93 "request_id": getattr(request.state, "request_id", None)}}, status_code=422) 105 "request_id": getattr(request.state, "request_id", None)}}, status_code=422)
94 106
95 @app.exception_handler(Exception) 107 @app.exception_handler(Exception)
96− async def any_exc(request: Request, exc: Exception): # noqa: ANN001 108+ async def any_exc(request: Request, exc: Exception):
97 from findata_core.redact import redact 109 from findata_core.redact import redact
98 log.error("unhandled", error=redact(repr(exc), s.secret_values()), path=request.url.path) 110 log.error("unhandled", error=redact(repr(exc), s.secret_values()), path=request.url.path)
99 return JSONResponse({"error": {"code": "internal_error", "message": "Internal error", "request_id": getattr(request.state, "request_id", None)}}, status_code=500) 111 return JSONResponse({"error": {"code": "internal_error", "message": "Internal error", "request_id": getattr(request.state, "request_id", None)}}, status_code=500)
modified services/api/src/findata_api/routers/admin.py +5 −5
@@ -5,14 +5,14 @@ from datetime import datetime
5 from typing import Any 5 from typing import Any
6 6
7 from fastapi import APIRouter, Depends, Query, Request 7 from fastapi import APIRouter, Depends, Query, Request
8−from pydantic import BaseModel, Field
9−
10−from findata_api import auth as A
11−from findata_api.errors import ApiError
12 from findata_core.config import get_settings 8 from findata_core.config import get_settings
13 from findata_core.db import connection, transaction 9 from findata_core.db import connection, transaction
14 from findata_ingest import queue 10 from findata_ingest import queue
15 from findata_ingest.lake import Lake 11 from findata_ingest.lake import Lake
12+from pydantic import BaseModel, Field
13+
14+from findata_api import auth as A
15+from findata_api.errors import ApiError
16 16
17 router = APIRouter(prefix="/admin", tags=["admin"]) 17 router = APIRouter(prefix="/admin", tags=["admin"])
18 18
@@ -155,8 +155,8 @@ def tracked(body: TrackedIn, request: Request, p: A.Principal = Depends(A.requir
155 @router.post("/discover") 155 @router.post("/discover")
156 def discover(request: Request, p: A.Principal = Depends(A.require("admin"))): 156 def discover(request: Request, p: A.Principal = Depends(A.require("admin"))):
157 """Sanitized live vendor probe (no bulk download).""" 157 """Sanitized live vendor probe (no bulk download)."""
158− from findata_market_firstrate import FirstRateMarketConnector
159 from findata_fundamentals import SecEdgarConnector 158 from findata_fundamentals import SecEdgarConnector
159+ from findata_market_firstrate import FirstRateMarketConnector
160 out = {"firstrate": FirstRateMarketConnector().discover(), "sec_edgar": SecEdgarConnector().status()} 160 out = {"firstrate": FirstRateMarketConnector().discover(), "sec_edgar": SecEdgarConnector().status()}
161 A.audit(request, "admin.discover") 161 A.audit(request, "admin.discover")
162 return out 162 return out
modified services/api/src/findata_api/routers/auth.py +2 −2
@@ -1,12 +1,12 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3 from fastapi import APIRouter, Depends, Request, Response 3 from fastapi import APIRouter, Depends, Request, Response
4+from findata_core.config import get_settings
5+from findata_core.db import connection
4 from pydantic import BaseModel, EmailStr 6 from pydantic import BaseModel, EmailStr
5 7
6 from findata_api import auth as A 8 from findata_api import auth as A
7 from findata_api.errors import ApiError 9 from findata_api.errors import ApiError
8−from findata_core.config import get_settings
9−from findata_core.db import connection
10 10
11 router = APIRouter(prefix="/auth", tags=["auth"]) 11 router = APIRouter(prefix="/auth", tags=["auth"])
12 12
modified services/api/src/findata_api/routers/catalog.py +1 −1
@@ -1,9 +1,9 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3 from fastapi import APIRouter, Depends 3 from fastapi import APIRouter, Depends
4+from findata_core.db import connection
4 5
5 from findata_api import auth as A 6 from findata_api import auth as A
6−from findata_core.db import connection
7 from findata_api.errors import ApiError 7 from findata_api.errors import ApiError
8 8
9 router = APIRouter(prefix="/catalog", tags=["catalog"]) 9 router = APIRouter(prefix="/catalog", tags=["catalog"])
modified services/api/src/findata_api/routers/coverage.py +1 −1
@@ -3,9 +3,9 @@ from __future__ import annotations
3 from datetime import date, timedelta 3 from datetime import date, timedelta
4 4
5 from fastapi import APIRouter, Depends, Query 5 from fastapi import APIRouter, Depends, Query
6+from findata_core.db import connection
6 7
7 from findata_api import auth as A 8 from findata_api import auth as A
8−from findata_core.db import connection
9 9
10 router = APIRouter(prefix="/coverage", tags=["coverage"]) 10 router = APIRouter(prefix="/coverage", tags=["coverage"])
11 11
modified services/api/src/findata_api/routers/exports.py +5 −6
@@ -3,7 +3,6 @@ from lake/exports with expiration. Dataset snapshot (version) is pinned at submi
3 from __future__ import annotations 3 from __future__ import annotations
4 4
5 import json 5 import json
6−import threading
7 import traceback 6 import traceback
8 from concurrent.futures import ThreadPoolExecutor 7 from concurrent.futures import ThreadPoolExecutor
9 from datetime import UTC, date, datetime, timedelta 8 from datetime import UTC, date, datetime, timedelta
@@ -12,15 +11,15 @@ from typing import Literal
12 import pyarrow.parquet as pq 11 import pyarrow.parquet as pq
13 from fastapi import APIRouter, Depends, Request 12 from fastapi import APIRouter, Depends, Request
14 from fastapi.responses import FileResponse 13 from fastapi.responses import FileResponse
14+from findata_core.config import get_settings
15+from findata_core.db import connection, transaction
16+from findata_ingest.lake import Lake
17+from findata_ingest.reader import apply_adjustment, read_bars
15 from pydantic import BaseModel, Field 18 from pydantic import BaseModel, Field
16 19
17 from findata_api import auth as A 20 from findata_api import auth as A
18 from findata_api.errors import ApiError 21 from findata_api.errors import ApiError
19 from findata_api.ratelimit import record_usage 22 from findata_api.ratelimit import record_usage
20−from findata_core.config import get_settings
21−from findata_core.db import connection, transaction
22−from findata_ingest.lake import Lake
23−from findata_ingest.reader import apply_adjustment, read_bars
24 23
25 router = APIRouter(prefix="/exports", tags=["exports"]) 24 router = APIRouter(prefix="/exports", tags=["exports"])
26 _pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="export") 25 _pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="export")
@@ -58,7 +57,7 @@ def _run(export_id: str, req: dict, principal: str) -> None:
58 (rel, out.stat().st_size, table.num_rows, datetime.now(UTC) + timedelta(hours=ttl), 57 (rel, out.stat().st_size, table.num_rows, datetime.now(UTC) + timedelta(hours=ttl),
59 json.dumps({"dataset_id": req["dataset_id"], "version": meta.get("version"), "files_scanned": meta.get("files_scanned")}), export_id)) 58 json.dumps({"dataset_id": req["dataset_id"], "version": meta.get("version"), "files_scanned": meta.get("files_scanned")}), export_id))
60 record_usage(principal, rows=table.num_rows, bytes_exported=out.stat().st_size) 59 record_usage(principal, rows=table.num_rows, bytes_exported=out.stat().st_size)
61− except Exception as e: # noqa: BLE001 60+ except Exception as e:
62 with transaction() as conn: 61 with transaction() as conn:
63 conn.execute("UPDATE exports SET status='failed', error=%s, finished_at=now() WHERE id=%s", (f"{e}\n{traceback.format_exc()[-800:]}"[:2000], export_id)) 62 conn.execute("UPDATE exports SET status='failed', error=%s, finished_at=now() WHERE id=%s", (f"{e}\n{traceback.format_exc()[-800:]}"[:2000], export_id))
64 63
modified services/api/src/findata_api/routers/fundamentals.py +1 −2
@@ -1,13 +1,12 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3 from datetime import UTC, datetime 3 from datetime import UTC, datetime
4−from typing import Any
5 4
6 from fastapi import APIRouter, Depends, Query 5 from fastapi import APIRouter, Depends, Query
6+from findata_core.db import connection
7 7
8 from findata_api import auth as A 8 from findata_api import auth as A
9 from findata_api.errors import ApiError 9 from findata_api.errors import ApiError
10−from findata_core.db import connection
11 10
12 router = APIRouter(prefix="/fundamentals", tags=["fundamentals"]) 11 router = APIRouter(prefix="/fundamentals", tags=["fundamentals"])
13 12
modified services/api/src/findata_api/routers/health.py +2 −4
@@ -1,12 +1,10 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3−from fastapi import APIRouter, Depends, Request, Response 3+from fastapi import APIRouter, Depends
4+from findata_core.db import connection
4 5
5 from findata_api import auth as A 6 from findata_api import auth as A
6−from findata_api.errors import ApiError
7 from findata_api.health import evaluate 7 from findata_api.health import evaluate
8−from findata_api.metrics import render
9−from findata_core.db import connection
10 8
11 router = APIRouter(tags=["health"]) 9 router = APIRouter(tags=["health"])
12 10
modified services/api/src/findata_api/routers/instruments.py +1 −1
@@ -1,9 +1,9 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3 from fastapi import APIRouter, Depends, Query 3 from fastapi import APIRouter, Depends, Query
4+from findata_core.db import connection
4 5
5 from findata_api import auth as A 6 from findata_api import auth as A
6−from findata_core.db import connection
7 7
8 router = APIRouter(prefix="/instruments", tags=["instruments"]) 8 router = APIRouter(prefix="/instruments", tags=["instruments"])
9 9
modified services/api/src/findata_api/routers/market.py +4 −5
@@ -6,17 +6,16 @@ from datetime import date, datetime
6 from typing import Literal 6 from typing import Literal
7 7
8 import pyarrow as pa 8 import pyarrow as pa
9−import pyarrow.ipc as ipc
10 from fastapi import APIRouter, Depends, Query, Request, Response 9 from fastapi import APIRouter, Depends, Query, Request, Response
11 from fastapi.responses import StreamingResponse 10 from fastapi.responses import StreamingResponse
11+from findata_core.db import connection
12+from findata_ingest.reader import apply_adjustment, read_bars
13+from pyarrow import ipc
12 14
13 from findata_api import auth as A 15 from findata_api import auth as A
14 from findata_api import metrics as M 16 from findata_api import metrics as M
15 from findata_api.errors import ApiError 17 from findata_api.errors import ApiError
16 from findata_api.ratelimit import record_usage 18 from findata_api.ratelimit import record_usage
17−from findata_core.config import get_settings
18−from findata_core.db import connection
19−from findata_ingest.reader import apply_adjustment, read_bars
20 19
21 router = APIRouter(prefix="/market", tags=["market"]) 20 router = APIRouter(prefix="/market", tags=["market"])
22 INTERVAL_DS = {"1m": "market_bars.{a}.1m.raw", "5m": "market_bars.{a}.5m.raw", "1d": "market_bars.{a}.1d.raw"} 21 INTERVAL_DS = {"1m": "market_bars.{a}.1m.raw", "5m": "market_bars.{a}.5m.raw", "1d": "market_bars.{a}.1d.raw"}
@@ -55,7 +54,7 @@ def bars(request: Request, response: Response,
55 if cursor: 54 if cursor:
56 try: 55 try:
57 after = datetime.fromisoformat(base64.urlsafe_b64decode(cursor.encode()).decode()) 56 after = datetime.fromisoformat(base64.urlsafe_b64decode(cursor.encode()).decode())
58− except Exception: # noqa: BLE001 57+ except Exception:
59 raise ApiError(400, "bad_cursor", "invalid cursor") from None 58 raise ApiError(400, "bad_cursor", "invalid cursor") from None
60 import pyarrow.compute as pc 59 import pyarrow.compute as pc
61 table = table.filter(pc.greater(table.column("timestamp_utc"), pa.scalar(after, pa.timestamp("us", tz="UTC")))) 60 table = table.filter(pc.greater(table.column("timestamp_utc"), pa.scalar(after, pa.timestamp("us", tz="UTC"))))
modified services/api/src/findata_api/routers/quality.py +1 −1
@@ -1,9 +1,9 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3 from fastapi import APIRouter, Depends, Query 3 from fastapi import APIRouter, Depends, Query
4+from findata_core.db import connection
4 5
5 from findata_api import auth as A 6 from findata_api import auth as A
6−from findata_core.db import connection
7 7
8 router = APIRouter(prefix="/quality", tags=["quality"]) 8 router = APIRouter(prefix="/quality", tags=["quality"])
9 9
modified services/api/src/findata_api/routers/query.py +3 −3
@@ -3,14 +3,14 @@ from __future__ import annotations
3 import time 3 import time
4 4
5 from fastapi import APIRouter, Depends, Request 5 from fastapi import APIRouter, Depends, Request
6+from findata_core.config import get_settings
7+from findata_ingest.reader import dataset_views, run_sql
6 from pydantic import BaseModel, Field 8 from pydantic import BaseModel, Field
7 9
8 from findata_api import auth as A 10 from findata_api import auth as A
9 from findata_api import metrics as M 11 from findata_api import metrics as M
10 from findata_api.errors import ApiError 12 from findata_api.errors import ApiError
11 from findata_api.ratelimit import record_usage 13 from findata_api.ratelimit import record_usage
12−from findata_core.config import get_settings
13−from findata_ingest.reader import dataset_views, run_sql
14 14
15 router = APIRouter(prefix="/query", tags=["query"]) 15 router = APIRouter(prefix="/query", tags=["query"])
16 16
@@ -31,7 +31,7 @@ def query(body: QueryIn, request: Request, p: A.Principal = Depends(A.require("q
31 raise ApiError(400, "query_rejected", str(e)) from None 31 raise ApiError(400, "query_rejected", str(e)) from None
32 except TimeoutError as e: 32 except TimeoutError as e:
33 raise ApiError(408, "query_timeout", str(e)) from None 33 raise ApiError(408, "query_timeout", str(e)) from None
34− except Exception as e: # noqa: BLE001 34+ except Exception as e:
35 raise ApiError(400, "query_error", str(e)[:500]) from None 35 raise ApiError(400, "query_error", str(e)[:500]) from None
36 rows = tbl.to_pylist() 36 rows = tbl.to_pylist()
37 for r in rows: 37 for r in rows:
modified services/ingest/src/findata_ingest/cli.py +7 −6
@@ -3,7 +3,6 @@ from __future__ import annotations
3 3
4 import json 4 import json
5 from pathlib import Path 5 from pathlib import Path
6−from typing import Optional
7 6
8 import typer 7 import typer
9 8
@@ -37,7 +36,7 @@ def discover(source: str = "all") -> None:
37 36
38 37
39 @app.command() 38 @app.command()
40−def worker(once: bool = False, no_scheduler: bool = False, kinds: Optional[str] = None) -> None: 39+def worker(once: bool = False, no_scheduler: bool = False, kinds: str | None = None) -> None:
41 """Run the single ingest worker (+ scheduler).""" 40 """Run the single ingest worker (+ scheduler)."""
42 from findata_ingest.worker import main 41 from findata_ingest.worker import main
43 main(once=once, kinds=kinds.split(",") if kinds else None, with_scheduler=not no_scheduler) 42 main(once=once, kinds=kinds.split(",") if kinds else None, with_scheduler=not no_scheduler)
@@ -60,7 +59,7 @@ def ingest_incremental(type_: str = typer.Option("stock", "--type"), period: str
60 59
61 @ingest_app.command("backfill") 60 @ingest_app.command("backfill")
62 def ingest_backfill(ticker_range: str, type_: str = typer.Option("stock", "--type"), timeframe: str = "1min", 61 def ingest_backfill(ticker_range: str, type_: str = typer.Option("stock", "--type"), timeframe: str = "1min",
63− adjustment: str = "UNADJUSTED", symbols: Optional[str] = None, max_symbols: int = 0) -> None: 62+ adjustment: str = "UNADJUSTED", symbols: str | None = None, max_symbols: int = 0) -> None:
64 """Bounded backfill of one ticker_range letter (full archive). Disk budget enforced by the lake.""" 63 """Bounded backfill of one ticker_range letter (full archive). Disk budget enforced by the lake."""
65 from findata_ingest import queue 64 from findata_ingest import queue
66 from findata_ingest.tasks.common import DATASET_ASSET 65 from findata_ingest.tasks.common import DATASET_ASSET
@@ -78,7 +77,7 @@ def ingest_fundamentals(tickers: str) -> None:
78 77
79 78
80 @ingest_app.command("derive") 79 @ingest_app.command("derive")
81−def ingest_derive(dates: Optional[str] = None, source: str = "market_bars.stock.1m.raw") -> None: 80+def ingest_derive(dates: str | None = None, source: str = "market_bars.stock.1m.raw") -> None:
82 from findata_ingest import queue 81 from findata_ingest import queue
83 typer.echo(queue.enqueue("derive", source.replace(".1m.", ".5m."), {"source_dataset": source, "target_interval": "5m", "dates": dates.split(",") if dates else []}, created_by="cli")) 82 typer.echo(queue.enqueue("derive", source.replace(".1m.", ".5m."), {"source_dataset": source, "target_interval": "5m", "dates": dates.split(",") if dates else []}, created_by="cli"))
84 83
@@ -93,7 +92,7 @@ def ingest_compaction(dataset: str = "market_bars.stock.1m.raw", min_age_days: i
93 def ingest_jobs(limit: int = 20) -> None: 92 def ingest_jobs(limit: int = 20) -> None:
94 from findata_core.db import fetch_all 93 from findata_core.db import fetch_all
95 for r in fetch_all("SELECT id, kind, dataset_id, status, attempts, created_at, finished_at, left(last_error, 80) err FROM ingest_jobs ORDER BY created_at DESC LIMIT %s", (limit,)): 94 for r in fetch_all("SELECT id, kind, dataset_id, status, attempts, created_at, finished_at, left(last_error, 80) err FROM ingest_jobs ORDER BY created_at DESC LIMIT %s", (limit,)):
96− typer.echo(f"{str(r['id'])[:8]} {r['kind']:<12} {str(r['dataset_id']):<28} {r['status']:<9} a={r['attempts']} {r['created_at']:%m-%d %H:%M} {r['err'] or ''}") 95+ typer.echo(f"{str(r['id'])[:8]} {r['kind']:<12} {r['dataset_id']!s:<28} {r['status']:<9} a={r['attempts']} {r['created_at']:%m-%d %H:%M} {r['err'] or ''}")
97 96
98 97
99 @app.command() 98 @app.command()
@@ -105,12 +104,13 @@ def backup(now: bool = True) -> None:
105 @app.command("restore-drill") 104 @app.command("restore-drill")
106 def restore_drill(dest: Path = Path("/tmp/findata-restore-drill"), dataset: str = "market_bars.stock.1m.raw") -> None: 105 def restore_drill(dest: Path = Path("/tmp/findata-restore-drill"), dataset: str = "market_bars.stock.1m.raw") -> None:
107 from findata_core.config import get_settings 106 from findata_core.config import get_settings
107+
108 from findata_ingest.tasks.backup import restore_drill as rd 108 from findata_ingest.tasks.backup import restore_drill as rd
109 typer.echo(json.dumps(rd(get_settings().backup_target, dest, dataset), indent=2, default=str)) 109 typer.echo(json.dumps(rd(get_settings().backup_target, dest, dataset), indent=2, default=str))
110 110
111 111
112 @app.command("create-owner") 112 @app.command("create-owner")
113−def create_owner(email: str, password: Optional[str] = None) -> None: 113+def create_owner(email: str, password: str | None = None) -> None:
114 """Create (or reset the password of) the owner account; prints the TOTP provisioning URI.""" 114 """Create (or reset the password of) the owner account; prints the TOTP provisioning URI."""
115 from findata_api.auth import create_owner as _co 115 from findata_api.auth import create_owner as _co
116 typer.echo(json.dumps(_co(email, password), indent=2)) 116 typer.echo(json.dumps(_co(email, password), indent=2))
@@ -125,6 +125,7 @@ def create_key(email: str, name: str, scopes: str = "read:market,read:fundamenta
125 @app.command() 125 @app.command()
126 def status() -> None: 126 def status() -> None:
127 from findata_core.db import fetch_all, fetch_one 127 from findata_core.db import fetch_all, fetch_one
128+
128 from findata_ingest.lake import Lake 129 from findata_ingest.lake import Lake
129 lake = Lake() 130 lake = Lake()
130 typer.echo(f"lake {lake.root} free {lake.free_gb():.1f} GB curated {lake.used_bytes('curated')/1e9:.2f} GB raw {lake.used_bytes('raw')/1e9:.2f} GB") 131 typer.echo(f"lake {lake.root} free {lake.free_gb():.1f} GB curated {lake.used_bytes('curated')/1e9:.2f} GB raw {lake.used_bytes('raw')/1e9:.2f} GB")
modified services/ingest/src/findata_ingest/lake.py +0 −1
@@ -27,7 +27,6 @@ from typing import Any
27 27
28 import pyarrow as pa 28 import pyarrow as pa
29 import pyarrow.parquet as pq 29 import pyarrow.parquet as pq
30−
31 from findata_core.config import get_settings 30 from findata_core.config import get_settings
32 from findata_core.db import transaction 31 from findata_core.db import transaction
33 32
modified services/ingest/src/findata_ingest/queue.py +2 −2
@@ -8,9 +8,9 @@ import socket
8 from datetime import UTC, datetime, timedelta 8 from datetime import UTC, datetime, timedelta
9 from typing import Any 9 from typing import Any
10 10
11+from findata_core.config import get_settings
11 from findata_core.db import connection, transaction 12 from findata_core.db import connection, transaction
12 from findata_core.redact import redact 13 from findata_core.redact import redact
13−from findata_core.config import get_settings
14 14
15 WORKER_ID = f"{socket.gethostname()}:{random.randint(1000, 9999)}" 15 WORKER_ID = f"{socket.gethostname()}:{random.randint(1000, 9999)}"
16 16
@@ -104,5 +104,5 @@ def _code_version() -> str:
104 try: 104 try:
105 return subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5, 105 return subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5,
106 cwd=str(get_settings().root)).stdout.strip() or "unknown" 106 cwd=str(get_settings().root)).stdout.strip() or "unknown"
107− except Exception: # noqa: BLE001 107+ except Exception:
108 return "unknown" 108 return "unknown"
modified services/ingest/src/findata_ingest/reader.py +4 −5
@@ -13,12 +13,11 @@ from typing import Any
13 13
14 import duckdb 14 import duckdb
15 import pyarrow as pa 15 import pyarrow as pa
16−
17−from findata_core.config import get_settings
18 from findata_core.db import connection 16 from findata_core.db import connection
19−from findata_ingest.lake import Lake, live_files
20 from findata_schema import BARS_SCHEMA 17 from findata_schema import BARS_SCHEMA
21 18
19+from findata_ingest.lake import Lake, live_files
20+
22 21
23 def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection: 22 def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection:
24 con = duckdb.connect(database=":memory:", config={"threads": 8, "memory_limit": "8GB", "TimeZone": "UTC"}) 23 con = duckdb.connect(database=":memory:", config={"threads": 8, "memory_limit": "8GB", "TimeZone": "UTC"})
@@ -120,10 +119,10 @@ def run_sql(sql: str, max_rows: int, timeout_s: int, views: dict[str, list[str]]
120 import re 119 import re
121 import threading 120 import threading
122 121
123− forbidden = re.compile(r"\b(attach|copy|export|import|install|load|create|insert|update|delete|drop|alter|pragma|set|call|read_csv|read_json|read_text|glob|httpfs)\b", re.I) 122+ forbidden = re.compile(r"\b(attach|copy|export|import|install|load|create|insert|update|delete|drop|alter|pragma|set|call|read_csv|read_json|read_text|glob|httpfs)\b", re.IGNORECASE)
124 if forbidden.search(sql) or ";" in sql.strip().rstrip(";"): 123 if forbidden.search(sql) or ";" in sql.strip().rstrip(";"):
125 raise ValueError("only a single read-only SELECT statement is allowed") 124 raise ValueError("only a single read-only SELECT statement is allowed")
126− if not re.match(r"^\s*(select|with|describe|show|summarize)\b", sql, re.I): 125+ if not re.match(r"^\s*(select|with|describe|show|summarize)\b", sql, re.IGNORECASE):
127 raise ValueError("statement must start with SELECT / WITH") 126 raise ValueError("statement must start with SELECT / WITH")
128 con = _con() 127 con = _con()
129 lake_root = str(Lake().root).replace("'", "''") 128 lake_root = str(Lake().root).replace("'", "''")
modified services/ingest/src/findata_ingest/tasks/__init__.py +10 −1
@@ -1,4 +1,13 @@
1−from findata_ingest.tasks import backfill, backup, compaction, derive, fundamentals, incremental, metadata, quality # noqa: F401 1+from findata_ingest.tasks import (
2+ backfill,
3+ backup,
4+ compaction,
5+ derive,
6+ fundamentals,
7+ incremental,
8+ metadata,
9+ quality,
10+)
2 11
3 REGISTRY = { 12 REGISTRY = {
4 "metadata": metadata.run, 13 "metadata": metadata.run,
modified services/ingest/src/findata_ingest/tasks/backfill.py +13 −5
@@ -10,15 +10,22 @@ from typing import Any
10 10
11 import pyarrow as pa 11 import pyarrow as pa
12 import pyarrow.compute as pc 12 import pyarrow.compute as pc
13−
14 from findata_core.db import connection, transaction 13 from findata_core.db import connection, transaction
15−from findata_ingest import queue
16−from findata_ingest.lake import Lake
17−from findata_ingest.tasks.common import DATASET_ASSET, live_delta_dates, record_anomalies, record_quality_report, record_raw_archive, upsert_coverage
18 from findata_market_firstrate import FirstRateMarketConnector 14 from findata_market_firstrate import FirstRateMarketConnector
19 from findata_schema import Interval 15 from findata_schema import Interval
20 from findata_schema.connector import FetchRequest 16 from findata_schema.connector import FetchRequest
21 17
18+from findata_ingest import queue
19+from findata_ingest.lake import Lake
20+from findata_ingest.tasks.common import (
21+ DATASET_ASSET,
22+ live_delta_dates,
23+ record_anomalies,
24+ record_quality_report,
25+ record_raw_archive,
26+ upsert_coverage,
27+)
28+
22 29
23 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]: 30 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
24 p = job.get("params") or {} 31 p = job.get("params") or {}
@@ -37,8 +44,9 @@ def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
37 done = set(ck.get("done_symbols") or []) 44 done = set(ck.get("done_symbols") or [])
38 raw_rel = ck.get("raw_path") 45 raw_rel = ck.get("raw_path")
39 if raw_rel and (lake.root / raw_rel).exists(): 46 if raw_rel and (lake.root / raw_rel).exists():
40− from findata_schema.connector import FetchResult
41 from datetime import datetime 47 from datetime import datetime
48+
49+ from findata_schema.connector import FetchResult
42 res = FetchResult(request=FetchRequest(dataset_id, {"type": asset, "period": "full", "ticker_range": letter, "timeframe": timeframe, "adjustment": adjustment}), 50 res = FetchResult(request=FetchRequest(dataset_id, {"type": asset, "period": "full", "ticker_range": letter, "timeframe": timeframe, "adjustment": adjustment}),
43 path=lake.root / raw_rel, sha256=ck["sha256"], bytes=ck["bytes"], fetched_at=datetime.fromisoformat(ck["fetched_at"]), 51 path=lake.root / raw_rel, sha256=ck["sha256"], bytes=ck["bytes"], fetched_at=datetime.fromisoformat(ck["fetched_at"]),
44 vendor_last_update=None) 52 vendor_last_update=None)
modified services/ingest/src/findata_ingest/tasks/backup.py +1 −0
@@ -10,6 +10,7 @@ from typing import Any
10 10
11 from findata_core.config import get_settings 11 from findata_core.config import get_settings
12 from findata_core.db import transaction 12 from findata_core.db import transaction
13+
13 from findata_ingest.lake import Lake 14 from findata_ingest.lake import Lake
14 15
15 16
modified services/ingest/src/findata_ingest/tasks/common.py +0 −3
@@ -5,9 +5,6 @@ from datetime import UTC, date, datetime
5 from typing import Any 5 from typing import Any
6 6
7 import pyarrow as pa 7 import pyarrow as pa
8−
9−from findata_core.db import transaction
10−from findata_ingest.lake import Lake, WrittenFile
11 from findata_quality.validators import session_coverage 8 from findata_quality.validators import session_coverage
12 from findata_schema import Interval 9 from findata_schema import Interval
13 10
modified services/ingest/src/findata_ingest/tasks/compaction.py +2 −2
@@ -7,11 +7,11 @@ from typing import Any
7 7
8 import duckdb 8 import duckdb
9 import pyarrow as pa 9 import pyarrow as pa
10−
11 from findata_core.db import connection 10 from findata_core.db import connection
12−from findata_ingest.lake import Lake
13 from findata_schema import BARS_SCHEMA 11 from findata_schema import BARS_SCHEMA
14 12
13+from findata_ingest.lake import Lake
14+
15 15
16 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]: 16 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
17 p = job.get("params") or {} 17 p = job.get("params") or {}
modified services/ingest/src/findata_ingest/tasks/derive.py +3 −4
@@ -4,14 +4,13 @@ from __future__ import annotations
4 from datetime import date 4 from datetime import date
5 from typing import Any 5 from typing import Any
6 6
7−import pyarrow as pa
8−
9 from findata_core.db import connection, transaction 7 from findata_core.db import connection, transaction
8+from findata_quality import aggregate_bars
9+from findata_schema import Interval
10+
10 from findata_ingest.lake import Lake, live_files 11 from findata_ingest.lake import Lake, live_files
11 from findata_ingest.reader import read_bars 12 from findata_ingest.reader import read_bars
12 from findata_ingest.tasks.common import live_delta_paths_for_dates, upsert_coverage 13 from findata_ingest.tasks.common import live_delta_paths_for_dates, upsert_coverage
13−from findata_quality import aggregate_bars
14−from findata_schema import Interval
15 14
16 15
17 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]: 16 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
modified services/ingest/src/findata_ingest/tasks/fundamentals.py +2 −1
@@ -8,6 +8,7 @@ from typing import Any
8 8
9 from findata_core.db import connection, transaction 9 from findata_core.db import connection, transaction
10 from findata_fundamentals import FirstRateFundamentalsConnector, facts_from_financials 10 from findata_fundamentals import FirstRateFundamentalsConnector, facts_from_financials
11+
11 from findata_ingest import queue 12 from findata_ingest import queue
12 from findata_ingest.lake import Lake 13 from findata_ingest.lake import Lake
13 14
@@ -34,7 +35,7 @@ def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
34 payload = fr.financials_all(tk, period) 35 payload = fr.financials_all(tk, period)
35 facts.extend(facts_from_financials(payload, observed, tk)) 36 facts.extend(facts_from_financials(payload, observed, tk))
36 earnings = fr.earnings(tk) 37 earnings = fr.earnings(tk)
37− except Exception as e: # noqa: BLE001 38+ except Exception as e:
38 stats["errors"] += 1 39 stats["errors"] += 1
39 log.warning("fundamentals.error", ticker=tk, error=str(e)[:200]) 40 log.warning("fundamentals.error", ticker=tk, error=str(e)[:200])
40 continue 41 continue
modified services/ingest/src/findata_ingest/tasks/incremental.py +11 −6
@@ -7,20 +7,25 @@ Checkpoints: dataset:<id> / last_archive, last_vendor_update.
7 from __future__ import annotations 7 from __future__ import annotations
8 8
9 from collections import defaultdict 9 from collections import defaultdict
10−from datetime import UTC, datetime
11 from typing import Any 10 from typing import Any
12 11
13 import pyarrow as pa 12 import pyarrow as pa
14−
15 from findata_core.db import connection, transaction 13 from findata_core.db import connection, transaction
16−from findata_ingest import queue
17−from findata_ingest.lake import Lake
18−from findata_ingest.tasks.common import (DATASET_ASSET, live_delta_paths_for_dates, record_anomalies, record_quality_report,
19− record_raw_archive, upsert_coverage)
20 from findata_market_firstrate import FirstRateMarketConnector 14 from findata_market_firstrate import FirstRateMarketConnector
21 from findata_schema import Interval 15 from findata_schema import Interval
22 from findata_schema.connector import FetchRequest 16 from findata_schema.connector import FetchRequest
23 17
18+from findata_ingest import queue
19+from findata_ingest.lake import Lake
20+from findata_ingest.tasks.common import (
21+ DATASET_ASSET,
22+ live_delta_paths_for_dates,
23+ record_anomalies,
24+ record_quality_report,
25+ record_raw_archive,
26+ upsert_coverage,
27+)
28+
24 29
25 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]: 30 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
26 p = job.get("params") or {} 31 p = job.get("params") or {}
modified services/ingest/src/findata_ingest/tasks/metadata.py +2 −1
@@ -4,9 +4,10 @@ from __future__ import annotations
4 from typing import Any 4 from typing import Any
5 5
6 from findata_core.db import transaction 6 from findata_core.db import transaction
7+from findata_market_firstrate import FirstRateMarketConnector
8+
7 from findata_ingest.lake import Lake 9 from findata_ingest.lake import Lake
8 from findata_ingest.tasks.common import record_raw_archive 10 from findata_ingest.tasks.common import record_raw_archive
9−from findata_market_firstrate import FirstRateMarketConnector
10 11
11 12
12 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]: 13 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
modified services/ingest/src/findata_ingest/tasks/quality.py +1 −1
@@ -2,10 +2,10 @@
2 from __future__ import annotations 2 from __future__ import annotations
3 3
4 import json 4 import json
5−from datetime import UTC, datetime, timedelta
6 from typing import Any 5 from typing import Any
7 6
8 from findata_core.db import connection, transaction 7 from findata_core.db import connection, transaction
8+
9 from findata_ingest.lake import Lake 9 from findata_ingest.lake import Lake
10 10
11 11
modified services/ingest/src/findata_ingest/worker.py +5 −4
@@ -5,11 +5,12 @@ from __future__ import annotations
5 import signal 5 import signal
6 import time 6 import time
7 import traceback 7 import traceback
8−from datetime import UTC, datetime 8+from datetime import datetime
9 from zoneinfo import ZoneInfo 9 from zoneinfo import ZoneInfo
10 10
11 from findata_core.config import get_settings 11 from findata_core.config import get_settings
12 from findata_core.logging import configure_logging, get_logger 12 from findata_core.logging import configure_logging, get_logger
13+
13 from findata_ingest import queue 14 from findata_ingest import queue
14 from findata_ingest.lake import Lake, WriterLock 15 from findata_ingest.lake import Lake, WriterLock
15 from findata_ingest.tasks import REGISTRY 16 from findata_ingest.tasks import REGISTRY
@@ -18,7 +19,7 @@ ET = ZoneInfo("America/New_York")
18 _stop = False 19 _stop = False
19 20
20 21
21−def _handle(sig, frame): # noqa: ANN001 22+def _handle(sig, frame):
22 global _stop 23 global _stop
23 _stop = True 24 _stop = True
24 25
@@ -63,7 +64,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
63 if with_scheduler and time.monotonic() - last_tick > 300: 64 if with_scheduler and time.monotonic() - last_tick > 300:
64 try: 65 try:
65 schedule_tick(log) 66 schedule_tick(log)
66− except Exception as e: # noqa: BLE001 67+ except Exception as e:
67 log.error("scheduler.error", error=str(e)) 68 log.error("scheduler.error", error=str(e))
68 last_tick = time.monotonic() 69 last_tick = time.monotonic()
69 job = queue.claim_next(kinds) 70 job = queue.claim_next(kinds)
@@ -80,7 +81,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
80 result = fn(job, lake, jl) 81 result = fn(job, lake, jl)
81 queue.finish(job, True, {**(result or {}), "seconds": round(time.monotonic() - t0, 1)}) 82 queue.finish(job, True, {**(result or {}), "seconds": round(time.monotonic() - t0, 1)})
82 jl.info("job.done", seconds=round(time.monotonic() - t0, 1)) 83 jl.info("job.done", seconds=round(time.monotonic() - t0, 1))
83− except Exception as e: # noqa: BLE001 84+ except Exception as e:
84 err = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}" 85 err = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}"
85 queue.finish(job, False, error=err) 86 queue.finish(job, False, error=err)
86 jl.error("job.failed", error=str(e)[:300], seconds=round(time.monotonic() - t0, 1)) 87 jl.error("job.failed", error=str(e)[:300], seconds=round(time.monotonic() - t0, 1))
modified services/quality/src/findata_quality/__init__.py +2 −2
@@ -1,3 +1,3 @@
1−from findata_quality.calendar import SessionCalendar, ET # noqa: F401
2−from findata_quality.validators import validate_bars # noqa: F401
3 from findata_quality.aggregate import aggregate_bars # noqa: F401 1 from findata_quality.aggregate import aggregate_bars # noqa: F401
2+from findata_quality.calendar import ET, SessionCalendar # noqa: F401
3+from findata_quality.validators import validate_bars # noqa: F401
modified services/quality/src/findata_quality/aggregate.py +0 −1
@@ -4,7 +4,6 @@ from __future__ import annotations
4 4
5 import polars as pl 5 import polars as pl
6 import pyarrow as pa 6 import pyarrow as pa
7−
8 from findata_schema import BARS_SCHEMA, INTERVAL_SECONDS, Interval 7 from findata_schema import BARS_SCHEMA, INTERVAL_SECONDS, Interval
9 8
10 9
modified services/quality/src/findata_quality/validators.py +1 −1
@@ -6,9 +6,9 @@ from typing import Any
6 6
7 import polars as pl 7 import polars as pl
8 import pyarrow as pa 8 import pyarrow as pa
9+from findata_schema import INTERVAL_SECONDS, Interval, QualityFlag
9 10
10 from findata_quality.calendar import SessionCalendar, xnys 11 from findata_quality.calendar import SessionCalendar, xnys
11−from findata_schema import INTERVAL_SECONDS, Interval, QualityFlag
12 12
13 PRICE_JUMP_THRESHOLD = 0.25 # 25 % between consecutive 1-minute closes inside a session (flag only) 13 PRICE_JUMP_THRESHOLD = 0.25 # 25 % between consecutive 1-minute closes inside a session (flag only)
14 14
modified tests/conftest.py +3 −3
@@ -25,8 +25,8 @@ os.environ.update({
25 "FINDATA_BACKUP_TARGET": "", 25 "FINDATA_BACKUP_TARGET": "",
26 }) 26 })
27 27
28−import httpx # noqa: E402 28+import httpx
29−import pytest # noqa: E402 29+import pytest
30 30
31 FIX = ROOT / "tests" / "fixtures" 31 FIX = ROOT / "tests" / "fixtures"
32 32
@@ -130,8 +130,8 @@ class VendorMock:
130 @pytest.fixture 130 @pytest.fixture
131 def vendor(monkeypatch): 131 def vendor(monkeypatch):
132 vm = VendorMock() 132 vm = VendorMock()
133− import findata_market_firstrate.client as C
134 import findata_fundamentals.firstrate as F 133 import findata_fundamentals.firstrate as F
134+ import findata_market_firstrate.client as C
135 monkeypatch.setattr(C, "DEFAULT_TRANSPORT", httpx.MockTransport(vm)) 135 monkeypatch.setattr(C, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))
136 monkeypatch.setattr(F, "DEFAULT_TRANSPORT", httpx.MockTransport(vm)) 136 monkeypatch.setattr(F, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))
137 return vm 137 return vm
modified tests/test_fundamentals_pit.py +6 −1
@@ -4,7 +4,12 @@ import json
4 from datetime import UTC, date, datetime 4 from datetime import UTC, date, datetime
5 from pathlib import Path 5 from pathlib import Path
6 6
7−from findata_fundamentals import SecEdgarConnector, available_at_policy, facts_from_companyfacts, facts_from_financials 7+from findata_fundamentals import (
8+ SecEdgarConnector,
9+ available_at_policy,
10+ facts_from_companyfacts,
11+ facts_from_financials,
12+)
8 13
9 FIX = Path(__file__).parent / "fixtures" / "fundamentals" 14 FIX = Path(__file__).parent / "fixtures" / "fundamentals"
10 NOW = datetime(2026, 11, 1, 12, 0, tzinfo=UTC) 15 NOW = datetime(2026, 11, 1, 12, 0, tzinfo=UTC)
modified tests/test_ingest_lake.py +3 −4
@@ -1,17 +1,16 @@
1 """Integration: lake publication, durable queue, incremental sync idempotency, resume, derive, compaction.""" 1 """Integration: lake publication, durable queue, incremental sync idempotency, resume, derive, compaction."""
2 from __future__ import annotations 2 from __future__ import annotations
3 3
4−import json
5 from datetime import UTC, date, datetime 4 from datetime import UTC, date, datetime
6 5
7 import pyarrow as pa 6 import pyarrow as pa
8 import pytest 7 import pytest
9−
10 from findata_core.db import connection, fetch_one, transaction 8 from findata_core.db import connection, fetch_one, transaction
11 from findata_ingest import queue 9 from findata_ingest import queue
12 from findata_ingest.lake import live_files 10 from findata_ingest.lake import live_files
13 from findata_ingest.reader import apply_adjustment, read_bars, run_sql 11 from findata_ingest.reader import apply_adjustment, read_bars, run_sql
14 from findata_ingest.tasks import backfill, compaction, derive, incremental, metadata 12 from findata_ingest.tasks import backfill, compaction, derive, incremental, metadata
13+
15 from tests.conftest import build_update_zip 14 from tests.conftest import build_update_zip
16 15
17 DS = "market_bars.stock.1m.raw" 16 DS = "market_bars.stock.1m.raw"
@@ -34,7 +33,7 @@ def test_queue_retry_backoff_and_dead_letter():
34 assert queue.enqueue("quality", None, {"x": 1}, dedupe_key="q-test") is None # deduped while queued 33 assert queue.enqueue("quality", None, {"x": 1}, dedupe_key="q-test") is None # deduped while queued
35 job = queue.claim_next(["quality"]) 34 job = queue.claim_next(["quality"])
36 assert job["attempts"] == 1 35 assert job["attempts"] == 1
37− queue.finish(job, False, error=f"boom userid=SECRETVALUE") 36+ queue.finish(job, False, error="boom userid=SECRETVALUE")
38 r = fetch_one("SELECT status, last_error, next_run_at > now() later FROM ingest_jobs WHERE id=%s", (jid,)) 37 r = fetch_one("SELECT status, last_error, next_run_at > now() later FROM ingest_jobs WHERE id=%s", (jid,))
39 assert r["status"] == "queued" and r["later"] and "SECRETVALUE" not in r["last_error"] 38 assert r["status"] == "queued" and r["later"] and "SECRETVALUE" not in r["last_error"]
40 with transaction() as conn: 39 with transaction() as conn:
@@ -71,7 +70,7 @@ def test_metadata_then_incremental_is_idempotent(vendor, lake, log):
71 assert r3.get("skipped") and r3["reason"] == "archive already ingested" and _rows() == v1 70 assert r3.get("skipped") and r3["reason"] == "archive already ingested" and _rows() == v1
72 71
73 # vendor republishes the same session with a corrected bar → supersede (no duplicates, version 2) 72 # vendor republishes the same session with a corrected bar → supersede (no duplicates, version 2)
74− fixed = (lake.root.parent / "x").exists() # noqa: F841 73+ fixed = (lake.root.parent / "x").exists()
75 import pathlib 74 import pathlib
76 txt = (pathlib.Path(__file__).parent / "fixtures" / "firstrate" / "AAPL_day_1min_UNADJUSTED.txt").read_text() 75 txt = (pathlib.Path(__file__).parent / "fixtures" / "firstrate" / "AAPL_day_1min_UNADJUSTED.txt").read_text()
77 txt = txt.replace("2026-10-30 09:30:00,346.63,348.51,346.62,348.26,1119437", "2026-10-30 09:30:00,346.63,348.51,346.62,348.27,1119437") 76 txt = txt.replace("2026-10-30 09:30:00,346.63,348.51,346.62,348.26,1119437", "2026-10-30 09:30:00,346.63,348.51,346.62,348.27,1119437")
modified tests/test_parse_firstrate.py +0 −2
@@ -3,8 +3,6 @@ from __future__ import annotations
3 from datetime import UTC, date, datetime 3 from datetime import UTC, date, datetime
4 from pathlib import Path 4 from pathlib import Path
5 5
6−import pyarrow as pa
7−
8 from findata_market_firstrate.archive import parse_bar_file, ticker_from_filename 6 from findata_market_firstrate.archive import parse_bar_file, ticker_from_filename
9 from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval 7 from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval
10 8
modified tests/test_redaction.py +1 −2
@@ -1,10 +1,9 @@
1 from __future__ import annotations 1 from __future__ import annotations
2 2
3−import httpx
4 import pytest 3 import pytest
5−
6 from findata_core.redact import redact 4 from findata_core.redact import redact
7 from findata_market_firstrate.client import FRDClient, FRDError 5 from findata_market_firstrate.client import FRDClient, FRDError
6+
8 from tests.conftest import TEST_USERID 7 from tests.conftest import TEST_USERID
9 8
10 9
modified tests/test_validators_aggregate.py +1 −3
@@ -3,8 +3,6 @@ from __future__ import annotations
3 from datetime import UTC, date, datetime 3 from datetime import UTC, date, datetime
4 from pathlib import Path 4 from pathlib import Path
5 5
6−import pyarrow as pa
7−
8 from findata_market_firstrate.archive import parse_bar_file 6 from findata_market_firstrate.archive import parse_bar_file
9 from findata_quality import aggregate_bars, validate_bars 7 from findata_quality import aggregate_bars, validate_bars
10 from findata_quality.calendar import xnys 8 from findata_quality.calendar import xnys
@@ -45,7 +43,7 @@ def test_row_level_flags():
45 ]) 43 ])
46 out, anomalies = validate_bars(_parse(rows), Interval.m1) 44 out, anomalies = validate_bars(_parse(rows), Interval.m1)
47 by_ts = {r["timestamp_utc"]: set(r["quality_flags"]) for r in out.to_pylist()} 45 by_ts = {r["timestamp_utc"]: set(r["quality_flags"]) for r in out.to_pylist()}
48− f = lambda h, m, d=30: by_ts[datetime(2026, 10, d, h, m, tzinfo=UTC)] # noqa: E731 46+ f = lambda h, m, d=30: by_ts[datetime(2026, 10, d, h, m, tzinfo=UTC)]
49 assert QualityFlag.ohlc_inconsistent in f(13, 30) 47 assert QualityFlag.ohlc_inconsistent in f(13, 30)
50 assert QualityFlag.negative_volume in f(13, 31) 48 assert QualityFlag.negative_volume in f(13, 31)
51 assert any(QualityFlag.off_interval in v for v in by_ts.values()) 49 assert any(QualityFlag.off_interval in v for v in by_ts.values())
modified tests/test_z_api_sdk.py +7 −6
@@ -2,21 +2,20 @@
2 from __future__ import annotations 2 from __future__ import annotations
3 3
4 import io 4 import io
5−import json
6 import time 5 import time
7 from datetime import UTC, date, datetime 6 from datetime import UTC, date, datetime
8 7
9 import httpx 8 import httpx
10−import pyarrow.ipc as ipc
11 import pyarrow.parquet as pq 9 import pyarrow.parquet as pq
12 import pyotp 10 import pyotp
13 import pytest 11 import pytest
14 from fastapi.testclient import TestClient 12 from fastapi.testclient import TestClient
15−
16 from findata_api import auth as A 13 from findata_api import auth as A
17 from findata_core.db import connection, fetch_one, transaction 14 from findata_core.db import connection, fetch_one, transaction
18 from findata_ingest import queue 15 from findata_ingest import queue
19 from findata_ingest.tasks import incremental, metadata 16 from findata_ingest.tasks import incremental, metadata
17+from pyarrow import ipc
18+
20 from tests.conftest import TEST_USERID 19 from tests.conftest import TEST_USERID
21 20
22 OWNER = "owner@example.com" 21 OWNER = "owner@example.com"
@@ -55,9 +54,10 @@ def app_client(vendor_module, lake_module):
55 54
56 @pytest.fixture(scope="module") 55 @pytest.fixture(scope="module")
57 def vendor_module(): 56 def vendor_module():
58− from tests.conftest import VendorMock
59− import findata_market_firstrate.client as C
60 import findata_fundamentals.firstrate as F 57 import findata_fundamentals.firstrate as F
58+ import findata_market_firstrate.client as C
59+
60+ from tests.conftest import VendorMock
61 vm = VendorMock() 61 vm = VendorMock()
62 old = (C.DEFAULT_TRANSPORT, F.DEFAULT_TRANSPORT) 62 old = (C.DEFAULT_TRANSPORT, F.DEFAULT_TRANSPORT)
63 C.DEFAULT_TRANSPORT = httpx.MockTransport(vm) 63 C.DEFAULT_TRANSPORT = httpx.MockTransport(vm)
@@ -225,9 +225,10 @@ def test_query_sandbox_and_quotas(app_client, api_key):
225 225
226 def test_export_roundtrip_and_sdk(app_client, api_key): 226 def test_export_roundtrip_and_sdk(app_client, api_key):
227 c = app_client 227 c = app_client
228− from findata_sdk import FinData
229 import threading 228 import threading
229+
230 import uvicorn 230 import uvicorn
231+ from findata_sdk import FinData
231 # the sync SDK needs a real HTTP server: run uvicorn on a loopback port in a thread 232 # the sync SDK needs a real HTTP server: run uvicorn on a loopback port in a thread
232 config = uvicorn.Config(c.app, host="127.0.0.1", port=8699, log_level="warning") 233 config = uvicorn.Config(c.app, host="127.0.0.1", port=8699, log_level="warning")
233 server = uvicorn.Server(config) 234 server = uvicorn.Server(config)
234 235
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere