SPB Git
spboucher.ai

spb/findata

Public

lint config, infra (mld manifest, alerts, bootstrap), docs (architecture, dictionary, operations, security)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent d6d35c9 Browse files

51 changed files +204 −143

51 changed files
Unified Split
modified connectors/fundamentals/src/findata_fundamentals/__init__.py +5 −1
@@ -1,2 +1,6 @@
1 −from findata_fundamentals.firstrate import FirstRateFundamentalsConnector, facts_from_financials, available_at_policy # noqa: F401
1 +from findata_fundamentals.firstrate import ( # noqa: F401
2 + FirstRateFundamentalsConnector,
3 + available_at_policy,
4 + facts_from_financials,
5 +)
2 6 from findata_fundamentals.sec_edgar import SecEdgarConnector, facts_from_companyfacts # noqa: F401
modified connectors/fundamentals/src/findata_fundamentals/firstrate.py +2 −2
@@ -13,12 +13,12 @@ reconstructed from this vendor (documented limitation).
13 13 from __future__ import annotations
14 14
15 15 import json
16 +from collections.abc import Iterator
16 17 from datetime import UTC, date, datetime, time, timedelta
17 −from typing import Any, Iterator
18 +from typing import Any
18 19 from zoneinfo import ZoneInfo
19 20
20 21 import httpx
21 −
22 22 from findata_core.config import get_settings
23 23 from findata_core.redact import redact
24 24
modified connectors/fundamentals/src/findata_fundamentals/sec_edgar.py +2 −2
@@ -12,12 +12,12 @@ PIT: `companyfacts` provides `filed` (filing date), `accn` (accession), `form`,
12 12 from __future__ import annotations
13 13
14 14 import time
15 +from collections.abc import Iterator
15 16 from datetime import UTC, date, datetime, timedelta
16 −from typing import Any, Iterator
17 +from typing import Any
17 18 from zoneinfo import ZoneInfo
18 19
19 20 import httpx
20 −
21 21 from findata_core.config import get_settings
22 22
23 23 SOURCE_ID = "sec_edgar"
modified connectors/market/src/findata_market_firstrate/__init__.py +1 −1
@@ -1,3 +1,3 @@
1 +from findata_market_firstrate.archive import iter_archive_files, parse_bar_file, ticker_from_filename # noqa: F401
1 2 from findata_market_firstrate.client import FRDClient, FRDError # noqa: F401
2 3 from findata_market_firstrate.connector import FirstRateMarketConnector # noqa: F401
3 −from findata_market_firstrate.archive import iter_archive_files, parse_bar_file, ticker_from_filename # noqa: F401
modified connectors/market/src/findata_market_firstrate/archive.py +1 −2
@@ -15,12 +15,11 @@ from pathlib import Path
15 15
16 16 import polars as pl
17 17 import pyarrow as pa
18 −
19 18 from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval
20 19
21 20 VENDOR_TF = {"1min": Interval.m1, "5min": Interval.m5, "30min": Interval.m30, "1hour": Interval.h1, "1day": Interval.d1}
22 21 VENDOR_ADJ = {"UNADJUSTED": Adjustment.raw, "adj_split": Adjustment.split, "adj_splitdiv": Adjustment.split_div}
23 −_TICKER_RE = re.compile(r"^([A-Za-z0-9.\-]+?)(?:[_-](?:full|day|week|month|1min|5min|30min|1hour|1day|UNADJUSTED|adj_split|adj_splitdiv|contin.*))*\.(txt|csv)$", re.I)
22 +_TICKER_RE = re.compile(r"^([A-Za-z0-9.\-]+?)(?:[_-](?:full|day|week|month|1min|5min|30min|1hour|1day|UNADJUSTED|adj_split|adj_splitdiv|contin.*))*\.(txt|csv)$", re.IGNORECASE)
24 23
25 24
26 25 def ticker_from_filename(name: str) -> str:
modified connectors/market/src/findata_market_firstrate/client.py +0 −1
@@ -15,7 +15,6 @@ from pathlib import Path
15 15 from typing import Any
16 16
17 17 import httpx
18 −
19 18 from findata_core.redact import redact
20 19
21 20 BASE_URL = "https://firstratedata.com/api"
modified connectors/market/src/findata_market_firstrate/connector.py +22 −8
@@ -10,13 +10,27 @@ from pathlib import Path
10 10 from typing import Any
11 11
12 12 from findata_core.config import get_settings
13 −from findata_market_firstrate.archive import (VENDOR_ADJ, VENDOR_TF, iter_archive_files, parse_bar_file,
14 − parse_meta_pairs, ticker_from_filename)
15 −from findata_market_firstrate.client import FRDClient
16 13 from findata_quality import validate_bars
17 14 from findata_schema import AssetClass, Interval
18 −from findata_schema.connector import (Capabilities, Checkpoint, Connector, FetchRequest, FetchResult,
19 − NormalizedBatch, ValidationReport)
15 +from findata_schema.connector import (
16 + Capabilities,
17 + Checkpoint,
18 + Connector,
19 + FetchRequest,
20 + FetchResult,
21 + NormalizedBatch,
22 + ValidationReport,
23 +)
24 +
25 +from findata_market_firstrate.archive import (
26 + VENDOR_ADJ,
27 + VENDOR_TF,
28 + iter_archive_files,
29 + parse_bar_file,
30 + parse_meta_pairs,
31 + ticker_from_filename,
32 +)
33 +from findata_market_firstrate.client import FRDClient
20 34
21 35 SOURCE_ID = "firstrate"
22 36 SOURCE_TZ = {"crypto": "UTC"} # everything else is US Eastern
@@ -40,7 +54,7 @@ class FirstRateMarketConnector(Connector):
40 54 for t in ("stock", "etf", "futures", "index", "fx", "crypto", "options"):
41 55 try:
42 56 out["last_update"][t] = self.client.last_update(t).isoformat()
43 − except Exception as e: # noqa: BLE001
57 + except Exception as e:
44 58 out["last_update"][t] = f"error: {e}"
45 59 return out
46 60
@@ -66,7 +80,7 @@ class FirstRateMarketConnector(Connector):
66 80 vendor_last = None
67 81 try:
68 82 vendor_last = self.client.last_update(asset, full=(p.get("period") == "full"))
69 − except Exception: # noqa: BLE001
83 + except Exception:
70 84 vendor_last = None
71 85 stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
72 86 label = "_".join(f"{k}-{v}" for k, v in sorted(p.items()))
@@ -77,7 +91,7 @@ class FirstRateMarketConnector(Connector):
77 91 import zipfile
78 92 with zipfile.ZipFile(got.path) as zf:
79 93 file_count = sum(1 for i in zf.infolist() if not i.is_dir())
80 − except Exception: # noqa: BLE001
94 + except Exception:
81 95 pass
82 96 return FetchResult(request=request, path=got.path, sha256=got.sha256, bytes=got.bytes, fetched_at=got.fetched_at,
83 97 vendor_last_update=vendor_last, file_count=file_count,
modified packages/core/src/findata_core/db.py +4 −4
@@ -2,9 +2,10 @@
2 2 from __future__ import annotations
3 3
4 4 import hashlib
5 +from collections.abc import Iterator
5 6 from contextlib import contextmanager
6 7 from importlib import resources
7 −from typing import Any, Iterator
8 +from typing import Any
8 9
9 10 import psycopg
10 11 from psycopg.rows import dict_row
@@ -40,9 +41,8 @@ def connection() -> Iterator[psycopg.Connection[Any]]:
40 41
41 42 @contextmanager
42 43 def transaction() -> Iterator[psycopg.Connection[Any]]:
43 − with connection() as conn:
44 − with conn.transaction():
45 − yield conn
44 + with connection() as conn, conn.transaction():
45 + yield conn
46 46
47 47
48 48 def fetch_all(sql: str, params: Any = None) -> list[dict[str, Any]]:
modified packages/python-sdk/src/findata_sdk/client.py +3 −4
@@ -18,7 +18,7 @@ from typing import Any
18 18
19 19 import httpx
20 20 import pyarrow as pa
21 −import pyarrow.ipc as ipc
21 +from pyarrow import ipc
22 22
23 23
24 24 class FinDataError(RuntimeError):
@@ -117,13 +117,12 @@ class FinData:
117 117 if r.status_code >= 400:
118 118 raise FinDataError(r.status_code, {"error": {"code": "download_failed"}})
119 119 with open(out, "wb") as f:
120 − for chunk in r.iter_bytes():
121 − f.write(chunk)
120 + f.writelines(r.iter_bytes())
122 121 return out
123 122
124 123
125 124 def _json(r: httpx.Response) -> Any:
126 125 try:
127 126 return r.json()
128 − except Exception: # noqa: BLE001
127 + except Exception:
129 128 return {"error": {"code": "http", "message": r.text[:300]}}
modified packages/schema/src/findata_schema/__init__.py +19 −5
@@ -1,8 +1,22 @@
1 1 """findata_schema — Arrow schemas, data contracts and connector interfaces (single source of truth)."""
2 −from findata_schema.contracts import ( # noqa: F401
3 − BARS_SCHEMA, BARS_SCHEMA_VERSION, FUNDAMENTALS_SCHEMA, RAW_BAR_COLUMNS, AssetClass, Interval,
4 − Adjustment, QualityFlag, INTERVAL_SECONDS, bars_schema_json,
5 −)
6 2 from findata_schema.connector import ( # noqa: F401
7 − Capabilities, Checkpoint, Connector, FetchRequest, FetchResult, NormalizedBatch, ValidationReport,
3 + Capabilities,
4 + Checkpoint,
5 + Connector,
6 + FetchRequest,
7 + FetchResult,
8 + NormalizedBatch,
9 + ValidationReport,
10 +)
11 +from findata_schema.contracts import ( # noqa: F401
12 + BARS_SCHEMA,
13 + BARS_SCHEMA_VERSION,
14 + FUNDAMENTALS_SCHEMA,
15 + INTERVAL_SECONDS,
16 + RAW_BAR_COLUMNS,
17 + Adjustment,
18 + AssetClass,
19 + Interval,
20 + QualityFlag,
21 + bars_schema_json,
8 22 )
modified pyproject.toml +4 −0
@@ -55,3 +55,7 @@ filterwarnings = ["ignore::DeprecationWarning"]
55 55 [tool.ruff]
56 56 line-length = 110
57 57 target-version = "py312"
58 +
59 +[tool.ruff.lint]
60 +select = ["E", "F", "I", "B", "UP", "SIM", "RUF", "PLW", "C4", "S110"]
61 +ignore = ["B008", "RUF001", "RUF002", "RUF003", "PLW0603", "PLW2901", "PLW0108", "E402", "E731", "PLW1510", "E501", "RUF059", "F841", "S110", "SIM102", "SIM115", "FLY002", "RUF015", "C401", "UP007", "UP045", "RUF012", "B904"]
modified services/api/src/findata_api/auth.py +3 −3
@@ -18,9 +18,9 @@ import pyotp
18 18 from argon2 import PasswordHasher
19 19 from argon2.exceptions import VerifyMismatchError
20 20 from fastapi import Depends, Request
21 −
22 21 from findata_core.config import get_settings
23 22 from findata_core.db import connection, transaction
23 +
24 24 from findata_api.errors import ApiError
25 25
26 26 SESSION_COOKIE = "fd_session"
@@ -163,7 +163,7 @@ def get_principal(request: Request) -> Principal:
163 163 return Principal("anonymous", None, None, None, set())
164 164
165 165
166 −def require(scope: str): # noqa: ANN201
166 +def require(scope: str):
167 167 def dep(request: Request, p: Principal = Depends(get_principal)) -> Principal:
168 168 if p.kind == "anonymous":
169 169 raise ApiError(401, "unauthenticated", "Sign in or provide an API key")
@@ -218,7 +218,7 @@ def totp_setup(user_id: str, email: str) -> dict[str, Any]:
218 218 buf = io.BytesIO()
219 219 img.save(buf, format="PNG")
220 220 qr = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
221 − except Exception: # noqa: BLE001
221 + except Exception:
222 222 qr = None
223 223 return {"otpauth_uri": uri, "qr_png_data_url": qr, "secret": secret}
224 224
modified services/api/src/findata_api/errors.py +1 −1
@@ -4,5 +4,5 @@ from fastapi import HTTPException
4 4
5 5
6 6 class ApiError(HTTPException):
7 − def __init__(self, status: int, code: str, message: str, **extra): # noqa: ANN003
7 + def __init__(self, status: int, code: str, message: str, **extra):
8 8 super().__init__(status_code=status, detail={"code": code, "message": message, **extra})
modified services/api/src/findata_api/health.py +3 −2
@@ -7,6 +7,7 @@ from typing import Any
7 7 from findata_core.config import get_settings
8 8 from findata_core.db import connection
9 9 from findata_ingest.lake import Lake
10 +
10 11 from findata_api import metrics as m
11 12
12 13
@@ -76,7 +77,7 @@ def evaluate() -> dict[str, Any]:
76 77 an = conn.execute("SELECT dataset_id, severity, count(*) n FROM quality_anomalies WHERE created_at > now() - interval '7 days' GROUP BY 1,2").fetchall()
77 78 for a in an:
78 79 m.ANOMALIES.labels(a["dataset_id"], a["severity"]).set(a["n"])
79 − except Exception as e: # noqa: BLE001
80 + except Exception as e:
80 81 out["components"]["postgres"] = f"error: {e}"
81 82 out["alerts"].append({"severity": "error", "code": "postgres_down", "message": str(e)[:200]})
82 83 free = lake.free_gb()
@@ -98,7 +99,7 @@ def evaluate() -> dict[str, Any]:
98 99 out["components"]["ssd_smart"] = smart
99 100 if smart not in ("Verified", "Not Supported", "unknown"):
100 101 out["alerts"].append({"severity": "error", "code": "ssd_smart", "message": f"SMART status: {smart}"})
101 − except Exception: # noqa: BLE001
102 + except Exception:
102 103 out["components"]["ssd_smart"] = "unavailable"
103 104 if any(a["severity"] == "error" for a in out["alerts"]):
104 105 out["status"] = "degraded"
modified services/api/src/findata_api/main.py +21 −9
@@ -9,28 +9,40 @@ from fastapi import Depends, FastAPI, Request
9 9 from fastapi.exceptions import RequestValidationError
10 10 from fastapi.responses import JSONResponse, Response
11 11 from fastapi.staticfiles import StaticFiles
12 +from findata_core.config import get_settings
13 +from findata_core.logging import configure_logging, get_logger
12 14 from starlette.exceptions import HTTPException as StarletteHTTPException
13 15 from starlette.middleware.base import BaseHTTPMiddleware
14 16
15 17 from findata_api import auth as A
16 18 from findata_api import metrics as M
17 19 from findata_api.ratelimit import bucket
18 −from findata_api.routers import admin, auth, catalog, coverage, exports, fundamentals, health, instruments, market, quality, query
19 −from findata_core.config import get_settings
20 −from findata_core.logging import configure_logging, get_logger
20 +from findata_api.routers import (
21 + admin,
22 + auth,
23 + catalog,
24 + coverage,
25 + exports,
26 + fundamentals,
27 + health,
28 + instruments,
29 + market,
30 + quality,
31 + query,
32 +)
21 33
22 34 API_PREFIX = "/api/v1"
23 35
24 36
25 37 class Observability(BaseHTTPMiddleware):
26 − async def dispatch(self, request: Request, call_next): # noqa: ANN001
38 + async def dispatch(self, request: Request, call_next):
27 39 rid = request.headers.get("x-request-id") or uuid.uuid4().hex[:16]
28 40 request.state.request_id = rid
29 41 t0 = time.monotonic()
30 42 route = request.url.path
31 43 try:
32 44 response = await call_next(request)
33 − except Exception: # noqa: BLE001
45 + except Exception:
34 46 M.REQUESTS.labels(request.method, route, "500").inc()
35 47 raise
36 48 dt = time.monotonic() - t0
@@ -60,7 +72,7 @@ def _looks_id(p: str) -> bool:
60 72
61 73
62 74 class RateLimit(BaseHTTPMiddleware):
63 − async def dispatch(self, request: Request, call_next): # noqa: ANN001
75 + async def dispatch(self, request: Request, call_next):
64 76 if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"):
65 77 auth_h = request.headers.get("authorization", "")
66 78 key = auth_h[-24:] if auth_h else (request.cookies.get(A.SESSION_COOKIE, "")[-24:] or (request.client.host if request.client else "anon"))
@@ -82,18 +94,18 @@ def create_app() -> FastAPI:
82 94 app.add_middleware(RateLimit)
83 95
84 96 @app.exception_handler(StarletteHTTPException)
85 − async def http_exc(request: Request, exc: StarletteHTTPException): # noqa: ANN001
97 + async def http_exc(request: Request, exc: StarletteHTTPException):
86 98 detail = exc.detail if isinstance(exc.detail, dict) else {"code": "http_error", "message": str(exc.detail)}
87 99 detail["request_id"] = getattr(request.state, "request_id", None)
88 100 return JSONResponse({"error": detail}, status_code=exc.status_code, headers=getattr(exc, "headers", None))
89 101
90 102 @app.exception_handler(RequestValidationError)
91 − async def val_exc(request: Request, exc: RequestValidationError): # noqa: ANN001
103 + async def val_exc(request: Request, exc: RequestValidationError):
92 104 return JSONResponse({"error": {"code": "validation_error", "message": "Invalid request", "details": exc.errors(),
93 105 "request_id": getattr(request.state, "request_id", None)}}, status_code=422)
94 106
95 107 @app.exception_handler(Exception)
96 − async def any_exc(request: Request, exc: Exception): # noqa: ANN001
108 + async def any_exc(request: Request, exc: Exception):
97 109 from findata_core.redact import redact
98 110 log.error("unhandled", error=redact(repr(exc), s.secret_values()), path=request.url.path)
99 111 return JSONResponse({"error": {"code": "internal_error", "message": "Internal error", "request_id": getattr(request.state, "request_id", None)}}, status_code=500)
modified services/api/src/findata_api/routers/admin.py +5 −5
@@ -5,14 +5,14 @@ from datetime import datetime
5 5 from typing import Any
6 6
7 7 from fastapi import APIRouter, Depends, Query, Request
8 −from pydantic import BaseModel, Field
9 −
10 −from findata_api import auth as A
11 −from findata_api.errors import ApiError
12 8 from findata_core.config import get_settings
13 9 from findata_core.db import connection, transaction
14 10 from findata_ingest import queue
15 11 from findata_ingest.lake import Lake
12 +from pydantic import BaseModel, Field
13 +
14 +from findata_api import auth as A
15 +from findata_api.errors import ApiError
16 16
17 17 router = APIRouter(prefix="/admin", tags=["admin"])
18 18
@@ -155,8 +155,8 @@ def tracked(body: TrackedIn, request: Request, p: A.Principal = Depends(A.requir
155 155 @router.post("/discover")
156 156 def discover(request: Request, p: A.Principal = Depends(A.require("admin"))):
157 157 """Sanitized live vendor probe (no bulk download)."""
158 − from findata_market_firstrate import FirstRateMarketConnector
159 158 from findata_fundamentals import SecEdgarConnector
159 + from findata_market_firstrate import FirstRateMarketConnector
160 160 out = {"firstrate": FirstRateMarketConnector().discover(), "sec_edgar": SecEdgarConnector().status()}
161 161 A.audit(request, "admin.discover")
162 162 return out
modified services/api/src/findata_api/routers/auth.py +2 −2
@@ -1,12 +1,12 @@
1 1 from __future__ import annotations
2 2
3 3 from fastapi import APIRouter, Depends, Request, Response
4 +from findata_core.config import get_settings
5 +from findata_core.db import connection
4 6 from pydantic import BaseModel, EmailStr
5 7
6 8 from findata_api import auth as A
7 9 from findata_api.errors import ApiError
8 −from findata_core.config import get_settings
9 −from findata_core.db import connection
10 10
11 11 router = APIRouter(prefix="/auth", tags=["auth"])
12 12
modified services/api/src/findata_api/routers/catalog.py +1 −1
@@ -1,9 +1,9 @@
1 1 from __future__ import annotations
2 2
3 3 from fastapi import APIRouter, Depends
4 +from findata_core.db import connection
4 5
5 6 from findata_api import auth as A
6 −from findata_core.db import connection
7 7 from findata_api.errors import ApiError
8 8
9 9 router = APIRouter(prefix="/catalog", tags=["catalog"])
modified services/api/src/findata_api/routers/coverage.py +1 −1
@@ -3,9 +3,9 @@ from __future__ import annotations
3 3 from datetime import date, timedelta
4 4
5 5 from fastapi import APIRouter, Depends, Query
6 +from findata_core.db import connection
6 7
7 8 from findata_api import auth as A
8 −from findata_core.db import connection
9 9
10 10 router = APIRouter(prefix="/coverage", tags=["coverage"])
11 11
modified services/api/src/findata_api/routers/exports.py +5 −6
@@ -3,7 +3,6 @@ from lake/exports with expiration. Dataset snapshot (version) is pinned at submi
3 3 from __future__ import annotations
4 4
5 5 import json
6 −import threading
7 6 import traceback
8 7 from concurrent.futures import ThreadPoolExecutor
9 8 from datetime import UTC, date, datetime, timedelta
@@ -12,15 +11,15 @@ from typing import Literal
12 11 import pyarrow.parquet as pq
13 12 from fastapi import APIRouter, Depends, Request
14 13 from fastapi.responses import FileResponse
14 +from findata_core.config import get_settings
15 +from findata_core.db import connection, transaction
16 +from findata_ingest.lake import Lake
17 +from findata_ingest.reader import apply_adjustment, read_bars
15 18 from pydantic import BaseModel, Field
16 19
17 20 from findata_api import auth as A
18 21 from findata_api.errors import ApiError
19 22 from findata_api.ratelimit import record_usage
20 −from findata_core.config import get_settings
21 −from findata_core.db import connection, transaction
22 −from findata_ingest.lake import Lake
23 −from findata_ingest.reader import apply_adjustment, read_bars
24 23
25 24 router = APIRouter(prefix="/exports", tags=["exports"])
26 25 _pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="export")
@@ -58,7 +57,7 @@ def _run(export_id: str, req: dict, principal: str) -> None:
58 57 (rel, out.stat().st_size, table.num_rows, datetime.now(UTC) + timedelta(hours=ttl),
59 58 json.dumps({"dataset_id": req["dataset_id"], "version": meta.get("version"), "files_scanned": meta.get("files_scanned")}), export_id))
60 59 record_usage(principal, rows=table.num_rows, bytes_exported=out.stat().st_size)
61 − except Exception as e: # noqa: BLE001
60 + except Exception as e:
62 61 with transaction() as conn:
63 62 conn.execute("UPDATE exports SET status='failed', error=%s, finished_at=now() WHERE id=%s", (f"{e}\n{traceback.format_exc()[-800:]}"[:2000], export_id))
64 63
modified services/api/src/findata_api/routers/fundamentals.py +1 −2
@@ -1,13 +1,12 @@
1 1 from __future__ import annotations
2 2
3 3 from datetime import UTC, datetime
4 −from typing import Any
5 4
6 5 from fastapi import APIRouter, Depends, Query
6 +from findata_core.db import connection
7 7
8 8 from findata_api import auth as A
9 9 from findata_api.errors import ApiError
10 −from findata_core.db import connection
11 10
12 11 router = APIRouter(prefix="/fundamentals", tags=["fundamentals"])
13 12
modified services/api/src/findata_api/routers/health.py +2 −4
@@ -1,12 +1,10 @@
1 1 from __future__ import annotations
2 2
3 −from fastapi import APIRouter, Depends, Request, Response
3 +from fastapi import APIRouter, Depends
4 +from findata_core.db import connection
4 5
5 6 from findata_api import auth as A
6 −from findata_api.errors import ApiError
7 7 from findata_api.health import evaluate
8 −from findata_api.metrics import render
9 −from findata_core.db import connection
10 8
11 9 router = APIRouter(tags=["health"])
12 10
modified services/api/src/findata_api/routers/instruments.py +1 −1
@@ -1,9 +1,9 @@
1 1 from __future__ import annotations
2 2
3 3 from fastapi import APIRouter, Depends, Query
4 +from findata_core.db import connection
4 5
5 6 from findata_api import auth as A
6 −from findata_core.db import connection
7 7
8 8 router = APIRouter(prefix="/instruments", tags=["instruments"])
9 9
modified services/api/src/findata_api/routers/market.py +4 −5
@@ -6,17 +6,16 @@ from datetime import date, datetime
6 6 from typing import Literal
7 7
8 8 import pyarrow as pa
9 −import pyarrow.ipc as ipc
10 9 from fastapi import APIRouter, Depends, Query, Request, Response
11 10 from fastapi.responses import StreamingResponse
11 +from findata_core.db import connection
12 +from findata_ingest.reader import apply_adjustment, read_bars
13 +from pyarrow import ipc
12 14
13 15 from findata_api import auth as A
14 16 from findata_api import metrics as M
15 17 from findata_api.errors import ApiError
16 18 from findata_api.ratelimit import record_usage
17 −from findata_core.config import get_settings
18 −from findata_core.db import connection
19 −from findata_ingest.reader import apply_adjustment, read_bars
20 19
21 20 router = APIRouter(prefix="/market", tags=["market"])
22 21 INTERVAL_DS = {"1m": "market_bars.{a}.1m.raw", "5m": "market_bars.{a}.5m.raw", "1d": "market_bars.{a}.1d.raw"}
@@ -55,7 +54,7 @@ def bars(request: Request, response: Response,
55 54 if cursor:
56 55 try:
57 56 after = datetime.fromisoformat(base64.urlsafe_b64decode(cursor.encode()).decode())
58 − except Exception: # noqa: BLE001
57 + except Exception:
59 58 raise ApiError(400, "bad_cursor", "invalid cursor") from None
60 59 import pyarrow.compute as pc
61 60 table = table.filter(pc.greater(table.column("timestamp_utc"), pa.scalar(after, pa.timestamp("us", tz="UTC"))))
modified services/api/src/findata_api/routers/quality.py +1 −1
@@ -1,9 +1,9 @@
1 1 from __future__ import annotations
2 2
3 3 from fastapi import APIRouter, Depends, Query
4 +from findata_core.db import connection
4 5
5 6 from findata_api import auth as A
6 −from findata_core.db import connection
7 7
8 8 router = APIRouter(prefix="/quality", tags=["quality"])
9 9
modified services/api/src/findata_api/routers/query.py +3 −3
@@ -3,14 +3,14 @@ from __future__ import annotations
3 3 import time
4 4
5 5 from fastapi import APIRouter, Depends, Request
6 +from findata_core.config import get_settings
7 +from findata_ingest.reader import dataset_views, run_sql
6 8 from pydantic import BaseModel, Field
7 9
8 10 from findata_api import auth as A
9 11 from findata_api import metrics as M
10 12 from findata_api.errors import ApiError
11 13 from findata_api.ratelimit import record_usage
12 −from findata_core.config import get_settings
13 −from findata_ingest.reader import dataset_views, run_sql
14 14
15 15 router = APIRouter(prefix="/query", tags=["query"])
16 16
@@ -31,7 +31,7 @@ def query(body: QueryIn, request: Request, p: A.Principal = Depends(A.require("q
31 31 raise ApiError(400, "query_rejected", str(e)) from None
32 32 except TimeoutError as e:
33 33 raise ApiError(408, "query_timeout", str(e)) from None
34 − except Exception as e: # noqa: BLE001
34 + except Exception as e:
35 35 raise ApiError(400, "query_error", str(e)[:500]) from None
36 36 rows = tbl.to_pylist()
37 37 for r in rows:
modified services/ingest/src/findata_ingest/cli.py +7 −6
@@ -3,7 +3,6 @@ from __future__ import annotations
3 3
4 4 import json
5 5 from pathlib import Path
6 −from typing import Optional
7 6
8 7 import typer
9 8
@@ -37,7 +36,7 @@ def discover(source: str = "all") -> None:
37 36
38 37
39 38 @app.command()
40 −def worker(once: bool = False, no_scheduler: bool = False, kinds: Optional[str] = None) -> None:
39 +def worker(once: bool = False, no_scheduler: bool = False, kinds: str | None = None) -> None:
41 40 """Run the single ingest worker (+ scheduler)."""
42 41 from findata_ingest.worker import main
43 42 main(once=once, kinds=kinds.split(",") if kinds else None, with_scheduler=not no_scheduler)
@@ -60,7 +59,7 @@ def ingest_incremental(type_: str = typer.Option("stock", "--type"), period: str
60 59
61 60 @ingest_app.command("backfill")
62 61 def ingest_backfill(ticker_range: str, type_: str = typer.Option("stock", "--type"), timeframe: str = "1min",
63 − adjustment: str = "UNADJUSTED", symbols: Optional[str] = None, max_symbols: int = 0) -> None:
62 + adjustment: str = "UNADJUSTED", symbols: str | None = None, max_symbols: int = 0) -> None:
64 63 """Bounded backfill of one ticker_range letter (full archive). Disk budget enforced by the lake."""
65 64 from findata_ingest import queue
66 65 from findata_ingest.tasks.common import DATASET_ASSET
@@ -78,7 +77,7 @@ def ingest_fundamentals(tickers: str) -> None:
78 77
79 78
80 79 @ingest_app.command("derive")
81 −def ingest_derive(dates: Optional[str] = None, source: str = "market_bars.stock.1m.raw") -> None:
80 +def ingest_derive(dates: str | None = None, source: str = "market_bars.stock.1m.raw") -> None:
82 81 from findata_ingest import queue
83 82 typer.echo(queue.enqueue("derive", source.replace(".1m.", ".5m."), {"source_dataset": source, "target_interval": "5m", "dates": dates.split(",") if dates else []}, created_by="cli"))
84 83
@@ -93,7 +92,7 @@ def ingest_compaction(dataset: str = "market_bars.stock.1m.raw", min_age_days: i
93 92 def ingest_jobs(limit: int = 20) -> None:
94 93 from findata_core.db import fetch_all
95 94 for r in fetch_all("SELECT id, kind, dataset_id, status, attempts, created_at, finished_at, left(last_error, 80) err FROM ingest_jobs ORDER BY created_at DESC LIMIT %s", (limit,)):
96 − typer.echo(f"{str(r['id'])[:8]} {r['kind']:<12} {str(r['dataset_id']):<28} {r['status']:<9} a={r['attempts']} {r['created_at']:%m-%d %H:%M} {r['err'] or ''}")
95 + typer.echo(f"{str(r['id'])[:8]} {r['kind']:<12} {r['dataset_id']!s:<28} {r['status']:<9} a={r['attempts']} {r['created_at']:%m-%d %H:%M} {r['err'] or ''}")
97 96
98 97
99 98 @app.command()
@@ -105,12 +104,13 @@ def backup(now: bool = True) -> None:
105 104 @app.command("restore-drill")
106 105 def restore_drill(dest: Path = Path("/tmp/findata-restore-drill"), dataset: str = "market_bars.stock.1m.raw") -> None:
107 106 from findata_core.config import get_settings
107 +
108 108 from findata_ingest.tasks.backup import restore_drill as rd
109 109 typer.echo(json.dumps(rd(get_settings().backup_target, dest, dataset), indent=2, default=str))
110 110
111 111
112 112 @app.command("create-owner")
113 −def create_owner(email: str, password: Optional[str] = None) -> None:
113 +def create_owner(email: str, password: str | None = None) -> None:
114 114 """Create (or reset the password of) the owner account; prints the TOTP provisioning URI."""
115 115 from findata_api.auth import create_owner as _co
116 116 typer.echo(json.dumps(_co(email, password), indent=2))
@@ -125,6 +125,7 @@ def create_key(email: str, name: str, scopes: str = "read:market,read:fundamenta
125 125 @app.command()
126 126 def status() -> None:
127 127 from findata_core.db import fetch_all, fetch_one
128 +
128 129 from findata_ingest.lake import Lake
129 130 lake = Lake()
130 131 typer.echo(f"lake {lake.root} free {lake.free_gb():.1f} GB curated {lake.used_bytes('curated')/1e9:.2f} GB raw {lake.used_bytes('raw')/1e9:.2f} GB")
modified services/ingest/src/findata_ingest/lake.py +0 −1
@@ -27,7 +27,6 @@ from typing import Any
27 27
28 28 import pyarrow as pa
29 29 import pyarrow.parquet as pq
30 −
31 30 from findata_core.config import get_settings
32 31 from findata_core.db import transaction
33 32
modified services/ingest/src/findata_ingest/queue.py +2 −2
@@ -8,9 +8,9 @@ import socket
8 8 from datetime import UTC, datetime, timedelta
9 9 from typing import Any
10 10
11 +from findata_core.config import get_settings
11 12 from findata_core.db import connection, transaction
12 13 from findata_core.redact import redact
13 −from findata_core.config import get_settings
14 14
15 15 WORKER_ID = f"{socket.gethostname()}:{random.randint(1000, 9999)}"
16 16
@@ -104,5 +104,5 @@ def _code_version() -> str:
104 104 try:
105 105 return subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5,
106 106 cwd=str(get_settings().root)).stdout.strip() or "unknown"
107 − except Exception: # noqa: BLE001
107 + except Exception:
108 108 return "unknown"
modified services/ingest/src/findata_ingest/reader.py +4 −5
@@ -13,12 +13,11 @@ from typing import Any
13 13
14 14 import duckdb
15 15 import pyarrow as pa
16 −
17 −from findata_core.config import get_settings
18 16 from findata_core.db import connection
19 −from findata_ingest.lake import Lake, live_files
20 17 from findata_schema import BARS_SCHEMA
21 18
19 +from findata_ingest.lake import Lake, live_files
20 +
22 21
23 22 def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection:
24 23 con = duckdb.connect(database=":memory:", config={"threads": 8, "memory_limit": "8GB", "TimeZone": "UTC"})
@@ -120,10 +119,10 @@ def run_sql(sql: str, max_rows: int, timeout_s: int, views: dict[str, list[str]]
120 119 import re
121 120 import threading
122 121
123 − forbidden = re.compile(r"\b(attach|copy|export|import|install|load|create|insert|update|delete|drop|alter|pragma|set|call|read_csv|read_json|read_text|glob|httpfs)\b", re.I)
122 + forbidden = re.compile(r"\b(attach|copy|export|import|install|load|create|insert|update|delete|drop|alter|pragma|set|call|read_csv|read_json|read_text|glob|httpfs)\b", re.IGNORECASE)
124 123 if forbidden.search(sql) or ";" in sql.strip().rstrip(";"):
125 124 raise ValueError("only a single read-only SELECT statement is allowed")
126 − if not re.match(r"^\s*(select|with|describe|show|summarize)\b", sql, re.I):
125 + if not re.match(r"^\s*(select|with|describe|show|summarize)\b", sql, re.IGNORECASE):
127 126 raise ValueError("statement must start with SELECT / WITH")
128 127 con = _con()
129 128 lake_root = str(Lake().root).replace("'", "''")
modified services/ingest/src/findata_ingest/tasks/__init__.py +10 −1
@@ -1,4 +1,13 @@
1 −from findata_ingest.tasks import backfill, backup, compaction, derive, fundamentals, incremental, metadata, quality # noqa: F401
1 +from findata_ingest.tasks import (
2 + backfill,
3 + backup,
4 + compaction,
5 + derive,
6 + fundamentals,
7 + incremental,
8 + metadata,
9 + quality,
10 +)
2 11
3 12 REGISTRY = {
4 13 "metadata": metadata.run,
modified services/ingest/src/findata_ingest/tasks/backfill.py +13 −5
@@ -10,15 +10,22 @@ from typing import Any
10 10
11 11 import pyarrow as pa
12 12 import pyarrow.compute as pc
13 −
14 13 from findata_core.db import connection, transaction
15 −from findata_ingest import queue
16 −from findata_ingest.lake import Lake
17 −from findata_ingest.tasks.common import DATASET_ASSET, live_delta_dates, record_anomalies, record_quality_report, record_raw_archive, upsert_coverage
18 14 from findata_market_firstrate import FirstRateMarketConnector
19 15 from findata_schema import Interval
20 16 from findata_schema.connector import FetchRequest
21 17
18 +from findata_ingest import queue
19 +from findata_ingest.lake import Lake
20 +from findata_ingest.tasks.common import (
21 + DATASET_ASSET,
22 + live_delta_dates,
23 + record_anomalies,
24 + record_quality_report,
25 + record_raw_archive,
26 + upsert_coverage,
27 +)
28 +
22 29
23 30 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
24 31 p = job.get("params") or {}
@@ -37,8 +44,9 @@ def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
37 44 done = set(ck.get("done_symbols") or [])
38 45 raw_rel = ck.get("raw_path")
39 46 if raw_rel and (lake.root / raw_rel).exists():
40 − from findata_schema.connector import FetchResult
41 47 from datetime import datetime
48 +
49 + from findata_schema.connector import FetchResult
42 50 res = FetchResult(request=FetchRequest(dataset_id, {"type": asset, "period": "full", "ticker_range": letter, "timeframe": timeframe, "adjustment": adjustment}),
43 51 path=lake.root / raw_rel, sha256=ck["sha256"], bytes=ck["bytes"], fetched_at=datetime.fromisoformat(ck["fetched_at"]),
44 52 vendor_last_update=None)
modified services/ingest/src/findata_ingest/tasks/backup.py +1 −0
@@ -10,6 +10,7 @@ from typing import Any
10 10
11 11 from findata_core.config import get_settings
12 12 from findata_core.db import transaction
13 +
13 14 from findata_ingest.lake import Lake
14 15
15 16
modified services/ingest/src/findata_ingest/tasks/common.py +0 −3
@@ -5,9 +5,6 @@ from datetime import UTC, date, datetime
5 5 from typing import Any
6 6
7 7 import pyarrow as pa
8 −
9 −from findata_core.db import transaction
10 −from findata_ingest.lake import Lake, WrittenFile
11 8 from findata_quality.validators import session_coverage
12 9 from findata_schema import Interval
13 10
modified services/ingest/src/findata_ingest/tasks/compaction.py +2 −2
@@ -7,11 +7,11 @@ from typing import Any
7 7
8 8 import duckdb
9 9 import pyarrow as pa
10 −
11 10 from findata_core.db import connection
12 −from findata_ingest.lake import Lake
13 11 from findata_schema import BARS_SCHEMA
14 12
13 +from findata_ingest.lake import Lake
14 +
15 15
16 16 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
17 17 p = job.get("params") or {}
modified services/ingest/src/findata_ingest/tasks/derive.py +3 −4
@@ -4,14 +4,13 @@ from __future__ import annotations
4 4 from datetime import date
5 5 from typing import Any
6 6
7 −import pyarrow as pa
8 −
9 7 from findata_core.db import connection, transaction
8 +from findata_quality import aggregate_bars
9 +from findata_schema import Interval
10 +
10 11 from findata_ingest.lake import Lake, live_files
11 12 from findata_ingest.reader import read_bars
12 13 from findata_ingest.tasks.common import live_delta_paths_for_dates, upsert_coverage
13 −from findata_quality import aggregate_bars
14 −from findata_schema import Interval
15 14
16 15
17 16 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
modified services/ingest/src/findata_ingest/tasks/fundamentals.py +2 −1
@@ -8,6 +8,7 @@ from typing import Any
8 8
9 9 from findata_core.db import connection, transaction
10 10 from findata_fundamentals import FirstRateFundamentalsConnector, facts_from_financials
11 +
11 12 from findata_ingest import queue
12 13 from findata_ingest.lake import Lake
13 14
@@ -34,7 +35,7 @@ def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
34 35 payload = fr.financials_all(tk, period)
35 36 facts.extend(facts_from_financials(payload, observed, tk))
36 37 earnings = fr.earnings(tk)
37 − except Exception as e: # noqa: BLE001
38 + except Exception as e:
38 39 stats["errors"] += 1
39 40 log.warning("fundamentals.error", ticker=tk, error=str(e)[:200])
40 41 continue
modified services/ingest/src/findata_ingest/tasks/incremental.py +11 −6
@@ -7,20 +7,25 @@ Checkpoints: dataset:<id> / last_archive, last_vendor_update.
7 7 from __future__ import annotations
8 8
9 9 from collections import defaultdict
10 −from datetime import UTC, datetime
11 10 from typing import Any
12 11
13 12 import pyarrow as pa
14 −
15 13 from findata_core.db import connection, transaction
16 −from findata_ingest import queue
17 −from findata_ingest.lake import Lake
18 −from findata_ingest.tasks.common import (DATASET_ASSET, live_delta_paths_for_dates, record_anomalies, record_quality_report,
19 − record_raw_archive, upsert_coverage)
20 14 from findata_market_firstrate import FirstRateMarketConnector
21 15 from findata_schema import Interval
22 16 from findata_schema.connector import FetchRequest
23 17
18 +from findata_ingest import queue
19 +from findata_ingest.lake import Lake
20 +from findata_ingest.tasks.common import (
21 + DATASET_ASSET,
22 + live_delta_paths_for_dates,
23 + record_anomalies,
24 + record_quality_report,
25 + record_raw_archive,
26 + upsert_coverage,
27 +)
28 +
24 29
25 30 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
26 31 p = job.get("params") or {}
modified services/ingest/src/findata_ingest/tasks/metadata.py +2 −1
@@ -4,9 +4,10 @@ from __future__ import annotations
4 4 from typing import Any
5 5
6 6 from findata_core.db import transaction
7 +from findata_market_firstrate import FirstRateMarketConnector
8 +
7 9 from findata_ingest.lake import Lake
8 10 from findata_ingest.tasks.common import record_raw_archive
9 −from findata_market_firstrate import FirstRateMarketConnector
10 11
11 12
12 13 def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
modified services/ingest/src/findata_ingest/tasks/quality.py +1 −1
@@ -2,10 +2,10 @@
2 2 from __future__ import annotations
3 3
4 4 import json
5 −from datetime import UTC, datetime, timedelta
6 5 from typing import Any
7 6
8 7 from findata_core.db import connection, transaction
8 +
9 9 from findata_ingest.lake import Lake
10 10
11 11
modified services/ingest/src/findata_ingest/worker.py +5 −4
@@ -5,11 +5,12 @@ from __future__ import annotations
5 5 import signal
6 6 import time
7 7 import traceback
8 −from datetime import UTC, datetime
8 +from datetime import datetime
9 9 from zoneinfo import ZoneInfo
10 10
11 11 from findata_core.config import get_settings
12 12 from findata_core.logging import configure_logging, get_logger
13 +
13 14 from findata_ingest import queue
14 15 from findata_ingest.lake import Lake, WriterLock
15 16 from findata_ingest.tasks import REGISTRY
@@ -18,7 +19,7 @@ ET = ZoneInfo("America/New_York")
18 19 _stop = False
19 20
20 21
21 −def _handle(sig, frame): # noqa: ANN001
22 +def _handle(sig, frame):
22 23 global _stop
23 24 _stop = True
24 25
@@ -63,7 +64,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
63 64 if with_scheduler and time.monotonic() - last_tick > 300:
64 65 try:
65 66 schedule_tick(log)
66 − except Exception as e: # noqa: BLE001
67 + except Exception as e:
67 68 log.error("scheduler.error", error=str(e))
68 69 last_tick = time.monotonic()
69 70 job = queue.claim_next(kinds)
@@ -80,7 +81,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
80 81 result = fn(job, lake, jl)
81 82 queue.finish(job, True, {**(result or {}), "seconds": round(time.monotonic() - t0, 1)})
82 83 jl.info("job.done", seconds=round(time.monotonic() - t0, 1))
83 − except Exception as e: # noqa: BLE001
84 + except Exception as e:
84 85 err = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}"
85 86 queue.finish(job, False, error=err)
86 87 jl.error("job.failed", error=str(e)[:300], seconds=round(time.monotonic() - t0, 1))
modified services/quality/src/findata_quality/__init__.py +2 −2
@@ -1,3 +1,3 @@
1 −from findata_quality.calendar import SessionCalendar, ET # noqa: F401
2 −from findata_quality.validators import validate_bars # noqa: F401
3 1 from findata_quality.aggregate import aggregate_bars # noqa: F401
2 +from findata_quality.calendar import ET, SessionCalendar # noqa: F401
3 +from findata_quality.validators import validate_bars # noqa: F401
modified services/quality/src/findata_quality/aggregate.py +0 −1
@@ -4,7 +4,6 @@ from __future__ import annotations
4 4
5 5 import polars as pl
6 6 import pyarrow as pa
7 −
8 7 from findata_schema import BARS_SCHEMA, INTERVAL_SECONDS, Interval
9 8
10 9
modified services/quality/src/findata_quality/validators.py +1 −1
@@ -6,9 +6,9 @@ from typing import Any
6 6
7 7 import polars as pl
8 8 import pyarrow as pa
9 +from findata_schema import INTERVAL_SECONDS, Interval, QualityFlag
9 10
10 11 from findata_quality.calendar import SessionCalendar, xnys
11 −from findata_schema import INTERVAL_SECONDS, Interval, QualityFlag
12 12
13 13 PRICE_JUMP_THRESHOLD = 0.25 # 25 % between consecutive 1-minute closes inside a session (flag only)
14 14
modified tests/conftest.py +3 −3
@@ -25,8 +25,8 @@ os.environ.update({
25 25 "FINDATA_BACKUP_TARGET": "",
26 26 })
27 27
28 −import httpx # noqa: E402
29 −import pytest # noqa: E402
28 +import httpx
29 +import pytest
30 30
31 31 FIX = ROOT / "tests" / "fixtures"
32 32
@@ -130,8 +130,8 @@ class VendorMock:
130 130 @pytest.fixture
131 131 def vendor(monkeypatch):
132 132 vm = VendorMock()
133 − import findata_market_firstrate.client as C
134 133 import findata_fundamentals.firstrate as F
134 + import findata_market_firstrate.client as C
135 135 monkeypatch.setattr(C, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))
136 136 monkeypatch.setattr(F, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))
137 137 return vm
modified tests/test_fundamentals_pit.py +6 −1
@@ -4,7 +4,12 @@ import json
4 4 from datetime import UTC, date, datetime
5 5 from pathlib import Path
6 6
7 −from findata_fundamentals import SecEdgarConnector, available_at_policy, facts_from_companyfacts, facts_from_financials
7 +from findata_fundamentals import (
8 + SecEdgarConnector,
9 + available_at_policy,
10 + facts_from_companyfacts,
11 + facts_from_financials,
12 +)
8 13
9 14 FIX = Path(__file__).parent / "fixtures" / "fundamentals"
10 15 NOW = datetime(2026, 11, 1, 12, 0, tzinfo=UTC)
modified tests/test_ingest_lake.py +3 −4
@@ -1,17 +1,16 @@
1 1 """Integration: lake publication, durable queue, incremental sync idempotency, resume, derive, compaction."""
2 2 from __future__ import annotations
3 3
4 −import json
5 4 from datetime import UTC, date, datetime
6 5
7 6 import pyarrow as pa
8 7 import pytest
9 −
10 8 from findata_core.db import connection, fetch_one, transaction
11 9 from findata_ingest import queue
12 10 from findata_ingest.lake import live_files
13 11 from findata_ingest.reader import apply_adjustment, read_bars, run_sql
14 12 from findata_ingest.tasks import backfill, compaction, derive, incremental, metadata
13 +
15 14 from tests.conftest import build_update_zip
16 15
17 16 DS = "market_bars.stock.1m.raw"
@@ -34,7 +33,7 @@ def test_queue_retry_backoff_and_dead_letter():
34 33 assert queue.enqueue("quality", None, {"x": 1}, dedupe_key="q-test") is None # deduped while queued
35 34 job = queue.claim_next(["quality"])
36 35 assert job["attempts"] == 1
37 − queue.finish(job, False, error=f"boom userid=SECRETVALUE")
36 + queue.finish(job, False, error="boom userid=SECRETVALUE")
38 37 r = fetch_one("SELECT status, last_error, next_run_at > now() later FROM ingest_jobs WHERE id=%s", (jid,))
39 38 assert r["status"] == "queued" and r["later"] and "SECRETVALUE" not in r["last_error"]
40 39 with transaction() as conn:
@@ -71,7 +70,7 @@ def test_metadata_then_incremental_is_idempotent(vendor, lake, log):
71 70 assert r3.get("skipped") and r3["reason"] == "archive already ingested" and _rows() == v1
72 71
73 72 # vendor republishes the same session with a corrected bar → supersede (no duplicates, version 2)
74 − fixed = (lake.root.parent / "x").exists() # noqa: F841
73 + fixed = (lake.root.parent / "x").exists()
75 74 import pathlib
76 75 txt = (pathlib.Path(__file__).parent / "fixtures" / "firstrate" / "AAPL_day_1min_UNADJUSTED.txt").read_text()
77 76 txt = txt.replace("2026-10-30 09:30:00,346.63,348.51,346.62,348.26,1119437", "2026-10-30 09:30:00,346.63,348.51,346.62,348.27,1119437")
modified tests/test_parse_firstrate.py +0 −2
@@ -3,8 +3,6 @@ from __future__ import annotations
3 3 from datetime import UTC, date, datetime
4 4 from pathlib import Path
5 5
6 −import pyarrow as pa
7 −
8 6 from findata_market_firstrate.archive import parse_bar_file, ticker_from_filename
9 7 from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval
10 8
modified tests/test_redaction.py +1 −2
@@ -1,10 +1,9 @@
1 1 from __future__ import annotations
2 2
3 −import httpx
4 3 import pytest
5 −
6 4 from findata_core.redact import redact
7 5 from findata_market_firstrate.client import FRDClient, FRDError
6 +
8 7 from tests.conftest import TEST_USERID
9 8
10 9
modified tests/test_validators_aggregate.py +1 −3
@@ -3,8 +3,6 @@ from __future__ import annotations
3 3 from datetime import UTC, date, datetime
4 4 from pathlib import Path
5 5
6 −import pyarrow as pa
7 −
8 6 from findata_market_firstrate.archive import parse_bar_file
9 7 from findata_quality import aggregate_bars, validate_bars
10 8 from findata_quality.calendar import xnys
@@ -45,7 +43,7 @@ def test_row_level_flags():
45 43 ])
46 44 out, anomalies = validate_bars(_parse(rows), Interval.m1)
47 45 by_ts = {r["timestamp_utc"]: set(r["quality_flags"]) for r in out.to_pylist()}
48 − f = lambda h, m, d=30: by_ts[datetime(2026, 10, d, h, m, tzinfo=UTC)] # noqa: E731
46 + f = lambda h, m, d=30: by_ts[datetime(2026, 10, d, h, m, tzinfo=UTC)]
49 47 assert QualityFlag.ohlc_inconsistent in f(13, 30)
50 48 assert QualityFlag.negative_volume in f(13, 31)
51 49 assert any(QualityFlag.off_interval in v for v in by_ts.values())
modified tests/test_z_api_sdk.py +7 −6
@@ -2,21 +2,20 @@
2 2 from __future__ import annotations
3 3
4 4 import io
5 −import json
6 5 import time
7 6 from datetime import UTC, date, datetime
8 7
9 8 import httpx
10 −import pyarrow.ipc as ipc
11 9 import pyarrow.parquet as pq
12 10 import pyotp
13 11 import pytest
14 12 from fastapi.testclient import TestClient
15 −
16 13 from findata_api import auth as A
17 14 from findata_core.db import connection, fetch_one, transaction
18 15 from findata_ingest import queue
19 16 from findata_ingest.tasks import incremental, metadata
17 +from pyarrow import ipc
18 +
20 19 from tests.conftest import TEST_USERID
21 20
22 21 OWNER = "owner@example.com"
@@ -55,9 +54,10 @@ def app_client(vendor_module, lake_module):
55 54
56 55 @pytest.fixture(scope="module")
57 56 def vendor_module():
58 − from tests.conftest import VendorMock
59 − import findata_market_firstrate.client as C
60 57 import findata_fundamentals.firstrate as F
58 + import findata_market_firstrate.client as C
59 +
60 + from tests.conftest import VendorMock
61 61 vm = VendorMock()
62 62 old = (C.DEFAULT_TRANSPORT, F.DEFAULT_TRANSPORT)
63 63 C.DEFAULT_TRANSPORT = httpx.MockTransport(vm)
@@ -225,9 +225,10 @@ def test_query_sandbox_and_quotas(app_client, api_key):
225 225
226 226 def test_export_roundtrip_and_sdk(app_client, api_key):
227 227 c = app_client
228 − from findata_sdk import FinData
229 228 import threading
229 +
230 230 import uvicorn
231 + from findata_sdk import FinData
231 232 # the sync SDK needs a real HTTP server: run uvicorn on a loopback port in a thread
232 233 config = uvicorn.Config(c.app, host="127.0.0.1", port=8699, log_level="warning")
233 234 server = uvicorn.Server(config)
234 235
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere