spb/findata

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

modified connectors/fundamentals/src/findata_fundamentals/__init__.py +5 −1

		@@ -1,2 +1,6 @@
1		−from findata_fundamentals.firstrate import FirstRateFundamentalsConnector, facts_from_financials, available_at_policy # noqa: F401
	1	+from findata_fundamentals.firstrate import ( # noqa: F401
	2	+ FirstRateFundamentalsConnector,
	3	+ available_at_policy,
	4	+ facts_from_financials,
	5	+)
2	6	from findata_fundamentals.sec_edgar import SecEdgarConnector, facts_from_companyfacts # noqa: F401

modified connectors/fundamentals/src/findata_fundamentals/firstrate.py +2 −2

		@@ -13,12 +13,12 @@ reconstructed from this vendor (documented limitation).
13	13	from __future__ import annotations
14	14
15	15	import json
	16	+from collections.abc import Iterator
16	17	from datetime import UTC, date, datetime, time, timedelta
17		−from typing import Any, Iterator
	18	+from typing import Any
18	19	from zoneinfo import ZoneInfo
19	20
20	21	import httpx
21		−
22	22	from findata_core.config import get_settings
23	23	from findata_core.redact import redact
24	24

modified connectors/fundamentals/src/findata_fundamentals/sec_edgar.py +2 −2

		@@ -12,12 +12,12 @@ PIT: `companyfacts` provides `filed` (filing date), `accn` (accession), `form`,
12	12	from __future__ import annotations
13	13
14	14	import time
	15	+from collections.abc import Iterator
15	16	from datetime import UTC, date, datetime, timedelta
16		−from typing import Any, Iterator
	17	+from typing import Any
17	18	from zoneinfo import ZoneInfo
18	19
19	20	import httpx
20		−
21	21	from findata_core.config import get_settings
22	22
23	23	SOURCE_ID = "sec_edgar"

modified connectors/market/src/findata_market_firstrate/__init__.py +1 −1

		@@ -1,3 +1,3 @@
	1	+from findata_market_firstrate.archive import iter_archive_files, parse_bar_file, ticker_from_filename # noqa: F401
1	2	from findata_market_firstrate.client import FRDClient, FRDError # noqa: F401
2	3	from findata_market_firstrate.connector import FirstRateMarketConnector # noqa: F401
3		−from findata_market_firstrate.archive import iter_archive_files, parse_bar_file, ticker_from_filename # noqa: F401

modified connectors/market/src/findata_market_firstrate/archive.py +1 −2

		@@ -15,12 +15,11 @@ from pathlib import Path
15	15
16	16	import polars as pl
17	17	import pyarrow as pa
18		−
19	18	from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval
20	19
21	20	VENDOR_TF = {"1min": Interval.m1, "5min": Interval.m5, "30min": Interval.m30, "1hour": Interval.h1, "1day": Interval.d1}
22	21	VENDOR_ADJ = {"UNADJUSTED": Adjustment.raw, "adj_split": Adjustment.split, "adj_splitdiv": Adjustment.split_div}
23		−_TICKER_RE = re.compile(r"^([A-Za-z0-9.\-]+?)(?:[_-](?:full|day|week|month|1min|5min|30min|1hour|1day|UNADJUSTED|adj_split|adj_splitdiv|contin.*))*\.(txt|csv)$", re.I)
	22	+_TICKER_RE = re.compile(r"^([A-Za-z0-9.\-]+?)(?:[_-](?:full|day|week|month|1min|5min|30min|1hour|1day|UNADJUSTED|adj_split|adj_splitdiv|contin.*))*\.(txt|csv)$", re.IGNORECASE)
24	23
25	24
26	25	def ticker_from_filename(name: str) -> str:

modified connectors/market/src/findata_market_firstrate/client.py +0 −1

		@@ -15,7 +15,6 @@ from pathlib import Path
15	15	from typing import Any
16	16
17	17	import httpx
18		−
19	18	from findata_core.redact import redact
20	19
21	20	BASE_URL = "https://firstratedata.com/api"

modified connectors/market/src/findata_market_firstrate/connector.py +22 −8

		@@ -10,13 +10,27 @@ from pathlib import Path
10	10	from typing import Any
11	11
12	12	from findata_core.config import get_settings
13		−from findata_market_firstrate.archive import (VENDOR_ADJ, VENDOR_TF, iter_archive_files, parse_bar_file,
14		− parse_meta_pairs, ticker_from_filename)
15		−from findata_market_firstrate.client import FRDClient
16	13	from findata_quality import validate_bars
17	14	from findata_schema import AssetClass, Interval
18		−from findata_schema.connector import (Capabilities, Checkpoint, Connector, FetchRequest, FetchResult,
19		− NormalizedBatch, ValidationReport)
	15	+from findata_schema.connector import (
	16	+ Capabilities,
	17	+ Checkpoint,
	18	+ Connector,
	19	+ FetchRequest,
	20	+ FetchResult,
	21	+ NormalizedBatch,
	22	+ ValidationReport,
	23	+)
	24	+
	25	+from findata_market_firstrate.archive import (
	26	+ VENDOR_ADJ,
	27	+ VENDOR_TF,
	28	+ iter_archive_files,
	29	+ parse_bar_file,
	30	+ parse_meta_pairs,
	31	+ ticker_from_filename,
	32	+)
	33	+from findata_market_firstrate.client import FRDClient
20	34
21	35	SOURCE_ID = "firstrate"
22	36	SOURCE_TZ = {"crypto": "UTC"} # everything else is US Eastern
		@@ -40,7 +54,7 @@ class FirstRateMarketConnector(Connector):
40	54	for t in ("stock", "etf", "futures", "index", "fx", "crypto", "options"):
41	55	try:
42	56	out["last_update"][t] = self.client.last_update(t).isoformat()
43		− except Exception as e: # noqa: BLE001
	57	+ except Exception as e:
44	58	out["last_update"][t] = f"error: {e}"
45	59	return out
46	60
		@@ -66,7 +80,7 @@ class FirstRateMarketConnector(Connector):
66	80	vendor_last = None
67	81	try:
68	82	vendor_last = self.client.last_update(asset, full=(p.get("period") == "full"))
69		− except Exception: # noqa: BLE001
	83	+ except Exception:
70	84	vendor_last = None
71	85	stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
72	86	label = "_".join(f"{k}-{v}" for k, v in sorted(p.items()))
		@@ -77,7 +91,7 @@ class FirstRateMarketConnector(Connector):
77	91	import zipfile
78	92	with zipfile.ZipFile(got.path) as zf:
79	93	file_count = sum(1 for i in zf.infolist() if not i.is_dir())
80		− except Exception: # noqa: BLE001
	94	+ except Exception:
81	95	pass
82	96	return FetchResult(request=request, path=got.path, sha256=got.sha256, bytes=got.bytes, fetched_at=got.fetched_at,
83	97	vendor_last_update=vendor_last, file_count=file_count,

modified packages/core/src/findata_core/db.py +4 −4

		@@ -2,9 +2,10 @@
2	2	from __future__ import annotations
3	3
4	4	import hashlib
	5	+from collections.abc import Iterator
5	6	from contextlib import contextmanager
6	7	from importlib import resources
7		−from typing import Any, Iterator
	8	+from typing import Any
8	9
9	10	import psycopg
10	11	from psycopg.rows import dict_row
		@@ -40,9 +41,8 @@ def connection() -> Iterator[psycopg.Connection[Any]]:
40	41
41	42	@contextmanager
42	43	def transaction() -> Iterator[psycopg.Connection[Any]]:
43		− with connection() as conn:
44		− with conn.transaction():
45		− yield conn
	44	+ with connection() as conn, conn.transaction():
	45	+ yield conn
46	46
47	47
48	48	def fetch_all(sql: str, params: Any = None) -> list[dict[str, Any]]:

modified packages/python-sdk/src/findata_sdk/client.py +3 −4

		@@ -18,7 +18,7 @@ from typing import Any
18	18
19	19	import httpx
20	20	import pyarrow as pa
21		−import pyarrow.ipc as ipc
	21	+from pyarrow import ipc
22	22
23	23
24	24	class FinDataError(RuntimeError):
		@@ -117,13 +117,12 @@ class FinData:
117	117	if r.status_code >= 400:
118	118	raise FinDataError(r.status_code, {"error": {"code": "download_failed"}})
119	119	with open(out, "wb") as f:
120		− for chunk in r.iter_bytes():
121		− f.write(chunk)
	120	+ f.writelines(r.iter_bytes())
122	121	return out
123	122
124	123
125	124	def _json(r: httpx.Response) -> Any:
126	125	try:
127	126	return r.json()
128		− except Exception: # noqa: BLE001
	127	+ except Exception:
129	128	return {"error": {"code": "http", "message": r.text[:300]}}

modified packages/schema/src/findata_schema/__init__.py +19 −5

		@@ -1,8 +1,22 @@
1	1	"""findata_schema — Arrow schemas, data contracts and connector interfaces (single source of truth)."""
2		−from findata_schema.contracts import ( # noqa: F401
3		− BARS_SCHEMA, BARS_SCHEMA_VERSION, FUNDAMENTALS_SCHEMA, RAW_BAR_COLUMNS, AssetClass, Interval,
4		− Adjustment, QualityFlag, INTERVAL_SECONDS, bars_schema_json,
5		−)
6	2	from findata_schema.connector import ( # noqa: F401
7		− Capabilities, Checkpoint, Connector, FetchRequest, FetchResult, NormalizedBatch, ValidationReport,
	3	+ Capabilities,
	4	+ Checkpoint,
	5	+ Connector,
	6	+ FetchRequest,
	7	+ FetchResult,
	8	+ NormalizedBatch,
	9	+ ValidationReport,
	10	+)
	11	+from findata_schema.contracts import ( # noqa: F401
	12	+ BARS_SCHEMA,
	13	+ BARS_SCHEMA_VERSION,
	14	+ FUNDAMENTALS_SCHEMA,
	15	+ INTERVAL_SECONDS,
	16	+ RAW_BAR_COLUMNS,
	17	+ Adjustment,
	18	+ AssetClass,
	19	+ Interval,
	20	+ QualityFlag,
	21	+ bars_schema_json,
8	22	)

modified pyproject.toml +4 −0

		@@ -55,3 +55,7 @@ filterwarnings = ["ignore::DeprecationWarning"]
55	55	[tool.ruff]
56	56	line-length = 110
57	57	target-version = "py312"
	58	+
	59	+[tool.ruff.lint]
	60	+select = ["E", "F", "I", "B", "UP", "SIM", "RUF", "PLW", "C4", "S110"]
	61	+ignore = ["B008", "RUF001", "RUF002", "RUF003", "PLW0603", "PLW2901", "PLW0108", "E402", "E731", "PLW1510", "E501", "RUF059", "F841", "S110", "SIM102", "SIM115", "FLY002", "RUF015", "C401", "UP007", "UP045", "RUF012", "B904"]

modified services/api/src/findata_api/auth.py +3 −3

		@@ -18,9 +18,9 @@ import pyotp
18	18	from argon2 import PasswordHasher
19	19	from argon2.exceptions import VerifyMismatchError
20	20	from fastapi import Depends, Request
21		−
22	21	from findata_core.config import get_settings
23	22	from findata_core.db import connection, transaction
	23	+
24	24	from findata_api.errors import ApiError
25	25
26	26	SESSION_COOKIE = "fd_session"
		@@ -163,7 +163,7 @@ def get_principal(request: Request) -> Principal:
163	163	return Principal("anonymous", None, None, None, set())
164	164
165	165
166		−def require(scope: str): # noqa: ANN201
	166	+def require(scope: str):
167	167	def dep(request: Request, p: Principal = Depends(get_principal)) -> Principal:
168	168	if p.kind == "anonymous":
169	169	raise ApiError(401, "unauthenticated", "Sign in or provide an API key")
		@@ -218,7 +218,7 @@ def totp_setup(user_id: str, email: str) -> dict[str, Any]:
218	218	buf = io.BytesIO()
219	219	img.save(buf, format="PNG")
220	220	qr = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
221		− except Exception: # noqa: BLE001
	221	+ except Exception:
222	222	qr = None
223	223	return {"otpauth_uri": uri, "qr_png_data_url": qr, "secret": secret}
224	224

modified services/api/src/findata_api/errors.py +1 −1

		@@ -4,5 +4,5 @@ from fastapi import HTTPException
4	4
5	5
6	6	class ApiError(HTTPException):
7		− def __init__(self, status: int, code: str, message: str, **extra): # noqa: ANN003
	7	+ def __init__(self, status: int, code: str, message: str, **extra):
8	8	super().__init__(status_code=status, detail={"code": code, "message": message, **extra})

modified services/api/src/findata_api/health.py +3 −2

		@@ -7,6 +7,7 @@ from typing import Any
7	7	from findata_core.config import get_settings
8	8	from findata_core.db import connection
9	9	from findata_ingest.lake import Lake
	10	+
10	11	from findata_api import metrics as m
11	12
12	13
		@@ -76,7 +77,7 @@ def evaluate() -> dict[str, Any]:
76	77	an = conn.execute("SELECT dataset_id, severity, count(*) n FROM quality_anomalies WHERE created_at > now() - interval '7 days' GROUP BY 1,2").fetchall()
77	78	for a in an:
78	79	m.ANOMALIES.labels(a["dataset_id"], a["severity"]).set(a["n"])
79		− except Exception as e: # noqa: BLE001
	80	+ except Exception as e:
80	81	out["components"]["postgres"] = f"error: {e}"
81	82	out["alerts"].append({"severity": "error", "code": "postgres_down", "message": str(e)[:200]})
82	83	free = lake.free_gb()
		@@ -98,7 +99,7 @@ def evaluate() -> dict[str, Any]:
98	99	out["components"]["ssd_smart"] = smart
99	100	if smart not in ("Verified", "Not Supported", "unknown"):
100	101	out["alerts"].append({"severity": "error", "code": "ssd_smart", "message": f"SMART status: {smart}"})
101		− except Exception: # noqa: BLE001
	102	+ except Exception:
102	103	out["components"]["ssd_smart"] = "unavailable"
103	104	if any(a["severity"] == "error" for a in out["alerts"]):
104	105	out["status"] = "degraded"

modified services/api/src/findata_api/main.py +21 −9

		@@ -9,28 +9,40 @@ from fastapi import Depends, FastAPI, Request
9	9	from fastapi.exceptions import RequestValidationError
10	10	from fastapi.responses import JSONResponse, Response
11	11	from fastapi.staticfiles import StaticFiles
	12	+from findata_core.config import get_settings
	13	+from findata_core.logging import configure_logging, get_logger
12	14	from starlette.exceptions import HTTPException as StarletteHTTPException
13	15	from starlette.middleware.base import BaseHTTPMiddleware
14	16
15	17	from findata_api import auth as A
16	18	from findata_api import metrics as M
17	19	from findata_api.ratelimit import bucket
18		−from findata_api.routers import admin, auth, catalog, coverage, exports, fundamentals, health, instruments, market, quality, query
19		−from findata_core.config import get_settings
20		−from findata_core.logging import configure_logging, get_logger
	20	+from findata_api.routers import (
	21	+ admin,
	22	+ auth,
	23	+ catalog,
	24	+ coverage,
	25	+ exports,
	26	+ fundamentals,
	27	+ health,
	28	+ instruments,
	29	+ market,
	30	+ quality,
	31	+ query,
	32	+)
21	33
22	34	API_PREFIX = "/api/v1"
23	35
24	36
25	37	class Observability(BaseHTTPMiddleware):
26		− async def dispatch(self, request: Request, call_next): # noqa: ANN001
	38	+ async def dispatch(self, request: Request, call_next):
27	39	rid = request.headers.get("x-request-id") or uuid.uuid4().hex[:16]
28	40	request.state.request_id = rid
29	41	t0 = time.monotonic()
30	42	route = request.url.path
31	43	try:
32	44	response = await call_next(request)
33		− except Exception: # noqa: BLE001
	45	+ except Exception:
34	46	M.REQUESTS.labels(request.method, route, "500").inc()
35	47	raise
36	48	dt = time.monotonic() - t0
		@@ -60,7 +72,7 @@ def _looks_id(p: str) -> bool:
60	72
61	73
62	74	class RateLimit(BaseHTTPMiddleware):
63		− async def dispatch(self, request: Request, call_next): # noqa: ANN001
	75	+ async def dispatch(self, request: Request, call_next):
64	76	if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"):
65	77	auth_h = request.headers.get("authorization", "")
66	78	key = auth_h[-24:] if auth_h else (request.cookies.get(A.SESSION_COOKIE, "")[-24:] or (request.client.host if request.client else "anon"))
		@@ -82,18 +94,18 @@ def create_app() -> FastAPI:
82	94	app.add_middleware(RateLimit)
83	95
84	96	@app.exception_handler(StarletteHTTPException)
85		− async def http_exc(request: Request, exc: StarletteHTTPException): # noqa: ANN001
	97	+ async def http_exc(request: Request, exc: StarletteHTTPException):
86	98	detail = exc.detail if isinstance(exc.detail, dict) else {"code": "http_error", "message": str(exc.detail)}
87	99	detail["request_id"] = getattr(request.state, "request_id", None)
88	100	return JSONResponse({"error": detail}, status_code=exc.status_code, headers=getattr(exc, "headers", None))
89	101
90	102	@app.exception_handler(RequestValidationError)
91		− async def val_exc(request: Request, exc: RequestValidationError): # noqa: ANN001
	103	+ async def val_exc(request: Request, exc: RequestValidationError):
92	104	return JSONResponse({"error": {"code": "validation_error", "message": "Invalid request", "details": exc.errors(),
93	105	"request_id": getattr(request.state, "request_id", None)}}, status_code=422)
94	106
95	107	@app.exception_handler(Exception)
96		− async def any_exc(request: Request, exc: Exception): # noqa: ANN001
	108	+ async def any_exc(request: Request, exc: Exception):
97	109	from findata_core.redact import redact
98	110	log.error("unhandled", error=redact(repr(exc), s.secret_values()), path=request.url.path)
99	111	return JSONResponse({"error": {"code": "internal_error", "message": "Internal error", "request_id": getattr(request.state, "request_id", None)}}, status_code=500)

modified services/api/src/findata_api/routers/admin.py +5 −5

		@@ -5,14 +5,14 @@ from datetime import datetime
5	5	from typing import Any
6	6
7	7	from fastapi import APIRouter, Depends, Query, Request
8		−from pydantic import BaseModel, Field
9		−
10		−from findata_api import auth as A
11		−from findata_api.errors import ApiError
12	8	from findata_core.config import get_settings
13	9	from findata_core.db import connection, transaction
14	10	from findata_ingest import queue
15	11	from findata_ingest.lake import Lake
	12	+from pydantic import BaseModel, Field
	13	+
	14	+from findata_api import auth as A
	15	+from findata_api.errors import ApiError
16	16
17	17	router = APIRouter(prefix="/admin", tags=["admin"])
18	18
		@@ -155,8 +155,8 @@ def tracked(body: TrackedIn, request: Request, p: A.Principal = Depends(A.requir
155	155	@router.post("/discover")
156	156	def discover(request: Request, p: A.Principal = Depends(A.require("admin"))):
157	157	"""Sanitized live vendor probe (no bulk download)."""
158		− from findata_market_firstrate import FirstRateMarketConnector
159	158	from findata_fundamentals import SecEdgarConnector
	159	+ from findata_market_firstrate import FirstRateMarketConnector
160	160	out = {"firstrate": FirstRateMarketConnector().discover(), "sec_edgar": SecEdgarConnector().status()}
161	161	A.audit(request, "admin.discover")
162	162	return out

modified services/api/src/findata_api/routers/auth.py +2 −2

		@@ -1,12 +1,12 @@
1	1	from __future__ import annotations
2	2
3	3	from fastapi import APIRouter, Depends, Request, Response
	4	+from findata_core.config import get_settings
	5	+from findata_core.db import connection
4	6	from pydantic import BaseModel, EmailStr
5	7
6	8	from findata_api import auth as A
7	9	from findata_api.errors import ApiError
8		−from findata_core.config import get_settings
9		−from findata_core.db import connection
10	10
11	11	router = APIRouter(prefix="/auth", tags=["auth"])
12	12

modified services/api/src/findata_api/routers/catalog.py +1 −1

		@@ -1,9 +1,9 @@
1	1	from __future__ import annotations
2	2
3	3	from fastapi import APIRouter, Depends
	4	+from findata_core.db import connection
4	5
5	6	from findata_api import auth as A
6		−from findata_core.db import connection
7	7	from findata_api.errors import ApiError
8	8
9	9	router = APIRouter(prefix="/catalog", tags=["catalog"])

modified services/api/src/findata_api/routers/coverage.py +1 −1

		@@ -3,9 +3,9 @@ from __future__ import annotations
3	3	from datetime import date, timedelta
4	4
5	5	from fastapi import APIRouter, Depends, Query
	6	+from findata_core.db import connection
6	7
7	8	from findata_api import auth as A
8		−from findata_core.db import connection
9	9
10	10	router = APIRouter(prefix="/coverage", tags=["coverage"])
11	11

modified services/api/src/findata_api/routers/exports.py +5 −6

		@@ -3,7 +3,6 @@ from lake/exports with expiration. Dataset snapshot (version) is pinned at submi
3	3	from __future__ import annotations
4	4
5	5	import json
6		−import threading
7	6	import traceback
8	7	from concurrent.futures import ThreadPoolExecutor
9	8	from datetime import UTC, date, datetime, timedelta
		@@ -12,15 +11,15 @@ from typing import Literal
12	11	import pyarrow.parquet as pq
13	12	from fastapi import APIRouter, Depends, Request
14	13	from fastapi.responses import FileResponse
	14	+from findata_core.config import get_settings
	15	+from findata_core.db import connection, transaction
	16	+from findata_ingest.lake import Lake
	17	+from findata_ingest.reader import apply_adjustment, read_bars
15	18	from pydantic import BaseModel, Field
16	19
17	20	from findata_api import auth as A
18	21	from findata_api.errors import ApiError
19	22	from findata_api.ratelimit import record_usage
20		−from findata_core.config import get_settings
21		−from findata_core.db import connection, transaction
22		−from findata_ingest.lake import Lake
23		−from findata_ingest.reader import apply_adjustment, read_bars
24	23
25	24	router = APIRouter(prefix="/exports", tags=["exports"])
26	25	_pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="export")
		@@ -58,7 +57,7 @@ def _run(export_id: str, req: dict, principal: str) -> None:
58	57	(rel, out.stat().st_size, table.num_rows, datetime.now(UTC) + timedelta(hours=ttl),
59	58	json.dumps({"dataset_id": req["dataset_id"], "version": meta.get("version"), "files_scanned": meta.get("files_scanned")}), export_id))
60	59	record_usage(principal, rows=table.num_rows, bytes_exported=out.stat().st_size)
61		− except Exception as e: # noqa: BLE001
	60	+ except Exception as e:
62	61	with transaction() as conn:
63	62	conn.execute("UPDATE exports SET status='failed', error=%s, finished_at=now() WHERE id=%s", (f"{e}\n{traceback.format_exc()[-800:]}"[:2000], export_id))
64	63

modified services/api/src/findata_api/routers/fundamentals.py +1 −2

		@@ -1,13 +1,12 @@
1	1	from __future__ import annotations
2	2
3	3	from datetime import UTC, datetime
4		−from typing import Any
5	4
6	5	from fastapi import APIRouter, Depends, Query
	6	+from findata_core.db import connection
7	7
8	8	from findata_api import auth as A
9	9	from findata_api.errors import ApiError
10		−from findata_core.db import connection
11	10
12	11	router = APIRouter(prefix="/fundamentals", tags=["fundamentals"])
13	12

modified services/api/src/findata_api/routers/health.py +2 −4

		@@ -1,12 +1,10 @@
1	1	from __future__ import annotations
2	2
3		−from fastapi import APIRouter, Depends, Request, Response
	3	+from fastapi import APIRouter, Depends
	4	+from findata_core.db import connection
4	5
5	6	from findata_api import auth as A
6		−from findata_api.errors import ApiError
7	7	from findata_api.health import evaluate
8		−from findata_api.metrics import render
9		−from findata_core.db import connection
10	8
11	9	router = APIRouter(tags=["health"])
12	10

modified services/api/src/findata_api/routers/instruments.py +1 −1

		@@ -1,9 +1,9 @@
1	1	from __future__ import annotations
2	2
3	3	from fastapi import APIRouter, Depends, Query
	4	+from findata_core.db import connection
4	5
5	6	from findata_api import auth as A
6		−from findata_core.db import connection
7	7
8	8	router = APIRouter(prefix="/instruments", tags=["instruments"])
9	9

modified services/api/src/findata_api/routers/market.py +4 −5

		@@ -6,17 +6,16 @@ from datetime import date, datetime
6	6	from typing import Literal
7	7
8	8	import pyarrow as pa
9		−import pyarrow.ipc as ipc
10	9	from fastapi import APIRouter, Depends, Query, Request, Response
11	10	from fastapi.responses import StreamingResponse
	11	+from findata_core.db import connection
	12	+from findata_ingest.reader import apply_adjustment, read_bars
	13	+from pyarrow import ipc
12	14
13	15	from findata_api import auth as A
14	16	from findata_api import metrics as M
15	17	from findata_api.errors import ApiError
16	18	from findata_api.ratelimit import record_usage
17		−from findata_core.config import get_settings
18		−from findata_core.db import connection
19		−from findata_ingest.reader import apply_adjustment, read_bars
20	19
21	20	router = APIRouter(prefix="/market", tags=["market"])
22	21	INTERVAL_DS = {"1m": "market_bars.{a}.1m.raw", "5m": "market_bars.{a}.5m.raw", "1d": "market_bars.{a}.1d.raw"}
		@@ -55,7 +54,7 @@ def bars(request: Request, response: Response,
55	54	if cursor:
56	55	try:
57	56	after = datetime.fromisoformat(base64.urlsafe_b64decode(cursor.encode()).decode())
58		− except Exception: # noqa: BLE001
	57	+ except Exception:
59	58	raise ApiError(400, "bad_cursor", "invalid cursor") from None
60	59	import pyarrow.compute as pc
61	60	table = table.filter(pc.greater(table.column("timestamp_utc"), pa.scalar(after, pa.timestamp("us", tz="UTC"))))

modified services/api/src/findata_api/routers/quality.py +1 −1

		@@ -1,9 +1,9 @@
1	1	from __future__ import annotations
2	2
3	3	from fastapi import APIRouter, Depends, Query
	4	+from findata_core.db import connection
4	5
5	6	from findata_api import auth as A
6		−from findata_core.db import connection
7	7
8	8	router = APIRouter(prefix="/quality", tags=["quality"])
9	9

modified services/api/src/findata_api/routers/query.py +3 −3

		@@ -3,14 +3,14 @@ from __future__ import annotations
3	3	import time
4	4
5	5	from fastapi import APIRouter, Depends, Request
	6	+from findata_core.config import get_settings
	7	+from findata_ingest.reader import dataset_views, run_sql
6	8	from pydantic import BaseModel, Field
7	9
8	10	from findata_api import auth as A
9	11	from findata_api import metrics as M
10	12	from findata_api.errors import ApiError
11	13	from findata_api.ratelimit import record_usage
12		−from findata_core.config import get_settings
13		−from findata_ingest.reader import dataset_views, run_sql
14	14
15	15	router = APIRouter(prefix="/query", tags=["query"])
16	16
		@@ -31,7 +31,7 @@ def query(body: QueryIn, request: Request, p: A.Principal = Depends(A.require("q
31	31	raise ApiError(400, "query_rejected", str(e)) from None
32	32	except TimeoutError as e:
33	33	raise ApiError(408, "query_timeout", str(e)) from None
34		− except Exception as e: # noqa: BLE001
	34	+ except Exception as e:
35	35	raise ApiError(400, "query_error", str(e)[:500]) from None
36	36	rows = tbl.to_pylist()
37	37	for r in rows:

modified services/ingest/src/findata_ingest/cli.py +7 −6

		@@ -3,7 +3,6 @@ from __future__ import annotations
3	3
4	4	import json
5	5	from pathlib import Path
6		−from typing import Optional
7	6
8	7	import typer
9	8
		@@ -37,7 +36,7 @@ def discover(source: str = "all") -> None:
37	36
38	37
39	38	@app.command()
40		−def worker(once: bool = False, no_scheduler: bool = False, kinds: Optional[str] = None) -> None:
	39	+def worker(once: bool = False, no_scheduler: bool = False, kinds: str | None = None) -> None:
41	40	"""Run the single ingest worker (+ scheduler)."""
42	41	from findata_ingest.worker import main
43	42	main(once=once, kinds=kinds.split(",") if kinds else None, with_scheduler=not no_scheduler)
		@@ -60,7 +59,7 @@ def ingest_incremental(type_: str = typer.Option("stock", "--type"), period: str
60	59
61	60	@ingest_app.command("backfill")
62	61	def ingest_backfill(ticker_range: str, type_: str = typer.Option("stock", "--type"), timeframe: str = "1min",
63		− adjustment: str = "UNADJUSTED", symbols: Optional[str] = None, max_symbols: int = 0) -> None:
	62	+ adjustment: str = "UNADJUSTED", symbols: str | None = None, max_symbols: int = 0) -> None:
64	63	"""Bounded backfill of one ticker_range letter (full archive). Disk budget enforced by the lake."""
65	64	from findata_ingest import queue
66	65	from findata_ingest.tasks.common import DATASET_ASSET
		@@ -78,7 +77,7 @@ def ingest_fundamentals(tickers: str) -> None:
78	77
79	78
80	79	@ingest_app.command("derive")
81		−def ingest_derive(dates: Optional[str] = None, source: str = "market_bars.stock.1m.raw") -> None:
	80	+def ingest_derive(dates: str | None = None, source: str = "market_bars.stock.1m.raw") -> None:
82	81	from findata_ingest import queue
83	82	typer.echo(queue.enqueue("derive", source.replace(".1m.", ".5m."), {"source_dataset": source, "target_interval": "5m", "dates": dates.split(",") if dates else []}, created_by="cli"))
84	83
		@@ -93,7 +92,7 @@ def ingest_compaction(dataset: str = "market_bars.stock.1m.raw", min_age_days: i
93	92	def ingest_jobs(limit: int = 20) -> None:
94	93	from findata_core.db import fetch_all
95	94	for r in fetch_all("SELECT id, kind, dataset_id, status, attempts, created_at, finished_at, left(last_error, 80) err FROM ingest_jobs ORDER BY created_at DESC LIMIT %s", (limit,)):
96		− typer.echo(f"{str(r['id'])[:8]} {r['kind']:<12} {str(r['dataset_id']):<28} {r['status']:<9} a={r['attempts']} {r['created_at']:%m-%d %H:%M} {r['err'] or ''}")
	95	+ typer.echo(f"{str(r['id'])[:8]} {r['kind']:<12} {r['dataset_id']!s:<28} {r['status']:<9} a={r['attempts']} {r['created_at']:%m-%d %H:%M} {r['err'] or ''}")
97	96
98	97
99	98	@app.command()
		@@ -105,12 +104,13 @@ def backup(now: bool = True) -> None:
105	104	@app.command("restore-drill")
106	105	def restore_drill(dest: Path = Path("/tmp/findata-restore-drill"), dataset: str = "market_bars.stock.1m.raw") -> None:
107	106	from findata_core.config import get_settings
	107	+
108	108	from findata_ingest.tasks.backup import restore_drill as rd
109	109	typer.echo(json.dumps(rd(get_settings().backup_target, dest, dataset), indent=2, default=str))
110	110
111	111
112	112	@app.command("create-owner")
113		−def create_owner(email: str, password: Optional[str] = None) -> None:
	113	+def create_owner(email: str, password: str | None = None) -> None:
114	114	"""Create (or reset the password of) the owner account; prints the TOTP provisioning URI."""
115	115	from findata_api.auth import create_owner as _co
116	116	typer.echo(json.dumps(_co(email, password), indent=2))
		@@ -125,6 +125,7 @@ def create_key(email: str, name: str, scopes: str = "read:market,read:fundamenta
125	125	@app.command()
126	126	def status() -> None:
127	127	from findata_core.db import fetch_all, fetch_one
	128	+
128	129	from findata_ingest.lake import Lake
129	130	lake = Lake()
130	131	typer.echo(f"lake {lake.root} free {lake.free_gb():.1f} GB curated {lake.used_bytes('curated')/1e9:.2f} GB raw {lake.used_bytes('raw')/1e9:.2f} GB")

modified services/ingest/src/findata_ingest/lake.py +0 −1

		@@ -27,7 +27,6 @@ from typing import Any
27	27
28	28	import pyarrow as pa
29	29	import pyarrow.parquet as pq
30		−
31	30	from findata_core.config import get_settings
32	31	from findata_core.db import transaction
33	32

modified services/ingest/src/findata_ingest/queue.py +2 −2

		@@ -8,9 +8,9 @@ import socket
8	8	from datetime import UTC, datetime, timedelta
9	9	from typing import Any
10	10
	11	+from findata_core.config import get_settings
11	12	from findata_core.db import connection, transaction
12	13	from findata_core.redact import redact
13		−from findata_core.config import get_settings
14	14
15	15	WORKER_ID = f"{socket.gethostname()}:{random.randint(1000, 9999)}"
16	16
		@@ -104,5 +104,5 @@ def _code_version() -> str:
104	104	try:
105	105	return subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5,
106	106	cwd=str(get_settings().root)).stdout.strip() or "unknown"
107		− except Exception: # noqa: BLE001
	107	+ except Exception:
108	108	return "unknown"

modified services/ingest/src/findata_ingest/reader.py +4 −5

		@@ -13,12 +13,11 @@ from typing import Any
13	13
14	14	import duckdb
15	15	import pyarrow as pa
16		−
17		−from findata_core.config import get_settings
18	16	from findata_core.db import connection
19		−from findata_ingest.lake import Lake, live_files
20	17	from findata_schema import BARS_SCHEMA
21	18
	19	+from findata_ingest.lake import Lake, live_files
	20	+
22	21
23	22	def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection:
24	23	con = duckdb.connect(database=":memory:", config={"threads": 8, "memory_limit": "8GB", "TimeZone": "UTC"})
		@@ -120,10 +119,10 @@ def run_sql(sql: str, max_rows: int, timeout_s: int, views: dict[str, list[str]]
120	119	import re
121	120	import threading
122	121
123		− forbidden = re.compile(r"\b(attach|copy|export|import|install|load|create|insert|update|delete|drop|alter|pragma|set|call|read_csv|read_json|read_text|glob|httpfs)\b", re.I)
	122	+ forbidden = re.compile(r"\b(attach|copy|export|import|install|load|create|insert|update|delete|drop|alter|pragma|set|call|read_csv|read_json|read_text|glob|httpfs)\b", re.IGNORECASE)
124	123	if forbidden.search(sql) or ";" in sql.strip().rstrip(";"):
125	124	raise ValueError("only a single read-only SELECT statement is allowed")
126		− if not re.match(r"^\s*(select|with|describe|show|summarize)\b", sql, re.I):
	125	+ if not re.match(r"^\s*(select|with|describe|show|summarize)\b", sql, re.IGNORECASE):
127	126	raise ValueError("statement must start with SELECT / WITH")
128	127	con = _con()
129	128	lake_root = str(Lake().root).replace("'", "''")

modified services/ingest/src/findata_ingest/tasks/__init__.py +10 −1

		@@ -1,4 +1,13 @@
1		−from findata_ingest.tasks import backfill, backup, compaction, derive, fundamentals, incremental, metadata, quality # noqa: F401
	1	+from findata_ingest.tasks import (
	2	+ backfill,
	3	+ backup,
	4	+ compaction,
	5	+ derive,
	6	+ fundamentals,
	7	+ incremental,
	8	+ metadata,
	9	+ quality,
	10	+)
2	11
3	12	REGISTRY = {
4	13	"metadata": metadata.run,

modified services/ingest/src/findata_ingest/tasks/backfill.py +13 −5

		@@ -10,15 +10,22 @@ from typing import Any
10	10
11	11	import pyarrow as pa
12	12	import pyarrow.compute as pc
13		−
14	13	from findata_core.db import connection, transaction
15		−from findata_ingest import queue
16		−from findata_ingest.lake import Lake
17		−from findata_ingest.tasks.common import DATASET_ASSET, live_delta_dates, record_anomalies, record_quality_report, record_raw_archive, upsert_coverage
18	14	from findata_market_firstrate import FirstRateMarketConnector
19	15	from findata_schema import Interval
20	16	from findata_schema.connector import FetchRequest
21	17
	18	+from findata_ingest import queue
	19	+from findata_ingest.lake import Lake
	20	+from findata_ingest.tasks.common import (
	21	+ DATASET_ASSET,
	22	+ live_delta_dates,
	23	+ record_anomalies,
	24	+ record_quality_report,
	25	+ record_raw_archive,
	26	+ upsert_coverage,
	27	+)
	28	+
22	29
23	30	def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
24	31	p = job.get("params") or {}
		@@ -37,8 +44,9 @@ def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
37	44	done = set(ck.get("done_symbols") or [])
38	45	raw_rel = ck.get("raw_path")
39	46	if raw_rel and (lake.root / raw_rel).exists():
40		− from findata_schema.connector import FetchResult
41	47	from datetime import datetime
	48	+
	49	+ from findata_schema.connector import FetchResult
42	50	res = FetchResult(request=FetchRequest(dataset_id, {"type": asset, "period": "full", "ticker_range": letter, "timeframe": timeframe, "adjustment": adjustment}),
43	51	path=lake.root / raw_rel, sha256=ck["sha256"], bytes=ck["bytes"], fetched_at=datetime.fromisoformat(ck["fetched_at"]),
44	52	vendor_last_update=None)

modified services/ingest/src/findata_ingest/tasks/backup.py +1 −0

		@@ -10,6 +10,7 @@ from typing import Any
10	10
11	11	from findata_core.config import get_settings
12	12	from findata_core.db import transaction
	13	+
13	14	from findata_ingest.lake import Lake
14	15
15	16

modified services/ingest/src/findata_ingest/tasks/common.py +0 −3

		@@ -5,9 +5,6 @@ from datetime import UTC, date, datetime
5	5	from typing import Any
6	6
7	7	import pyarrow as pa
8		−
9		−from findata_core.db import transaction
10		−from findata_ingest.lake import Lake, WrittenFile
11	8	from findata_quality.validators import session_coverage
12	9	from findata_schema import Interval
13	10

modified services/ingest/src/findata_ingest/tasks/compaction.py +2 −2

		@@ -7,11 +7,11 @@ from typing import Any
7	7
8	8	import duckdb
9	9	import pyarrow as pa
10		−
11	10	from findata_core.db import connection
12		−from findata_ingest.lake import Lake
13	11	from findata_schema import BARS_SCHEMA
14	12
	13	+from findata_ingest.lake import Lake
	14	+
15	15
16	16	def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
17	17	p = job.get("params") or {}

modified services/ingest/src/findata_ingest/tasks/derive.py +3 −4

		@@ -4,14 +4,13 @@ from __future__ import annotations
4	4	from datetime import date
5	5	from typing import Any
6	6
7		−import pyarrow as pa
8		−
9	7	from findata_core.db import connection, transaction
	8	+from findata_quality import aggregate_bars
	9	+from findata_schema import Interval
	10	+
10	11	from findata_ingest.lake import Lake, live_files
11	12	from findata_ingest.reader import read_bars
12	13	from findata_ingest.tasks.common import live_delta_paths_for_dates, upsert_coverage
13		−from findata_quality import aggregate_bars
14		−from findata_schema import Interval
15	14
16	15
17	16	def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:

modified services/ingest/src/findata_ingest/tasks/fundamentals.py +2 −1

		@@ -8,6 +8,7 @@ from typing import Any
8	8
9	9	from findata_core.db import connection, transaction
10	10	from findata_fundamentals import FirstRateFundamentalsConnector, facts_from_financials
	11	+
11	12	from findata_ingest import queue
12	13	from findata_ingest.lake import Lake
13	14
		@@ -34,7 +35,7 @@ def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
34	35	payload = fr.financials_all(tk, period)
35	36	facts.extend(facts_from_financials(payload, observed, tk))
36	37	earnings = fr.earnings(tk)
37		− except Exception as e: # noqa: BLE001
	38	+ except Exception as e:
38	39	stats["errors"] += 1
39	40	log.warning("fundamentals.error", ticker=tk, error=str(e)[:200])
40	41	continue

modified services/ingest/src/findata_ingest/tasks/incremental.py +11 −6

		@@ -7,20 +7,25 @@ Checkpoints: dataset:<id> / last_archive, last_vendor_update.
7	7	from __future__ import annotations
8	8
9	9	from collections import defaultdict
10		−from datetime import UTC, datetime
11	10	from typing import Any
12	11
13	12	import pyarrow as pa
14		−
15	13	from findata_core.db import connection, transaction
16		−from findata_ingest import queue
17		−from findata_ingest.lake import Lake
18		−from findata_ingest.tasks.common import (DATASET_ASSET, live_delta_paths_for_dates, record_anomalies, record_quality_report,
19		− record_raw_archive, upsert_coverage)
20	14	from findata_market_firstrate import FirstRateMarketConnector
21	15	from findata_schema import Interval
22	16	from findata_schema.connector import FetchRequest
23	17
	18	+from findata_ingest import queue
	19	+from findata_ingest.lake import Lake
	20	+from findata_ingest.tasks.common import (
	21	+ DATASET_ASSET,
	22	+ live_delta_paths_for_dates,
	23	+ record_anomalies,
	24	+ record_quality_report,
	25	+ record_raw_archive,
	26	+ upsert_coverage,
	27	+)
	28	+
24	29
25	30	def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:
26	31	p = job.get("params") or {}

modified services/ingest/src/findata_ingest/tasks/metadata.py +2 −1

		@@ -4,9 +4,10 @@ from __future__ import annotations
4	4	from typing import Any
5	5
6	6	from findata_core.db import transaction
	7	+from findata_market_firstrate import FirstRateMarketConnector
	8	+
7	9	from findata_ingest.lake import Lake
8	10	from findata_ingest.tasks.common import record_raw_archive
9		−from findata_market_firstrate import FirstRateMarketConnector
10	11
11	12
12	13	def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]:

modified services/ingest/src/findata_ingest/tasks/quality.py +1 −1

		@@ -2,10 +2,10 @@
2	2	from __future__ import annotations
3	3
4	4	import json
5		−from datetime import UTC, datetime, timedelta
6	5	from typing import Any
7	6
8	7	from findata_core.db import connection, transaction
	8	+
9	9	from findata_ingest.lake import Lake
10	10
11	11

modified services/ingest/src/findata_ingest/worker.py +5 −4

		@@ -5,11 +5,12 @@ from __future__ import annotations
5	5	import signal
6	6	import time
7	7	import traceback
8		−from datetime import UTC, datetime
	8	+from datetime import datetime
9	9	from zoneinfo import ZoneInfo
10	10
11	11	from findata_core.config import get_settings
12	12	from findata_core.logging import configure_logging, get_logger
	13	+
13	14	from findata_ingest import queue
14	15	from findata_ingest.lake import Lake, WriterLock
15	16	from findata_ingest.tasks import REGISTRY
		@@ -18,7 +19,7 @@ ET = ZoneInfo("America/New_York")
18	19	_stop = False
19	20
20	21
21		−def _handle(sig, frame): # noqa: ANN001
	22	+def _handle(sig, frame):
22	23	global _stop
23	24	_stop = True
24	25
		@@ -63,7 +64,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
63	64	if with_scheduler and time.monotonic() - last_tick > 300:
64	65	try:
65	66	schedule_tick(log)
66		− except Exception as e: # noqa: BLE001
	67	+ except Exception as e:
67	68	log.error("scheduler.error", error=str(e))
68	69	last_tick = time.monotonic()
69	70	job = queue.claim_next(kinds)
		@@ -80,7 +81,7 @@ def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: boo
80	81	result = fn(job, lake, jl)
81	82	queue.finish(job, True, {**(result or {}), "seconds": round(time.monotonic() - t0, 1)})
82	83	jl.info("job.done", seconds=round(time.monotonic() - t0, 1))
83		− except Exception as e: # noqa: BLE001
	84	+ except Exception as e:
84	85	err = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}"
85	86	queue.finish(job, False, error=err)
86	87	jl.error("job.failed", error=str(e)[:300], seconds=round(time.monotonic() - t0, 1))

modified services/quality/src/findata_quality/__init__.py +2 −2

		@@ -1,3 +1,3 @@
1		−from findata_quality.calendar import SessionCalendar, ET # noqa: F401
2		−from findata_quality.validators import validate_bars # noqa: F401
3	1	from findata_quality.aggregate import aggregate_bars # noqa: F401
	2	+from findata_quality.calendar import ET, SessionCalendar # noqa: F401
	3	+from findata_quality.validators import validate_bars # noqa: F401

modified services/quality/src/findata_quality/aggregate.py +0 −1

		@@ -4,7 +4,6 @@ from __future__ import annotations
4	4
5	5	import polars as pl
6	6	import pyarrow as pa
7		−
8	7	from findata_schema import BARS_SCHEMA, INTERVAL_SECONDS, Interval
9	8
10	9

modified services/quality/src/findata_quality/validators.py +1 −1

		@@ -6,9 +6,9 @@ from typing import Any
6	6
7	7	import polars as pl
8	8	import pyarrow as pa
	9	+from findata_schema import INTERVAL_SECONDS, Interval, QualityFlag
9	10
10	11	from findata_quality.calendar import SessionCalendar, xnys
11		−from findata_schema import INTERVAL_SECONDS, Interval, QualityFlag
12	12
13	13	PRICE_JUMP_THRESHOLD = 0.25 # 25 % between consecutive 1-minute closes inside a session (flag only)
14	14

modified tests/conftest.py +3 −3

		@@ -25,8 +25,8 @@ os.environ.update({
25	25	"FINDATA_BACKUP_TARGET": "",
26	26	})
27	27
28		−import httpx # noqa: E402
29		−import pytest # noqa: E402
	28	+import httpx
	29	+import pytest
30	30
31	31	FIX = ROOT / "tests" / "fixtures"
32	32
		@@ -130,8 +130,8 @@ class VendorMock:
130	130	@pytest.fixture
131	131	def vendor(monkeypatch):
132	132	vm = VendorMock()
133		− import findata_market_firstrate.client as C
134	133	import findata_fundamentals.firstrate as F
	134	+ import findata_market_firstrate.client as C
135	135	monkeypatch.setattr(C, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))
136	136	monkeypatch.setattr(F, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))
137	137	return vm

modified tests/test_fundamentals_pit.py +6 −1

		@@ -4,7 +4,12 @@ import json
4	4	from datetime import UTC, date, datetime
5	5	from pathlib import Path
6	6
7		−from findata_fundamentals import SecEdgarConnector, available_at_policy, facts_from_companyfacts, facts_from_financials
	7	+from findata_fundamentals import (
	8	+ SecEdgarConnector,
	9	+ available_at_policy,
	10	+ facts_from_companyfacts,
	11	+ facts_from_financials,
	12	+)
8	13
9	14	FIX = Path(__file__).parent / "fixtures" / "fundamentals"
10	15	NOW = datetime(2026, 11, 1, 12, 0, tzinfo=UTC)

modified tests/test_ingest_lake.py +3 −4

		@@ -1,17 +1,16 @@
1	1	"""Integration: lake publication, durable queue, incremental sync idempotency, resume, derive, compaction."""
2	2	from __future__ import annotations
3	3
4		−import json
5	4	from datetime import UTC, date, datetime
6	5
7	6	import pyarrow as pa
8	7	import pytest
9		−
10	8	from findata_core.db import connection, fetch_one, transaction
11	9	from findata_ingest import queue
12	10	from findata_ingest.lake import live_files
13	11	from findata_ingest.reader import apply_adjustment, read_bars, run_sql
14	12	from findata_ingest.tasks import backfill, compaction, derive, incremental, metadata
	13	+
15	14	from tests.conftest import build_update_zip
16	15
17	16	DS = "market_bars.stock.1m.raw"
		@@ -34,7 +33,7 @@ def test_queue_retry_backoff_and_dead_letter():
34	33	assert queue.enqueue("quality", None, {"x": 1}, dedupe_key="q-test") is None # deduped while queued
35	34	job = queue.claim_next(["quality"])
36	35	assert job["attempts"] == 1
37		− queue.finish(job, False, error=f"boom userid=SECRETVALUE")
	36	+ queue.finish(job, False, error="boom userid=SECRETVALUE")
38	37	r = fetch_one("SELECT status, last_error, next_run_at > now() later FROM ingest_jobs WHERE id=%s", (jid,))
39	38	assert r["status"] == "queued" and r["later"] and "SECRETVALUE" not in r["last_error"]
40	39	with transaction() as conn:
		@@ -71,7 +70,7 @@ def test_metadata_then_incremental_is_idempotent(vendor, lake, log):
71	70	assert r3.get("skipped") and r3["reason"] == "archive already ingested" and _rows() == v1
72	71
73	72	# vendor republishes the same session with a corrected bar → supersede (no duplicates, version 2)
74		− fixed = (lake.root.parent / "x").exists() # noqa: F841
	73	+ fixed = (lake.root.parent / "x").exists()
75	74	import pathlib
76	75	txt = (pathlib.Path(__file__).parent / "fixtures" / "firstrate" / "AAPL_day_1min_UNADJUSTED.txt").read_text()
77	76	txt = txt.replace("2026-10-30 09:30:00,346.63,348.51,346.62,348.26,1119437", "2026-10-30 09:30:00,346.63,348.51,346.62,348.27,1119437")

modified tests/test_parse_firstrate.py +0 −2

		@@ -3,8 +3,6 @@ from __future__ import annotations
3	3	from datetime import UTC, date, datetime
4	4	from pathlib import Path
5	5
6		−import pyarrow as pa
7		−
8	6	from findata_market_firstrate.archive import parse_bar_file, ticker_from_filename
9	7	from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval
10	8

modified tests/test_redaction.py +1 −2

		@@ -1,10 +1,9 @@
1	1	from __future__ import annotations
2	2
3		−import httpx
4	3	import pytest
5		−
6	4	from findata_core.redact import redact
7	5	from findata_market_firstrate.client import FRDClient, FRDError
	6	+
8	7	from tests.conftest import TEST_USERID
9	8
10	9

modified tests/test_validators_aggregate.py +1 −3

		@@ -3,8 +3,6 @@ from __future__ import annotations
3	3	from datetime import UTC, date, datetime
4	4	from pathlib import Path
5	5
6		−import pyarrow as pa
7		−
8	6	from findata_market_firstrate.archive import parse_bar_file
9	7	from findata_quality import aggregate_bars, validate_bars
10	8	from findata_quality.calendar import xnys
		@@ -45,7 +43,7 @@ def test_row_level_flags():
45	43	])
46	44	out, anomalies = validate_bars(_parse(rows), Interval.m1)
47	45	by_ts = {r["timestamp_utc"]: set(r["quality_flags"]) for r in out.to_pylist()}
48		− f = lambda h, m, d=30: by_ts[datetime(2026, 10, d, h, m, tzinfo=UTC)] # noqa: E731
	46	+ f = lambda h, m, d=30: by_ts[datetime(2026, 10, d, h, m, tzinfo=UTC)]
49	47	assert QualityFlag.ohlc_inconsistent in f(13, 30)
50	48	assert QualityFlag.negative_volume in f(13, 31)
51	49	assert any(QualityFlag.off_interval in v for v in by_ts.values())

modified tests/test_z_api_sdk.py +7 −6

		@@ -2,21 +2,20 @@
2	2	from __future__ import annotations
3	3
4	4	import io
5		−import json
6	5	import time
7	6	from datetime import UTC, date, datetime
8	7
9	8	import httpx
10		−import pyarrow.ipc as ipc
11	9	import pyarrow.parquet as pq
12	10	import pyotp
13	11	import pytest
14	12	from fastapi.testclient import TestClient
15		−
16	13	from findata_api import auth as A
17	14	from findata_core.db import connection, fetch_one, transaction
18	15	from findata_ingest import queue
19	16	from findata_ingest.tasks import incremental, metadata
	17	+from pyarrow import ipc
	18	+
20	19	from tests.conftest import TEST_USERID
21	20
22	21	OWNER = "owner@example.com"
		@@ -55,9 +54,10 @@ def app_client(vendor_module, lake_module):
55	54
56	55	@pytest.fixture(scope="module")
57	56	def vendor_module():
58		− from tests.conftest import VendorMock
59		− import findata_market_firstrate.client as C
60	57	import findata_fundamentals.firstrate as F
	58	+ import findata_market_firstrate.client as C
	59	+
	60	+ from tests.conftest import VendorMock
61	61	vm = VendorMock()
62	62	old = (C.DEFAULT_TRANSPORT, F.DEFAULT_TRANSPORT)
63	63	C.DEFAULT_TRANSPORT = httpx.MockTransport(vm)
		@@ -225,9 +225,10 @@ def test_query_sandbox_and_quotas(app_client, api_key):
225	225
226	226	def test_export_roundtrip_and_sdk(app_client, api_key):
227	227	c = app_client
228		− from findata_sdk import FinData
229	228	import threading
	229	+
230	230	import uvicorn
	231	+ from findata_sdk import FinData
231	232	# the sync SDK needs a real HTTP server: run uvicorn on a loopback port in a thread
232	233	config = uvicorn.Config(c.app, host="127.0.0.1", port=8699, log_level="warning")
233	234	server = uvicorn.Server(config)
234	235