1"""Test harness: isolated lake in a temp dir, a fresh `findata_test` PostgreSQL database (localhost), and a mocked2FirstRate transport built from the licensed fixture excerpts (one session, three tickers). No real vendor calls."""3from __future__ import annotations45import io6import os7import subprocess8import tempfile9import zipfile10from pathlib import Path1112TEST_USERID = "test-userid-SECRET-9f8e7d6c"13ROOT = Path(__file__).resolve().parents[1]14LAKE_DIR = Path(os.environ.get("FINDATA_LAKE") or tempfile.mkdtemp(prefix="findata-lake-"))15os.environ.setdefault("FINDATA_LAKE", str(LAKE_DIR))16os.environ.update({17 "FINDATA_ENV": "test",18 "FINDATA_ROOT": str(ROOT),19 "FINDATA_DATABASE_URL": os.environ.get("FINDATA_TEST_DATABASE_URL", "postgresql://127.0.0.1:5432/findata_test"),20 "FINDATA_SECRET_KEY": "test-secret-key-test-secret-key-0123456789",21 "FINDATA_PUBLIC_URL": "http://testserver",22 "FRD_USERID": TEST_USERID,23 "SEC_EDGAR_USER_AGENT": "",24 "FINDATA_LAKE_HARD_MIN_FREE_GB": "1",25 "FINDATA_BACKUP_TARGET": "",26})2728import httpx29import pytest3031FIX = ROOT / "tests" / "fixtures"323334def _reset_db() -> None:35 url = os.environ["FINDATA_DATABASE_URL"]36 subprocess.run(["psql", url, "-q", "-c", "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"], check=True, capture_output=True)37 from findata_core.db import close_pool, migrate38 close_pool()39 migrate(verbose=False)404142@pytest.fixture(scope="session", autouse=True)43def database():44 _reset_db()45 yield46 from findata_core.db import close_pool47 close_pool()484950def build_update_zip(tickers: list[str] | None = None, mutate: dict[str, str] | None = None, period: str = "day") -> bytes:51 """Zip shaped like the vendor's `stock_update_day_1min_UNADJUSTED.zip`."""52 tickers = tickers or ["AAPL", "A", "AAL"]53 buf = io.BytesIO()54 with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:55 for t in tickers:56 data = (FIX / "firstrate" / f"{t}_day_1min_UNADJUSTED.txt").read_text()57 if mutate and t in mutate:58 data = mutate[t]59 zf.writestr(zipfile.ZipInfo(f"{t}_{period}_1min_UNADJUSTED.txt", date_time=(2026, 10, 30, 18, 6, 0)), data, compress_type=zipfile.ZIP_DEFLATED)60 return buf.getvalue()616263def build_full_zip(tickers: list[str]) -> bytes:64 buf = io.BytesIO()65 with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:66 for t in tickers:67 zf.writestr(zipfile.ZipInfo(f"{t}_1min.txt", date_time=(2026, 10, 30, 18, 6, 0)), (FIX / "firstrate" / f"{t}_day_1min_UNADJUSTED.txt").read_text(), compress_type=zipfile.ZIP_DEFLATED)68 return buf.getvalue()697071class VendorMock:72 """httpx.MockTransport handler emulating FirstRate endpoints; records sanitized calls; asserts auth."""7374 def __init__(self):75 self.calls: list[str] = []76 self.last_update = "2026-10-30"77 self.archive_override: bytes | None = None78 self.fail_next_download = 07980 def __call__(self, request: httpx.Request) -> httpx.Response:81 url = request.url82 assert url.params.get("userid") == TEST_USERID, "vendor calls must carry the userid"83 self.calls.append(f"{url.path}?{'&'.join(f'{k}={v}' for k, v in sorted(url.params.items()) if k != 'userid')}")84 if url.path == "/api/last_update":85 return httpx.Response(200, text=self.last_update if url.params.get("type") != "fx" else self.last_update.replace("-", ""))86 if url.path == "/api/ticker_listing":87 t = url.params.get("type")88 body = "Ticker,Name,First Date,Last Date\n"89 if t == "stock":90 body += "A,Agilent Technologies Inc,2000-01-04,2026-10-30\nAAL,American Airlines,2005-09-27,2026-10-30\nAAPL,Apple Inc,2000-01-03,2026-10-30\nTWTR-DELISTED,Twitter Inc,2013-11-07,2022-10-27\n"91 else:92 body += "SPY,SPDR S&P 500,2000-01-03,2026-10-30\n"93 return httpx.Response(200, text=body)94 if url.path == "/api/update_log":95 return httpx.Response(200, text="Ticker,Last Date\nTWTR,2022-10-27\n")96 if url.path == "/api/meta_file":97 kind = url.params.get("metafile_type")98 buf = io.BytesIO()99 with zipfile.ZipFile(buf, "w") as zf:100 if kind == "splits":101 zf.writestr("AAPL.txt", "2020-08-31,4\n2014-06-09,7\n2005-02-28,2\n2000-06-21,2\n")102 zf.writestr("_splits_readme.txt", "readme")103 else:104 zf.writestr("AAPL_divs.txt", "2026-08-10,0.27\n2026-05-11,0.27\n")105 return httpx.Response(302, headers={"location": "https://cdn.example/meta.zip"}) if False else httpx.Response(200, content=buf.getvalue(), headers={"content-type": "application/zip"})106 if url.path == "/api/data_file":107 if self.fail_next_download:108 self.fail_next_download -= 1109 return httpx.Response(503, text="busy")110 if url.params.get("period") == "full":111 letter = url.params.get("ticker_range")112 tick = [t for t in ["A", "AAL", "AAPL"] if t.startswith(letter)]113 return httpx.Response(200, content=build_full_zip(tick), headers={"content-type": "application/zip"})114 body = self.archive_override or build_update_zip(period=url.params.get("period", "day"))115 # emulate the CDN redirect116 if url.host == "firstratedata.com":117 return httpx.Response(302, headers={"location": f"https://x01.frd-data.com/stock_update_{url.params.get('period')}_1min_UNADJUSTED_tok.zip?userid={TEST_USERID}"})118 return httpx.Response(200, content=body, headers={"content-type": "application/zip"})119 if url.host == "x01.frd-data.com":120 body = self.archive_override or build_update_zip()121 return httpx.Response(200, content=body, headers={"content-type": "application/zip"})122 if url.path.startswith("/api-f/v1/financials/all/"):123 p = (FIX / "fundamentals" / "financials_all_AAPL_quarterly.json").read_text()124 return httpx.Response(200, text=p, headers={"content-type": "application/json"})125 if url.path.startswith("/api-f/v1/earnings_release/t/"):126 return httpx.Response(200, text=(FIX / "fundamentals" / "earnings_AAPL.json").read_text(), headers={"content-type": "application/json"})127 return httpx.Response(404, text=f"no route {url.path} userid={url.params.get('userid')}")128129130@pytest.fixture131def vendor(monkeypatch):132 vm = VendorMock()133 import findata_fundamentals.firstrate as F134 import findata_market_firstrate.client as C135 monkeypatch.setattr(C, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))136 monkeypatch.setattr(F, "DEFAULT_TRANSPORT", httpx.MockTransport(vm))137 return vm138139140@pytest.fixture141def lake():142 from findata_ingest.lake import Lake143 return Lake(LAKE_DIR)144145146class Log:147 def __getattr__(self, name):148 def f(*a, **k):149 return None150 return f151152 def bind(self, **k):153 return self154155156@pytest.fixture157def log():158 return Log()159