1from __future__ import annotations23from datetime import UTC, date, datetime4from pathlib import Path56from findata_market_firstrate.archive import parse_bar_file, ticker_from_filename7from findata_schema import BARS_SCHEMA, Adjustment, AssetClass, Interval89FIX = Path(__file__).parent / "fixtures" / "firstrate"10NOW = datetime(2026, 11, 1, 12, 0, tzinfo=UTC)111213def _parse(text: str, symbol="TEST", interval=Interval.m1):14 return parse_bar_file(text.encode(), symbol, AssetClass.stock, interval, Adjustment.raw, "firstrate", NOW)151617def test_ticker_from_filename_variants():18 assert ticker_from_filename("AAPL_day_1min_UNADJUSTED.txt") == "AAPL"19 assert ticker_from_filename("AAPL_1min.txt") == "AAPL"20 assert ticker_from_filename("BRK.B_full_1min_adj_split.txt") == "BRK.B"21 assert ticker_from_filename("TWTR-DELISTED_1min.txt") == "TWTR-DELISTED"22 assert ticker_from_filename("nested.zip!/ES_contin_adj_ratio_1min.txt") == "ES"232425def test_parse_real_session_bar_open_convention():26 t, raw = parse_bar_file((FIX / "AAPL_day_1min_UNADJUSTED.txt").read_bytes(), "AAPL", AssetClass.stock, Interval.m1, Adjustment.raw, "firstrate", NOW)27 assert raw == 900 and t.num_rows == 90028 assert t.schema.equals(BARS_SCHEMA)29 first = t.slice(0, 1).to_pylist()[0]30 # 2026-10-30 04:00 America/New_York (EDT, UTC-4) == 08:00 UTC ; timestamp marks the bar OPEN31 assert first["timestamp_utc"] == datetime(2026, 10, 30, 8, 0, tzinfo=UTC)32 assert first["session_date"] == date(2026, 10, 30)33 assert first["open"] == 350.76 and first["volume"] == 4802.034 assert first["instrument_id"] == "firstrate:stock:AAPL" and first["adjustment_type"] == "raw"35 last = t.slice(899, 1).to_pylist()[0]36 assert last["timestamp_utc"] == datetime(2026, 10, 30, 23, 59, tzinfo=UTC) # 19:59 EDT373839def test_dst_spring_forward_and_fall_back():40 # 2026-03-06 (EST, UTC-5): 09:30 ET = 14:30 UTC ; 2026-03-09 (EDT, UTC-4): 09:30 ET = 13:30 UTC41 t, _ = _parse("2026-03-06 09:30:00,1,2,0.5,1.5,10\n2026-03-09 09:30:00,1,2,0.5,1.5,10\n")42 ts = t.column("timestamp_utc").to_pylist()43 assert ts == [datetime(2026, 3, 6, 14, 30, tzinfo=UTC), datetime(2026, 3, 9, 13, 30, tzinfo=UTC)]44 # fall back 2026-11-01: 2026-10-30 09:30 EDT = 13:30 UTC ; 2026-11-02 09:30 EST = 14:30 UTC45 t, _ = _parse("2026-10-30 09:30:00,1,2,0.5,1.5,10\n2026-11-02 09:30:00,1,2,0.5,1.5,10\n")46 ts = t.column("timestamp_utc").to_pylist()47 assert ts == [datetime(2026, 10, 30, 13, 30, tzinfo=UTC), datetime(2026, 11, 2, 14, 30, tzinfo=UTC)]484950def test_nonexistent_local_time_is_dropped_and_counted():51 # 02:30 on 2026-03-08 does not exist in America/New_York52 t, raw = _parse("2026-03-08 02:30:00,1,2,0.5,1.5,10\n2026-03-08 04:00:00,1,2,0.5,1.5,10\n")53 assert raw == 2 and t.num_rows == 1545556def test_crlf_header_and_fx_two_field_layout():57 t, raw = _parse("DateTime,Open,High,Low,Close,Volume\r\n2026-10-30 09:30:00,1,2,0.5,1.5,10\r\n2026-10-30 09:31:00,1,2,0.5,1.5,0\r\n")58 assert raw == 2 and t.num_rows == 259 t, raw = _parse("20261030,09:30:00,1.1,1.2,1.0,1.15,0\n")60 assert t.num_rows == 1 and t.column("timestamp_utc")[0].as_py() == datetime(2026, 10, 30, 13, 30, tzinfo=UTC)616263def test_daily_layout_with_open_interest_column():64 t, raw = _parse("2026-10-30,1,2,0.5,1.5,10,1234\n", interval=Interval.d1)65 assert t.num_rows == 1 and t.column("session_date")[0].as_py() == date(2026, 10, 30)666768def test_duplicates_are_kept_for_flagging_not_dropped():69 t, raw = _parse("2026-10-30 09:30:00,1,2,0.5,1.5,10\n2026-10-30 09:30:00,1,2,0.5,1.5,10\n")70 assert raw == 2 and t.num_rows == 2717273def test_ragged_lines_do_not_break_parsing():74 txt = "2026-10-30 09:30:00,1,2,0.5,1.5,10\n2026-10-30 09:31:00,1,2,0.5,1.5,10,EXTRA\n2026-10-30 09:32:00,1,2,0.5,1.5\n2026-10-30 09:33:00,1,2,0.5,1.5,10\n"75 t, raw = _parse(txt)76 assert raw == 4 and t.num_rows >= 3 # extra field truncated; short line yields nulls and is dropped or kept with volume 077