spb/hfmarketdata

1"""Unit tests of the fundamentals engine: mapping, fiscal calendar, resolution, derivation, TTM, versioning, ratios."""2from __future__ import annotations34from datetime import date5from pathlib import Path67import pandas as pd8import pytest910from tests.conftest import PROTO_CIKS, load_gz1112ROOT = Path(__file__).resolve().parents[1]131415def _mods():16    from fundamentals import mapping as M17    from fundamentals import normalize as N18    from fundamentals import ratios as R19    from fundamentals import screener as SC20    return M, N, R, SC212223def _fact(tag, val, start, end, accn, filed, *, fy=2024, fp="Q1", form="10-Q", unit="USD", taxonomy="us-gaap", frame=None):24    return dict(cik=1, taxonomy=taxonomy, tag=tag, unit=unit, fy=fy, fp=fp, form=form, start=start, end=end, val=val,25                accn=accn, filed=filed, frame=frame)262728def _frame(rows):29    _, N, _, _ = _mods()30    df = pd.DataFrame(rows, columns=N.FACT_COLUMNS)31    for c in ("start", "end", "filed"):32        df[c] = pd.to_datetime(df[c]).dt.date33    df["fy"] = df["fy"].astype("Int64")34    return df353637# ------------------------------------------------------------------------------------------- mapping38def test_mapping_is_prioritized_and_unique(app):39    M, _, _, _ = _mods()40    rows = M.mapping_rows()41    assert len(rows) > 12042    for a in M.ACCOUNTS:43        tags = [(t.taxonomy, t.tag) for t in a.tags]44        assert len(tags) == len(set(tags)), a.name45        if not a.computed:46            assert tags, a.name47        else:48            assert a.formula49    # the spec's chart of accounts, exactly50    assert [a.name for a in M.accounts_for(M.INCOME) if not a.auxiliary] == [51        "revenue", "cost_of_revenue", "gross_profit", "rnd_expense", "sga_expense", "operating_income", "interest_expense",52        "pretax_income", "income_tax", "net_income", "eps_basic", "eps_diluted", "shares_basic", "shares_diluted", "ebitda",53        "dividends_paid"]54    assert [a.name for a in M.accounts_for(M.BALANCE) if not a.auxiliary] == [55        "cash_and_equivalents", "short_term_investments", "receivables", "inventory", "total_current_assets", "ppe_net",56        "goodwill", "intangibles", "total_assets", "accounts_payable", "short_term_debt", "total_current_liabilities",57        "long_term_debt", "total_liabilities", "retained_earnings", "total_equity", "total_debt", "net_debt", "working_capital"]58    assert [a.name for a in M.accounts_for(M.CASHFLOW) if not a.auxiliary] == [59        "operating_cash_flow", "capex", "free_cash_flow", "acquisitions", "investing_cash_flow", "debt_issued", "debt_repaid",60        "buybacks", "dividends", "financing_cash_flow", "net_change_in_cash", "stock_based_compensation"]61    assert M.ACCOUNT_BY_NAME["revenue"].tags[0].tag == "Revenues"62    assert M.ACCOUNT_BY_NAME["revenue"].tags[1].tag == "RevenueFromContractWithCustomerExcludingAssessedTax"63    assert M.is_mapped("us-gaap", "Assets") and not M.is_mapped("us-gaap", "OperatingExpenses")646566def test_mapping_seeded_in_db(fundamentals_data):67    from sqlalchemy import func, select6869    from core.db import session70    from fundamentals import mapping as M71    from fundamentals.models import FundMapping72    with session() as s:73        n = s.scalar(select(func.count()).select_from(FundMapping).where(FundMapping.version == M.MAPPING_VERSION))74    assert n == len(M.mapping_rows())757677# ------------------------------------------------------------------------------------ fiscal calendar78def test_fiscal_calendar_apple_and_microsoft(app):79    _, N, _, _ = _mods()80    apple = N.FiscalCalendar("0928", {2023: date(2023, 9, 30), 2024: date(2024, 9, 28)})81    assert apple.locate(date(2024, 3, 30)) == (2024, 2)        # Apple Q2 FY202482    assert apple.locate(date(2023, 12, 30)) == (2024, 1)83    assert apple.locate(date(2024, 6, 29)) == (2024, 3)84    assert apple.locate(date(2024, 9, 28)) == (2024, 4)85    assert apple.locate(date(2025, 3, 29)) == (2025, 2)        # extrapolated from MMDD with ±7-day snap86    assert apple.locate(date(2025, 9, 27)) == (2025, 4)87    msft = N.FiscalCalendar("0630", {2024: date(2024, 6, 30)})88    assert msft.locate(date(2024, 12, 31)) == (2025, 2)89    assert msft.locate(date(2025, 6, 30)) == (2025, 4)90    assert msft.locate(date(2024, 9, 30)) == (2025, 1)91    # 52/53-week retailer ending late December92    shak = N.FiscalCalendar("1231", {2023: date(2023, 12, 27), 2024: date(2024, 12, 25)})93    assert shak.locate(date(2024, 3, 27)) == (2024, 1)94    assert shak.locate(date(2024, 12, 25)) == (2024, 4)959697def test_calendar_quarter_and_span(app):98    _, N, _, _ = _mods()99    assert N.calendar_quarter(date(2024, 3, 30)) == "2024Q1"100    assert N.calendar_quarter(date(2023, 12, 30)) == "2023Q4"101    assert N.calendar_quarter(date(2025, 1, 3)) == "2024Q4"          # 53-week year ending in early January102    assert N.calendar_quarter(date(2024, 6, 29)) == "2024Q2"103    assert N.span_type(date(2023, 12, 31), date(2024, 3, 30)) == N.SPAN_Q104    assert N.span_type(date(2023, 10, 1), date(2024, 3, 30)) == N.SPAN_H105    assert N.span_type(date(2023, 10, 1), date(2024, 6, 29)) == N.SPAN_9M106    assert N.span_type(date(2023, 10, 1), date(2024, 9, 28)) == N.SPAN_FY107    assert N.span_type(None, date(2024, 9, 28)) is None108    assert N.span_type(date(2024, 8, 1), date(2024, 9, 28)) is None   # odd stub period109110111# ---------------------------------------------------------------------------------------- resolution112def _one_filing(rows, fye="1231", fy_ends=None):113    _, N, _, _ = _mods()114    facts = _frame(rows)115    filings = {"A1": N.Filing("A1", "10-Q", date(2024, 5, 1), date(2024, 3, 31))}116    cal = N.FiscalCalendar(fye, fy_ends or {2023: date(2023, 12, 31)})117    return N.resolve_filing(facts, cal, filings["A1"])118119120def test_priority_and_fallback(app):121    periods, _, _ = _one_filing([122        _fact("RevenueFromContractWithCustomerExcludingAssessedTax", 100, "2024-01-01", "2024-03-31", "A1", "2024-05-01"),123        _fact("SalesRevenueNet", 90, "2024-01-01", "2024-03-31", "A1", "2024-05-01"),124    ])125    inc = periods[(2024, 1, "Q")]["income"]126    assert inc.values["revenue"] == 100 and inc.coverage["revenue"]["priority"] == 2127    periods, _, _ = _one_filing([128        _fact("Revenues", 120, "2024-01-01", "2024-03-31", "A1", "2024-05-01"),129        _fact("RevenueFromContractWithCustomerExcludingAssessedTax", 100, "2024-01-01", "2024-03-31", "A1", "2024-05-01"),130    ])131    inc = periods[(2024, 1, "Q")]["income"]132    assert inc.values["revenue"] == 120 and inc.coverage["revenue"]["tag"] == "us-gaap:Revenues"133134135def test_component_sum_and_unit_normalisation(app):136    periods, _, _ = _one_filing([137        _fact("GeneralAndAdministrativeExpense", 30, "2024-01-01", "2024-03-31", "A1", "2024-05-01"),138        _fact("SellingAndMarketingExpense", 20, "2024-01-01", "2024-03-31", "A1", "2024-05-01"),139        _fact("EarningsPerShareDiluted", 1.5, "2024-01-01", "2024-03-31", "A1", "2024-05-01", unit="USD/shares"),140        _fact("WeightedAverageNumberOfDilutedSharesOutstanding", 1_000, "2024-01-01", "2024-03-31", "A1", "2024-05-01", unit="shares"),141        _fact("Assets", 5_000, None, "2024-03-31", "A1", "2024-05-01", unit="CAD"),142        _fact("Assets", 999, None, "2024-03-31", "A1", "2024-05-01", unit="pure"),      # wrong unit: ignored143        _fact("EarningsPerShareDiluted", 7, "2024-01-01", "2024-03-31", "A1", "2024-05-01", unit="USD"),  # wrong unit: ignored144    ])145    inc = periods[(2024, 1, "Q")]["income"]146    assert inc.values["sga_expense"] == 50 and inc.coverage["sga_expense"]["components"] is True147    assert inc.values["eps_diluted"] == 1.5 and inc.values["shares_diluted"] == 1_000148    bal = periods[(2024, 1, "I")]["balance"]149    assert bal.values["total_assets"] == 5_000 and bal.currency == "CAD" and bal.coverage["total_assets"]["currency"] == "CAD"150151152# -------------------------------------------------------------------------------- derivation / versions153def _company(rows, fye="1231"):154    _, N, _, _ = _mods()155    facts = _frame(rows)156    return N.normalize_company(1, "TEST", facts, {}, fye)157158159def test_q4_derivation_requires_three_quarters_same_year(app):160    def q(tag, val, s, e, accn, filed, fp, fy=2023, form="10-Q"):161        return _fact(tag, val, s, e, accn, filed, fy=fy, fp=fp, form=form)162    rows = [q("Revenues", 10, "2023-01-01", "2023-03-31", "Q1", "2023-05-01", "Q1"),163            q("Revenues", 20, "2023-04-01", "2023-06-30", "Q2", "2023-08-01", "Q2"),164            q("Revenues", 30, "2023-07-01", "2023-09-30", "Q3", "2023-11-01", "Q3"),165            q("Revenues", 100, "2023-01-01", "2023-12-31", "K", "2024-02-15", "FY", form="10-K"),166            q("WeightedAverageNumberOfDilutedSharesOutstanding", 100, "2023-01-01", "2023-03-31", "Q1", "2023-05-01", "Q1"),167            q("WeightedAverageNumberOfDilutedSharesOutstanding", 100, "2023-04-01", "2023-06-30", "Q2", "2023-08-01", "Q2"),168            q("WeightedAverageNumberOfDilutedSharesOutstanding", 100, "2023-07-01", "2023-09-30", "Q3", "2023-11-01", "Q3"),169            q("WeightedAverageNumberOfDilutedSharesOutstanding", 95, "2023-01-01", "2023-12-31", "K", "2024-02-15", "FY", form="10-K")]170    for r in rows:171        r["unit"] = "shares" if "Shares" in r["tag"] else "USD"172    res = _company(rows)173    q4 = [r for r in res.rows if r["statement"] == "income" and r["fiscal_quarter"] == 4]174    assert len(q4) == 1 and q4[0]["derived"] is True175    assert q4[0]["revenue"] == 40 and q4[0]["coverage"]["revenue"]["derived"] == "FY-(Q1+Q2+Q3)"176    assert q4[0]["shares_diluted"] == 4 * 95 - 300 and q4[0]["coverage"]["shares_diluted"]["approx"] is True177    assert q4[0]["form"] == "10-K" and str(q4[0]["filed_date"]) == "2024-02-15" and str(q4[0]["period_end"]) == "2023-12-31"178    # drop Q2 → Q4 cannot be derived, no invented value179    res2 = _company([r for r in rows if r["accn"] != "Q2"])180    assert not [r for r in res2.rows if r["statement"] == "income" and r["fiscal_quarter"] == 4]181182183def test_restatement_versioning_and_as_of(app):184    _, N, _, _ = _mods()185    rows = [_fact("Revenues", 100, "2023-01-01", "2023-03-31", "Q1", "2023-05-01", fy=2023, fp="Q1"),186            _fact("Revenues", 110, "2023-01-01", "2023-03-31", "Q1A", "2023-09-01", fy=2023, fp="Q1", form="10-Q/A"),187            _fact("Revenues", 110, "2023-01-01", "2023-03-31", "Q1N", "2024-05-01", fy=2024, fp="Q1")]   # comparative, unchanged188    res = _company(rows)189    versions = sorted([r for r in res.rows if r["statement"] == "income"], key=lambda r: r["filed_date"])190    assert [(v["revenue"], v["restated"], v["form"]) for v in versions] == [(100, False, "10-Q"), (110, True, "10-Q/A")]191    assert N.select_as_of(res.rows, date(2023, 6, 1))[0]["revenue"] == 100192    assert N.select_as_of(res.rows, date(2023, 9, 1))[0]["revenue"] == 110193    assert N.select_as_of(res.rows, None)[0]["revenue"] == 110194    assert N.select_as_of(res.rows, date(2023, 4, 30)) == []       # nothing was public yet195196197def test_lower_priority_comparative_does_not_restate(app):198    """A later filing re-reporting a period with a worse concept (cash incl. restricted) must not override."""199    rows = [_fact("CashAndCashEquivalentsAtCarryingValue", 100, None, "2024-03-30", "A", "2024-05-03", fy=2024, fp="Q2"),200            _fact("CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents", 105, None, "2024-03-30", "B", "2025-05-02", fy=2025, fp="Q2")]201    res = _company(rows, fye="0930")202    bal = [r for r in res.rows if r["statement"] == "balance" and r["fiscal_quarter"] != 0]203    assert len(bal) == 1 and bal[0]["cash_and_equivalents"] == 100 and bal[0]["restated"] is False204205206# ------------------------------------------------------------------------------------ real fixtures207@pytest.fixture(scope="module")208def apple_rows(app):209    _, N, _, _ = _mods()210    cf = load_gz(f"companyfacts_CIK{PROTO_CIKS['AAPL']:010d}.json.gz")211    sub = load_gz(f"submissions_CIK{PROTO_CIKS['AAPL']:010d}.json.gz")212    res = N.normalize_company(320193, "AAPL", N.facts_frame(cf), N.filings_from_submissions(sub), sub["fiscalYearEnd"])213    assert res.stats["fiscal_mismatches"] == []214    return res.rows215216217def _row(rows, statement, fy, fq):218    return next(r for r in rows if r["statement"] == statement and r["fiscal_year"] == fy and r["fiscal_quarter"] == fq)219220221def test_apple_q2_fy2024_real_numbers(apple_rows):222    """Apple 10-Q filed 2024-05-03 (quarter ended 2024-03-30): the reference numbers of the spec."""223    _, N, _, _ = _mods()224    latest = N.select_as_of(apple_rows, None)225    inc = _row(latest, "income", 2024, 2)226    assert str(inc["period_end"]) == "2024-03-30" and inc["calendar_quarter"] == "2024Q1"227    assert inc["revenue"] == 90_753_000_000 and inc["net_income"] == 23_636_000_000 and inc["eps_diluted"] == 1.53228    assert inc["derived"] is False and inc["coverage"]["revenue"]["tag"] == "us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax"229    bal = _row(latest, "balance", 2024, 2)230    assert bal["total_assets"] == 337_411_000_000231    cf = _row(latest, "cashflow", 2024, 2)232    assert cf["operating_cash_flow"] == 22_690_000_000            # de-cumulated: YTD6 (62,585) − Q1 (39,895)233    assert cf["derived"] is True and cf["coverage"]["operating_cash_flow"]["derived"] == "YTD6-Q1"234    q4 = _row(latest, "income", 2024, 4)235    # Apple reports 9-month YTD income facts, so Q4 = FY − YTD9 (identical to FY − (Q1+Q2+Q3), fewer inputs)236    assert q4["derived"] is True and q4["revenue"] == 94_930_000_000 and q4["coverage"]["revenue"]["derived"] == "FY-YTD9"237    fy = _row(latest, "income", 2024, 0)238    assert fy["revenue"] == 391_035_000_000 and fy["eps_diluted"] == 6.08239    # nulls are explained, never guessed240    assert inc["interest_expense"] is None and inc["coverage"]["interest_expense"]["reason"] == "no_mapped_tag"241242243def test_apple_ttm_and_point_in_time(apple_rows):244    _, N, _, _ = _mods()245    latest = N.select_as_of(apple_rows, None)246    t = N.ttm(latest, "income")247    w = next(x for x in t if x["fiscal_year"] == 2024 and x["fiscal_quarter"] == 2)248    assert w["revenue"] == 119_575_000_000 + 90_753_000_000 + 89_498_000_000 + 81_797_000_000 == 381_623_000_000249    assert w["ttm"] is True and w["shares_diluted"] == _row(latest, "income", 2024, 2)["shares_diluted"]250    assert w["coverage"]["eps_diluted"]["approx"]251    bt = N.ttm(latest, "balance")252    assert next(x for x in bt if x["fiscal_year"] == 2024 and x["fiscal_quarter"] == 2)["total_assets"] == 337_411_000_000253    # anti look-ahead254    assert not [r for r in N.select_as_of(apple_rows, date(2024, 5, 2)) if r["statement"] == "income" and (r["fiscal_year"], r["fiscal_quarter"]) == (2024, 2)]255    assert [r for r in N.select_as_of(apple_rows, date(2024, 5, 3)) if r["statement"] == "income" and (r["fiscal_year"], r["fiscal_quarter"]) == (2024, 2)]256257258def test_ratio_formulas_on_apple_q2_fy2024(apple_rows):259    """Formulas checked against the values of the recorded fixture (TTM to 2024-03-30, close 183.38 on 2024-05-03)."""260    _, N, R, _ = _mods()261    from fundamentals.service import ratio_inputs262    rows = N.select_as_of(apple_rows, date(2024, 5, 3))263    price = 183.38264    f, info = ratio_inputs(rows, price=price)265    assert str(info["period_end"]) == "2024-03-30" and info["shares_source"] == "dei:EntityCommonStockSharesOutstanding"266    values, reasons = R.compute_all(f)267    assert f["revenue"] == 381_623_000_000 and f["net_income"] == 100_389_000_000268    assert f["shares_outstanding"] == 15_334_082_000                   # cover page of the 10-Q (2024-04-19)269    assert values["market_cap"] == pytest.approx(price * 15_334_082_000)270    assert values["pe"] == pytest.approx(price / f["eps_diluted"]) and f["eps_diluted"] == pytest.approx(6.43)271    assert values["pb"] == pytest.approx(values["market_cap"] / 74_194_000_000)272    assert values["gross_margin"] == pytest.approx(173_966 / 381_623, rel=1e-6)273    assert values["net_margin"] == pytest.approx(100_389 / 381_623, rel=1e-6)274    assert values["roe"] == pytest.approx(100_389 / 74_194, rel=1e-6)275    assert values["enterprise_value"] == pytest.approx(values["market_cap"] + 104_590e6 - 32_695e6 - 34_455e6)276    assert values["ev_ebitda"] == pytest.approx(values["enterprise_value"] / f["ebitda"])277    assert values["fcf_yield"] == pytest.approx(f["free_cash_flow"] / values["market_cap"])278    assert values["current_ratio"] == pytest.approx(128_416 / 123_822, rel=1e-6)279    assert values["debt_to_equity"] == pytest.approx(104_590 / 74_194, rel=1e-6)280    assert values["forward_pe"] is None and reasons["forward_pe"] == "no_estimates"281    assert values["interest_coverage"] is None and reasons["interest_coverage"].startswith("missing:")282    assert values["revenue_growth_yoy"] == pytest.approx(381_623 / 385_095 - 1, rel=1e-6)283    assert values["book_value_ps"] == pytest.approx(74_194_000_000 / 15_334_082_000)284285286def test_ratio_docs_are_in_sync(app):287    _, _, R, _ = _mods()288    assert (ROOT / "docs" / "fundamentals-ratios.md").read_text(encoding="utf-8").strip() == R.render_docs().strip()289    assert len(R.RATIOS) == len(set(r.name for r in R.RATIOS)) >= 45290291292def test_null_ratios_never_invented(app):293    _, _, R, _ = _mods()294    values, reasons = R.compute_all({"price": 10.0, "eps_diluted": -1.0, "total_equity": 0.0, "revenue": None})295    assert values["pe"] is None and reasons["pe"] == "denominator_not_positive"296    assert values["pb"] is None and values["ps"] is None and reasons["ps"].startswith("missing:")297298299# -------------------------------------------------------------------------------------- filter grammar300def test_filter_grammar(app):301    from core.errors import ApiError302    _, _, _, SC = _mods()303    f = SC.parse_filters("pe<15, roe>=15%,market_cap>1.5b,ev_ebitda=5..12,exchange=Nasdaq|NYSE,ticker!=AAPL")304    assert [(x.field, x.op, x.value) for x in f] == [("pe", "<", 15.0), ("roe", ">=", 0.15), ("market_cap", ">", 1.5e9),305                                                     ("ev_ebitda", "range", (5.0, 12.0)), ("exchange", "=", ["NASDAQ", "NYSE"]),306                                                     ("ticker", "!=", ["AAPL"])]307    assert SC.parse_sort("fcf_yield:desc") == ("fcf_yield", True) and SC.parse_sort(None) == ("market_cap", True)308    for bad in ("pe<<15", "nope>1", "pe>", "pe=abc", "pe=20..10", "exchange>1", "pe~1"):309        with pytest.raises(ApiError) as e:310            SC.parse_filters(bad)311        assert e.value.code == "INVALID_FILTER"312    with pytest.raises(ApiError):313        SC.parse_sort("pe:sideways")314315316# ------------------------------------------------------------------------------------ edgar client317def test_token_bucket_and_backoff(app):318    import httpx319    import respx320321    from fundamentals import edgar_client as ec322    calls = {"n": 0}323324    def handler(request):325        calls["n"] += 1326        if calls["n"] == 1:327            return httpx.Response(429, headers={"Retry-After": "0"})328        return httpx.Response(200, json={"ok": True})329330    with respx.mock(assert_all_mocked=True) as m:331        m.get("https://data.sec.gov/x.json").mock(side_effect=handler)332        c = ec.EdgarClient(rate=10_000, raw_dir=Path("/tmp/hfmd-test-raw"))333        assert c.get_json("https://data.sec.gov/x.json") == {"ok": True}334        assert c.stats.retries == 1 and calls["n"] == 2335        assert c._http.headers["User-Agent"].startswith("HF Market Data (Simon-Pierre Boucher")336    b = ec.TokenBucket(rate=1000, capacity=3)337    for _ in range(10):338        b.acquire()339340341def test_parse_feeds(app):342    from fundamentals import edgar_client as ec343    from fundamentals import ingest344    atom = (ROOT / "tests" / "fixtures" / "edgar" / "atom_10-Q.xml").read_text()345    entries = ec.parse_atom(atom)346    assert len(entries) == 3 and all(e["accn"] and e["cik"] and e["form"] == "10-Q" for e in entries)347    idx = ingest.parse_master_index((ROOT / "tests" / "fixtures" / "edgar" / "master.idx").read_text())348    assert idx and all(set(e) == {"cik", "name", "form", "filed", "accn"} for e in idx)349    assert ingest.master_index_url(date(2026, 9, 3)).endswith("/2026/QTR3/master.20260903.idx")350