spb/hfmarketdata

1"""Integration tests of /v1/fundamentals/* (TestClient over the ingested prototype companies)."""2from __future__ import annotations34import io56import pandas as pd7import pytest89pytestmark = pytest.mark.usefixtures("fundamentals_data")101112def _rows(body):13    return body["data"]141516def _find(rows, fy, fq, statement=None):17    return next(r for r in rows if r["fiscal_year"] == fy and r["fiscal_quarter"] == fq and (statement is None or r["statement"] == statement))181920def test_universe_and_share_classes(fundamentals_data):21    from fundamentals.service import resolve_company22    u = fundamentals_data["universe"]23    assert u.companies == 4 and u.added == 4                     # AAPL, MSFT, SHAK, Alphabet (GOOG+GOOGL)24    goog, googl = resolve_company("GOOG"), resolve_company("GOOGL")25    assert goog.cik == googl.cik == 1652044 and sorted(goog.tickers) == ["GOOG", "GOOGL"]26    assert resolve_company("aapl").cik == 320193272829def test_statements_quarterly_json(client):30    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&limit=200")31    assert r.status_code == 200 and r.headers["X-Row-Count"] == str(r.json()["meta"]["count"])32    body = r.json()33    assert body["meta"]["ticker"] == "AAPL" and body["meta"]["statement"] == "income"34    row = _find(_rows(body), 2024, 2)35    assert row["period_end"] == "2024-03-30" and row["revenue"] == 90_753_000_000 and row["net_income"] == 23_636_000_00036    assert row["eps_diluted"] == 1.53 and row["form"] == "10-Q" and row["filed_date"] == "2024-05-03"37    assert row["coverage"]["interest_expense"] == {"reason": "no_mapped_tag"}38    assert "shares_outstanding" not in row and "depreciation_amortization" not in row       # auxiliary accounts hidden39    q4 = _find(_rows(body), 2024, 4)40    assert q4["derived"] is True and q4["coverage"]["revenue"]["derived"] in ("FY-YTD9", "FY-(Q1+Q2+Q3)")41    # rows are newest first and only quarters42    assert all(r["fiscal_quarter"] in (1, 2, 3, 4) for r in _rows(body))43    assert _rows(body)[0]["period_end"] >= _rows(body)[-1]["period_end"]444546def test_statements_all_statements_and_pagination(client):47    r = client.get("/v1/fundamentals/AAPL/statements?statement=all&period=quarterly&limit=5")48    body = r.json()49    assert body["meta"]["count"] == 5 and body["meta"]["next_cursor"]50    r2 = client.get(f"/v1/fundamentals/AAPL/statements?statement=all&period=quarterly&limit=5&cursor={body['meta']['next_cursor']}")51    assert r2.status_code == 200 and r2.json()["data"][0] != body["data"][0]52    assert {r["statement"] for r in body["data"]} <= {"income", "balance", "cashflow"}53    assert "total_assets" in body["data"][0] and "operating_cash_flow" in body["data"][0]545556def test_statements_annual_ttm_and_balance(client):57    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=annual")58    fy = _find(_rows(r.json()), 2024, 0)59    assert fy["revenue"] == 391_035_000_000 and fy["period_start"] == "2023-10-01" and fy["period_end"] == "2024-09-28"60    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=ttm&to=2024-03-30&limit=1")61    row = _rows(r.json())[0]62    assert row["ttm"] is True and row["fiscal_year"] == 2024 and row["fiscal_quarter"] == 2 and row["revenue"] == 381_623_000_00063    r = client.get("/v1/fundamentals/AAPL/statements?statement=balance&period=quarterly&from=2024-03-30&to=2024-03-30")64    bal = _rows(r.json())[0]65    assert bal["total_assets"] == 337_411_000_000 and bal["period_start"] is None66    assert bal["total_debt"] == bal["short_term_debt"] + bal["long_term_debt"]67    r = client.get("/v1/fundamentals/AAPL/statements?statement=cashflow&period=quarterly&from=2024-03-30&to=2024-03-30")68    cf = _rows(r.json())[0]69    assert cf["operating_cash_flow"] == 22_690_000_000 and cf["derived"] is True70    assert cf["free_cash_flow"] == cf["operating_cash_flow"] - cf["capex"]717273def test_statements_as_of_point_in_time(client):74    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&as_of=2024-05-02&limit=1")75    assert r.status_code == 20076    top = _rows(r.json())[0]77    assert (top["fiscal_year"], top["fiscal_quarter"]) == (2024, 1) and r.json()["meta"]["as_of"] == "2024-05-02"78    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&as_of=2024-05-03&limit=1")79    top = _rows(r.json())[0]80    assert (top["fiscal_year"], top["fiscal_quarter"]) == (2024, 2)81    r = client.get("/v1/fundamentals/AAPL/statements?as_of=2000-01-01")82    assert r.status_code == 404 and r.json()["error"]["code"] == "FUNDAMENTALS_NOT_AVAILABLE"838485def test_statements_formats(client):86    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=annual&format=csv")87    assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv")88    df = pd.read_csv(io.StringIO(r.text))89    assert "revenue" in df.columns and len(df) == int(r.headers["X-Row-Count"])90    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=annual&format=parquet")91    assert r.status_code == 200 and r.headers["content-type"] == "application/vnd.apache.parquet"92    df = pd.read_parquet(io.BytesIO(r.content))93    assert df["revenue"].max() > 3e1194    r = client.get("/v1/fundamentals/AAPL/statements?format=xml")95    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"969798def test_statements_as_reported_view(client):99    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&view=as_reported&from=2024-03-30&to=2024-03-30")100    assert r.status_code == 200101    rows = _rows(r.json())102    rev = next(x for x in rows if x["account"] == "revenue")103    assert rev["tag"] == "RevenueFromContractWithCustomerExcludingAssessedTax" and rev["val"] == 90_753_000_000104    assert rev["accn"] == "0000320193-24-000069" and rev["unit"] == "USD"105106107def test_errors_envelope(client):108    r = client.get("/v1/fundamentals/NOPE/statements")109    assert r.status_code == 404110    body = r.json()111    assert body["error"]["code"] == "FUNDAMENTALS_NOT_AVAILABLE" and body["error"]["docs"].endswith("#fundamentals_not_available")112    assert body["detail"] and body["error"]["details"]["ticker"] == "NOPE"113    r = client.get("/v1/fundamentals/SPY/statements")           # in the price lake but not an SEC filer114    assert r.status_code == 404 and r.json()["error"]["code"] == "FUNDAMENTALS_NOT_AVAILABLE"115    r = client.get("/v1/fundamentals/AAPL/statements?statement=equity")116    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"117    r = client.get("/v1/fundamentals/AAPL/statements?as_of=yesterday")118    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"119    r = client.get("/v1/fundamentals/AAPL/statements?limit=0")120    assert r.status_code == 422 and r.json()["error"]["code"] == "VALIDATION_ERROR"121122123def test_facts_standardized_and_raw(client):124    r = client.get("/v1/fundamentals/AAPL/facts/revenue?from=2024-03-30&to=2024-03-30")125    assert r.status_code == 200126    rows = _rows(r.json())127    q2 = next(x for x in rows if x["fiscal_quarter"] == 2)128    assert q2["value"] == 90_753_000_000 and q2["coverage"]["tag"].endswith("RevenueFromContractWithCustomerExcludingAssessedTax")129    assert r.json()["meta"]["kind"] == "standardized" and r.json()["meta"]["tags"][0] == "us-gaap:Revenues"130    r = client.get("/v1/fundamentals/AAPL/facts/us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax?from=2024-03-30&to=2024-03-30")131    assert r.status_code == 200132    raw = _rows(r.json())133    assert any(x["val"] == 90_753_000_000 and x["accn"] == "0000320193-24-000069" for x in raw)134    assert any(x["frame"] == "CY2024Q1" for x in raw) and r.json()["meta"]["kind"] == "xbrl_fact"135    r = client.get("/v1/fundamentals/AAPL/facts/us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax?as_of=2024-05-02&from=2024-03-30&to=2024-03-30")136    assert all(x["end"] != "2024-03-30" or x["start"] != "2023-12-31" for x in _rows(r.json()))137    r = client.get("/v1/fundamentals/AAPL/facts/us-gaap:NoSuchConcept")138    assert r.status_code == 404 and r.json()["error"]["code"] == "CONCEPT_NOT_FOUND"139    r = client.get("/v1/fundamentals/AAPL/facts/dei:EntityCommonStockSharesOutstanding?format=csv")140    assert r.status_code == 200 and "EntityCommonStockSharesOutstanding" in r.text141142143def test_ratios(client):144    r = client.get("/v1/fundamentals/AAPL/ratios?as_of=2024-05-03")145    assert r.status_code == 200146    d, m = r.json()["data"], r.json()["meta"]147    assert d["fundamentals_period_end"] == "2024-03-30" and d["price"] is not None and d["price_date"] <= "2024-05-03"148    assert d["valuation"]["market_cap"] == pytest.approx(d["price"] * 15_334_082_000)149    assert d["valuation"]["pe"] == pytest.approx(d["price"] / 6.43, rel=1e-3)150    assert d["valuation"]["forward_pe"] is None and m["reasons"]["forward_pe"] == "no_estimates"151    assert d["profitability"]["gross_margin"] == pytest.approx(173_966 / 381_623, rel=1e-6)152    assert set(d) >= {"valuation", "profitability", "liquidity", "solvency", "efficiency", "growth", "per_share", "inputs"}153    assert m["shares_source"] == "dei:EntityCommonStockSharesOutstanding" and m["price_source"].startswith("stock/1day/")154    r = client.get("/v1/fundamentals/AAPL/ratios?period=annual")155    assert r.status_code == 200 and r.json()["meta"]["period"] == "annual"156    r = client.get("/v1/fundamentals/AAPL/ratios?period=weekly")157    assert r.status_code == 400158    r = client.get("/v1/fundamentals/SHAK/ratios")159    d = r.json()["data"]160    assert d["profitability"]["gross_margin"] is None and r.json()["meta"]["reasons"]["gross_margin"].startswith("missing:")161162163def test_ratios_daily_point_in_time(client):164    r = client.get("/v1/fundamentals/AAPL/ratios/daily?from=2024-04-25&to=2024-05-10&fields=pe,pb,market_cap")165    assert r.status_code == 200166    rows = _rows(r.json())167    assert rows and list(rows[0]) == ["date", "close", "fundamentals_as_of", "fundamentals_period_end", "pe", "pb", "market_cap"]168    before = [x for x in rows if x["date"] < "2024-05-03"]169    after = [x for x in rows if x["date"] >= "2024-05-03"]170    assert before and after171    assert all(x["fundamentals_period_end"] == "2023-12-30" for x in before)          # Q1 FY2024 was the latest public data172    assert all(x["fundamentals_period_end"] == "2024-03-30" for x in after)           # the 10-Q filed May 3 applies from May 3173    assert all(x["fundamentals_as_of"] <= x["date"] for x in rows)174    assert r.json()["meta"]["point_in_time"] is True175    r = client.get("/v1/fundamentals/AAPL/ratios/daily?from=2024-04-25&to=2024-05-10&fields=nope")176    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"177    r = client.get("/v1/fundamentals/AAPL/ratios/daily?from=2024-04-25&to=2024-05-10&format=parquet")178    assert r.status_code == 200 and len(pd.read_parquet(io.BytesIO(r.content))) == int(r.headers["X-Row-Count"])179180181def test_screener(client_hu):182    r = client_hu.get("/v1/fundamentals/screener?filters=market_cap>1b&sort=market_cap:desc")183    assert r.status_code == 200184    body = r.json()185    tickers = [x["ticker"] for x in body["data"]]186    assert tickers[:2] == ["AAPL", "MSFT"] and body["meta"]["total"] >= 2187    r = client_hu.get("/v1/fundamentals/screener?filters=ticker=SHAK&columns=ticker,revenue,gross_margin,net_margin")188    row = r.json()["data"][0]189    assert row["ticker"] == "SHAK" and row["gross_margin"] is None and row["net_margin"] is not None190    r = client_hu.get("/v1/fundamentals/screener?filters=revenue=1..2,pe>0")191    assert r.status_code == 200 and r.json()["meta"]["count"] == 0192    r = client_hu.get("/v1/fundamentals/screener?filters=roe>10000%25")193    assert r.status_code == 200 and r.json()["meta"]["count"] == 0194    r = client_hu.get("/v1/fundamentals/screener?filters=pe<<15")195    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_FILTER"196    r = client_hu.get("/v1/fundamentals/screener?filters=unknown>1")197    assert r.status_code == 400 and "numeric_fields" in r.json()["error"]["details"]198    r = client_hu.get("/v1/fundamentals/screener?limit=1&format=csv")199    assert r.status_code == 200 and r.headers["X-Row-Count"] == "1"200    r = client_hu.get("/v1/fundamentals/screener?limit=1")201    nxt = r.json()["meta"]["next_cursor"]202    assert nxt and client_hu.get(f"/v1/fundamentals/screener?limit=1&cursor={nxt}").json()["data"][0]["ticker"] != r.json()["data"][0]["ticker"]203204205def test_frames(client):206    r = client.get("/v1/fundamentals/frames/revenue?calendar_quarter=2024Q1")207    assert r.status_code == 200208    rows = _rows(r.json())209    by = {x["ticker"]: x for x in rows}210    assert by["AAPL"]["value"] == 90_753_000_000 and by["AAPL"]["fiscal_quarter"] == 2       # Apple's fiscal Q2211    assert by["MSFT"]["fiscal_quarter"] == 3 and by["SHAK"]["fiscal_quarter"] == 1212    assert [x["value"] for x in rows] == sorted([x["value"] for x in rows], reverse=True)213    r = client.get("/v1/fundamentals/frames/revenue?fiscal_year=2024&fiscal_quarter=0&format=csv")214    assert r.status_code == 200 and "AAPL" in r.text215    r = client.get("/v1/fundamentals/frames/revenue?calendar_quarter=2024Q1&as_of=2024-05-02")216    assert "AAPL" not in {x["ticker"] for x in _rows(r.json())}217    r = client.get("/v1/fundamentals/frames/nope?calendar_quarter=2024Q1")218    assert r.status_code == 404 and r.json()["error"]["code"] == "CONCEPT_NOT_FOUND"219    r = client.get("/v1/fundamentals/frames/revenue")220    assert r.status_code == 400221    r = client.get("/v1/fundamentals/frames/revenue?calendar_quarter=2024-Q1x")222    assert r.status_code == 400223224225def test_filings(client):226    r = client.get("/v1/fundamentals/AAPL/filings?form=10-Q&from=2024-01-01&to=2024-12-31")227    assert r.status_code == 200228    rows = _rows(r.json())229    assert rows and all(x["form"] == "10-Q" for x in rows)230    q2 = next(x for x in rows if x["accn"] == "0000320193-24-000069")231    assert q2["filed_date"] == "2024-05-03" and q2["period_of_report"] == "2024-03-30" and q2["is_xbrl"] is True232    assert q2["primary_doc_url"] == "https://www.sec.gov/Archives/edgar/data/320193/000032019324000069/aapl-20240330.htm"233    assert q2["index_url"].endswith("/0000320193-24-000069-index.htm")234    r = client.get("/v1/fundamentals/AAPL/filings?format=parquet&limit=3")235    assert r.status_code == 200 and r.headers["X-Row-Count"] == "3"236237238def test_coverage_and_custom_extensions(client):239    r = client.get("/v1/fundamentals/SHAK/coverage")240    assert r.status_code == 200241    d = r.json()["data"]242    assert d["cik"] == 1620533 and d["quarters"] > 8 and d["completeness"] is not None243    assert d["missing_accounts"]["cost_of_revenue"]["reason"] == "no_mapped_tag"244    ext = {e["tag"] for e in d["custom_extensions"]}245    assert "shak:OperatingMaterialsExpense" in ext246    hint = next(e for e in d["custom_extensions"] if e["tag"] == "shak:OperatingMaterialsExpense")["hint_account"]247    assert hint == "cost_of_revenue"248    r = client.get("/v1/fundamentals/AAPL/coverage")249    d = r.json()["data"]250    assert d["fiscal_year_end"] and d["completeness_by_statement"]["income"] > 80 and d["derived_quarters"] > 0251    assert d["gaps"] == []252253254def test_mapping_and_health(client):255    r = client.get("/v1/fundamentals/_mapping")256    assert r.status_code == 200257    rev = next(a for a in r.json()["data"] if a["account"] == "revenue")258    assert rev["tags"][0]["tag"] == "Revenues" and rev["tags"][1]["priority"] == 2259    r = client.get("/v1/fundamentals/_health")260    d = r.json()["data"]261    assert d["companies"] == 4 and d["statement_versions"] > 200 and d["screener_rows"] == 3262    assert d["jobs"]["backfill"]["companies_done"] == 3 and d["jobs"]["backfill"]["mapping_failure_rate"] is not None263264265def test_openapi_declares_fundamentals(client):266    spec = client.get("/openapi.json").json()267    op = spec["paths"]["/v1/fundamentals/{ticker}/statements"]["get"]268    assert op["tags"] == ["fundamentals"] and "404" in op["responses"] and "FUNDAMENTALS_NOT_AVAILABLE" in op["responses"]["404"]["description"]269    assert "/v1/fundamentals/screener" in spec["paths"] and "/v1/bulk/fundamentals/{year}.parquet" in spec["paths"]270    assert "/v1/fundamentals/_health" not in spec["paths"]271