"""Integration tests of /v1/fundamentals/* (TestClient over the ingested prototype companies)."""
from __future__ import annotations

import io

import pandas as pd
import pytest

pytestmark = pytest.mark.usefixtures("fundamentals_data")


def _rows(body):
    return body["data"]


def _find(rows, fy, fq, statement=None):
    return next(r for r in rows if r["fiscal_year"] == fy and r["fiscal_quarter"] == fq and (statement is None or r["statement"] == statement))


def test_universe_and_share_classes(fundamentals_data):
    from fundamentals.service import resolve_company
    u = fundamentals_data["universe"]
    assert u.companies == 4 and u.added == 4                     # AAPL, MSFT, SHAK, Alphabet (GOOG+GOOGL)
    goog, googl = resolve_company("GOOG"), resolve_company("GOOGL")
    assert goog.cik == googl.cik == 1652044 and sorted(goog.tickers) == ["GOOG", "GOOGL"]
    assert resolve_company("aapl").cik == 320193


def test_statements_quarterly_json(client):
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&limit=200")
    assert r.status_code == 200 and r.headers["X-Row-Count"] == str(r.json()["meta"]["count"])
    body = r.json()
    assert body["meta"]["ticker"] == "AAPL" and body["meta"]["statement"] == "income"
    row = _find(_rows(body), 2024, 2)
    assert row["period_end"] == "2024-03-30" and row["revenue"] == 90_753_000_000 and row["net_income"] == 23_636_000_000
    assert row["eps_diluted"] == 1.53 and row["form"] == "10-Q" and row["filed_date"] == "2024-05-03"
    assert row["coverage"]["interest_expense"] == {"reason": "no_mapped_tag"}
    assert "shares_outstanding" not in row and "depreciation_amortization" not in row       # auxiliary accounts hidden
    q4 = _find(_rows(body), 2024, 4)
    assert q4["derived"] is True and q4["coverage"]["revenue"]["derived"] in ("FY-YTD9", "FY-(Q1+Q2+Q3)")
    # rows are newest first and only quarters
    assert all(r["fiscal_quarter"] in (1, 2, 3, 4) for r in _rows(body))
    assert _rows(body)[0]["period_end"] >= _rows(body)[-1]["period_end"]


def test_statements_all_statements_and_pagination(client):
    r = client.get("/v1/fundamentals/AAPL/statements?statement=all&period=quarterly&limit=5")
    body = r.json()
    assert body["meta"]["count"] == 5 and body["meta"]["next_cursor"]
    r2 = client.get(f"/v1/fundamentals/AAPL/statements?statement=all&period=quarterly&limit=5&cursor={body['meta']['next_cursor']}")
    assert r2.status_code == 200 and r2.json()["data"][0] != body["data"][0]
    assert {r["statement"] for r in body["data"]} <= {"income", "balance", "cashflow"}
    assert "total_assets" in body["data"][0] and "operating_cash_flow" in body["data"][0]


def test_statements_annual_ttm_and_balance(client):
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=annual")
    fy = _find(_rows(r.json()), 2024, 0)
    assert fy["revenue"] == 391_035_000_000 and fy["period_start"] == "2023-10-01" and fy["period_end"] == "2024-09-28"
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=ttm&to=2024-03-30&limit=1")
    row = _rows(r.json())[0]
    assert row["ttm"] is True and row["fiscal_year"] == 2024 and row["fiscal_quarter"] == 2 and row["revenue"] == 381_623_000_000
    r = client.get("/v1/fundamentals/AAPL/statements?statement=balance&period=quarterly&from=2024-03-30&to=2024-03-30")
    bal = _rows(r.json())[0]
    assert bal["total_assets"] == 337_411_000_000 and bal["period_start"] is None
    assert bal["total_debt"] == bal["short_term_debt"] + bal["long_term_debt"]
    r = client.get("/v1/fundamentals/AAPL/statements?statement=cashflow&period=quarterly&from=2024-03-30&to=2024-03-30")
    cf = _rows(r.json())[0]
    assert cf["operating_cash_flow"] == 22_690_000_000 and cf["derived"] is True
    assert cf["free_cash_flow"] == cf["operating_cash_flow"] - cf["capex"]


def test_statements_as_of_point_in_time(client):
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&as_of=2024-05-02&limit=1")
    assert r.status_code == 200
    top = _rows(r.json())[0]
    assert (top["fiscal_year"], top["fiscal_quarter"]) == (2024, 1) and r.json()["meta"]["as_of"] == "2024-05-02"
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&as_of=2024-05-03&limit=1")
    top = _rows(r.json())[0]
    assert (top["fiscal_year"], top["fiscal_quarter"]) == (2024, 2)
    r = client.get("/v1/fundamentals/AAPL/statements?as_of=2000-01-01")
    assert r.status_code == 404 and r.json()["error"]["code"] == "FUNDAMENTALS_NOT_AVAILABLE"


def test_statements_formats(client):
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=annual&format=csv")
    assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv")
    df = pd.read_csv(io.StringIO(r.text))
    assert "revenue" in df.columns and len(df) == int(r.headers["X-Row-Count"])
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=annual&format=parquet")
    assert r.status_code == 200 and r.headers["content-type"] == "application/vnd.apache.parquet"
    df = pd.read_parquet(io.BytesIO(r.content))
    assert df["revenue"].max() > 3e11
    r = client.get("/v1/fundamentals/AAPL/statements?format=xml")
    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"


def test_statements_as_reported_view(client):
    r = client.get("/v1/fundamentals/AAPL/statements?statement=income&period=quarterly&view=as_reported&from=2024-03-30&to=2024-03-30")
    assert r.status_code == 200
    rows = _rows(r.json())
    rev = next(x for x in rows if x["account"] == "revenue")
    assert rev["tag"] == "RevenueFromContractWithCustomerExcludingAssessedTax" and rev["val"] == 90_753_000_000
    assert rev["accn"] == "0000320193-24-000069" and rev["unit"] == "USD"


def test_errors_envelope(client):
    r = client.get("/v1/fundamentals/NOPE/statements")
    assert r.status_code == 404
    body = r.json()
    assert body["error"]["code"] == "FUNDAMENTALS_NOT_AVAILABLE" and body["error"]["docs"].endswith("#fundamentals_not_available")
    assert body["detail"] and body["error"]["details"]["ticker"] == "NOPE"
    r = client.get("/v1/fundamentals/SPY/statements")           # in the price lake but not an SEC filer
    assert r.status_code == 404 and r.json()["error"]["code"] == "FUNDAMENTALS_NOT_AVAILABLE"
    r = client.get("/v1/fundamentals/AAPL/statements?statement=equity")
    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
    r = client.get("/v1/fundamentals/AAPL/statements?as_of=yesterday")
    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
    r = client.get("/v1/fundamentals/AAPL/statements?limit=0")
    assert r.status_code == 422 and r.json()["error"]["code"] == "VALIDATION_ERROR"


def test_facts_standardized_and_raw(client):
    r = client.get("/v1/fundamentals/AAPL/facts/revenue?from=2024-03-30&to=2024-03-30")
    assert r.status_code == 200
    rows = _rows(r.json())
    q2 = next(x for x in rows if x["fiscal_quarter"] == 2)
    assert q2["value"] == 90_753_000_000 and q2["coverage"]["tag"].endswith("RevenueFromContractWithCustomerExcludingAssessedTax")
    assert r.json()["meta"]["kind"] == "standardized" and r.json()["meta"]["tags"][0] == "us-gaap:Revenues"
    r = client.get("/v1/fundamentals/AAPL/facts/us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax?from=2024-03-30&to=2024-03-30")
    assert r.status_code == 200
    raw = _rows(r.json())
    assert any(x["val"] == 90_753_000_000 and x["accn"] == "0000320193-24-000069" for x in raw)
    assert any(x["frame"] == "CY2024Q1" for x in raw) and r.json()["meta"]["kind"] == "xbrl_fact"
    r = client.get("/v1/fundamentals/AAPL/facts/us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax?as_of=2024-05-02&from=2024-03-30&to=2024-03-30")
    assert all(x["end"] != "2024-03-30" or x["start"] != "2023-12-31" for x in _rows(r.json()))
    r = client.get("/v1/fundamentals/AAPL/facts/us-gaap:NoSuchConcept")
    assert r.status_code == 404 and r.json()["error"]["code"] == "CONCEPT_NOT_FOUND"
    r = client.get("/v1/fundamentals/AAPL/facts/dei:EntityCommonStockSharesOutstanding?format=csv")
    assert r.status_code == 200 and "EntityCommonStockSharesOutstanding" in r.text


def test_ratios(client):
    r = client.get("/v1/fundamentals/AAPL/ratios?as_of=2024-05-03")
    assert r.status_code == 200
    d, m = r.json()["data"], r.json()["meta"]
    assert d["fundamentals_period_end"] == "2024-03-30" and d["price"] is not None and d["price_date"] <= "2024-05-03"
    assert d["valuation"]["market_cap"] == pytest.approx(d["price"] * 15_334_082_000)
    assert d["valuation"]["pe"] == pytest.approx(d["price"] / 6.43, rel=1e-3)
    assert d["valuation"]["forward_pe"] is None and m["reasons"]["forward_pe"] == "no_estimates"
    assert d["profitability"]["gross_margin"] == pytest.approx(173_966 / 381_623, rel=1e-6)
    assert set(d) >= {"valuation", "profitability", "liquidity", "solvency", "efficiency", "growth", "per_share", "inputs"}
    assert m["shares_source"] == "dei:EntityCommonStockSharesOutstanding" and m["price_source"].startswith("stock/1day/")
    r = client.get("/v1/fundamentals/AAPL/ratios?period=annual")
    assert r.status_code == 200 and r.json()["meta"]["period"] == "annual"
    r = client.get("/v1/fundamentals/AAPL/ratios?period=weekly")
    assert r.status_code == 400
    r = client.get("/v1/fundamentals/SHAK/ratios")
    d = r.json()["data"]
    assert d["profitability"]["gross_margin"] is None and r.json()["meta"]["reasons"]["gross_margin"].startswith("missing:")


def test_ratios_daily_point_in_time(client):
    r = client.get("/v1/fundamentals/AAPL/ratios/daily?from=2024-04-25&to=2024-05-10&fields=pe,pb,market_cap")
    assert r.status_code == 200
    rows = _rows(r.json())
    assert rows and list(rows[0]) == ["date", "close", "fundamentals_as_of", "fundamentals_period_end", "pe", "pb", "market_cap"]
    before = [x for x in rows if x["date"] < "2024-05-03"]
    after = [x for x in rows if x["date"] >= "2024-05-03"]
    assert before and after
    assert all(x["fundamentals_period_end"] == "2023-12-30" for x in before)          # Q1 FY2024 was the latest public data
    assert all(x["fundamentals_period_end"] == "2024-03-30" for x in after)           # the 10-Q filed May 3 applies from May 3
    assert all(x["fundamentals_as_of"] <= x["date"] for x in rows)
    assert r.json()["meta"]["point_in_time"] is True
    r = client.get("/v1/fundamentals/AAPL/ratios/daily?from=2024-04-25&to=2024-05-10&fields=nope")
    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
    r = client.get("/v1/fundamentals/AAPL/ratios/daily?from=2024-04-25&to=2024-05-10&format=parquet")
    assert r.status_code == 200 and len(pd.read_parquet(io.BytesIO(r.content))) == int(r.headers["X-Row-Count"])


def test_screener(client_hu):
    r = client_hu.get("/v1/fundamentals/screener?filters=market_cap>1b&sort=market_cap:desc")
    assert r.status_code == 200
    body = r.json()
    tickers = [x["ticker"] for x in body["data"]]
    assert tickers[:2] == ["AAPL", "MSFT"] and body["meta"]["total"] >= 2
    r = client_hu.get("/v1/fundamentals/screener?filters=ticker=SHAK&columns=ticker,revenue,gross_margin,net_margin")
    row = r.json()["data"][0]
    assert row["ticker"] == "SHAK" and row["gross_margin"] is None and row["net_margin"] is not None
    r = client_hu.get("/v1/fundamentals/screener?filters=revenue=1..2,pe>0")
    assert r.status_code == 200 and r.json()["meta"]["count"] == 0
    r = client_hu.get("/v1/fundamentals/screener?filters=roe>10000%25")
    assert r.status_code == 200 and r.json()["meta"]["count"] == 0
    r = client_hu.get("/v1/fundamentals/screener?filters=pe<<15")
    assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_FILTER"
    r = client_hu.get("/v1/fundamentals/screener?filters=unknown>1")
    assert r.status_code == 400 and "numeric_fields" in r.json()["error"]["details"]
    r = client_hu.get("/v1/fundamentals/screener?limit=1&format=csv")
    assert r.status_code == 200 and r.headers["X-Row-Count"] == "1"
    r = client_hu.get("/v1/fundamentals/screener?limit=1")
    nxt = r.json()["meta"]["next_cursor"]
    assert nxt and client_hu.get(f"/v1/fundamentals/screener?limit=1&cursor={nxt}").json()["data"][0]["ticker"] != r.json()["data"][0]["ticker"]


def test_frames(client):
    r = client.get("/v1/fundamentals/frames/revenue?calendar_quarter=2024Q1")
    assert r.status_code == 200
    rows = _rows(r.json())
    by = {x["ticker"]: x for x in rows}
    assert by["AAPL"]["value"] == 90_753_000_000 and by["AAPL"]["fiscal_quarter"] == 2       # Apple's fiscal Q2
    assert by["MSFT"]["fiscal_quarter"] == 3 and by["SHAK"]["fiscal_quarter"] == 1
    assert [x["value"] for x in rows] == sorted([x["value"] for x in rows], reverse=True)
    r = client.get("/v1/fundamentals/frames/revenue?fiscal_year=2024&fiscal_quarter=0&format=csv")
    assert r.status_code == 200 and "AAPL" in r.text
    r = client.get("/v1/fundamentals/frames/revenue?calendar_quarter=2024Q1&as_of=2024-05-02")
    assert "AAPL" not in {x["ticker"] for x in _rows(r.json())}
    r = client.get("/v1/fundamentals/frames/nope?calendar_quarter=2024Q1")
    assert r.status_code == 404 and r.json()["error"]["code"] == "CONCEPT_NOT_FOUND"
    r = client.get("/v1/fundamentals/frames/revenue")
    assert r.status_code == 400
    r = client.get("/v1/fundamentals/frames/revenue?calendar_quarter=2024-Q1x")
    assert r.status_code == 400


def test_filings(client):
    r = client.get("/v1/fundamentals/AAPL/filings?form=10-Q&from=2024-01-01&to=2024-12-31")
    assert r.status_code == 200
    rows = _rows(r.json())
    assert rows and all(x["form"] == "10-Q" for x in rows)
    q2 = next(x for x in rows if x["accn"] == "0000320193-24-000069")
    assert q2["filed_date"] == "2024-05-03" and q2["period_of_report"] == "2024-03-30" and q2["is_xbrl"] is True
    assert q2["primary_doc_url"] == "https://www.sec.gov/Archives/edgar/data/320193/000032019324000069/aapl-20240330.htm"
    assert q2["index_url"].endswith("/0000320193-24-000069-index.htm")
    r = client.get("/v1/fundamentals/AAPL/filings?format=parquet&limit=3")
    assert r.status_code == 200 and r.headers["X-Row-Count"] == "3"


def test_coverage_and_custom_extensions(client):
    r = client.get("/v1/fundamentals/SHAK/coverage")
    assert r.status_code == 200
    d = r.json()["data"]
    assert d["cik"] == 1620533 and d["quarters"] > 8 and d["completeness"] is not None
    assert d["missing_accounts"]["cost_of_revenue"]["reason"] == "no_mapped_tag"
    ext = {e["tag"] for e in d["custom_extensions"]}
    assert "shak:OperatingMaterialsExpense" in ext
    hint = next(e for e in d["custom_extensions"] if e["tag"] == "shak:OperatingMaterialsExpense")["hint_account"]
    assert hint == "cost_of_revenue"
    r = client.get("/v1/fundamentals/AAPL/coverage")
    d = r.json()["data"]
    assert d["fiscal_year_end"] and d["completeness_by_statement"]["income"] > 80 and d["derived_quarters"] > 0
    assert d["gaps"] == []


def test_mapping_and_health(client):
    r = client.get("/v1/fundamentals/_mapping")
    assert r.status_code == 200
    rev = next(a for a in r.json()["data"] if a["account"] == "revenue")
    assert rev["tags"][0]["tag"] == "Revenues" and rev["tags"][1]["priority"] == 2
    r = client.get("/v1/fundamentals/_health")
    d = r.json()["data"]
    assert d["companies"] == 4 and d["statement_versions"] > 200 and d["screener_rows"] == 3
    assert d["jobs"]["backfill"]["companies_done"] == 3 and d["jobs"]["backfill"]["mapping_failure_rate"] is not None


def test_openapi_declares_fundamentals(client):
    spec = client.get("/openapi.json").json()
    op = spec["paths"]["/v1/fundamentals/{ticker}/statements"]["get"]
    assert op["tags"] == ["fundamentals"] and "404" in op["responses"] and "FUNDAMENTALS_NOT_AVAILABLE" in op["responses"]["404"]["description"]
    assert "/v1/fundamentals/screener" in spec["paths"] and "/v1/bulk/fundamentals/{year}.parquet" in spec["paths"]
    assert "/v1/fundamentals/_health" not in spec["paths"]