Repository Public
1"""Version-2 report: schema, version-1 compatibility, checks, benchmarks, trial counting, CLIs."""2from __future__ import annotations34import copy5import json67import jsonschema8import numpy as np9import pandas as pd10import pytest1112from conftest import ROOT, write_strategy13from perp_helpers import perp_data, perp_meta, settlements14from spq import backtest as spot_cli15from spq import perp_backtest as cli16from spq import perp_pipeline as pipeline17from spq.data.core import normalise18from spq.data.hyperliquid import normalise_funding19from spq.perp_engine import PerpConfig20from spq.perp_metrics import PERP_KEYS, compute_perp_metrics, holding_runs21from spq.perp_report import build_perp_report, load_schema_v2, returns_from_report, validate_report_v222from spq.perp_research import PerpExperiment, funding_regimes23from spq.report import load_schema24from spq.research import read_ledger25from spq.strategy import from_function2627SYMS = ("BTC", "ETH")28CONFIG = PerpConfig("1h", rebalance_threshold=0.05, venue_max_leverage={"BTC": 40, "ETH": 25},29 size_decimals={"BTC": 5, "ETH": 4})303132def trend(d, p):33 out = {}34 for s, df in d.items():35 c = df["close"].to_numpy()36 w = int(p["window"])37 out[s] = 0.0 if len(c) <= w else float(np.sign(c[-1] - c[-1 - w])) * p["gross"] / len(d)38 return out394041def strategy(lookback=60, **params):42 return from_function(trend, perp_meta(SYMS, lookback=lookback, max_leverage=3.0, max_gross=3.0,43 params={"window": 24, "gross": 1.0, **params}, name="trend_perp"))444546def inputs(n=2000, seed=1):47 data = perp_data(n, SYMS, seed=seed, sigma=0.01)48 as_of = data["BTC"].index[-1] + pd.Timedelta(hours=2)49 quality = {s: normalise(df[["open", "high", "low", "close", "volume"]], symbol=s, interval="1h",50 source="hyperliquid", as_of=as_of)[1] for s, df in data.items()}51 raw = {s: settlements(df) for s, df in data.items()}52 fq = {s: normalise_funding(f.reset_index(drop=True), symbol=s, as_of=as_of)[1] for s, f in raw.items()}53 spot = {s: df[["open", "high", "low", "close", "volume"]] * 1.0005 for s, df in data.items()}54 return data, quality, fq, raw, spot555657@pytest.fixture(scope="module")58def report():59 data, quality, fq, raw, spot = inputs()60 rep, exp = build_perp_report(strategy(), data, CONFIG, quality=quality, funding_quality=fq, funding_raw=raw,61 spot_data=spot, n_resamples=200, source="hyperliquid")62 return rep, exp, data636465def test_report_follows_schema_v2_and_keeps_every_v1_key(report):66 rep, _, _ = report67 validate_report_v2(rep)68 assert rep["schema_version"] == 2 and rep["kind"] == "perp"69 json.dumps(rep, allow_nan=False)70 # every version-1 key is there, with a value a version-1 consumer accepts: validate against the version-171 # schema with only its version constant, its closed key lists and its symbol pattern relaxed72 v1 = copy.deepcopy(load_schema())73 v1["properties"].pop("schema_version")7475 def relax(node):76 if isinstance(node, dict):77 node.pop("additionalProperties", None)78 if node.get("description") == "average invested fraction of equity":79 node.pop("maximum", None) # gross exposure can exceed 1 with leverage80 for v in node.values():81 relax(v)82 elif isinstance(node, list):83 for v in node:84 relax(v)8586 relax(v1)87 jsonschema.Draft202012Validator(v1).validate(rep)88 v1m = load_schema()["$defs"]["metrics"]89 for split in ("train", "validation", "test"):90 assert set(v1m["required"]) <= set(rep["metrics"][split]) and set(PERP_KEYS) <= set(rep["metrics"][split])91 assert set(load_schema()["properties"]["config"]["required"]) <= set(rep["config"])92 assert set(load_schema()["properties"]["robustness"]["required"]) <= set(rep["robustness"])93 # ... and the strict version-1 schema refuses it (schema_version is 2): no silent confusion94 assert not jsonschema.Draft202012Validator(load_schema()).is_valid(rep)959697def test_schema_v2_refuses_reports_without_the_new_checks(report):98 rep, _, _ = report99 for path in (("checks", "ledger"), ("checks", "funding_alignment"), ("benchmarks", "perp_long_1x"),100 ("metrics", "test", "funding_pnl"), ("metrics", "test", "liquidations"), ("ledger",), ("kind",)):101 bad = copy.deepcopy(rep)102 node = bad103 for k in path[:-1]:104 node = node[k]105 node.pop(path[-1])106 with pytest.raises(jsonschema.ValidationError):107 validate_report_v2(bad)108 bad = copy.deepcopy(rep)109 bad["checks"]["ledger"] = "maybe"110 with pytest.raises(jsonschema.ValidationError):111 validate_report_v2(bad)112 bad = copy.deepcopy(rep)113 bad["schema_version"] = 1114 with pytest.raises(jsonschema.ValidationError):115 validate_report_v2(bad)116 assert load_schema_v2()["properties"]["checks"]["required"][-2:] == ["ledger", "funding_alignment"]117118119def test_report_contents(report):120 rep, exp, data = report121 c = rep["checks"]122 assert (c["lookahead"], c["data_quality"], c["ledger"], c["funding_alignment"]) == ("pass",) * 4123 t = rep["metrics"]["test"]124 assert t["bars"] == 400 and t["trades"] > 0 and t["funding_payments"] > 0 and t["wiped_out"] is False125 assert 0 < t["gross_leverage_mean"] <= t["gross_leverage_max"] <= 3.0 + 1e-9126 assert t["long_exposure"] >= 0 and t["short_exposure"] >= 0127 assert t["exposure"] == pytest.approx(t["long_exposure"] + t["short_exposure"])128 assert t["net_leverage_mean"] == pytest.approx(t["long_exposure"] - t["short_exposure"])129 assert 0 < t["time_in_market"] <= 1 and t["avg_holding_hours"] == t["avg_holding_bars"] > 0130 assert t["worst_intrabar_drawdown"] >= t["max_drawdown"] - 1e-12131 assert 0 < t["margin_utilisation_mean"] <= t["margin_utilisation_max"]132 # ledger: exact strings that add up133 from decimal import Decimal as D134135 from spq.margin import exact136137 for split in ("train", "validation", "test"):138 led = rep["ledger"][split]139 assert led["reconciles"] is True140 with exact(): # a sum that raises instead of rounding141 total = (D(led["initial"]) + D(led["realised_pnl"]) + D(led["funding"]) - D(led["fees"])142 - D(led["slippage"]) + D(led["unrealised_pnl"]) + D(led["bad_debt_written_off"]))143 assert total == D(led["final_equity"])144 assert float(led["final_equity"]) == pytest.approx(rep["metrics"][split]["final_equity"], rel=1e-9)145 assert float(led["funding"]) == pytest.approx(rep["metrics"][split]["funding_pnl"], rel=1e-9, abs=1e-12)146 # benchmarks: spot buy-and-hold (spot engine: v1 metrics) and perp long 1x including funding (v2 metrics)147 b = rep["benchmarks"]148 assert {"buy_and_hold", "perp_long_1x", "perp_long_1x_BTC", "perp_long_1x_ETH"} <= set(b)149 assert "funding_pnl" not in b["buy_and_hold"] and b["perp_long_1x"]["funding_pnl"] < 0 # longs paid (rate > 0)150 assert b["perp_long_1x"]["trades"] == 2 and b["perp_long_1x"]["gross_leverage_max"] < 1.2151 # the 1x perp differs from spot by the funding it paid (plus the small price and cost differences of the fixture)152 assert b["perp_long_1x"]["total_return"] < b["buy_and_hold"]["total_return"] + 0.01153 rob = rep["robustness"]154 assert len(rob["bootstrap"]["perp_long_1x_sharpe_ci"]) == 2 and len(rob["bootstrap"]["buy_and_hold_sharpe_ci"]) == 2155 assert rob["trials_tested"] == 5 and rob["test_evaluations"] == 1 and len(rob["walk_forward"]) == 4156 assert len(rob["parameter_sensitivity"]) == 5 and all("error" not in r for r in rob["parameter_sensitivity"])157 assert set(rob["benchmarks_by_split"]) == {"train", "validation"}158 regimes = [w["regime"] for w in rob["funding_stress_windows"]]159 assert "highest funding" in regimes and "lowest funding" in regimes160 for w in rob["funding_stress_windows"]:161 assert rep["splits"]["test"][0] <= w["window"][0] <= w["window"][1] <= rep["splits"]["test"][1]162 f = rep["funding"]["test"]["BTC"]163 assert f["bars_with_funding"] == 400 and f["mean_rate_annualised"] == pytest.approx(f["mean_rate_per_bar"] * 8760)164 assert rep["config"]["maintenance_fraction"] == {"BTC": 0.0125, "ETH": 0.02} # half the IM at 40x / 25x165 assert rep["config"]["fee_bps"] == 4.5 and rep["config"]["leverage"] == 3.0166 assert np.allclose(returns_from_report(rep), exp.final_test(strategy()).returns.to_numpy(), atol=1e-6)167168169def test_checks_fail_when_they_should():170 data, quality, fq, raw, spot = inputs(900, seed=2)171 kw = dict(n_resamples=50, n_folds=2)172 # no settlement series -> the alignment cannot be shown; no quality record -> data_quality fails173 rep, _ = build_perp_report(strategy(), data, CONFIG, **kw)174 assert rep["checks"]["funding_alignment"] == "fail" and rep["checks"]["data_quality"] == "fail"175 assert rep["checks"]["ledger"] == "pass"176 assert any("WITHOUT funding" in n for n in rep["checks"]["notes"]) # no spot data either177 assert "funding_pnl" in rep["benchmarks"]["buy_and_hold"] and rep["benchmarks"]["buy_and_hold"]["funding_pnl"] == 0178 # a funding column shifted by one bar (the settlement of the bar's END): caught179 shifted = {s: df.assign(funding=df["funding"].shift(-1)) for s, df in data.items()}180 rep, _ = build_perp_report(strategy(), shifted, CONFIG, quality=quality, funding_quality=fq, funding_raw=raw,181 spot_data=spot, **kw)182 assert rep["checks"]["funding_alignment"] == "fail" and rep["checks"]["data_quality"] == "pass"183 # a funding series with a long hole: data quality fails184 holed = {s: f.drop(f.index[100:140]) for s, f in raw.items()}185 fq_bad = {s: normalise_funding(f.reset_index(drop=True), symbol=s, as_of="2100-01-01")[1] for s, f in holed.items()}186 rep, _ = build_perp_report(strategy(), data, CONFIG, quality=quality, funding_quality=fq_bad, funding_raw=raw,187 spot_data=spot, **kw)188 assert rep["checks"]["data_quality"] == "fail" and any("missing settlements" in n for n in rep["checks"]["notes"])189 with pytest.raises(ValueError, match="not a perp strategy"):190 spot_meta = {"name": "s", "version": "1", "universe": list(SYMS), "interval": "1h", "lookback": 10, "params": {}}191 build_perp_report(from_function(lambda d, p: {}, spot_meta), data, CONFIG)192193194def test_liquidations_and_wipe_out_are_reported(tmp_path):195 data, quality, fq, raw, spot = inputs(900, seed=5)196 reckless = from_function(lambda d, p: {"BTC": 20.0, "ETH": -19.0},197 perp_meta(SYMS, lookback=5, max_leverage=20.0, max_gross=40.0, name="reckless_perp"))198 rep, _ = build_perp_report(reckless, data, CONFIG, quality=quality, funding_quality=fq, funding_raw=raw,199 spot_data=spot, n_resamples=50, n_folds=2)200 validate_report_v2(rep)201 t = rep["metrics"]["test"]202 assert t["liquidations"] >= 1 and len(rep["liquidations"]["test"]) == t["liquidations"]203 first = rep["liquidations"]["test"][0]204 assert first["positions"] and first["reason"] and first["ts"] >= rep["splits"]["test"][0]205 assert rep["checks"]["ledger"] == "pass"206 if t["wiped_out"]:207 assert t["total_return"] == -1.0 and rep["equity_curve"][-1][1] == 0.0208 assert float(rep["ledger"]["test"]["bad_debt_written_off"]) > 0209210211def test_every_trial_is_counted_across_runs(tmp_path):212 data, quality, fq, raw, spot = inputs(900, seed=6)213 ledger = tmp_path / "ledger.jsonl"214 kw = dict(quality=quality, funding_quality=fq, funding_raw=raw, spot_data=spot, n_resamples=50, n_folds=2,215 ledger=ledger)216 rep1, _ = build_perp_report(strategy(), data, CONFIG, **kw)217 assert rep1["robustness"]["trials_tested"] == 5218 other = from_function(trend, perp_meta(SYMS, lookback=60, max_leverage=3.0, max_gross=3.0,219 params={"window": 36, "gross": 1.0}, name="trend_perp_b"))220 rep2, _ = build_perp_report(other, data, CONFIG, **kw)221 assert rep2["robustness"]["trials_tested"] == 10 # its 5 + the 5 tried before222 assert rep2["robustness"]["deflated_sharpe_detail"]["n_trials"] == 10223 recs = read_ledger(ledger)224 assert sum(r["kind"] == "trial" for r in recs) == 10 and sum(r["kind"] == "test" for r in recs) == 2225 rep3, _ = build_perp_report(strategy(), data, CONFIG, **kw) # a second look at the same test split226 assert rep3["robustness"]["test_evaluations"] == 2227228229def test_holding_runs_and_funding_regimes():230 w = np.array([[0, 1], [1, 1], [1, -1], [1, -1], [0, 0], [-1, 0], [-1, 2]], dtype=float)231 assert sorted(holding_runs(w)) == [1, 2, 2, 2, 3]232 idx = pd.date_range("2024-01-01", periods=400, freq="1h", tz="UTC")233 f = np.zeros(400)234 f[50:90] = 1e-3235 f[300:340] = -1e-3236 data = {"X": pd.DataFrame({"funding": f}, index=idx), "X.SPOT": pd.DataFrame({"close": 1.0}, index=idx)}237 regimes = funding_regimes(data, idx[0], idx[-1] + pd.Timedelta(hours=1), window_bars=40, top=1)238 assert [r["regime"] for r in regimes] == ["highest funding", "lowest funding"]239 assert regimes[0]["start"] == idx[50] and regimes[0]["end"] == idx[89] and regimes[0]["mean_rate"] == pytest.approx(1e-3)240 assert regimes[1]["start"] == idx[300] and regimes[1]["mean_rate"] == pytest.approx(-1e-3)241 assert funding_regimes({"X": data["X.SPOT"]}, idx[0], idx[-1], window_bars=40) == []242243244# ------------------------------------------------------------------------------------------------------- CLIs245246BODY = """import numpy as np247out = {}248for s, df in data.items():249 if s.endswith('.SPOT'):250 out[s] = 0.2251 continue252 c = df['close'].to_numpy()253 f = df['funding'].to_numpy()254 out[s] = -0.2 if np.nanmean(f[-12:]) > 0 else float(np.sign(c[-1] - c[0])) * params['gross'] / 2255return out256"""257258259def store(tmp_path, n=900, seed=3):260 """A local store in the layout of spq.data.hyperliquid / spq.data.fetch."""261 data = perp_data(n, SYMS, seed=seed, sigma=0.01)262 for s, df in data.items():263 (tmp_path / "hyperliquid" / "funding").mkdir(parents=True, exist_ok=True)264 (tmp_path / "coinbase").mkdir(exist_ok=True)265 df[["open", "high", "low", "close", "volume"]].reset_index().to_parquet(tmp_path / "hyperliquid" / f"{s}_1h.parquet", index=False)266 (df[["open", "high", "low", "close", "volume"]] * 1.0003).reset_index().to_parquet(tmp_path / "coinbase" / f"{s}_1h.parquet", index=False)267 settlements(df).reset_index(drop=True).to_parquet(tmp_path / "hyperliquid" / "funding" / f"{s}.parquet", index=False)268 (tmp_path / "hyperliquid" / "meta.json").write_text(json.dumps({"universe": {269 "BTC": {"max_leverage": 40.0, "size_decimals": 5}, "ETH": {"max_leverage": 25.0, "size_decimals": 4}}}))270 return data271272273def test_perp_backtest_cli_writes_a_valid_report(tmp_path, capsys):274 store(tmp_path)275 d = write_strategy(tmp_path / "strategies" / "carry_perp" / "1", BODY,276 meta=perp_meta(("BTC", "ETH", "BTC.SPOT"), lookback=40, max_leverage=2.0, max_gross=2.0,277 params={"gross": 1.0}, name="carry_perp"))278 out = tmp_path / "report.json"279 code = cli.main(["--strategy", str(d), "--data-dir", str(tmp_path), "--out", str(out), "--resamples", "50",280 "--folds", "2", "--ledger", str(tmp_path / "l.jsonl")])281 printed = capsys.readouterr()282 assert code == 0, printed.err283 rep = json.loads(out.read_text())284 validate_report_v2(rep)285 line = json.loads(printed.out)286 assert line["checks"] == {"lookahead": "pass", "data_quality": "pass", "ledger": "pass", "funding_alignment": "pass"}287 assert rep["config"]["symbols"] == ["BTC", "ETH", "BTC.SPOT"] and rep["config"]["source"] == "hyperliquid"288 assert rep["config"]["venue_max_leverage"] == {"BTC": 40.0, "ETH": 25.0} # read from meta.json289 assert rep["config"]["size_decimals"] == {"BTC": 5, "ETH": 4, "BTC.SPOT": 6}290 assert rep["metrics"]["test"]["long_exposure"] > 0.15 # the spot leg is held291 assert any("spot benchmark from coinbase" in n for n in rep["checks"]["notes"])292 # refusals: wrong hash, a spot strategy, and the spot CLI given a perp strategy293 assert cli.main(["--strategy", str(d), "--sha256", "0" * 64, "--data-dir", str(tmp_path), "--out", str(out)]) == 2294 assert "sha256" in capsys.readouterr().err295 assert cli.main(["--strategy", str(ROOT / "strategies" / "ma_trend" / "1"), "--data-dir", str(tmp_path),296 "--out", str(out)]) == 2297 assert "spq.backtest" in capsys.readouterr().err298 assert spot_cli.main(["--strategy", str(d), "--data-dir", str(tmp_path), "--out", str(out)]) == 2299 assert "spq.perp_backtest" in capsys.readouterr().err300301302def test_perp_pipeline_shares_bars_and_counts_all_trials(tmp_path, capsys):303 store(tmp_path, n=800)304 for name, universe, gross in (("a_perp", ("BTC", "ETH"), 1.0), ("b_perp", ("BTC", "BTC.SPOT"), 0.5)):305 write_strategy(tmp_path / "strategies" / name / "1", BODY,306 meta=perp_meta(universe, lookback=40, max_leverage=2.0, max_gross=2.0, params={"gross": gross},307 name=name))308 write_strategy(tmp_path / "strategies" / "spot_one" / "1", "return {s: 0.5 for s in data}\n",309 meta={"name": "spot_one", "version": "1", "universe": ["BTC", "ETH"], "interval": "1h",310 "lookback": 10, "params": {}})311 out = tmp_path / "results"312 code = pipeline.main(["--strategies", str(tmp_path / "strategies"), "--data-dir", str(tmp_path), "--out", str(out),313 "--resamples", "50", "--workers", "1", "--decision-every", "b_perp=6"])314 md = capsys.readouterr().out315 assert code == 0316 reports = {p.stem: json.loads(p.read_text()) for p in out.glob("*-1.json")}317 assert set(reports) == {"a_perp-1", "b_perp-1"} # the spot strategy is not run here318 for rep in reports.values():319 validate_report_v2(rep)320 assert rep["robustness"]["trials_tested"] == 6 # 3 + 3 configurations, counted together321 assert rep["robustness"]["deflated_sharpe_detail"]["scope"].startswith("all perp strategies")322 assert reports["a_perp-1"]["splits"] == reports["b_perp-1"]["splits"] # same bars, same dates323 assert reports["b_perp-1"]["config"]["decision_every"] == 6324 summary = json.loads((out / "summary.json").read_text())325 names = [r["strategy"] for r in summary["rows"]]326 assert names == ["a_perp/1", "b_perp/1", "benchmark: perp long 1x incl. funding", "benchmark: buy and hold spot"]327 assert summary["benchmarks"]["symbols"] == ["BTC", "ETH"] and summary["benchmarks"]["buy_and_hold_is_spot"] is True328 assert "## Test split" in md and "Liquidations" in md and (out / "summary.md").read_text().strip() == md.strip()329 assert len((out / "trials.jsonl").read_text().splitlines()) == 8 # 6 trials + 2 test evaluations330