Repository Public
1"""The JSON report: content, schema validation, and the backtest CLI end to end."""2from __future__ import annotations34import copy5import json67import jsonschema8import numpy as np9import pandas as pd10import pytest1112from conftest import ROOT, make_data13from spq import backtest as backtest_cli14from spq.data import normalise15from spq.engine import BacktestConfig16from spq.report import build_report, load_schema, validate_report17from spq.research import TestSplitError, read_ledger18from spq.strategy import load_strategy1920ARCHITECTURE_KEYS = {21 "top": {"strategy_sha256", "config", "splits", "metrics", "benchmarks", "robustness", "checks", "equity_curve"},22 "config": {"symbols", "interval", "start", "end", "fee_bps", "slippage_bps", "initial_cash"},23 "metrics": {"total_return", "cagr", "volatility", "sharpe", "sortino", "max_drawdown", "calmar", "turnover",24 "trades", "win_rate", "profit_factor", "fees_paid", "exposure"},25 "robustness": {"walk_forward", "bootstrap_sharpe_ci", "parameter_sensitivity", "trials_tested",26 "deflated_sharpe"},27 "checks": {"lookahead", "data_quality", "notes"},28}29STRATEGY = ROOT / "strategies" / "ma_trend" / "1"30SYMBOLS = ("ETH", "LINK")313233@pytest.fixture(scope="module")34def built():35 data = make_data(4000, SYMBOLS, seed=4, drift=0.0001)36 quality = {s: normalise(df, symbol=s, interval="1h", source="synthetic", as_of="2030-01-01")[1]37 for s, df in data.items()}38 strategy = load_strategy(STRATEGY)39 cfg = BacktestConfig("1h", fee_bps=10, slippage_bps=10, rebalance_threshold=0.05)40 report, exp = build_report(strategy, data, cfg, quality=quality, n_resamples=300, source="synthetic")41 return report, exp, strategy, data424344def test_report_has_everything_the_architecture_document_lists(built):45 report, exp, strategy, data = built46 assert ARCHITECTURE_KEYS["top"] <= set(report)47 assert ARCHITECTURE_KEYS["config"] <= set(report["config"])48 assert set(report["splits"]) == set(report["metrics"]) == {"train", "validation", "test"}49 for split in report["metrics"].values():50 assert ARCHITECTURE_KEYS["metrics"] <= set(split)51 assert ARCHITECTURE_KEYS["metrics"] <= set(report["benchmarks"]["buy_and_hold"])52 assert {"equal_weight", "buy_and_hold_ETH", "buy_and_hold_LINK"} <= set(report["benchmarks"])53 assert ARCHITECTURE_KEYS["robustness"] <= set(report["robustness"])54 assert ARCHITECTURE_KEYS["checks"] <= set(report["checks"])55 assert report["strategy_sha256"] == strategy.sha25656 assert report["config"]["symbols"] == list(SYMBOLS) and report["config"]["fee_bps"] == 1057 assert report["checks"]["lookahead"] == "pass" and report["checks"]["data_quality"] == "pass"58 json.dumps(report, allow_nan=False) # strict JSON: no NaN, no Infinity596061def test_report_numbers_are_consistent(built):62 report, exp, strategy, data = built63 idx = data["ETH"].index64 assert report["splits"]["train"] == [idx[0].isoformat(), idx[2399].isoformat()]65 assert report["splits"]["test"] == [idx[3200].isoformat(), idx[-1].isoformat()]66 curve = report["equity_curve"]67 assert [curve[0][0], curve[-1][0]] == report["splits"]["test"] and len(curve) == 80068 assert curve[0][1] == 1000.069 assert curve[-1][1] / 1000 - 1 == pytest.approx(report["metrics"]["test"]["total_return"], abs=1e-8)70 assert report["metrics"]["test"]["bars"] == 800 and report["benchmarks"]["buy_and_hold"]["bars"] == 80071 rob = report["robustness"]72 grid = rob["parameter_sensitivity"]73 assert grid[0]["params"] == strategy.params and len(grid) == 5 # base + fast/slow halved and doubled74 assert rob["trials_tested"] == 5 == rob["deflated_sharpe_detail"]["n_trials"]75 assert rob["test_evaluations"] == 176 assert 0 <= rob["deflated_sharpe"] <= rob["deflated_sharpe_detail"]["probabilistic_sharpe_vs_zero"] <= 177 lo, hi = rob["bootstrap_sharpe_ci"]78 assert lo < report["metrics"]["test"]["sharpe"] < hi79 assert len(rob["walk_forward"]) == 4 and all(f["out_of_sample"][1] <= report["splits"]["test"][0]80 for f in rob["walk_forward"])81 assert len(rob["stress_windows"]) == 3 and rob["stress_windows"][0]["benchmark_return"] < 082 assert rob["monte_carlo"]["bar_returns"]["total_return_p05"] < rob["monte_carlo"]["bar_returns"]["total_return_p95"]83 # a strategy that is in cash part of the time has less exposure than buy-and-hold84 assert report["metrics"]["test"]["exposure"] < report["benchmarks"]["buy_and_hold"]["exposure"]858687def test_the_experiment_is_sealed_after_the_report(built):88 report, exp, strategy, data = built89 with pytest.raises(TestSplitError):90 exp.trial(strategy, {"fast": 10, "slow": 100})919293def test_schema_accepts_the_report_and_rejects_broken_ones(built):94 report = built[0]95 schema = load_schema()96 jsonschema.Draft202012Validator.check_schema(schema)97 validate_report(report)9899 def broken(fn):100 r = copy.deepcopy(report)101 fn(r)102 with pytest.raises(jsonschema.ValidationError):103 validate_report(r)104105 for key in ARCHITECTURE_KEYS["top"]:106 broken(lambda r, key=key: r.pop(key))107 for key in ARCHITECTURE_KEYS["config"]:108 broken(lambda r, key=key: r["config"].pop(key))109 for key in ARCHITECTURE_KEYS["metrics"]:110 broken(lambda r, key=key: r["metrics"]["test"].pop(key))111 broken(lambda r, key=key: r["benchmarks"]["buy_and_hold"].pop(key))112 for key in ARCHITECTURE_KEYS["robustness"]:113 broken(lambda r, key=key: r["robustness"].pop(key))114 for key in ARCHITECTURE_KEYS["checks"]:115 broken(lambda r, key=key: r["checks"].pop(key))116 broken(lambda r: r.update(strategy_sha256="abc"))117 broken(lambda r: r.update(strategy_sha256=r["strategy_sha256"].upper()))118 broken(lambda r: r["checks"].update(lookahead="probably"))119 broken(lambda r: r["checks"].update(data_quality=True))120 broken(lambda r: r["metrics"].pop("validation"))121 broken(lambda r: r["metrics"].update(holdout=r["metrics"]["test"]))122 broken(lambda r: r["metrics"]["test"].update(sharpe="1.2"))123 broken(lambda r: r["metrics"]["test"].update(max_drawdown=-0.2))124 broken(lambda r: r["metrics"]["test"].update(win_rate=1.5))125 broken(lambda r: r["metrics"]["test"].update(trades=3.5))126 broken(lambda r: r["metrics"]["test"].update(profitable=True))127 broken(lambda r: r["robustness"].update(trials_tested=0))128 broken(lambda r: r["robustness"].update(deflated_sharpe=1.2))129 broken(lambda r: r["robustness"].update(bootstrap_sharpe_ci=[0.1]))130 broken(lambda r: r["splits"].update(test=["2024-01-01T00:00:00+00:00"]))131 broken(lambda r: r["config"].update(symbols=[]))132 broken(lambda r: r["config"].update(fee_bps=-1))133 broken(lambda r: r.update(equity_curve=[["2024-01-01T00:00:00+00:00", 1000.0]]))134 broken(lambda r: r["equity_curve"].__setitem__(0, ["2024-01-01T00:00:00+00:00", "1000"]))135 # undefined ratios are null, which is valid136 r = copy.deepcopy(report)137 r["metrics"]["test"].update(sharpe=None, profit_factor=None, win_rate=None, calmar=None)138 r["robustness"].update(deflated_sharpe=None, bootstrap_sharpe_ci=[None, None])139 validate_report(r)140141142def test_missing_quality_record_fails_the_data_check():143 data = make_data(1500, SYMBOLS, seed=5)144 report, _ = build_report(load_strategy(STRATEGY), data, BacktestConfig("1h", rebalance_threshold=0.05),145 n_resamples=100, grid=[{"fast": 72, "slow": 480}])146 assert report["checks"]["data_quality"] == "fail" and report["robustness"]["trials_tested"] == 1147 assert any("no data-quality record" in n for n in report["checks"]["notes"])148149150def test_backtest_cli_end_to_end(tmp_path, capsys):151 data = make_data(3000, SYMBOLS, seed=6)152 gap = data["LINK"].index[[700, 701]]153 for sym, df in data.items():154 p = tmp_path / "data" / "coinbase" / f"{sym}_1h.parquet"155 p.parent.mkdir(parents=True, exist_ok=True)156 (df.drop(index=gap) if sym == "LINK" else df).reset_index().to_parquet(p, index=False)157 out, ledger = tmp_path / "out" / "report.json", tmp_path / "trials.jsonl"158 argv = ["--strategy", str(STRATEGY), "--symbols", "ETH,LINK", "--interval", "1h", "--start", "2024-01-01",159 "--end", "2024-05-01", "--fee-bps", "10", "--slippage-bps", "10", "--data-dir", str(tmp_path / "data"),160 "--grid", '{"fast": [48, 72], "slow": [240, 480]}', "--resamples", "200", "--ledger", str(ledger),161 "--out", str(out)]162 assert backtest_cli.main(argv) == 0163 line = json.loads(capsys.readouterr().out)164 report = json.loads(out.read_text())165 validate_report(report)166 assert line["report"] == str(out) and line["strategy_sha256"] == report["strategy_sha256"]167 assert report["config"]["bars"] == 2902 and report["config"]["rebalance_threshold"] == 0.05 # 2904 - 2 missing168 assert report["config"]["start"] == "2024-01-01T00:00:00+00:00"169 assert pd.Timestamp(report["config"]["end"]) == pd.Timestamp("2024-04-30T23:00", tz="UTC")170 assert report["robustness"]["trials_tested"] == 4 and len(read_ledger(ledger)) == 5171 assert any("LINK: 2 missing bars" in n for n in report["checks"]["notes"])172 assert any("common bars kept: 2902" in n for n in report["checks"]["notes"])173 assert report["checks"]["data_quality"] == "pass"174175 # a second run adds to the ledger: the trial count is unchanged (same configurations), the test look is counted176 assert backtest_cli.main(argv) == 0177 capsys.readouterr()178 again = json.loads(out.read_text())179 assert again["robustness"]["trials_tested"] == 4 and again["robustness"]["test_evaluations"] == 2180 assert again["metrics"] == report["metrics"] and again["equity_curve"] == report["equity_curve"]181182 # refusals: wrong hash, unknown data, bad split183 assert backtest_cli.main(argv + ["--sha256", "0" * 64]) == 2184 assert backtest_cli.main([a if a != "ETH,LINK" else "ETH,DOGE" for a in argv]) == 2185 assert backtest_cli.main(argv + ["--split", "0.9,0.2,0.1"]) == 2186 assert "spq.backtest:" in capsys.readouterr().err187188189def test_report_never_claims_profitability(built):190 text = json.dumps(built[0]).lower()191 for word in ("profitable", "recommend", "guarantee", "will make"):192 assert word not in text193 assert np.isfinite(built[0]["metrics"]["test"]["total_return"])194