Repository Public
1"""Backtest report, schema version 2 (perpetual futures): schema/backtest-report-v2.schema.json.23Every key of the version-1 report is present with the same meaning, so a consumer written for version 1 keeps4working; version 2 adds the perp metrics, the benchmark `perp_long_1x`, the exact ledger, the liquidation log, the5funding summary and two checks (`ledger`, `funding_alignment`). How to read it: docs/v4/quant-perps.md.67A report states what was measured, on which data, with which costs. It never says a strategy is profitable.8"""9from __future__ import annotations1011import json12from datetime import datetime, timezone13from pathlib import Path1415import numpy as np1617from .data.core import Quality, bars_per_year, interval_seconds18from .data.hyperliquid import FundingQuality, check_funding_alignment19from .dsr import deflate_for_job, deflated_sharpe_from_returns20from .metrics import compute_metrics, equity_points21from .perp_engine import PerpConfig, PerpResult, run_perp_backtest22from .perp_metrics import compute_perp_metrics, ledger_report, perp_lot_pnls23from .perp_research import PerpExperiment, funding_stress24from .report import _unique_trials25from .research import one_at_a_time_grid, read_job_trials, read_ledger26from .robustness import (bootstrap_sharpe_ci, lookahead_check, monte_carlo, parameter_sensitivity, stress_windows,27 walk_forward)28from .strategy import Strategy, is_spot_leg2930SCHEMA_V2_PATH = Path(__file__).resolve().parents[2] / "schema" / "backtest-report-v2.schema.json"31SCHEMA_VERSION = 23233MODEL_NOTES = [34 "cross-margin account in USDC; weights are signed fractions of account equity (negative = short)",35 "decision on the closed bar, fill at the next bar's open; taker fee on traded notional and slippage in bps",36 "funding: every hourly settlement is charged at the open of the bar that starts at its hour, on the position "37 "held coming into that bar: amount = -signed size x perpetual open x rate (Hyperliquid pays on the ORACLE "38 "price; historical oracle prices are not served, the perpetual's own price is used instead)",39 "initial margin = notional / leverage, checked at order time: reducing orders always execute, orders adding "40 "exposure are cut pro rata to the free collateral or refused below the minimum order",41 "maintenance margin = notional x flat fraction (Hyperliquid: half of the initial margin at maximum leverage); "42 "NO margin tiers by size",43 "liquidation is intrabar: with every long at the bar's low and every short at its high together, the account "44 "is liquidated where margin equity = maintenance margin, at that price, plus taker fee and a penalty; the WHOLE "45 "account is closed (no partial liquidation, no backstop, no auto-deleveraging)",46 "highs and lows are trade prices of the perpetual, not the venue's mark price (which is smoother): wicks "47 "liquidate here that might not on the venue",48 "equity <= 0 marks the run wiped out: equity stays at 0, the unpaid loss is reported as bad debt",49 "ledger in exact decimals; average entry price rounded half-even at 18 decimals, sizes truncated to the lot",50]515253def load_schema_v2(path=None) -> dict:54 return json.loads(Path(path or SCHEMA_V2_PATH).read_text())555657def validate_report_v2(report: dict, schema: dict | None = None) -> None:58 import jsonschema5960 jsonschema.Draft202012Validator(schema or load_schema_v2()).validate(report)616263def funding_summary(data: dict, start, end, interval: str) -> dict:64 """Per perpetual, on [start, end): what funding looked like (rates per bar; annualised with the bar count)."""65 N = bars_per_year(interval)66 out = {}67 for s, df in data.items():68 if is_spot_leg(s) or "funding" not in df.columns:69 continue70 f = df["funding"][(df.index >= start) & (df.index < end)]71 v = f.dropna().to_numpy()72 if len(v) == 0:73 out[s] = {"bars": int(len(f)), "bars_with_funding": 0}74 continue75 out[s] = {"bars": int(len(f)), "bars_with_funding": int(len(v)), "mean_rate_per_bar": float(v.mean()),76 "mean_rate_annualised": float(v.mean() * N), "min_rate": float(v.min()), "max_rate": float(v.max()),77 "positive_fraction": float((v > 0).mean()), "sum_of_rates": float(v.sum())}78 if "premium" in df.columns:79 p = df["premium"][(df.index >= start) & (df.index < end)].dropna().abs()80 out[s]["median_abs_premium"] = float(p.median()) if len(p) else None81 return out828384def _liquidation_log(res: PerpResult) -> list[dict]:85 return [{"ts": e["ts"].isoformat(), "reason": e["reason"], "positions": e["positions"],86 "margin_equity_before": e["margin_equity_before"], "equity_after": e["equity_after"]}87 for e in res.liquidations]888990def build_perp_report(91 strategy: Strategy,92 data: dict,93 config: PerpConfig,94 *,95 quality: dict[str, Quality] | None = None,96 funding_quality: dict[str, FundingQuality] | None = None,97 funding_raw: dict | None = None,98 spot_data: dict | None = None,99 fractions=(0.6, 0.2, 0.2),100 grid: list[dict] | None = None,101 ledger=None,102 job_ledger=None,103 n_folds: int = 4,104 n_resamples: int = 2000,105 seed: int = 0,106 source: str | None = None,107 notes: list[str] | None = None,108 extra: dict | None = None,109) -> tuple[dict, PerpExperiment]:110 """Full pipeline for ONE perp strategy version with the parameters of its META (same order as build_report:111 selection analyses on train + validation, then the single look at the test split, then benchmarks and checks)."""112 if strategy.kind != "perp":113 raise ValueError(f"strategy {strategy.name!r} is not a perp strategy (META['kind'] = 'perp')")114 notes = list(notes or [])115 exp = PerpExperiment(data, config, fractions=fractions, ledger=ledger, spot_data=spot_data)116 symbols = exp.symbols117 if sorted(symbols) != sorted(strategy.universe):118 notes.append(f"symbols {symbols} differ from META['universe'] {strategy.universe}")119 if config.interval != strategy.meta["interval"]:120 notes.append(f"interval {config.interval} differs from META['interval'] {strategy.meta['interval']}")121122 # 1. development range only -------------------------------------------------------------------------------123 grid = grid if grid is not None else one_at_a_time_grid(strategy.params)124 if strategy.params not in grid:125 grid = [strategy.params, *grid]126 sensitivity = parameter_sensitivity(exp, strategy, grid)127 folds = walk_forward(exp, strategy, grid, n_folds=n_folds)128 runs = {s: exp.run(strategy, s) for s in ("train", "validation")}129130 # 2. the single look at the test split --------------------------------------------------------------------131 test = exp.final_test(strategy)132 runs["test"] = test133 metrics = {s: compute_perp_metrics(res) for s, res in runs.items()}134 r = test.returns.to_numpy()135136 # 3. benchmarks, resampling, checks -----------------------------------------------------------------------137 bench = exp.benchmarks("test")138 benchmarks = {name: (compute_perp_metrics(res) if isinstance(res, PerpResult) else compute_metrics(res))139 for name, res in bench.items()}140 split_benchmarks = {}141 for split in ("train", "validation"):142 b = exp.benchmarks(split)143 split_benchmarks[split] = {name: {k: m[k] for k in ("total_return", "sharpe", "max_drawdown")}144 for name, m in ((n_, compute_perp_metrics(x) if isinstance(x, PerpResult)145 else compute_metrics(x)) for n_, x in b.items())146 if name in ("perp_long_1x", "buy_and_hold")}147 boot = bootstrap_sharpe_ci(r, config.interval, n_samples=n_resamples, seed=seed)148 bh_boot = bootstrap_sharpe_ci(bench["buy_and_hold"].returns.to_numpy(), config.interval,149 n_samples=n_resamples, seed=seed)150 perp_boot = bootstrap_sharpe_ci(bench["perp_long_1x"].returns.to_numpy(), config.interval,151 n_samples=n_resamples, seed=seed)152 mc = monte_carlo(r, perp_lot_pnls(test), config.initial_cash, n_samples=n_resamples, seed=seed)153 stress = stress_windows(test.equity, bench["perp_long_1x"].equity) if not test.wiped_out else []154 full = exp.lookahead_data()155 t0, t1 = exp.splits.window("test")156 week = max(2, int(7 * 86400 // interval_seconds(config.interval)))157 fstress = funding_stress(test, bench["perp_long_1x"], full, t0, t1, window_bars=week,158 bars_per_year=bars_per_year(config.interval))159160 trials = _unique_trials(read_ledger(ledger), config.interval) if ledger else exp.trials161 dsr = deflated_sharpe_from_returns(r, [t.get("validation_sharpe_per_bar") for t in trials])162 dsr["n_trials"] = max(len(trials), 1)163 if job_ledger: # v7: a research job's trials count against this strategy (see spq.dsr.deflate_for_job)164 dsr = deflate_for_job(dsr, r, [t.get("validation_sharpe_per_bar") for t in trials],165 read_job_trials(job_ledger))166 dsr["sharpe_split"], dsr["trial_sharpes_split"] = "test", "validation"167 test_looks = (len([t for t in read_ledger(ledger) if t.get("kind") == "test" and t["strategy"] == strategy.name])168 if ledger else len(exp.test_evaluations))169170 look = lookahead_check(strategy, full, config, start=t0, runner=run_perp_backtest)171 ledger_ok = all(res.ledger["reconciles"] for res in runs.values()) and all(172 res.ledger["reconciles"] for res in bench.values() if isinstance(res, PerpResult))173 align = (check_funding_alignment(full, funding_raw) if funding_raw174 else {"status": "fail", "detail": "no funding settlement series was supplied"})175 quality = quality or {}176 funding_quality = funding_quality or {}177 dq_ok = bool(quality) and all(q.ok() for q in quality.values()) and all(q.ok() for q in funding_quality.values())178 if not quality:179 notes.append("no data-quality record was supplied: data_quality is reported as fail")180 for s, q in quality.items():181 if q.missing_bars or q.duplicates_dropped or q.ohlc_violations or q.off_grid:182 notes.append(f"{s}: {q.missing_bars} missing bars (longest gap {q.max_gap_bars}), "183 f"{q.duplicates_dropped} duplicates dropped, {q.ohlc_violations} OHLC violations, "184 f"{q.off_grid} off-grid timestamps; missing bars are not filled")185 for s, q in funding_quality.items():186 if q.missing or q.off_grid or q.duplicates_dropped or q.non_finite:187 notes.append(f"{s} funding: {q.missing} missing settlements (longest gap {q.max_gap}), {q.off_grid} "188 f"off-grid, {q.duplicates_dropped} duplicates dropped; missing settlements are charged as 0")189 if not getattr(exp, "buy_and_hold_is_spot", False):190 notes.append("no spot candles were supplied: benchmarks.buy_and_hold is the perpetual's own price held "191 "at 1x WITHOUT funding, not a spot position")192 if align["status"] != "pass":193 notes.append(f"funding alignment: {json.dumps(align['detail'], default=str)[:400]}")194 index = next(iter(full.values())).index195 notes += [196 "equity_curve is the test split, valued at the close of each bar (timestamps are bar-open times, UTC)",197 *MODEL_NOTES,198 "max_drawdown and worst_intrabar_drawdown are positive fractions; sharpe/sortino/volatility are annualised "199 f"with {bars_per_year(config.interval):.0f} bars per year (24/7 market), risk-free rate 0",200 "parameters on the test split are META['params']; the sensitivity grid never selects them",201 f"look-ahead check: {look['detail']}",202 ]203 perps = [s for s in symbols if not is_spot_leg(s)]204205 report = {206 "schema_version": SCHEMA_VERSION,207 "kind": "perp",208 "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),209 "strategy_sha256": strategy.sha256,210 "strategy": {"name": strategy.name, "version": strategy.meta["version"], "meta": strategy.meta},211 "config": {212 "symbols": symbols, "interval": config.interval,213 "start": index[0].isoformat(), "end": index[-1].isoformat(),214 "fee_bps": config.fee_bps, "slippage_bps": config.slippage_bps, "initial_cash": config.initial_cash,215 "min_order": config.min_order, "rebalance_threshold": config.rebalance_threshold,216 "decision_every": config.decision_every, "params": strategy.params, "source": source,217 "bars": int(len(index)), "split_fractions": list(fractions),218 # ------------------------------------------------------------------------------------- perps219 "venue": source, "taker_fee_bps": config.taker_fee_bps, "maker_fee_bps": config.maker_fee_bps,220 "liquidity": config.liquidity,221 "leverage": config.leverage or float(strategy.meta["max_leverage"]),222 "max_leverage": float(strategy.meta["max_leverage"]),223 "max_gross_exposure": float(strategy.meta["max_gross_exposure"]),224 "venue_max_leverage": {s: config.venue_leverage(s) for s in perps},225 "maintenance_fraction": {s: config.maintenance(s) for s in perps},226 "liquidation_penalty_bps": config.liquidation_penalty_bps,227 "size_decimals": {s: int(config.size_decimals.get(s, config.default_size_decimals)) for s in symbols},228 "spot_fee_bps": config.spot_fee_bps, "spot_slippage_bps": config.spot_slippage_bps,229 "funding_reference_price": "index column" if any("index" in full[s].columns for s in perps)230 else "perpetual open at the settlement hour",231 },232 "splits": exp.splits.to_report(),233 "metrics": metrics,234 "benchmarks": benchmarks,235 "robustness": {236 "walk_forward": folds,237 "bootstrap_sharpe_ci": boot["ci"],238 "bootstrap": {**{k: v for k, v in boot.items() if k != "ci"}, "split": "test",239 "buy_and_hold_sharpe_ci": bh_boot["ci"], "perp_long_1x_sharpe_ci": perp_boot["ci"]},240 "parameter_sensitivity": sensitivity,241 "trials_tested": dsr["n_trials"],242 "deflated_sharpe": dsr["deflated_sharpe"],243 "deflated_sharpe_detail": dsr,244 "monte_carlo": mc,245 "stress_windows": stress,246 "funding_stress_windows": fstress,247 "test_evaluations": int(test_looks),248 "benchmarks_by_split": split_benchmarks,249 },250 "checks": {251 "lookahead": look["status"],252 "data_quality": "pass" if dq_ok else "fail",253 "ledger": "pass" if ledger_ok else "fail",254 "funding_alignment": align["status"],255 "notes": notes,256 "data_quality_detail": {s: q.to_dict() for s, q in quality.items()},257 "funding_quality_detail": {s: q.to_dict() for s, q in funding_quality.items()},258 "funding_alignment_detail": align["detail"],259 },260 "ledger": {s: ledger_report(res) for s, res in runs.items()},261 "liquidations": {s: _liquidation_log(res) for s, res in runs.items()},262 "funding": {s: funding_summary(full if s == "test" else exp.dev_data(), *exp.splits.window(s),263 config.interval) for s in ("train", "validation", "test")},264 "equity_curve": equity_points(test.equity),265 }266 if extra:267 report.update(extra)268 validate_report_v2(report)269 return report, exp270271272def write_perp_report(report: dict, path) -> None:273 validate_report_v2(report)274 path = Path(path)275 path.parent.mkdir(parents=True, exist_ok=True)276 path.write_text(json.dumps(report, indent=1, allow_nan=False) + "\n")277278279def returns_from_report(report: dict) -> np.ndarray:280 eq = np.array([p[1] for p in report["equity_curve"]], dtype="float64")281 with np.errstate(divide="ignore", invalid="ignore"):282 r = eq[1:] / eq[:-1] - 1.0283 return np.where(np.isfinite(r), r, 0.0)284