Repository Public
1"""Build and validate the JSON backtest report of docs/v3/ARCHITECTURE.md ("Backtest report").23A report states what was measured, on which data, with which costs. It never says a strategy is profitable.4"""5from __future__ import annotations67import json8from datetime import datetime, timezone9from pathlib import Path1011import numpy as np1213from .data.core import Quality, bars_per_year14from .dsr import deflate_for_job, deflated_sharpe_from_returns15from .engine import BacktestConfig16from .metrics import compute_metrics, equity_points, lot_pnls17from .research import Experiment, one_at_a_time_grid, read_job_trials, read_ledger18from .robustness import (bootstrap_sharpe_ci, lookahead_check, monte_carlo, parameter_sensitivity, stress_windows,19 walk_forward)20from .strategy import Strategy2122SCHEMA_PATH = Path(__file__).resolve().parents[2] / "schema" / "backtest-report.schema.json"23SCHEMA_VERSION = 1242526def load_schema(path=None) -> dict:27 return json.loads(Path(path or SCHEMA_PATH).read_text())282930def validate_report(report: dict, schema: dict | None = None) -> None:31 """Raise jsonschema.ValidationError if the report does not follow the committed schema."""32 import jsonschema3334 jsonschema.Draft202012Validator(schema or load_schema()).validate(report)353637def _unique_trials(records: list[dict], interval: str) -> list[dict]:38 seen, out = set(), []39 for t in records:40 if t.get("kind") != "trial" or t.get("interval") != interval:41 continue42 key = (t["sha256"], json.dumps(t["params"], sort_keys=True, default=str), tuple(t.get("symbols", [])))43 if key not in seen:44 seen.add(key)45 out.append(t)46 return out474849def build_report(50 strategy: Strategy,51 data: dict,52 config: BacktestConfig,53 *,54 quality: dict[str, Quality] | None = None,55 fractions=(0.6, 0.2, 0.2),56 grid: list[dict] | None = None,57 ledger=None,58 job_ledger=None,59 n_folds: int = 4,60 n_resamples: int = 2000,61 seed: int = 0,62 source: str | None = None,63 notes: list[str] | None = None,64 extra: dict | None = None,65) -> tuple[dict, Experiment]:66 """Full pipeline for ONE strategy version with the parameters of its META.6768 Order matters and is fixed: (1) everything that compares configurations runs on train + validation only,69 (2) the test split is evaluated once with META['params'], (3) benchmarks, resampling and checks.70 The grid is a robustness analysis: it never changes the parameters used on the test split. If the grid shows71 better parameters, write a new strategy version (new hash) and count it as a new trial.72 """73 notes = list(notes or [])74 exp = Experiment(data, config, fractions=fractions, ledger=ledger)75 symbols = exp.symbols76 if not strategy.generic and sorted(symbols) != sorted(strategy.universe): # v5: a generic universe has no list to differ from77 notes.append(f"symbols {symbols} differ from META['universe'] {strategy.universe}")78 if config.interval != strategy.meta["interval"]:79 notes.append(f"interval {config.interval} differs from META['interval'] {strategy.meta['interval']}")8081 # 1. development range only -------------------------------------------------------------------------------82 grid = grid if grid is not None else one_at_a_time_grid(strategy.params)83 if strategy.params not in grid:84 grid = [strategy.params, *grid]85 sensitivity = parameter_sensitivity(exp, strategy, grid)86 folds = walk_forward(exp, strategy, grid, n_folds=n_folds)87 metrics = {s: compute_metrics(exp.run(strategy, s)) for s in ("train", "validation")}8889 # 2. the single look at the test split --------------------------------------------------------------------90 test = exp.final_test(strategy)91 metrics["test"] = compute_metrics(test)92 r = test.returns.to_numpy()9394 # 3. benchmarks, resampling, checks -----------------------------------------------------------------------95 bench = exp.benchmarks("test")96 benchmarks = {name: compute_metrics(res) for name, res in bench.items()}97 boot = bootstrap_sharpe_ci(r, config.interval, n_samples=n_resamples, seed=seed)98 bench_boot = bootstrap_sharpe_ci(bench["buy_and_hold"].returns.to_numpy(), config.interval,99 n_samples=n_resamples, seed=seed)100 mc = monte_carlo(r, lot_pnls(test), config.initial_cash, n_samples=n_resamples, seed=seed)101 stress = stress_windows(test.equity, bench["buy_and_hold"].equity)102103 trials = _unique_trials(read_ledger(ledger), config.interval) if ledger else exp.trials104 dsr = deflated_sharpe_from_returns(r, [t.get("validation_sharpe_per_bar") for t in trials])105 dsr["n_trials"] = max(len(trials), 1)106 if job_ledger: # v7: a research job's trials count against this strategy (see spq.dsr.deflate_for_job)107 dsr = deflate_for_job(dsr, r, [t.get("validation_sharpe_per_bar") for t in trials],108 read_job_trials(job_ledger))109 dsr["sharpe_split"], dsr["trial_sharpes_split"] = "test", "validation"110 test_looks = (len([t for t in read_ledger(ledger) if t.get("kind") == "test" and t["strategy"] == strategy.name])111 if ledger else len(exp.test_evaluations))112113 full = exp.lookahead_data()114 look = lookahead_check(strategy, full, config, start=exp.splits.test[0])115 quality = quality or {}116 dq_ok = all(q.ok() for q in quality.values()) if quality else False117 if not quality:118 notes.append("no data-quality record was supplied: data_quality is reported as fail")119 for s, q in quality.items():120 if q.missing_bars or q.duplicates_dropped or q.ohlc_violations or q.off_grid:121 notes.append(f"{s}: {q.missing_bars} missing bars (longest gap {q.max_gap_bars}), "122 f"{q.duplicates_dropped} duplicates dropped, {q.ohlc_violations} OHLC violations, "123 f"{q.off_grid} off-grid timestamps; missing bars are not filled")124 index = next(iter(full.values())).index125 notes += [126 "equity_curve is the test split, valued at the close of each bar (timestamps are bar-open times, UTC)",127 "decisions use closed bars only; fills at the next bar's open; fee and slippage on traded notional",128 "max_drawdown is a positive fraction; sharpe/sortino/volatility are annualised with "129 f"{bars_per_year(config.interval):.0f} bars per year (24/7 market), risk-free rate 0",130 "parameters on the test split are META['params']; the sensitivity grid never selects them",131 f"look-ahead check: {look['detail']}",132 ]133134 report = {135 "schema_version": SCHEMA_VERSION,136 "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),137 "strategy_sha256": strategy.sha256,138 "strategy": {"name": strategy.name, "version": strategy.meta["version"], "meta": strategy.meta},139 "config": {140 "symbols": symbols, "interval": config.interval,141 "start": index[0].isoformat(), "end": index[-1].isoformat(),142 "fee_bps": config.fee_bps, "slippage_bps": config.slippage_bps, "initial_cash": config.initial_cash,143 "min_order": config.min_order, "rebalance_threshold": config.rebalance_threshold,144 "decision_every": config.decision_every, "params": strategy.params, "source": source,145 "bars": int(len(index)), "split_fractions": list(fractions),146 },147 "splits": exp.splits.to_report(),148 "metrics": metrics,149 "benchmarks": benchmarks,150 "robustness": {151 "walk_forward": folds,152 "bootstrap_sharpe_ci": boot["ci"],153 "bootstrap": {**{k: v for k, v in boot.items() if k != "ci"}, "split": "test",154 "buy_and_hold_sharpe_ci": bench_boot["ci"]},155 "parameter_sensitivity": sensitivity,156 "trials_tested": dsr["n_trials"],157 "deflated_sharpe": dsr["deflated_sharpe"],158 "deflated_sharpe_detail": dsr,159 "monte_carlo": mc,160 "stress_windows": stress,161 "test_evaluations": int(test_looks),162 },163 "checks": {164 "lookahead": look["status"],165 "data_quality": "pass" if dq_ok else "fail",166 "notes": notes,167 "data_quality_detail": {s: q.to_dict() for s, q in quality.items()},168 },169 "equity_curve": equity_points(test.equity),170 }171 if extra:172 report.update(extra)173 validate_report(report)174 return report, exp175176177def write_report(report: dict, path) -> None:178 validate_report(report)179 path = Path(path)180 path.parent.mkdir(parents=True, exist_ok=True)181 path.write_text(json.dumps(report, indent=1, allow_nan=False) + "\n")182183184def test_returns_from_report(report: dict) -> np.ndarray:185 eq = np.array([p[1] for p in report["equity_curve"]], dtype="float64")186 return eq[1:] / eq[:-1] - 1.0187188189test_returns_from_report.__test__ = False190