Repository Public
1"""Strategy contract of docs/v3/ARCHITECTURE.md: loading, identity, validation and purity guard.23A strategy version is a directory containing `strategy.py` with45 META = {"name", "version", "universe", "interval", "lookback", "params"}6 def target_weights(data: dict[str, pandas.DataFrame], params: dict) -> dict[str, float]78Its identity is the SHA-256 of the bytes of `strategy.py`.910Purity ("no network, no clock, no files") is enforced in three layers, none of which is a security sandbox:11121. Static check of the source (AST): imports are limited to an allow-list of numerical libraries, and a short list of13 names is refused (open, eval, exec, __import__, .now(), .today(), read_csv, ...).142. Runtime guard around every call: socket creation, name resolution, builtins.open / io.open / os.open, subprocess,15 time.time & co, the `random` module and the numpy global seedless generators raise `PurityError`.163. Behavioural checks done by the backtester: the inputs are handed over copy-on-write (mutating them cannot reach the17 engine), and the look-ahead check re-runs the strategy on truncated data and requires identical decisions (this18 proves that the engine leaks no future bar; it cannot see data obtained from outside the inputs).1920Known limits: Python cannot confine code running in the same process. C-implemented entry points cannot be patched21(`datetime.datetime.now()` and `pandas.Timestamp.now()` are only caught by the static check), a determined author can22reach the originals through `ctypes`, `gc` or object introspection, and module-level code runs at import time before23the guard is active (it is only covered by the static check). The real protection is the trust-zone design: the24research zone holds no keys, the engine must run the signal process with an OS-level sandbox and a timeout, and the25signer enforces the mandate whatever the strategy says.26"""27from __future__ import annotations2829import ast30import builtins31import hashlib32import importlib.util33import io34import math35import os36import socket37import subprocess38import sys39import time40from contextlib import contextmanager41from dataclasses import dataclass, field42from pathlib import Path43from typing import Callable4445import numpy as np4647from .data.core import interval_seconds4849WEIGHT_TOL = 1e-950META_KEYS = {"name", "version", "universe", "interval", "lookback", "params"}51# Contract v2 (perpetual futures, docs/v4/quant-perps.md). A META without "kind" (or with "kind": "spot") is the52# spot contract above, unchanged. "kind": "perp" adds two mandatory keys: the per-symbol bound of a weight and the53# bound of the sum of absolute weights. A universe symbol ending in ".SPOT" is a spot leg (long-only, fully paid).54KINDS = ("spot", "perp")55PERP_META_KEYS = {"max_leverage", "max_gross_exposure"}56MAX_LEVERAGE_LIMIT = 50.057SPOT_LEG_SUFFIX = ".SPOT"58# --- v5: universe-generic strategies ------------------------------------------------------------------------------59# META["universe"] == ["*"] means "any set of symbols, chosen by the agent": the symbol list then comes from the input60# (the candles' keys for a signal, --symbols for a backtest), between MIN_SYMBOLS and MAX_SYMBOLS of them, optionally61# narrowed by META["min_symbols"] / META["max_symbols"]. The code hash still pins the behaviour; the agent's mandate62# pins the asset list. A fixed universe (a list of symbols) works exactly as before.63GENERIC = ["*"]64OPTIONAL_META_KEYS = {"min_symbols", "max_symbols"}65MIN_SYMBOLS, MAX_SYMBOLS = 1, 12666768def is_generic(meta) -> bool:69 return meta.get("universe") == GENERIC707172def symbol_bounds(meta) -> tuple[int, int]:73 return int(meta.get("min_symbols", MIN_SYMBOLS)), int(meta.get("max_symbols", MAX_SYMBOLS))747576def _is_symbol(s) -> bool:77 return isinstance(s, str) and 2 <= len(s) <= 10 and s.isascii() and s.isalnum() and s == s.upper()787980def resolve_symbols(meta, symbols) -> list[str]:81 """The symbols a run uses. Generic: the given list, validated (distinct tickers, count within the bounds).82 Fixed universe: META['universe'] when nothing is given, else the given list unchanged (as before v5)."""83 if not is_generic(meta):84 return list(meta["universe"]) if symbols is None else list(symbols)85 if symbols is None:86 raise StrategyError("this strategy is universe-generic (META['universe'] == ['*']): the symbols must be given")87 symbols = list(symbols)88 lo, hi = symbol_bounds(meta)89 if len(symbols) < lo or len(symbols) > hi:90 raise StrategyError(f"{len(symbols)} symbols given, this strategy takes between {lo} and {hi}")91 if len(set(symbols)) != len(symbols) or not all(_is_symbol(x) for x in symbols):92 raise StrategyError("symbols must be distinct upper-case tickers (2 to 10 letters or digits)")93 return symbols94# --- end v5 -------------------------------------------------------------------------------------------------------95ALLOWED_IMPORTS = {96 "__future__", "math", "typing", "collections", "itertools", "functools", "dataclasses", "statistics", "warnings",97 "numpy", "pandas", "scipy", "statsmodels", "arch", "sklearn",98}99FORBIDDEN_NAMES = {"open", "eval", "exec", "compile", "__import__", "input", "breakpoint", "globals", "vars",100 "getattr", "setattr", "delattr", "__builtins__"}101FORBIDDEN_ATTRS = {102 "now", "utcnow", "today", "read_csv", "read_parquet", "read_json", "read_sql", "read_pickle", "read_excel",103 "read_table", "read_html", "to_csv", "to_parquet", "to_pickle", "to_json", "to_sql", "to_excel", "load", "save",104 "savez", "loadtxt", "savetxt", "genfromtxt", "fromfile", "tofile", "memmap", "system", "popen", "urlopen",105 "__globals__", "__builtins__", "__subclasses__", "__code__", "__dict__", "__class__",106}107108109class StrategyError(Exception):110 """The strategy directory, META or source is not acceptable."""111112113class HashMismatch(StrategyError):114 """strategy.py does not have the expected SHA-256."""115116117class PurityError(RuntimeError):118 """The strategy tried to reach the network, a file, the clock or a random source."""119120121class WeightsError(ValueError):122 """target_weights returned something that is not a valid long-only allocation."""123124125# ----------------------------------------------------------------------------------------------- identity and META126127def sha256_file(path) -> str:128 return hashlib.sha256(Path(path).read_bytes()).hexdigest()129130131def validate_meta(meta) -> dict:132 if not isinstance(meta, dict):133 raise StrategyError("META must be a dict")134 kind = meta.get("kind", "spot")135 if kind not in KINDS:136 raise StrategyError(f"META['kind'] must be one of {list(KINDS)} (absent = spot), got {kind!r}")137 allowed = META_KEYS | {"kind"} | OPTIONAL_META_KEYS | (PERP_META_KEYS if kind == "perp" else set())138 required = META_KEYS | (PERP_META_KEYS if kind == "perp" else set())139 missing = required - meta.keys()140 extra = meta.keys() - allowed141 if missing or extra:142 raise StrategyError(f"META keys: missing {sorted(missing)}, unexpected {sorted(extra)}")143 for k in ("name", "version", "interval"):144 if not isinstance(meta[k], str) or not meta[k]:145 raise StrategyError(f"META[{k!r}] must be a non-empty string")146 try:147 interval_seconds(meta["interval"])148 except ValueError as e:149 raise StrategyError(str(e)) from e150 u = meta["universe"]151 if u == GENERIC: # v5: any set of symbols, chosen by the agent152 lo, hi = meta.get("min_symbols", MIN_SYMBOLS), meta.get("max_symbols", MAX_SYMBOLS)153 if (any(not isinstance(x, int) or isinstance(x, bool) for x in (lo, hi))154 or not MIN_SYMBOLS <= lo <= hi <= MAX_SYMBOLS):155 raise StrategyError(f"META['min_symbols'] / META['max_symbols'] must be integers with "156 f"{MIN_SYMBOLS} <= min <= max <= {MAX_SYMBOLS}")157 elif OPTIONAL_META_KEYS & meta.keys():158 raise StrategyError("META['min_symbols'] / META['max_symbols'] only apply to a generic universe ['*']")159 elif (not isinstance(u, list) or not u or len(set(u)) != len(u) or "*" in u160 or not all(isinstance(s, str) and s and s == s.upper() for s in u)):161 raise StrategyError("META['universe'] must be a non-empty list of distinct upper-case symbols, or ['*']")162 lb = meta["lookback"]163 if not isinstance(lb, int) or isinstance(lb, bool) or lb < 1:164 raise StrategyError("META['lookback'] must be a positive integer (number of closed bars the strategy needs)")165 if not isinstance(meta["params"], dict) or not all(isinstance(k, str) for k in meta["params"]):166 raise StrategyError("META['params'] must be a dict with string keys")167 if kind == "perp":168 _validate_perp_meta(meta)169 elif any(s.endswith(SPOT_LEG_SUFFIX) for s in u):170 raise StrategyError(f"'{SPOT_LEG_SUFFIX}' symbols (spot legs) only exist in META['kind'] = 'perp' strategies")171 try:172 import json173174 json.dumps(meta, allow_nan=False)175 except (TypeError, ValueError) as e:176 raise StrategyError(f"META must be JSON-serialisable: {e}") from e177 return meta178179180def _validate_perp_meta(meta: dict) -> None:181 for k in sorted(PERP_META_KEYS):182 v = meta[k]183 if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v <= 0:184 raise StrategyError(f"META[{k!r}] must be a positive finite number")185 if meta["max_leverage"] > MAX_LEVERAGE_LIMIT:186 raise StrategyError(f"META['max_leverage'] above {MAX_LEVERAGE_LIMIT:g} is refused")187 u = meta["universe"]188 perps = [s for s in u if not s.endswith(SPOT_LEG_SUFFIX)]189 if not perps:190 raise StrategyError("a perp strategy needs at least one perpetual in its universe")191 for s in u:192 if s.endswith(SPOT_LEG_SUFFIX) and s[: -len(SPOT_LEG_SUFFIX)] not in perps:193 raise StrategyError(f"spot leg {s!r} has no perpetual {s[: -len(SPOT_LEG_SUFFIX)]!r} in the universe")194 n_spot = len(u) - len(perps)195 if meta["max_gross_exposure"] > meta["max_leverage"] * len(perps) + n_spot + WEIGHT_TOL:196 raise StrategyError("META['max_gross_exposure'] cannot exceed max_leverage x number of perpetuals "197 "(+ 1 per spot leg)")198199200def static_check(source: str) -> list[str]:201 """Return the list of purity problems found in the source (empty list = nothing found)."""202 problems: list[str] = []203 try:204 tree = ast.parse(source)205 except SyntaxError as e:206 return [f"syntax error: {e}"]207 for node in ast.walk(tree):208 if isinstance(node, ast.Import):209 for a in node.names:210 if a.name.split(".")[0] not in ALLOWED_IMPORTS:211 problems.append(f"line {node.lineno}: import of {a.name!r} is not allowed")212 elif isinstance(node, ast.ImportFrom):213 if node.level or (node.module or "").split(".")[0] not in ALLOWED_IMPORTS:214 problems.append(f"line {node.lineno}: import from {node.module!r} is not allowed")215 elif isinstance(node, ast.Name) and node.id in FORBIDDEN_NAMES:216 problems.append(f"line {node.lineno}: use of {node.id!r} is not allowed")217 elif isinstance(node, ast.Attribute) and node.attr in FORBIDDEN_ATTRS:218 problems.append(f"line {node.lineno}: attribute .{node.attr} is not allowed")219 elif isinstance(node, (ast.Global, ast.Nonlocal)):220 problems.append(f"line {node.lineno}: global/nonlocal state is not allowed in a pure function")221 return problems222223224# ------------------------------------------------------------------------------------------------- runtime guard225226def _deny(what: str) -> Callable:227 def denied(*_a, **_k):228 raise PurityError(f"strategy code tried to use {what}: target_weights must be a pure function of its inputs")229 return denied230231232_PATCHES: list[tuple[object, str, str]] = [233 (socket, "socket", "the network (socket)"),234 (socket, "create_connection", "the network (socket)"),235 (socket, "getaddrinfo", "the network (DNS)"),236 (socket, "gethostbyname", "the network (DNS)"),237 (builtins, "open", "a file (open)"),238 (io, "open", "a file (io.open)"),239 (os, "open", "a file (os.open)"),240 (os, "system", "a subprocess"),241 (os, "urandom", "a random source (os.urandom)"),242 (subprocess, "Popen", "a subprocess"),243 (time, "time", "the clock (time.time)"),244 (time, "time_ns", "the clock (time.time_ns)"),245 (time, "localtime", "the clock (time.localtime)"),246 (time, "gmtime", "the clock (time.gmtime)"),247 (time, "sleep", "the clock (time.sleep)"),248 (np.random, "default_rng", "a random source (numpy.random.default_rng)"),249 (np.random, "seed", "a random source (numpy.random.seed)"),250 (np.random, "random", "a random source (numpy.random)"),251 (np.random, "rand", "a random source (numpy.random)"),252 (np.random, "randn", "a random source (numpy.random)"),253 (np.random, "randint", "a random source (numpy.random)"),254 (np.random, "normal", "a random source (numpy.random)"),255 (np.random, "uniform", "a random source (numpy.random)"),256 (np.random, "choice", "a random source (numpy.random)"),257 (np.random, "permutation", "a random source (numpy.random)"),258 (np.random, "shuffle", "a random source (numpy.random)"),259]260_DENIERS = [(obj, name, _deny(what)) for obj, name, what in _PATCHES]261_guard_depth = 0262263264@contextmanager265def purity_guard():266 """Disable network, file, clock, subprocess and random entry points while a strategy runs (re-entrant)."""267 global _guard_depth268 saved = None269 if _guard_depth == 0:270 saved = [(obj, name, getattr(obj, name)) for obj, name, _ in _DENIERS]271 for obj, name, denier in _DENIERS:272 setattr(obj, name, denier)273 _guard_depth += 1274 try:275 yield276 finally:277 _guard_depth -= 1278 if saved is not None:279 for obj, name, original in saved:280 setattr(obj, name, original)281282283# ------------------------------------------------------------------------------------------------------ weights284285def validate_weights(weights, symbols) -> dict[str, float]:286 """Check a long-only allocation: dict symbol -> finite number in [0, 1], total <= 1, known symbols only.287288 Symbols that are not returned get weight 0. Values within 1e-9 of the bounds are snapped onto them.289 """290 if not isinstance(weights, dict):291 raise WeightsError(f"target_weights must return a dict, got {type(weights).__name__}")292 unknown = [s for s in weights if s not in symbols]293 if unknown:294 raise WeightsError(f"weights for symbols outside the universe: {sorted(map(str, unknown))}")295 out: dict[str, float] = {}296 for s in symbols:297 w = weights.get(s, 0.0)298 if isinstance(w, bool) or not isinstance(w, (int, float, np.integer, np.floating)):299 raise WeightsError(f"weight of {s} is not a number: {w!r}")300 w = float(w)301 if not math.isfinite(w):302 raise WeightsError(f"weight of {s} is not finite: {w!r}")303 if w < -WEIGHT_TOL or w > 1.0 + WEIGHT_TOL:304 raise WeightsError(f"weight of {s} is outside [0, 1]: {w!r} (long-only, no leverage)")305 out[s] = min(max(w, 0.0), 1.0)306 total = math.fsum(out.values())307 if total > 1.0 + WEIGHT_TOL:308 raise WeightsError(f"weights sum to {total!r} > 1 (no leverage)")309 if total > 1.0:310 out = {s: w / total for s, w in out.items()}311 return out312313314def is_spot_leg(symbol: str) -> bool:315 return symbol.endswith(SPOT_LEG_SUFFIX)316317318def validate_perp_weights(weights, symbols, max_leverage: float, max_gross_exposure: float) -> dict[str, float]:319 """Check a contract-v2 allocation: dict symbol -> finite number, negative = short.320321 A perpetual's weight is in [-max_leverage, max_leverage]; a spot leg ('X.SPOT') is in [0, 1]; the sum of322 absolute weights is at most max_gross_exposure. Weights are fractions of account equity (notional / equity).323 Symbols that are not returned get weight 0. Values within 1e-9 of a bound are snapped onto it; anything beyond324 is refused, never scaled down silently.325 """326 if not isinstance(weights, dict):327 raise WeightsError(f"target_weights must return a dict, got {type(weights).__name__}")328 unknown = [s for s in weights if s not in symbols]329 if unknown:330 raise WeightsError(f"weights for symbols outside the universe: {sorted(map(str, unknown))}")331 lev = float(max_leverage)332 out: dict[str, float] = {}333 for s in symbols:334 w = weights.get(s, 0.0)335 if isinstance(w, bool) or not isinstance(w, (int, float, np.integer, np.floating)):336 raise WeightsError(f"weight of {s} is not a number: {w!r}")337 w = float(w)338 if not math.isfinite(w):339 raise WeightsError(f"weight of {s} is not finite: {w!r}")340 lo, hi = (0.0, 1.0) if is_spot_leg(s) else (-lev, lev)341 if w < lo - WEIGHT_TOL or w > hi + WEIGHT_TOL:342 what = "a spot leg is long-only and unlevered" if is_spot_leg(s) else f"max_leverage is {lev:g}"343 raise WeightsError(f"weight of {s} is outside [{lo:g}, {hi:g}]: {w!r} ({what})")344 out[s] = min(max(w, lo), hi)345 gross = math.fsum(abs(w) for w in out.values())346 if gross > float(max_gross_exposure) + WEIGHT_TOL:347 raise WeightsError(f"gross exposure {gross!r} exceeds max_gross_exposure {float(max_gross_exposure):g}")348 return out349350351# ------------------------------------------------------------------------------------------------------- loader352353@dataclass354class Strategy:355 """A loaded strategy version. Call it with (data, params=None) to get validated weights."""356357 func: Callable358 meta: dict359 sha256: str360 path: Path | None = None361 guard: bool = True362 static_problems: list[str] = field(default_factory=list)363364 @property365 def name(self) -> str:366 return self.meta["name"]367368 @property369 def kind(self) -> str:370 return self.meta.get("kind", "spot")371372 @property373 def universe(self) -> list[str]:374 return list(self.meta["universe"])375376 @property377 def generic(self) -> bool:378 """v5: True when META['universe'] == ['*'] (the symbols come from the input)."""379 return is_generic(self.meta)380381 def symbols_for(self, symbols=None) -> list[str]:382 """v5: the symbols of a run, see resolve_symbols."""383 return resolve_symbols(self.meta, symbols)384385 @property386 def lookback(self) -> int:387 return int(self.meta["lookback"])388389 @property390 def params(self) -> dict:391 return dict(self.meta["params"])392393 def raw(self, data: dict, params: dict | None = None):394 p = dict(self.meta["params"] if params is None else params)395 if not self.guard:396 return self.func(data, p)397 with purity_guard():398 return self.func(data, p)399400 def validate(self, raw, symbols) -> dict[str, float]:401 """Weight validation of this strategy's contract (spot: [0, 1], sum <= 1; perp: signed, leverage caps)."""402 if self.kind == "perp":403 return validate_perp_weights(raw, list(symbols), self.meta["max_leverage"],404 self.meta["max_gross_exposure"])405 return validate_weights(raw, list(symbols))406407 def __call__(self, data: dict, params: dict | None = None) -> dict[str, float]:408 return self.validate(self.raw(data, params), list(data.keys()))409410411def load_strategy(directory, expected_sha256: str | None = None, *, strict: bool = True) -> Strategy:412 """Import `strategy.py` from a strategy version directory.413414 The file is hashed first; with `expected_sha256` a different hash raises HashMismatch BEFORE any strategy code415 runs. strict=True (default) refuses sources that fail the static purity check.416 """417 directory = Path(directory)418 path = directory / "strategy.py" if directory.is_dir() else directory419 if path.name != "strategy.py" or not path.is_file():420 raise StrategyError(f"no strategy.py in {directory}")421 blob = path.read_bytes()422 digest = hashlib.sha256(blob).hexdigest()423 if expected_sha256 is not None and digest != expected_sha256.strip().lower():424 raise HashMismatch(f"strategy.py sha256 is {digest}, expected {expected_sha256}")425 try:426 source = blob.decode("utf-8")427 except UnicodeDecodeError as e:428 raise StrategyError(f"strategy.py is not UTF-8: {e}") from e429 problems = static_check(source)430 if problems and strict:431 raise StrategyError("strategy.py fails the purity check: " + "; ".join(problems))432 # Executed from the bytes that were hashed (not re-read from disk), without writing bytecode.433 module_name = f"spq_strategy_{digest[:16]}"434 spec = importlib.util.spec_from_loader(module_name, loader=None, origin=str(path))435 module = importlib.util.module_from_spec(spec)436 module.__file__ = str(path)437 try:438 exec(builtins.compile(source, str(path), "exec"), module.__dict__) # noqa: S102 - this is the loader439 except Exception as e: # noqa: BLE001440 raise StrategyError(f"strategy.py failed at import: {type(e).__name__}: {e}") from e441 sys.modules.pop(module_name, None)442 if not hasattr(module, "META"):443 raise StrategyError("strategy.py does not define META")444 meta = validate_meta(module.META)445 func = getattr(module, "target_weights", None)446 if not callable(func):447 raise StrategyError("strategy.py does not define target_weights(data, params)")448 return Strategy(func=func, meta=meta, sha256=digest, path=path, static_problems=problems)449450451def from_function(func: Callable, meta: dict, *, guard: bool = False) -> Strategy:452 """Wrap an in-memory function (tests, benchmarks). Its identity is the hash of its META, not of a file."""453 import json454455 validate_meta(meta)456 digest = hashlib.sha256(("inline:" + json.dumps(meta, sort_keys=True)).encode()).hexdigest()457 return Strategy(func=func, meta=meta, sha256=digest, guard=guard)458