spb/anomaly-atlas Public License

1# =============================================================================2#  Project   : anomaly-atlas3#  File      : experiments/micro/expH_oos_stability/benchmark.py4#  Purpose   : Out-of-sample stability on the untouched validation split5#  Author    : Simon-Pierre Boucher6#  Contact   : contact@spboucher.ai7#  Data src  : hfmarketdata.io (sole data source)8#  Created   : 2026-08-129#  Modified  : 2026-08-1210#  Platform  : macOS / Apple Silicon (arm64)11#  License   : All rights reserved (research code)12# =============================================================================13"""Experiment H — the validation split (2016-2021), opened for the first time.1415Re-evaluates, with machinery identical to expC/expD/expG (imported from16their committed benchmarks, not re-implemented): the expG pool net of17costs, the ES->SPY basis effect, and the daily reversal family. The sealed18HOLDOUT (2022->) is not touched.19"""2021from __future__ import annotations2223import importlib.util24import json25import sys26from datetime import UTC, datetime27from pathlib import Path2829import numpy as np3031REPO_ROOT = Path(__file__).resolve().parents[3]32sys.path.insert(0, str(REPO_ROOT / "benchmarks"))33sys.path.insert(0, str(REPO_ROOT / "src"))3435from hardware_manifest import collect_manifest  # noqa: E4023637from anomaly_atlas.data.cleaning import rth_day_grids  # noqa: E40238from anomaly_atlas.data.hf_client import HFMarketDataClient  # noqa: E40239from anomaly_atlas.data.universe import LIQUID_ETF, LIQUID_STOCK, VALIDATION  # noqa: E40240from anomaly_atlas.stats.bootstrap import moving_block_bootstrap, percentile_ci  # noqa: E40241from anomaly_atlas.stats.multiple_testing import bootstrap_pvalue  # noqa: E40242from anomaly_atlas.stats.reversion import ac1, variance_ratio  # noqa: E40243from anomaly_atlas.validation.artifacts import edge_spread  # noqa: E402444546def load_module(name: str, rel: str):47    spec = importlib.util.spec_from_file_location(name, REPO_ROOT / rel)48    mod = importlib.util.module_from_spec(spec)49    spec.loader.exec_module(mod)50    return mod515253expg = load_module("expg_bench", "experiments/micro/expG_cost_frontier/benchmark.py")54expd = load_module("expd_bench", "experiments/micro/expD_leadlag_scan/benchmark.py")5556ADJ = "adj_split"57VAL_SUBS = {"2016-2018": ("2016-01-01", "2019-01-01"),58            "2019-2021": ("2019-01-01", "2022-01-01")}59KAPPA_CHECK = ("0.25", "1.0")606162def eval_pool_on(client, rev_pool, ll_pool, start, end, label) -> list[dict]:63    items = []64    for r in rev_pool:65        bars = client.get_bars(r["asset"], r["ticker"], r["timeframe"], ADJ, start, end)66        stream = expg.contrarian_stream(bars, r["timeframe"])67        hs = expg.half_spread(client, r["asset"], r["ticker"], ADJ, start, end)68        if len(stream) < 150:69            continue70        item = {"rule": f"R:{r['ticker']}:{r['timeframe']}", "family": "reversion",71                "window": label} | expg.sweep(stream, hs)72        items.append(item)73    needed = {"SPY"} | {p["leader"] for p in ll_pool} | {p["follower"] for p in ll_pool}74    grids = {}75    for name in sorted(needed):76        if name == "ES":77            g = rth_day_grids(client.get_bars("futures", "ES", "1min",78                                              "contin_adj_ratio", start, end))79        else:80            asset = ("etf" if name in ("SPY", "QQQ") or name.startswith("XL") else "stock")81            g = rth_day_grids(client.get_bars(asset, name, "1min", ADJ, start, end))82        grids[name] = g83    day_list = sorted(grids["SPY"].keys())84    for p in ll_pool:85        if p["leader"] not in grids or p["follower"] not in grids:86            continue87        stream = expg.leadlag_stream(grids[p["leader"]], grids[p["follower"]],88                                     day_list, p["sign"])89        if len(stream) < 150:90            continue91        traded = p["follower"]92        asset = ("futures" if traded == "ES" else93                 "etf" if traded in ("SPY", "QQQ") or traded.startswith("XL") else "stock")94        adj = "contin_adj_ratio" if traded == "ES" else ADJ95        hs = expg.half_spread(client, asset, traded, adj, start, end)96        item = {"rule": f"L:{p['pair']}:{'+' if p['sign'] > 0 else '-'}",97                "family": "leadlag", "window": label} | expg.sweep(stream, hs)98        items.append(item)99    return items, grids, day_list100101102def main() -> None:103    run_utc = datetime.now(UTC)104    client = HFMarketDataClient()105    rev_pool, ll_pool = expg.pool_from_committed_results()106    s, e = VALIDATION107108    # (a,b) pool on validation109    items, grids, day_list = eval_pool_on(client, rev_pool, ll_pool, s, e, "validation")110    for it in items:111        print(it["rule"], "kappa* =", it.get("kappa_star"))112113    # CKX sub-period check (Q-a robustness clause)114    ckx_subs = {}115    for sub, (ss, ee) in VAL_SUBS.items():116        bars = client.get_bars("stock", "CKX", "1day", ADJ, ss, ee)117        stream = expg.contrarian_stream(bars, "1day")118        hs = expg.half_spread(client, "stock", "CKX", ADJ, ss, ee)119        ckx_subs[sub] = expg.sweep(stream, hs) if len(stream) >= 100 else {"n_days": len(stream)}120121    # (c) ES->SPY and SPX->SPY fresh xcorr on validation122    xcorr_out = {}123    spy_mat = expd.day_matrix(grids["SPY"], day_list)124    for name in ("ES", "SPX"):125        if name == "SPX":126            g = rth_day_grids(client.get_bars("index", "SPX", "1min", None, s, e))127        else:128            g = grids.get("ES") or rth_day_grids(129                client.get_bars("futures", "ES", "1min", "contin_adj_ratio", s, e))130        mat = expd.day_matrix(g, day_list)131        m = expd.analyze_pair(mat[0], mat[1], spy_mat[0], spy_mat[1])132        if m:133            xcorr_out[f"{name}->SPY"] = {k: m[k] for k in134                                         ("fresh_xcorr", "raw_xcorr", "p_fresh_+1",135                                          "p_fresh_-1", "n_fresh_pairs")}136    # ES splice invariance137    for adj in ("contin_adj_absolute", "contin_UNadj"):138        g = rth_day_grids(client.get_bars("futures", "ES", "1min", adj, s, e))139        mat = expd.day_matrix(g, day_list)140        m = expd.analyze_pair(mat[0], mat[1], spy_mat[0], spy_mat[1])141        if m:142            xcorr_out[f"ES[{adj}]->SPY"] = {"fresh_-1": m["fresh_xcorr"]["-1"],143                                            "fresh_+1": m["fresh_xcorr"]["1"]}144145    # (d) daily reversal family on validation vs train values146    rc = expg.latest(str(REPO_ROOT / "results/expC_reversion_scan/*/results.json"))147    train_daily = {c["ticker"]: c for c in rc["cells"]148                   if c["timeframe"] == "1day" and c["period"] == "2008-2015"}149    family = []150    for asset, ticker in [("stock", t) for t in LIQUID_STOCK] + \151                         [("etf", t) for t in LIQUID_ETF]:152        bars = client.get_bars(asset, ticker, "1day", ADJ, s, e)153        if len(bars) < 500:154            continue155        closes = np.array([b["close"] for b in bars])156        r = np.diff(np.log(closes))157        a = ac1(r)158        vr30 = variance_ratio(r, 30)159        o = np.array([b["open"] for b in bars])160        h = np.array([b["high"] for b in bars])161        lo = np.array([b["low"] for b in bars])162        c = closes163        sp = edge_spread(o, h, lo, c)164        var = r.var()165        bounce = -(sp**2) / 4 / var if np.isfinite(sp) and var > 0 else 0.0166        boot = moving_block_bootstrap(r, ac1, block=21, n_boot=300, seed=42)167        lo_ci, hi_ci = percentile_ci(boot - bounce)168        tr = train_daily.get(ticker, {})169        family.append({170            "ticker": ticker,171            "val_ac1": round(a, 5), "val_excess_ac1": round(a - bounce, 5),172            "val_excess_ci95": [round(lo_ci, 5), round(hi_ci, 5)],173            "val_p_excess": bootstrap_pvalue(boot - bounce, 0.0),174            "val_vr30_excess": round(vr30 - (1 + 2 * a * (1 - 1 / 30)), 4),175            "train_excess_ac1": tr.get("excess_ac1"),176            "train_vr30_excess": round(tr["vr30"] - (1 + 2 * tr["ac1"] * (1 - 1 / 30)), 4)177            if tr else None,178        })179180    ck = {str(k): [i["rule"] for i in items if i["net"].get(str(k), {}).get("positive")181                   and i["net"].get(str(k), {}).get("ci95_bp", [1])[0] > 0]182          for k in (0.25, 1.0)}183    med_val = float(np.median([f["val_vr30_excess"] for f in family]))184    med_train = float(np.median([f["train_vr30_excess"] for f in family185                                 if f["train_vr30_excess"] is not None]))186    summary = {187        "pool_evaluated": len(items),188        "net_pos_ci_at": ck,189        "ckx_validation": next((i for i in items if i["rule"].startswith("R:CKX")), None),190        "ckx_subperiods": ckx_subs,191        "es_spy_val_fresh_-1": xcorr_out.get("ES->SPY", {}).get("fresh_xcorr", {}).get("-1"),192        "es_spy_val_p_-1": xcorr_out.get("ES->SPY", {}).get("p_fresh_-1"),193        "spx_spy_val_fresh_-1": xcorr_out.get("SPX->SPY", {}).get("fresh_xcorr", {}).get("-1"),194        "daily_family_median_vr30_excess": {"train_2008_2015": round(med_train, 4),195                                            "validation": round(med_val, 4)},196    }197198    results = {199        "experiment": "expH_oos_stability",200        "run_utc": run_utc.isoformat(),201        "author": "Simon-Pierre Boucher",202        "contact": "contact@spboucher.ai",203        "data_source": "hfmarketdata.io",204        "confidence_level": "evaluates Level-1/2 claims on the validation split",205        "protocol": {"validation": VALIDATION, "subs": VAL_SUBS,206                     "holdout_untouched": True},207        "pool_items": items,208        "xcorr": xcorr_out,209        "daily_family": family,210        "summary": summary,211        "client_stats": vars(client.stats) | {"refreshes": list(client.stats.refreshes)},212        "manifest": collect_manifest(),213    }214    out_dir = REPO_ROOT / "results" / "expH_oos_stability" / run_utc.strftime("%Y%m%dT%H%M%SZ")215    out_dir.mkdir(parents=True)216    (out_dir / "results.json").write_text(217        json.dumps(expg.sanitize(results), indent=2, allow_nan=False) + "\n")218    print(f"\nwrote {out_dir.relative_to(REPO_ROOT)}/results.json")219    print(json.dumps(expg.sanitize(summary), indent=1))220221222if __name__ == "__main__":223    main()224