#!/usr/bin/env python3
"""Generates the synthetic 1-minute OHLCV fixtures used by CI (no vendor data) and the reference values
(numpy/statsmodels/exchange_calendars/arch) consumed by tests/reference. Deterministic (seed 20261102).

Fixture: 2 symbols × 5 sessions around Thanksgiving 2024 (2024-11-25 … 2024-11-29, incl. the early close on 11-29
and the holiday 11-28), extended hours 04:00–19:59 ET with regular bars 09:30–16:00 inclusive; ~3 % of regular
minutes randomly absent (zero-volume minutes are absent at the source); the 16:00 bar is present on every session.
"""
import json, math, os, sys
import numpy as np
import pandas as pd

root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
out_dir = os.path.join(root, "datasets", "fixtures"); os.makedirs(out_dir, exist_ok=True)
rng = np.random.default_rng(20261102)
sessions = ["2024-11-25", "2024-11-26", "2024-11-27", "2024-11-29", "2024-12-02"]
early = {"2024-11-29"}
rows = []
truth = {}
for sym, p0, vol in [("SYN1", 100.0, 0.0008), ("SYN2", 25.0, 0.0015)]:
    price = p0
    truth[sym] = {}
    for s in sessions:
        close_min = 13 * 60 if s in early else 16 * 60
        ext_end = 17 * 60 if s in early else 20 * 60
        minutes = list(range(4 * 60, ext_end))
        day = pd.Timestamp(s, tz="America/New_York")
        sess_rv = 0.0; n_reg = 0; closes = []
        for m in minutes:
            regular = 9 * 60 + 30 <= m <= close_min
            # drop ~3 % of regular minutes and ~40 % of extended minutes, but never the open/close prints
            if m not in (9 * 60 + 30, close_min):
                if regular and rng.random() < 0.03: continue
                if not regular and rng.random() < 0.40: continue
            r = rng.normal(0, vol * (1.0 if regular else 0.5))
            o = price; c = price * math.exp(r); h = max(o, c) * (1 + abs(rng.normal(0, 1e-4))); l = min(o, c) * (1 - abs(rng.normal(0, 1e-4)))
            v = float(rng.integers(100, 5000)) if regular else float(rng.integers(0, 300) + 1)
            ts = (day + pd.Timedelta(minutes=m)).tz_convert("UTC")
            rows.append((sym, ts.strftime("%Y-%m-%dT%H:%M:%SZ"), round(o, 4), round(h, 4), round(l, 4), round(c, 4), v))
            price = c
            if regular:
                closes.append(round(c, 4)); n_reg += 1   # truth from the rounded prices written to the CSV
        lr = np.diff(np.log(np.array(closes)))
        truth[sym][s] = {"n_regular_bars": n_reg, "rv": float(np.sum(lr ** 2)), "bpv": float((math.pi / 2) * (len(lr) / (len(lr) - 1)) * np.sum(np.abs(lr[1:]) * np.abs(lr[:-1]))),
                         "rsv_neg": float(np.sum(lr[lr < 0] ** 2)), "rsv_pos": float(np.sum(lr[lr > 0] ** 2)), "rq": float(len(lr) / 3 * np.sum(lr ** 4)),
                         "expected_bars": 211 if s in early else 391}
df = pd.DataFrame(rows, columns=["symbol", "timestamp_utc", "open", "high", "low", "close", "volume"])
df.to_csv(os.path.join(out_dir, "synthetic_bars_1m.csv"), index=False)
json.dump(truth, open(os.path.join(out_dir, "synthetic_bars_1m.expected.json"), "w"), indent=1)
print("wrote", len(df), "bars for", df.symbol.nunique(), "symbols;", df.groupby("symbol").size().to_dict())

# --- reference values from established software -------------------------------------------------------------
ref = {}
try:
    import exchange_calendars as xc
    cal = xc.get_calendar("XNYS", start="2000-01-01", end="2026-12-31")
    sess = cal.sessions_in_range("2000-01-01", "2026-12-31")
    ref["xnys"] = {"n_sessions_2000_2026": int(len(sess)), "sessions_2024": [str(d.date()) for d in cal.sessions_in_range("2024-01-01", "2024-12-31")],
                   "early_closes_2000_2026": [str(d.date()) for d in cal.early_closes.sessions_in_range("2000-01-01", "2026-12-31")] if hasattr(cal, "early_closes") else None,
                   "holidays_2000_2026": [str(d.date()) for d in pd.bdate_range("2000-01-01", "2026-12-31") if d not in sess],
                   "exchange_calendars_version": xc.__version__}
except Exception as e:
    ref["xnys"] = {"error": str(e)}
import statsmodels.api as sm
from scipy import stats as sps
X = np.column_stack([np.ones(500), rng.normal(size=500), rng.normal(size=500) * 2 + 1]); y = 0.5 + 1.5 * X[:, 1] - 0.7 * X[:, 2] + rng.normal(size=500) * (1 + 0.5 * np.abs(X[:, 1]))
ols = sm.OLS(y, X).fit(); hc0 = sm.OLS(y, X).fit(cov_type="HC0"); hc3 = sm.OLS(y, X).fit(cov_type="HC3"); hac = sm.OLS(y, X).fit(cov_type="HAC", cov_kwds={"maxlags": 5, "use_correction": False})
groups = np.arange(500) // 25; cl = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": groups})
np.savetxt(os.path.join(out_dir, "ols_design.csv"), np.column_stack([y, X[:, 1], X[:, 2]]), delimiter=",", header="y,x1,x2", comments="", fmt="%.17g")
ref["ols"] = {"params": ols.params.tolist(), "bse": ols.bse.tolist(), "pvalues": ols.pvalues.tolist(), "rsquared": float(ols.rsquared), "ssr": float(ols.ssr),
              "hc0_bse": hc0.bse.tolist(), "hc3_bse": hc3.bse.tolist(), "hac5_bse": hac.bse.tolist(), "cluster_bse": cl.bse.tolist(), "statsmodels_version": sm.__version__,
              "t_pvalues": {"1.96_inf": float(2 * sps.norm.sf(1.96)), "2.0_10": float(2 * sps.t.sf(2.0, 10)), "-3.0_30": float(2 * sps.t.sf(3.0, 30))}}
try:
    from arch import arch_model
    import arch
    s2 = 0.04 / (1 - 0.05 - 0.9); e = 0.0; r = np.empty(3000); g = np.random.default_rng(7)
    for t in range(3000):
        s2 = 0.04 + 0.05 * e * e + 0.9 * s2; e = math.sqrt(s2) * g.standard_normal(); r[t] = 0.01 + e
    np.savetxt(os.path.join(out_dir, "garch_returns.csv"), r, header="r", comments="", fmt="%.17g")
    out = {"arch_version": arch.__version__}
    for name, kw in [("garch", dict(vol="GARCH", p=1, q=1)), ("gjr", dict(vol="GARCH", p=1, o=1, q=1)), ("egarch", dict(vol="EGARCH", p=1, o=1, q=1))]:
        for dist in ["normal", "t"]:
            am = arch_model(r, mean="Constant", dist=dist, rescale=False, **kw); res = am.fit(disp="off")
            out[f"{name}.{dist}"] = {"params": {k: float(v) for k, v in res.params.items()}, "std_err": {k: float(v) for k, v in res.std_err.items()}, "loglik": float(res.loglikelihood),
                                     "converged": bool(res.convergence_flag == 0), "forecast_h5": res.forecast(horizon=5, reindex=False).variance.values[-1].tolist() if name != "egarch" else res.forecast(horizon=5, method="simulation", simulations=20000, reindex=False).variance.values[-1].tolist()}
    ref["garch"] = out
except Exception as e:
    ref["garch"] = {"error": repr(e)}
json.dump(ref, open(os.path.join(out_dir, "reference_values.json"), "w"), indent=1)
print("reference values written:", list(ref.keys()))