1"""Baseline ladder B0-B6 with train-only preprocessing and an equal tuning budget per information set."""2from __future__ import annotations34import json5import random6from dataclasses import dataclass, field78import numpy as np9import polars as pl10from sklearn.linear_model import LogisticRegression, Ridge11from sklearn.metrics import log_loss1213INFO_SETS = {14 "B1": ["fundamentals"],15 "B2": ["fundamentals", "daily"],16 "B3": ["fundamentals", "daily", "intraday_30m"],17 "B4": ["fundamentals", "daily", "intraday_5m"],18 "B5": ["fundamentals", "daily", "intraday_1m"],19 "B5all": ["fundamentals", "daily", "intraday_30m", "intraday_5m", "intraday_1m"],20}212223@dataclass24class Preprocessor:25 """Winsorise (1/99 %), impute (median) and standardise using TRAINING statistics only."""26 cols: list[str]27 lo: dict = field(default_factory=dict)28 hi: dict = field(default_factory=dict)29 med: dict = field(default_factory=dict)30 mu: dict = field(default_factory=dict)31 sd: dict = field(default_factory=dict)32 fitted_on_rows: int = 03334 def fit(self, X: pl.DataFrame) -> "Preprocessor":35 for c in self.cols:36 s = X[c].cast(pl.Float64)37 s = s.filter(s.is_finite())38 if s.len() == 0:39 self.lo[c], self.hi[c], self.med[c], self.mu[c], self.sd[c] = 0.0, 0.0, 0.0, 0.0, 1.040 continue41 lo, hi = float(s.quantile(0.01)), float(s.quantile(0.99))42 w = s.clip(lo, hi)43 self.lo[c], self.hi[c], self.med[c] = lo, hi, float(w.median())44 self.mu[c] = float(w.mean())45 sd = float(w.std()) if w.len() > 1 else 1.046 self.sd[c] = sd if sd and sd > 1e-12 else 1.047 self.fitted_on_rows = X.height48 return self4950 def transform(self, X: pl.DataFrame) -> np.ndarray:51 out = np.empty((X.height, len(self.cols)), dtype=np.float64)52 for j, c in enumerate(self.cols):53 v = X[c].cast(pl.Float64).to_numpy().astype(np.float64)54 v = np.where(np.isfinite(v), v, np.nan)55 v = np.clip(v, self.lo[c], self.hi[c])56 v = np.where(np.isnan(v), self.med[c], v)57 out[:, j] = (v - self.mu[c]) / self.sd[c]58 return out5960 def to_dict(self) -> dict:61 return {"cols": self.cols, "lo": self.lo, "hi": self.hi, "med": self.med, "mu": self.mu, "sd": self.sd,62 "fitted_on_rows": self.fitted_on_rows}636465def columns_for(info_set: str, fc: dict[str, list[str]]) -> list[str]:66 cols = []67 for fam in INFO_SETS[info_set]:68 cols += fc[fam]69 return cols707172# ----------------------------------------------------------------------------- B073def base_rate_predict(train_y: np.ndarray, n: int) -> np.ndarray:74 return np.full(n, float(np.clip(train_y.mean(), 1e-4, 1 - 1e-4)))757677def firm_history_predict(train: pl.DataFrame, test: pl.DataFrame, y: str) -> np.ndarray:78 """B0b: trailing firm-level adverse rate (from training rows only, shrunk to the pooled rate with prior weight 8)."""79 pooled = float(train[y].mean())80 g = train.group_by("entity_id").agg([pl.col(y).sum().alias("k"), pl.len().alias("n")])81 t = test.select("entity_id").join(g, on="entity_id", how="left").with_columns([pl.col("k").fill_null(0), pl.col("n").fill_null(0)])82 k, n = t["k"].to_numpy().astype(float), t["n"].to_numpy().astype(float)83 return np.clip((k + 8 * pooled) / (n + 8), 1e-4, 1 - 1e-4)848586# ------------------------------------------------------------------------ logistic87def fit_logistic(Xtr: np.ndarray, ytr: np.ndarray, Xva: np.ndarray, yva: np.ndarray, C_grid: list[float], seed: int) -> tuple:88 best, best_ll, best_C = None, np.inf, None89 for C in C_grid:90 m = LogisticRegression(C=C, penalty="l2", solver="lbfgs", max_iter=2000, random_state=seed)91 m.fit(Xtr, ytr)92 ll = log_loss(yva, np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6))93 if ll < best_ll:94 best, best_ll, best_C = m, ll, C95 return best, {"C": best_C, "val_logloss": best_ll}969798def fit_ridge(Xtr, ytr, Xva, yva, alpha_grid=(0.1, 1, 10, 100, 1000)):99 best, best_mse, best_a = None, np.inf, None100 for a in alpha_grid:101 m = Ridge(alpha=a).fit(Xtr, ytr)102 mse = float(np.mean((m.predict(Xva) - yva) ** 2))103 if mse < best_mse:104 best, best_mse, best_a = m, mse, a105 return best, {"alpha": best_a, "val_mse": best_mse}106107108# ------------------------------------------------------------------------ LightGBM109def _sample_params(space: dict, rng: random.Random) -> dict:110 return {k: rng.choice(v) for k, v in space.items()}111112113def fit_lightgbm(Xtr, ytr, Xva, yva, cfg: dict, seed: int, objective: str = "binary", n_jobs: int = 8) -> tuple:114 import lightgbm as lgb115 rng = random.Random(seed)116 best, best_score, best_params, trials = None, np.inf, None, []117 for t in range(int(cfg["budget_trials"])):118 p = _sample_params(cfg["search_space"], rng)119 params = dict(objective=objective, learning_rate=p["learning_rate"], num_leaves=p["num_leaves"],120 min_child_samples=p["min_child_samples"], feature_fraction=p["feature_fraction"],121 bagging_fraction=p["bagging_fraction"], bagging_freq=1 if p["bagging_fraction"] < 1 else 0,122 lambda_l2=p["lambda_l2"], n_estimators=int(cfg["n_estimators_max"]), random_state=seed + t,123 n_jobs=n_jobs, verbose=-1)124 m = lgb.LGBMClassifier(**params) if objective == "binary" else lgb.LGBMRegressor(**params)125 m.fit(Xtr, ytr, eval_set=[(Xva, yva)], callbacks=[lgb.early_stopping(int(cfg["early_stopping_rounds"]), verbose=False)])126 if objective == "binary":127 score = log_loss(yva, np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6))128 else:129 score = float(np.mean((m.predict(Xva) - yva) ** 2))130 trials.append({"trial": t, "params": p, "best_iteration": int(m.best_iteration_ or 0), "val_score": score})131 if score < best_score:132 best, best_score, best_params = m, score, {**p, "best_iteration": int(m.best_iteration_ or 0)}133 return best, {"val_score": best_score, "params": best_params, "trials": trials}134135136def predict_proba(model, X: np.ndarray) -> np.ndarray:137 return np.clip(model.predict_proba(X)[:, 1], 1e-6, 1 - 1e-6)138