spb/wp15_uqo

1"""Stage D.2 — monthly universe membership with STRICTLY LAGGED screens.23For each month m (1992-02 onward), membership is decided from month m-1 daily4metadata only (no look-ahead, CLAUDE.md §6.2):5  * days_present >= 10;6  * median RTH close >= $5 (config: sample_filters.baseline.min_price_lagged);7  * median RTH bars/day >= 100 (min_bars_per_day_lagged);8  * rank by total RTH dollar volume among eligible -> top500 / top1000 / top3000.910A crude common-stock heuristic flags likely non-common instruments from ticker11shape (documented limitation; no security master exists in the source):12  * '.'/'-' class/series suffixes are kept (share classes are common stock);13  * 5-letter tickers ending in F or Y -> foreign/ADR OTC;14  * tickers ending in Q (bankruptcy), W/WS (warrants), U (units), P/PR (preferred).1516Output: data/metadata/universe_monthly.parquet17  (month, ticker, days, med_close, med_bars, dollar_vol, eligible,18   top500, top1000, top3000, flag_noncommon)19"""2021from __future__ import annotations2223from pathlib import Path2425import polars as pl26import yaml2728ROOT = Path(__file__).resolve().parents[1]29DAILY = ROOT / "data" / "metadata" / "daily"3031cfg = yaml.safe_load((ROOT / "config" / "sample_filters.yaml").read_text())["baseline"]32MIN_PRICE = float(cfg["min_price_lagged"])33MIN_BARS = int(cfg["min_bars_per_day_lagged"])34MIN_DAYS = 10353637def month_key(p: Path) -> str:38    return p.stem.replace("daily_", "")394041def next_month(m: str) -> str:42    y, mm = int(m[:4]), int(m[5:7])43    mm += 144    if mm == 13:45        y, mm = y + 1, 146    return f"{y:04d}-{mm:02d}"474849NONCOMMON = (50    (pl.col("ticker").str.len_chars() == 5)51    & pl.col("ticker").str.slice(4, 1).is_in(["F", "Y", "Q", "W", "U"])52) | pl.col("ticker").str.contains(r"(\.WS|\.W|\.U|\.P|-P|\.PR)$")535455def main() -> None:56    files = sorted(DAILY.glob("daily_*.parquet"))57    frames = []58    for f in files:59        m_prev = month_key(f)60        m = next_month(m_prev)61        agg = (62            pl.scan_parquet(f)63            .filter(pl.col("rth_last_close").is_not_null())64            .group_by("ticker")65            .agg(66                pl.len().alias("days"),67                pl.col("rth_last_close").median().alias("med_close"),68                pl.col("bars_rth").median().alias("med_bars"),69                pl.col("rth_dollar_vol").sum().alias("dollar_vol"),70            )71            .with_columns(72                pl.lit(m).alias("month"),73                ((pl.col("days") >= MIN_DAYS)74                 & (pl.col("med_close") >= MIN_PRICE)75                 & (pl.col("med_bars") >= MIN_BARS)).alias("eligible"),76                NONCOMMON.alias("flag_noncommon"),77            )78            .with_columns(79                pl.col("dollar_vol")80                .rank("ordinal", descending=True)81                .over(pl.col("eligible"))  # rank within eligible group82                .alias("rank_dv"),83            )84            .with_columns([85                (pl.col("eligible") & (pl.col("rank_dv") <= n)).alias(f"top{n}")86                for n in (500, 1000, 3000)87            ])88            .collect()89        )90        frames.append(agg)91    out = pl.concat(frames).sort("month", "rank_dv")92    dest = ROOT / "data" / "metadata" / "universe_monthly.parquet"93    out.write_parquet(dest)94    last = out.filter(pl.col("month") == out["month"].max())95    print(f"months={out['month'].n_unique()} rows={out.height}")96    print(f"last month: eligible={last['eligible'].sum()} "97          f"top500={last['top500'].sum()} noncommon={last['flag_noncommon'].sum()}")9899100if __name__ == "__main__":101    main()102