WP39: phase 0-2 scaffold — data audit, frozen design, PIT protocol, client, features, tests
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
76 changed files +6,968 −0
added
.env.example
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +# WP39 UQO — environment-based authorization. Never commit a real .env. | |
| 2 | +# FinData OS (mandatory primary data source). Key file is read if FINDATA_API_KEY is unset. | |
| 3 | +FINDATA_BASE_URL=https://findata.spboucher.ai/api/v1 | |
| 4 | +FINDATA_API_KEY= | |
| 5 | +FINDATA_API_KEY_FILE=~/findata-secrets/api-key-wp39.txt | |
| 6 | +# Scratch cache for licensed raw extracts (outside the repository, never committed) | |
| 7 | +WP39_CACHE=~/wp39-cache | |
| 8 | +# SEC EDGAR fair-access User-Agent (public data, no secret) for ticker->CIK->SIC crosswalk | |
| 9 | +SEC_USER_AGENT="UQO WP39 research (mailto:simon-pierre.boucher@uqo.ca)" | |
| 10 | +# Set to 1 to run the live API contract tests | |
| 11 | +WP39_LIVE=0 | |
added
.gitignore
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +# Raw & processed data — never committed (see data/raw/README.md for provenance) | |
| 2 | +data/raw/* | |
| 3 | +!data/raw/README.md | |
| 4 | +data/processed/* | |
| 5 | +!data/processed/.gitkeep | |
| 6 | + | |
| 7 | +# Python | |
| 8 | +__pycache__/ | |
| 9 | +*.pyc | |
| 10 | +.pytest_cache/ | |
| 11 | +.venv/ | |
| 12 | +*.egg-info/ | |
| 13 | + | |
| 14 | +# LaTeX build artifacts (paper/main.pdf IS committed) | |
| 15 | +paper/*.aux | |
| 16 | +paper/*.log | |
| 17 | +paper/*.out | |
| 18 | +paper/*.toc | |
| 19 | +paper/*.lof | |
| 20 | +paper/*.lot | |
| 21 | +paper/*.fls | |
| 22 | +paper/*.fdb_latexmk | |
| 23 | +paper/*.bbl | |
| 24 | +paper/*.blg | |
| 25 | +paper/*.spl | |
| 26 | +paper/*.synctex.gz | |
| 27 | +paper/*.run.xml | |
| 28 | +paper/*.bcf | |
| 29 | + | |
| 30 | +# Logs | |
| 31 | +logs/ | |
| 32 | +*.log | |
| 33 | + | |
| 34 | +# macOS | |
| 35 | +.DS_Store | |
added
CLAUDE.md
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +# CLAUDE.md — UQO Research WP39 | |
| 2 | + | |
| 3 | +## THE INVISIBLE BALANCE SHEET | |
| 4 | +### Can Intraday Financial Markets Detect Corporate Deterioration Before Accounting Disclosures? | |
| 5 | + | |
| 6 | +**Project ID:** `wp39_uqo` · **Institution:** Université du Québec en Outaouais (UQO) · **PI:** Simon-Pierre Boucher | |
| 7 | +**Status:** research project built 2026-11-02 under the owner's specification (reproduced below in condensed form; the | |
| 8 | +owner's original brief is the authoritative text). All findings are hypotheses until reproducibly estimated. | |
| 9 | + | |
| 10 | +Owner instructions added during the build (2026-11-02): (1) the FinData **fundamentals API** is the accounting source and | |
| 11 | +must be used; (2) **minimise AI usage** — no LLM features, no neural/text models, no AI-generated statistics. | |
| 12 | + | |
| 13 | +## Non-negotiable execution rules | |
| 14 | +1. FinData OS (`https://findata.spboucher.ai/api/v1`) is the primary source of prices, volumes, earnings events and | |
| 15 | + fundamentals. No silent substitution; every exception is documented in `DATA_AUDIT.md` / `docs/API_INTEGRATION.md`. | |
| 16 | +2. Inspect before changing; preserve all other working papers; this repository is its own git repo. | |
| 17 | +3. Credentials from the protected store only (`~/findata-secrets/api-key-wp39.txt` on the node); never printed, never | |
| 18 | + in URLs, git, artifacts, notebooks, figures or manuscript. No destructive ingestion, no unapproved backfills. | |
| 19 | +4. FirstRate Data is privately licensed: no raw records in git, public endpoints, replication packages or logs. | |
| 20 | +5. Hypotheses, variable definitions, PIT rules, baselines, splits and significance plan are frozen in | |
| 21 | + `docs/RESEARCH_DESIGN.md` and `docs/PIT_PROTOCOL.md` before interpreting results. Prediction ≠ causation. | |
| 22 | +6. Every claim traces to a saved table/figure/log/test. No fabricated statistics or references. | |
| 23 | +7. Phases with gates; `PROGRESS.md`, `DECISIONS.md`, `DATA_AUDIT.md`, `LIMITATIONS.md` are kept current. | |
| 24 | +8. English everywhere; LaTeX article compiled with `latexmk` (`paper/`). | |
| 25 | + | |
| 26 | +## Where things are | |
| 27 | +| Item | Location | | |
| 28 | +|---|---| | |
| 29 | +| Design, PIT rules, novelty | `docs/RESEARCH_DESIGN.md`, `docs/PIT_PROTOCOL.md`, `docs/NOVELTY_MEMO.md` | | |
| 30 | +| Data audit, limitations, decisions, progress | root `*.md` files | | |
| 31 | +| Code | `src/wp39/` (client, provenance, calendar, instruments, PIT fundamentals, corporate actions, ingest, features, labels, samples, splits, models, inference, portfolio, robustness, figures, pipeline) | | |
| 32 | +| Pipeline | `scripts/00_…09_` in order; `scripts/01a_extract_node_tables.sh` runs on M3U96a | | |
| 33 | +| Tests | `tests/` (`pytest`; live API tests need `WP39_LIVE=1`) | | |
| 34 | +| Artifacts | `artifacts/manifests` (hashes, versions), `artifacts/evaluation`, `artifacts/logs`, `results/`, `figures/` | | |
| 35 | +| Paper | `paper/main.tex` + `sections/` + generated `tables/`, `figures/` | | |
| 36 | + | |
| 37 | +## How to run (compute node M3U96a, API on localhost) | |
| 38 | +```bash | |
| 39 | +rsync -a --exclude .venv --exclude artifacts/logs ~/Desktop/UQO/Recherche/Recherche-UQO/finance/invisible-balance-sheet/ M3U96a:~/wp39_uqo/ | |
| 40 | +ssh M3U96a 'cd ~/wp39_uqo && uv venv -p 3.12 .venv && uv pip install -e ".[dev]" && export FINDATA_BASE_URL=http://127.0.0.1:8600/api/v1 && \ | |
| 41 | + bash scripts/01a_extract_node_tables.sh && for s in 00 01 02 03 04 05 06 07 08; do .venv/bin/python scripts/${s}_*.py; done' | |
| 42 | +``` | |
| 43 | +Then `rsync` `results/ figures/ artifacts/ paper/tables paper/figures` back and `cd paper && make`. | |
added
DATA_AUDIT.md
+167 −0
@@ -0,0 +1,167 @@ | ||
| 1 | +# DATA_AUDIT.md — WP39 `wp39_uqo` | |
| 2 | + | |
| 3 | +Phase 0 discovery and feasibility audit of the **FinData OS** API (`https://findata.spboucher.ai/api/v1`) as the primary | |
| 4 | +source for *The Invisible Balance Sheet*. All numbers below were measured on **2026-11-02** (America/Toronto) with the | |
| 5 | +owner's Claude Code key (admin scope, laptop) and then with a dedicated least-privilege key `wp39_uqo` | |
| 6 | +(scopes `read:market, read:fundamentals, query, export`; stored outside git in `~/findata-secrets/api-key-wp39.txt` on | |
| 7 | +M3U96a and `~/Desktop/Cluster/secrets/findata/api-key-wp39.txt` on the laptop, mode 600). No credential appears in | |
| 8 | +this repository, in logs or in artifacts. Probe transcripts: `artifacts/logs/phase0_probe.json` (produced by | |
| 9 | +`scripts/00_probe_findata.py`). | |
| 10 | + | |
| 11 | +## 0. Viability conclusion | |
| 12 | + | |
| 13 | +**YES — viable, with two structural caveats that shape the paper.** | |
| 14 | + | |
| 15 | +1. A licensed, point-in-time usable cohort exists: **5,451 US-listed entities** with quarterly fundamentals | |
| 16 | + (**67 metrics**, `filing_date` 2012-01-03 → 2026-10-30, median 45 quarterly periods per firm), **373,326 earnings-release | |
| 17 | + events** with release date and (for 65 %) before-open/after-close timing, and **6.70 billion raw 1-minute bars** for | |
| 18 | + 16,195 stock symbols (2000-01-03 → 2026-10-30) including 8,069 `-DELISTED` symbols. Exports of 1-minute history are | |
| 19 | + fast (≈ 1 s per symbol-decade, see §6). | |
| 20 | +2. **Caveat A — survivorship in fundamentals, not in prices.** The fundamentals/earnings tables contain **zero delisted | |
| 21 | + entities** (ingestion ran for *active* tickers only). Minute bars do cover delisted names, so the price side is | |
| 22 | + survivorship-free, but the accounting panel is conditioned on being listed on 2026-11-01. The vendor does serve | |
| 23 | + fundamentals for at least some delisted tickers (probe: `FMD` → HTTP 200 with data; `CWZ` → 404), so a scoped | |
| 24 | + delisted-fundamentals backfill is *possible* but requires owner approval and a capacity plan (not run; see §8). | |
| 25 | +3. **Caveat B — PIT is a proxy.** `available_at = filing_date + 1 day` is a policy, `revision_id = 1` for every fact | |
| 26 | + (`observed_at` = first ingestion, 2026-11-01/02). Values are the vendor's *current* history, so restatements that | |
| 27 | + happened before ingestion are invisible. The design therefore (i) uses only prior-quarter fundamentals as inputs, | |
| 28 | + (ii) uses the **earnings-release date** (not the filing date) as the reveal date, and (iii) reserves a stringent | |
| 29 | + specification using SEC-filing-level reconstruction for a subset (Phase 5, conditional on time). | |
| 30 | + | |
| 31 | +## 1. API contract — documented vs observed | |
| 32 | + | |
| 33 | +| Purpose | Route (observed in `openapi.json`, OpenAPI 3.1.0, API 1.0.0) | Observed behaviour | | |
| 34 | +|---|---|---| | |
| 35 | +| Datasets | `GET /catalog/datasets` | 64 datasets; 62 `active`, `fundamentals.sec_companyfacts` = `pending_configuration`, `market_ticks` = `unsupported` | | |
| 36 | +| Dataset detail | `GET /catalog/datasets/{id}` | version, `license_class = frd-private` | | |
| 37 | +| Bars | `GET /market/bars` | `interval ∈ {1m, 5m, 1d}` only (no 30m/1h on REST); `adjustment ∈ {raw, split, split_div}`; `limit ≤ 10 000`; `format ∈ {json, arrow, csv}`; `version` pin | | |
| 38 | +| Bars schema | `GET /catalog/schema/bars` | schema v1, `timestamp_convention = bar_open`, `timezone_storage = UTC`, 20 fields (see §3) | | |
| 39 | +| Fundamentals | `GET /fundamentals/{entity}` | `as_of`, `metrics`, `period_type ∈ {q,a}`, `include_superseded`, **`limit ≤ 20 000`** (default 2 000) | | |
| 40 | +| Fundamental periods | `GET /fundamentals/{entity}/periods` | one row per fiscal period with `filing_date`, `available_at`, `max_revision`, `restated` | | |
| 41 | +| SQL | `POST /query` | one read-only statement, `max_rows ≤ 100 000`, 30 s; views = dataset ids with `.`→`_` | | |
| 42 | +| Exports | `POST /exports` → `GET /exports/{id}` → `/download` | body `{dataset_id, symbols ≤ 500, start, end, adjustment, format}`; **range ≤ 12 years per request (HTTP 413 otherwise)**; Parquet with schema metadata and pinned `X-Dataset-Version` | | |
| 43 | +| Coverage | `GET /coverage`, `/coverage/symbols` (`limit ≤ 5 000`, cursor), `/coverage/heatmap` | per-session bar counts vs `expected_bars` (391 regular) | | |
| 44 | +| Instruments | `GET /instruments`, `/instruments/{symbol}` | status `active`/`delisted`, first/last date | | |
| 45 | +| Quality | `GET /quality`, `/quality/anomalies` | row-level flags (`price_jump`, `outside_session`, …) | | |
| 46 | +| **Earnings events** | **no route** | dataset `fundamentals.earnings` is listed (0 rows reported) but rows live in Postgres `earnings_events` → read-only extraction on the node (documented exception, §7) | | |
| 47 | +| Corporate actions | **no route** | Postgres `corporate_actions`: 6,131 splits (2000-01-03 → 2026-10-15), 243,209 cash dividends; applied at query time via `adjustment=` | | |
| 48 | + | |
| 49 | +Reconciliation notes: the catalog reports `row_count = 0` for Postgres-backed datasets (`fundamentals.*`, `corporate_actions`, | |
| 50 | +`reference.instruments`); that is a catalog limitation, not an absence of data. The REST `interval` enum is narrower than the | |
| 51 | +lake (30m/1h exist only as vendor-adjusted datasets); WP39 rebuilds 5m/30m/1d bars **from the raw 1m export** so that every | |
| 52 | +frequency shares one source and one adjustment path. | |
| 53 | + | |
| 54 | +## 2. Market bars — coverage | |
| 55 | + | |
| 56 | +Counts of distinct stock symbols with at least one bar on a mid-June session (SQL playground, `session_date =` predicate): | |
| 57 | + | |
| 58 | +| Session | 1m symbols | 1m bars | 1d symbols | of which `-DELISTED` in 1d | | |
| 59 | +|---|---:|---:|---:|---:| | |
| 60 | +| 2001-06-15 | — | — | 1,088 | 0 | | |
| 61 | +| 2005-06-15 | 3,498 (340,665 bars delisted) | 631,260 | 1,504 | 0 | | |
| 62 | +| 2010-06-15 | — | — | 2,309 | 0 | | |
| 63 | +| 2015-06-15 | 6,489 | 1,207,920 | 2,985 | 0 | | |
| 64 | +| 2020-06-15 | — | — | 4,144 | 0 | | |
| 65 | +| 2025-06-16 | 6,918 | 1,317,343 | 6,485 | 0 | | |
| 66 | + | |
| 67 | +Finding: the vendor **1d dataset is incomplete** relative to the 1m dataset (about half the symbols in 2005–2015 and no | |
| 68 | +delisted names). Decision D-003: daily bars are *derived* from the 1m export (regular session 09:30–16:00 ET) — never taken | |
| 69 | +from `market_bars.stock.1d.raw`. Delisted symbols are present in the 1m lake (e.g. `AA.B-DELISTED` 531 sessions | |
| 70 | +2014–2016; `CWZ-DELISTED` 2000–2014). | |
| 71 | + | |
| 72 | +Bar conventions verified on `XOM` and `SPY` exports: timestamps are bar **open** in UTC (`2012-01-03 09:01Z` = 04:01 ET), | |
| 73 | +extended hours 04:00–19:59 ET are present, zero-volume minutes are absent (bar counts per session 405–798 vs 391 expected | |
| 74 | +regular bars), `quality_flags` list is empty on normal rows, `revision_id = 1`. | |
| 75 | + | |
| 76 | +## 3. Bars schema (Arrow, schema v1) | |
| 77 | + | |
| 78 | +`instrument_id, symbol_as_reported, venue (null), asset_class, interval, timestamp_utc (timestamp[us, UTC], bar open), | |
| 79 | +session_date (date32, exchange tz), open, high, low, close, volume (double), currency, trade_count (int32?, derived only), | |
| 80 | +vwap (double?, derived only), adjustment_type, source_id, ingested_at, quality_flags (list<string>), revision_id (int32)`. | |
| 81 | +Parquet exports carry metadata `findata.schema = bars`, `findata.schema_version = 1`, `findata.timestamp_convention = bar_open`. | |
| 82 | + | |
| 83 | +## 4. Fundamentals — coverage and PIT fields | |
| 84 | + | |
| 85 | +- Entities: **5,451** (`frd:<TICKER>`, `security_id = firstrate:stock:<TICKER>`, none delisted, none with `security_id` null). | |
| 86 | +- Metrics observed per period: **67** (the vendor documents 76; 9 are never populated in the API output). Names: | |
| 87 | + `acc_oci, assets_c, assets_nc, assets_tot, capex, cash, cash_business_acq, cash_change, cash_debt_sec, cash_div, | |
| 88 | + cash_equity, cash_fin, cash_inv, cash_inv_acq, cash_op, cogs, cos_inc, debt, debt_c, debt_nc, def_rev, depr, dps, ebit, | |
| 89 | + ebitda, ebt, eps, eps_d, equity, fx_impact_cash, gm, gp, intangibles, interest_exp, intr_cover, inv_c, inv_nc, inventory, | |
| 90 | + investments, liab_c, liab_nc, liab_tot, net_inc, net_inc_com, net_inc_non_control, net_loss_discontinued, om, op_ex, | |
| 91 | + op_inc, payables, ppe, pref_divs, receivables, research_dev, ret_earnings, rev, roa, roe, roic, sgna, share_comp, shares, | |
| 92 | + shares_av, shares_av_diluted, tax_asset, tax_expense, tax_liability`. | |
| 93 | +- Quarterly `rev` facts by fiscal year (entities / facts): 2011: 1,798 / 1,812 · 2012: 2,461 / 8,725 · 2013: 2,623 / 9,868 · | |
| 94 | + 2014: 2,771 / 10,560 · 2015: 2,916 / 11,145 · 2016: 3,076 / 11,656 · 2017: 3,276 / 12,340 · 2018: 3,505 / 13,209 · | |
| 95 | + 2019: 3,763 / 14,034 · 2020: 4,309 / 15,488 · 2021: 4,663 / 17,458 · 2022: 4,851 / 18,289 · 2023: 5,043 / 18,904 · | |
| 96 | + 2024: 5,244 / 19,467 · 2025: 5,370 / 19,625 · 2026: 4,928 / 10,915. `filing_date` is never null (no `pit_fallback`). | |
| 97 | +- Firms by number of quarterly periods: ≥ 8: 5,018 · ≥ 20: 4,331 · ≥ 40: 2,891. First quarter year: 2011: 2,141 · | |
| 98 | + 2012: 317 · 2013–2019: 148–256 per year · 2020: 551 · 2021: 353 · 2022–2025: 164–215 per year. | |
| 99 | +- Filing lag (filing_date − fiscal_period_end, quarterly): p10 = 30 d, median = 40 d, p90 = 77 d. | |
| 100 | +- Q4 values are genuine quarterly flows (AAPL FY2019 revenue 260.17 bn = 84.31 + 58.02 + 53.81 + 64.04 bn). | |
| 101 | +- PIT fields per fact: `filing_date`, `published_at` (null), `available_at = filing_date + 1 d 00:00 ET`, `observed_at` | |
| 102 | + (= 2026-11-01 18:24 ET for the first ingestion), `revision_id = 1`, `restatement_flag = false`, `superseded_at = null`. | |
| 103 | + The API `as_of` semantics (`available_at ≤ as_of` and (`revision_id = 1` or `observed_at ≤ as_of`)) were exercised: | |
| 104 | + AAPL `as_of = 2020-01-01` returns 2,000 facts (limit) and reports `facts_hidden_by_as_of = 2211`. | |
| 105 | +- Entity master: `cik`, `sector`, `industry` are **null for all 5,451 entities** → external SEC crosswalk needed (§7). | |
| 106 | + | |
| 107 | +## 5. Earnings-release events (reveal dates) | |
| 108 | + | |
| 109 | +Postgres `earnings_events` (`entity_id, event_date, eps_estimate, eps_reported, release_time, source_id, observed_at`): | |
| 110 | +373,326 rows, 5,451 entities, 2000-01-01 → 2027-04-29, one row per (entity, date). `release_time`: `after_close` 124,832 · | |
| 111 | +`before_open` 120,586 · null 127,908. For 2012–2024 the null share is 35–36 %; `eps_estimate` is null for 24–27 % of | |
| 112 | +events (vendor estimate; vintage unknown → exploratory only). Row counts for 2025–2026 are inflated by forward-dated | |
| 113 | +placeholders (2026: 35,361 events), which are dropped at the sample end (2026-10-30). | |
| 114 | + | |
| 115 | +Matching quarterly filings to the first release event in `[period_end, filing_date + 10 d]` (213,954 filings 2012→): | |
| 116 | +96.3 % matched; 1,493 (0.7 %) have `filing_date < event_date`; 16,868 (7.9 %) have a lag > 30 d (likely mismatches or | |
| 117 | +late filers); median `filing_date − event_date` = **1 day**. Rule (PIT_PROTOCOL §3): reveal date `A_q` = matched release | |
| 118 | +date; a firm-quarter is **excluded** when no release is matched, when `filing_date < event_date`, or when the lag exceeds | |
| 119 | +30 days; the prediction cutoff always satisfies `c < min(A_q, F_q)` in trading days. | |
| 120 | + | |
| 121 | +## 6. Throughput and capacity (measured on M3U96a, API on localhost) | |
| 122 | + | |
| 123 | +| Operation | Measured | | |
| 124 | +|---|---| | |
| 125 | +| Export 1m raw, 1 symbol, 2012-01-01→2019-12-31 (`XOM`) | ready in 1.0 s, 859,354 rows, 13.6 MB Parquet, dataset version 17375 | | |
| 126 | +| Export 1m raw, 1 symbol, 2020-01-01→2026-10-30 | ready in 1.0 s, 884,983 rows, 15.4 MB | | |
| 127 | +| Full-range export (2012→2026, 14.8 years) | **HTTP 413** — limit 12 years per request → two requests per symbol | | |
| 128 | +| SQL cross-section (`session_date =` on 1m) | ≈ 3 s | | |
| 129 | +| Coverage per symbol (`/coverage/symbols`) | ms, cursor-paged | | |
| 130 | +| Fundamentals, one entity, all quarterly facts | one call (≤ 20 000 rows) | | |
| 131 | + | |
| 132 | +Capacity model for the main sample (≈ 4,300 firms with ≥ 8 quarters and 1m coverage): 2 exports × ~1.5 s ≈ 3.6 h of | |
| 133 | +export wall-time (sequential; 4 concurrent streams bring it under 1 h), ≈ 30 MB raw Parquet per firm (≈ 130 GB if kept). | |
| 134 | +Decision D-004: raw extracts are processed **streaming** (export → daily feature rows at 1m/5m/30m/1d → delete raw file, | |
| 135 | +keep SHA-256 + row count in the manifest), so steady-state scratch usage stays below 5 GB. The storage node M3U96a | |
| 136 | +(28 cores, 96 GB RAM, 319 GB free on the internal SSD, hard floor 60 GB for FinData) hosts the computation; the feature | |
| 137 | +panel (≈ 4,300 firms × ~3,700 sessions × 4 frequencies) is < 2 GB. | |
| 138 | + | |
| 139 | +## 7. External sources and exceptions | |
| 140 | + | |
| 141 | +| Need | Source | Status | Why not FinData | | |
| 142 | +|---|---|---|---| | |
| 143 | +| Reveal dates / release timing | FinData Postgres `earnings_events` (read-only `psql` on M3U96a) | available | not exposed by the API (gap logged for the FinData backlog) | | |
| 144 | +| CIK, SIC industry | SEC `company_tickers.json` (10,423 rows) + `submissions/CIK##########.json` (`sic`, `sicDescription`, `exchanges`, `formerNames`) | verified reachable (fair-access UA) | entity master has no CIK/sector | | |
| 145 | +| Daily factors (H5 only) | Kenneth French library, FF5 2×3 daily + momentum daily | verified (HTTP 200, updated 2026-10-26) | not a FinData dataset | | |
| 146 | +| Analyst consensus | vendor `eps_estimate` in `earnings_events` (24–27 % missing, vintage unknown) | exploratory only | no point-in-time consensus history available | | |
| 147 | +| TAQ / quotes / order flow | none | **not available** (vendor supplies bars only) | bar-based proxies are *never* called order flow, spreads or informed trading | | |
| 148 | + | |
| 149 | +## 8. Gaps, risks and recommendations to the owner | |
| 150 | + | |
| 151 | +1. **Delisted fundamentals backfill (recommended, not run).** A one-off `fundamentals` job for the 8,069 `-DELISTED` | |
| 152 | + tickers (vendor returns data for at least part of them) would remove the survivorship conditioning of the accounting | |
| 153 | + panel. Estimated cost: ≤ 8,069 vendor calls (1 req/s policy ≈ 2.3 h), a few hundred MB in Postgres. Requires: owner | |
| 154 | + approval, `findata ingest fundamentals --delisted` (does not exist yet; connector change on M3U96a `~/apps/findata`). | |
| 155 | +2. **Earnings events API route** — expose `fundamentals.earnings` through `/api/v1/earnings/{entity}` so that WP39 (and | |
| 156 | + future projects) stop reading Postgres directly. | |
| 157 | +3. **SEC EDGAR connector** (`fundamentals.sec_companyfacts`, pending a User-Agent setting) would give filing-level | |
| 158 | + as-originally-reported values for the stringent PIT specification. | |
| 159 | +4. API bug found: `POST /admin/keys` returns HTTP 500 (`api_keys_user_id_fkey` violation) when the caller is an API key | |
| 160 | + whose `user_id` is not in `users`; the CLI `findata create-key` works. Logged for the FinData backlog, not fixed here. | |
| 161 | + | |
| 162 | +## 9. License and privacy constraints applied | |
| 163 | + | |
| 164 | +FirstRate Data: perpetual private use, research and derived statistics allowed with attribution, **no redistribution of raw | |
| 165 | +records**. Consequently: no raw bars or fundamentals are committed; `data/raw` stays empty; the replication package ships | |
| 166 | +code, manifests (hashes, versions, counts), derived aggregate tables and figures only; the manuscript attributes FirstRate | |
| 167 | +Data via FinData OS. SEC and Kenneth French data are public. | |
added
DECISIONS.md
+14 −0
@@ -0,0 +1,14 @@ | ||
| 1 | +# DECISIONS.md — dated design and engineering decisions (WP39) | |
| 2 | + | |
| 3 | +| ID | Date (ET) | Decision | Reason | Consequence | | |
| 4 | +|---|---|---|---|---| | |
| 5 | +| D-001 | 2026-11-02 13:00 | Compute on **M3U96a** (FinData host) with a dedicated least-privilege key `wp39_uqo`; repository lives in the UQO Research workspace (`Recherche-UQO/finance/invisible-balance-sheet`, dir name follows the series convention; project id `wp39_uqo`) and is rsync'd to `M3U96a:~/wp39_uqo` for runs | API on localhost, lake co-located, 28 cores / 96 GB; secrets never leave `~/findata-secrets` / `~/Desktop/Cluster/secrets` | results and manifests synced back; raw cache stays on the node | | |
| 6 | +| D-002 | 2026-11-02 13:05 | **Earnings release dates** (`earnings_events.event_date`) define the reveal date `A_q`; the filing date `F_q` defines structured-fact availability (`available_at = F_q + 1`) | press releases precede 10-Q filings by a median of 1 day; cutoffs must precede the *first* public reveal | cutoff rule `c < min(A_q, F_q)`; release rows read from Postgres (no API route) — documented exception | | |
| 7 | +| D-003 | 2026-11-02 13:10 | Daily bars are **rebuilt from the 1-minute raw export** (regular session), not taken from `market_bars.stock.1d.raw` | vendor 1d dataset covers about half the symbols before 2016 and no delisted names | one source, one adjustment path for all four frequencies | | |
| 8 | +| D-004 | 2026-11-02 13:30 | Streaming feature extraction (export → features → delete raw Parquet; manifest keeps SHA-256) | disk budget on the FinData host (hard floor 60 GB free) | re-runs re-export from the pinned dataset version | | |
| 9 | +| D-005 | 2026-11-02 14:00 | Universe = all FinData fundamentals entities (survivor-conditioned); delisted-fundamentals backfill **proposed, not run** | vendor ingestion covered active tickers only; backfill needs owner approval + capacity plan (rule 3 of the brief) | survivorship caveat stated in abstract, data section, limitations; price side is survivorship-free | | |
| 10 | +| D-006 | 2026-11-02 14:05 | **No neural (B7) nor text/multimodal (B8) models; no LLM anywhere in the pipeline** | owner instruction received mid-session on 2026-11-02 ("limite le plus possible l'utilisation AI"); also keeps the frequency comparison about information sets rather than capacity | ladder is B0–B6; LightGBM provides the nonlinear benchmark with equal budgets | | |
| 11 | +| D-007 | 2026-11-02 14:10 | Primary target = adverse **operating-income** surprise vs a seasonal random walk with drift, scaled by total assets, threshold −0.5 % | no point-in-time consensus exists in FinData; operating income is less affected by one-off items than net income; Foster (1977) model is the classic honest expectation | consensus-based miss (vendor `eps_estimate`) is exploratory only | | |
| 12 | +| D-008 | 2026-11-02 14:15 | Frequency sets B3/B4/B5 use **identical feature families** (one column per family) computed from bars of 30 m / 5 m / 1 m | compare information sets, not dimensionality | a supplementary "all frequencies" model is run in Phase 4 | | |
| 13 | +| D-009 | 2026-11-02 14:20 | Literature verified with **Crossref metadata** (no AI summarisation); references without a verifiable DOI are cited without one (Grossman–Stiglitz 1980, Holm 1979) | rule 7 of the brief (no fabricated references) | `artifacts/logs/crossref_batch*.json` are the evidence | | |
| 14 | +| D-010 | 2026-11-02 14:30 | Phase gates that require "owner approval" are recorded as **self-approved under the autonomous mandate** with the memo attached; the owner can veto any of them before Phase 6 | the owner asked for the whole project to be built in one autonomous session and is not available mid-task | each gate entry in PROGRESS.md names the memo and the date | | |
added
LIMITATIONS.md
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +# LIMITATIONS.md — living list (WP39) | |
| 2 | + | |
| 3 | +## Data | |
| 4 | +- **Survivor-conditioned accounting panel.** Fundamentals and earnings-release rows exist only for the 5,451 tickers active | |
| 5 | + on 2026-11-01. Firms that delisted during 2012–2026 (including those that deteriorated and failed) are absent from the | |
| 6 | + firm-quarter sample, although their minute bars exist. Base rates of severe deterioration are therefore understated and | |
| 7 | + H3/H5 effects for the most vulnerable firms are not identified. A delisted-fundamentals backfill is proposed (DATA_AUDIT §8). | |
| 8 | +- **PIT is a documented proxy.** `available_at = filing_date + 1 day` is a policy; all facts are `revision_id = 1` observed | |
| 9 | + on 2026-11-01/02, i.e. the vendor's current (possibly restated) history. Original-as-reported values are not available. | |
| 10 | + Inputs use prior quarters only, so restatements affect the *measurement of inputs and targets*, not the direction of time. | |
| 11 | +- **OHLCV bars, not ticks.** No quotes, no trades, no order flow. Bar-based statistics (down-bar volume share, bar | |
| 12 | + autocorrelation) are descriptive proxies and are never interpreted as informed trading or spreads. | |
| 13 | +- Vendor merges ticker changes under the latest ticker (fully since 2022); CIK/SIC come from the SEC's *current* | |
| 14 | + `company_tickers.json`, so historical ticker reuse before 2022 is only partially auditable (`formerNames` check). | |
| 15 | +- `release_time` is unknown for ~35 % of release events; we never use release-day bars, so this affects only the | |
| 16 | + descriptive timing exhibit. | |
| 17 | +- The vendor `eps_estimate` has an unknown vintage and is missing for a quarter of events → exploratory only. | |
| 18 | +- 1-minute coverage is thinner before ~2005 and for illiquid names; the sample starts in 2012 and requires ≥ 100 regular | |
| 19 | + bars per session, which excludes the most illiquid firm-days (a documented selection on liquidity). | |
| 20 | + | |
| 21 | +## Design | |
| 22 | +- The event-time cohort conditions on the *existence* of a release in the next 65 trading days; a hypothetical observer | |
| 23 | + knows the approximate schedule but not the exact date. The fixed-calendar design is reported as the robustness check. | |
| 24 | +- Prediction, not causation: a positive H1 shows that bar-level information is statistically informative; it does not | |
| 25 | + establish who traded on private information. | |
| 26 | +- No B7/B8 (neural/text) models by owner instruction; the frequency comparison is therefore within linear and | |
| 27 | + gradient-boosted families only. | |
| 28 | + | |
| 29 | +## To be completed as results arrive | |
| 30 | +- (Phase 3) achieved sample sizes, base rates, coverage gaps by year/size. | |
| 31 | +- (Phase 5) which placebo/stress tests failed or were inconclusive. | |
added
PROGRESS.md
+22 −0
@@ -0,0 +1,22 @@ | ||
| 1 | +# PROGRESS.md — WP39 `wp39_uqo` | |
| 2 | + | |
| 3 | +## Phase 0 — Discovery and feasibility (2026-11-02) — GATE PASSED | |
| 4 | +- Inspected the UQO Research workspace (`Recherche-UQO/`, 39 papers, harmonisation standard); no `wp39` existed; created | |
| 5 | + `finance/invisible-balance-sheet/` from the template; git initialised on a dedicated repository (no other paper touched). | |
| 6 | +- Probed FinData: `/health` ok, 64 datasets, OpenAPI 3.1 (45 routes), bars schema v1, fundamentals (67 metrics, 5,451 | |
| 7 | + entities, 2012→2026), earnings events (Postgres), coverage endpoints, export throughput (≈ 1 s per symbol-decade; | |
| 8 | + 12-year cap per request). Created least-privilege key `wp39_uqo`. | |
| 9 | +- Verified external public sources: SEC `company_tickers.json` + `submissions` (CIK/SIC), Kenneth French daily factors. | |
| 10 | +- Verified 94 bibliography entries on Crossref; wrote `docs/NOVELTY_MEMO.md`, `literature/RELATED_WORK_MATRIX.csv`. | |
| 11 | +- Outputs: `DATA_AUDIT.md` (viability = YES with survivorship + PIT caveats), `configs/*.yaml`, `docs/API_INTEGRATION.md`, | |
| 12 | + `docs/COMPUTE_BUDGET.md`, `docs/PIT_PROTOCOL.md`, `docs/RESEARCH_DESIGN.md` (frozen), `DECISIONS.md` D-001…D-010. | |
| 13 | +- Owner instructions received during the session: (1) the fundamentals API must be used — it is the accounting source; | |
| 14 | + (2) minimise AI usage — no LLM features, no neural models (D-006). | |
| 15 | +- Blockers: none. Next safe step: Phase 1 gate memo, then code + tests (Phase 2). | |
| 16 | + | |
| 17 | +## Phase 1 — Research design and novelty — GATE (self-approved under the autonomous mandate, D-010) | |
| 18 | +- `docs/RESEARCH_DESIGN.md` frozen 2026-11-02 14:40 ET; hypotheses H1–H5 with null/alternative, primary endpoint (H1, | |
| 19 | + h = 20, log loss, binary adverse operating-income surprise), decision rules, splits, inference, power plan. | |
| 20 | +- Owner review pending; nothing in Phases 2–5 depends on test-period data until the Phase 3 gate. | |
| 21 | + | |
| 22 | +## Phase 2 — Data acquisition and verification — IN PROGRESS | |
added
configs/data_contract.yaml
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +# Data contract — verified against the live FinData OS API on 2026-11-02 (see DATA_AUDIT.md) | |
| 2 | +findata: | |
| 3 | + base_url: https://findata.spboucher.ai/api/v1 | |
| 4 | + openapi: /openapi.json | |
| 5 | + datasets: | |
| 6 | + bars_1m: market_bars.stock.1m.raw # raw, bar-open timestamps in UTC, 04:00-20:00 ET, zero-volume minutes absent | |
| 7 | + bars_1m_etf: market_bars.etf.1m.raw # SPY used as market proxy | |
| 8 | + bars_1d: market_bars.stock.1d.raw # INCOMPLETE vs 1m (fewer symbols) -> daily bars are rebuilt from 1m | |
| 9 | + fundamentals: fundamentals.financials # Postgres-backed; GET /fundamentals/{entity} | |
| 10 | + earnings: fundamentals.earnings # Postgres table earnings_events (no API route yet) -> documented exception | |
| 11 | + limits: | |
| 12 | + bars_1m_days_per_call: 31 | |
| 13 | + rows_per_page: 10000 | |
| 14 | + export_years_per_request: 12 | |
| 15 | + export_symbols_per_request: 500 | |
| 16 | + query_timeout_s: 30 | |
| 17 | + query_max_rows: 100000 | |
| 18 | + rate_limit_per_min: 600 | |
| 19 | + fundamentals_limit_max: 20000 | |
| 20 | + bars_schema_version: 1 | |
| 21 | + timestamp_convention: bar_open | |
| 22 | + timezone_storage: UTC | |
| 23 | + exchange_tz: America/New_York | |
| 24 | + regular_session: ["09:30", "16:00"] # the 16:00 bar is the closing print | |
| 25 | + extended_session: ["04:00", "20:00"] | |
| 26 | + license_class: frd-private | |
| 27 | +pit: | |
| 28 | + available_at_policy: "filing_date + 1 day 00:00 America/New_York (vendor proxy; fallback period_end + 91 d flagged pit_fallback)" | |
| 29 | + revision_semantics: "revision_id=1 observed 2026-11-01/02; no pre-ingestion restatement history (vendor limitation)" | |
| 30 | + reveal_date: "earnings_events.event_date (press release), release_time in {before_open, after_close, null}" | |
| 31 | +universe: | |
| 32 | + entity_source: fundamentals entities (5,451 active tickers as of 2026-11-01; NO delisted entities -> survivorship caveat) | |
| 33 | + min_quarters: 8 | |
| 34 | + first_filing_year: 2012 | |
| 35 | + last_session: 2026-10-30 | |
| 36 | +external: | |
| 37 | + sec_company_tickers: https://www.sec.gov/files/company_tickers.json | |
| 38 | + sec_submissions: https://data.sec.gov/submissions/CIK{cik:010d}.json | |
| 39 | + ff5_daily: https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/F-F_Research_Data_5_Factors_2x3_daily_CSV.zip | |
| 40 | + mom_daily: https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/F-F_Momentum_Factor_daily_CSV.zip | |
added
configs/models.yaml
+27 −0
@@ -0,0 +1,27 @@ | ||
| 1 | +# Equal tuning budget across information sets. Hyper-parameters are tuned ONLY on the validation period. | |
| 2 | +baselines: | |
| 3 | + B0: {type: base_rate} | |
| 4 | + B1: {type: logistic, inputs: [fundamentals, controls]} | |
| 5 | + B2: {type: logistic, inputs: [fundamentals, controls, daily]} | |
| 6 | + B3: {type: logistic, inputs: [fundamentals, controls, daily, intraday_30m]} | |
| 7 | + B4: {type: logistic, inputs: [fundamentals, controls, daily, intraday_5m]} | |
| 8 | + B5: {type: logistic, inputs: [fundamentals, controls, daily, intraday_1m]} | |
| 9 | +logistic: | |
| 10 | + penalty: l2 | |
| 11 | + C_grid: [0.01, 0.03, 0.1, 0.3, 1.0, 3.0] | |
| 12 | + standardize: train_only | |
| 13 | + impute: train_median | |
| 14 | +lightgbm: | |
| 15 | + information_sets: [B1, B2, B3, B4, B5] | |
| 16 | + budget_trials: 24 # identical random-search budget for every information set | |
| 17 | + seed: 20261102 | |
| 18 | + search_space: | |
| 19 | + num_leaves: [7, 15, 31, 63] | |
| 20 | + learning_rate: [0.01, 0.03, 0.05] | |
| 21 | + min_child_samples: [50, 100, 200] | |
| 22 | + feature_fraction: [0.5, 0.7, 0.9] | |
| 23 | + bagging_fraction: [0.6, 0.8, 1.0] | |
| 24 | + lambda_l2: [0.0, 1.0, 10.0] | |
| 25 | + n_estimators_max: 2000 | |
| 26 | + early_stopping_rounds: 100 | |
| 27 | +neural: {enabled: false, reason: "owner instruction 2026-11-02: minimise AI usage; B7/B8 not run"} | |
added
configs/research.yaml
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +# Frozen research design parameters (see docs/RESEARCH_DESIGN.md). Changing a value requires a DECISIONS.md entry. | |
| 2 | +seed: 20261102 | |
| 3 | +sample: | |
| 4 | + start: 2012-01-01 | |
| 5 | + end: 2026-10-30 | |
| 6 | + min_quarters_per_firm: 8 | |
| 7 | + min_price_usd: 1.0 # at cutoff, raw close | |
| 8 | + min_regular_bars_per_day: 100 # sessions with fewer 1m regular-session bars are "sparse" | |
| 9 | + min_valid_days_in_window: 15 # out of the 20-trading-day feature window | |
| 10 | +cutoffs: | |
| 11 | + event_time_horizons_td: [40, 20, 10, 5, 1] # trading days before the earnings release date A_q | |
| 12 | + primary_horizon_td: 20 | |
| 13 | + fixed_calendar: month_end # last trading day of each month | |
| 14 | + embargo_rule: "cutoff c < min(A_q, F_q); bars with session_date <= c only; fundamentals with available_at <= c only" | |
| 15 | +feature_window: | |
| 16 | + short_td: 20 | |
| 17 | + long_td: 60 | |
| 18 | +frequencies: [1d, 30m, 5m, 1m] | |
| 19 | +targets: | |
| 20 | + primary: | |
| 21 | + name: adverse_opinc_surprise | |
| 22 | + variable: op_inc | |
| 23 | + expectation: seasonal_random_walk_with_drift # E[X_q] = X_{q-4} + mean(X_{q-1-k} - X_{q-5-k}, k=0..3), PIT | |
| 24 | + scale: assets_tot (latest available) | |
| 25 | + threshold: -0.005 # surprise <= -0.5% of total assets | |
| 26 | + continuous_primary: scaled_opinc_surprise | |
| 27 | + secondary: | |
| 28 | + - {name: roa_yoy_decline, rule: "roa_q - roa_{q-4} <= -0.01"} | |
| 29 | + - {name: ocf_innovation_negative, rule: "(cash_op_q - cash_op_{q-4})/assets <= -0.01"} | |
| 30 | + - {name: liquidity_deterioration, rule: "current ratio falls >= 10% yoy OR interest coverage falls below 1 from >= 1"} | |
| 31 | + - {name: eps_miss, rule: "eps_reported < eps_estimate (vendor estimate, exploratory)"} | |
| 32 | +splits: | |
| 33 | + train: [2012-01-01, 2018-12-31] | |
| 34 | + validation: [2019-01-01, 2020-12-31] | |
| 35 | + test: [2021-01-01, 2026-10-30] | |
| 36 | + walk_forward: {first_test_year: 2016, embargo_quarters: 1} | |
| 37 | + firm_holdout_fraction: 0.2 | |
| 38 | +inference: | |
| 39 | + cluster: [entity_id, calendar_quarter] | |
| 40 | + bootstrap_blocks: calendar_quarter | |
| 41 | + n_bootstrap: 1000 | |
| 42 | + multiple_testing: holm | |
added
data/README.md
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +# data/ | |
| 2 | + | |
| 3 | +No licensed raw records are stored in this repository. | |
| 4 | + | |
| 5 | +| Path | Content | Committed? | | |
| 6 | +|---|---|---| | |
| 7 | +| `data/raw/` | nothing (raw FirstRate bars live only in the ignored scratch cache `$WP39_CACHE` on the compute node) | no | | |
| 8 | +| `data/processed/` | derived firm-quarter panel, feature tables and labels (Parquet). Derived aggregates only; still **not** committed (size, license caution) | no | | |
| 9 | +| `artifacts/manifests/` | hashed dataset manifests: dataset ids, `X-Dataset-Version`, extraction dates, row counts, SHA-256 of extracts, code SHA | yes | | |
| 10 | + | |
| 11 | +Provenance rules (see `DATA_AUDIT.md`, `docs/API_INTEGRATION.md`): | |
| 12 | +- Market bars and fundamentals come from the owner's FinData OS API (`https://findata.spboucher.ai/api/v1`), a private | |
| 13 | + lakehouse of FirstRate Data (license: private use, no redistribution of raw records). | |
| 14 | +- Earnings-release calendar rows are read from the FinData Postgres table `earnings_events` on the storage node because the | |
| 15 | + API does not expose that dataset yet (documented exception). | |
| 16 | +- External public augmentations: SEC EDGAR `company_tickers.json` and `submissions` (CIK, SIC), Kenneth French data library | |
| 17 | + (daily factors). Both are public, versioned by download date in the manifests. | |
added
data/processed/.gitkeep
+0 −0
added
data/raw/README.md
+4 −0
@@ -0,0 +1,4 @@ | ||
| 1 | +# data/raw — intentionally empty | |
| 2 | +Raw FirstRate Data records (via FinData OS) are licensed for private use and are never stored in this repository. | |
| 3 | +They live only in the ignored scratch cache on the compute node (`$WP39_CACHE`), are deleted after feature extraction, | |
| 4 | +and are described by hashed manifests in `artifacts/manifests/`. | |
added
docs/API_INTEGRATION.md
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +# API_INTEGRATION.md — how WP39 talks to FinData OS | |
| 2 | + | |
| 3 | +## Authorization | |
| 4 | +- Header `Authorization: Bearer <key>`; the key is read from `FINDATA_API_KEY` or the file `FINDATA_API_KEY_FILE` | |
| 5 | + (default `~/findata-secrets/api-key-wp39.txt`, mode 600). The client never logs headers, never puts the key in a URL, | |
| 6 | + and redacts any string starting with `fd_` in exceptions (`src/wp39/api_client.py::_redact`). | |
| 7 | +- Dedicated key `wp39_uqo` (scopes `read:market, read:fundamentals, query, export`; **no admin**), created 2026-11-02 with | |
| 8 | + the node CLI `findata create-key` because `POST /admin/keys` returned HTTP 500 for API-key callers (bug logged in | |
| 9 | + DATA_AUDIT §8). | |
| 10 | +- Compute runs on **M3U96a** against `http://127.0.0.1:8600/api/v1` (same host as the lake; the public URL | |
| 11 | + `https://findata.spboucher.ai/api/v1` is used from the laptop for probes only). | |
| 12 | + | |
| 13 | +## Client (`src/wp39/api_client.py`) | |
| 14 | +- `httpx.Client` with 60 s timeout, retry with bounded exponential backoff (5 attempts, 0.5 → 8 s, jitter) on 429/5xx and | |
| 15 | + network errors; idempotent GETs only are retried automatically; export creation is retried only when no export id was | |
| 16 | + received. | |
| 17 | +- Every response's `X-Request-ID`, `X-Dataset-Id`, `X-Dataset-Version` are recorded in the provenance log | |
| 18 | + (`artifacts/logs/api_calls.jsonl`, one line per call, no payloads). | |
| 19 | +- Cursor iteration for `/market/bars` and `/coverage/symbols`; export polling (`/exports/{id}`) every 1 s up to 10 min. | |
| 20 | +- Arrow/Parquet schema validation against `GET /catalog/schema/bars` (schema v1) before any file is accepted. | |
| 21 | + | |
| 22 | +## Calls used by the pipeline | |
| 23 | +| Step | Call | Scope | Volume | | |
| 24 | +|---|---|---|---| | |
| 25 | +| 00 probe | `/health`, `/catalog/datasets`, `/catalog/schema/bars`, `/openapi.json` | read | 4 | | |
| 26 | +| 01 universe | `/coverage/symbols?dataset_id=market_bars.stock.1m.raw` (cursor, 5,000/page), `/instruments/{symbol}` | read | ≈ 4 + N | | |
| 27 | +| 02 PIT panel | `/fundamentals/{entity}?period_type=q&limit=20000` (all facts, local as-of join), `/fundamentals/{entity}/periods` | read:fundamentals | 2 × N entities | | |
| 28 | +| 03 features | `POST /exports` (1m raw, ≤ 12 y, 1 symbol) ×2 per symbol, `/exports/{id}`, `/download`; SPY from `market_bars.etf.1m.raw` | export | 2 × N + 2 | | |
| 29 | +| checks | `POST /query` (cross-section counts, spot checks ≤ 100 k rows) | query | few | | |
| 30 | + | |
| 31 | +Exception (documented): earnings-release rows are extracted once on M3U96a with | |
| 32 | +`psql -d findata -c "\copy (select ... from earnings_events) to '…'"` (read-only). The extract's SHA-256, row count and | |
| 33 | +timestamp are in `artifacts/manifests/earnings_events.json`. | |
| 34 | + | |
| 35 | +## Idempotent partitioning and manifests | |
| 36 | +- Exports are cached per `(symbol, chunk)` as `$WP39_CACHE/bars/<symbol>__<start>__<end>.parquet` and deleted after the | |
| 37 | + feature rows are written; `artifacts/manifests/bars_manifest.jsonl` keeps `symbol, start, end, dataset_id, | |
| 38 | + dataset_version, row_count, bytes, sha256, exported_at, request_id`. | |
| 39 | +- Fundamentals extracts: `$WP39_CACHE/fundamentals/<entity>.parquet` with the same manifest fields. | |
| 40 | +- Re-running a script skips completed partitions (manifest lookup) and never overwrites a partition with a different | |
| 41 | + dataset version without logging a `version_drift` warning. | |
added
docs/COMPUTE_BUDGET.md
+16 −0
@@ -0,0 +1,16 @@ | ||
| 1 | +# COMPUTE_BUDGET.md | |
| 2 | + | |
| 3 | +| Resource | Plan | Measured / status | | |
| 4 | +|---|---|---| | |
| 5 | +| Compute node | M3U96a (Mac Studio M3 Ultra, 28 cores, 96 GB RAM) — FinData API on localhost | load 1.5–1.9 before start | | |
| 6 | +| Scratch | `~/wp39-cache` on the internal SSD (319 GB free; FinData hard floor 60 GB) | streaming design keeps < 5 GB | | |
| 7 | +| API exports | ≈ 2 × 4,300 symbol-chunks, ≈ 1 s each server-side, 4 concurrent | pilot measured 1.0 s / 860 k rows | | |
| 8 | +| Fundamentals calls | 5,451 × 2 | ms each | | |
| 9 | +| Feature computation | Polars per symbol (≈ 1.7 M rows → 4 frequency tables), 8 worker processes | to be measured in the pilot | | |
| 10 | +| Models | logistic ridge (seconds) ; LightGBM 24 trials × 5 information sets × (1 chronological + 11 walk-forward folds) ≈ 1,440 fits, early stopping; ≈ 2–4 h on 28 cores | — | | |
| 11 | +| Bootstrap | 1,000 stationary-bootstrap draws over 23 test quarters per contrast — seconds | — | | |
| 12 | +| LaTeX | laptop (TeX Live, `latexmk`) | available | | |
| 13 | +| AI usage | none in the pipeline (no LLM features, no neural B7/B8); Claude Code used only as the engineer | owner instruction 2026-11-02 | | |
| 14 | + | |
| 15 | +Pilot (Phase 2 gate): 300 firms stratified by first-fundamentals year and size, full history → validates throughput, | |
| 16 | +feature sanity and label base rates before the full run. | |
added
docs/NOVELTY_MEMO.md
+53 −0
@@ -0,0 +1,53 @@ | ||
| 1 | +# NOVELTY_MEMO.md — the three closest papers and why WP39 is not them | |
| 2 | + | |
| 3 | +Prepared 2026-11-02 from publisher metadata verified on Crossref (`literature/RELATED_WORK_MATRIX.csv`, | |
| 4 | +`artifacts/logs/crossref_batch*.json`). Full texts were not machine-summarised; the points of distinction below are | |
| 5 | +based on the papers' stated designs as reported in their abstracts and our own reading of their published versions. | |
| 6 | + | |
| 7 | +## Closest paper 1 — Grégoire & Martineau (2022), *How Is Earnings News Transmitted to Stock Prices?*, JAR 60(1) | |
| 8 | +- **What it does.** Decomposes price discovery around earnings announcements using TAQ quotes *and* trades, showing that | |
| 9 | + most of the reaction to after-hours announcements occurs in quotes with little trading. | |
| 10 | +- **Strongest objection to WP39 from this paper.** "Information arrives at the announcement; what you call anticipation is | |
| 11 | + the pre-announcement drift already documented, and bars cannot even see quotes." | |
| 12 | +- **How WP39 differs.** WP39 does not study the announcement reaction at all: all features end strictly *before* the | |
| 13 | + release and the outcome is the accounting number itself, not the return. We make no claim about quotes versus trades | |
| 14 | + (our data are OHLCV bars, stated plainly) and we ask a forecasting question — whether the *resolution* of pre-release | |
| 15 | + bar data carries incremental information about fundamentals — which their event-window decomposition does not address. | |
| 16 | + | |
| 17 | +## Closest paper 2 — Chen, Cho, Dou & Lev (2022), *Predicting Future Earnings Changes Using Machine Learning and Detailed Financial Data*, JAR 60(2) | |
| 18 | +- **What it does.** Shows that gradient-boosted trees on detailed XBRL financial-statement items predict the direction of | |
| 19 | + next-year earnings changes better than parsimonious accounting models, and that the predictions are priced slowly. | |
| 20 | +- **Strongest objection.** "Fundamentals-based ML already predicts earnings changes; adding market data is Ou–Penman with | |
| 21 | + a new model, and daily returns are known to lead earnings (Beaver–Lambert–Morse 1980; Kothari–Sloan 1992)." | |
| 22 | +- **How WP39 differs.** Their inputs are accounting items; ours hold the accounting information set fixed (B1) and | |
| 23 | + measure the *marginal* value of market data at four sampling frequencies with equal model capacity (B2–B5, B6). The | |
| 24 | + stock-prices-lead-earnings literature establishes the daily/annual lead; WP39 tests whether anything remains at the | |
| 25 | + minute scale once daily information is in, with a strict PIT availability clock and a release-date reveal clock. | |
| 26 | + | |
| 27 | +## Closest paper 3 — Zhang, Zhang, Cucuringu & Qian (2023/2024), *Volatility Forecasting with Machine Learning and Intraday Commonality*, JFEC | |
| 28 | +- **What it does.** Pools intraday realized-volatility features across many stocks to forecast volatility with ML, showing | |
| 29 | + that commonality in intraday patterns improves volatility forecasts. | |
| 30 | +- **Strongest objection.** "Intraday features forecast volatility, and earnings surprises are volatility events, so your | |
| 31 | + deterioration signal is a volatility forecast in disguise." | |
| 32 | +- **How WP39 differs.** Our target is an accounting quantity with a *sign* (adverse surprise), not a variance; the design | |
| 33 | + includes the volatility families explicitly in every frequency set so that the test isolates information beyond | |
| 34 | + predicted volatility (placebo: a symmetric |surprise| target must show the pattern a volatility channel implies, a signed | |
| 35 | + target must not). We also report directional asymmetry (negative semivariance, down-bar volume) as pre-specified | |
| 36 | + mechanism variables, not as the headline. | |
| 37 | + | |
| 38 | +## Other adjacent work and the recycling check | |
| 39 | +- UQO WP15 (*Faster, Wider, More Fragile?*) studies intraday *shock propagation* across stocks on the same vendor data; | |
| 40 | + it never links bars to accounting outcomes. WP39 shares the data vendor and the realized-measure toolkit only; no | |
| 41 | + result or figure is reused. | |
| 42 | +- Pre-announcement informed trading (Christophe–Ferri–Angel 2004; Berkman et al. 2009; Kaniel et al. 2012; | |
| 43 | + Hendershott–Livdan–Schürhoff 2015) uses trader-type data we do not have; our contribution is orthogonal: whether | |
| 44 | + *anonymous* OHLCV at fine resolution is informative, and whether the information is attributable to the frequency. | |
| 45 | +- Patton & Verardo (2012) and Bollerslev–Li–Xue (2018) model announcement-day betas and volume-volatility; they are | |
| 46 | + announcement-window studies. | |
| 47 | +- Earnings-prediction ML (Easton et al. 2024; Cao et al. 2024) uses accounting, analyst and text inputs, not intraday bars. | |
| 48 | + | |
| 49 | +## Verdict | |
| 50 | +No published paper, to our verified knowledge, measures the marginal forecasting value of 1-minute versus 5-, 30-minute | |
| 51 | +and daily OHLCV for *forthcoming accounting deterioration* with a point-in-time information set and matched model | |
| 52 | +capacity. The question is therefore open. If the empirical answer is a precise null, that is itself informative given | |
| 53 | +the practitioner belief that high-frequency activity "sees" fundamentals early. | |
added
docs/PIT_PROTOCOL.md
+75 −0
@@ -0,0 +1,75 @@ | ||
| 1 | +# PIT_PROTOCOL.md — point-in-time rules for WP39 | |
| 2 | + | |
| 3 | +Frozen 2026-11-02 before any model was estimated. Any change is an amendment logged in `DECISIONS.md`. | |
| 4 | + | |
| 5 | +## 1. Clocks | |
| 6 | + | |
| 7 | +All instants are converted to **America/New_York** for session logic and kept in UTC in storage. Trading days follow the | |
| 8 | +XNYS calendar (`exchange_calendars`). For firm `i` and fiscal quarter `q`: | |
| 9 | + | |
| 10 | +| Symbol | Meaning | Source | | |
| 11 | +|---|---|---| | |
| 12 | +| `E_q` | fiscal period end | `fundamentals_facts.fiscal_period_end` | | |
| 13 | +| `A_q` | earnings-release date (press release) — the **reveal date** of quarter-`q` results | `earnings_events.event_date`, first event in `[E_q, F_q + 10 d]` | | |
| 14 | +| `τ_q` | release timing: `before_open`, `after_close`, or `unknown` | `earnings_events.release_time` | | |
| 15 | +| `F_q` | SEC filing date of the 10-Q/10-K | `fundamentals_facts.filing_date` | | |
| 16 | +| `V_q` | vendor-proxy availability of the structured facts | `available_at = F_q + 1 d 00:00 ET` | | |
| 17 | +| `c` | prediction cutoff (a trading day; features use information up to the **close** of `c`) | design | | |
| 18 | + | |
| 19 | +## 2. The information set at cutoff `c` | |
| 20 | + | |
| 21 | +`I(c)` contains exactly: | |
| 22 | + | |
| 23 | +1. **Market bars** with `session_date ≤ c` (regular session 09:30–16:00 ET for the main specification; the 16:00 bar is the | |
| 24 | + closing print; extended hours 04:00–09:29 and 16:01–19:59 enter only the "extended" variants, never mixed). | |
| 25 | +2. **Fundamentals** with `available_at ≤ c 00:00 ET`, i.e. `F_j + 1 ≤ c`. The latest such quarter is `j*(c)`, and the | |
| 26 | + model also receives the *age* `c − A_{j*}` in trading days. Because `V_q > F_q ≥ A_q > c` by construction (§3), quarter-`q` | |
| 27 | + facts can never enter `I(c)`. | |
| 28 | +3. **Earnings-release history** with `event_date < c` (prior realized releases, used for the historical base-rate feature B0). | |
| 29 | +4. **Static crosswalk** (CIK, SIC) — time-invariant by assumption; `formerNames` are checked for ticker reuse (Phase 2 audit). | |
| 30 | +5. **Nothing else.** No consensus estimates for quarter `q` (vintage unknown), no index-membership backfills, no | |
| 31 | + contemporaneous factors. | |
| 32 | + | |
| 33 | +## 3. Cutoff rules | |
| 34 | + | |
| 35 | +- **Event-time design** (primary, H1–H4): for horizon `h ∈ {40, 20, 10, 5, 1}` trading days, `c = A_q − h` trading days. | |
| 36 | + Admissible only if `c < min(A_q, F_q)` *and* `c ≥ A_{q−1} + 1` (the previous release has already happened, so the | |
| 37 | + cohort window never straddles two reveals). With `τ_q = before_open`, bars of `A_q` contain the reaction and are excluded | |
| 38 | + (`c ≤ A_q − 1` guarantees it); with `τ_q = after_close` or `unknown` the same rule applies — we never use `A_q` bars. | |
| 39 | +- **Fixed-calendar design** (H1 robustness, H5): `c` = last trading day of each month; the target is the *next* release | |
| 40 | + after `c` (first `A_q > c`), restricted to `A_q − c ≤ 65` trading days. The identity of "the next quarter" is known ex | |
| 41 | + ante; its date is not needed by the forecaster. | |
| 42 | +- **Firm-quarter exclusions** (frozen): no matched release; `F_q < A_q`; `F_q − A_q > 30` calendar days; `E_q` not within | |
| 43 | + 60–120 days of `E_{q−1}` (irregular fiscal calendars); quarter `q − 4` or `q − 1` not available at `c` (expectation | |
| 44 | + undefined); raw close at `c` < 1 USD; fewer than 15 valid sessions in the 20-day feature window. | |
| 45 | + | |
| 46 | +## 4. Label construction (no price-based labels) | |
| 47 | + | |
| 48 | +Primary: `S_q = (X_q − E_c[X_q]) / TA_{j*}`, `X = op_inc` (quarterly operating income), `TA` = total assets of the latest | |
| 49 | +available quarter, `E_c[X_q] = X_{q−4} + (1/4) Σ_{k=0..3} (X_{q−1−k} − X_{q−5−k})` using only quarters available at `c` | |
| 50 | +(seasonal random walk with drift; Foster 1977). `D_q = 1[S_q ≤ −0.005]`. The secondary targets are in | |
| 51 | +`configs/research.yaml`. `X_q` itself is taken from the **first ingested revision** of quarter `q` (the only one available), | |
| 52 | +which is the vendor's current value — see LIMITATIONS §PIT. | |
| 53 | + | |
| 54 | +## 5. Leakage battery (each item is a unit test in `tests/`) | |
| 55 | + | |
| 56 | +| # | Risk | Guard | Test | | |
| 57 | +|---|---|---|---| | |
| 58 | +| L1 | quarter-`q` facts used as inputs | as-of join on `available_at ≤ c` | `test_pit_leakage.py::test_no_future_fundamentals` | | |
| 59 | +| L2 | selection by fiscal end instead of availability | join key is `available_at`, never `fiscal_period_end` | `test_pit_leakage.py::test_join_key_is_available_at` | | |
| 60 | +| L3 | post-release bars in the feature window | `session_date ≤ c < A_q` | `test_pit_leakage.py::test_no_bars_after_cutoff` | | |
| 61 | +| L4 | global standardisation / imputation / tuning on test data | transformers fit on train rows only; tuning on validation period only | `test_train_only_transforms.py` | | |
| 62 | +| L5 | DST / timezone errors | UTC → ET conversion, 09:30 and 16:00 bars identified on both DST regimes | `test_timestamp_dsts.py` | | |
| 63 | +| L6 | ticker reuse / identifier drift | crosswalk with effective dates; SEC `formerNames` audit | `test_instrument_ids.py` | | |
| 64 | +| L7 | splits/dividends creating mechanical returns | split-adjusted returns from `corporate_actions` (ex-date rule), raw bars immutable | `test_corporate_actions.py` | | |
| 65 | +| L8 | release timing assumed intraday | release-day bars never used; timing enters only as a descriptive variable | `test_labels.py::test_release_day_excluded` | | |
| 66 | +| L9 | overlapping target windows across splits | purge + 1-quarter embargo at every split boundary | `test_splits.py` | | |
| 67 | +| L10 | survivorship by construction | universe = all fundamentals entities (documented survivor conditioning), delisted bars kept where present | `01_audit_universe.py` report | | |
| 68 | +| L11 | vendor ingestion date mistaken for knowledge date | `observed_at` is never used as a knowledge date; `available_at` only | `test_pit_leakage.py::test_observed_at_not_used` | | |
| 69 | +| L12 | randomised placebo | labels shuffled within calendar quarter → gain must vanish; cutoffs shifted −45 td → H4 gain must vanish | `08_run_robustness.py` | | |
| 70 | + | |
| 71 | +## 6. Equivalence check of the local as-of join with the API | |
| 72 | + | |
| 73 | +For 25 random entities and 10 random `as_of` instants, the set of (metric, period) pairs returned by | |
| 74 | +`GET /fundamentals/{entity}?as_of=` must equal the set produced by the local filter `available_at ≤ as_of` on the full | |
| 75 | +extract. Implemented in `tests/test_api_contract.py::test_asof_equivalence` (live test, `WP39_LIVE=1`). | |
added
docs/REPLICATION.md
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +# REPLICATION.md — how to reproduce WP39 | |
| 2 | + | |
| 3 | +## Requirements | |
| 4 | +- Python ≥ 3.12 with `uv` (`uv venv -p 3.12 .venv && uv pip install -e ".[dev]"`). | |
| 5 | +- Access to FinData OS (owner-issued key with scopes `read:market, read:fundamentals, query, export`) exposed as | |
| 6 | + `FINDATA_API_KEY` or a mode-600 file at `FINDATA_API_KEY_FILE`. Without access, Phases 2–5 cannot run; the saved | |
| 7 | + evaluation artifacts in `artifacts/evaluation/` and `results/` allow Phase 6 (tables/figures/paper) to be regenerated. | |
| 8 | +- Read-only `psql` access to the FinData Postgres database on the storage node for `scripts/01a_extract_node_tables.sh` | |
| 9 | + (earnings events, entities, corporate actions — not yet exposed by the API). | |
| 10 | +- Internet access to `sec.gov` (CIK/SIC) and the Kenneth French library (daily factors). | |
| 11 | +- TeX Live with `latexmk` for the paper. | |
| 12 | + | |
| 13 | +## Order of execution (compute node, API on localhost) | |
| 14 | +```bash | |
| 15 | +export FINDATA_BASE_URL=http://127.0.0.1:8600/api/v1 WP39_CACHE=~/wp39-cache | |
| 16 | +bash scripts/01a_extract_node_tables.sh # node extracts + manifests | |
| 17 | +python scripts/00_probe_findata.py # contract reconciliation -> artifacts/logs/phase0_probe.json | |
| 18 | +python scripts/01_audit_universe.py # universe.parquet, results/universe_audit.json | |
| 19 | +python scripts/02_build_pit_panel.py # quarters.parquet (labels), design freeze hash | |
| 20 | +python scripts/03_build_features.py --workers 8 # streaming exports -> session features -> rolling windows | |
| 21 | +python scripts/04_build_labels.py # sample_event.parquet, sample_calendar.parquet, cascades | |
| 22 | +python scripts/05_train_baselines.py # B0-B5 logistic, ridge, firm hold-out, walk-forward | |
| 23 | +python scripts/06_train_advanced.py # LightGBM equal-budget, horizon ladder, OOF signals | |
| 24 | +python scripts/07_evaluate.py # metrics, paired contrasts, GW tests, Holm, power | |
| 25 | +python scripts/08_run_robustness.py # placebos, subgroups, panel regressions, H5 | |
| 26 | +python scripts/09_make_manuscript.py # tables, figures, results_macros.tex | |
| 27 | +cd paper && make # main.pdf | |
| 28 | +pytest -q # 29 tests; WP39_LIVE=1 adds the live API contract tests | |
| 29 | +``` | |
| 30 | +Runtime on a Mac Studio M3 Ultra (28 cores): see `PROGRESS.md` for measured wall-times of each step. | |
| 31 | + | |
| 32 | +## What is and is not in the package | |
| 33 | +- In: code, configs, docs, tests, manifests (hashes/versions/counts), derived evaluation tables, figures, paper. | |
| 34 | +- Out: raw bars and fundamentals (FirstRate Data license), the per-symbol feature cache (derived from licensed data; | |
| 35 | + regenerable), model binaries. | |
| 36 | +- Determinism: all seeds come from `configs/research.yaml` (`seed: 20261102`); LightGBM uses `n_jobs=12` with | |
| 37 | + deterministic histogram construction at the default settings, so results are reproducible to numerical precision on | |
| 38 | + the same architecture. | |
added
docs/RESEARCH_DESIGN.md
+111 −0
@@ -0,0 +1,111 @@ | ||
| 1 | +# RESEARCH_DESIGN.md — preregistered specification (internal) | |
| 2 | + | |
| 3 | +Frozen **2026-11-02 14:40 ET**, before the first test-period observation was opened. Hash of this file at freeze is | |
| 4 | +recorded in `artifacts/manifests/design_freeze.json` by `scripts/02_build_pit_panel.py`. Later changes are amendments in | |
| 5 | +`DECISIONS.md` (dated, with reason) and never alter frozen primary endpoints. | |
| 6 | + | |
| 7 | +## 1. Question | |
| 8 | + | |
| 9 | +Does one-minute OHLCV activity contain **incremental** information about a firm's *forthcoming* adverse accounting | |
| 10 | +outcome, beyond (i) point-in-time fundamentals, (ii) daily OHLCV, (iii) coarser intraday aggregates of the same bars, and | |
| 11 | +(iv) calendar/industry/size controls? Anticipation (prices move before the release) is not the claim; the claim is about | |
| 12 | +the marginal forecasting value of **sampling frequency** holding the information set, calendar coverage, universe, | |
| 13 | +cutoff, horizon and training budget fixed. | |
| 14 | + | |
| 15 | +## 2. Hypotheses — null, alternative, primary endpoint, decision rule | |
| 16 | + | |
| 17 | +| | Null `H0` | Alternative `H1` | Primary endpoint | Decision rule | | |
| 18 | +|---|---|---|---|---| | |
| 19 | +| **H1** incremental information | OOS log loss of B5 (fundamentals + daily + 1m) ≥ that of B2 (fundamentals + daily) | B5 < B2 | Δ log loss (B5 − B2) on the untouched test period, 2021-01 → 2026-10, horizon h = 20 | 95 % block-bootstrap CI (blocks = calendar quarters, 1,000 draws) excludes 0 **and** Giacomini–White conditional test p < 0.05 (two-sided) | | |
| 20 | +| **H2** temporal resolution | Δ(B5 − B2) ≤ Δ(B4 − B2) and ≤ Δ(B3 − B2) | 1m gain strictly exceeds 5m and 30m gains | paired Δ log loss B5 − B4 and B5 − B3 | both CIs exclude 0 after Holm correction over the 3 frequency contrasts | | |
| 21 | +| **H3** heterogeneity | the B5 − B2 gain is equal across terciles of size, illiquidity, leverage | gain larger in small, illiquid, levered firms | interaction coefficient `β_HF×tercile` in the panel regression (§5) | Holm-corrected p < 0.05 for the pre-specified ordering | | |
| 22 | +| **H4** timing | the B5 − B2 gain does not increase as h shrinks (40→20→10→5→1) | monotone increase | slope of Δ log loss on −h across horizons | sign test on the five ordered horizons + placebo cutoffs (−45 td) showing no gain | | |
| 23 | +| **H5** pricing | the LFDS does not predict the announcement-window abnormal return nor post-cutoff factor-adjusted returns | it does | (a) regression of CAR[c+1, A_q+1] on LFDS with controls; (b) decile long-short FF5+MOM alpha with costs | CI excludes 0; economic value reported with turnover and costs regardless of significance | | |
| 24 | + | |
| 25 | +Primary endpoint for the headline sentence: **H1 at h = 20 on the binary primary target** with log loss. Brier, AUC, | |
| 26 | +PR-AUC, calibration slope/intercept, MAE/RMSE/OOS-R² (continuous target) are secondary metrics reported for every model. | |
| 27 | +The headline sentence will read: *"At a 20-trading-day horizon, on a 2012–2026 panel of U.S. listed firms with a strictly | |
| 28 | +point-in-time information set, one-minute OHLCV features change the out-of-sample log loss of forecasts of an adverse | |
| 29 | +operating-income surprise by [Δ, 95 % CI] relative to a daily-market-and-accounting benchmark; the gain is | |
| 30 | +[consistent/inconsistent] with [mechanisms] and has [measured/absent] economic value."* | |
| 31 | + | |
| 32 | +## 3. Sample | |
| 33 | + | |
| 34 | +Universe: all FinData fundamentals entities (5,451; survivor-conditioned, see LIMITATIONS) with ≥ 8 quarterly periods | |
| 35 | +and 1-minute bars; firm-quarters 2012-Q1 → 2026-Q2 with reveal dates ≤ 2026-10-30 and admissible cutoffs (PIT_PROTOCOL | |
| 36 | +§3). Sector = SEC SIC division (2-digit), size = log market cap at `c` (raw close × latest available `shares`). | |
| 37 | + | |
| 38 | +Expected sample (to be reported, not assumed): ~4,300 firms × ~40 quarters ≈ 150–170 k firm-quarters before exclusions. | |
| 39 | + | |
| 40 | +## 4. Variables | |
| 41 | + | |
| 42 | +**Targets** — `configs/research.yaml`. Primary binary `D_q = 1[S_q ≤ −0.5 % of assets]`, primary continuous `S_q` | |
| 43 | +(winsorised at 1/99 % on the training period). | |
| 44 | + | |
| 45 | +**PIT fundamentals (B1)** — latest available quarter at `c`: `rev` growth (yoy), gross margin `gp/rev`, operating margin | |
| 46 | +`op_inc/rev`, `roa`, `net_inc/assets_tot`, `cash_op/assets_tot`, accruals `(net_inc − cash_op)/assets_tot`, | |
| 47 | +`cash/assets_tot`, leverage `debt/assets_tot`, interest coverage `intr_cover` (clipped), current ratio `assets_c/liab_c`, | |
| 48 | +net issuance (Δ`shares` yoy), capex/assets, R&D/rev, `S_{j*}` (last surprise), last-four-quarter surprise mean, loss | |
| 49 | +indicator, log assets, log market cap, information age `c − A_{j*}`, fiscal-quarter dummies, sector dummies, year-quarter | |
| 50 | +calendar dummies (in-sample only; for prediction, calendar effects are replaced by the trailing cross-sectional base rate). | |
| 51 | + | |
| 52 | +**Daily OHLCV (B2)** — from 1m regular-session bars aggregated to daily: 20- and 60-day cumulative split-adjusted returns, | |
| 53 | +daily return volatility (20d), Parkinson range volatility, skewness and kurtosis of daily returns (60d), daily volume | |
| 54 | +relative to 60-day median (20d mean and last 5d), Amihud illiquidity (60d), share of zero-return days, overnight | |
| 55 | +(close-to-open) vs intraday (open-to-close) return decomposition (20d sums), max drawdown (60d), beta and residual | |
| 56 | +volatility against SPY (60d daily). | |
| 57 | + | |
| 58 | +**Intraday families (B3 = 30m, B4 = 5m, B5 = 1m)** — the *same* families computed on bars of the given length, | |
| 59 | +each aggregated per session then averaged (or summed) over the 20-day window: | |
| 60 | +realized variance (RV), bipower variation (BPV), relative jump `max(0, RV − BPV)/RV`, negative semivariance share | |
| 61 | +`RS⁻/RV`, signed-jump variation `(RS⁺ − RS⁻)/RV`, realized skewness and kurtosis, maximum absolute bar-return share | |
| 62 | +of RV (concentration), first-order autocorrelation of bar returns, variance ratio `RV_freq / RV_daily`, intraday drift | |
| 63 | +`(close − open)/open`, time-of-day volume profile (share of volume in the first and last 30 minutes), volume dispersion | |
| 64 | +across bars (Gini), down-bar volume share (volume on bars with negative return / total — *a bar statistic, not order | |
| 65 | +flow*), volume surprise relative to the firm's 60-day time-of-day seasonal profile, fraction of bars present (activity), | |
| 66 | +RV on market-down bars vs market-up bars (asymmetry vs SPY), realized beta to SPY at that frequency, vol-of-vol | |
| 67 | +(dispersion of daily RV within the window). At 30m the within-day counts are small (13 bars), so skewness/kurtosis and | |
| 68 | +autocorrelation are computed on the pooled window. | |
| 69 | + | |
| 70 | +Note on frequency comparison: B3/B4/B5 have the **same number of feature columns** (one per family); dimensionality does | |
| 71 | +not grow with frequency. A supplementary experiment (Phase 4) concatenates all three to test whether 1m dominates when | |
| 72 | +coarser aggregates are also present. | |
| 73 | + | |
| 74 | +## 5. Models and inference | |
| 75 | + | |
| 76 | +Baseline ladder B0–B6 (`configs/models.yaml`). Logistic ridge for B1–B5 with `C` tuned on the validation period only; | |
| 77 | +LightGBM with an identical random-search budget (24 trials, early stopping on validation) for every information set. | |
| 78 | +B7/B8 (temporal neural, text) are **not run** (owner instruction to minimise AI use; declared in DECISIONS D-006). | |
| 79 | + | |
| 80 | +Panel regression for interpretability and H3: | |
| 81 | +`S_{i,q} = α_i + δ_{s(i),t(q)} + β·LFDS_{i,c} + γ'X_{i,c} + ε_{i,q}`, where `LFDS` is the B5 out-of-fold predicted | |
| 82 | +probability (walk-forward, so it is never fitted on the same quarter), with firm fixed effects and sector×year-quarter | |
| 83 | +effects, two-way clustered SEs (firm, calendar quarter). H3 interactions use ex-ante terciles formed at `c`. | |
| 84 | + | |
| 85 | +Comparisons: Diebold–Mariano / Giacomini–White on per-quarter mean loss differentials with HAC; stationary block | |
| 86 | +bootstrap over calendar quarters for paired CIs; Holm within each hypothesis family; model confidence set over the ladder | |
| 87 | +as a supplementary exhibit. | |
| 88 | + | |
| 89 | +## 6. Validation protocol | |
| 90 | + | |
| 91 | +- Chronological: train 2012-01 → 2018-12, validation 2019-01 → 2020-12, test 2021-01 → 2026-10 (never opened until Phase 3 | |
| 92 | + gate). Purge: training observations whose `A_q` falls after the validation start minus one quarter are dropped | |
| 93 | + (embargo = 1 calendar quarter) and likewise at the test boundary. | |
| 94 | +- Walk-forward: expanding window, annual refits, first test year 2016, 1-quarter embargo; this provides the out-of-fold | |
| 95 | + LFDS for the panel regressions and the H4/H5 exercises. | |
| 96 | +- Firm hold-out: 20 % of firms (hash of entity id) never seen in training, evaluated separately from the time hold-out. | |
| 97 | +- Train-only transforms: winsorisation bounds, medians for imputation, standardisation, sector-time demeaning, tercile | |
| 98 | + cut-points — all computed on the training rows of the current fold. | |
| 99 | + | |
| 100 | +## 7. Power | |
| 101 | + | |
| 102 | +With ≈ 40 k test firm-quarters, base rate ≈ 20–30 %, and quarter-level clustering (23 test quarters), the minimum | |
| 103 | +detectable paired Δ log loss is roughly 0.002–0.004 (to be computed from the validation-period variance of the loss | |
| 104 | +differential and reported in `artifacts/evaluation/power.json` before the test set is opened). | |
| 105 | + | |
| 106 | +## 8. What would make us abandon or pivot | |
| 107 | + | |
| 108 | +If, on the validation period, B2 already captures all of B5 (Δ within ±0.001 and CI covering 0), H1 is likely null; we | |
| 109 | +proceed to the test anyway (a precise null is a result) and devote Phase 5 to characterising *where* intraday information | |
| 110 | +does and does not help. If the earnings-release matching proves unreliable (> 10 % irregular lags after cleaning), the | |
| 111 | +event-time design is demoted and the fixed-calendar design becomes primary (amendment required). | |
added
literature/RELATED_WORK_MATRIX.csv
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +key,authors,year,journal,doi_or_url,data_source,frequency,sample,research_question,identification,findings,point_of_distinction_for_WP39 | |
| 2 | +gregoire2022,Grégoire & Martineau,2022,Journal of Accounting Research,10.1111/1475-679X.12394,TAQ quotes and trades,intraday (quotes/trades),US earnings announcements,How is earnings news incorporated into prices?,event-window decomposition of price discovery,most price discovery for after-hours announcements occurs through quote updates with little trading,"WP39 uses only pre-release bars, forecasts the accounting outcome (not the return), never claims quote/trade mechanisms" | |
| 3 | +chen2022,Chen; Cho; Dou; Lev,2022,Journal of Accounting Research,10.1111/1475-679X.12429,XBRL detailed financial statement items,annual,US firms,Can ML on detailed financials predict earnings changes?,OOS prediction + return tests,boosted trees beat parsimonious models; slow pricing,"WP39 holds accounting inputs fixed and measures the marginal value of market data by sampling frequency" | |
| 4 | +zhang2023volatility,Zhang; Zhang; Cucuringu; Qian,2024,Journal of Financial Econometrics,10.1093/jjfinec/nbad005,intraday equity returns,intraday,US stocks,Does pooling intraday commonality improve ML volatility forecasts?,OOS forecasting,commonality improves volatility forecasts,"WP39 targets a signed accounting quantity, includes volatility families as controls, uses a signed vs absolute placebo" | |
| 5 | +beaver1980,Beaver; Lambert; Morse,1980,Journal of Accounting and Economics,10.1016/0165-4101(80)90013-0,CRSP/Compustat,annual,US,Do prices contain information about future earnings?,regression of earnings changes on returns,prices lead earnings,"establishes the daily/annual lead; WP39 asks what remains at minute resolution" | |
| 6 | +kothari1992,Kothari & Sloan,1992,Journal of Accounting and Economics,10.1016/0165-4101(92)90016-U,CRSP/Compustat,annual,US,Information in prices about future earnings (ERC),leading returns,ERCs rise with leading returns,"same" | |
| 7 | +collins1994,Collins; Kothari; Shanken; Sloan,1994,Journal of Accounting and Economics,10.1016/0165-4101(94)90024-8,CRSP/Compustat,annual,US,Why is the contemporaneous return-earnings association low?,lack of timeliness vs noise,prices anticipate earnings several years ahead,"same" | |
| 8 | +foster1977,Foster,1977,The Accounting Review,10.2308/tar-4493420,quarterly earnings,quarterly,US,Time-series properties of quarterly earnings,forecast evaluation,seasonal random walk with drift forecasts well,"WP39's expectation model for the surprise" | |
| 9 | +bernard1989,Bernard & Thomas,1989,Journal of Accounting Research,10.2307/2491062,CRSP/Compustat,daily,US,Post-earnings-announcement drift,event study,drift after SUE,"WP39 studies pre-release information, not post-announcement drift" | |
| 10 | +ou1989,Ou & Penman,1989,Journal of Accounting and Economics,10.1016/0165-4101(89)90017-7,Compustat,annual,US,Can financial statement analysis predict earnings and returns?,logit on ratios,yes,"WP39 B1 baseline follows this tradition" | |
| 11 | +sloan1996,Sloan,1996,The Accounting Review,10.2308/tar-9608042309,Compustat/CRSP,annual,US,Do prices reflect accruals information?,hedge portfolio,accrual anomaly,"accruals are a B1 control" | |
| 12 | +christophe2004,Christophe; Ferri; Angel,2004,Journal of Finance,10.1111/j.1540-6261.2004.00681.x,Nasdaq short sales,daily,US,Do short sellers anticipate earnings news?,pre-announcement short selling,yes,"trader-type data unavailable to WP39; WP39 tests anonymous OHLCV" | |
| 13 | +berkman2009,Berkman; Dimitrov; Jain; Koch; Tice,2009,Journal of Financial Economics,10.1016/j.jfineco.2008.04.009,CRSP/IBES,daily,US,Differences of opinion and returns around announcements,event study,prices fall around announcements for high-disagreement stocks,"announcement-window return study" | |
| 14 | +kaniel2012,Kaniel; Liu; Saar; Titman,2012,Journal of Finance,10.1111/j.1540-6261.2012.01727.x,NYSE CAUD individual trades,daily,US,Do individual investors trade informatively before earnings?,pre-event trading,yes partly,"trader-type data" | |
| 15 | +hendershott2015,Hendershott; Livdan; Schürhoff,2015,Journal of Financial Economics,10.1016/j.jfineco.2015.03.007,institutional order flow,daily,US,Are institutions informed about news?,order flow before news,yes,"order flow unavailable; WP39 never labels bar statistics as order flow" | |
| 16 | +patton2012,Patton & Verardo,2012,Review of Financial Studies,10.1093/rfs/hhs073,TAQ,intraday,US,Does beta move with news?,announcement-day realized beta,beta rises on announcement days,"announcement-day study" | |
| 17 | +bollerslev2018volume,Bollerslev; Li; Xue,2018,Review of Economic Studies,10.1093/restud/rdy003,TAQ,intraday,US,Volume-volatility relation around public news,structural model,announcement effects on volume-volatility elasticity,"announcement-window study" | |
| 18 | +weller2018,Weller,2018,Review of Financial Studies,10.1093/rfs/hhx137,TAQ + earnings,daily/intraday,US,Does algorithmic trading reduce information acquisition?,AT and pre-announcement price informativeness,AT reduces pre-announcement price informativeness,"closest mechanism paper: WP39 reports pre-release information by era as a mechanism exhibit" | |
| 19 | +lee2012,Lee,2012,Review of Financial Studies,10.1093/rfs/hhr084,TAQ,intraday,US,Jumps and information flow,jump tests around news,jumps cluster at news,"jump proxies are WP39 features" | |
| 20 | +easton2024,Easton; Kapons; Monahan; Schütt; Weisbrod,2024,The Accounting Review,10.2308/tar-2021-0478,Compustat,annual,US,Forecast earnings with k-NN,OOS forecasting,kNN competitive with analysts,"accounting inputs only" | |
| 21 | +cao2024,Cao; Jiang; Wang; Yang,2024,Journal of Financial Economics,10.1016/j.jfineco.2024.103910,analyst + ML,monthly,US,Man vs machine in stock analysis,OOS forecasts,AI complements analysts,"no intraday data" | |
| 22 | +gu2020,Gu; Kelly; Xiu,2020,Review of Financial Studies,10.1093/rfs/hhaa009,CRSP/Compustat characteristics,monthly,US,ML for return prediction,OOS R2,trees and NNs best,"methodological reference for equal-budget comparisons" | |
| 23 | +giacomini2006,Giacomini & White,2006,Econometrica,10.1111/j.1468-0262.2006.00718.x,—,—,—,Conditional predictive ability tests,—,—,"inference for H1/H2" | |
| 24 | +liu2015,Liu; Patton; Sheppard,2015,Journal of Econometrics,10.1016/j.jeconom.2015.02.008,TAQ,intraday,multi-asset,Does anything beat 5-minute RV?,forecast comparison,5-min RV hard to beat,"frequency comparison for variance measurement; WP39 asks the analogous question for fundamentals" | |
| 25 | +boucher2026wp15,Boucher,2026,UQO Working Paper 15,—,FirstRate Data 1-minute bars,1-minute,US 1992-2026,Intraday shock propagation,event studies,shocks propagate narrowly,"same vendor; different question; nothing reused" | |
added
literature/references.bib
+1021 −0
@@ -0,0 +1,1021 @@ | ||
| 1 | +% WP39 UQO bibliography. Journal metadata verified on Crossref 2026-11-02 (artifacts/logs/crossref_batch*.json). | |
| 2 | + | |
| 3 | +@article{gregoire2022, | |
| 4 | + author = {Grégoire, Vincent and Martineau, Charles}, | |
| 5 | + title = {{How is Earnings News Transmitted to Stock Prices?}}, | |
| 6 | + journal = {Journal of Accounting Research}, | |
| 7 | + year = {2022}, | |
| 8 | + volume = {60}, | |
| 9 | + number = {1}, | |
| 10 | + pages = {261--297}, | |
| 11 | + doi = {10.1111/1475-679x.12394} | |
| 12 | +} | |
| 13 | + | |
| 14 | +@article{zhang2023volatility, | |
| 15 | + author = {Zhang, Chao and Zhang, Yihuang and Cucuringu, Mihai and Qian, Zhongmin}, | |
| 16 | + title = {{Volatility Forecasting with Machine Learning and Intraday Commonality}}, | |
| 17 | + journal = {Journal of Financial Econometrics}, | |
| 18 | + year = {2024}, | |
| 19 | + volume = {22}, | |
| 20 | + number = {2}, | |
| 21 | + pages = {492--530}, | |
| 22 | + doi = {10.1093/jjfinec/nbad005} | |
| 23 | +} | |
| 24 | + | |
| 25 | +@article{ball1968, | |
| 26 | + author = {Ball, Ray and Brown, Philip}, | |
| 27 | + title = {{An Empirical Evaluation of Accounting Income Numbers}}, | |
| 28 | + journal = {Journal of Accounting Research}, | |
| 29 | + year = {1968}, | |
| 30 | + volume = {6}, | |
| 31 | + number = {2}, | |
| 32 | + pages = {159}, | |
| 33 | + doi = {10.2307/2490232} | |
| 34 | +} | |
| 35 | + | |
| 36 | +@article{bernard1989, | |
| 37 | + author = {Bernard, Victor L. and Thomas, Jacob K.}, | |
| 38 | + title = {{Post-Earnings-Announcement Drift: Delayed Price Response or Risk Premium?}}, | |
| 39 | + journal = {Journal of Accounting Research}, | |
| 40 | + year = {1989}, | |
| 41 | + volume = {27}, | |
| 42 | + pages = {1}, | |
| 43 | + doi = {10.2307/2491062} | |
| 44 | +} | |
| 45 | + | |
| 46 | +@article{bernard1990, | |
| 47 | + author = {Bernard, Victor L. and Thomas, Jacob K.}, | |
| 48 | + title = {{Evidence that stock prices do not fully reflect the implications of current earnings for future earnings}}, | |
| 49 | + journal = {Journal of Accounting and Economics}, | |
| 50 | + year = {1990}, | |
| 51 | + volume = {13}, | |
| 52 | + number = {4}, | |
| 53 | + pages = {305--340}, | |
| 54 | + doi = {10.1016/0165-4101(90)90008-r} | |
| 55 | +} | |
| 56 | + | |
| 57 | +@article{sloan1996, | |
| 58 | + author = {Sloan, Richard G.}, | |
| 59 | + title = {{Do Stock Prices Fully Reflect Information in Accruals and Cash Flows About Future Earnings?}}, | |
| 60 | + journal = {The Accounting Review}, | |
| 61 | + year = {1996}, | |
| 62 | + volume = {71}, | |
| 63 | + number = {3}, | |
| 64 | + pages = {289--315}, | |
| 65 | + doi = {10.2308/tar-9608042309} | |
| 66 | +} | |
| 67 | + | |
| 68 | +@article{ou1989, | |
| 69 | + author = {Ou, Jane A. and Penman, Stephen H.}, | |
| 70 | + title = {{Financial statement analysis and the prediction of stock returns}}, | |
| 71 | + journal = {Journal of Accounting and Economics}, | |
| 72 | + year = {1989}, | |
| 73 | + volume = {11}, | |
| 74 | + number = {4}, | |
| 75 | + pages = {295--329}, | |
| 76 | + doi = {10.1016/0165-4101(89)90017-7} | |
| 77 | +} | |
| 78 | + | |
| 79 | +@article{kyle1985, | |
| 80 | + author = {Kyle, Albert S.}, | |
| 81 | + title = {{Continuous Auctions and Insider Trading}}, | |
| 82 | + journal = {Econometrica}, | |
| 83 | + year = {1985}, | |
| 84 | + volume = {53}, | |
| 85 | + number = {6}, | |
| 86 | + pages = {1315}, | |
| 87 | + doi = {10.2307/1913210} | |
| 88 | +} | |
| 89 | + | |
| 90 | +@article{easley1987, | |
| 91 | + author = {Easley, David and O'Hara, Maureen}, | |
| 92 | + title = {{Price, trade size, and information in securities markets}}, | |
| 93 | + journal = {Journal of Financial Economics}, | |
| 94 | + year = {1987}, | |
| 95 | + volume = {19}, | |
| 96 | + number = {1}, | |
| 97 | + pages = {69--90}, | |
| 98 | + doi = {10.1016/0304-405x(87)90029-8} | |
| 99 | +} | |
| 100 | + | |
| 101 | +@article{andersen2003, | |
| 102 | + author = {Andersen, Torben G. and Bollerslev, Tim and Diebold, Francis X. and Labys, Paul}, | |
| 103 | + title = {{Modeling and Forecasting Realized Volatility}}, | |
| 104 | + journal = {Econometrica}, | |
| 105 | + year = {2003}, | |
| 106 | + volume = {71}, | |
| 107 | + number = {2}, | |
| 108 | + pages = {579--625}, | |
| 109 | + doi = {10.1111/1468-0262.00418} | |
| 110 | +} | |
| 111 | + | |
| 112 | +@article{barndorff2004, | |
| 113 | + author = {Barndorff-Nielsen, O. E.}, | |
| 114 | + title = {{Power and Bipower Variation with Stochastic Volatility and Jumps}}, | |
| 115 | + journal = {Journal of Financial Econometrics}, | |
| 116 | + year = {2004}, | |
| 117 | + volume = {2}, | |
| 118 | + number = {1}, | |
| 119 | + pages = {1--37}, | |
| 120 | + doi = {10.1093/jjfinec/nbh001} | |
| 121 | +} | |
| 122 | + | |
| 123 | +@article{gu2020, | |
| 124 | + author = {Gu, Shihao and Kelly, Bryan and Xiu, Dacheng}, | |
| 125 | + title = {{Empirical Asset Pricing via Machine Learning}}, | |
| 126 | + journal = {The Review of Financial Studies}, | |
| 127 | + year = {2020}, | |
| 128 | + volume = {33}, | |
| 129 | + number = {5}, | |
| 130 | + pages = {2223--2273}, | |
| 131 | + doi = {10.1093/rfs/hhaa009} | |
| 132 | +} | |
| 133 | + | |
| 134 | +@article{giacomini2006, | |
| 135 | + author = {Giacomini, Raffaella and White, Halbert}, | |
| 136 | + title = {{Tests of Conditional Predictive Ability}}, | |
| 137 | + journal = {Econometrica}, | |
| 138 | + year = {2006}, | |
| 139 | + volume = {74}, | |
| 140 | + number = {6}, | |
| 141 | + pages = {1545--1578}, | |
| 142 | + doi = {10.1111/j.1468-0262.2006.00718.x} | |
| 143 | +} | |
| 144 | + | |
| 145 | +@article{diebold1995, | |
| 146 | + author = {Diebold, Francis X. and Mariano, Roberto S.}, | |
| 147 | + title = {{Comparing Predictive Accuracy}}, | |
| 148 | + journal = {Journal of Business \& Economic Statistics}, | |
| 149 | + year = {1995}, | |
| 150 | + volume = {13}, | |
| 151 | + number = {3}, | |
| 152 | + pages = {253--263}, | |
| 153 | + doi = {10.1080/07350015.1995.10524599} | |
| 154 | +} | |
| 155 | + | |
| 156 | +@article{hansen2005, | |
| 157 | + author = {Hansen, Peter Reinhard}, | |
| 158 | + title = {{A Test for Superior Predictive Ability}}, | |
| 159 | + journal = {Journal of Business \& Economic Statistics}, | |
| 160 | + year = {2005}, | |
| 161 | + volume = {23}, | |
| 162 | + number = {4}, | |
| 163 | + pages = {365--380}, | |
| 164 | + doi = {10.1198/073500105000000063} | |
| 165 | +} | |
| 166 | + | |
| 167 | +@article{fama2015, | |
| 168 | + author = {Fama, Eugene F. and French, Kenneth R.}, | |
| 169 | + title = {{A five-factor asset pricing model}}, | |
| 170 | + journal = {Journal of Financial Economics}, | |
| 171 | + year = {2015}, | |
| 172 | + volume = {116}, | |
| 173 | + number = {1}, | |
| 174 | + pages = {1--22}, | |
| 175 | + doi = {10.1016/j.jfineco.2014.10.010} | |
| 176 | +} | |
| 177 | + | |
| 178 | +@article{bollerslev2020good, | |
| 179 | + author = {Bollerslev, Tim and Li, Sophia Zhengzi and Zhao, Bingzhi}, | |
| 180 | + title = {{Good Volatility, Bad Volatility, and the Cross Section of Stock Returns}}, | |
| 181 | + journal = {Journal of Financial and Quantitative Analysis}, | |
| 182 | + year = {2020}, | |
| 183 | + volume = {55}, | |
| 184 | + number = {3}, | |
| 185 | + pages = {751--781}, | |
| 186 | + doi = {10.1017/s0022109019000097} | |
| 187 | +} | |
| 188 | + | |
| 189 | +@article{patton2015, | |
| 190 | + author = {Patton, Andrew J. and Sheppard, Kevin}, | |
| 191 | + title = {{Good Volatility, Bad Volatility: Signed Jumps and The Persistence of Volatility}}, | |
| 192 | + journal = {Review of Economics and Statistics}, | |
| 193 | + year = {2015}, | |
| 194 | + volume = {97}, | |
| 195 | + number = {3}, | |
| 196 | + pages = {683--697}, | |
| 197 | + doi = {10.1162/rest_a_00503} | |
| 198 | +} | |
| 199 | + | |
| 200 | +@article{chen2022, | |
| 201 | + author = {CHEN, XI and CHO, YANG HA (TONY) and DOU, YIWEI and LEV, BARUCH}, | |
| 202 | + title = {{Predicting Future Earnings Changes Using Machine Learning and Detailed Financial Data}}, | |
| 203 | + journal = {Journal of Accounting Research}, | |
| 204 | + year = {2022}, | |
| 205 | + volume = {60}, | |
| 206 | + number = {2}, | |
| 207 | + pages = {467--515}, | |
| 208 | + doi = {10.1111/1475-679x.12429} | |
| 209 | +} | |
| 210 | + | |
| 211 | +@article{piotroski2000, | |
| 212 | + author = {Piotroski, Joseph D.}, | |
| 213 | + title = {{Value Investing: The Use of Historical Financial Statement Information to Separate Winners from Losers}}, | |
| 214 | + journal = {Journal of Accounting Research}, | |
| 215 | + year = {2000}, | |
| 216 | + volume = {38}, | |
| 217 | + pages = {1}, | |
| 218 | + doi = {10.2307/2672906} | |
| 219 | +} | |
| 220 | + | |
| 221 | +@article{hirshleifer2009, | |
| 222 | + author = {HIRSHLEIFER, DAVID and LIM, SONYA SEONGYEON and TEOH, SIEW HONG}, | |
| 223 | + title = {{Driven to Distraction: Extraneous Events and Underreaction to Earnings News}}, | |
| 224 | + journal = {The Journal of Finance}, | |
| 225 | + year = {2009}, | |
| 226 | + volume = {64}, | |
| 227 | + number = {5}, | |
| 228 | + pages = {2289--2325}, | |
| 229 | + doi = {10.1111/j.1540-6261.2009.01501.x} | |
| 230 | +} | |
| 231 | + | |
| 232 | +@article{dellavigna2009, | |
| 233 | + author = {DELLAVIGNA, STEFANO and POLLET, JOSHUA M.}, | |
| 234 | + title = {{Investor Inattention and Friday Earnings Announcements}}, | |
| 235 | + journal = {The Journal of Finance}, | |
| 236 | + year = {2009}, | |
| 237 | + volume = {64}, | |
| 238 | + number = {2}, | |
| 239 | + pages = {709--749}, | |
| 240 | + doi = {10.1111/j.1540-6261.2009.01447.x} | |
| 241 | +} | |
| 242 | + | |
| 243 | +@article{loughran2011, | |
| 244 | + author = {LOUGHRAN, TIM and MCDONALD, BILL}, | |
| 245 | + title = {{When Is a Liability Not a Liability? Textual Analysis, Dictionaries, and 10-Ks}}, | |
| 246 | + journal = {The Journal of Finance}, | |
| 247 | + year = {2011}, | |
| 248 | + volume = {66}, | |
| 249 | + number = {1}, | |
| 250 | + pages = {35--65}, | |
| 251 | + doi = {10.1111/j.1540-6261.2010.01625.x} | |
| 252 | +} | |
| 253 | + | |
| 254 | +@article{bajgrowicz2016, | |
| 255 | + author = {Bajgrowicz, Pierre and Scaillet, Olivier and Treccani, Adrien}, | |
| 256 | + title = {{Jumps in High-Frequency Data: Spurious Detections, Dynamics, and News}}, | |
| 257 | + journal = {Management Science}, | |
| 258 | + year = {2016}, | |
| 259 | + volume = {62}, | |
| 260 | + number = {8}, | |
| 261 | + pages = {2198--2217}, | |
| 262 | + doi = {10.1287/mnsc.2015.2234} | |
| 263 | +} | |
| 264 | + | |
| 265 | +@article{lee2008, | |
| 266 | + author = {Lee, Suzanne S. and Mykland, Per A.}, | |
| 267 | + title = {{Jumps in Financial Markets: A New Nonparametric Test and Jump Dynamics}}, | |
| 268 | + journal = {Review of Financial Studies}, | |
| 269 | + year = {2008}, | |
| 270 | + volume = {21}, | |
| 271 | + number = {6}, | |
| 272 | + pages = {2535--2563}, | |
| 273 | + doi = {10.1093/rfs/hhm056} | |
| 274 | +} | |
| 275 | + | |
| 276 | +@article{bollerslev2018risk, | |
| 277 | + author = {Bollerslev, Tim and Hood, Benjamin and Huss, John and Pedersen, Lasse Heje}, | |
| 278 | + title = {{Risk Everywhere: Modeling and Managing Volatility}}, | |
| 279 | + journal = {The Review of Financial Studies}, | |
| 280 | + year = {2018}, | |
| 281 | + volume = {31}, | |
| 282 | + number = {7}, | |
| 283 | + pages = {2729--2773}, | |
| 284 | + doi = {10.1093/rfs/hhy041} | |
| 285 | +} | |
| 286 | + | |
| 287 | +@article{hendershott2011, | |
| 288 | + author = {HENDERSHOTT, TERRENCE and JONES, CHARLES M. and MENKVELD, ALBERT J.}, | |
| 289 | + title = {{Does Algorithmic Trading Improve Liquidity?}}, | |
| 290 | + journal = {The Journal of Finance}, | |
| 291 | + year = {2011}, | |
| 292 | + volume = {66}, | |
| 293 | + number = {1}, | |
| 294 | + pages = {1--33}, | |
| 295 | + doi = {10.1111/j.1540-6261.2010.01624.x} | |
| 296 | +} | |
| 297 | + | |
| 298 | +@article{richardson2005, | |
| 299 | + author = {Richardson, Scott A. and Sloan, Richard G. and Soliman, Mark T. and Tuna, İrem}, | |
| 300 | + title = {{Accrual reliability, earnings persistence and stock prices}}, | |
| 301 | + journal = {Journal of Accounting and Economics}, | |
| 302 | + year = {2005}, | |
| 303 | + volume = {39}, | |
| 304 | + number = {3}, | |
| 305 | + pages = {437--485}, | |
| 306 | + doi = {10.1016/j.jacceco.2005.04.005} | |
| 307 | +} | |
| 308 | + | |
| 309 | +@article{beneish1999, | |
| 310 | + author = {Beneish, Messod D.}, | |
| 311 | + title = {{The Detection of Earnings Manipulation}}, | |
| 312 | + journal = {Financial Analysts Journal}, | |
| 313 | + year = {1999}, | |
| 314 | + volume = {55}, | |
| 315 | + number = {5}, | |
| 316 | + pages = {24--36}, | |
| 317 | + doi = {10.2469/faj.v55.n5.2296} | |
| 318 | +} | |
| 319 | + | |
| 320 | +@article{campbell2008, | |
| 321 | + author = {CAMPBELL, JOHN Y. and HILSCHER, JENS and SZILAGYI, JAN}, | |
| 322 | + title = {{In Search of Distress Risk}}, | |
| 323 | + journal = {The Journal of Finance}, | |
| 324 | + year = {2008}, | |
| 325 | + volume = {63}, | |
| 326 | + number = {6}, | |
| 327 | + pages = {2899--2939}, | |
| 328 | + doi = {10.1111/j.1540-6261.2008.01416.x} | |
| 329 | +} | |
| 330 | + | |
| 331 | +@article{fama1973, | |
| 332 | + author = {Fama, Eugene F. and MacBeth, James D.}, | |
| 333 | + title = {{Risk, Return, and Equilibrium: Empirical Tests}}, | |
| 334 | + journal = {Journal of Political Economy}, | |
| 335 | + year = {1973}, | |
| 336 | + volume = {81}, | |
| 337 | + number = {3}, | |
| 338 | + pages = {607--636}, | |
| 339 | + doi = {10.1086/260061} | |
| 340 | +} | |
| 341 | + | |
| 342 | +@article{petersen2009, | |
| 343 | + author = {Petersen, Mitchell A.}, | |
| 344 | + title = {{Estimating Standard Errors in Finance Panel Data Sets: Comparing Approaches}}, | |
| 345 | + journal = {Review of Financial Studies}, | |
| 346 | + year = {2009}, | |
| 347 | + volume = {22}, | |
| 348 | + number = {1}, | |
| 349 | + pages = {435--480}, | |
| 350 | + doi = {10.1093/rfs/hhn053} | |
| 351 | +} | |
| 352 | + | |
| 353 | +@article{cameron2011, | |
| 354 | + author = {Cameron, A. Colin and Gelbach, Jonah B. and Miller, Douglas L.}, | |
| 355 | + title = {{Robust Inference With Multiway Clustering}}, | |
| 356 | + journal = {Journal of Business \& Economic Statistics}, | |
| 357 | + year = {2011}, | |
| 358 | + volume = {29}, | |
| 359 | + number = {2}, | |
| 360 | + pages = {238--249}, | |
| 361 | + doi = {10.1198/jbes.2010.07136} | |
| 362 | +} | |
| 363 | + | |
| 364 | +@article{novymarx2016, | |
| 365 | + author = {Novy-Marx, Robert and Velikov, Mihail}, | |
| 366 | + title = {{A Taxonomy of Anomalies and Their Trading Costs}}, | |
| 367 | + journal = {Review of Financial Studies}, | |
| 368 | + year = {2016}, | |
| 369 | + volume = {29}, | |
| 370 | + number = {1}, | |
| 371 | + pages = {104--147}, | |
| 372 | + doi = {10.1093/rfs/hhv063} | |
| 373 | +} | |
| 374 | + | |
| 375 | +@article{bollerslev2016beta, | |
| 376 | + author = {Bollerslev, Tim and Li, Sophia Zhengzi and Todorov, Viktor}, | |
| 377 | + title = {{Roughing up beta: Continuous versus discontinuous betas and the cross section of expected stock returns}}, | |
| 378 | + journal = {Journal of Financial Economics}, | |
| 379 | + year = {2016}, | |
| 380 | + volume = {120}, | |
| 381 | + number = {3}, | |
| 382 | + pages = {464--490}, | |
| 383 | + doi = {10.1016/j.jfineco.2016.02.001} | |
| 384 | +} | |
| 385 | + | |
| 386 | +@article{lo1988, | |
| 387 | + author = {Lo, Andrew W. and MacKinlay, A. Craig}, | |
| 388 | + title = {{Stock Market Prices Do Not Follow Random Walks: Evidence from a Simple Specification Test}}, | |
| 389 | + journal = {Review of Financial Studies}, | |
| 390 | + year = {1988}, | |
| 391 | + volume = {1}, | |
| 392 | + number = {1}, | |
| 393 | + pages = {41--66}, | |
| 394 | + doi = {10.1093/rfs/1.1.41} | |
| 395 | +} | |
| 396 | + | |
| 397 | +@article{freyberger2020, | |
| 398 | + author = {Freyberger, Joachim and Neuhierl, Andreas and Weber, Michael}, | |
| 399 | + title = {{Dissecting Characteristics Nonparametrically}}, | |
| 400 | + journal = {The Review of Financial Studies}, | |
| 401 | + year = {2020}, | |
| 402 | + volume = {33}, | |
| 403 | + number = {5}, | |
| 404 | + pages = {2326--2377}, | |
| 405 | + doi = {10.1093/rfs/hhz123} | |
| 406 | +} | |
| 407 | + | |
| 408 | +@article{bollerslev2016errors, | |
| 409 | + author = {Bollerslev, Tim and Patton, Andrew J. and Quaedvlieg, Rogier}, | |
| 410 | + title = {{Exploiting the errors: A simple approach for improved volatility forecasting}}, | |
| 411 | + journal = {Journal of Econometrics}, | |
| 412 | + year = {2016}, | |
| 413 | + volume = {192}, | |
| 414 | + number = {1}, | |
| 415 | + pages = {1--18}, | |
| 416 | + doi = {10.1016/j.jeconom.2015.10.007} | |
| 417 | +} | |
| 418 | + | |
| 419 | +@article{liu2015, | |
| 420 | + author = {Liu, Lily Y. and Patton, Andrew J. and Sheppard, Kevin}, | |
| 421 | + title = {{Does anything beat 5-minute RV? A comparison of realized measures across multiple asset classes}}, | |
| 422 | + journal = {Journal of Econometrics}, | |
| 423 | + year = {2015}, | |
| 424 | + volume = {187}, | |
| 425 | + number = {1}, | |
| 426 | + pages = {293--311}, | |
| 427 | + doi = {10.1016/j.jeconom.2015.02.008} | |
| 428 | +} | |
| 429 | + | |
| 430 | +@article{andersen1998, | |
| 431 | + author = {Andersen, Torben G. and Bollerslev, Tim}, | |
| 432 | + title = {{Answering the Skeptics: Yes, Standard Volatility Models do Provide Accurate Forecasts}}, | |
| 433 | + journal = {International Economic Review}, | |
| 434 | + year = {1998}, | |
| 435 | + volume = {39}, | |
| 436 | + number = {4}, | |
| 437 | + pages = {885}, | |
| 438 | + doi = {10.2307/2527343} | |
| 439 | +} | |
| 440 | + | |
| 441 | +@article{aboody2010, | |
| 442 | + author = {Aboody, David and Lehavy, Reuven and Trueman, Brett}, | |
| 443 | + title = {{Limited attention and the earnings announcement returns of past stock market winners}}, | |
| 444 | + journal = {Review of Accounting Studies}, | |
| 445 | + year = {2010}, | |
| 446 | + volume = {15}, | |
| 447 | + number = {2}, | |
| 448 | + pages = {317--344}, | |
| 449 | + doi = {10.1007/s11142-009-9104-9} | |
| 450 | +} | |
| 451 | + | |
| 452 | +@article{kothari2001, | |
| 453 | + author = {Kothari, S.P}, | |
| 454 | + title = {{Capital markets research in accounting}}, | |
| 455 | + journal = {Journal of Accounting and Economics}, | |
| 456 | + year = {2001}, | |
| 457 | + volume = {31}, | |
| 458 | + number = {1-3}, | |
| 459 | + pages = {105--231}, | |
| 460 | + doi = {10.1016/s0165-4101(01)00030-1} | |
| 461 | +} | |
| 462 | + | |
| 463 | +@article{easley2012, | |
| 464 | + author = {Easley, David and López de Prado, Marcos M. and O'Hara, Maureen}, | |
| 465 | + title = {{Flow Toxicity and Liquidity in a High-frequency World}}, | |
| 466 | + journal = {Review of Financial Studies}, | |
| 467 | + year = {2012}, | |
| 468 | + volume = {25}, | |
| 469 | + number = {5}, | |
| 470 | + pages = {1457--1493}, | |
| 471 | + doi = {10.1093/rfs/hhs053} | |
| 472 | +} | |
| 473 | + | |
| 474 | +@article{hou2015, | |
| 475 | + author = {Hou, Kewei and Xue, Chen and Zhang, Lu}, | |
| 476 | + title = {{Digesting Anomalies: An Investment Approach}}, | |
| 477 | + journal = {Review of Financial Studies}, | |
| 478 | + year = {2015}, | |
| 479 | + volume = {28}, | |
| 480 | + number = {3}, | |
| 481 | + pages = {650--705}, | |
| 482 | + doi = {10.1093/rfs/hhu068} | |
| 483 | +} | |
| 484 | + | |
| 485 | +@article{dechow2010, | |
| 486 | + author = {Dechow, Patricia and Ge, Weili and Schrand, Catherine}, | |
| 487 | + title = {{Understanding earnings quality: A review of the proxies, their determinants and their consequences}}, | |
| 488 | + journal = {Journal of Accounting and Economics}, | |
| 489 | + year = {2010}, | |
| 490 | + volume = {50}, | |
| 491 | + number = {2-3}, | |
| 492 | + pages = {344--401}, | |
| 493 | + doi = {10.1016/j.jacceco.2010.09.001} | |
| 494 | +} | |
| 495 | + | |
| 496 | +@article{fama1970, | |
| 497 | + author = {Fama, Eugene F.}, | |
| 498 | + title = {{Efficient Capital Markets: A Review of Theory and Empirical Work}}, | |
| 499 | + journal = {The Journal of Finance}, | |
| 500 | + year = {1970}, | |
| 501 | + volume = {25}, | |
| 502 | + number = {2}, | |
| 503 | + pages = {383}, | |
| 504 | + doi = {10.2307/2325486} | |
| 505 | +} | |
| 506 | + | |
| 507 | +@article{bollerslev2011tails, | |
| 508 | + author = {BOLLERSLEV, TIM and TODOROV, VIKTOR}, | |
| 509 | + title = {{Tails, Fears, and Risk Premia}}, | |
| 510 | + journal = {The Journal of Finance}, | |
| 511 | + year = {2011}, | |
| 512 | + volume = {66}, | |
| 513 | + number = {6}, | |
| 514 | + pages = {2165--2211}, | |
| 515 | + doi = {10.1111/j.1540-6261.2011.01695.x} | |
| 516 | +} | |
| 517 | + | |
| 518 | +@article{hirshleifer2003, | |
| 519 | + author = {Hirshleifer, David and Teoh, Siew Hong}, | |
| 520 | + title = {{Limited attention, information disclosure, and financial reporting}}, | |
| 521 | + journal = {Journal of Accounting and Economics}, | |
| 522 | + year = {2003}, | |
| 523 | + volume = {36}, | |
| 524 | + number = {1-3}, | |
| 525 | + pages = {337--386}, | |
| 526 | + doi = {10.1016/j.jacceco.2003.10.002} | |
| 527 | +} | |
| 528 | + | |
| 529 | +@article{savor2016, | |
| 530 | + author = {SAVOR, PAVEL and WILSON, MUNGO}, | |
| 531 | + title = {{Earnings Announcements and Systematic Risk}}, | |
| 532 | + journal = {The Journal of Finance}, | |
| 533 | + year = {2016}, | |
| 534 | + volume = {71}, | |
| 535 | + number = {1}, | |
| 536 | + pages = {83--138}, | |
| 537 | + doi = {10.1111/jofi.12361} | |
| 538 | +} | |
| 539 | + | |
| 540 | +@article{budish2015, | |
| 541 | + author = {Budish, Eric and Cramton, Peter and Shim, John}, | |
| 542 | + title = {{The High-Frequency Trading Arms Race: Frequent Batch Auctions as a Market Design Response *}}, | |
| 543 | + journal = {The Quarterly Journal of Economics}, | |
| 544 | + year = {2015}, | |
| 545 | + volume = {130}, | |
| 546 | + number = {4}, | |
| 547 | + pages = {1547--1621}, | |
| 548 | + doi = {10.1093/qje/qjv027} | |
| 549 | +} | |
| 550 | + | |
| 551 | +@article{chen2024deep, | |
| 552 | + author = {Chen, Luyang and Pelger, Markus and Zhu, Jason}, | |
| 553 | + title = {{Deep Learning in Asset Pricing}}, | |
| 554 | + journal = {Management Science}, | |
| 555 | + year = {2024}, | |
| 556 | + volume = {70}, | |
| 557 | + number = {2}, | |
| 558 | + pages = {714--750}, | |
| 559 | + doi = {10.1287/mnsc.2023.4695} | |
| 560 | +} | |
| 561 | + | |
| 562 | +@article{kozak2020, | |
| 563 | + author = {Kozak, Serhiy and Nagel, Stefan and Santosh, Shrihari}, | |
| 564 | + title = {{Shrinking the cross-section}}, | |
| 565 | + journal = {Journal of Financial Economics}, | |
| 566 | + year = {2020}, | |
| 567 | + volume = {135}, | |
| 568 | + number = {2}, | |
| 569 | + pages = {271--292}, | |
| 570 | + doi = {10.1016/j.jfineco.2019.06.008} | |
| 571 | +} | |
| 572 | + | |
| 573 | +@article{brogaard2019, | |
| 574 | + author = {BROGAARD, JONATHAN and HENDERSHOTT, TERRENCE and RIORDAN, RYAN}, | |
| 575 | + title = {{Price Discovery without Trading: Evidence from Limit Orders}}, | |
| 576 | + journal = {The Journal of Finance}, | |
| 577 | + year = {2019}, | |
| 578 | + volume = {74}, | |
| 579 | + number = {4}, | |
| 580 | + pages = {1621--1658}, | |
| 581 | + doi = {10.1111/jofi.12769} | |
| 582 | +} | |
| 583 | + | |
| 584 | +@article{ang2006, | |
| 585 | + author = {ANG, ANDREW and HODRICK, ROBERT J. and XING, YUHANG and ZHANG, XIAOYAN}, | |
| 586 | + title = {{The Cross-Section of Volatility and Expected Returns}}, | |
| 587 | + journal = {The Journal of Finance}, | |
| 588 | + year = {2006}, | |
| 589 | + volume = {61}, | |
| 590 | + number = {1}, | |
| 591 | + pages = {259--299}, | |
| 592 | + doi = {10.1111/j.1540-6261.2006.00836.x} | |
| 593 | +} | |
| 594 | + | |
| 595 | +@article{skinner2002, | |
| 596 | + author = {Skinner, Douglas J. and Sloan, Richard G.}, | |
| 597 | + title = {{Earnings Surprises, Growth Expectations, and Stock Returns or Don't Let an Earnings Torpedo Sink Your Portfolio}}, | |
| 598 | + journal = {Review of Accounting Studies}, | |
| 599 | + year = {2002}, | |
| 600 | + volume = {7}, | |
| 601 | + number = {2-3}, | |
| 602 | + pages = {289--312}, | |
| 603 | + doi = {10.1023/a:1020294523516} | |
| 604 | +} | |
| 605 | + | |
| 606 | +@article{levi2015, | |
| 607 | + author = {Levi, Shai and Zhang, Xiao-Jun}, | |
| 608 | + title = {{Asymmetric decrease in liquidity trading before earnings announcements and the announcement return premium}}, | |
| 609 | + journal = {Journal of Financial Economics}, | |
| 610 | + year = {2015}, | |
| 611 | + volume = {118}, | |
| 612 | + number = {2}, | |
| 613 | + pages = {383--398}, | |
| 614 | + doi = {10.1016/j.jfineco.2015.08.003} | |
| 615 | +} | |
| 616 | + | |
| 617 | +@article{campbell1993, | |
| 618 | + author = {Campbell, J. Y. and Grossman, S. J. and Wang, J.}, | |
| 619 | + title = {{Trading Volume and Serial Correlation in Stock Returns}}, | |
| 620 | + journal = {The Quarterly Journal of Economics}, | |
| 621 | + year = {1993}, | |
| 622 | + volume = {108}, | |
| 623 | + number = {4}, | |
| 624 | + pages = {905--939}, | |
| 625 | + doi = {10.2307/2118454} | |
| 626 | +} | |
| 627 | + | |
| 628 | +@article{fang2009, | |
| 629 | + author = {FANG, LILY and PERESS, JOEL}, | |
| 630 | + title = {{Media Coverage and the Cross-section of Stock Returns}}, | |
| 631 | + journal = {The Journal of Finance}, | |
| 632 | + year = {2009}, | |
| 633 | + volume = {64}, | |
| 634 | + number = {5}, | |
| 635 | + pages = {2023--2052}, | |
| 636 | + doi = {10.1111/j.1540-6261.2009.01493.x} | |
| 637 | +} | |
| 638 | + | |
| 639 | +@article{beaver1980, | |
| 640 | + author = {Beaver, William and Lambert, Richard and Morse, Dale}, | |
| 641 | + title = {{The information content of security prices}}, | |
| 642 | + journal = {Journal of Accounting and Economics}, | |
| 643 | + year = {1980}, | |
| 644 | + volume = {2}, | |
| 645 | + number = {1}, | |
| 646 | + pages = {3--28}, | |
| 647 | + doi = {10.1016/0165-4101(80)90013-0} | |
| 648 | +} | |
| 649 | + | |
| 650 | +@article{kothari1992, | |
| 651 | + author = {Kothari, S.P and Sloan, Richard G}, | |
| 652 | + title = {{Information in prices about future earnings}}, | |
| 653 | + journal = {Journal of Accounting and Economics}, | |
| 654 | + year = {1992}, | |
| 655 | + volume = {15}, | |
| 656 | + number = {2-3}, | |
| 657 | + pages = {143--171}, | |
| 658 | + doi = {10.1016/0165-4101(92)90016-u} | |
| 659 | +} | |
| 660 | + | |
| 661 | +@article{collins1994, | |
| 662 | + author = {Collins, Daniel W. and Kothari, S.P. and Shanken, Jay and Sloan, Richard G.}, | |
| 663 | + title = {{Lack of timeliness and noise as explanations for the low contemporaneuos return-earnings association}}, | |
| 664 | + journal = {Journal of Accounting and Economics}, | |
| 665 | + year = {1994}, | |
| 666 | + volume = {18}, | |
| 667 | + number = {3}, | |
| 668 | + pages = {289--324}, | |
| 669 | + doi = {10.1016/0165-4101(94)90024-8} | |
| 670 | +} | |
| 671 | + | |
| 672 | +@article{foster1977, | |
| 673 | + author = {Foster, George}, | |
| 674 | + title = {{Quarterly Accounting Data: Time-Series Properties and Predicative-Ability Results.}}, | |
| 675 | + journal = {The Accounting Review}, | |
| 676 | + year = {1977}, | |
| 677 | + volume = {52}, | |
| 678 | + number = {1}, | |
| 679 | + pages = {1--21}, | |
| 680 | + doi = {10.2308/tar-4493420} | |
| 681 | +} | |
| 682 | + | |
| 683 | +@article{weller2018, | |
| 684 | + author = {Weller, Brian M.}, | |
| 685 | + title = {{Does Algorithmic Trading Reduce Information Acquisition?}}, | |
| 686 | + journal = {The Review of Financial Studies}, | |
| 687 | + year = {2018}, | |
| 688 | + volume = {31}, | |
| 689 | + number = {6}, | |
| 690 | + pages = {2184--2226}, | |
| 691 | + doi = {10.1093/rfs/hhx137} | |
| 692 | +} | |
| 693 | + | |
| 694 | +@article{bollerslev2018volume, | |
| 695 | + author = {Bollerslev, Tim and Li, Jia and Xue, Yuan}, | |
| 696 | + title = {{Volume, Volatility, and Public News Announcements}}, | |
| 697 | + journal = {The Review of Economic Studies}, | |
| 698 | + year = {2018}, | |
| 699 | + volume = {85}, | |
| 700 | + number = {4}, | |
| 701 | + pages = {2005--2041}, | |
| 702 | + doi = {10.1093/restud/rdy003} | |
| 703 | +} | |
| 704 | + | |
| 705 | +@article{patton2012, | |
| 706 | + author = {Patton, Andrew J. and Verardo, Michela}, | |
| 707 | + title = {{Does Beta Move with News? Firm-Specific Information Flows and Learning about Profitability}}, | |
| 708 | + journal = {Review of Financial Studies}, | |
| 709 | + year = {2012}, | |
| 710 | + volume = {25}, | |
| 711 | + number = {9}, | |
| 712 | + pages = {2789--2839}, | |
| 713 | + doi = {10.1093/rfs/hhs073} | |
| 714 | +} | |
| 715 | + | |
| 716 | +@article{kaniel2012, | |
| 717 | + author = {KANIEL, RON and LIU, SHUMING and SAAR, GIDEON and TITMAN, SHERIDAN}, | |
| 718 | + title = {{Individual Investor Trading and Return Patterns around Earnings Announcements}}, | |
| 719 | + journal = {The Journal of Finance}, | |
| 720 | + year = {2012}, | |
| 721 | + volume = {67}, | |
| 722 | + number = {2}, | |
| 723 | + pages = {639--680}, | |
| 724 | + doi = {10.1111/j.1540-6261.2012.01727.x} | |
| 725 | +} | |
| 726 | + | |
| 727 | +@article{christophe2004, | |
| 728 | + author = {Christophe, Stephen E. and Ferri, Michael G. and Angel, James J.}, | |
| 729 | + title = {{Short-Selling Prior to Earnings Announcements}}, | |
| 730 | + journal = {The Journal of Finance}, | |
| 731 | + year = {2004}, | |
| 732 | + volume = {59}, | |
| 733 | + number = {4}, | |
| 734 | + pages = {1845--1876}, | |
| 735 | + doi = {10.1111/j.1540-6261.2004.00681.x} | |
| 736 | +} | |
| 737 | + | |
| 738 | +@article{berkman2009, | |
| 739 | + author = {Berkman, Henk and Dimitrov, Valentin and Jain, Prem C. and Koch, Paul D. and Tice, Sheri}, | |
| 740 | + title = {{Sell on the news: Differences of opinion, short-sales constraints, and returns around earnings announcements☆}}, | |
| 741 | + journal = {Journal of Financial Economics}, | |
| 742 | + year = {2009}, | |
| 743 | + volume = {92}, | |
| 744 | + number = {3}, | |
| 745 | + pages = {376--399}, | |
| 746 | + doi = {10.1016/j.jfineco.2008.04.009} | |
| 747 | +} | |
| 748 | + | |
| 749 | +@article{hendershott2015, | |
| 750 | + author = {Hendershott, Terrence and Livdan, Dmitry and Schürhoff, Norman}, | |
| 751 | + title = {{Are institutions informed about news?}}, | |
| 752 | + journal = {Journal of Financial Economics}, | |
| 753 | + year = {2015}, | |
| 754 | + volume = {117}, | |
| 755 | + number = {2}, | |
| 756 | + pages = {249--287}, | |
| 757 | + doi = {10.1016/j.jfineco.2015.03.007} | |
| 758 | +} | |
| 759 | + | |
| 760 | +@article{lee2012, | |
| 761 | + author = {Lee, Suzanne S.}, | |
| 762 | + title = {{Jumps and Information Flow in Financial Markets}}, | |
| 763 | + journal = {Review of Financial Studies}, | |
| 764 | + year = {2012}, | |
| 765 | + volume = {25}, | |
| 766 | + number = {2}, | |
| 767 | + pages = {439--479}, | |
| 768 | + doi = {10.1093/rfs/hhr084} | |
| 769 | +} | |
| 770 | + | |
| 771 | +@article{cao2024, | |
| 772 | + author = {Cao, Sean and Jiang, Wei and Wang, Junbo and Yang, Baozhong}, | |
| 773 | + title = {{From Man vs. Machine to Man + Machine: The art and AI of stock analyses}}, | |
| 774 | + journal = {Journal of Financial Economics}, | |
| 775 | + year = {2024}, | |
| 776 | + volume = {160}, | |
| 777 | + pages = {103910}, | |
| 778 | + doi = {10.1016/j.jfineco.2024.103910} | |
| 779 | +} | |
| 780 | + | |
| 781 | +@article{bali2023, | |
| 782 | + author = {Bali, Turan G and Beckmeyer, Heiner and Mörke, Mathis and Weigert, Florian}, | |
| 783 | + title = {{Option Return Predictability with Machine Learning and Big Data}}, | |
| 784 | + journal = {The Review of Financial Studies}, | |
| 785 | + year = {2023}, | |
| 786 | + volume = {36}, | |
| 787 | + number = {9}, | |
| 788 | + pages = {3548--3602}, | |
| 789 | + doi = {10.1093/rfs/hhad017} | |
| 790 | +} | |
| 791 | + | |
| 792 | +@article{easton2024, | |
| 793 | + author = {Easton, Peter D. and Kapons, Martin M. and Monahan, Steven J. and Schütt, Harm H. and Weisbrod, Eric H.}, | |
| 794 | + title = {{Forecasting Earnings Using \emph{k}-Nearest Neighbors}}, | |
| 795 | + journal = {The Accounting Review}, | |
| 796 | + year = {2024}, | |
| 797 | + volume = {99}, | |
| 798 | + number = {3}, | |
| 799 | + pages = {115--140}, | |
| 800 | + doi = {10.2308/tar-2021-0478} | |
| 801 | +} | |
| 802 | + | |
| 803 | +@article{so2013, | |
| 804 | + author = {So, Eric C.}, | |
| 805 | + title = {{A new approach to predicting analyst forecast errors: Do investors overweight analyst forecasts?}}, | |
| 806 | + journal = {Journal of Financial Economics}, | |
| 807 | + year = {2013}, | |
| 808 | + volume = {108}, | |
| 809 | + number = {3}, | |
| 810 | + pages = {615--640}, | |
| 811 | + doi = {10.1016/j.jfineco.2013.02.002} | |
| 812 | +} | |
| 813 | + | |
| 814 | +@article{hou2012, | |
| 815 | + author = {Hou, Kewei and van Dijk, Mathijs A. and Zhang, Yinglei}, | |
| 816 | + title = {{The implied cost of capital: A new approach}}, | |
| 817 | + journal = {Journal of Accounting and Economics}, | |
| 818 | + year = {2012}, | |
| 819 | + volume = {53}, | |
| 820 | + number = {3}, | |
| 821 | + pages = {504--526}, | |
| 822 | + doi = {10.1016/j.jacceco.2011.12.001} | |
| 823 | +} | |
| 824 | + | |
| 825 | +@article{hansen2006, | |
| 826 | + author = {Hansen, Peter R and Lunde, Asger}, | |
| 827 | + title = {{Realized Variance and Market Microstructure Noise}}, | |
| 828 | + journal = {Journal of Business \& Economic Statistics}, | |
| 829 | + year = {2006}, | |
| 830 | + volume = {24}, | |
| 831 | + number = {2}, | |
| 832 | + pages = {127--161}, | |
| 833 | + doi = {10.1198/073500106000000071} | |
| 834 | +} | |
| 835 | + | |
| 836 | +@article{barndorff2008, | |
| 837 | + author = {Barndorff-Nielsen, Ole E. and Hansen, Peter Reinhard and Lunde, Asger and Shephard, Neil}, | |
| 838 | + title = {{Designing Realized Kernels to Measure the ex post Variation of Equity Prices in the Presence of Noise}}, | |
| 839 | + journal = {Econometrica}, | |
| 840 | + year = {2008}, | |
| 841 | + volume = {76}, | |
| 842 | + number = {6}, | |
| 843 | + pages = {1481--1536}, | |
| 844 | + doi = {10.3982/ecta6495} | |
| 845 | +} | |
| 846 | + | |
| 847 | +@article{vankervel2019, | |
| 848 | + author = {VAN KERVEL, VINCENT and MENKVELD, ALBERT J.}, | |
| 849 | + title = {{High-Frequency Trading around Large Institutional Orders}}, | |
| 850 | + journal = {The Journal of Finance}, | |
| 851 | + year = {2019}, | |
| 852 | + volume = {74}, | |
| 853 | + number = {3}, | |
| 854 | + pages = {1091--1137}, | |
| 855 | + doi = {10.1111/jofi.12759} | |
| 856 | +} | |
| 857 | + | |
| 858 | +@article{avramov2023, | |
| 859 | + author = {Avramov, Doron and Cheng, Si and Metzker, Lior}, | |
| 860 | + title = {{Machine Learning vs. Economic Restrictions: Evidence from Stock Return Predictability}}, | |
| 861 | + journal = {Management Science}, | |
| 862 | + year = {2023}, | |
| 863 | + volume = {69}, | |
| 864 | + number = {5}, | |
| 865 | + pages = {2587--2619}, | |
| 866 | + doi = {10.1287/mnsc.2022.4449} | |
| 867 | +} | |
| 868 | + | |
| 869 | +@article{chordia2020, | |
| 870 | + author = {Chordia, Tarun and Goyal, Amit and Saretto, Alessio}, | |
| 871 | + title = {{Anomalies and False Rejections}}, | |
| 872 | + journal = {The Review of Financial Studies}, | |
| 873 | + year = {2020}, | |
| 874 | + volume = {33}, | |
| 875 | + number = {5}, | |
| 876 | + pages = {2134--2179}, | |
| 877 | + doi = {10.1093/rfs/hhaa018} | |
| 878 | +} | |
| 879 | + | |
| 880 | +@article{barber2013, | |
| 881 | + author = {Barber, Brad M. and De George, Emmanuel T. and Lehavy, Reuven and Trueman, Brett}, | |
| 882 | + title = {{The earnings announcement premium around the globe}}, | |
| 883 | + journal = {Journal of Financial Economics}, | |
| 884 | + year = {2013}, | |
| 885 | + volume = {108}, | |
| 886 | + number = {1}, | |
| 887 | + pages = {118--138}, | |
| 888 | + doi = {10.1016/j.jfineco.2012.10.006} | |
| 889 | +} | |
| 890 | + | |
| 891 | +@article{politis1994, | |
| 892 | + author = {Politis, Dimitris N. and Romano, Joseph P.}, | |
| 893 | + title = {{The Stationary Bootstrap}}, | |
| 894 | + journal = {Journal of the American Statistical Association}, | |
| 895 | + year = {1994}, | |
| 896 | + volume = {89}, | |
| 897 | + number = {428}, | |
| 898 | + pages = {1303--1313}, | |
| 899 | + doi = {10.1080/01621459.1994.10476870} | |
| 900 | +} | |
| 901 | + | |
| 902 | +@article{hansen2011mcs, | |
| 903 | + author = {Hansen, Peter R. and Lunde, Asger and Nason, James M.}, | |
| 904 | + title = {{The Model Confidence Set}}, | |
| 905 | + journal = {Econometrica}, | |
| 906 | + year = {2011}, | |
| 907 | + volume = {79}, | |
| 908 | + number = {2}, | |
| 909 | + pages = {453--497}, | |
| 910 | + doi = {10.3982/ecta5771} | |
| 911 | +} | |
| 912 | + | |
| 913 | +@article{benjamini1995, | |
| 914 | + author = {Benjamini, Yoav and Hochberg, Yosef}, | |
| 915 | + title = {{Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing}}, | |
| 916 | + journal = {Journal of the Royal Statistical Society Series B: Statistical Methodology}, | |
| 917 | + year = {1995}, | |
| 918 | + volume = {57}, | |
| 919 | + number = {1}, | |
| 920 | + pages = {289--300}, | |
| 921 | + doi = {10.1111/j.2517-6161.1995.tb02031.x} | |
| 922 | +} | |
| 923 | + | |
| 924 | +@article{amihud2002, | |
| 925 | + author = {Amihud, Yakov}, | |
| 926 | + title = {{Illiquidity and stock returns: cross-section and time-series effects}}, | |
| 927 | + journal = {Journal of Financial Markets}, | |
| 928 | + year = {2002}, | |
| 929 | + volume = {5}, | |
| 930 | + number = {1}, | |
| 931 | + pages = {31--56}, | |
| 932 | + doi = {10.1016/s1386-4181(01)00024-6} | |
| 933 | +} | |
| 934 | + | |
| 935 | +@article{fama1993, | |
| 936 | + author = {Fama, Eugene F. and French, Kenneth R.}, | |
| 937 | + title = {{Common risk factors in the returns on stocks and bonds}}, | |
| 938 | + journal = {Journal of Financial Economics}, | |
| 939 | + year = {1993}, | |
| 940 | + volume = {33}, | |
| 941 | + number = {1}, | |
| 942 | + pages = {3--56}, | |
| 943 | + doi = {10.1016/0304-405x(93)90023-5} | |
| 944 | +} | |
| 945 | + | |
| 946 | +@article{grossman1980, | |
| 947 | + author = {Grossman, Sanford J. and Stiglitz, Joseph E.}, | |
| 948 | + title = {{On the Impossibility of Informationally Efficient Markets}}, | |
| 949 | + journal = {American Economic Review}, | |
| 950 | + year = {1980}, | |
| 951 | + volume = {70}, | |
| 952 | + number = {3}, | |
| 953 | + pages = {393--408} | |
| 954 | +} | |
| 955 | +@article{holm1979, | |
| 956 | + author = {Holm, Sture}, | |
| 957 | + title = {{A Simple Sequentially Rejective Multiple Test Procedure}}, | |
| 958 | + journal = {Scandinavian Journal of Statistics}, | |
| 959 | + year = {1979}, | |
| 960 | + volume = {6}, | |
| 961 | + number = {2}, | |
| 962 | + pages = {65--70} | |
| 963 | +} | |
| 964 | +@article{kunsch1989, | |
| 965 | + author = {K{\"u}nsch, Hans R.}, | |
| 966 | + title = {{The Jackknife and the Bootstrap for General Stationary Observations}}, | |
| 967 | + journal = {Annals of Statistics}, | |
| 968 | + year = {1989}, | |
| 969 | + volume = {17}, | |
| 970 | + number = {3}, | |
| 971 | + pages = {1217--1241}, | |
| 972 | + doi = {10.1214/aos/1176347265} | |
| 973 | +} | |
| 974 | +@inproceedings{ke2017lightgbm, | |
| 975 | + author = {Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan}, | |
| 976 | + title = {{LightGBM: A Highly Efficient Gradient Boosting Decision Tree}}, | |
| 977 | + booktitle = {Advances in Neural Information Processing Systems 30}, | |
| 978 | + year = {2017}, | |
| 979 | + pages = {3146--3154} | |
| 980 | +} | |
| 981 | +@book{lopezdeprado2018, | |
| 982 | + author = {L{\'o}pez de Prado, Marcos}, | |
| 983 | + title = {{Advances in Financial Machine Learning}}, | |
| 984 | + publisher = {Wiley}, | |
| 985 | + address = {Hoboken, NJ}, | |
| 986 | + year = {2018} | |
| 987 | +} | |
| 988 | +@techreport{frazzini2018costs, | |
| 989 | + author = {Frazzini, Andrea and Israel, Ronen and Moskowitz, Tobias J.}, | |
| 990 | + title = {{Trading Costs}}, | |
| 991 | + institution = {SSRN Working Paper 3229719}, | |
| 992 | + year = {2018}, | |
| 993 | + doi = {10.2139/ssrn.3229719} | |
| 994 | +} | |
| 995 | +@incollection{barndorff2010semivariance, | |
| 996 | + author = {Barndorff-Nielsen, Ole E. and Kinnebrock, Silja and Shephard, Neil}, | |
| 997 | + title = {{Measuring Downside Risk: Realised Semivariance}}, | |
| 998 | + booktitle = {Volatility and Time Series Econometrics: Essays in Honor of Robert F. Engle}, | |
| 999 | + editor = {Bollerslev, Tim and Russell, Jeffrey and Watson, Mark}, | |
| 1000 | + publisher = {Oxford University Press}, | |
| 1001 | + year = {2010}, | |
| 1002 | + pages = {117--136}, | |
| 1003 | + doi = {10.1093/acprof:oso/9780199549498.003.0007} | |
| 1004 | +} | |
| 1005 | +@article{boucher2026wp15, | |
| 1006 | + author = {Boucher, Simon-Pierre}, | |
| 1007 | + title = {{Faster, Wider, More Fragile? The Evolution of Intraday Shock Propagation in U.S. Equity Markets, 1992--2026}}, | |
| 1008 | + journal = {UQO Working Paper No. 15}, | |
| 1009 | + year = {2026} | |
| 1010 | +} | |
| 1011 | + | |
| 1012 | +@article{harvey2016, | |
| 1013 | + author = {Harvey, Campbell R. and Liu, Yan and Zhu, Heqing}, | |
| 1014 | + title = {{\ldots and the Cross-Section of Expected Returns}}, | |
| 1015 | + journal = {The Review of Financial Studies}, | |
| 1016 | + year = {2016}, | |
| 1017 | + volume = {29}, | |
| 1018 | + number = {1}, | |
| 1019 | + pages = {5--68}, | |
| 1020 | + doi = {10.1093/rfs/hhv059} | |
| 1021 | +} | |
added
paper/.latexmkrc
+5 −0
@@ -0,0 +1,5 @@ | ||
| 1 | +$pdf_mode = 1; | |
| 2 | +$pdflatex = 'pdflatex -interaction=nonstopmode -halt-on-error -synctex=1 %O %S'; | |
| 3 | +$bibtex_use = 2; | |
| 4 | +$clean_ext = 'synctex.gz run.xml bbl bcf fdb_latexmk fls log aux out toc lof lot blg'; | |
| 5 | +@default_files = ('main.tex'); | |
added
paper/Makefile
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +.PHONY: all clean distclean | |
| 2 | + | |
| 3 | +all: | |
| 4 | + latexmk | |
| 5 | + | |
| 6 | +clean: | |
| 7 | + latexmk -c | |
| 8 | + | |
| 9 | +distclean: | |
| 10 | + latexmk -C | |
| 11 | + rm -f *.synctex.gz *.run.xml *.bbl *.bcf | |
added
paper/appendix/appendix.tex
+54 −0
@@ -0,0 +1,54 @@ | ||
| 1 | +\newpage | |
| 2 | +\begin{appendices} | |
| 3 | +\section{Variable definitions and data lineage} | |
| 4 | +\label{app:variables} | |
| 5 | +\begin{table}[H] | |
| 6 | +\centering\scriptsize | |
| 7 | +\caption{Variable definitions and lineage} | |
| 8 | +\begin{tabular}{p{3.4cm}p{8.2cm}p{3.2cm}} | |
| 9 | +\toprule | |
| 10 | +Variable & Definition & Source / step \\ | |
| 11 | +\midrule | |
| 12 | +$A_{i,q}$ & Earnings-release date: first \texttt{earnings\_events.event\_date} in $[E_q, F_q+10\,\mathrm{d}]$ & FinData Postgres extract (01a) \\ | |
| 13 | +$F_{i,q}$, $V_{i,q}$ & SEC filing date; availability $=F+1$ day 00:00 ET & \texttt{/fundamentals} (02) \\ | |
| 14 | +$S_{i,q}$ & $(X_q-\hat X_q)/TA_{q-1}$, $X=$ operating income, $\hat X_q=X_{q-4}+\frac14\sum_{k=0}^3(X_{q-1-k}-X_{q-5-k})$ & 02, \texttt{labels.py} \\ | |
| 15 | +$D_{i,q}$ & $\mathbb 1[S_{i,q}\le-0.005]$ & 02 \\ | |
| 16 | +Secondary targets & net-income surprise; $\Delta$ROA$_{yoy}\le-0.01$; $(CFO_q-CFO_{q-4})/TA\le-0.01$; current ratio $-10\%$ or coverage $<1$ from $\ge1$; $|S|\ge0.005$; EPS $<$ estimate & 02 \\ | |
| 17 | +Intraday families (per bar length $f\in\{1,5,30\}$ min) & RV $=\sum r^2$; BPV $=\frac{\pi}{2}\sum|r_t||r_{t-1}|$; RJ $=\max(0,\mathrm{RV}-\mathrm{BPV})/\mathrm{RV}$; RS$^-$/RV; SJV $=(\mathrm{RS}^+-\mathrm{RS}^-)/\mathrm{RV}$; realized skewness/kurtosis; $\max r^2/\mathrm{RV}$; AC(1); drift $=\ln(C/O)$; opening/closing 30-min volume shares; volume Gini; down-bar volume share; present-bar share; RV share on SPY-down bars; $\beta=\sum r\,r^{SPY}/\sum (r^{SPY})^2$; vol-of-vol; RV/Var(daily) & 03, \texttt{features.py}, \texttt{samples.py} \\ | |
| 18 | +Daily block & 20/60-day log returns, SD, Parkinson, skewness, kurtosis, overnight/intraday sums, volume ratios (20d, 5d vs 60d median), Amihud (60d), zero-return share, log dollar volume, max drawdown (60d), bars per day & 03 \\ | |
| 19 | +Accounting block & gross/operating margin, NI/TA, CFO/TA, accruals, cash/TA, debt/TA, current ratio, capex/TA, R\&D/revenue, coverage, loss dummy, log assets, ROA, revenue growth, net issuance, log market cap, last surprise, mean of last four surprises, trailing adverse rate, information age & 02--04 \\ | |
| 20 | +Returns & split- and dividend-adjusted close-to-close from raw bars and \texttt{corporate\_actions} (ex-date rule) & 03 \\ | |
| 21 | +\bottomrule | |
| 22 | +\end{tabular} | |
| 23 | +\end{table} | |
| 24 | + | |
| 25 | +\section{Leakage battery} | |
| 26 | +\label{app:leakage} | |
| 27 | +Each guard is a unit test in \texttt{tests/}: (L1) future facts, (L2) fiscal-end joins, (L3) post-cutoff bars, | |
| 28 | +(L4) test-period transforms, (L5) DST/timezone, (L6) identifier drift, (L7) splits/dividends, (L8) release-day bars, | |
| 29 | +(L9) overlapping windows across splits, (L10) survivorship report, (L11) ingestion dates, (L12) randomised placebos. | |
| 30 | +The live test \texttt{test\_asof\_equivalence} verifies that the local as-of join reproduces the API's | |
| 31 | +\texttt{as\_of} output on random entities and instants. | |
| 32 | + | |
| 33 | +\section{Additional tables} | |
| 34 | +\label{app:tables} | |
| 35 | +\begin{table}[H]\centering\footnotesize | |
| 36 | +\caption{Continuous target: ridge forecasts of the winsorised surprise (test period)} | |
| 37 | +\input{tables/tab10b_continuous} | |
| 38 | +\end{table} | |
| 39 | +\begin{table}[H]\centering\footnotesize | |
| 40 | +\caption{Transfer to held-out firms (20\% of firms never seen in training)} | |
| 41 | +\input{tables/tab09b_firmholdout} | |
| 42 | +\end{table} | |
| 43 | +\begin{table}[H]\centering\footnotesize | |
| 44 | +\caption{Announcement-window abnormal returns on the ex-ante signal} | |
| 45 | +\input{tables/tab11b_car} | |
| 46 | +\begin{minipage}{\textwidth}\vspace{0.3em}\footnotesize\emph{Notes.} CAR from $c+1$ to $A_q+1$, market-adjusted, winsorised 1/99\%; firm and sector$\times$quarter fixed effects; two-way clustered SEs. $^{*}$, $^{**}$, $^{***}$: 10, 5, 1\%.\end{minipage} | |
| 47 | +\end{table} | |
| 48 | + | |
| 49 | +\section{Reproducibility} | |
| 50 | +\label{app:repro} | |
| 51 | +The pipeline (\texttt{scripts/00}--\texttt{09}) runs on the FinData storage node against the local API; manifests in | |
| 52 | +\texttt{artifacts/manifests} pin dataset versions and hashes; the design freeze hash is recorded before the first | |
| 53 | +model fit. Raw extracts are deleted after feature extraction and are reproducible from the pinned versions. | |
| 54 | +\end{appendices} | |
added
paper/main.tex
+141 −0
@@ -0,0 +1,141 @@ | ||
| 1 | +% ============================================================================ | |
| 2 | +% UQO Working Paper Template — Economics / Finance | |
| 3 | +% ============================================================================ | |
| 4 | +\documentclass[12pt,letterpaper]{article} | |
| 5 | + | |
| 6 | +% --- Encoding & Language --- | |
| 7 | +\usepackage[utf8]{inputenc} | |
| 8 | +\usepackage[T1]{fontenc} | |
| 9 | +\usepackage[english]{babel} | |
| 10 | + | |
| 11 | +% --- Page Layout --- | |
| 12 | +\usepackage[ | |
| 13 | + letterpaper, | |
| 14 | + top=1in, | |
| 15 | + bottom=1in, | |
| 16 | + left=1in, | |
| 17 | + right=1in | |
| 18 | +]{geometry} | |
| 19 | + | |
| 20 | +% --- Typography --- | |
| 21 | +\usepackage{mathptmx} % Times-like font (text + math) | |
| 22 | +\usepackage{microtype} % Improved typographic refinements | |
| 23 | +\usepackage{setspace} | |
| 24 | +\onehalfspacing % 1.5 line spacing | |
| 25 | + | |
| 26 | +% --- Math --- | |
| 27 | +\usepackage{amsmath,amssymb,amsthm} | |
| 28 | + | |
| 29 | +% --- Tables --- | |
| 30 | +\usepackage{booktabs} | |
| 31 | +\usepackage{tabularx} | |
| 32 | +\usepackage{multirow} | |
| 33 | +\usepackage{threeparttable} | |
| 34 | +\usepackage{siunitx} | |
| 35 | +\sisetup{ | |
| 36 | + input-symbols = {()}, | |
| 37 | + table-format = -1.3, | |
| 38 | + parse-numbers = false | |
| 39 | +} | |
| 40 | + | |
| 41 | +% --- Figures --- | |
| 42 | +\usepackage{graphicx} | |
| 43 | +\usepackage{subcaption} | |
| 44 | +\usepackage{float} | |
| 45 | +\usepackage{longtable} | |
| 46 | +\usepackage{adjustbox} | |
| 47 | + | |
| 48 | +% --- Colors & Links --- | |
| 49 | +\usepackage[dvipsnames]{xcolor} | |
| 50 | +\usepackage[ | |
| 51 | + pdftitle={The Invisible Balance Sheet}, | |
| 52 | + pdfauthor={Simon-Pierre Boucher}, | |
| 53 | + colorlinks = true, | |
| 54 | + linkcolor = NavyBlue, | |
| 55 | + citecolor = NavyBlue, | |
| 56 | + urlcolor = NavyBlue, | |
| 57 | + breaklinks = true | |
| 58 | +]{hyperref} | |
| 59 | + | |
| 60 | +% --- Bibliography --- | |
| 61 | +\usepackage[authoryear,round,semicolon]{natbib} | |
| 62 | +\bibliographystyle{aer} | |
| 63 | + | |
| 64 | +% --- Captions --- | |
| 65 | +\usepackage[ | |
| 66 | + font = small, | |
| 67 | + labelfont = bf, | |
| 68 | + labelsep = period, | |
| 69 | + justification = justified, | |
| 70 | + singlelinecheck = false | |
| 71 | +]{caption} | |
| 72 | + | |
| 73 | +% --- Headers & Footers --- | |
| 74 | +\usepackage{fancyhdr} | |
| 75 | +\pagestyle{fancy} | |
| 76 | +\fancyhf{} | |
| 77 | +\renewcommand{\headrulewidth}{0pt} | |
| 78 | +\fancyfoot[C]{\thepage} | |
| 79 | + | |
| 80 | +% --- Section Formatting --- | |
| 81 | +\usepackage{titlesec} | |
| 82 | +\titleformat{\section}{\large\bfseries}{\thesection.}{0.5em}{} | |
| 83 | +\titleformat{\subsection}{\normalsize\bfseries}{\thesubsection.}{0.5em}{} | |
| 84 | +\titleformat{\subsubsection}{\normalsize\itshape}{\thesubsubsection.}{0.5em}{} | |
| 85 | + | |
| 86 | +% --- Appendix Support --- | |
| 87 | +\usepackage[toc,page]{appendix} | |
| 88 | + | |
| 89 | +% --- Misc --- | |
| 90 | +\usepackage{enumitem} | |
| 91 | +\setlist{nosep,leftmargin=*} | |
| 92 | + | |
| 93 | +% ============================================================================ | |
| 94 | +% METADATA | |
| 95 | +% ============================================================================ | |
| 96 | +\input{results_macros} | |
| 97 | +\newcommand{\WPnumber}{39} | |
| 98 | +\newcommand{\WPtitle}{The Invisible Balance Sheet} | |
| 99 | +\newcommand{\WPsubtitle}{Can Intraday Financial Markets Detect Corporate Deterioration Before Accounting Disclosures?} | |
| 100 | +\newcommand{\WPdate}{November 2026} | |
| 101 | +\newcommand{\WPversion}{1.0} | |
| 102 | +\newcommand{\WPabstract}{\input{sections/abstract}} | |
| 103 | +\newcommand{\WPkeywords}{intraday data, sampling frequency, earnings surprises, fundamental analysis, point-in-time data, machine learning, market efficiency} | |
| 104 | +\newcommand{\WPjel}{G14, G12, G17, M41, C53, C58} | |
| 105 | + | |
| 106 | +% --- Author --- | |
| 107 | +\newcommand{\WPauthor}{Simon-Pierre Boucher} | |
| 108 | +\newcommand{\WPaffiliation}{% | |
| 109 | + Département des sciences administratives\\ | |
| 110 | + Université du Québec en Outaouais% | |
| 111 | +} | |
| 112 | +\newcommand{\WPemail}{simon-pierre.boucher@uqo.ca} | |
| 113 | +\newcommand{\WPaddress}{% | |
| 114 | + Gatineau -- Pavillon Alexandre-Taché\\ | |
| 115 | + 283, boulevard Alexandre-Taché\\ | |
| 116 | + Gatineau, Québec, Canada J9A 1L8% | |
| 117 | +} | |
| 118 | + | |
| 119 | +% ============================================================================ | |
| 120 | +% DOCUMENT | |
| 121 | +% ============================================================================ | |
| 122 | +\begin{document} | |
| 123 | + | |
| 124 | +\input{sections/titlepage} | |
| 125 | + | |
| 126 | +\input{sections/introduction} | |
| 127 | +\input{sections/theory_hypotheses} | |
| 128 | +\input{sections/literature} | |
| 129 | +\input{sections/data} | |
| 130 | +\input{sections/methodology} | |
| 131 | +\input{sections/results} | |
| 132 | +\input{sections/mechanisms} | |
| 133 | +\input{sections/robustness} | |
| 134 | +\input{sections/conclusion} | |
| 135 | + | |
| 136 | +\newpage | |
| 137 | +\bibliography{references} | |
| 138 | + | |
| 139 | +\input{appendix/appendix} | |
| 140 | + | |
| 141 | +\end{document} | |
added
paper/references.bib
+1021 −0
@@ -0,0 +1,1021 @@ | ||
| 1 | +% WP39 UQO bibliography. Journal metadata verified on Crossref 2026-11-02 (artifacts/logs/crossref_batch*.json). | |
| 2 | + | |
| 3 | +@article{gregoire2022, | |
| 4 | + author = {Grégoire, Vincent and Martineau, Charles}, | |
| 5 | + title = {{How is Earnings News Transmitted to Stock Prices?}}, | |
| 6 | + journal = {Journal of Accounting Research}, | |
| 7 | + year = {2022}, | |
| 8 | + volume = {60}, | |
| 9 | + number = {1}, | |
| 10 | + pages = {261--297}, | |
| 11 | + doi = {10.1111/1475-679x.12394} | |
| 12 | +} | |
| 13 | + | |
| 14 | +@article{zhang2023volatility, | |
| 15 | + author = {Zhang, Chao and Zhang, Yihuang and Cucuringu, Mihai and Qian, Zhongmin}, | |
| 16 | + title = {{Volatility Forecasting with Machine Learning and Intraday Commonality}}, | |
| 17 | + journal = {Journal of Financial Econometrics}, | |
| 18 | + year = {2024}, | |
| 19 | + volume = {22}, | |
| 20 | + number = {2}, | |
| 21 | + pages = {492--530}, | |
| 22 | + doi = {10.1093/jjfinec/nbad005} | |
| 23 | +} | |
| 24 | + | |
| 25 | +@article{ball1968, | |
| 26 | + author = {Ball, Ray and Brown, Philip}, | |
| 27 | + title = {{An Empirical Evaluation of Accounting Income Numbers}}, | |
| 28 | + journal = {Journal of Accounting Research}, | |
| 29 | + year = {1968}, | |
| 30 | + volume = {6}, | |
| 31 | + number = {2}, | |
| 32 | + pages = {159}, | |
| 33 | + doi = {10.2307/2490232} | |
| 34 | +} | |
| 35 | + | |
| 36 | +@article{bernard1989, | |
| 37 | + author = {Bernard, Victor L. and Thomas, Jacob K.}, | |
| 38 | + title = {{Post-Earnings-Announcement Drift: Delayed Price Response or Risk Premium?}}, | |
| 39 | + journal = {Journal of Accounting Research}, | |
| 40 | + year = {1989}, | |
| 41 | + volume = {27}, | |
| 42 | + pages = {1}, | |
| 43 | + doi = {10.2307/2491062} | |
| 44 | +} | |
| 45 | + | |
| 46 | +@article{bernard1990, | |
| 47 | + author = {Bernard, Victor L. and Thomas, Jacob K.}, | |
| 48 | + title = {{Evidence that stock prices do not fully reflect the implications of current earnings for future earnings}}, | |
| 49 | + journal = {Journal of Accounting and Economics}, | |
| 50 | + year = {1990}, | |
| 51 | + volume = {13}, | |
| 52 | + number = {4}, | |
| 53 | + pages = {305--340}, | |
| 54 | + doi = {10.1016/0165-4101(90)90008-r} | |
| 55 | +} | |
| 56 | + | |
| 57 | +@article{sloan1996, | |
| 58 | + author = {Sloan, Richard G.}, | |
| 59 | + title = {{Do Stock Prices Fully Reflect Information in Accruals and Cash Flows About Future Earnings?}}, | |
| 60 | + journal = {The Accounting Review}, | |
| 61 | + year = {1996}, | |
| 62 | + volume = {71}, | |
| 63 | + number = {3}, | |
| 64 | + pages = {289--315}, | |
| 65 | + doi = {10.2308/tar-9608042309} | |
| 66 | +} | |
| 67 | + | |
| 68 | +@article{ou1989, | |
| 69 | + author = {Ou, Jane A. and Penman, Stephen H.}, | |
| 70 | + title = {{Financial statement analysis and the prediction of stock returns}}, | |
| 71 | + journal = {Journal of Accounting and Economics}, | |
| 72 | + year = {1989}, | |
| 73 | + volume = {11}, | |
| 74 | + number = {4}, | |
| 75 | + pages = {295--329}, | |
| 76 | + doi = {10.1016/0165-4101(89)90017-7} | |
| 77 | +} | |
| 78 | + | |
| 79 | +@article{kyle1985, | |
| 80 | + author = {Kyle, Albert S.}, | |
| 81 | + title = {{Continuous Auctions and Insider Trading}}, | |
| 82 | + journal = {Econometrica}, | |
| 83 | + year = {1985}, | |
| 84 | + volume = {53}, | |
| 85 | + number = {6}, | |
| 86 | + pages = {1315}, | |
| 87 | + doi = {10.2307/1913210} | |
| 88 | +} | |
| 89 | + | |
| 90 | +@article{easley1987, | |
| 91 | + author = {Easley, David and O'Hara, Maureen}, | |
| 92 | + title = {{Price, trade size, and information in securities markets}}, | |
| 93 | + journal = {Journal of Financial Economics}, | |
| 94 | + year = {1987}, | |
| 95 | + volume = {19}, | |
| 96 | + number = {1}, | |
| 97 | + pages = {69--90}, | |
| 98 | + doi = {10.1016/0304-405x(87)90029-8} | |
| 99 | +} | |
| 100 | + | |
| 101 | +@article{andersen2003, | |
| 102 | + author = {Andersen, Torben G. and Bollerslev, Tim and Diebold, Francis X. and Labys, Paul}, | |
| 103 | + title = {{Modeling and Forecasting Realized Volatility}}, | |
| 104 | + journal = {Econometrica}, | |
| 105 | + year = {2003}, | |
| 106 | + volume = {71}, | |
| 107 | + number = {2}, | |
| 108 | + pages = {579--625}, | |
| 109 | + doi = {10.1111/1468-0262.00418} | |
| 110 | +} | |
| 111 | + | |
| 112 | +@article{barndorff2004, | |
| 113 | + author = {Barndorff-Nielsen, O. E.}, | |
| 114 | + title = {{Power and Bipower Variation with Stochastic Volatility and Jumps}}, | |
| 115 | + journal = {Journal of Financial Econometrics}, | |
| 116 | + year = {2004}, | |
| 117 | + volume = {2}, | |
| 118 | + number = {1}, | |
| 119 | + pages = {1--37}, | |
| 120 | + doi = {10.1093/jjfinec/nbh001} | |
| 121 | +} | |
| 122 | + | |
| 123 | +@article{gu2020, | |
| 124 | + author = {Gu, Shihao and Kelly, Bryan and Xiu, Dacheng}, | |
| 125 | + title = {{Empirical Asset Pricing via Machine Learning}}, | |
| 126 | + journal = {The Review of Financial Studies}, | |
| 127 | + year = {2020}, | |
| 128 | + volume = {33}, | |
| 129 | + number = {5}, | |
| 130 | + pages = {2223--2273}, | |
| 131 | + doi = {10.1093/rfs/hhaa009} | |
| 132 | +} | |
| 133 | + | |
| 134 | +@article{giacomini2006, | |
| 135 | + author = {Giacomini, Raffaella and White, Halbert}, | |
| 136 | + title = {{Tests of Conditional Predictive Ability}}, | |
| 137 | + journal = {Econometrica}, | |
| 138 | + year = {2006}, | |
| 139 | + volume = {74}, | |
| 140 | + number = {6}, | |
| 141 | + pages = {1545--1578}, | |
| 142 | + doi = {10.1111/j.1468-0262.2006.00718.x} | |
| 143 | +} | |
| 144 | + | |
| 145 | +@article{diebold1995, | |
| 146 | + author = {Diebold, Francis X. and Mariano, Roberto S.}, | |
| 147 | + title = {{Comparing Predictive Accuracy}}, | |
| 148 | + journal = {Journal of Business \& Economic Statistics}, | |
| 149 | + year = {1995}, | |
| 150 | + volume = {13}, | |
| 151 | + number = {3}, | |
| 152 | + pages = {253--263}, | |
| 153 | + doi = {10.1080/07350015.1995.10524599} | |
| 154 | +} | |
| 155 | + | |
| 156 | +@article{hansen2005, | |
| 157 | + author = {Hansen, Peter Reinhard}, | |
| 158 | + title = {{A Test for Superior Predictive Ability}}, | |
| 159 | + journal = {Journal of Business \& Economic Statistics}, | |
| 160 | + year = {2005}, | |
| 161 | + volume = {23}, | |
| 162 | + number = {4}, | |
| 163 | + pages = {365--380}, | |
| 164 | + doi = {10.1198/073500105000000063} | |
| 165 | +} | |
| 166 | + | |
| 167 | +@article{fama2015, | |
| 168 | + author = {Fama, Eugene F. and French, Kenneth R.}, | |
| 169 | + title = {{A five-factor asset pricing model}}, | |
| 170 | + journal = {Journal of Financial Economics}, | |
| 171 | + year = {2015}, | |
| 172 | + volume = {116}, | |
| 173 | + number = {1}, | |
| 174 | + pages = {1--22}, | |
| 175 | + doi = {10.1016/j.jfineco.2014.10.010} | |
| 176 | +} | |
| 177 | + | |
| 178 | +@article{bollerslev2020good, | |
| 179 | + author = {Bollerslev, Tim and Li, Sophia Zhengzi and Zhao, Bingzhi}, | |
| 180 | + title = {{Good Volatility, Bad Volatility, and the Cross Section of Stock Returns}}, | |
| 181 | + journal = {Journal of Financial and Quantitative Analysis}, | |
| 182 | + year = {2020}, | |
| 183 | + volume = {55}, | |
| 184 | + number = {3}, | |
| 185 | + pages = {751--781}, | |
| 186 | + doi = {10.1017/s0022109019000097} | |
| 187 | +} | |
| 188 | + | |
| 189 | +@article{patton2015, | |
| 190 | + author = {Patton, Andrew J. and Sheppard, Kevin}, | |
| 191 | + title = {{Good Volatility, Bad Volatility: Signed Jumps and The Persistence of Volatility}}, | |
| 192 | + journal = {Review of Economics and Statistics}, | |
| 193 | + year = {2015}, | |
| 194 | + volume = {97}, | |
| 195 | + number = {3}, | |
| 196 | + pages = {683--697}, | |
| 197 | + doi = {10.1162/rest_a_00503} | |
| 198 | +} | |
| 199 | + | |
| 200 | +@article{chen2022, | |
| 201 | + author = {CHEN, XI and CHO, YANG HA (TONY) and DOU, YIWEI and LEV, BARUCH}, | |
| 202 | + title = {{Predicting Future Earnings Changes Using Machine Learning and Detailed Financial Data}}, | |
| 203 | + journal = {Journal of Accounting Research}, | |
| 204 | + year = {2022}, | |
| 205 | + volume = {60}, | |
| 206 | + number = {2}, | |
| 207 | + pages = {467--515}, | |
| 208 | + doi = {10.1111/1475-679x.12429} | |
| 209 | +} | |
| 210 | + | |
| 211 | +@article{piotroski2000, | |
| 212 | + author = {Piotroski, Joseph D.}, | |
| 213 | + title = {{Value Investing: The Use of Historical Financial Statement Information to Separate Winners from Losers}}, | |
| 214 | + journal = {Journal of Accounting Research}, | |
| 215 | + year = {2000}, | |
| 216 | + volume = {38}, | |
| 217 | + pages = {1}, | |
| 218 | + doi = {10.2307/2672906} | |
| 219 | +} | |
| 220 | + | |
| 221 | +@article{hirshleifer2009, | |
| 222 | + author = {HIRSHLEIFER, DAVID and LIM, SONYA SEONGYEON and TEOH, SIEW HONG}, | |
| 223 | + title = {{Driven to Distraction: Extraneous Events and Underreaction to Earnings News}}, | |
| 224 | + journal = {The Journal of Finance}, | |
| 225 | + year = {2009}, | |
| 226 | + volume = {64}, | |
| 227 | + number = {5}, | |
| 228 | + pages = {2289--2325}, | |
| 229 | + doi = {10.1111/j.1540-6261.2009.01501.x} | |
| 230 | +} | |
| 231 | + | |
| 232 | +@article{dellavigna2009, | |
| 233 | + author = {DELLAVIGNA, STEFANO and POLLET, JOSHUA M.}, | |
| 234 | + title = {{Investor Inattention and Friday Earnings Announcements}}, | |
| 235 | + journal = {The Journal of Finance}, | |
| 236 | + year = {2009}, | |
| 237 | + volume = {64}, | |
| 238 | + number = {2}, | |
| 239 | + pages = {709--749}, | |
| 240 | + doi = {10.1111/j.1540-6261.2009.01447.x} | |
| 241 | +} | |
| 242 | + | |
| 243 | +@article{loughran2011, | |
| 244 | + author = {LOUGHRAN, TIM and MCDONALD, BILL}, | |
| 245 | + title = {{When Is a Liability Not a Liability? Textual Analysis, Dictionaries, and 10-Ks}}, | |
| 246 | + journal = {The Journal of Finance}, | |
| 247 | + year = {2011}, | |
| 248 | + volume = {66}, | |
| 249 | + number = {1}, | |
| 250 | + pages = {35--65}, | |
| 251 | + doi = {10.1111/j.1540-6261.2010.01625.x} | |
| 252 | +} | |
| 253 | + | |
| 254 | +@article{bajgrowicz2016, | |
| 255 | + author = {Bajgrowicz, Pierre and Scaillet, Olivier and Treccani, Adrien}, | |
| 256 | + title = {{Jumps in High-Frequency Data: Spurious Detections, Dynamics, and News}}, | |
| 257 | + journal = {Management Science}, | |
| 258 | + year = {2016}, | |
| 259 | + volume = {62}, | |
| 260 | + number = {8}, | |
| 261 | + pages = {2198--2217}, | |
| 262 | + doi = {10.1287/mnsc.2015.2234} | |
| 263 | +} | |
| 264 | + | |
| 265 | +@article{lee2008, | |
| 266 | + author = {Lee, Suzanne S. and Mykland, Per A.}, | |
| 267 | + title = {{Jumps in Financial Markets: A New Nonparametric Test and Jump Dynamics}}, | |
| 268 | + journal = {Review of Financial Studies}, | |
| 269 | + year = {2008}, | |
| 270 | + volume = {21}, | |
| 271 | + number = {6}, | |
| 272 | + pages = {2535--2563}, | |
| 273 | + doi = {10.1093/rfs/hhm056} | |
| 274 | +} | |
| 275 | + | |
| 276 | +@article{bollerslev2018risk, | |
| 277 | + author = {Bollerslev, Tim and Hood, Benjamin and Huss, John and Pedersen, Lasse Heje}, | |
| 278 | + title = {{Risk Everywhere: Modeling and Managing Volatility}}, | |
| 279 | + journal = {The Review of Financial Studies}, | |
| 280 | + year = {2018}, | |
| 281 | + volume = {31}, | |
| 282 | + number = {7}, | |
| 283 | + pages = {2729--2773}, | |
| 284 | + doi = {10.1093/rfs/hhy041} | |
| 285 | +} | |
| 286 | + | |
| 287 | +@article{hendershott2011, | |
| 288 | + author = {HENDERSHOTT, TERRENCE and JONES, CHARLES M. and MENKVELD, ALBERT J.}, | |
| 289 | + title = {{Does Algorithmic Trading Improve Liquidity?}}, | |
| 290 | + journal = {The Journal of Finance}, | |
| 291 | + year = {2011}, | |
| 292 | + volume = {66}, | |
| 293 | + number = {1}, | |
| 294 | + pages = {1--33}, | |
| 295 | + doi = {10.1111/j.1540-6261.2010.01624.x} | |
| 296 | +} | |
| 297 | + | |
| 298 | +@article{richardson2005, | |
| 299 | + author = {Richardson, Scott A. and Sloan, Richard G. and Soliman, Mark T. and Tuna, İrem}, | |
| 300 | + title = {{Accrual reliability, earnings persistence and stock prices}}, | |
| 301 | + journal = {Journal of Accounting and Economics}, | |
| 302 | + year = {2005}, | |
| 303 | + volume = {39}, | |
| 304 | + number = {3}, | |
| 305 | + pages = {437--485}, | |
| 306 | + doi = {10.1016/j.jacceco.2005.04.005} | |
| 307 | +} | |
| 308 | + | |
| 309 | +@article{beneish1999, | |
| 310 | + author = {Beneish, Messod D.}, | |
| 311 | + title = {{The Detection of Earnings Manipulation}}, | |
| 312 | + journal = {Financial Analysts Journal}, | |
| 313 | + year = {1999}, | |
| 314 | + volume = {55}, | |
| 315 | + number = {5}, | |
| 316 | + pages = {24--36}, | |
| 317 | + doi = {10.2469/faj.v55.n5.2296} | |
| 318 | +} | |
| 319 | + | |
| 320 | +@article{campbell2008, | |
| 321 | + author = {CAMPBELL, JOHN Y. and HILSCHER, JENS and SZILAGYI, JAN}, | |
| 322 | + title = {{In Search of Distress Risk}}, | |
| 323 | + journal = {The Journal of Finance}, | |
| 324 | + year = {2008}, | |
| 325 | + volume = {63}, | |
| 326 | + number = {6}, | |
| 327 | + pages = {2899--2939}, | |
| 328 | + doi = {10.1111/j.1540-6261.2008.01416.x} | |
| 329 | +} | |
| 330 | + | |
| 331 | +@article{fama1973, | |
| 332 | + author = {Fama, Eugene F. and MacBeth, James D.}, | |
| 333 | + title = {{Risk, Return, and Equilibrium: Empirical Tests}}, | |
| 334 | + journal = {Journal of Political Economy}, | |
| 335 | + year = {1973}, | |
| 336 | + volume = {81}, | |
| 337 | + number = {3}, | |
| 338 | + pages = {607--636}, | |
| 339 | + doi = {10.1086/260061} | |
| 340 | +} | |
| 341 | + | |
| 342 | +@article{petersen2009, | |
| 343 | + author = {Petersen, Mitchell A.}, | |
| 344 | + title = {{Estimating Standard Errors in Finance Panel Data Sets: Comparing Approaches}}, | |
| 345 | + journal = {Review of Financial Studies}, | |
| 346 | + year = {2009}, | |
| 347 | + volume = {22}, | |
| 348 | + number = {1}, | |
| 349 | + pages = {435--480}, | |
| 350 | + doi = {10.1093/rfs/hhn053} | |
| 351 | +} | |
| 352 | + | |
| 353 | +@article{cameron2011, | |
| 354 | + author = {Cameron, A. Colin and Gelbach, Jonah B. and Miller, Douglas L.}, | |
| 355 | + title = {{Robust Inference With Multiway Clustering}}, | |
| 356 | + journal = {Journal of Business \& Economic Statistics}, | |
| 357 | + year = {2011}, | |
| 358 | + volume = {29}, | |
| 359 | + number = {2}, | |
| 360 | + pages = {238--249}, | |
| 361 | + doi = {10.1198/jbes.2010.07136} | |
| 362 | +} | |
| 363 | + | |
| 364 | +@article{novymarx2016, | |
| 365 | + author = {Novy-Marx, Robert and Velikov, Mihail}, | |
| 366 | + title = {{A Taxonomy of Anomalies and Their Trading Costs}}, | |
| 367 | + journal = {Review of Financial Studies}, | |
| 368 | + year = {2016}, | |
| 369 | + volume = {29}, | |
| 370 | + number = {1}, | |
| 371 | + pages = {104--147}, | |
| 372 | + doi = {10.1093/rfs/hhv063} | |
| 373 | +} | |
| 374 | + | |
| 375 | +@article{bollerslev2016beta, | |
| 376 | + author = {Bollerslev, Tim and Li, Sophia Zhengzi and Todorov, Viktor}, | |
| 377 | + title = {{Roughing up beta: Continuous versus discontinuous betas and the cross section of expected stock returns}}, | |
| 378 | + journal = {Journal of Financial Economics}, | |
| 379 | + year = {2016}, | |
| 380 | + volume = {120}, | |
| 381 | + number = {3}, | |
| 382 | + pages = {464--490}, | |
| 383 | + doi = {10.1016/j.jfineco.2016.02.001} | |
| 384 | +} | |
| 385 | + | |
| 386 | +@article{lo1988, | |
| 387 | + author = {Lo, Andrew W. and MacKinlay, A. Craig}, | |
| 388 | + title = {{Stock Market Prices Do Not Follow Random Walks: Evidence from a Simple Specification Test}}, | |
| 389 | + journal = {Review of Financial Studies}, | |
| 390 | + year = {1988}, | |
| 391 | + volume = {1}, | |
| 392 | + number = {1}, | |
| 393 | + pages = {41--66}, | |
| 394 | + doi = {10.1093/rfs/1.1.41} | |
| 395 | +} | |
| 396 | + | |
| 397 | +@article{freyberger2020, | |
| 398 | + author = {Freyberger, Joachim and Neuhierl, Andreas and Weber, Michael}, | |
| 399 | + title = {{Dissecting Characteristics Nonparametrically}}, | |
| 400 | + journal = {The Review of Financial Studies}, | |
| 401 | + year = {2020}, | |
| 402 | + volume = {33}, | |
| 403 | + number = {5}, | |
| 404 | + pages = {2326--2377}, | |
| 405 | + doi = {10.1093/rfs/hhz123} | |
| 406 | +} | |
| 407 | + | |
| 408 | +@article{bollerslev2016errors, | |
| 409 | + author = {Bollerslev, Tim and Patton, Andrew J. and Quaedvlieg, Rogier}, | |
| 410 | + title = {{Exploiting the errors: A simple approach for improved volatility forecasting}}, | |
| 411 | + journal = {Journal of Econometrics}, | |
| 412 | + year = {2016}, | |
| 413 | + volume = {192}, | |
| 414 | + number = {1}, | |
| 415 | + pages = {1--18}, | |
| 416 | + doi = {10.1016/j.jeconom.2015.10.007} | |
| 417 | +} | |
| 418 | + | |
| 419 | +@article{liu2015, | |
| 420 | + author = {Liu, Lily Y. and Patton, Andrew J. and Sheppard, Kevin}, | |
| 421 | + title = {{Does anything beat 5-minute RV? A comparison of realized measures across multiple asset classes}}, | |
| 422 | + journal = {Journal of Econometrics}, | |
| 423 | + year = {2015}, | |
| 424 | + volume = {187}, | |
| 425 | + number = {1}, | |
| 426 | + pages = {293--311}, | |
| 427 | + doi = {10.1016/j.jeconom.2015.02.008} | |
| 428 | +} | |
| 429 | + | |
| 430 | +@article{andersen1998, | |
| 431 | + author = {Andersen, Torben G. and Bollerslev, Tim}, | |
| 432 | + title = {{Answering the Skeptics: Yes, Standard Volatility Models do Provide Accurate Forecasts}}, | |
| 433 | + journal = {International Economic Review}, | |
| 434 | + year = {1998}, | |
| 435 | + volume = {39}, | |
| 436 | + number = {4}, | |
| 437 | + pages = {885}, | |
| 438 | + doi = {10.2307/2527343} | |
| 439 | +} | |
| 440 | + | |
| 441 | +@article{aboody2010, | |
| 442 | + author = {Aboody, David and Lehavy, Reuven and Trueman, Brett}, | |
| 443 | + title = {{Limited attention and the earnings announcement returns of past stock market winners}}, | |
| 444 | + journal = {Review of Accounting Studies}, | |
| 445 | + year = {2010}, | |
| 446 | + volume = {15}, | |
| 447 | + number = {2}, | |
| 448 | + pages = {317--344}, | |
| 449 | + doi = {10.1007/s11142-009-9104-9} | |
| 450 | +} | |
| 451 | + | |
| 452 | +@article{kothari2001, | |
| 453 | + author = {Kothari, S.P}, | |
| 454 | + title = {{Capital markets research in accounting}}, | |
| 455 | + journal = {Journal of Accounting and Economics}, | |
| 456 | + year = {2001}, | |
| 457 | + volume = {31}, | |
| 458 | + number = {1-3}, | |
| 459 | + pages = {105--231}, | |
| 460 | + doi = {10.1016/s0165-4101(01)00030-1} | |
| 461 | +} | |
| 462 | + | |
| 463 | +@article{easley2012, | |
| 464 | + author = {Easley, David and López de Prado, Marcos M. and O'Hara, Maureen}, | |
| 465 | + title = {{Flow Toxicity and Liquidity in a High-frequency World}}, | |
| 466 | + journal = {Review of Financial Studies}, | |
| 467 | + year = {2012}, | |
| 468 | + volume = {25}, | |
| 469 | + number = {5}, | |
| 470 | + pages = {1457--1493}, | |
| 471 | + doi = {10.1093/rfs/hhs053} | |
| 472 | +} | |
| 473 | + | |
| 474 | +@article{hou2015, | |
| 475 | + author = {Hou, Kewei and Xue, Chen and Zhang, Lu}, | |
| 476 | + title = {{Digesting Anomalies: An Investment Approach}}, | |
| 477 | + journal = {Review of Financial Studies}, | |
| 478 | + year = {2015}, | |
| 479 | + volume = {28}, | |
| 480 | + number = {3}, | |
| 481 | + pages = {650--705}, | |
| 482 | + doi = {10.1093/rfs/hhu068} | |
| 483 | +} | |
| 484 | + | |
| 485 | +@article{dechow2010, | |
| 486 | + author = {Dechow, Patricia and Ge, Weili and Schrand, Catherine}, | |
| 487 | + title = {{Understanding earnings quality: A review of the proxies, their determinants and their consequences}}, | |
| 488 | + journal = {Journal of Accounting and Economics}, | |
| 489 | + year = {2010}, | |
| 490 | + volume = {50}, | |
| 491 | + number = {2-3}, | |
| 492 | + pages = {344--401}, | |
| 493 | + doi = {10.1016/j.jacceco.2010.09.001} | |
| 494 | +} | |
| 495 | + | |
| 496 | +@article{fama1970, | |
| 497 | + author = {Fama, Eugene F.}, | |
| 498 | + title = {{Efficient Capital Markets: A Review of Theory and Empirical Work}}, | |
| 499 | + journal = {The Journal of Finance}, | |
| 500 | + year = {1970}, | |
| 501 | + volume = {25}, | |
| 502 | + number = {2}, | |
| 503 | + pages = {383}, | |
| 504 | + doi = {10.2307/2325486} | |
| 505 | +} | |
| 506 | + | |
| 507 | +@article{bollerslev2011tails, | |
| 508 | + author = {BOLLERSLEV, TIM and TODOROV, VIKTOR}, | |
| 509 | + title = {{Tails, Fears, and Risk Premia}}, | |
| 510 | + journal = {The Journal of Finance}, | |
| 511 | + year = {2011}, | |
| 512 | + volume = {66}, | |
| 513 | + number = {6}, | |
| 514 | + pages = {2165--2211}, | |
| 515 | + doi = {10.1111/j.1540-6261.2011.01695.x} | |
| 516 | +} | |
| 517 | + | |
| 518 | +@article{hirshleifer2003, | |
| 519 | + author = {Hirshleifer, David and Teoh, Siew Hong}, | |
| 520 | + title = {{Limited attention, information disclosure, and financial reporting}}, | |
| 521 | + journal = {Journal of Accounting and Economics}, | |
| 522 | + year = {2003}, | |
| 523 | + volume = {36}, | |
| 524 | + number = {1-3}, | |
| 525 | + pages = {337--386}, | |
| 526 | + doi = {10.1016/j.jacceco.2003.10.002} | |
| 527 | +} | |
| 528 | + | |
| 529 | +@article{savor2016, | |
| 530 | + author = {SAVOR, PAVEL and WILSON, MUNGO}, | |
| 531 | + title = {{Earnings Announcements and Systematic Risk}}, | |
| 532 | + journal = {The Journal of Finance}, | |
| 533 | + year = {2016}, | |
| 534 | + volume = {71}, | |
| 535 | + number = {1}, | |
| 536 | + pages = {83--138}, | |
| 537 | + doi = {10.1111/jofi.12361} | |
| 538 | +} | |
| 539 | + | |
| 540 | +@article{budish2015, | |
| 541 | + author = {Budish, Eric and Cramton, Peter and Shim, John}, | |
| 542 | + title = {{The High-Frequency Trading Arms Race: Frequent Batch Auctions as a Market Design Response *}}, | |
| 543 | + journal = {The Quarterly Journal of Economics}, | |
| 544 | + year = {2015}, | |
| 545 | + volume = {130}, | |
| 546 | + number = {4}, | |
| 547 | + pages = {1547--1621}, | |
| 548 | + doi = {10.1093/qje/qjv027} | |
| 549 | +} | |
| 550 | + | |
| 551 | +@article{chen2024deep, | |
| 552 | + author = {Chen, Luyang and Pelger, Markus and Zhu, Jason}, | |
| 553 | + title = {{Deep Learning in Asset Pricing}}, | |
| 554 | + journal = {Management Science}, | |
| 555 | + year = {2024}, | |
| 556 | + volume = {70}, | |
| 557 | + number = {2}, | |
| 558 | + pages = {714--750}, | |
| 559 | + doi = {10.1287/mnsc.2023.4695} | |
| 560 | +} | |
| 561 | + | |
| 562 | +@article{kozak2020, | |
| 563 | + author = {Kozak, Serhiy and Nagel, Stefan and Santosh, Shrihari}, | |
| 564 | + title = {{Shrinking the cross-section}}, | |
| 565 | + journal = {Journal of Financial Economics}, | |
| 566 | + year = {2020}, | |
| 567 | + volume = {135}, | |
| 568 | + number = {2}, | |
| 569 | + pages = {271--292}, | |
| 570 | + doi = {10.1016/j.jfineco.2019.06.008} | |
| 571 | +} | |
| 572 | + | |
| 573 | +@article{brogaard2019, | |
| 574 | + author = {BROGAARD, JONATHAN and HENDERSHOTT, TERRENCE and RIORDAN, RYAN}, | |
| 575 | + title = {{Price Discovery without Trading: Evidence from Limit Orders}}, | |
| 576 | + journal = {The Journal of Finance}, | |
| 577 | + year = {2019}, | |
| 578 | + volume = {74}, | |
| 579 | + number = {4}, | |
| 580 | + pages = {1621--1658}, | |
| 581 | + doi = {10.1111/jofi.12769} | |
| 582 | +} | |
| 583 | + | |
| 584 | +@article{ang2006, | |
| 585 | + author = {ANG, ANDREW and HODRICK, ROBERT J. and XING, YUHANG and ZHANG, XIAOYAN}, | |
| 586 | + title = {{The Cross-Section of Volatility and Expected Returns}}, | |
| 587 | + journal = {The Journal of Finance}, | |
| 588 | + year = {2006}, | |
| 589 | + volume = {61}, | |
| 590 | + number = {1}, | |
| 591 | + pages = {259--299}, | |
| 592 | + doi = {10.1111/j.1540-6261.2006.00836.x} | |
| 593 | +} | |
| 594 | + | |
| 595 | +@article{skinner2002, | |
| 596 | + author = {Skinner, Douglas J. and Sloan, Richard G.}, | |
| 597 | + title = {{Earnings Surprises, Growth Expectations, and Stock Returns or Don't Let an Earnings Torpedo Sink Your Portfolio}}, | |
| 598 | + journal = {Review of Accounting Studies}, | |
| 599 | + year = {2002}, | |
| 600 | + volume = {7}, | |
| 601 | + number = {2-3}, | |
| 602 | + pages = {289--312}, | |
| 603 | + doi = {10.1023/a:1020294523516} | |
| 604 | +} | |
| 605 | + | |
| 606 | +@article{levi2015, | |
| 607 | + author = {Levi, Shai and Zhang, Xiao-Jun}, | |
| 608 | + title = {{Asymmetric decrease in liquidity trading before earnings announcements and the announcement return premium}}, | |
| 609 | + journal = {Journal of Financial Economics}, | |
| 610 | + year = {2015}, | |
| 611 | + volume = {118}, | |
| 612 | + number = {2}, | |
| 613 | + pages = {383--398}, | |
| 614 | + doi = {10.1016/j.jfineco.2015.08.003} | |
| 615 | +} | |
| 616 | + | |
| 617 | +@article{campbell1993, | |
| 618 | + author = {Campbell, J. Y. and Grossman, S. J. and Wang, J.}, | |
| 619 | + title = {{Trading Volume and Serial Correlation in Stock Returns}}, | |
| 620 | + journal = {The Quarterly Journal of Economics}, | |
| 621 | + year = {1993}, | |
| 622 | + volume = {108}, | |
| 623 | + number = {4}, | |
| 624 | + pages = {905--939}, | |
| 625 | + doi = {10.2307/2118454} | |
| 626 | +} | |
| 627 | + | |
| 628 | +@article{fang2009, | |
| 629 | + author = {FANG, LILY and PERESS, JOEL}, | |
| 630 | + title = {{Media Coverage and the Cross-section of Stock Returns}}, | |
| 631 | + journal = {The Journal of Finance}, | |
| 632 | + year = {2009}, | |
| 633 | + volume = {64}, | |
| 634 | + number = {5}, | |
| 635 | + pages = {2023--2052}, | |
| 636 | + doi = {10.1111/j.1540-6261.2009.01493.x} | |
| 637 | +} | |
| 638 | + | |
| 639 | +@article{beaver1980, | |
| 640 | + author = {Beaver, William and Lambert, Richard and Morse, Dale}, | |
| 641 | + title = {{The information content of security prices}}, | |
| 642 | + journal = {Journal of Accounting and Economics}, | |
| 643 | + year = {1980}, | |
| 644 | + volume = {2}, | |
| 645 | + number = {1}, | |
| 646 | + pages = {3--28}, | |
| 647 | + doi = {10.1016/0165-4101(80)90013-0} | |
| 648 | +} | |
| 649 | + | |
| 650 | +@article{kothari1992, | |
| 651 | + author = {Kothari, S.P and Sloan, Richard G}, | |
| 652 | + title = {{Information in prices about future earnings}}, | |
| 653 | + journal = {Journal of Accounting and Economics}, | |
| 654 | + year = {1992}, | |
| 655 | + volume = {15}, | |
| 656 | + number = {2-3}, | |
| 657 | + pages = {143--171}, | |
| 658 | + doi = {10.1016/0165-4101(92)90016-u} | |
| 659 | +} | |
| 660 | + | |
| 661 | +@article{collins1994, | |
| 662 | + author = {Collins, Daniel W. and Kothari, S.P. and Shanken, Jay and Sloan, Richard G.}, | |
| 663 | + title = {{Lack of timeliness and noise as explanations for the low contemporaneuos return-earnings association}}, | |
| 664 | + journal = {Journal of Accounting and Economics}, | |
| 665 | + year = {1994}, | |
| 666 | + volume = {18}, | |
| 667 | + number = {3}, | |
| 668 | + pages = {289--324}, | |
| 669 | + doi = {10.1016/0165-4101(94)90024-8} | |
| 670 | +} | |
| 671 | + | |
| 672 | +@article{foster1977, | |
| 673 | + author = {Foster, George}, | |
| 674 | + title = {{Quarterly Accounting Data: Time-Series Properties and Predicative-Ability Results.}}, | |
| 675 | + journal = {The Accounting Review}, | |
| 676 | + year = {1977}, | |
| 677 | + volume = {52}, | |
| 678 | + number = {1}, | |
| 679 | + pages = {1--21}, | |
| 680 | + doi = {10.2308/tar-4493420} | |
| 681 | +} | |
| 682 | + | |
| 683 | +@article{weller2018, | |
| 684 | + author = {Weller, Brian M.}, | |
| 685 | + title = {{Does Algorithmic Trading Reduce Information Acquisition?}}, | |
| 686 | + journal = {The Review of Financial Studies}, | |
| 687 | + year = {2018}, | |
| 688 | + volume = {31}, | |
| 689 | + number = {6}, | |
| 690 | + pages = {2184--2226}, | |
| 691 | + doi = {10.1093/rfs/hhx137} | |
| 692 | +} | |
| 693 | + | |
| 694 | +@article{bollerslev2018volume, | |
| 695 | + author = {Bollerslev, Tim and Li, Jia and Xue, Yuan}, | |
| 696 | + title = {{Volume, Volatility, and Public News Announcements}}, | |
| 697 | + journal = {The Review of Economic Studies}, | |
| 698 | + year = {2018}, | |
| 699 | + volume = {85}, | |
| 700 | + number = {4}, | |
| 701 | + pages = {2005--2041}, | |
| 702 | + doi = {10.1093/restud/rdy003} | |
| 703 | +} | |
| 704 | + | |
| 705 | +@article{patton2012, | |
| 706 | + author = {Patton, Andrew J. and Verardo, Michela}, | |
| 707 | + title = {{Does Beta Move with News? Firm-Specific Information Flows and Learning about Profitability}}, | |
| 708 | + journal = {Review of Financial Studies}, | |
| 709 | + year = {2012}, | |
| 710 | + volume = {25}, | |
| 711 | + number = {9}, | |
| 712 | + pages = {2789--2839}, | |
| 713 | + doi = {10.1093/rfs/hhs073} | |
| 714 | +} | |
| 715 | + | |
| 716 | +@article{kaniel2012, | |
| 717 | + author = {KANIEL, RON and LIU, SHUMING and SAAR, GIDEON and TITMAN, SHERIDAN}, | |
| 718 | + title = {{Individual Investor Trading and Return Patterns around Earnings Announcements}}, | |
| 719 | + journal = {The Journal of Finance}, | |
| 720 | + year = {2012}, | |
| 721 | + volume = {67}, | |
| 722 | + number = {2}, | |
| 723 | + pages = {639--680}, | |
| 724 | + doi = {10.1111/j.1540-6261.2012.01727.x} | |
| 725 | +} | |
| 726 | + | |
| 727 | +@article{christophe2004, | |
| 728 | + author = {Christophe, Stephen E. and Ferri, Michael G. and Angel, James J.}, | |
| 729 | + title = {{Short-Selling Prior to Earnings Announcements}}, | |
| 730 | + journal = {The Journal of Finance}, | |
| 731 | + year = {2004}, | |
| 732 | + volume = {59}, | |
| 733 | + number = {4}, | |
| 734 | + pages = {1845--1876}, | |
| 735 | + doi = {10.1111/j.1540-6261.2004.00681.x} | |
| 736 | +} | |
| 737 | + | |
| 738 | +@article{berkman2009, | |
| 739 | + author = {Berkman, Henk and Dimitrov, Valentin and Jain, Prem C. and Koch, Paul D. and Tice, Sheri}, | |
| 740 | + title = {{Sell on the news: Differences of opinion, short-sales constraints, and returns around earnings announcements☆}}, | |
| 741 | + journal = {Journal of Financial Economics}, | |
| 742 | + year = {2009}, | |
| 743 | + volume = {92}, | |
| 744 | + number = {3}, | |
| 745 | + pages = {376--399}, | |
| 746 | + doi = {10.1016/j.jfineco.2008.04.009} | |
| 747 | +} | |
| 748 | + | |
| 749 | +@article{hendershott2015, | |
| 750 | + author = {Hendershott, Terrence and Livdan, Dmitry and Schürhoff, Norman}, | |
| 751 | + title = {{Are institutions informed about news?}}, | |
| 752 | + journal = {Journal of Financial Economics}, | |
| 753 | + year = {2015}, | |
| 754 | + volume = {117}, | |
| 755 | + number = {2}, | |
| 756 | + pages = {249--287}, | |
| 757 | + doi = {10.1016/j.jfineco.2015.03.007} | |
| 758 | +} | |
| 759 | + | |
| 760 | +@article{lee2012, | |
| 761 | + author = {Lee, Suzanne S.}, | |
| 762 | + title = {{Jumps and Information Flow in Financial Markets}}, | |
| 763 | + journal = {Review of Financial Studies}, | |
| 764 | + year = {2012}, | |
| 765 | + volume = {25}, | |
| 766 | + number = {2}, | |
| 767 | + pages = {439--479}, | |
| 768 | + doi = {10.1093/rfs/hhr084} | |
| 769 | +} | |
| 770 | + | |
| 771 | +@article{cao2024, | |
| 772 | + author = {Cao, Sean and Jiang, Wei and Wang, Junbo and Yang, Baozhong}, | |
| 773 | + title = {{From Man vs. Machine to Man + Machine: The art and AI of stock analyses}}, | |
| 774 | + journal = {Journal of Financial Economics}, | |
| 775 | + year = {2024}, | |
| 776 | + volume = {160}, | |
| 777 | + pages = {103910}, | |
| 778 | + doi = {10.1016/j.jfineco.2024.103910} | |
| 779 | +} | |
| 780 | + | |
| 781 | +@article{bali2023, | |
| 782 | + author = {Bali, Turan G and Beckmeyer, Heiner and Mörke, Mathis and Weigert, Florian}, | |
| 783 | + title = {{Option Return Predictability with Machine Learning and Big Data}}, | |
| 784 | + journal = {The Review of Financial Studies}, | |
| 785 | + year = {2023}, | |
| 786 | + volume = {36}, | |
| 787 | + number = {9}, | |
| 788 | + pages = {3548--3602}, | |
| 789 | + doi = {10.1093/rfs/hhad017} | |
| 790 | +} | |
| 791 | + | |
| 792 | +@article{easton2024, | |
| 793 | + author = {Easton, Peter D. and Kapons, Martin M. and Monahan, Steven J. and Schütt, Harm H. and Weisbrod, Eric H.}, | |
| 794 | + title = {{Forecasting Earnings Using \emph{k}-Nearest Neighbors}}, | |
| 795 | + journal = {The Accounting Review}, | |
| 796 | + year = {2024}, | |
| 797 | + volume = {99}, | |
| 798 | + number = {3}, | |
| 799 | + pages = {115--140}, | |
| 800 | + doi = {10.2308/tar-2021-0478} | |
| 801 | +} | |
| 802 | + | |
| 803 | +@article{so2013, | |
| 804 | + author = {So, Eric C.}, | |
| 805 | + title = {{A new approach to predicting analyst forecast errors: Do investors overweight analyst forecasts?}}, | |
| 806 | + journal = {Journal of Financial Economics}, | |
| 807 | + year = {2013}, | |
| 808 | + volume = {108}, | |
| 809 | + number = {3}, | |
| 810 | + pages = {615--640}, | |
| 811 | + doi = {10.1016/j.jfineco.2013.02.002} | |
| 812 | +} | |
| 813 | + | |
| 814 | +@article{hou2012, | |
| 815 | + author = {Hou, Kewei and van Dijk, Mathijs A. and Zhang, Yinglei}, | |
| 816 | + title = {{The implied cost of capital: A new approach}}, | |
| 817 | + journal = {Journal of Accounting and Economics}, | |
| 818 | + year = {2012}, | |
| 819 | + volume = {53}, | |
| 820 | + number = {3}, | |
| 821 | + pages = {504--526}, | |
| 822 | + doi = {10.1016/j.jacceco.2011.12.001} | |
| 823 | +} | |
| 824 | + | |
| 825 | +@article{hansen2006, | |
| 826 | + author = {Hansen, Peter R and Lunde, Asger}, | |
| 827 | + title = {{Realized Variance and Market Microstructure Noise}}, | |
| 828 | + journal = {Journal of Business \& Economic Statistics}, | |
| 829 | + year = {2006}, | |
| 830 | + volume = {24}, | |
| 831 | + number = {2}, | |
| 832 | + pages = {127--161}, | |
| 833 | + doi = {10.1198/073500106000000071} | |
| 834 | +} | |
| 835 | + | |
| 836 | +@article{barndorff2008, | |
| 837 | + author = {Barndorff-Nielsen, Ole E. and Hansen, Peter Reinhard and Lunde, Asger and Shephard, Neil}, | |
| 838 | + title = {{Designing Realized Kernels to Measure the ex post Variation of Equity Prices in the Presence of Noise}}, | |
| 839 | + journal = {Econometrica}, | |
| 840 | + year = {2008}, | |
| 841 | + volume = {76}, | |
| 842 | + number = {6}, | |
| 843 | + pages = {1481--1536}, | |
| 844 | + doi = {10.3982/ecta6495} | |
| 845 | +} | |
| 846 | + | |
| 847 | +@article{vankervel2019, | |
| 848 | + author = {VAN KERVEL, VINCENT and MENKVELD, ALBERT J.}, | |
| 849 | + title = {{High-Frequency Trading around Large Institutional Orders}}, | |
| 850 | + journal = {The Journal of Finance}, | |
| 851 | + year = {2019}, | |
| 852 | + volume = {74}, | |
| 853 | + number = {3}, | |
| 854 | + pages = {1091--1137}, | |
| 855 | + doi = {10.1111/jofi.12759} | |
| 856 | +} | |
| 857 | + | |
| 858 | +@article{avramov2023, | |
| 859 | + author = {Avramov, Doron and Cheng, Si and Metzker, Lior}, | |
| 860 | + title = {{Machine Learning vs. Economic Restrictions: Evidence from Stock Return Predictability}}, | |
| 861 | + journal = {Management Science}, | |
| 862 | + year = {2023}, | |
| 863 | + volume = {69}, | |
| 864 | + number = {5}, | |
| 865 | + pages = {2587--2619}, | |
| 866 | + doi = {10.1287/mnsc.2022.4449} | |
| 867 | +} | |
| 868 | + | |
| 869 | +@article{chordia2020, | |
| 870 | + author = {Chordia, Tarun and Goyal, Amit and Saretto, Alessio}, | |
| 871 | + title = {{Anomalies and False Rejections}}, | |
| 872 | + journal = {The Review of Financial Studies}, | |
| 873 | + year = {2020}, | |
| 874 | + volume = {33}, | |
| 875 | + number = {5}, | |
| 876 | + pages = {2134--2179}, | |
| 877 | + doi = {10.1093/rfs/hhaa018} | |
| 878 | +} | |
| 879 | + | |
| 880 | +@article{barber2013, | |
| 881 | + author = {Barber, Brad M. and De George, Emmanuel T. and Lehavy, Reuven and Trueman, Brett}, | |
| 882 | + title = {{The earnings announcement premium around the globe}}, | |
| 883 | + journal = {Journal of Financial Economics}, | |
| 884 | + year = {2013}, | |
| 885 | + volume = {108}, | |
| 886 | + number = {1}, | |
| 887 | + pages = {118--138}, | |
| 888 | + doi = {10.1016/j.jfineco.2012.10.006} | |
| 889 | +} | |
| 890 | + | |
| 891 | +@article{politis1994, | |
| 892 | + author = {Politis, Dimitris N. and Romano, Joseph P.}, | |
| 893 | + title = {{The Stationary Bootstrap}}, | |
| 894 | + journal = {Journal of the American Statistical Association}, | |
| 895 | + year = {1994}, | |
| 896 | + volume = {89}, | |
| 897 | + number = {428}, | |
| 898 | + pages = {1303--1313}, | |
| 899 | + doi = {10.1080/01621459.1994.10476870} | |
| 900 | +} | |
| 901 | + | |
| 902 | +@article{hansen2011mcs, | |
| 903 | + author = {Hansen, Peter R. and Lunde, Asger and Nason, James M.}, | |
| 904 | + title = {{The Model Confidence Set}}, | |
| 905 | + journal = {Econometrica}, | |
| 906 | + year = {2011}, | |
| 907 | + volume = {79}, | |
| 908 | + number = {2}, | |
| 909 | + pages = {453--497}, | |
| 910 | + doi = {10.3982/ecta5771} | |
| 911 | +} | |
| 912 | + | |
| 913 | +@article{benjamini1995, | |
| 914 | + author = {Benjamini, Yoav and Hochberg, Yosef}, | |
| 915 | + title = {{Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing}}, | |
| 916 | + journal = {Journal of the Royal Statistical Society Series B: Statistical Methodology}, | |
| 917 | + year = {1995}, | |
| 918 | + volume = {57}, | |
| 919 | + number = {1}, | |
| 920 | + pages = {289--300}, | |
| 921 | + doi = {10.1111/j.2517-6161.1995.tb02031.x} | |
| 922 | +} | |
| 923 | + | |
| 924 | +@article{amihud2002, | |
| 925 | + author = {Amihud, Yakov}, | |
| 926 | + title = {{Illiquidity and stock returns: cross-section and time-series effects}}, | |
| 927 | + journal = {Journal of Financial Markets}, | |
| 928 | + year = {2002}, | |
| 929 | + volume = {5}, | |
| 930 | + number = {1}, | |
| 931 | + pages = {31--56}, | |
| 932 | + doi = {10.1016/s1386-4181(01)00024-6} | |
| 933 | +} | |
| 934 | + | |
| 935 | +@article{fama1993, | |
| 936 | + author = {Fama, Eugene F. and French, Kenneth R.}, | |
| 937 | + title = {{Common risk factors in the returns on stocks and bonds}}, | |
| 938 | + journal = {Journal of Financial Economics}, | |
| 939 | + year = {1993}, | |
| 940 | + volume = {33}, | |
| 941 | + number = {1}, | |
| 942 | + pages = {3--56}, | |
| 943 | + doi = {10.1016/0304-405x(93)90023-5} | |
| 944 | +} | |
| 945 | + | |
| 946 | +@article{grossman1980, | |
| 947 | + author = {Grossman, Sanford J. and Stiglitz, Joseph E.}, | |
| 948 | + title = {{On the Impossibility of Informationally Efficient Markets}}, | |
| 949 | + journal = {American Economic Review}, | |
| 950 | + year = {1980}, | |
| 951 | + volume = {70}, | |
| 952 | + number = {3}, | |
| 953 | + pages = {393--408} | |
| 954 | +} | |
| 955 | +@article{holm1979, | |
| 956 | + author = {Holm, Sture}, | |
| 957 | + title = {{A Simple Sequentially Rejective Multiple Test Procedure}}, | |
| 958 | + journal = {Scandinavian Journal of Statistics}, | |
| 959 | + year = {1979}, | |
| 960 | + volume = {6}, | |
| 961 | + number = {2}, | |
| 962 | + pages = {65--70} | |
| 963 | +} | |
| 964 | +@article{kunsch1989, | |
| 965 | + author = {K{\"u}nsch, Hans R.}, | |
| 966 | + title = {{The Jackknife and the Bootstrap for General Stationary Observations}}, | |
| 967 | + journal = {Annals of Statistics}, | |
| 968 | + year = {1989}, | |
| 969 | + volume = {17}, | |
| 970 | + number = {3}, | |
| 971 | + pages = {1217--1241}, | |
| 972 | + doi = {10.1214/aos/1176347265} | |
| 973 | +} | |
| 974 | +@inproceedings{ke2017lightgbm, | |
| 975 | + author = {Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan}, | |
| 976 | + title = {{LightGBM: A Highly Efficient Gradient Boosting Decision Tree}}, | |
| 977 | + booktitle = {Advances in Neural Information Processing Systems 30}, | |
| 978 | + year = {2017}, | |
| 979 | + pages = {3146--3154} | |
| 980 | +} | |
| 981 | +@book{lopezdeprado2018, | |
| 982 | + author = {L{\'o}pez de Prado, Marcos}, | |
| 983 | + title = {{Advances in Financial Machine Learning}}, | |
| 984 | + publisher = {Wiley}, | |
| 985 | + address = {Hoboken, NJ}, | |
| 986 | + year = {2018} | |
| 987 | +} | |
| 988 | +@techreport{frazzini2018costs, | |
| 989 | + author = {Frazzini, Andrea and Israel, Ronen and Moskowitz, Tobias J.}, | |
| 990 | + title = {{Trading Costs}}, | |
| 991 | + institution = {SSRN Working Paper 3229719}, | |
| 992 | + year = {2018}, | |
| 993 | + doi = {10.2139/ssrn.3229719} | |
| 994 | +} | |
| 995 | +@incollection{barndorff2010semivariance, | |
| 996 | + author = {Barndorff-Nielsen, Ole E. and Kinnebrock, Silja and Shephard, Neil}, | |
| 997 | + title = {{Measuring Downside Risk: Realised Semivariance}}, | |
| 998 | + booktitle = {Volatility and Time Series Econometrics: Essays in Honor of Robert F. Engle}, | |
| 999 | + editor = {Bollerslev, Tim and Russell, Jeffrey and Watson, Mark}, | |
| 1000 | + publisher = {Oxford University Press}, | |
| 1001 | + year = {2010}, | |
| 1002 | + pages = {117--136}, | |
| 1003 | + doi = {10.1093/acprof:oso/9780199549498.003.0007} | |
| 1004 | +} | |
| 1005 | +@article{boucher2026wp15, | |
| 1006 | + author = {Boucher, Simon-Pierre}, | |
| 1007 | + title = {{Faster, Wider, More Fragile? The Evolution of Intraday Shock Propagation in U.S. Equity Markets, 1992--2026}}, | |
| 1008 | + journal = {UQO Working Paper No. 15}, | |
| 1009 | + year = {2026} | |
| 1010 | +} | |
| 1011 | + | |
| 1012 | +@article{harvey2016, | |
| 1013 | + author = {Harvey, Campbell R. and Liu, Yan and Zhu, Heqing}, | |
| 1014 | + title = {{\ldots and the Cross-Section of Expected Returns}}, | |
| 1015 | + journal = {The Review of Financial Studies}, | |
| 1016 | + year = {2016}, | |
| 1017 | + volume = {29}, | |
| 1018 | + number = {1}, | |
| 1019 | + pages = {5--68}, | |
| 1020 | + doi = {10.1093/rfs/hhv059} | |
| 1021 | +} | |
added
paper/sections/data.tex
+83 −0
@@ -0,0 +1,83 @@ | ||
| 1 | +\section{Data} | |
| 2 | +\label{sec:data} | |
| 3 | + | |
| 4 | +\subsection{Source and license} | |
| 5 | + | |
| 6 | +All market and accounting data come from FinData OS, a private lakehouse of FirstRate Data operated by the author and | |
| 7 | +accessed through its versioned API (\texttt{findata.spboucher.ai/api/v1}). The lake holds 6.7 billion raw 1-minute | |
| 8 | +bars for 16,195 U.S. stock symbols (2000--2026, including 8,069 delisted symbols), quarterly fundamentals for | |
| 9 | +\nEntities{} entities (67 metrics, filing dates 2012--2026) and 373,326 earnings-release events. Bars are timestamped | |
| 10 | +at the bar open in UTC and cover 04:00--20:00 Eastern; zero-volume minutes are absent at the source and are never | |
| 11 | +imputed. FirstRate Data is licensed for private research use; no raw record is redistributed, and the replication | |
| 12 | +package contains code, hashed extraction manifests (dataset identifiers and versions, row counts, SHA-256 digests) and | |
| 13 | +derived aggregates only. Public augmentations are the SEC's \texttt{company\_tickers.json} and \texttt{submissions} | |
| 14 | +files (CIK, SIC industry), and the Kenneth French daily factor library. | |
| 15 | + | |
| 16 | +\subsection{Universe and survivorship} | |
| 17 | + | |
| 18 | +The accounting panel contains the \nEntities{} entities for which the vendor supplies quarterly statements; it has | |
| 19 | +\emph{no delisted entity}, because the vendor ingestion covered tickers active on 1 November 2026. The price side is | |
| 20 | +survivorship-free (delisted names are present in the bar lake), but the firm-quarter sample is conditioned on survival | |
| 21 | +to the end of the sample. This is a structural caveat: firms that deteriorated and delisted are absent, so adverse | |
| 22 | +base rates are understated and the most vulnerable firms are under-represented. We state it in the abstract and | |
| 23 | +treat it as the first limitation. Table~\ref{tab:universe} reports the construction of the universe: \nEligible{} | |
| 24 | +firms with at least eight quarterly periods, at least 250 sessions of 1-minute bars after 2012 and a non-governmental | |
| 25 | +SIC division; \nWithCik{} are matched to a CIK. Identifiers are vendor tickers; the vendor merges ticker changes under | |
| 26 | +the latest ticker, and we audit reuse with the SEC's former-name history. | |
| 27 | + | |
| 28 | +\subsection{Reveal dates and point-in-time discipline} | |
| 29 | + | |
| 30 | +Earnings-release dates are matched to fiscal quarters as the first release in $[E_q, F_q+10\text{ days}]$; | |
| 31 | +\releaseMatchedShare\% of firm-quarters are matched with a filing lag of at most 30 days (median lag between release | |
| 32 | +and filing: \filingLagMedian{} day). Release timing is known (before-open or after-close) for about two thirds of events. | |
| 33 | +Structured facts carry \texttt{filing\_date}, \texttt{available\_at} ($=$ filing date $+1$ day, a documented policy | |
| 34 | +rather than a historical vintage) and a revision identifier; because every fact was first ingested in November 2026 and | |
| 35 | +carries revision~1, restatements that occurred before ingestion are not reconstructible. Inputs therefore use prior | |
| 36 | +quarters only, outcomes use the vendor's current value of quarter $q$, and the stringent as-originally-reported | |
| 37 | +specification is left for a future SEC-filing-level reconstruction (Section~\ref{sec:limitations}). | |
| 38 | + | |
| 39 | +\subsection{Sample} | |
| 40 | + | |
| 41 | +The quarter panel has \nFirmQuarters{} firm-quarters for \nFirmsPanel{} firms. After the admissibility and exclusion | |
| 42 | +cascade of Table~\ref{tab:cascade} (market window of at least 15 valid sessions, raw close of at least one dollar, | |
| 43 | +available fundamentals, prediction of the next quarter, defined expectation, regular fiscal calendar), the event-time | |
| 44 | +sample has \nSampleEvent{} observations across the five horizons (\nSampleFirms{} firms, \nSampleQuarters{} firm-quarters) | |
| 45 | +and the primary $h=20$ cohort has \nObsPrimary{} observations with cutoffs from \firstCutoff{} to \lastCutoff; the | |
| 46 | +fixed-calendar sample has \nSampleCalendar{} firm-months. The adverse outcome occurs in \baseRatePrimary\% of | |
| 47 | +primary-cohort firm-quarters. Table~\ref{tab:summary} reports summary statistics and Figure~\ref{fig:coverage} the | |
| 48 | +coverage by year together with the adverse rate. | |
| 49 | + | |
| 50 | +\begin{table}[H] | |
| 51 | +\centering\footnotesize | |
| 52 | +\caption{Construction of the universe and of the primary sample} | |
| 53 | +\label{tab:universe} | |
| 54 | +\begin{minipage}[t]{0.42\textwidth}\centering | |
| 55 | +\input{tables/tab02_universe} | |
| 56 | +\end{minipage}\hfill | |
| 57 | +\begin{minipage}[t]{0.56\textwidth}\centering | |
| 58 | +\label{tab:cascade} | |
| 59 | +\input{tables/tab02b_sample_cascade} | |
| 60 | +\end{minipage} | |
| 61 | +\begin{minipage}{\textwidth}\vspace{0.3em}\footnotesize | |
| 62 | +\emph{Notes.} Left: firms. Right: event-time observations (five horizons pooled); steps are applied in the frozen order | |
| 63 | +of \texttt{docs/PIT\_PROTOCOL.md}. | |
| 64 | +\end{minipage} | |
| 65 | +\end{table} | |
| 66 | + | |
| 67 | +\begin{table}[H] | |
| 68 | +\centering\footnotesize | |
| 69 | +\caption{Summary statistics, primary cohort ($h=20$)} | |
| 70 | +\label{tab:summary} | |
| 71 | +\adjustbox{max width=\textwidth}{\input{tables/tab03_summary}} | |
| 72 | +\begin{minipage}{\textwidth}\vspace{0.3em}\footnotesize | |
| 73 | +\emph{Notes.} Intraday families are daily values averaged over the 20-session window ending at the cutoff; realized | |
| 74 | +variances are sums of squared bar log-returns within the regular session. Raw, unwinsorised values. | |
| 75 | +\end{minipage} | |
| 76 | +\end{table} | |
| 77 | + | |
| 78 | +\begin{figure}[H] | |
| 79 | +\centering | |
| 80 | +\includegraphics[width=\textwidth]{figures/fig02_coverage.pdf} | |
| 81 | +\caption{Sample coverage and adverse-outcome rate by year of cutoff ($h=20$ cohort).} | |
| 82 | +\label{fig:coverage} | |
| 83 | +\end{figure} | |
added
paper/sections/literature.tex
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +\section{Related literature} | |
| 2 | +\label{sec:literature} | |
| 3 | + | |
| 4 | +\paragraph{Prices lead earnings.} Since \citet{ball1968}, accounting research has documented that stock prices | |
| 5 | +anticipate accounting income: annual returns forecast earnings changes \citep{beaver1980}, leading returns raise | |
| 6 | +earnings-response coefficients \citep{kothari1992}, and the low contemporaneous return--earnings association reflects | |
| 7 | +prices' timeliness rather than noise \citep{collins1994}. The post-earnings-announcement drift \citep{bernard1989, | |
| 8 | +bernard1990} and the accrual anomaly \citep{sloan1996, richardson2005} show that the incorporation of accounting news | |
| 9 | +is incomplete, while fundamental analysis \citep{ou1989, piotroski2000} extracts forecasts of future earnings from | |
| 10 | +statements alone. Recent work replaces hand-crafted ratios by machine learning on detailed statement items | |
| 11 | +\citep{chen2022, easton2024, cao2024}. All of these studies operate at daily or lower frequency and treat the market side | |
| 12 | +as a return; our contribution is to fix the accounting side and vary the \emph{resolution} of the market side. | |
| 13 | + | |
| 14 | +\paragraph{Information in intraday activity.} Market microstructure theory ties the composition of trading to private | |
| 15 | +information \citep{kyle1985, easley1987}, and realized-measure econometrics separates continuous variation, jumps and | |
| 16 | +their signs from high-frequency bars \citep{andersen1998, andersen2003, barndorff2004, barndorff2010semivariance, | |
| 17 | +patton2015, lee2008}. Signed jump variation and downside semivariance price the cross-section | |
| 18 | +\citep{bollerslev2020good, bollerslev2016beta}, jumps cluster around news \citep{lee2012, bajgrowicz2016}, and | |
| 19 | +announcement days change betas and the volume--volatility relation \citep{patton2012, bollerslev2018volume, | |
| 20 | +savor2016}. The question of the best sampling frequency has been settled for \emph{variance measurement} -- five-minute | |
| 21 | +realized variance is hard to beat \citep{liu2015, bollerslev2016errors, bollerslev2018risk} -- but not for the | |
| 22 | +\emph{forecasting of fundamentals}, which is our object. \citet{zhang2023volatility} show that pooling intraday | |
| 23 | +commonality improves machine-learning volatility forecasts; we include volatility families at every frequency and use a | |
| 24 | +symmetric placebo target to separate a volatility channel from a signed-information channel. | |
| 25 | + | |
| 26 | +\paragraph{Pre-announcement trading and price discovery.} Short sellers \citep{christophe2004}, individual investors | |
| 27 | +\citep{kaniel2012}, institutions \citep{hendershott2015} and option traders trade informatively before earnings | |
| 28 | +announcements; differences of opinion shape announcement returns \citep{berkman2009}; algorithmic trading may reduce | |
| 29 | +pre-announcement information acquisition \citep{weller2018}; and the announcement reaction itself occurs largely in | |
| 30 | +quotes rather than trades \citep{gregoire2022, brogaard2019}. These studies use trader-type or quote data that bars do | |
| 31 | +not contain. We ask a complementary question with anonymous OHLCV only: how much of the forthcoming accounting outcome | |
| 32 | +is visible in the shape of pre-release trading, and does that visibility depend on resolution? | |
| 33 | + | |
| 34 | +\paragraph{Attention and processing.} Limited attention \citep{hirshleifer2003, hirshleifer2009, dellavigna2009, | |
| 35 | +aboody2010} and media coverage \citep{fang2009} govern how fast public information is processed; the timing and | |
| 36 | +heterogeneity hypotheses H3--H4 are motivated by these channels and by distress-risk measurement \citep{campbell2008, | |
| 37 | +beneish1999}. | |
| 38 | + | |
| 39 | +\paragraph{Forecast evaluation and machine learning in finance.} We follow the evaluation standards of | |
| 40 | +\citet{diebold1995}, \citet{giacomini2006}, \citet{hansen2005} and \citet{hansen2011mcs}, with multiple-testing control | |
| 41 | +\citep{holm1979, benjamini1995}, cluster-robust inference \citep{petersen2009, cameron2011}, block bootstrap | |
| 42 | +\citep{kunsch1989, politis1994} and purged, embargoed time splits \citep{lopezdeprado2018}. Equal-budget comparisons of | |
| 43 | +information sets are inspired by the asset-pricing machine-learning literature \citep{gu2020, freyberger2020, kozak2020, | |
| 44 | +chen2024deep, avramov2023} and its warnings about false discoveries \citep{harvey2016, chordia2020}. Gradient-boosted | |
| 45 | +trees \citep{ke2017lightgbm} provide the nonlinear benchmark. The companion UQO working paper on intraday shock | |
| 46 | +propagation \citep{boucher2026wp15} uses the same vendor data for a different question; no result is reused here. | |
added
paper/sections/methodology.tex
+78 −0
@@ -0,0 +1,78 @@ | ||
| 1 | +\section{Research design} | |
| 2 | +\label{sec:method} | |
| 3 | + | |
| 4 | +\subsection{Clocks, cutoffs and the information set} | |
| 5 | + | |
| 6 | +Figure~\ref{fig:timeline} summarises the event-time design. For firm $i$ and fiscal quarter $q$ we observe the period end | |
| 7 | +$E_{i,q}$, the earnings-release date $A_{i,q}$ (the reveal date, with its before-open/after-close timing when known), the | |
| 8 | +10-Q/10-K filing date $F_{i,q}$ and the vendor availability instant $V_{i,q}=F_{i,q}+1$ day. The prediction cutoff is | |
| 9 | +$c=A_{i,q}-h$ trading days for $h\in\{40,20,10,5,1\}$, with $h=20$ as the preregistered primary horizon. A cutoff is | |
| 10 | +admissible only if $c<\min(A_{i,q},F_{i,q})$ and $c>A_{i,q-1}$, so the feature window never contains the release of | |
| 11 | +quarter $q$ and never straddles two reveals. The information set $\mathcal{I}(c)$ contains regular-session bars with | |
| 12 | +session date $\le c$ and accounting facts with $V_{i,j}\le c$ only; the latest such quarter is $j^*(c)=q-1$ in the | |
| 13 | +retained sample (we require that the forecaster predicts the quarter immediately following the latest available one). | |
| 14 | +Twelve leakage guards (future facts, fiscal-end joins, post-cutoff bars, test-period transforms, daylight-saving errors, | |
| 15 | +identifier drift, split artefacts, release-day bars, overlapping windows across splits, survivorship, ingestion dates, | |
| 16 | +randomised placebos) are implemented as unit tests in the replication package; the local as-of join was verified live | |
| 17 | +against the FinData API's \texttt{as\_of} semantics on random entities and instants. | |
| 18 | + | |
| 19 | +A fixed-calendar design (month-end cutoffs, target = the next release within 65 trading days) is used for the pricing | |
| 20 | +tests and as a robustness check because it does not condition on knowledge of the release date. | |
| 21 | + | |
| 22 | +\subsection{Targets} | |
| 23 | + | |
| 24 | +The primary outcome is the scaled operating-income surprise $S_{i,q}$ of equation~\eqref{eq:surprise}, with | |
| 25 | +$\mathbb{E}[X_q]=X_{q-4}+\tfrac14\sum_{k=0}^{3}(X_{q-1-k}-X_{q-5-k})$ computed from quarters available at $c$ and scaled by | |
| 26 | +the latest available total assets. The binary adverse outcome is $D_{i,q}=\mathbb{1}[S_{i,q}\le-0.5\%]$, i.e., a | |
| 27 | +shortfall of at least half a percent of assets (roughly a fifth of a typical quarterly return on assets). Secondary | |
| 28 | +outcomes are the net-income analogue, a year-on-year ROA decline of at least one point, a negative operating-cash-flow | |
| 29 | +innovation of at least one percent of assets, a liquidity/coverage deterioration (current ratio down 10\% or interest | |
| 30 | +coverage falling below one), a symmetric placebo $|S_{i,q}|\ge0.5\%$ that a pure volatility channel would predict, and an | |
| 31 | +exploratory EPS miss against the vendor's estimate (available for \epsEstimateShare\% of releases; vintage unknown). | |
| 32 | +The continuous surprise, winsorised at the training-period 1st/99th percentiles, is modelled by ridge regression. No | |
| 33 | +target uses prices. | |
| 34 | + | |
| 35 | +\subsection{Features by sampling frequency} | |
| 36 | + | |
| 37 | +All market features are derived from the same raw 1-minute bars of the regular session (09:30 to the closing print), | |
| 38 | +aggregated to 5-minute and 30-minute buckets and to daily bars. Each intraday \emph{family} is computed per session | |
| 39 | +on bars of the given length and averaged (or summed) over the 20-session window ending at $c$: realized variance, | |
| 40 | +bipower variation, relative jump, negative-semivariance share, signed-jump variation, realized skewness and kurtosis, | |
| 41 | +concentration of variance in the largest bar, first-order autocorrelation, intraday drift, opening and closing volume | |
| 42 | +shares, volume dispersion (Gini), down-bar volume share, share of expected bars present, share of variance on | |
| 43 | +market-down bars, realized beta to SPY (ratio of summed cross-products), variance ratio against the daily return variance, | |
| 44 | +and volatility-of-volatility. The 30-minute, 5-minute and 1-minute sets therefore have the \emph{same} \kIntra{} columns. | |
| 45 | +Daily features (\kDaily{} columns) are 20- and 60-day returns, volatilities, Parkinson range, skewness, kurtosis, | |
| 46 | +overnight and intraday return components, volume ratios, Amihud illiquidity, zero-return share, dollar volume, maximum | |
| 47 | +drawdown and bar activity. Point-in-time accounting features (\kFund{} columns) are margins, ROA, accruals, cash, | |
| 48 | +leverage, current ratio, coverage, capex, R\&D, issuance, revenue growth, size, the last available surprise, the trailing | |
| 49 | +mean surprise, the trailing adverse rate and the age of the information. Bar statistics are descriptive; nothing is | |
| 50 | +interpreted as order flow, spread or informed trading. | |
| 51 | + | |
| 52 | +\subsection{Baseline ladder and equal budgets} | |
| 53 | + | |
| 54 | +Table~\ref{tab:infosets} nests the information sets. B0 is the training base rate (and a shrunk firm-history rate); | |
| 55 | +B1--B5 are $\ell_2$-penalised logistic regressions whose penalty is tuned on the validation period only; LB1--LB5 are | |
| 56 | +gradient-boosted trees \citep{ke2017lightgbm} with an identical random-search budget (24 trials, early stopping on the | |
| 57 | +validation period) for every information set, so that the frequency comparison is a comparison of information sets, | |
| 58 | +not of capacity. Winsorisation bounds, imputation medians and standardisation constants are estimated on training rows | |
| 59 | +only. A supplementary set (B5all) concatenates all three intraday resolutions. Neural and text models are not used. | |
| 60 | + | |
| 61 | +\subsection{Validation and inference} | |
| 62 | + | |
| 63 | +The chronological split trains on cutoffs in 2012--2018, tunes on 2019--2020 and evaluates on 2021--2026, with purging | |
| 64 | +of training outcomes revealed after the next split's start minus a one-quarter embargo \citep{lopezdeprado2018}. | |
| 65 | +Expanding-window annual refits from 2016 provide out-of-fold predictions for the panel regressions and the timing and | |
| 66 | +pricing exercises; a 20\% firm hold-out evaluates transfer to unseen firms. The primary metric is log loss; Brier score, | |
| 67 | +AUC, PR-AUC and calibration are reported for every model. Paired differences (B5$-$B2, B5$-$B4, B5$-$B3) are assessed | |
| 68 | +with a block bootstrap over calendar quarters of the reveal date \citep{kunsch1989, politis1994} and with the | |
| 69 | +Giacomini--White test on quarterly mean loss differentials with Newey--West variance \citep{diebold1995, | |
| 70 | +giacomini2006}; Holm's correction is applied within each hypothesis family \citep{holm1979}. Panel regressions | |
| 71 | +\begin{equation} | |
| 72 | + S_{i,q}=\alpha_i+\delta_{s(i)\times t(q)}+\beta\,\mathrm{LFDS}_{i,c}+\gamma'X_{i,c}+\varepsilon_{i,q} | |
| 73 | + \label{eq:panel} | |
| 74 | +\end{equation} | |
| 75 | +use the out-of-fold B5 probability as the Latent Fundamental Deterioration Signal (LFDS), firm and sector-by-quarter | |
| 76 | +fixed effects, and two-way clustering by firm and calendar quarter \citep{petersen2009, cameron2011}. Mechanism tests | |
| 77 | +(H3) interact the \emph{incremental} signal $\mathrm{LFDS}^{\mathrm{B5}}-\mathrm{LFDS}^{\mathrm{B2}}$ with ex-ante | |
| 78 | +terciles of size, illiquidity and leverage formed within each calendar quarter. | |
added
paper/sections/theory_hypotheses.tex
+64 −0
@@ -0,0 +1,64 @@ | ||
| 1 | +\section{Economic framework and hypotheses} | |
| 2 | +\label{sec:theory} | |
| 3 | + | |
| 4 | +\subsection{Anticipation versus revelation} | |
| 5 | + | |
| 6 | +Let $X_{i,q}$ denote an accounting flow of firm $i$ for fiscal quarter $q$ (operating income in our primary | |
| 7 | +specification), revealed to the public at the earnings-release date $A_{i,q}$ and structured in the SEC filing of date | |
| 8 | +$F_{i,q}\ge A_{i,q}$. A forecaster standing at a trading day $c<A_{i,q}$ holds an information set $\mathcal{I}(c)$ that | |
| 9 | +contains, at most, prices and volumes up to the close of $c$ and the accounting statements that were \emph{available} at | |
| 10 | +$c$, i.e., quarters $j$ with $F_{i,j}+1\le c$. The quantity of interest is the conditional distribution of the surprise | |
| 11 | +\begin{equation} | |
| 12 | + S_{i,q} \;=\; \frac{X_{i,q}-\mathbb{E}\!\left[X_{i,q}\mid \mathcal{A}(c)\right]}{TA_{i,j^*(c)}}, | |
| 13 | + \label{eq:surprise} | |
| 14 | +\end{equation} | |
| 15 | +where $\mathcal{A}(c)\subset\mathcal{I}(c)$ is the accounting information alone, $j^*(c)$ is the latest available quarter, | |
| 16 | +and the expectation is a seasonal random walk with drift \citep{foster1977}, the classic honest benchmark when no | |
| 17 | +point-in-time consensus exists. Semi-strong efficiency \citep{fama1970} implies that prices at $c$ impound all public | |
| 18 | +information, including $\mathcal{A}(c)$; the ``prices lead earnings'' literature \citep{beaver1980, kothari1992, | |
| 19 | +collins1994} documents that daily and annual returns indeed forecast accounting changes. None of this is the question | |
| 20 | +we ask. Our question is whether the \emph{resolution} at which the market is observed carries incremental information, | |
| 21 | +holding the information set, the universe, the calendar coverage, the horizon and the model capacity fixed. | |
| 22 | + | |
| 23 | +\subsection{Why resolution could matter, and why it might not} | |
| 24 | + | |
| 25 | +Three mechanisms make fine-grained activity informative beyond daily aggregates. First, in sequential-trade models | |
| 26 | +\citep{kyle1985, easley1987}, informed traders split orders and trade when liquidity is deep, so the \emph{composition} of | |
| 27 | +a day's volume and return (intraday autocorrelation, concentration of variance in a few minutes, volume on down-bars) can | |
| 28 | +reveal private information that the daily net return hides. Second, realized measures built from fine bars separate the | |
| 29 | +continuous and jump components of variation \citep{barndorff2004, andersen2003} and their sign | |
| 30 | +\citep{barndorff2010semivariance, patton2015}; negative semivariance and signed jumps have been linked to the | |
| 31 | +cross-section of expected returns \citep{bollerslev2020good} and may proxy for the arrival of adverse firm-specific news. | |
| 32 | +Third, limited attention \citep{hirshleifer2003, hirshleifer2009, dellavigna2009} predicts that the incorporation of | |
| 33 | +public but costly-to-process information is gradual, so that the time path of trading within the day (opening and closing | |
| 34 | +concentration, overnight versus intraday returns) carries information about ongoing processing. | |
| 35 | + | |
| 36 | +Against these channels stand two forces. The Epps effect and microstructure noise contaminate 1-minute returns | |
| 37 | +\citep{hansen2006, barndorff2008}, so that the finest sampling is not the most informative for variance measurement | |
| 38 | +\citep{liu2015}; and whatever information intraday activity contains about fundamentals may be fully summarised by the | |
| 39 | +daily close, in which case the frequency premium is zero. Algorithmic trading may further have reduced the amount of | |
| 40 | +pre-announcement information acquisition \citep{weller2018}. The sign and size of the frequency premium are therefore | |
| 41 | +empirical questions, and a precise null is as informative as a positive estimate. | |
| 42 | + | |
| 43 | +\subsection{Hypotheses} | |
| 44 | + | |
| 45 | +All hypotheses, endpoints and decision rules were frozen before the test period was opened (\texttt{docs/RESEARCH\_DESIGN.md} | |
| 46 | +in the replication package). Let $\mathcal{L}(M)$ denote the out-of-sample log loss of model $M$ for the binary adverse | |
| 47 | +outcome $D_{i,q}=\mathbb{1}[S_{i,q}\le -0.5\%]$, and let B1--B5 be the nested information sets of Table~\ref{tab:infosets}. | |
| 48 | + | |
| 49 | +\begin{description}[leftmargin=1.5em] | |
| 50 | +\item[H1 (incremental information).] $\mathcal{L}(\mathrm{B5})<\mathcal{L}(\mathrm{B2})$ at the 20-trading-day horizon: | |
| 51 | + one-minute features improve forecasts beyond point-in-time fundamentals and daily OHLCV. Null: no improvement. | |
| 52 | +\item[H2 (temporal resolution).] The gain of B5 over B2 exceeds the gains of B4 (5-minute) and B3 (30-minute) over B2, | |
| 53 | + with the same feature families and the same number of inputs. Null: the gains are equal. | |
| 54 | +\item[H3 (heterogeneity).] The B5$-$B2 gain is larger for small, illiquid and highly levered firms, the groups in which | |
| 55 | + information asymmetry and processing costs are highest. Null: homogeneous gain. | |
| 56 | +\item[H4 (timing).] The gain increases as the cutoff approaches the release ($h=40\to1$) and is absent at placebo cutoffs. | |
| 57 | + Null: no gradient. | |
| 58 | +\item[H5 (pricing).] The ex-ante deterioration probability predicts the announcement-window abnormal return and | |
| 59 | + post-cutoff factor-adjusted returns. Null: no return predictability after risk adjustment and costs. | |
| 60 | +\end{description} | |
| 61 | + | |
| 62 | +A positive H1 does not identify \emph{who} trades on what: observational prediction is not causal evidence of informed | |
| 63 | +trading, and we do not have quotes, trades or trader identities \citep{gregoire2022, hendershott2015}. The paper's claims | |
| 64 | +are confined to the forecasting content of anonymous bars and to the frequency at which that content appears. | |
added
paper/sections/titlepage.tex
+77 −0
@@ -0,0 +1,77 @@ | ||
| 1 | +% ============================================================================ | |
| 2 | +% Title Page | |
| 3 | +% ============================================================================ | |
| 4 | +\thispagestyle{empty} | |
| 5 | + | |
| 6 | +\begin{center} | |
| 7 | + | |
| 8 | +% --- Logo --- | |
| 9 | +\includegraphics[width=3.5cm]{uq_logo.jpg} | |
| 10 | + | |
| 11 | +\vspace{0.6cm} | |
| 12 | + | |
| 13 | +{\footnotesize\textsc{Université du Québec en Outaouais}}\\[0.15cm] | |
| 14 | +{\footnotesize\textsc{Département des sciences administratives}} | |
| 15 | + | |
| 16 | +\vspace{0.8cm} | |
| 17 | + | |
| 18 | +{\footnotesize\textsc{Working Paper No.~\WPnumber}} | |
| 19 | + | |
| 20 | +\vspace{1.2cm} | |
| 21 | + | |
| 22 | +% --- Title --- | |
| 23 | +{\LARGE\bfseries \WPtitle\par} | |
| 24 | + | |
| 25 | +\ifx\WPsubtitle\empty\else | |
| 26 | + \vspace{0.3cm} | |
| 27 | + {\large\itshape \WPsubtitle\par} | |
| 28 | +\fi | |
| 29 | + | |
| 30 | +\vspace{1.2cm} | |
| 31 | + | |
| 32 | +% --- Author --- | |
| 33 | +{\large \WPauthor}\\[0.3cm] | |
| 34 | +{\normalsize \WPaffiliation}\\[0.15cm] | |
| 35 | +{\normalsize \href{mailto:\WPemail}{\WPemail}}\\[0.15cm] | |
| 36 | +{\small \WPaddress} | |
| 37 | + | |
| 38 | +\vspace{0.8cm} | |
| 39 | + | |
| 40 | +% --- Date & Version --- | |
| 41 | +{\normalsize \WPdate}\\[0.1cm] | |
| 42 | +{\small Version~\WPversion} | |
| 43 | + | |
| 44 | +\end{center} | |
| 45 | + | |
| 46 | +\vfill | |
| 47 | + | |
| 48 | +\newpage | |
| 49 | + | |
| 50 | +% ============================================================================ | |
| 51 | +% Abstract Page | |
| 52 | +% ============================================================================ | |
| 53 | +\thispagestyle{empty} | |
| 54 | + | |
| 55 | +\vspace*{1cm} | |
| 56 | + | |
| 57 | +\noindent\rule{\textwidth}{0.4pt} | |
| 58 | +\vspace{0.3cm} | |
| 59 | + | |
| 60 | +\noindent\textbf{Abstract} | |
| 61 | + | |
| 62 | +\vspace{0.15cm} | |
| 63 | + | |
| 64 | +\noindent\WPabstract | |
| 65 | + | |
| 66 | +\vspace{0.4cm} | |
| 67 | + | |
| 68 | +\noindent\textbf{Keywords:} \WPkeywords | |
| 69 | + | |
| 70 | +\vspace{0.15cm} | |
| 71 | + | |
| 72 | +\noindent\textbf{JEL Classification:} \WPjel | |
| 73 | + | |
| 74 | +\vspace{0.3cm} | |
| 75 | +\noindent\rule{\textwidth}{0.4pt} | |
| 76 | + | |
| 77 | +\newpage | |
added
paper/sections/titlepage_anon.tex
+53 −0
@@ -0,0 +1,53 @@ | ||
| 1 | +% ============================================================================ | |
| 2 | +% ANONYMOUS TITLE PAGE — for double-anonymized journal submission | |
| 3 | +% Used by paper/main_anon.tex (same macros as main.tex, no author block, | |
| 4 | +% no institution, no logo). Abstract/keywords/JEL page identical to titlepage.tex. | |
| 5 | +% ============================================================================ | |
| 6 | +\thispagestyle{empty} | |
| 7 | +\begin{center} | |
| 8 | + | |
| 9 | +\vspace*{3cm} | |
| 10 | + | |
| 11 | +{\LARGE\bfseries \WPtitle\par} | |
| 12 | + | |
| 13 | +\ifx\WPsubtitle\empty\else | |
| 14 | + \vspace{0.3cm} | |
| 15 | + {\large\itshape \WPsubtitle\par} | |
| 16 | +\fi | |
| 17 | + | |
| 18 | +\vspace{1.5cm} | |
| 19 | + | |
| 20 | +{\normalsize [Author identity withheld for review]} | |
| 21 | + | |
| 22 | +\vspace{0.8cm} | |
| 23 | + | |
| 24 | +{\normalsize \WPdate} | |
| 25 | + | |
| 26 | +\end{center} | |
| 27 | + | |
| 28 | +\vfill | |
| 29 | + | |
| 30 | +\newpage | |
| 31 | +\thispagestyle{empty} | |
| 32 | +\vspace*{1cm} | |
| 33 | + | |
| 34 | +\noindent\rule{\textwidth}{0.4pt} | |
| 35 | +\vspace{0.3cm} | |
| 36 | + | |
| 37 | +\noindent\textbf{Abstract} | |
| 38 | +\vspace{0.15cm} | |
| 39 | + | |
| 40 | +\noindent\WPabstract | |
| 41 | + | |
| 42 | +\vspace{0.4cm} | |
| 43 | + | |
| 44 | +\noindent\textbf{Keywords:} \WPkeywords | |
| 45 | + | |
| 46 | +\vspace{0.15cm} | |
| 47 | + | |
| 48 | +\noindent\textbf{JEL Classification:} \WPjel | |
| 49 | + | |
| 50 | +\vspace{0.3cm} | |
| 51 | +\noindent\rule{\textwidth}{0.4pt} | |
| 52 | + | |
| 53 | +\newpage | |
added
paper/uq_logo.jpg
+0 −0
Binary file not shown.
added
pyproject.toml
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +[project] | |
| 2 | +name = "wp39-uqo" | |
| 3 | +version = "0.1.0" | |
| 4 | +description = "WP39 UQO - The Invisible Balance Sheet: can intraday markets detect corporate deterioration before accounting disclosures?" | |
| 5 | +readme = "README.md" | |
| 6 | +requires-python = ">=3.12" | |
| 7 | +license = { text = "Research code: MIT. Data: FirstRate Data private license (not redistributed)." } | |
| 8 | +authors = [{ name = "Simon-Pierre Boucher", email = "simon-pierre.boucher@uqo.ca" }] | |
| 9 | +dependencies = [ | |
| 10 | + "httpx>=0.27", | |
| 11 | + "pyarrow>=17", | |
| 12 | + "polars>=1.0", | |
| 13 | + "duckdb>=1.0", | |
| 14 | + "numpy>=1.26", | |
| 15 | + "pandas>=2.2", | |
| 16 | + "scikit-learn>=1.5", | |
| 17 | + "lightgbm>=4.3", | |
| 18 | + "statsmodels>=0.14", | |
| 19 | + "scipy>=1.13", | |
| 20 | + "pyyaml>=6", | |
| 21 | + "matplotlib>=3.9", | |
| 22 | + "exchange-calendars>=4.5", | |
| 23 | +] | |
| 24 | + | |
| 25 | +[project.optional-dependencies] | |
| 26 | +dev = ["pytest>=8", "pytest-cov", "ruff"] | |
| 27 | + | |
| 28 | +[build-system] | |
| 29 | +requires = ["setuptools>=68", "wheel"] | |
| 30 | +build-backend = "setuptools.build_meta" | |
| 31 | + | |
| 32 | +[tool.setuptools.packages.find] | |
| 33 | +where = ["src"] | |
| 34 | + | |
| 35 | +[tool.pytest.ini_options] | |
| 36 | +testpaths = ["tests"] | |
| 37 | +addopts = "-q" | |
| 38 | +markers = ["live: tests that call the live FinData API (skipped without WP39_LIVE=1)"] | |
| 39 | + | |
| 40 | +[tool.ruff] | |
| 41 | +line-length = 110 | |
| 42 | +target-version = "py312" | |
added
scripts/00_probe_findata.py
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 0: authenticated probe of FinData OS — contract reconciliation, no raw data, no credentials printed.""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import json | |
| 6 | +import sys | |
| 7 | +from pathlib import Path | |
| 8 | + | |
| 9 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 10 | +from wp39.api_client import FinDataClient # noqa: E402 | |
| 11 | +from wp39.catalog import probe # noqa: E402 | |
| 12 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 13 | +from wp39.provenance import git_sha, now_iso, write_json # noqa: E402 | |
| 14 | + | |
| 15 | + | |
| 16 | +def main() -> int: | |
| 17 | + P = Paths.default() | |
| 18 | + log = get_logger("00_probe", P.logs / "00_probe.log") | |
| 19 | + contract = load_config("data_contract") | |
| 20 | + client = FinDataClient(log_path=P.logs / "api_calls.jsonl") | |
| 21 | + rep = probe(client, contract) | |
| 22 | + rep["probed_at"], rep["code_sha"] = now_iso(), git_sha(P.root) | |
| 23 | + # spot-check fundamentals + export schema on one well-known entity, counts only | |
| 24 | + f = client.fundamentals("frd:AAPL", period_type="q", limit=20000) | |
| 25 | + rep["aapl_quarterly_facts"] = len(f.get("data", [])) | |
| 26 | + rep["aapl_metrics"] = sorted({r["metric"] for r in f.get("data", [])}) | |
| 27 | + rep["pit_policy"] = f.get("meta", {}).get("pit_policy") | |
| 28 | + periods = client.fundamentals_periods("frd:AAPL") | |
| 29 | + rep["aapl_periods"] = len(periods) | |
| 30 | + rep["aapl_first_filing"] = min(p["filing_date"] for p in periods if p.get("filing_date")) | |
| 31 | + write_json(P.logs / "phase0_probe.json", rep) | |
| 32 | + log.info("health=%s datasets=%d schema_ok=%s earnings_route=%s", rep["health_status"], rep["n_datasets"], | |
| 33 | + rep["bars_fields_ok"], rep["has_earnings_route"]) | |
| 34 | + for k, v in rep["required_datasets"].items(): | |
| 35 | + log.info("dataset %s: %s", k, v) | |
| 36 | + print(json.dumps({k: rep[k] for k in ("health_status", "n_datasets", "bars_fields_ok", "has_earnings_route", | |
| 37 | + "aapl_quarterly_facts", "aapl_periods", "lake_free_gb")}, indent=1)) | |
| 38 | + return 0 | |
| 39 | + | |
| 40 | + | |
| 41 | +if __name__ == "__main__": | |
| 42 | + raise SystemExit(main()) | |
added
scripts/01_audit_universe.py
+83 −0
@@ -0,0 +1,83 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 2a: universe audit — fundamentals entities x 1-minute coverage x SEC crosswalk; exclusions documented. | |
| 3 | + | |
| 4 | +Inputs: node extracts produced by scripts/01a_extract_node_tables.sh (entities, earnings_events, corporate_actions — | |
| 5 | +read-only Postgres copies, documented exception), FinData /coverage/symbols and /instruments, SEC public files. | |
| 6 | +""" | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import json | |
| 10 | +import sys | |
| 11 | +from pathlib import Path | |
| 12 | + | |
| 13 | +import polars as pl | |
| 14 | + | |
| 15 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 16 | +from wp39.api_client import FinDataClient # noqa: E402 | |
| 17 | +from wp39.instruments import build_crosswalk, fetch_sec_submissions, fetch_sec_tickers # noqa: E402 | |
| 18 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 19 | +from wp39.provenance import now_iso, sha256_file, write_json # noqa: E402 | |
| 20 | + | |
| 21 | + | |
| 22 | +def main() -> int: | |
| 23 | + P = Paths.default() | |
| 24 | + log = get_logger("01_audit", P.logs / "01_audit.log") | |
| 25 | + cfg = load_config("research"); contract = load_config("data_contract") | |
| 26 | + ext = P.cache / "external" | |
| 27 | + ents = pl.read_csv(ext / "entities.csv") | |
| 28 | + ents = ents.with_columns(pl.col("entity_id").str.replace("^frd:", "").alias("ticker")) | |
| 29 | + log.info("fundamentals entities: %d", ents.height) | |
| 30 | + | |
| 31 | + client = FinDataClient(log_path=P.logs / "api_calls.jsonl") | |
| 32 | + cov = pl.DataFrame(list(client.coverage_symbols(contract["findata"]["datasets"]["bars_1m"], min_sessions=1))) | |
| 33 | + cov = cov.rename({"symbol": "ticker", "sessions": "n_sessions_1m", "d0": "first_session_1m", "d1": "last_session_1m", "bars": "n_bars_1m"}) | |
| 34 | + cov = cov.with_columns([pl.col("first_session_1m").str.to_date(), pl.col("last_session_1m").str.to_date()]) | |
| 35 | + log.info("1m coverage rows: %d (incl. %d delisted)", cov.height, cov.filter(pl.col("ticker").str.ends_with("-DELISTED")).height) | |
| 36 | + | |
| 37 | + sec = fetch_sec_tickers(ext) | |
| 38 | + u = ents.join(cov, on="ticker", how="left") | |
| 39 | + u = u.join(sec.unique(subset=["ticker"]), on="ticker", how="left") | |
| 40 | + subs = fetch_sec_submissions(sorted(u["cik"].drop_nulls().unique().to_list()), ext, log=log) | |
| 41 | + u = u.join(subs, on="cik", how="left") | |
| 42 | + u = u.with_columns([ | |
| 43 | + pl.col("first_q").str.to_date(), pl.col("last_q").str.to_date(), pl.col("first_filing").str.to_date(), | |
| 44 | + pl.col("cik").is_not_null().alias("has_cik"), | |
| 45 | + pl.col("sic_division").fill_null("unknown"), | |
| 46 | + ]) | |
| 47 | + # eligibility (frozen): >= min quarters, 1m coverage overlapping 2012+, SIC not public administration | |
| 48 | + u = u.with_columns([ | |
| 49 | + (pl.col("n_q") >= cfg["sample"]["min_quarters_per_firm"]).alias("ok_quarters"), | |
| 50 | + (pl.col("n_sessions_1m").fill_null(0) >= 250).alias("ok_coverage"), | |
| 51 | + (pl.col("last_session_1m") >= pl.date(2012, 6, 30)).fill_null(False).alias("ok_period"), | |
| 52 | + (pl.col("sic_division") != "J_public_admin").alias("ok_sic"), | |
| 53 | + ]) | |
| 54 | + u = u.with_columns((pl.col("ok_quarters") & pl.col("ok_coverage") & pl.col("ok_period") & pl.col("ok_sic")).alias("eligible")) | |
| 55 | + u.write_parquet(P.processed / "universe.parquet") | |
| 56 | + cascade = [ | |
| 57 | + {"step": "fundamentals_entities", "n": u.height}, | |
| 58 | + {"step": ">= 8 quarterly periods", "n": u.filter(pl.col("ok_quarters")).height}, | |
| 59 | + {"step": "+ 1m bars (>= 250 sessions)", "n": u.filter(pl.col("ok_quarters") & pl.col("ok_coverage")).height}, | |
| 60 | + {"step": "+ bars after 2012-06", "n": u.filter(pl.col("ok_quarters") & pl.col("ok_coverage") & pl.col("ok_period")).height}, | |
| 61 | + {"step": "+ not public administration", "n": u.filter(pl.col("eligible")).height}, | |
| 62 | + ] | |
| 63 | + rep = { | |
| 64 | + "built_at": now_iso(), "entities": u.height, "eligible": u.filter(pl.col("eligible")).height, | |
| 65 | + "with_cik": u.filter(pl.col("has_cik")).height, "with_sic": u.filter(pl.col("sic").is_not_null()).height, | |
| 66 | + "sic_division_counts": u.filter(pl.col("eligible")).group_by("sic_division").len().sort("len", descending=True).to_dicts(), | |
| 67 | + "first_q_year_counts": u.filter(pl.col("eligible")).with_columns(pl.col("first_q").dt.year().alias("y")).group_by("y").len().sort("y").to_dicts(), | |
| 68 | + "exchanges": u.filter(pl.col("eligible")).explode("exchanges").group_by("exchanges").len().sort("len", descending=True).head(8).to_dicts(), | |
| 69 | + "delisted_symbols_in_1m_lake": int(cov.filter(pl.col("ticker").str.ends_with("-DELISTED")).height), | |
| 70 | + "delisted_entities_with_fundamentals": 0, | |
| 71 | + "cascade": cascade, | |
| 72 | + "n_former_names_gt0": u.filter(pl.col("eligible") & (pl.col("n_former_names") > 0)).height, | |
| 73 | + "sec_tickers_snapshot_sha256": sha256_file(ext / "sec_company_tickers.json"), | |
| 74 | + } | |
| 75 | + write_json(P.results / "universe_audit.json", rep) | |
| 76 | + pl.DataFrame(cascade).write_csv(P.results / "table_universe_cascade.csv") | |
| 77 | + log.info("eligible firms: %d / %d; with CIK %d; with SIC %d", rep["eligible"], rep["entities"], rep["with_cik"], rep["with_sic"]) | |
| 78 | + print(json.dumps({k: rep[k] for k in ("entities", "eligible", "with_cik", "with_sic", "cascade")}, indent=1, default=str)) | |
| 79 | + return 0 | |
| 80 | + | |
| 81 | + | |
| 82 | +if __name__ == "__main__": | |
| 83 | + raise SystemExit(main()) | |
added
scripts/01a_extract_node_tables.sh
+16 −0
@@ -0,0 +1,16 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# Phase 2a (run ON the FinData storage node M3U96a): read-only Postgres extracts of three tables the API does not expose. | |
| 3 | +# Documented exception (docs/API_INTEGRATION.md). Outputs go to $WP39_CACHE/external and are hashed into manifests. | |
| 4 | +set -euo pipefail | |
| 5 | +OUT="${WP39_CACHE:-$HOME/wp39-cache}/external"; mkdir -p "$OUT" | |
| 6 | +MAN="$(cd "$(dirname "$0")/.." && pwd)/artifacts/manifests"; mkdir -p "$MAN" | |
| 7 | +psql -d findata -Atc "\copy (select entity_id, security_id, min(fiscal_period_end) first_q, max(fiscal_period_end) last_q, count(distinct fiscal_period_end) n_q, min(filing_date) first_filing from fundamentals_facts where fiscal_period_type='q' and metric='rev' group by 1,2 order by 1) to '$OUT/entities.csv' csv header" | |
| 8 | +psql -d findata -Atc "\copy (select entity_id, event_date, eps_estimate, eps_reported, release_time, source_id from earnings_events where event_date <= date '2026-10-30' order by 1,2) to '$OUT/earnings_events.csv' csv header" | |
| 9 | +psql -d findata -Atc "\copy (select symbol, action_type, ex_date, ratio, amount from corporate_actions order by 1,3) to '$OUT/corporate_actions.csv' csv header" 2>/dev/null || \ | |
| 10 | +psql -d findata -Atc "\copy (select * from corporate_actions order by 1) to '$OUT/corporate_actions_raw.csv' csv header" | |
| 11 | +for f in entities earnings_events corporate_actions corporate_actions_raw; do | |
| 12 | + [ -f "$OUT/$f.csv" ] || continue | |
| 13 | + rows=$(($(wc -l < "$OUT/$f.csv") - 1)); sha=$(shasum -a 256 "$OUT/$f.csv" | cut -d' ' -f1) | |
| 14 | + printf '{"table":"%s","rows":%d,"sha256":"%s","extracted_at":"%s","source":"findata postgres (read-only copy)","license":"frd-private"}\n' "$f" "$rows" "$sha" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$MAN/node_extract_$f.json" | |
| 15 | + echo "$f: $rows rows" | |
| 16 | +done | |
added
scripts/02_build_pit_panel.py
+75 −0
@@ -0,0 +1,75 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 2b: fundamentals extraction per eligible entity (API, cached), wide quarterly panel, reveal dates, labels. | |
| 3 | +Also freezes the research design (hash of docs/RESEARCH_DESIGN.md + configs) before any model is fitted.""" | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import json | |
| 7 | +import sys | |
| 8 | +from concurrent.futures import ThreadPoolExecutor | |
| 9 | +from pathlib import Path | |
| 10 | + | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 14 | +from wp39.api_client import FinDataClient # noqa: E402 | |
| 15 | +from wp39.labels import build_quarter_table, finalize_labels, match_release_dates # noqa: E402 | |
| 16 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 17 | +from wp39.pit_fundamentals import add_quarter_index, fetch_entity_facts, wide_quarterly # noqa: E402 | |
| 18 | +from wp39.provenance import git_sha, now_iso, sha256_file, write_json # noqa: E402 | |
| 19 | + | |
| 20 | + | |
| 21 | +def main() -> int: | |
| 22 | + P = Paths.default() | |
| 23 | + log = get_logger("02_pit", P.logs / "02_pit.log") | |
| 24 | + cfg = load_config("research") | |
| 25 | + # design freeze | |
| 26 | + freeze = {"frozen_at": now_iso(), "code_sha": git_sha(P.root), | |
| 27 | + "files": {f: sha256_file(P.root / f) for f in ("docs/RESEARCH_DESIGN.md", "docs/PIT_PROTOCOL.md", | |
| 28 | + "configs/research.yaml", "configs/models.yaml")}} | |
| 29 | + fz = P.manifests / "design_freeze.json" | |
| 30 | + if not fz.exists(): | |
| 31 | + write_json(fz, freeze) | |
| 32 | + uni = pl.read_parquet(P.processed / "universe.parquet").filter(pl.col("eligible")) | |
| 33 | + ents = uni["entity_id"].to_list() | |
| 34 | + log.info("fetching quarterly facts for %d entities", len(ents)) | |
| 35 | + man = P.manifests / "fundamentals_manifest.jsonl" | |
| 36 | + | |
| 37 | + def one(e: str): | |
| 38 | + c = FinDataClient(log_path=P.logs / "api_calls.jsonl") | |
| 39 | + try: | |
| 40 | + return fetch_entity_facts(c, e, P.cache / "fundamentals" / f"{e.replace(':', '_')}.parquet", man) | |
| 41 | + finally: | |
| 42 | + c.close() | |
| 43 | + | |
| 44 | + frames = [] | |
| 45 | + with ThreadPoolExecutor(max_workers=6) as ex: | |
| 46 | + for i, df in enumerate(ex.map(one, ents)): | |
| 47 | + frames.append(df) | |
| 48 | + if i % 500 == 0: | |
| 49 | + log.info("fundamentals %d/%d", i, len(ents)) | |
| 50 | + facts = pl.concat([f for f in frames if f.height], how="diagonal_relaxed") | |
| 51 | + log.info("facts: %d rows, revisions>1: %d", facts.height, facts.filter(pl.col("revision_id") > 1).height) | |
| 52 | + wide = add_quarter_index(wide_quarterly(facts)) | |
| 53 | + events = pl.read_csv(P.cache / "external" / "earnings_events.csv", try_parse_dates=True) | |
| 54 | + wide = match_release_dates(wide, events) | |
| 55 | + q = finalize_labels(build_quarter_table(wide), threshold=cfg["targets"]["primary"]["threshold"]) | |
| 56 | + q.write_parquet(P.processed / "quarters.parquet") | |
| 57 | + rep = { | |
| 58 | + "built_at": now_iso(), "entities": q["entity_id"].n_unique(), "firm_quarters": q.height, | |
| 59 | + "release_match": q.group_by("release_match").len().sort("len", descending=True).to_dicts(), | |
| 60 | + "release_time": q.group_by("release_time").len().sort("len", descending=True).to_dicts(), | |
| 61 | + "filing_minus_release_days_quantiles": {str(p): q["filing_minus_release_days"].quantile(p) for p in (0.1, 0.5, 0.9)}, | |
| 62 | + "label_base_rates_all": {y: float(q[y].mean()) for y in ("y_adverse_opinc", "y_adverse_netinc", "y_roa_decline", | |
| 63 | + "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise")}, | |
| 64 | + "eps_miss_rate_where_available": float(q["y_eps_miss"].drop_nulls().mean()) if q["y_eps_miss"].drop_nulls().len() else None, | |
| 65 | + "eps_estimate_available_share": float(q["eps_estimate"].is_not_null().mean()), | |
| 66 | + "S_opinc_quantiles": {str(p): q["S_opinc"].drop_nulls().quantile(p) for p in (0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99)}, | |
| 67 | + "regular_calendar_share": float((q["_gap_ok8"] == 8).mean()), | |
| 68 | + } | |
| 69 | + write_json(P.results / "pit_panel_audit.json", rep) | |
| 70 | + print(json.dumps(rep, indent=1, default=str)) | |
| 71 | + return 0 | |
| 72 | + | |
| 73 | + | |
| 74 | +if __name__ == "__main__": | |
| 75 | + raise SystemExit(main()) | |
added
scripts/03_build_features.py
+116 −0
@@ -0,0 +1,116 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 2c: streaming export -> session features at 1m/5m/30m/1d -> rolling windows, per symbol (multiprocess). | |
| 3 | +Raw Parquet is deleted after the feature rows are written; manifests keep hashes and dataset versions.""" | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import argparse | |
| 7 | +import datetime as dt | |
| 8 | +import json | |
| 9 | +import os | |
| 10 | +import sys | |
| 11 | +import time | |
| 12 | +from concurrent.futures import ProcessPoolExecutor, as_completed | |
| 13 | +from pathlib import Path | |
| 14 | + | |
| 15 | +import polars as pl | |
| 16 | + | |
| 17 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 18 | +from wp39.api_client import FinDataClient # noqa: E402 | |
| 19 | +from wp39.corporate_actions import load_actions # noqa: E402 | |
| 20 | +from wp39.features import compute_spy_buckets, compute_symbol_features, session_bounds_table # noqa: E402 | |
| 21 | +from wp39.ingest import export_symbol, mark_consumed # noqa: E402 | |
| 22 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 23 | +from wp39.provenance import append_jsonl, now_iso, write_json # noqa: E402 | |
| 24 | +from wp39.samples import rolling_window_features # noqa: E402 | |
| 25 | + | |
| 26 | +START, END = dt.date(2011, 1, 1), dt.date(2026, 10, 30) | |
| 27 | + | |
| 28 | + | |
| 29 | +def work(symbol: str, keep_raw: bool) -> dict: | |
| 30 | + P = Paths.default() | |
| 31 | + contract = load_config("data_contract") | |
| 32 | + ds = contract["findata"]["datasets"]["bars_1m"] | |
| 33 | + out = P.cache / "features" / f"{symbol}.parquet" | |
| 34 | + win_out = P.cache / "windows" / f"{symbol}.parquet" | |
| 35 | + win_out.parent.mkdir(exist_ok=True) | |
| 36 | + if out.exists() and win_out.exists(): | |
| 37 | + return {"symbol": symbol, "status": "cached"} | |
| 38 | + t0 = time.time() | |
| 39 | + client = FinDataClient(log_path=P.logs / f"api_calls_worker_{os.getpid()}.jsonl") | |
| 40 | + try: | |
| 41 | + paths = export_symbol(client, symbol, START, END, ds, P.cache / "bars", P.manifests / "bars_manifest.jsonl", keep=keep_raw) | |
| 42 | + finally: | |
| 43 | + client.close() | |
| 44 | + if not paths: | |
| 45 | + return {"symbol": symbol, "status": "no_export"} | |
| 46 | + bounds = pl.read_parquet(P.cache / "external" / "session_bounds.parquet") | |
| 47 | + spy = {k: pl.read_parquet(P.cache / "external" / f"spy_buckets_{k}.parquet") for k in ("1m", "5m", "30m")} | |
| 48 | + actions = pl.read_parquet(P.cache / "external" / "actions.parquet") | |
| 49 | + feat = compute_symbol_features(paths, symbol, bounds, spy, actions) | |
| 50 | + if feat.height == 0: | |
| 51 | + return {"symbol": symbol, "status": "empty"} | |
| 52 | + feat.write_parquet(out) | |
| 53 | + rolling_window_features(feat).write_parquet(win_out) | |
| 54 | + if not keep_raw: | |
| 55 | + for p in paths: | |
| 56 | + p.unlink(missing_ok=True) | |
| 57 | + mark_consumed(P.manifests / "bars_manifest.jsonl", symbol, ds) | |
| 58 | + return {"symbol": symbol, "status": "ok", "sessions": feat.height, "first": str(feat["session_date"].min()), | |
| 59 | + "last": str(feat["session_date"].max()), "secs": round(time.time() - t0, 1)} | |
| 60 | + | |
| 61 | + | |
| 62 | +def main() -> int: | |
| 63 | + ap = argparse.ArgumentParser() | |
| 64 | + ap.add_argument("--workers", type=int, default=6) | |
| 65 | + ap.add_argument("--limit", type=int, default=0, help="pilot: only the first N eligible symbols") | |
| 66 | + ap.add_argument("--symbols", type=str, default="", help="comma-separated explicit list") | |
| 67 | + ap.add_argument("--keep-raw", action="store_true") | |
| 68 | + args = ap.parse_args() | |
| 69 | + P = Paths.default() | |
| 70 | + log = get_logger("03_features", P.logs / "03_features.log") | |
| 71 | + contract = load_config("data_contract") | |
| 72 | + ext = P.cache / "external" | |
| 73 | + # shared inputs: session bounds, SPY buckets, corporate actions | |
| 74 | + if not (ext / "session_bounds.parquet").exists(): | |
| 75 | + session_bounds_table(START, END).write_parquet(ext / "session_bounds.parquet") | |
| 76 | + if not (ext / "spy_buckets_1m.parquet").exists(): | |
| 77 | + client = FinDataClient(log_path=P.logs / "api_calls.jsonl") | |
| 78 | + paths = export_symbol(client, "SPY", START, END, contract["findata"]["datasets"]["bars_1m_etf"], P.cache / "bars", | |
| 79 | + P.manifests / "bars_manifest.jsonl", keep=True) | |
| 80 | + client.close() | |
| 81 | + bounds = pl.read_parquet(ext / "session_bounds.parquet") | |
| 82 | + for k, v in compute_spy_buckets(paths, bounds).items(): | |
| 83 | + v.write_parquet(ext / f"spy_buckets_{k}.parquet") | |
| 84 | + log.info("SPY buckets written") | |
| 85 | + if not (ext / "actions.parquet").exists(): | |
| 86 | + load_actions(ext / "corporate_actions_raw.csv").write_parquet(ext / "actions.parquet") | |
| 87 | + if args.symbols: | |
| 88 | + symbols = args.symbols.split(",") | |
| 89 | + else: | |
| 90 | + uni = pl.read_parquet(P.processed / "universe.parquet").filter(pl.col("eligible")).sort("ticker") | |
| 91 | + symbols = uni["ticker"].to_list() | |
| 92 | + if args.limit: | |
| 93 | + symbols = symbols[: args.limit] | |
| 94 | + log.info("symbols to process: %d (workers=%d)", len(symbols), args.workers) | |
| 95 | + status = P.logs / "03_features_status.jsonl" | |
| 96 | + done = {json.loads(l)["symbol"] for l in open(status)} if status.exists() else set() | |
| 97 | + todo = [s for s in symbols if s not in done] | |
| 98 | + t0 = time.time(); n_ok = 0 | |
| 99 | + with ProcessPoolExecutor(max_workers=args.workers) as ex: | |
| 100 | + futs = {ex.submit(work, s, args.keep_raw): s for s in todo} | |
| 101 | + for i, f in enumerate(as_completed(futs)): | |
| 102 | + try: | |
| 103 | + r = f.result() | |
| 104 | + except Exception as exc: # noqa: BLE001 | |
| 105 | + r = {"symbol": futs[f], "status": "error", "error": str(exc)[:300]} | |
| 106 | + r["ts"] = now_iso() | |
| 107 | + append_jsonl(status, r) | |
| 108 | + n_ok += r["status"] in ("ok", "cached") | |
| 109 | + if i % 50 == 0: | |
| 110 | + log.info("%d/%d done (ok=%d) elapsed=%.0fs last=%s", i + 1, len(todo), n_ok, time.time() - t0, r) | |
| 111 | + log.info("finished: %d symbols, %.0fs", len(todo), time.time() - t0) | |
| 112 | + return 0 | |
| 113 | + | |
| 114 | + | |
| 115 | +if __name__ == "__main__": | |
| 116 | + raise SystemExit(main()) | |
added
scripts/04_build_labels.py
+83 −0
@@ -0,0 +1,83 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 2d/3a: cohorts (event-time per horizon + fixed calendar), PIT joins, exclusion cascade, base rates.""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import datetime as dt | |
| 6 | +import json | |
| 7 | +import sys | |
| 8 | +from pathlib import Path | |
| 9 | + | |
| 10 | +import polars as pl | |
| 11 | + | |
| 12 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 13 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 14 | +from wp39.provenance import now_iso, write_json # noqa: E402 | |
| 15 | +from wp39.samples import (apply_admissibility, calendar_cutoffs, event_cutoffs, exclusion_cascade, join_fundamentals, # noqa: E402 | |
| 16 | + join_market) | |
| 17 | + | |
| 18 | + | |
| 19 | +def hist_features(sample: pl.DataFrame, quarters: pl.DataFrame) -> pl.DataFrame: | |
| 20 | + """Information age, last available surprise, mean of last 4 surprises, trailing adverse rate — all from quarters | |
| 21 | + available at the cutoff (pit_qidx and earlier).""" | |
| 22 | + qq = quarters.select(["entity_id", "qidx", "S_opinc", "y_adverse_opinc", "A_q"]).sort(["entity_id", "qidx"]) | |
| 23 | + qq = qq.with_columns([ | |
| 24 | + pl.col("S_opinc").alias("last_surprise"), | |
| 25 | + pl.col("S_opinc").rolling_mean(window_size=4, min_samples=2).over("entity_id").alias("mean_surprise4"), | |
| 26 | + pl.col("y_adverse_opinc").cast(pl.Float64).rolling_mean(window_size=8, min_samples=2).over("entity_id").alias("hist_adverse_rate"), | |
| 27 | + pl.col("A_q").alias("A_pit"), | |
| 28 | + ]).select(["entity_id", pl.col("qidx").alias("pit_qidx"), "last_surprise", "mean_surprise4", "hist_adverse_rate", "A_pit"]) | |
| 29 | + s = sample.join(qq, on=["entity_id", "pit_qidx"], how="left") | |
| 30 | + return s.with_columns((pl.col("cutoff") - pl.col("A_pit")).dt.total_days().alias("info_age_td")) | |
| 31 | + | |
| 32 | + | |
| 33 | +def build(design: str, cfg: dict, P: Paths, quarters: pl.DataFrame, windows: pl.DataFrame, sym: pl.DataFrame, log) -> pl.DataFrame: | |
| 34 | + if design == "event": | |
| 35 | + cut = event_cutoffs(quarters, cfg["cutoffs"]["event_time_horizons_td"]) | |
| 36 | + else: | |
| 37 | + s0, s1 = dt.date.fromisoformat(str(cfg["sample"]["start"])), dt.date.fromisoformat(str(cfg["sample"]["end"])) | |
| 38 | + cut = calendar_cutoffs(quarters, s0, s1) | |
| 39 | + cut = apply_admissibility(cut) | |
| 40 | + log.info("%s cutoffs after admissibility: %d", design, cut.height) | |
| 41 | + j = join_market(cut, windows, sym) | |
| 42 | + j = join_fundamentals(j, quarters) | |
| 43 | + # attach the outcome of quarter q (the row keyed by entity/fiscal_period_end) | |
| 44 | + ycols = ["S_opinc", "S_netinc", "S_ocf", "S_rev", "ocf_innov", "roa_yoy", "y_adverse_opinc", "y_adverse_netinc", "y_roa_decline", | |
| 45 | + "y_ocf_negative", "y_liquidity_deterioration", "y_eps_miss", "y_abs_surprise", "assets_tot_lag1", "_gap_ok8", | |
| 46 | + "release_time", "eps_estimate"] | |
| 47 | + j = j.join(quarters.select(["entity_id", "fiscal_period_end"] + ycols), on=["entity_id", "fiscal_period_end"], how="left") | |
| 48 | + j = hist_features(j, quarters) | |
| 49 | + sample, cascade = exclusion_cascade(j, cfg["sample"]["min_price_usd"], cfg["sample"]["min_valid_days_in_window"]) | |
| 50 | + cascade.write_csv(P.results / f"table_sample_cascade_{design}.csv") | |
| 51 | + log.info("%s sample: %d rows; cascade=%s", design, sample.height, cascade.to_dicts()) | |
| 52 | + return sample | |
| 53 | + | |
| 54 | + | |
| 55 | +def main() -> int: | |
| 56 | + P = Paths.default() | |
| 57 | + log = get_logger("04_labels", P.logs / "04_labels.log") | |
| 58 | + cfg = load_config("research") | |
| 59 | + quarters = pl.read_parquet(P.processed / "quarters.parquet") | |
| 60 | + uni = pl.read_parquet(P.processed / "universe.parquet").filter(pl.col("eligible")) | |
| 61 | + sym = uni.select(["entity_id", "ticker", "sic_division", "sic"]).rename({"ticker": "symbol"}) | |
| 62 | + wins = [p for p in (P.cache / "windows").glob("*.parquet") if p.stem in set(sym["symbol"].to_list())] | |
| 63 | + windows = pl.concat([pl.read_parquet(p) for p in wins], how="diagonal_relaxed") | |
| 64 | + log.info("windows: %d rows, %d symbols", windows.height, len(wins)) | |
| 65 | + quarters = quarters.filter(pl.col("entity_id").is_in(sym["entity_id"].to_list())) | |
| 66 | + out = {} | |
| 67 | + for design in ("event", "calendar"): | |
| 68 | + s = build(design, cfg, P, quarters, windows, sym, log) | |
| 69 | + s.write_parquet(P.processed / f"sample_{design}.parquet") | |
| 70 | + out[design] = { | |
| 71 | + "rows": s.height, "firms": s["entity_id"].n_unique(), "quarters": s.select(["entity_id", "fiscal_period_end"]).unique().height, | |
| 72 | + "by_h": s.group_by("h").agg([pl.len().alias("n"), pl.col("y_adverse_opinc").mean().alias("base_rate")]).sort("h").to_dicts() if design == "event" else None, | |
| 73 | + "base_rates": {y: float(s[y].mean()) for y in ("y_adverse_opinc", "y_adverse_netinc", "y_roa_decline", "y_ocf_negative", | |
| 74 | + "y_liquidity_deterioration", "y_abs_surprise")}, | |
| 75 | + "years": s.with_columns(pl.col("cutoff").dt.year().alias("y")).group_by("y").len().sort("y").to_dicts(), | |
| 76 | + } | |
| 77 | + write_json(P.results / "sample_audit.json", {"built_at": now_iso(), **out}) | |
| 78 | + print(json.dumps(out, indent=1, default=str)) | |
| 79 | + return 0 | |
| 80 | + | |
| 81 | + | |
| 82 | +if __name__ == "__main__": | |
| 83 | + raise SystemExit(main()) | |
added
scripts/05_train_baselines.py
+108 −0
@@ -0,0 +1,108 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 3: baseline ladder B0-B5 (base rates, logistic ridge per information set) on the chronological split, every | |
| 3 | +horizon, primary + secondary targets; walk-forward out-of-fold predictions; firm hold-out variant. | |
| 4 | +Predictions are saved per (design, h, target, model) so that 07_evaluate.py computes every metric from files.""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import json | |
| 8 | +import sys | |
| 9 | +from pathlib import Path | |
| 10 | + | |
| 11 | +import numpy as np | |
| 12 | +import polars as pl | |
| 13 | + | |
| 14 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 15 | +from wp39.models import INFO_SETS, Preprocessor, base_rate_predict, columns_for, fit_logistic, fit_ridge, firm_history_predict, predict_proba # noqa: E402 | |
| 16 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 17 | +from wp39.provenance import now_iso, write_json # noqa: E402 | |
| 18 | +from wp39.samples import feature_columns # noqa: E402 | |
| 19 | +from wp39.splits import chronological, firm_holdout_mask, overlap_check, walk_forward # noqa: E402 | |
| 20 | + | |
| 21 | +KEY = ["entity_id", "symbol", "fiscal_period_end", "h", "cutoff", "A_q", "sic_division", "release_time"] | |
| 22 | +TARGETS = ["y_adverse_opinc", "y_adverse_netinc", "y_roa_decline", "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise"] | |
| 23 | + | |
| 24 | + | |
| 25 | +def save_preds(P: Paths, df: pl.DataFrame, p: np.ndarray, tag: str, design: str, h: int, target: str, model: str, part: str) -> None: | |
| 26 | + out = df.select(KEY + [target]).with_columns([pl.Series("p", p), pl.lit(model).alias("model"), pl.lit(part).alias("part"), | |
| 27 | + pl.lit(tag).alias("tag")]) | |
| 28 | + d = P.evaluation / "preds"; d.mkdir(exist_ok=True) | |
| 29 | + out.write_parquet(d / f"{design}_h{h}_{target}_{model}_{part}_{tag}.parquet") | |
| 30 | + | |
| 31 | + | |
| 32 | +def run_split(P, cfg, mcfg, fc, design, h, target, train, valid, test, tag, log, info_sets, record): | |
| 33 | + ytr, yva, yte = (x[target].to_numpy().astype(int) for x in (train, valid, test)) | |
| 34 | + # B0 | |
| 35 | + save_preds(P, test, base_rate_predict(ytr, test.height), tag, design, h, target, "B0", "test") | |
| 36 | + save_preds(P, valid, base_rate_predict(ytr, valid.height), tag, design, h, target, "B0", "validation") | |
| 37 | + save_preds(P, test, firm_history_predict(train, test, target), tag, design, h, target, "B0b", "test") | |
| 38 | + for name in info_sets: | |
| 39 | + cols = columns_for(name, fc) | |
| 40 | + pp = Preprocessor(cols).fit(train) | |
| 41 | + Xtr, Xva, Xte = pp.transform(train), pp.transform(valid), pp.transform(test) | |
| 42 | + m, info = fit_logistic(Xtr, ytr, Xva, yva, mcfg["logistic"]["C_grid"], cfg["seed"]) | |
| 43 | + save_preds(P, valid, predict_proba(m, Xva), tag, design, h, target, name, "validation") | |
| 44 | + save_preds(P, test, predict_proba(m, Xte), tag, design, h, target, name, "test") | |
| 45 | + record.append({"design": design, "h": h, "target": target, "model": name, "tag": tag, "n_train": train.height, | |
| 46 | + "n_valid": valid.height, "n_test": test.height, "n_features": len(cols), **info, | |
| 47 | + "coef": dict(zip(cols, m.coef_[0].round(4).tolist()))}) | |
| 48 | + log.info("%s h=%d %s %s %s: C=%s val_ll=%.4f (p=%d)", design, h, target, name, tag, info["C"], info["val_logloss"], len(cols)) | |
| 49 | + | |
| 50 | + | |
| 51 | +def main() -> int: | |
| 52 | + P = Paths.default() | |
| 53 | + log = get_logger("05_baselines", P.logs / "05_baselines.log") | |
| 54 | + cfg, mcfg = load_config("research"), load_config("models") | |
| 55 | + record = [] | |
| 56 | + for design in ("event", "calendar"): | |
| 57 | + df = pl.read_parquet(P.processed / f"sample_{design}.parquet") | |
| 58 | + if design == "calendar": | |
| 59 | + df = df.with_columns(pl.lit(0).alias("h")) # calendar: h varies; pooled model | |
| 60 | + fc = feature_columns(df) | |
| 61 | + write_json(P.evaluation / f"feature_columns_{design}.json", fc) | |
| 62 | + horizons = sorted(df["h"].unique().to_list()) if design == "event" else [0] | |
| 63 | + for h in horizons: | |
| 64 | + d = df.filter(pl.col("h") == h) if design == "event" else df | |
| 65 | + targets = TARGETS if (design == "event" and h == cfg["cutoffs"]["primary_horizon_td"]) else ["y_adverse_opinc"] | |
| 66 | + for target in targets: | |
| 67 | + dd = d.filter(pl.col(target).is_not_null()) | |
| 68 | + sp = chronological(dd, cfg["splits"]) | |
| 69 | + chk = overlap_check(sp["train"], sp["test"]) | |
| 70 | + assert chk["shared_observations"] == 0 and chk["train_outcomes_revealed_in_test"] == 0, chk | |
| 71 | + sets = list(INFO_SETS) if target == "y_adverse_opinc" else ["B1", "B2", "B5"] | |
| 72 | + run_split(P, cfg, mcfg, fc, design, h, target, sp["train"], sp["validation"], sp["test"], "chrono", log, sets, record) | |
| 73 | + if design == "event" and h == cfg["cutoffs"]["primary_horizon_td"] and target == "y_adverse_opinc": | |
| 74 | + # continuous target (ridge) on the same split | |
| 75 | + for name in ("B2", "B5"): | |
| 76 | + cols = columns_for(name, fc); pp = Preprocessor(cols).fit(sp["train"]) | |
| 77 | + ytr = sp["train"]["S_opinc"].to_numpy(); lo, hi = np.quantile(ytr, [0.01, 0.99]); ytr = np.clip(ytr, lo, hi) | |
| 78 | + m, info = fit_ridge(pp.transform(sp["train"]), ytr, pp.transform(sp["validation"]), np.clip(sp["validation"]["S_opinc"].to_numpy(), lo, hi)) | |
| 79 | + save_preds(P, sp["test"], m.predict(pp.transform(sp["test"])), "chrono_cont", design, h, "S_opinc", name, "test") | |
| 80 | + record.append({"design": design, "h": h, "target": "S_opinc", "model": name, "tag": "chrono_cont", "winsor": [float(lo), float(hi)], | |
| 81 | + "train_mean": float(ytr.mean()), **info}) | |
| 82 | + # firm hold-out: train/validate on non-held-out firms, test on both groups separately | |
| 83 | + hold = firm_holdout_mask(dd["entity_id"], cfg["splits"]["firm_holdout_fraction"]) | |
| 84 | + dd2 = dd.with_columns(pl.Series("_hold", hold)) | |
| 85 | + sp2 = chronological(dd2.filter(~pl.col("_hold")), cfg["splits"]) | |
| 86 | + te_hold = dd2.filter(pl.col("_hold") & (pl.col("cutoff") >= sp2["test"]["cutoff"].min())) | |
| 87 | + run_split(P, cfg, mcfg, fc, design, h, target, sp2["train"], sp2["validation"], te_hold, "firmhold_heldout", log, ["B2", "B5"], record) | |
| 88 | + run_split(P, cfg, mcfg, fc, design, h, target, sp2["train"], sp2["validation"], sp2["test"], "firmhold_seen", log, ["B2", "B5"], record) | |
| 89 | + # walk-forward out-of-fold predictions (logistic, C fixed at the chronological choice per set) | |
| 90 | + for name in ("B1", "B2", "B3", "B4", "B5"): | |
| 91 | + cols = columns_for(name, fc) | |
| 92 | + C = next(r["C"] for r in record if r["model"] == name and r["tag"] == "chrono" and r["h"] == h and r["target"] == target) | |
| 93 | + parts = [] | |
| 94 | + for y, tr, te in walk_forward(dd, cfg["splits"], 2026): | |
| 95 | + pp = Preprocessor(cols).fit(tr) | |
| 96 | + from sklearn.linear_model import LogisticRegression | |
| 97 | + m = LogisticRegression(C=C, max_iter=2000, random_state=cfg["seed"]).fit(pp.transform(tr), tr[target].to_numpy().astype(int)) | |
| 98 | + parts.append(te.select(KEY + [target]).with_columns([pl.Series("p", predict_proba(m, pp.transform(te))), pl.lit(y).alias("fold")])) | |
| 99 | + oof = pl.concat(parts).with_columns([pl.lit(name).alias("model"), pl.lit("oof").alias("part"), pl.lit("walkforward").alias("tag")]) | |
| 100 | + oof.write_parquet(P.evaluation / "preds" / f"{design}_h{h}_{target}_{name}_oof_walkforward.parquet") | |
| 101 | + log.info("walk-forward %s: %d oof rows", name, oof.height) | |
| 102 | + write_json(P.evaluation / "baselines_fit_log.json", {"run_at": now_iso(), "fits": record}) | |
| 103 | + print(json.dumps({"fits": len(record)}, indent=1)) | |
| 104 | + return 0 | |
| 105 | + | |
| 106 | + | |
| 107 | +if __name__ == "__main__": | |
| 108 | + raise SystemExit(main()) | |
added
scripts/06_train_advanced.py
+116 −0
@@ -0,0 +1,116 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 4: LightGBM (B6) with an identical random-search budget for every information set on the chronological split | |
| 3 | +(primary horizon, primary target + secondary targets), horizon ladder and walk-forward refits with a fixed configuration | |
| 4 | +(no tuning inside folds, so earlier folds never see later data). No neural/text models (DECISIONS D-006).""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import json | |
| 8 | +import sys | |
| 9 | +from pathlib import Path | |
| 10 | + | |
| 11 | +import numpy as np | |
| 12 | +import polars as pl | |
| 13 | + | |
| 14 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 15 | +from wp39.models import INFO_SETS, Preprocessor, columns_for, fit_lightgbm, predict_proba # noqa: E402 | |
| 16 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 17 | +from wp39.provenance import now_iso, write_json # noqa: E402 | |
| 18 | +from wp39.samples import feature_columns # noqa: E402 | |
| 19 | +from wp39.splits import chronological, firm_holdout_mask, walk_forward # noqa: E402 | |
| 20 | + | |
| 21 | +KEY = ["entity_id", "symbol", "fiscal_period_end", "h", "cutoff", "A_q", "sic_division", "release_time"] | |
| 22 | +FIXED = {"num_leaves": 15, "learning_rate": 0.03, "min_child_samples": 100, "feature_fraction": 0.7, "bagging_fraction": 0.8, | |
| 23 | + "bagging_freq": 1, "lambda_l2": 1.0, "n_estimators": 600, "verbose": -1, "n_jobs": 12} | |
| 24 | + | |
| 25 | + | |
| 26 | +def save(P, df, p, design, h, target, model, part, tag, extra=None): | |
| 27 | + out = df.select(KEY + [target]).with_columns([pl.Series("p", p), pl.lit(model).alias("model"), pl.lit(part).alias("part"), pl.lit(tag).alias("tag")]) | |
| 28 | + if extra is not None: | |
| 29 | + out = out.with_columns(extra) | |
| 30 | + out.write_parquet(P.evaluation / "preds" / f"{design}_h{h}_{target}_{model}_{part}_{tag}.parquet") | |
| 31 | + | |
| 32 | + | |
| 33 | +def tuned(P, cfg, mcfg, fc, design, h, target, sp, tag, sets, log, record): | |
| 34 | + ytr, yva = sp["train"][target].to_numpy().astype(int), sp["validation"][target].to_numpy().astype(int) | |
| 35 | + for name in sets: | |
| 36 | + cols = columns_for(name, fc); pp = Preprocessor(cols).fit(sp["train"]) | |
| 37 | + m, info = fit_lightgbm(pp.transform(sp["train"]), ytr, pp.transform(sp["validation"]), yva, mcfg["lightgbm"], cfg["seed"], n_jobs=12) | |
| 38 | + save(P, sp["validation"], predict_proba(m, pp.transform(sp["validation"])), design, h, target, "L" + name, "validation", tag) | |
| 39 | + save(P, sp["test"], predict_proba(m, pp.transform(sp["test"])), design, h, target, "L" + name, "test", tag) | |
| 40 | + imp = dict(zip(cols, (m.booster_.feature_importance("gain") / max(m.booster_.feature_importance("gain").sum(), 1)).round(4).tolist())) | |
| 41 | + record.append({"design": design, "h": h, "target": target, "model": "L" + name, "tag": tag, "n_features": len(cols), | |
| 42 | + "val_score": info["val_score"], "params": info["params"], "trials": len(info["trials"]), "importance_gain": imp}) | |
| 43 | + log.info("%s h=%d %s L%s %s: val_ll=%.4f params=%s", design, h, target, name, tag, info["val_score"], info["params"]) | |
| 44 | + | |
| 45 | + | |
| 46 | +def fixed_fit(train, test, cols, target, seed): | |
| 47 | + import lightgbm as lgb | |
| 48 | + pp = Preprocessor(cols).fit(train) | |
| 49 | + # early stopping on the last 4 calendar quarters of the training window (purged by construction: still before test) | |
| 50 | + cut = train["cutoff"].quantile(0.85) | |
| 51 | + tr, va = train.filter(pl.col("cutoff") < cut), train.filter(pl.col("cutoff") >= cut) | |
| 52 | + m = lgb.LGBMClassifier(random_state=seed, **FIXED) | |
| 53 | + m.fit(pp.transform(tr), tr[target].to_numpy().astype(int), eval_set=[(pp.transform(va), va[target].to_numpy().astype(int))], | |
| 54 | + callbacks=[lgb.early_stopping(50, verbose=False)]) | |
| 55 | + return predict_proba(m, pp.transform(test)), int(m.best_iteration_ or 0) | |
| 56 | + | |
| 57 | + | |
| 58 | +def main() -> int: | |
| 59 | + P = Paths.default() | |
| 60 | + log = get_logger("06_advanced", P.logs / "06_advanced.log") | |
| 61 | + cfg, mcfg = load_config("research"), load_config("models") | |
| 62 | + hp = cfg["cutoffs"]["primary_horizon_td"] | |
| 63 | + record = [] | |
| 64 | + df = pl.read_parquet(P.processed / "sample_event.parquet") | |
| 65 | + fc = feature_columns(df) | |
| 66 | + # 1) primary: tuned LightGBM, all information sets, h = 20, primary target | |
| 67 | + d = df.filter((pl.col("h") == hp) & pl.col("y_adverse_opinc").is_not_null()) | |
| 68 | + sp = chronological(d, cfg["splits"]) | |
| 69 | + tuned(P, cfg, mcfg, fc, "event", hp, "y_adverse_opinc", sp, "chrono", list(INFO_SETS), log, record) | |
| 70 | + # 2) secondary targets (B2 vs B5 only), tuned | |
| 71 | + for target in ("y_adverse_netinc", "y_roa_decline", "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise"): | |
| 72 | + dt_ = df.filter((pl.col("h") == hp) & pl.col(target).is_not_null()) | |
| 73 | + tuned(P, cfg, mcfg, fc, "event", hp, target, chronological(dt_, cfg["splits"]), "chrono", ["B2", "B5"], log, record) | |
| 74 | + # 3) horizon ladder (fixed config): B2 vs B5 at every h on the chronological split | |
| 75 | + for h in sorted(df["h"].unique().to_list()): | |
| 76 | + dh = df.filter((pl.col("h") == h) & pl.col("y_adverse_opinc").is_not_null()) | |
| 77 | + sph = chronological(dh, cfg["splits"]) | |
| 78 | + for name in ("B2", "B5"): | |
| 79 | + p, it = fixed_fit(pl.concat([sph["train"], sph["validation"]]), sph["test"], columns_for(name, fc), "y_adverse_opinc", cfg["seed"]) | |
| 80 | + save(P, sph["test"], p, "event", h, "y_adverse_opinc", "L" + name, "test", "fixed") | |
| 81 | + record.append({"design": "event", "h": h, "target": "y_adverse_opinc", "model": "L" + name, "tag": "fixed", "best_iteration": it}) | |
| 82 | + # 4) firm hold-out (fixed config) | |
| 83 | + hold = firm_holdout_mask(d["entity_id"], cfg["splits"]["firm_holdout_fraction"]) | |
| 84 | + d2 = d.with_columns(pl.Series("_hold", hold)) | |
| 85 | + sp2 = chronological(d2.filter(~pl.col("_hold")), cfg["splits"]) | |
| 86 | + te_hold = d2.filter(pl.col("_hold") & (pl.col("cutoff") >= sp2["test"]["cutoff"].min())) | |
| 87 | + for name in ("B2", "B5"): | |
| 88 | + trv = pl.concat([sp2["train"], sp2["validation"]]) | |
| 89 | + p, _ = fixed_fit(trv, te_hold, columns_for(name, fc), "y_adverse_opinc", cfg["seed"]); save(P, te_hold, p, "event", hp, "y_adverse_opinc", "L" + name, "test", "firmhold_heldout") | |
| 90 | + p, _ = fixed_fit(trv, sp2["test"], columns_for(name, fc), "y_adverse_opinc", cfg["seed"]); save(P, sp2["test"], p, "event", hp, "y_adverse_opinc", "L" + name, "test", "firmhold_seen") | |
| 91 | + # 5) walk-forward out-of-fold (fixed config) for B1..B5 at h = 20 -> LFDS for the panel regressions / H4 / H5 | |
| 92 | + for name in ("B1", "B2", "B3", "B4", "B5"): | |
| 93 | + parts = [] | |
| 94 | + for y, tr, te in walk_forward(d, cfg["splits"], 2026): | |
| 95 | + p, it = fixed_fit(tr, te, columns_for(name, fc), "y_adverse_opinc", cfg["seed"]) | |
| 96 | + parts.append(te.select(KEY + ["y_adverse_opinc"]).with_columns([pl.Series("p", p), pl.lit(y).alias("fold")])) | |
| 97 | + oof = pl.concat(parts).with_columns([pl.lit("L" + name).alias("model"), pl.lit("oof").alias("part"), pl.lit("walkforward").alias("tag")]) | |
| 98 | + oof.write_parquet(P.evaluation / "preds" / f"event_h{hp}_y_adverse_opinc_L{name}_oof_walkforward.parquet") | |
| 99 | + log.info("walk-forward L%s: %d oof rows", name, oof.height) | |
| 100 | + # 6) calendar design: walk-forward OOF for B2/B5 (scores for H5 portfolio sorts) | |
| 101 | + dc = pl.read_parquet(P.processed / "sample_calendar.parquet").with_columns(pl.lit(0).alias("h")) | |
| 102 | + fcc = feature_columns(dc) | |
| 103 | + for name in ("B2", "B5"): | |
| 104 | + parts = [] | |
| 105 | + for y, tr, te in walk_forward(dc, cfg["splits"], 2026): | |
| 106 | + p, _ = fixed_fit(tr, te, columns_for(name, fcc), "y_adverse_opinc", cfg["seed"]) | |
| 107 | + parts.append(te.select(KEY + ["y_adverse_opinc"]).with_columns([pl.Series("p", p), pl.lit(y).alias("fold")])) | |
| 108 | + oof = pl.concat(parts).with_columns([pl.lit("L" + name).alias("model"), pl.lit("oof").alias("part"), pl.lit("walkforward").alias("tag")]) | |
| 109 | + oof.write_parquet(P.evaluation / "preds" / f"calendar_h0_y_adverse_opinc_L{name}_oof_walkforward.parquet") | |
| 110 | + write_json(P.evaluation / "advanced_fit_log.json", {"run_at": now_iso(), "fixed_config": FIXED, "fits": record}) | |
| 111 | + print(json.dumps({"fits": len(record)}, indent=1)) | |
| 112 | + return 0 | |
| 113 | + | |
| 114 | + | |
| 115 | +if __name__ == "__main__": | |
| 116 | + raise SystemExit(main()) | |
added
scripts/07_evaluate.py
+209 −0
@@ -0,0 +1,209 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 3/4 evaluation: every metric and every paired contrast is computed from saved prediction files. | |
| 3 | +Outputs (artifacts/evaluation + results/): metrics_all.csv, contrasts.csv, horizon_ladder.csv, secondary_targets.csv, | |
| 4 | +firm_holdout.csv, walkforward_by_year.csv, reliability.csv, continuous.csv, power.json.""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import json | |
| 8 | +import sys | |
| 9 | +from pathlib import Path | |
| 10 | + | |
| 11 | +import numpy as np | |
| 12 | +import pandas as pd | |
| 13 | +import polars as pl | |
| 14 | + | |
| 15 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 16 | +from wp39.inference import (binary_metrics, continuous_metrics, dm_gw_test, holm, paired_block_bootstrap, # noqa: E402 | |
| 17 | + pointwise_logloss, quarter_blocks, reliability_table) | |
| 18 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 19 | +from wp39.provenance import now_iso, write_json # noqa: E402 | |
| 20 | + | |
| 21 | +CONTRASTS = [("B5", "B2"), ("B4", "B2"), ("B3", "B2"), ("B5", "B4"), ("B5", "B3"), ("B5all", "B2"), ("B5all", "B5"), ("B2", "B1"), ("B1", "B0")] | |
| 22 | +FAMILIES = {"freq_vs_daily": [("B5", "B2"), ("B4", "B2"), ("B3", "B2")], "fine_vs_coarse": [("B5", "B4"), ("B5", "B3")]} | |
| 23 | + | |
| 24 | + | |
| 25 | +def load_preds(P: Paths) -> pl.DataFrame: | |
| 26 | + files = sorted((P.evaluation / "preds").glob("*.parquet")) | |
| 27 | + frames = [] | |
| 28 | + for f in files: | |
| 29 | + d = pl.read_parquet(f) | |
| 30 | + target = [c for c in d.columns if c.startswith("y_") or c == "S_opinc"][0] | |
| 31 | + d = d.rename({target: "y"}).with_columns(pl.lit(target).alias("target")) | |
| 32 | + if "fold" not in d.columns: | |
| 33 | + d = d.with_columns(pl.lit(None, dtype=pl.Int64).alias("fold")) | |
| 34 | + frames.append(d.select(["entity_id", "symbol", "fiscal_period_end", "h", "cutoff", "A_q", "sic_division", "release_time", | |
| 35 | + "y", "p", "model", "part", "tag", "target", "fold"])) | |
| 36 | + return pl.concat(frames) | |
| 37 | + | |
| 38 | + | |
| 39 | +def contrast(a: pl.DataFrame, b: pl.DataFrame, n_boot: int, seed: int) -> dict: | |
| 40 | + key = ["entity_id", "fiscal_period_end", "cutoff"] | |
| 41 | + j = a.select(key + ["y", "p", "A_q"]).join(b.select(key + [pl.col("p").alias("pb")]), on=key, how="inner") | |
| 42 | + y = j["y"].to_numpy().astype(int); pa, pb = j["p"].to_numpy(), j["pb"].to_numpy() | |
| 43 | + d_ll = pointwise_logloss(y, pa) - pointwise_logloss(y, pb) | |
| 44 | + d_br = (pa - y) ** 2 - (pb - y) ** 2 | |
| 45 | + blocks = quarter_blocks(j["A_q"].to_numpy()) | |
| 46 | + bll, bbr = paired_block_bootstrap(d_ll, blocks, n_boot, seed), paired_block_bootstrap(d_br, blocks, n_boot, seed) | |
| 47 | + gw = dm_gw_test(d_ll, blocks) | |
| 48 | + return {"n": int(len(y)), "d_logloss": bll["mean"], "ci_lo": bll["ci_lo"], "ci_hi": bll["ci_hi"], "p_boot": bll["p_boot"], | |
| 49 | + "gw_t": gw["t"], "gw_p": gw["p"], "n_quarters": gw["T"], "d_brier": bbr["mean"], "brier_ci_lo": bbr["ci_lo"], "brier_ci_hi": bbr["ci_hi"]} | |
| 50 | + | |
| 51 | + | |
| 52 | +def main() -> int: | |
| 53 | + P = Paths.default() | |
| 54 | + log = get_logger("07_eval", P.logs / "07_eval.log") | |
| 55 | + cfg = load_config("research") | |
| 56 | + hp, nb, seed = cfg["cutoffs"]["primary_horizon_td"], cfg["inference"]["n_bootstrap"], cfg["seed"] | |
| 57 | + preds = load_preds(P) | |
| 58 | + log.info("prediction rows: %d (%d files)", preds.height, len(list((P.evaluation / "preds").glob("*.parquet")))) | |
| 59 | + | |
| 60 | + # 1) metrics for every binary prediction set | |
| 61 | + rows = [] | |
| 62 | + grp = preds.filter(pl.col("target") != "S_opinc").with_columns(pl.col("cutoff").dt.year().alias("year")) | |
| 63 | + for keys, g in grp.group_by(["h", "target", "model", "part", "tag"], maintain_order=True): | |
| 64 | + h, target, model, part, tag = keys | |
| 65 | + m = binary_metrics(g["y"].to_numpy().astype(int), g["p"].to_numpy()) | |
| 66 | + rows.append({"h": h, "target": target, "model": model, "part": part, "tag": tag, **m}) | |
| 67 | + metrics = pd.DataFrame(rows).sort_values(["target", "h", "tag", "part", "model"]) | |
| 68 | + metrics.to_csv(P.evaluation / "metrics_all.csv", index=False) | |
| 69 | + | |
| 70 | + # 2) primary contrasts (h = hp, primary target, chronological test) for logistic and LightGBM | |
| 71 | + out = [] | |
| 72 | + base = preds.filter((pl.col("h") == hp) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "test") & (pl.col("tag") == "chrono")) | |
| 73 | + for prefix in ("", "L"): | |
| 74 | + pv = {} | |
| 75 | + for a, b in CONTRASTS: | |
| 76 | + A, B = base.filter(pl.col("model") == prefix + a), base.filter(pl.col("model") == (prefix + b if b != "B0" else "B0")) | |
| 77 | + if A.height == 0 or B.height == 0: | |
| 78 | + continue | |
| 79 | + c = contrast(A, B, nb, seed); c.update({"family": "logistic" if prefix == "" else "lightgbm", "a": prefix + a, "b": prefix + b}) | |
| 80 | + out.append(c); pv[(a, b)] = c["gw_p"] | |
| 81 | + for fam, pairs in FAMILIES.items(): | |
| 82 | + adj = holm({f"{a}-{b}": pv[(a, b)] for a, b in pairs if (a, b) in pv}) | |
| 83 | + for r in out: | |
| 84 | + if r["family"] == ("logistic" if prefix == "" else "lightgbm") and f"{r['a'][len(prefix):]}-{r['b'][len(prefix):]}" in adj: | |
| 85 | + r["holm_family"], r["gw_p_holm"] = fam, adj[f"{r['a'][len(prefix):]}-{r['b'][len(prefix):]}"] | |
| 86 | + contrasts = pd.DataFrame(out); contrasts.to_csv(P.evaluation / "contrasts.csv", index=False) | |
| 87 | + log.info("primary contrasts:\n%s", contrasts[["family", "a", "b", "n", "d_logloss", "ci_lo", "ci_hi", "gw_p"]].to_string()) | |
| 88 | + | |
| 89 | + # 3) horizon ladder (B5 vs B2) logistic chrono + LightGBM fixed | |
| 90 | + lad = [] | |
| 91 | + for h in sorted(preds["h"].unique().to_list()): | |
| 92 | + for fam, tag, a, b in (("logistic", "chrono", "B5", "B2"), ("lightgbm", "fixed", "LB5", "LB2")): | |
| 93 | + s = preds.filter((pl.col("h") == h) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "test") & (pl.col("tag") == tag)) | |
| 94 | + A, B = s.filter(pl.col("model") == a), s.filter(pl.col("model") == b) | |
| 95 | + if A.height and B.height: | |
| 96 | + c = contrast(A, B, nb, seed); c.update({"h": h, "family": fam}) | |
| 97 | + c["auc_a"] = binary_metrics(A["y"].to_numpy().astype(int), A["p"].to_numpy()).get("auc") | |
| 98 | + c["auc_b"] = binary_metrics(B["y"].to_numpy().astype(int), B["p"].to_numpy()).get("auc") | |
| 99 | + lad.append(c) | |
| 100 | + ladder = pd.DataFrame(lad); ladder.to_csv(P.evaluation / "horizon_ladder.csv", index=False) | |
| 101 | + h4 = {} | |
| 102 | + for fam, g in ladder.groupby("family"): | |
| 103 | + g = g.sort_values("h", ascending=False) | |
| 104 | + from scipy.stats import spearmanr | |
| 105 | + rho, p = spearmanr(-g["h"], -g["d_logloss"]) # gain (= -d) should rise as h falls | |
| 106 | + h4[fam] = {"spearman_gain_vs_minus_h": float(rho), "p": float(p), "horizons": g["h"].tolist(), "d_logloss": g["d_logloss"].round(5).tolist()} | |
| 107 | + write_json(P.evaluation / "h4_timing.json", h4) | |
| 108 | + | |
| 109 | + # 4) secondary targets (B5 vs B2) logistic + lightgbm at hp | |
| 110 | + sec = [] | |
| 111 | + for target in ("y_adverse_netinc", "y_roa_decline", "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise", "y_adverse_opinc"): | |
| 112 | + for fam, a, b in (("logistic", "B5", "B2"), ("lightgbm", "LB5", "LB2")): | |
| 113 | + s = preds.filter((pl.col("h") == hp) & (pl.col("target") == target) & (pl.col("part") == "test") & (pl.col("tag") == "chrono")) | |
| 114 | + A, B = s.filter(pl.col("model") == a), s.filter(pl.col("model") == b) | |
| 115 | + if A.height and B.height: | |
| 116 | + c = contrast(A, B, nb, seed); c.update({"target": target, "family": fam, "base_rate": float(A["y"].mean()), | |
| 117 | + "auc_a": binary_metrics(A["y"].to_numpy().astype(int), A["p"].to_numpy()).get("auc"), | |
| 118 | + "auc_b": binary_metrics(B["y"].to_numpy().astype(int), B["p"].to_numpy()).get("auc")}) | |
| 119 | + sec.append(c) | |
| 120 | + pd.DataFrame(sec).to_csv(P.evaluation / "secondary_targets.csv", index=False) | |
| 121 | + | |
| 122 | + # 5) firm hold-out | |
| 123 | + fh = [] | |
| 124 | + for tag in ("firmhold_heldout", "firmhold_seen"): | |
| 125 | + for fam, a, b in (("logistic", "B5", "B2"), ("lightgbm", "LB5", "LB2")): | |
| 126 | + s = preds.filter((pl.col("h") == hp) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "test") & (pl.col("tag") == tag)) | |
| 127 | + A, B = s.filter(pl.col("model") == a), s.filter(pl.col("model") == b) | |
| 128 | + if A.height and B.height: | |
| 129 | + c = contrast(A, B, nb, seed); c.update({"tag": tag, "family": fam}); fh.append(c) | |
| 130 | + pd.DataFrame(fh).to_csv(P.evaluation / "firm_holdout.csv", index=False) | |
| 131 | + | |
| 132 | + # 6) walk-forward by year | |
| 133 | + wf = [] | |
| 134 | + oof = preds.filter((pl.col("tag") == "walkforward") & (pl.col("h") == hp)).with_columns(pl.col("cutoff").dt.year().alias("year")) | |
| 135 | + for keys, g in oof.group_by(["model", "year"], maintain_order=True): | |
| 136 | + m = binary_metrics(g["y"].to_numpy().astype(int), g["p"].to_numpy()); wf.append({"model": keys[0], "year": keys[1], **m}) | |
| 137 | + wfd = pd.DataFrame(wf).sort_values(["model", "year"]); wfd.to_csv(P.evaluation / "walkforward_by_year.csv", index=False) | |
| 138 | + wfc = [] | |
| 139 | + for fam, a, b in (("logistic", "B5", "B2"), ("lightgbm", "LB5", "LB2")): | |
| 140 | + A, B = oof.filter(pl.col("model") == a), oof.filter(pl.col("model") == b) | |
| 141 | + if A.height and B.height: | |
| 142 | + for y in sorted(A["year"].unique().to_list()): | |
| 143 | + c = contrast(A.filter(pl.col("year") == y), B.filter(pl.col("year") == y), 300, seed); c.update({"family": fam, "year": y}); wfc.append(c) | |
| 144 | + c = contrast(A, B, nb, seed); c.update({"family": fam, "year": "all"}); wfc.append(c) | |
| 145 | + pd.DataFrame(wfc).to_csv(P.evaluation / "walkforward_contrasts.csv", index=False) | |
| 146 | + | |
| 147 | + # 7) reliability (test, chrono) for B2, B5, LB2, LB5 | |
| 148 | + rel = [] | |
| 149 | + for model in ("B2", "B5", "LB2", "LB5"): | |
| 150 | + s = base.filter(pl.col("model") == model) | |
| 151 | + if s.height: | |
| 152 | + t = reliability_table(s["y"].to_numpy().astype(int), s["p"].to_numpy()); t["model"] = model; rel.append(t) | |
| 153 | + if rel: | |
| 154 | + pd.concat(rel).to_csv(P.evaluation / "reliability.csv", index=False) | |
| 155 | + | |
| 156 | + # 8) continuous target | |
| 157 | + cont = [] | |
| 158 | + fitlog = json.load(open(P.evaluation / "baselines_fit_log.json"))["fits"] | |
| 159 | + for model in ("B2", "B5"): | |
| 160 | + s = preds.filter((pl.col("target") == "S_opinc") & (pl.col("model") == model) & (pl.col("tag") == "chrono_cont")) | |
| 161 | + if s.height: | |
| 162 | + rec = next(r for r in fitlog if r.get("tag") == "chrono_cont" and r["model"] == model) | |
| 163 | + lo, hi = rec["winsor"] | |
| 164 | + y = np.clip(s["y"].to_numpy(), lo, hi) | |
| 165 | + cont.append({"model": model, **continuous_metrics(y, s["p"].to_numpy(), rec["train_mean"])}) | |
| 166 | + if cont: | |
| 167 | + A = preds.filter((pl.col("target") == "S_opinc") & (pl.col("model") == "B5") & (pl.col("tag") == "chrono_cont")) | |
| 168 | + B = preds.filter((pl.col("target") == "S_opinc") & (pl.col("model") == "B2") & (pl.col("tag") == "chrono_cont")) | |
| 169 | + key = ["entity_id", "fiscal_period_end", "cutoff"] | |
| 170 | + j = A.join(B.select(key + [pl.col("p").alias("pb")]), on=key) | |
| 171 | + lo, hi = next(r for r in fitlog if r.get("tag") == "chrono_cont" and r["model"] == "B5")["winsor"] | |
| 172 | + y = np.clip(j["y"].to_numpy(), lo, hi) | |
| 173 | + d = (j["p"].to_numpy() - y) ** 2 - (j["pb"].to_numpy() - y) ** 2 | |
| 174 | + bb = paired_block_bootstrap(d, quarter_blocks(j["A_q"].to_numpy()), nb, seed) | |
| 175 | + cont.append({"model": "B5-B2 d_mse", **bb}) | |
| 176 | + pd.DataFrame(cont).to_csv(P.evaluation / "continuous.csv", index=False) | |
| 177 | + | |
| 178 | + # 9) power (validation period, logistic B5 vs B2): MDE at 80 % power, alpha 5 %, quarter clustering | |
| 179 | + v = preds.filter((pl.col("h") == hp) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "validation") & (pl.col("tag") == "chrono")) | |
| 180 | + A, B = v.filter(pl.col("model") == "B5"), v.filter(pl.col("model") == "B2") | |
| 181 | + power = {} | |
| 182 | + if A.height and B.height: | |
| 183 | + key = ["entity_id", "fiscal_period_end", "cutoff"] | |
| 184 | + j = A.join(B.select(key + [pl.col("p").alias("pb")]), on=key) | |
| 185 | + y = j["y"].to_numpy().astype(int) | |
| 186 | + d = pointwise_logloss(y, j["p"].to_numpy()) - pointwise_logloss(y, j["pb"].to_numpy()) | |
| 187 | + bl = quarter_blocks(j["A_q"].to_numpy()); ub = np.unique(bl) | |
| 188 | + dq = np.array([d[bl == b].mean() for b in ub]); se = dq.std(ddof=1) / np.sqrt(len(ub)) | |
| 189 | + n_test_q = preds.filter((pl.col("part") == "test") & (pl.col("tag") == "chrono") & (pl.col("h") == hp))["A_q"].to_numpy() | |
| 190 | + T_test = len(np.unique(quarter_blocks(n_test_q))) if len(n_test_q) else 23 | |
| 191 | + se_test = dq.std(ddof=1) / np.sqrt(T_test) | |
| 192 | + power = {"validation_quarters": int(len(ub)), "sd_quarterly_diff": float(dq.std(ddof=1)), "test_quarters_assumed": int(T_test), | |
| 193 | + "mde_logloss_80pct_power": float((1.96 + 0.84) * se_test), "validation_mean_diff": float(d.mean())} | |
| 194 | + write_json(P.evaluation / "power.json", power) | |
| 195 | + | |
| 196 | + # copies to results/ | |
| 197 | + for f in ("metrics_all", "contrasts", "horizon_ladder", "secondary_targets", "firm_holdout", "walkforward_by_year", | |
| 198 | + "walkforward_contrasts", "reliability", "continuous"): | |
| 199 | + src = P.evaluation / f"{f}.csv" | |
| 200 | + if src.exists(): | |
| 201 | + (P.results / f"{f}.csv").write_text(src.read_text()) | |
| 202 | + write_json(P.results / "evaluation_summary.json", {"run_at": now_iso(), "power": power, "h4": h4, | |
| 203 | + "primary": contrasts.to_dict(orient="records") if len(contrasts) else []}) | |
| 204 | + print(contrasts[["family", "a", "b", "n", "d_logloss", "ci_lo", "ci_hi", "gw_p"]].to_string() if len(contrasts) else "no contrasts") | |
| 205 | + return 0 | |
| 206 | + | |
| 207 | + | |
| 208 | +if __name__ == "__main__": | |
| 209 | + raise SystemExit(main()) | |
added
scripts/08_run_robustness.py
+156 −0
@@ -0,0 +1,156 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 5: negative controls, subgroup stress tests (H3), panel regressions (H3), pricing tests (H5).""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import json | |
| 6 | +import sys | |
| 7 | +from pathlib import Path | |
| 8 | + | |
| 9 | +import numpy as np | |
| 10 | +import pandas as pd | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 14 | +from wp39.inference import binary_metrics, paired_block_bootstrap, pointwise_logloss, quarter_blocks, two_way_cluster_ols # noqa: E402 | |
| 15 | +from wp39.models import Preprocessor, columns_for, predict_proba # noqa: E402 | |
| 16 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 17 | +from wp39.portfolio import car_window, download_french, factor_alpha, monthly_decile_portfolios, read_french_daily # noqa: E402 | |
| 18 | +from wp39.provenance import now_iso, write_json # noqa: E402 | |
| 19 | +from wp39.robustness import add_terciles, shuffle_labels_within_quarter, subgroup_masks # noqa: E402 | |
| 20 | +from wp39.samples import feature_columns # noqa: E402 | |
| 21 | +from wp39.splits import chronological # noqa: E402 | |
| 22 | + | |
| 23 | +KEY = ["entity_id", "fiscal_period_end", "cutoff"] | |
| 24 | +FIXED = {"num_leaves": 15, "learning_rate": 0.03, "min_child_samples": 100, "feature_fraction": 0.7, "bagging_fraction": 0.8, | |
| 25 | + "bagging_freq": 1, "lambda_l2": 1.0, "n_estimators": 600, "verbose": -1, "n_jobs": 12} | |
| 26 | + | |
| 27 | + | |
| 28 | +def fixed_fit(train, test, cols, target, seed): | |
| 29 | + import lightgbm as lgb | |
| 30 | + pp = Preprocessor(cols).fit(train) | |
| 31 | + cut = train["cutoff"].quantile(0.85) | |
| 32 | + tr, va = train.filter(pl.col("cutoff") < cut), train.filter(pl.col("cutoff") >= cut) | |
| 33 | + m = lgb.LGBMClassifier(random_state=seed, **FIXED) | |
| 34 | + m.fit(pp.transform(tr), tr[target].to_numpy().astype(int), eval_set=[(pp.transform(va), va[target].to_numpy().astype(int))], | |
| 35 | + callbacks=[lgb.early_stopping(50, verbose=False)]) | |
| 36 | + return predict_proba(m, pp.transform(test)) | |
| 37 | + | |
| 38 | + | |
| 39 | +def paired(A: pl.DataFrame, B: pl.DataFrame, y: str, nb: int, seed: int) -> dict: | |
| 40 | + j = A.select(KEY + [y, "p", "A_q"]).join(B.select(KEY + [pl.col("p").alias("pb")]), on=KEY) | |
| 41 | + yy = j[y].to_numpy().astype(int) | |
| 42 | + d = pointwise_logloss(yy, j["p"].to_numpy()) - pointwise_logloss(yy, j["pb"].to_numpy()) | |
| 43 | + r = paired_block_bootstrap(d, quarter_blocks(j["A_q"].to_numpy()), nb, seed) | |
| 44 | + r["auc_a"] = binary_metrics(yy, j["p"].to_numpy()).get("auc"); r["auc_b"] = binary_metrics(yy, j["pb"].to_numpy()).get("auc") | |
| 45 | + r["base_rate"] = float(yy.mean()) | |
| 46 | + return r | |
| 47 | + | |
| 48 | + | |
| 49 | +def main() -> int: | |
| 50 | + P = Paths.default() | |
| 51 | + log = get_logger("08_robust", P.logs / "08_robust.log") | |
| 52 | + cfg = load_config("research"); hp, nb, seed = cfg["cutoffs"]["primary_horizon_td"], cfg["inference"]["n_bootstrap"], cfg["seed"] | |
| 53 | + y = "y_adverse_opinc" | |
| 54 | + df = pl.read_parquet(P.processed / "sample_event.parquet").filter((pl.col("h") == hp) & pl.col(y).is_not_null()) | |
| 55 | + fc = feature_columns(df) | |
| 56 | + sp = chronological(df, cfg["splits"]) | |
| 57 | + trv = pl.concat([sp["train"], sp["validation"]]) | |
| 58 | + | |
| 59 | + # ---------------------------------------------------------------- placebo 1: labels shuffled within calendar quarter | |
| 60 | + plc = [] | |
| 61 | + for s in range(3): | |
| 62 | + shuf_tr = shuffle_labels_within_quarter(trv, y, seed + s); shuf_te = shuffle_labels_within_quarter(sp["test"], y, seed + 100 + s) | |
| 63 | + A = shuf_te.with_columns(pl.Series("p", fixed_fit(shuf_tr, shuf_te, columns_for("B5", fc), y, seed))) | |
| 64 | + B = shuf_te.with_columns(pl.Series("p", fixed_fit(shuf_tr, shuf_te, columns_for("B2", fc), y, seed))) | |
| 65 | + r = paired(A, B, y, nb, seed); r.update({"placebo": "shuffled_labels", "draw": s}); plc.append(r) | |
| 66 | + log.info("placebo shuffled labels draw %d: d=%.5f [%.5f, %.5f] auc5=%.3f", s, r["mean"], r["ci_lo"], r["ci_hi"], r["auc_a"]) | |
| 67 | + pd.DataFrame(plc).to_csv(P.evaluation / "placebo_shuffled.csv", index=False) | |
| 68 | + | |
| 69 | + # ---------------------------------------------------------------- subgroups on the chronological LightGBM test predictions | |
| 70 | + preds = {m: pl.read_parquet(P.evaluation / "preds" / f"event_h{hp}_{y}_{m}_test_chrono.parquet") for m in ("LB2", "LB5", "B2", "B5")} | |
| 71 | + tr_t, te_t = add_terciles(sp["train"], [sp["test"]]) | |
| 72 | + te_t = te_t.with_columns([ | |
| 73 | + ((pl.col("cutoff") - pl.col("A_prev")).dt.total_days() >= 35).fill_null(True).alias("window_clean"), | |
| 74 | + (pl.col("w1m_present_share") >= 0.98).alias("dense_bars"), | |
| 75 | + ]) | |
| 76 | + extra = {"window_clean": pl.col("window_clean"), "window_straddles_prev_release": ~pl.col("window_clean"), | |
| 77 | + "dense_bars": pl.col("dense_bars"), "sparse_bars": ~pl.col("dense_bars")} | |
| 78 | + sub = [] | |
| 79 | + for fam, a, b in (("lightgbm", "LB5", "LB2"), ("logistic", "B5", "B2")): | |
| 80 | + for name, mask in {**subgroup_masks(te_t), **extra}.items(): | |
| 81 | + ids = te_t.filter(mask).select(KEY) | |
| 82 | + A, B = preds[a].join(ids, on=KEY), preds[b].join(ids, on=KEY) | |
| 83 | + if A.height < 500: | |
| 84 | + continue | |
| 85 | + r = paired(A, B, y, nb, seed); r.update({"family": fam, "subgroup": name}); sub.append(r) | |
| 86 | + subdf = pd.DataFrame(sub); subdf.to_csv(P.evaluation / "subgroups.csv", index=False) | |
| 87 | + log.info("subgroups:\n%s", subdf[subdf.family == "lightgbm"][["subgroup", "n", "mean", "ci_lo", "ci_hi", "auc_a", "auc_b"]].to_string()) | |
| 88 | + | |
| 89 | + # ---------------------------------------------------------------- H3 panel regressions on walk-forward OOF signals | |
| 90 | + oof5 = pl.read_parquet(P.evaluation / "preds" / f"event_h{hp}_{y}_LB5_oof_walkforward.parquet").select(KEY + [pl.col("p").alias("lfds5")]) | |
| 91 | + oof2 = pl.read_parquet(P.evaluation / "preds" / f"event_h{hp}_{y}_LB2_oof_walkforward.parquet").select(KEY + [pl.col("p").alias("lfds2")]) | |
| 92 | + pan = df.join(oof5, on=KEY).join(oof2, on=KEY) | |
| 93 | + pan = pan.with_columns([ | |
| 94 | + (pl.col("cutoff").dt.year().cast(pl.Utf8) + "Q" + ((pl.col("cutoff").dt.month() - 1) // 3 + 1).cast(pl.Utf8)).alias("cq"), | |
| 95 | + ]) | |
| 96 | + pan = pan.with_columns((pl.col("sic_division") + "|" + pl.col("cq")).alias("sector_q")) | |
| 97 | + # ex-ante cross-sectional terciles within calendar quarter | |
| 98 | + for name, col in (("size", "f_log_mcap"), ("illiq", "wd_amihud60"), ("lev", "f_leverage")): | |
| 99 | + pan = pan.with_columns(((pl.col(col).rank("average").over("cq") - 1) / pl.len().over("cq") * 3).floor().clip(0, 2).cast(pl.Int8).alias(f"t_{name}")) | |
| 100 | + pd_ = pan.to_pandas() | |
| 101 | + lo, hi = np.quantile(pd_["S_opinc"].dropna(), [0.01, 0.99]); pd_["S_w"] = pd_["S_opinc"].clip(lo, hi) | |
| 102 | + pd_["d_lfds"] = pd_["lfds5"] - pd_["lfds2"] | |
| 103 | + for name in ("size", "illiq", "lev"): | |
| 104 | + for k in (0, 2): | |
| 105 | + pd_[f"lfds5_x_{name}{k}"] = pd_["lfds5"] * (pd_[f"t_{name}"] == k) | |
| 106 | + pd_[f"dlfds_x_{name}{k}"] = pd_["d_lfds"] * (pd_[f"t_{name}"] == k) | |
| 107 | + regs = {} | |
| 108 | + controls = ["f_roa", "f_leverage", "f_log_mcap", "wd_ret20", "wd_vol20", "last_surprise", "mean_surprise4"] | |
| 109 | + regs["S_on_lfds5"] = two_way_cluster_ols(pd_, "S_w", ["lfds5"] + controls, "entity_id", "sector_q", "entity_id", "cq") | |
| 110 | + regs["S_on_lfds2_and_dlfds"] = two_way_cluster_ols(pd_, "S_w", ["lfds2", "d_lfds"] + controls, "entity_id", "sector_q", "entity_id", "cq") | |
| 111 | + regs["y_on_lfds5"] = two_way_cluster_ols(pd_.assign(yb=pd_[y].astype(float)), "yb", ["lfds5"] + controls, "entity_id", "sector_q", "entity_id", "cq") | |
| 112 | + for name in ("size", "illiq", "lev"): | |
| 113 | + regs[f"dlfds_x_{name}"] = two_way_cluster_ols(pd_, "S_w", ["lfds2", "d_lfds", f"dlfds_x_{name}0", f"dlfds_x_{name}2"] + controls, | |
| 114 | + "entity_id", "sector_q", "entity_id", "cq") | |
| 115 | + allreg = pd.concat([r.assign(spec=k) for k, r in regs.items()]); allreg.to_csv(P.evaluation / "panel_regressions.csv", index=False) | |
| 116 | + log.info("panel:\n%s", allreg[allreg.spec.isin(["S_on_lfds5", "S_on_lfds2_and_dlfds"])][["spec", "var", "coef", "se", "t", "p", "n"]].head(12).to_string()) | |
| 117 | + | |
| 118 | + # ---------------------------------------------------------------- H5a: announcement-window CAR on the ex-ante signal | |
| 119 | + ext = P.cache / "external" | |
| 120 | + ff = read_french_daily(download_french(load_config("data_contract")["external"]["ff5_daily"], ext / "ff5_daily.zip")) | |
| 121 | + mom = read_french_daily(download_french(load_config("data_contract")["external"]["mom_daily"], ext / "mom_daily.zip")) | |
| 122 | + ff = ff.join(mom.iloc[:, :1].rename(columns={mom.columns[0]: "Mom"}), how="left") | |
| 123 | + daily = pl.concat([pl.read_parquet(p).select(["symbol", "session_date", "d_ret_cc", "d_close", "d_dollar_vol"]) for p in (P.cache / "features").glob("*.parquet")]) | |
| 124 | + syms = set(pan["symbol"].unique().to_list()) | |
| 125 | + daily = daily.filter(pl.col("symbol").is_in(list(syms))) | |
| 126 | + car = car_window(daily, ff, pan.select(["entity_id", "symbol", "cutoff", "A_q", "fiscal_period_end"])) | |
| 127 | + pc = pd_.merge(car.to_pandas(), on=["entity_id", "symbol", "cutoff", "A_q", "fiscal_period_end"], how="inner") | |
| 128 | + lo, hi = np.quantile(pc["car"].dropna(), [0.01, 0.99]); pc["car_w"] = pc["car"].clip(lo, hi) | |
| 129 | + h5 = {"car_on_lfds5": two_way_cluster_ols(pc, "car_w", ["lfds5"] + controls, "entity_id", "sector_q", "entity_id", "cq"), | |
| 130 | + "car_on_lfds2_dlfds": two_way_cluster_ols(pc, "car_w", ["lfds2", "d_lfds"] + controls, "entity_id", "sector_q", "entity_id", "cq")} | |
| 131 | + pd.concat([r.assign(spec=k) for k, r in h5.items()]).to_csv(P.evaluation / "h5_car_regressions.csv", index=False) | |
| 132 | + log.info("H5 CAR:\n%s", h5["car_on_lfds2_dlfds"][["var", "coef", "se", "t", "p", "n"]].head(3).to_string()) | |
| 133 | + | |
| 134 | + # ---------------------------------------------------------------- H5b: monthly decile sorts (calendar design, OOF scores) | |
| 135 | + port_out = {} | |
| 136 | + for m in ("LB5", "LB2"): | |
| 137 | + f = P.evaluation / "preds" / f"calendar_h0_{y}_{m}_oof_walkforward.parquet" | |
| 138 | + if not f.exists(): | |
| 139 | + continue | |
| 140 | + sig = pl.read_parquet(f).select(["symbol", "cutoff", pl.col("p").alias("score")]).to_pandas() | |
| 141 | + res = monthly_decile_portfolios(sig, daily.to_pandas(), ff, cost_bps=20.0) | |
| 142 | + port = res["portfolios"]; port.to_csv(P.evaluation / f"portfolios_{m}.csv", index=False) | |
| 143 | + port_out[m] = {"gross": factor_alpha(port, ff, 1, 10, net=False), "net20bps": factor_alpha(port, ff, 1, 10, net=True), | |
| 144 | + "decile_means": port.groupby("decile")["ret"].mean().round(5).to_dict(), | |
| 145 | + "avg_turnover": float(port["turnover"].mean())} | |
| 146 | + log.info("portfolio %s: %s", m, port_out[m]["gross"]) | |
| 147 | + write_json(P.evaluation / "h5_portfolios.json", port_out) | |
| 148 | + | |
| 149 | + for f in ("placebo_shuffled", "subgroups", "panel_regressions", "h5_car_regressions"): | |
| 150 | + (P.results / f"{f}.csv").write_text((P.evaluation / f"{f}.csv").read_text()) | |
| 151 | + write_json(P.results / "robustness_summary.json", {"run_at": now_iso(), "placebo": plc, "portfolios": port_out}) | |
| 152 | + return 0 | |
| 153 | + | |
| 154 | + | |
| 155 | +if __name__ == "__main__": | |
| 156 | + raise SystemExit(main()) | |
added
scripts/09_make_manuscript.py
+203 −0
@@ -0,0 +1,203 @@ | ||
| 1 | +#!/usr/bin/env python | |
| 2 | +"""Phase 6: generate LaTeX tables, figures and a results-macro file from saved artifacts (never from memory).""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import json | |
| 6 | +import sys | |
| 7 | +from pathlib import Path | |
| 8 | + | |
| 9 | +import numpy as np | |
| 10 | +import pandas as pd | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 14 | +from wp39 import figures as F # noqa: E402 | |
| 15 | +from wp39.pipeline import Paths, get_logger, load_config # noqa: E402 | |
| 16 | + | |
| 17 | +T = {} # macros | |
| 18 | + | |
| 19 | + | |
| 20 | +def macro(name: str, value, fmt: str = "{}"): | |
| 21 | + T[name] = fmt.format(value) | |
| 22 | + | |
| 23 | + | |
| 24 | +def fnum(x, d=4): | |
| 25 | + return "" if x is None or (isinstance(x, float) and np.isnan(x)) else f"{x:.{d}f}" | |
| 26 | + | |
| 27 | + | |
| 28 | +def stars(p): | |
| 29 | + return "***" if p < 0.01 else "**" if p < 0.05 else "*" if p < 0.1 else "" | |
| 30 | + | |
| 31 | + | |
| 32 | +def tex_table(df: pd.DataFrame, path: Path, cols: list[tuple[str, str, str]], note: str = ""): | |
| 33 | + """cols: (column, header, fmt) — fmt is a python format or 'int'/'str'.""" | |
| 34 | + lines = ["\\begin{tabular}{l" + "r" * (len(cols) - 1) + "}", "\\toprule", " & ".join(h for _, h, _ in cols) + " \\\\", "\\midrule"] | |
| 35 | + for _, r in df.iterrows(): | |
| 36 | + cells = [] | |
| 37 | + for c, _, fmt in cols: | |
| 38 | + v = r[c] | |
| 39 | + if fmt == "int": | |
| 40 | + cells.append(f"{int(v):,}" if pd.notna(v) else "") | |
| 41 | + elif fmt == "str": | |
| 42 | + cells.append(str(v).replace("_", " ").replace("%", "\\%")) | |
| 43 | + else: | |
| 44 | + cells.append(fmt.format(v) if pd.notna(v) else "") | |
| 45 | + lines.append(" & ".join(cells) + " \\\\") | |
| 46 | + lines += ["\\bottomrule", "\\end{tabular}"] | |
| 47 | + path.write_text("\n".join(lines) + "\n") | |
| 48 | + | |
| 49 | + | |
| 50 | +def main() -> int: | |
| 51 | + P = Paths.default() | |
| 52 | + log = get_logger("09_manuscript", P.logs / "09_manuscript.log") | |
| 53 | + cfg = load_config("research"); hp = cfg["cutoffs"]["primary_horizon_td"] | |
| 54 | + E, R, TB, FG = P.evaluation, P.results, P.paper_tables, P.paper_figures | |
| 55 | + ua = json.load(open(R / "universe_audit.json")); pa = json.load(open(R / "pit_panel_audit.json")); sa = json.load(open(R / "sample_audit.json")) | |
| 56 | + ev = pd.read_parquet(P.processed / "sample_event.parquet", columns=["entity_id", "fiscal_period_end", "h", "cutoff", "A_q", "y_adverse_opinc", | |
| 57 | + "S_opinc", "f_log_mcap", "wd_amihud60", "f_leverage", "release_time", "sic_division"]) | |
| 58 | + e20 = ev[ev.h == hp] | |
| 59 | + # ---------------- macros: sample | |
| 60 | + macro("nEntities", ua["entities"], "{:,}"); macro("nEligible", ua["eligible"], "{:,}"); macro("nWithCik", ua["with_cik"], "{:,}") | |
| 61 | + macro("nFirmQuarters", pa["firm_quarters"], "{:,}"); macro("nFirmsPanel", pa["entities"], "{:,}") | |
| 62 | + macro("nSampleEvent", sa["event"]["rows"], "{:,}"); macro("nSampleFirms", sa["event"]["firms"], "{:,}") | |
| 63 | + macro("nSampleQuarters", sa["event"]["quarters"], "{:,}"); macro("nSampleCalendar", sa["calendar"]["rows"], "{:,}") | |
| 64 | + macro("nObsPrimary", len(e20), "{:,}"); macro("baseRatePrimary", 100 * e20.y_adverse_opinc.mean(), "{:.1f}") | |
| 65 | + macro("firstCutoff", str(e20.cutoff.min())[:10]); macro("lastCutoff", str(e20.cutoff.max())[:10]) | |
| 66 | + macro("releaseMatchedShare", 100 * next(r["len"] for r in pa["release_match"] if r["release_match"] == "ok") / pa["firm_quarters"], "{:.1f}") | |
| 67 | + macro("filingLagMedian", pa["filing_minus_release_days_quantiles"]["0.5"], "{:.0f}") | |
| 68 | + macro("epsEstimateShare", 100 * pa["eps_estimate_available_share"], "{:.0f}") | |
| 69 | + # ---------------- Table 2: universe cascade | |
| 70 | + tex_table(pd.read_csv(R / "table_universe_cascade.csv"), TB / "tab02_universe.tex", [("step", "Step", "str"), ("n", "Firms", "int")]) | |
| 71 | + casc = pd.read_csv(R / "table_sample_cascade_event.csv") | |
| 72 | + tex_table(casc, TB / "tab02b_sample_cascade.tex", [("step", "Exclusion step", "str"), ("remaining", "Remaining", "int"), ("dropped", "Dropped", "int")]) | |
| 73 | + # ---------------- Table 3: summary statistics (h = 20 sample) | |
| 74 | + feat = pd.read_parquet(P.processed / "sample_event.parquet") | |
| 75 | + feat = feat[feat.h == hp] | |
| 76 | + desc_cols = {"S_opinc": "Scaled operating-income surprise $S_q$", "y_adverse_opinc": "Adverse surprise ($S_q \\le -0.5\\%$)", "f_roa": "ROA (latest available)", | |
| 77 | + "f_leverage": "Debt / assets", "f_log_mcap": "log market cap", "wd_ret20": "20-day return", "wd_vol20": "20-day daily volatility", | |
| 78 | + "wd_amihud60": "Amihud illiquidity (60d, $\\times 10^6$)", "w1m_rv": "1-min realized variance (daily mean)", "w1m_rj": "1-min relative jump", | |
| 79 | + "w1m_rsneg_share": "1-min negative semivariance share", "w1m_down_vol_share": "1-min down-bar volume share", "w1m_present_share": "1-min bars present (share)", | |
| 80 | + "w30m_rv": "30-min realized variance", "w5m_rv": "5-min realized variance", "info_age_td": "Days since last release"} | |
| 81 | + rows = [] | |
| 82 | + for c, lab in desc_cols.items(): | |
| 83 | + s = pd.to_numeric(feat[c], errors="coerce").replace([np.inf, -np.inf], np.nan).dropna() | |
| 84 | + rows.append({"var": lab, "n": len(s), "mean": s.mean(), "sd": s.std(), "p10": s.quantile(0.1), "p50": s.median(), "p90": s.quantile(0.9)}) | |
| 85 | + tex_table(pd.DataFrame(rows), TB / "tab03_summary.tex", [("var", "Variable", "str"), ("n", "N", "int"), ("mean", "Mean", "{:.4g}"), ("sd", "SD", "{:.4g}"), | |
| 86 | + ("p10", "P10", "{:.4g}"), ("p50", "Median", "{:.4g}"), ("p90", "P90", "{:.4g}")]) | |
| 87 | + # ---------------- Table 4: information sets | |
| 88 | + fc = json.load(open(E / "feature_columns_event.json")) | |
| 89 | + info = pd.DataFrame([{"set": "B1", "desc": "PIT fundamentals + history + size", "k": len(fc["fundamentals"])}, | |
| 90 | + {"set": "B2", "desc": "B1 + daily OHLCV (20/60-day)", "k": len(fc["fundamentals"]) + len(fc["daily"])}, | |
| 91 | + {"set": "B3", "desc": "B2 + 30-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_30m"])}, | |
| 92 | + {"set": "B4", "desc": "B2 + 5-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_5m"])}, | |
| 93 | + {"set": "B5", "desc": "B2 + 1-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_1m"])}, | |
| 94 | + {"set": "B5all", "desc": "B2 + 30m + 5m + 1m", "k": sum(len(v) for v in fc.values())}]) | |
| 95 | + tex_table(info, TB / "tab04_info_sets.tex", [("set", "Set", "str"), ("desc", "Information set", "str"), ("k", "Features", "int")]) | |
| 96 | + macro("kFund", len(fc["fundamentals"])); macro("kDaily", len(fc["daily"])); macro("kIntra", len(fc["intraday_1m"])) | |
| 97 | + # ---------------- Table 5: headline OOS metrics (h=20, chrono test) | |
| 98 | + met = pd.read_csv(E / "metrics_all.csv") | |
| 99 | + m20 = met[(met.h == hp) & (met.target == "y_adverse_opinc") & (met.part == "test") & (met.tag == "chrono")].copy() | |
| 100 | + order = ["B0", "B0b", "B1", "B2", "B3", "B4", "B5", "B5all", "LB1", "LB2", "LB3", "LB4", "LB5", "LB5all"] | |
| 101 | + m20["o"] = m20.model.map({m: i for i, m in enumerate(order)}); m20 = m20.sort_values("o") | |
| 102 | + tex_table(m20, TB / "tab05_headline.tex", [("model", "Model", "str"), ("n", "N", "int"), ("logloss", "Log loss", "{:.4f}"), ("brier", "Brier", "{:.4f}"), | |
| 103 | + ("auc", "AUC", "{:.3f}"), ("pr_auc", "PR-AUC", "{:.3f}"), ("cal_slope", "Cal. slope", "{:.2f}"), ("cal_intercept", "Cal. int.", "{:.2f}")]) | |
| 104 | + for _, r in m20.iterrows(): | |
| 105 | + macro(f"ll{r.model}", r.logloss, "{:.4f}"); macro(f"auc{r.model}", r.auc, "{:.3f}") | |
| 106 | + # ---------------- Table 5b: paired contrasts | |
| 107 | + con = pd.read_csv(E / "contrasts.csv") | |
| 108 | + con["ci"] = con.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1) | |
| 109 | + con["gwp"] = con.apply(lambda r: f"{r.gw_p:.3f}{stars(r.gw_p)}", axis=1) | |
| 110 | + con["holm"] = con["gw_p_holm"].map(lambda v: "" if pd.isna(v) else f"{v:.3f}") | |
| 111 | + tex_table(con, TB / "tab05b_contrasts.tex", [("family", "Family", "str"), ("a", "A", "str"), ("b", "B", "str"), ("n", "N", "int"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), | |
| 112 | + ("ci", "95\\% CI", "str"), ("gwp", "GW $p$", "str"), ("holm", "Holm", "str"), ("d_brier", "$\\Delta$ Brier", "{:+.5f}")]) | |
| 113 | + for _, r in con.iterrows(): | |
| 114 | + k = f"{r.family[:3]}{r.a}vs{r.b}".replace("B5all", "Ball") | |
| 115 | + macro(f"d{k}", r.d_logloss, "{:+.4f}"); macro(f"lo{k}", r.ci_lo, "{:+.4f}"); macro(f"hi{k}", r.ci_hi, "{:+.4f}"); macro(f"p{k}", r.gw_p, "{:.3f}") | |
| 116 | + macro(f"n{k}", r.n, "{:,}") | |
| 117 | + # ---------------- Table 7: horizon ladder, secondary targets, firm holdout | |
| 118 | + lad = pd.read_csv(E / "horizon_ladder.csv") | |
| 119 | + lad["ci"] = lad.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1) | |
| 120 | + tex_table(lad.sort_values(["family", "h"], ascending=[True, False]), TB / "tab07_horizons.tex", [("family", "Family", "str"), ("h", "$h$", "int"), ("n", "N", "int"), | |
| 121 | + ("auc_b", "AUC B2", "{:.3f}"), ("auc_a", "AUC B5", "{:.3f}"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")]) | |
| 122 | + h4 = json.load(open(E / "h4_timing.json")) | |
| 123 | + for fam, v in h4.items(): | |
| 124 | + macro(f"h4rho{fam}", v["spearman_gain_vs_minus_h"], "{:+.2f}"); macro(f"h4p{fam}", v["p"], "{:.2f}") | |
| 125 | + sec = pd.read_csv(E / "secondary_targets.csv") | |
| 126 | + sec["ci"] = sec.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1) | |
| 127 | + tex_table(sec, TB / "tab10_targets.tex", [("target", "Target", "str"), ("family", "Family", "str"), ("n", "N", "int"), ("base_rate", "Base rate", "{:.3f}"), | |
| 128 | + ("auc_b", "AUC B2", "{:.3f}"), ("auc_a", "AUC B5", "{:.3f}"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")]) | |
| 129 | + fh = pd.read_csv(E / "firm_holdout.csv"); fh["ci"] = fh.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1) | |
| 130 | + tex_table(fh, TB / "tab09b_firmholdout.tex", [("tag", "Firms", "str"), ("family", "Family", "str"), ("n", "N", "int"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")]) | |
| 131 | + # ---------------- Table 6: panel regressions | |
| 132 | + pr = pd.read_csv(E / "panel_regressions.csv") | |
| 133 | + pr = pr[pr["var"].isin(["lfds5", "lfds2", "d_lfds", "dlfds_x_size0", "dlfds_x_size2", "dlfds_x_illiq0", "dlfds_x_illiq2", "dlfds_x_lev0", "dlfds_x_lev2"])].copy() | |
| 134 | + pr["coef_s"] = pr.apply(lambda r: f"{r.coef:.4f}{stars(r.p)}", axis=1); pr["se_s"] = pr.se.map(lambda v: f"({v:.4f})") | |
| 135 | + tex_table(pr, TB / "tab06_panel.tex", [("spec", "Specification", "str"), ("var", "Regressor", "str"), ("coef_s", "Coef.", "str"), ("se_s", "SE", "str"), ("n", "N", "int"), ("n_firms", "Firms", "int"), ("n_quarters", "Quarters", "int")]) | |
| 136 | + for _, r in pr.iterrows(): | |
| 137 | + macro(f"pan{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.coef, "{:+.4f}"); macro(f"pant{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.t, "{:.2f}") | |
| 138 | + # ---------------- Table 7b: mechanisms / subgroups; Table 8: placebos | |
| 139 | + sub = pd.read_csv(E / "subgroups.csv"); sub["ci"] = sub.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1) | |
| 140 | + tex_table(sub[sub.family == "lightgbm"], TB / "tab07b_subgroups.tex", [("subgroup", "Subgroup", "str"), ("n", "N", "int"), ("base_rate", "Base rate", "{:.3f}"), | |
| 141 | + ("auc_b", "AUC LB2", "{:.3f}"), ("auc_a", "AUC LB5", "{:.3f}"), ("mean", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str")]) | |
| 142 | + plc = pd.read_csv(E / "placebo_shuffled.csv"); plc["ci"] = plc.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1) | |
| 143 | + tex_table(plc, TB / "tab08_placebo.tex", [("placebo", "Placebo", "str"), ("draw", "Draw", "int"), ("n", "N", "int"), ("auc_a", "AUC LB5", "{:.3f}"), ("auc_b", "AUC LB2", "{:.3f}"), ("mean", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str")]) | |
| 144 | + macro("placeboMeanD", plc["mean"].mean(), "{:+.4f}"); macro("placeboAUC", plc["auc_a"].mean(), "{:.3f}") | |
| 145 | + # ---------------- Table 11: economic value | |
| 146 | + h5 = json.load(open(E / "h5_portfolios.json")); car = pd.read_csv(E / "h5_car_regressions.csv") | |
| 147 | + rows = [] | |
| 148 | + for m, v in h5.items(): | |
| 149 | + for k in ("gross", "net20bps"): | |
| 150 | + a = v[k]; rows.append({"model": m, "variant": k, "alpha": 100 * a["alpha_monthly"], "t": a["alpha_t"], "mean_ls": 100 * a["mean_ls"], "sharpe": a["sharpe_ann"], "T": a["T"], "turnover": v["avg_turnover"]}) | |
| 151 | + tex_table(pd.DataFrame(rows), TB / "tab11_economic.tex", [("model", "Score", "str"), ("variant", "Variant", "str"), ("mean_ls", "Mean L/S (\\%/mo)", "{:.2f}"), ("alpha", "FF5+MOM $\\alpha$ (\\%/mo)", "{:+.2f}"), | |
| 152 | + ("t", "$t$ (HAC)", "{:.2f}"), ("sharpe", "Sharpe (ann.)", "{:.2f}"), ("turnover", "Turnover", "{:.2f}"), ("T", "Months", "int")]) | |
| 153 | + for m, v in h5.items(): | |
| 154 | + macro(f"alpha{m}gross", 100 * v["gross"]["alpha_monthly"], "{:+.2f}"); macro(f"alphat{m}gross", v["gross"]["alpha_t"], "{:.2f}") | |
| 155 | + macro(f"alpha{m}net", 100 * v["net20bps"]["alpha_monthly"], "{:+.2f}"); macro(f"alphat{m}net", v["net20bps"]["alpha_t"], "{:.2f}") | |
| 156 | + carr = car[car["var"].isin(["lfds5", "lfds2", "d_lfds"])].copy(); carr["coef_s"] = carr.apply(lambda r: f"{r.coef:.4f}{stars(r.p)}", axis=1); carr["se_s"] = carr.se.map(lambda v: f"({v:.4f})") | |
| 157 | + tex_table(carr, TB / "tab11b_car.tex", [("spec", "Specification", "str"), ("var", "Regressor", "str"), ("coef_s", "Coef.", "str"), ("se_s", "SE", "str"), ("n", "N", "int")]) | |
| 158 | + for _, r in carr.iterrows(): | |
| 159 | + macro(f"car{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.coef, "{:+.4f}"); macro(f"cart{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.t, "{:.2f}") | |
| 160 | + # continuous + power | |
| 161 | + cont = pd.read_csv(E / "continuous.csv") | |
| 162 | + tex_table(cont[cont.model.isin(["B2", "B5"])], TB / "tab10b_continuous.tex", [("model", "Model", "str"), ("n", "N", "int"), ("mae", "MAE", "{:.5f}"), ("rmse", "RMSE", "{:.5f}"), ("oos_r2", "OOS $R^2$", "{:.4f}")]) | |
| 163 | + for _, r in cont.iterrows(): | |
| 164 | + if r.model in ("B2", "B5"): | |
| 165 | + macro(f"r2{r.model}", r.oos_r2, "{:.4f}") | |
| 166 | + pw = json.load(open(E / "power.json")); macro("mde", pw.get("mde_logloss_80pct_power", float("nan")), "{:.4f}"); macro("valQuarters", pw.get("validation_quarters", 0)) | |
| 167 | + # ---------------- figures | |
| 168 | + F.fig_timeline(FG) | |
| 169 | + yrs = e20.assign(year=pd.to_datetime(e20.cutoff).dt.year).groupby("year").agg(firms=("entity_id", "nunique"), firm_quarters=("entity_id", "size"), adverse_rate=("y_adverse_opinc", "mean")).reset_index() | |
| 170 | + F.fig_coverage(FG, yrs) | |
| 171 | + te = e20[pd.to_datetime(e20.cutoff) >= pd.Timestamp(str(cfg["splits"]["test"][0]))].copy() | |
| 172 | + prev_rows = [{"label": "All firm-quarters", "sel": np.ones(len(te), bool)}] | |
| 173 | + for name, col, lab in (("size", "f_log_mcap", "Market cap"), ("illiq", "wd_amihud60", "Illiquidity"), ("lev", "f_leverage", "Leverage")): | |
| 174 | + q = te[col].rank(pct=True) | |
| 175 | + prev_rows += [{"label": f"{lab}: bottom tercile", "sel": (q <= 1 / 3).values}, {"label": f"{lab}: top tercile", "sel": (q > 2 / 3).values}] | |
| 176 | + for rt in ("before_open", "after_close"): | |
| 177 | + prev_rows.append({"label": f"Release {rt.replace('_', ' ')}", "sel": (te.release_time == rt).values}) | |
| 178 | + pr_ = [] | |
| 179 | + for r in prev_rows: | |
| 180 | + y = te.y_adverse_opinc.values[r["sel"]]; n = len(y); p = y.mean() if n else np.nan; se = np.sqrt(p * (1 - p) / n) if n else np.nan | |
| 181 | + pr_.append({"label": r["label"], "rate": p, "lo": p - 1.96 * se, "hi": p + 1.96 * se}) | |
| 182 | + F.fig_prevalence(FG, pd.DataFrame(pr_)) | |
| 183 | + F.fig_frequency_gains(FG, con) | |
| 184 | + F.fig_reliability(FG, pd.read_csv(E / "reliability.csv")) | |
| 185 | + F.fig_signal_evolution(FG, lad) | |
| 186 | + F.fig_heterogeneity(FG, sub, ["all", "size_small", "size_large", "illiq_high", "illiq_low", "lev_high", "lev_low", "loss_firms", "profit_firms", | |
| 187 | + "covid_2020", "ex_covid", "window_clean", "window_straddles_prev_release", "dense_bars", "sparse_bars", "sector_finance", "sector_nonfinance"]) | |
| 188 | + F.fig_rolling(FG, pd.read_csv(E / "walkforward_by_year.csv"), pd.read_csv(E / "walkforward_contrasts.csv")) | |
| 189 | + if "LB5" in h5: | |
| 190 | + F.fig_economic(FG, pd.read_csv(E / "portfolios_LB5.csv"), h5["LB5"]) | |
| 191 | + # ---------------- macros file | |
| 192 | + lines = ["% Auto-generated by scripts/09_make_manuscript.py — do not edit"] | |
| 193 | + for k, v in T.items(): | |
| 194 | + words = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"} | |
| 195 | + kk = "".join(words.get(ch, ch) if ch.isdigit() else ch for ch in k if ch.isalnum()) | |
| 196 | + lines.append(f"\\newcommand{{\\{kk}}}{{{v}}}") | |
| 197 | + (P.root / "paper" / "results_macros.tex").write_text("\n".join(lines) + "\n") | |
| 198 | + log.info("wrote %d macros, %d tables, figures to %s", len(T), len(list(TB.glob('*.tex'))), FG) | |
| 199 | + return 0 | |
| 200 | + | |
| 201 | + | |
| 202 | +if __name__ == "__main__": | |
| 203 | + raise SystemExit(main()) | |
added
src/wp39/__init__.py
+2 −0
@@ -0,0 +1,2 @@ | ||
| 1 | +"""WP39 UQO — The Invisible Balance Sheet (research package).""" | |
| 2 | +__version__ = "0.1.0" | |
added
src/wp39/api_client.py
+226 −0
@@ -0,0 +1,226 @@ | ||
| 1 | +"""Typed, redacting client for the FinData OS API (https://findata.spboucher.ai/api/v1). | |
| 2 | + | |
| 3 | +Design rules (docs/API_INTEGRATION.md): | |
| 4 | +- the bearer token is read from the environment or a mode-600 file, never logged, never placed in a URL; | |
| 5 | +- bounded exponential backoff with jitter on 429/5xx and transport errors (idempotent GETs; export creation only when | |
| 6 | + no export id was obtained); | |
| 7 | +- every call appends one provenance line (method, path, status, request id, dataset id/version, elapsed) to a JSONL log; | |
| 8 | +- Parquet exports are validated against the published bars schema before being accepted. | |
| 9 | +""" | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import hashlib | |
| 13 | +import json | |
| 14 | +import os | |
| 15 | +import random | |
| 16 | +import re | |
| 17 | +import time | |
| 18 | +from dataclasses import dataclass, asdict | |
| 19 | +from pathlib import Path | |
| 20 | +from typing import Any, Iterator | |
| 21 | + | |
| 22 | +import httpx | |
| 23 | + | |
| 24 | +_TOKEN_RE = re.compile(r"fd_[A-Za-z0-9_.\-]+") | |
| 25 | + | |
| 26 | + | |
| 27 | +def _redact(text: str) -> str: | |
| 28 | + return _TOKEN_RE.sub("fd_<REDACTED>", text) | |
| 29 | + | |
| 30 | + | |
| 31 | +class FinDataError(RuntimeError): | |
| 32 | + def __init__(self, status: int, code: str, message: str, request_id: str | None = None): | |
| 33 | + super().__init__(f"HTTP {status} {code}: {_redact(message)} (request_id={request_id})") | |
| 34 | + self.status, self.code, self.request_id = status, code, request_id | |
| 35 | + | |
| 36 | + | |
| 37 | +def load_api_key() -> str: | |
| 38 | + key = os.environ.get("FINDATA_API_KEY") | |
| 39 | + if key: | |
| 40 | + return key.strip() | |
| 41 | + path = os.path.expanduser(os.environ.get("FINDATA_API_KEY_FILE", "~/findata-secrets/api-key-wp39.txt")) | |
| 42 | + if not os.path.exists(path): | |
| 43 | + path = os.path.expanduser("~/Desktop/Cluster/secrets/findata/api-key-wp39.txt") | |
| 44 | + with open(path) as fh: | |
| 45 | + return fh.read().strip() | |
| 46 | + | |
| 47 | + | |
| 48 | +@dataclass | |
| 49 | +class ExportResult: | |
| 50 | + export_id: str | |
| 51 | + dataset_id: str | |
| 52 | + dataset_version: int | None | |
| 53 | + symbols: list[str] | |
| 54 | + start: str | |
| 55 | + end: str | |
| 56 | + adjustment: str | |
| 57 | + row_count: int | |
| 58 | + bytes: int | |
| 59 | + sha256: str | |
| 60 | + path: str | |
| 61 | + request_id: str | None | |
| 62 | + exported_at: str | |
| 63 | + | |
| 64 | + | |
| 65 | +class FinDataClient: | |
| 66 | + def __init__( | |
| 67 | + self, | |
| 68 | + base_url: str | None = None, | |
| 69 | + api_key: str | None = None, | |
| 70 | + log_path: str | os.PathLike | None = None, | |
| 71 | + timeout: float = 60.0, | |
| 72 | + max_attempts: int = 5, | |
| 73 | + ): | |
| 74 | + self.base_url = (base_url or os.environ.get("FINDATA_BASE_URL") or "https://findata.spboucher.ai/api/v1").rstrip("/") | |
| 75 | + self._key = api_key or load_api_key() | |
| 76 | + self.max_attempts = max_attempts | |
| 77 | + self.log_path = Path(log_path) if log_path else None | |
| 78 | + self._client = httpx.Client(timeout=timeout, headers={"Authorization": f"Bearer {self._key}", "User-Agent": "wp39-uqo/0.1"}) | |
| 79 | + | |
| 80 | + # ------------------------------------------------------------------ core | |
| 81 | + def _log(self, rec: dict[str, Any]) -> None: | |
| 82 | + if not self.log_path: | |
| 83 | + return | |
| 84 | + self.log_path.parent.mkdir(parents=True, exist_ok=True) | |
| 85 | + with open(self.log_path, "a") as fh: | |
| 86 | + fh.write(json.dumps(rec, default=str) + "\n") | |
| 87 | + | |
| 88 | + def _request(self, method: str, path: str, *, params=None, json_body=None, stream_to: Path | None = None, | |
| 89 | + retry: bool = True) -> httpx.Response: | |
| 90 | + url = path if path.startswith("http") else f"{self.base_url}{path}" | |
| 91 | + attempt, delay = 0, 0.5 | |
| 92 | + while True: | |
| 93 | + attempt += 1 | |
| 94 | + t0 = time.perf_counter() | |
| 95 | + try: | |
| 96 | + if stream_to is not None: | |
| 97 | + with self._client.stream(method, url, params=params, json=json_body) as r: | |
| 98 | + if r.status_code >= 400: | |
| 99 | + r.read() | |
| 100 | + resp = r | |
| 101 | + else: | |
| 102 | + stream_to.parent.mkdir(parents=True, exist_ok=True) | |
| 103 | + h = hashlib.sha256() | |
| 104 | + with open(stream_to, "wb") as fh: | |
| 105 | + for chunk in r.iter_bytes(): | |
| 106 | + fh.write(chunk) | |
| 107 | + h.update(chunk) | |
| 108 | + r.sha256 = h.hexdigest() # type: ignore[attr-defined] | |
| 109 | + resp = r | |
| 110 | + else: | |
| 111 | + resp = self._client.request(method, url, params=params, json=json_body) | |
| 112 | + elapsed = time.perf_counter() - t0 | |
| 113 | + self._log({ | |
| 114 | + "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "status": resp.status_code, | |
| 115 | + "request_id": resp.headers.get("X-Request-ID"), "dataset_id": resp.headers.get("X-Dataset-Id"), | |
| 116 | + "dataset_version": resp.headers.get("X-Dataset-Version"), "elapsed_s": round(elapsed, 3), "attempt": attempt, | |
| 117 | + }) | |
| 118 | + if resp.status_code < 400: | |
| 119 | + return resp | |
| 120 | + retryable = resp.status_code in (429, 500, 502, 503, 504) | |
| 121 | + if not (retry and retryable) or attempt >= self.max_attempts: | |
| 122 | + try: | |
| 123 | + err = resp.json().get("error", {}) | |
| 124 | + except Exception: | |
| 125 | + err = {"code": "http_error", "message": resp.text[:300]} | |
| 126 | + raise FinDataError(resp.status_code, err.get("code", "http_error"), err.get("message", ""), err.get("request_id")) | |
| 127 | + except (httpx.TransportError, httpx.TimeoutException) as exc: | |
| 128 | + self._log({"ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "status": None, | |
| 129 | + "error": _redact(type(exc).__name__), "attempt": attempt}) | |
| 130 | + if not retry or attempt >= self.max_attempts: | |
| 131 | + raise RuntimeError(f"FinData transport failure after {attempt} attempts on {path}: {_redact(str(exc))}") from None | |
| 132 | + time.sleep(delay + random.uniform(0, delay / 2)) | |
| 133 | + delay = min(delay * 2, 8.0) | |
| 134 | + | |
| 135 | + def get(self, path: str, **params) -> Any: | |
| 136 | + return self._request("GET", path, params={k: v for k, v in params.items() if v is not None}).json() | |
| 137 | + | |
| 138 | + def post(self, path: str, body: dict, retry: bool = False) -> Any: | |
| 139 | + return self._request("POST", path, json_body=body, retry=retry).json() | |
| 140 | + | |
| 141 | + # --------------------------------------------------------------- catalog | |
| 142 | + def health(self) -> dict: | |
| 143 | + return self.get("/health") | |
| 144 | + | |
| 145 | + def openapi(self) -> dict: | |
| 146 | + return self.get("/openapi.json") | |
| 147 | + | |
| 148 | + def datasets(self) -> list[dict]: | |
| 149 | + return self.get("/catalog/datasets")["data"] | |
| 150 | + | |
| 151 | + def dataset(self, dataset_id: str) -> dict: | |
| 152 | + return self.get(f"/catalog/datasets/{dataset_id}") | |
| 153 | + | |
| 154 | + def schema_bars(self) -> dict: | |
| 155 | + return self.get("/catalog/schema/bars") | |
| 156 | + | |
| 157 | + # ----------------------------------------------------------- instruments | |
| 158 | + def instrument(self, symbol: str) -> dict: | |
| 159 | + return self.get(f"/instruments/{symbol}") | |
| 160 | + | |
| 161 | + def instruments_search(self, query: str | None = None, asset_class: str | None = None, status: str | None = None, | |
| 162 | + limit: int = 200) -> Iterator[dict]: | |
| 163 | + cursor = None | |
| 164 | + while True: | |
| 165 | + page = self.get("/instruments", query=query, asset_class=asset_class, status=status, limit=limit, cursor=cursor) | |
| 166 | + yield from page.get("data", []) | |
| 167 | + cursor = page.get("next_cursor") | |
| 168 | + if not cursor: | |
| 169 | + return | |
| 170 | + | |
| 171 | + def coverage_symbols(self, dataset_id: str, min_sessions: int = 1, limit: int = 5000) -> Iterator[dict]: | |
| 172 | + cursor = None | |
| 173 | + while True: | |
| 174 | + page = self.get("/coverage/symbols", dataset_id=dataset_id, min_sessions=min_sessions, limit=limit, cursor=cursor) | |
| 175 | + yield from page.get("data", []) | |
| 176 | + cursor = page.get("next_cursor") | |
| 177 | + if not cursor: | |
| 178 | + return | |
| 179 | + | |
| 180 | + def coverage(self, dataset_id: str, symbol: str, start: str, end: str, limit: int = 5000) -> list[dict]: | |
| 181 | + return self.get("/coverage", dataset_id=dataset_id, symbol=symbol, start=start, end=end, limit=limit)["data"] | |
| 182 | + | |
| 183 | + # ---------------------------------------------------------- fundamentals | |
| 184 | + def fundamentals(self, entity_id: str, as_of: str | None = None, metrics: list[str] | None = None, | |
| 185 | + period_type: str | None = "q", limit: int = 20000, include_superseded: bool = False) -> dict: | |
| 186 | + return self.get(f"/fundamentals/{entity_id}", as_of=as_of, metrics=",".join(metrics) if metrics else None, | |
| 187 | + period_type=period_type, limit=limit, include_superseded=str(include_superseded).lower()) | |
| 188 | + | |
| 189 | + def fundamentals_periods(self, entity_id: str) -> list[dict]: | |
| 190 | + return self.get(f"/fundamentals/{entity_id}/periods")["periods"] | |
| 191 | + | |
| 192 | + # ----------------------------------------------------------------- query | |
| 193 | + def query(self, sql: str, max_rows: int = 1000) -> tuple[list[str], list[list[Any]], dict]: | |
| 194 | + d = self.post("/query", {"sql": sql, "max_rows": max_rows}, retry=True) | |
| 195 | + return [c["name"] for c in d["columns"]], d["rows"], d.get("meta", {}) | |
| 196 | + | |
| 197 | + # --------------------------------------------------------------- exports | |
| 198 | + def export(self, dataset_id: str, symbols: list[str], start: str, end: str, dest: Path, adjustment: str = "raw", | |
| 199 | + poll_s: float = 1.0, max_wait_s: float = 600.0) -> ExportResult: | |
| 200 | + body = {"dataset_id": dataset_id, "symbols": symbols, "start": start, "end": end, "adjustment": adjustment, "format": "parquet"} | |
| 201 | + r = self._request("POST", "/exports", json_body=body, retry=False) | |
| 202 | + created = r.json() | |
| 203 | + export_id, request_id = created["id"], r.headers.get("X-Request-ID") | |
| 204 | + t0 = time.time() | |
| 205 | + while True: | |
| 206 | + st = self.get(f"/exports/{export_id}") | |
| 207 | + if st["status"] == "ready": | |
| 208 | + break | |
| 209 | + if st["status"] in ("failed", "expired"): | |
| 210 | + raise FinDataError(500, "export_" + st["status"], str(st.get("error")), request_id) | |
| 211 | + if time.time() - t0 > max_wait_s: | |
| 212 | + raise FinDataError(504, "export_timeout", f"export {export_id} not ready after {max_wait_s}s", request_id) | |
| 213 | + time.sleep(poll_s) | |
| 214 | + resp = self._request("GET", f"/exports/{export_id}/download", stream_to=dest) | |
| 215 | + snap = st.get("snapshot") or {} | |
| 216 | + return ExportResult(export_id=export_id, dataset_id=dataset_id, dataset_version=snap.get("version"), symbols=symbols, | |
| 217 | + start=start, end=end, adjustment=adjustment, row_count=int(st.get("row_count") or 0), | |
| 218 | + bytes=int(st.get("bytes") or 0), sha256=getattr(resp, "sha256", ""), path=str(dest), | |
| 219 | + request_id=request_id, exported_at=time.strftime("%Y-%m-%dT%H:%M:%S%z")) | |
| 220 | + | |
| 221 | + def close(self) -> None: | |
| 222 | + self._client.close() | |
| 223 | + | |
| 224 | + | |
| 225 | +def export_result_dict(res: ExportResult) -> dict: | |
| 226 | + return asdict(res) | |
added
src/wp39/calendar.py
+78 −0
@@ -0,0 +1,78 @@ | ||
| 1 | +"""XNYS trading calendar helpers (America/New_York sessions, UTC storage).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import datetime as dt | |
| 5 | +from functools import lru_cache | |
| 6 | +from zoneinfo import ZoneInfo | |
| 7 | + | |
| 8 | +import numpy as np | |
| 9 | +import pandas as pd | |
| 10 | + | |
| 11 | +ET = ZoneInfo("America/New_York") | |
| 12 | +UTC = ZoneInfo("UTC") | |
| 13 | + | |
| 14 | + | |
| 15 | +@lru_cache(maxsize=1) | |
| 16 | +def _xnys(): | |
| 17 | + import exchange_calendars as xc | |
| 18 | + return xc.get_calendar("XNYS", start="2000-01-01", end="2027-12-31") | |
| 19 | + | |
| 20 | + | |
| 21 | +@lru_cache(maxsize=1) | |
| 22 | +def sessions() -> pd.DatetimeIndex: | |
| 23 | + """All XNYS sessions 2000→2027 as tz-naive dates.""" | |
| 24 | + return _xnys().sessions.tz_localize(None) if _xnys().sessions.tz is not None else _xnys().sessions | |
| 25 | + | |
| 26 | + | |
| 27 | +def session_array() -> np.ndarray: | |
| 28 | + return sessions().to_numpy().astype("datetime64[D]") | |
| 29 | + | |
| 30 | + | |
| 31 | +def is_session(day: dt.date) -> bool: | |
| 32 | + return np.datetime64(day) in set(session_array().tolist()) | |
| 33 | + | |
| 34 | + | |
| 35 | +def shift_trading_days(day: dt.date | np.datetime64 | pd.Timestamp, n: int) -> dt.date: | |
| 36 | + """Shift by n trading days; if `day` is not a session, start from the previous session (n<=0) or next (n>0).""" | |
| 37 | + arr = session_array() | |
| 38 | + d = np.datetime64(pd.Timestamp(day).date()) | |
| 39 | + pos = int(np.searchsorted(arr, d)) | |
| 40 | + if pos < len(arr) and arr[pos] == d: | |
| 41 | + base = pos | |
| 42 | + else: | |
| 43 | + base = pos - 1 if n <= 0 else pos | |
| 44 | + idx = base + n | |
| 45 | + if idx < 0 or idx >= len(arr): | |
| 46 | + raise ValueError("date out of calendar range") | |
| 47 | + return arr[idx].astype("datetime64[D]").astype(dt.date) | |
| 48 | + | |
| 49 | + | |
| 50 | +def trading_days_between(a: dt.date, b: dt.date) -> int: | |
| 51 | + """Number of sessions strictly after a up to and including b (negative if b<a).""" | |
| 52 | + arr = session_array() | |
| 53 | + ia, ib = np.searchsorted(arr, np.datetime64(a), side="right"), np.searchsorted(arr, np.datetime64(b), side="right") | |
| 54 | + return int(ib - ia) | |
| 55 | + | |
| 56 | + | |
| 57 | +def month_end_sessions(start: dt.date, end: dt.date) -> list[dt.date]: | |
| 58 | + s = pd.Series(sessions()) | |
| 59 | + s = s[(s >= pd.Timestamp(start)) & (s <= pd.Timestamp(end))] | |
| 60 | + return [d.date() for d in s.groupby(s.dt.to_period("M")).max()] | |
| 61 | + | |
| 62 | + | |
| 63 | +def close_time_utc(session: dt.date) -> pd.Timestamp: | |
| 64 | + """Open instant (UTC) of the closing-print bar (16:00 ET, or 13:00 ET on early-close days).""" | |
| 65 | + cal = _xnys() | |
| 66 | + ts = pd.Timestamp(session) | |
| 67 | + close = cal.session_close(ts) | |
| 68 | + return pd.Timestamp(close).tz_convert("UTC") if close.tz is not None else pd.Timestamp(close).tz_localize("UTC") | |
| 69 | + | |
| 70 | + | |
| 71 | +def regular_session_bounds_utc(session: dt.date) -> tuple[pd.Timestamp, pd.Timestamp]: | |
| 72 | + """[open 09:30 ET, close bar open] in UTC for the session (inclusive bar-open bounds).""" | |
| 73 | + cal = _xnys() | |
| 74 | + ts = pd.Timestamp(session) | |
| 75 | + o, c = cal.session_open(ts), cal.session_close(ts) | |
| 76 | + o = pd.Timestamp(o).tz_convert("UTC") if o.tz is not None else pd.Timestamp(o).tz_localize("UTC") | |
| 77 | + c = pd.Timestamp(c).tz_convert("UTC") if c.tz is not None else pd.Timestamp(c).tz_localize("UTC") | |
| 78 | + return o, c | |
added
src/wp39/catalog.py
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +"""Catalog reconciliation: documented contract vs observed API.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +from .api_client import FinDataClient | |
| 5 | + | |
| 6 | +EXPECTED_BARS_FIELDS = ["instrument_id", "symbol_as_reported", "venue", "asset_class", "interval", "timestamp_utc", | |
| 7 | + "session_date", "open", "high", "low", "close", "volume", "currency", "trade_count", "vwap", | |
| 8 | + "adjustment_type", "source_id", "ingested_at", "quality_flags", "revision_id"] | |
| 9 | + | |
| 10 | + | |
| 11 | +def probe(client: FinDataClient, contract: dict) -> dict: | |
| 12 | + health = client.health() | |
| 13 | + datasets = {d["id"]: d for d in client.datasets()} | |
| 14 | + schema = client.schema_bars() | |
| 15 | + spec = client.openapi() | |
| 16 | + routes = sorted(f"{m.upper()} {p}" for p, ops in spec["paths"].items() for m in ops) | |
| 17 | + needed = [contract["findata"]["datasets"][k] for k in ("bars_1m", "bars_1m_etf")] | |
| 18 | + report = { | |
| 19 | + "health_status": health.get("status"), | |
| 20 | + "n_datasets": len(datasets), | |
| 21 | + "schema_version": schema.get("schema_version"), | |
| 22 | + "timestamp_convention": schema.get("timestamp_convention"), | |
| 23 | + "bars_fields_ok": [f["name"] for f in schema["fields"]] == EXPECTED_BARS_FIELDS, | |
| 24 | + "required_datasets": {d: {"status": datasets.get(d, {}).get("status"), "rows": datasets.get(d, {}).get("row_count"), | |
| 25 | + "first": datasets.get(d, {}).get("first_session"), "last": datasets.get(d, {}).get("last_session"), | |
| 26 | + "license": datasets.get(d, {}).get("license_class")} for d in needed}, | |
| 27 | + "has_earnings_route": any("/earnings" in r for r in routes), | |
| 28 | + "routes": routes, | |
| 29 | + "lake_free_gb": (health.get("components", {}).get("lake") or {}).get("free_gb"), | |
| 30 | + } | |
| 31 | + return report | |
added
src/wp39/corporate_actions.py
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +"""Splits and cash dividends (FinData `corporate_actions`, vendor meta files) -> per-symbol ex-date adjustment table. | |
| 2 | + | |
| 3 | +The extract is produced once on the storage node (read-only `psql \\copy`, see docs/API_INTEGRATION.md) because the API | |
| 4 | +has no corporate-actions route; its SHA-256 is in artifacts/manifests/corporate_actions.json. Raw bars stay unadjusted; | |
| 5 | +adjustments are applied explicitly in features.daily_block (ex-date rule). | |
| 6 | +""" | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +from pathlib import Path | |
| 10 | + | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | + | |
| 14 | +def load_actions(path: Path) -> pl.DataFrame: | |
| 15 | + """Raw Postgres copy: id, instrument_id, action_type (split|dividend), ex_date, value, ... . | |
| 16 | + Vendor quirk: dividends are keyed `firstrate:stock:<SYM>_DIVS`; `value` = split ratio (new per old) or cash amount.""" | |
| 17 | + if not path.exists(): | |
| 18 | + return pl.DataFrame(schema={"symbol": pl.Utf8, "ex_date": pl.Date, "split_ratio": pl.Float64, "dividend": pl.Float64}) | |
| 19 | + a = pl.read_csv(path, try_parse_dates=True, infer_schema_length=10000) | |
| 20 | + a = a.with_columns([ | |
| 21 | + pl.col("instrument_id").str.replace("^firstrate:[a-z]+:", "").str.replace("_DIVS$", "").alias("symbol"), | |
| 22 | + pl.col("ex_date").cast(pl.Date), pl.col("value").cast(pl.Float64), | |
| 23 | + ]) | |
| 24 | + splits = a.filter(pl.col("action_type") == "split").group_by(["symbol", "ex_date"]).agg(pl.col("value").product().alias("split_ratio")) | |
| 25 | + divs = a.filter(pl.col("action_type") == "dividend").group_by(["symbol", "ex_date"]).agg(pl.col("value").sum().alias("dividend")) | |
| 26 | + out = splits.join(divs, on=["symbol", "ex_date"], how="full", coalesce=True).with_columns([ | |
| 27 | + pl.col("split_ratio").fill_null(1.0), pl.col("dividend").fill_null(0.0)]) | |
| 28 | + out = out.filter((pl.col("split_ratio") > 0) & (pl.col("dividend") >= 0)) | |
| 29 | + return out.sort(["symbol", "ex_date"]) | |
| 30 | + | |
| 31 | + | |
| 32 | +def split_events_in_window(actions: pl.DataFrame, symbol: str, start, end) -> int: | |
| 33 | + if actions.height == 0: | |
| 34 | + return 0 | |
| 35 | + return actions.filter((pl.col("symbol") == symbol) & (pl.col("ex_date") >= start) & (pl.col("ex_date") <= end) | |
| 36 | + & (pl.col("split_ratio") != 1.0)).height | |
added
src/wp39/features.py
+181 −0
@@ -0,0 +1,181 @@ | ||
| 1 | +"""Frequency-controlled OHLCV features computed per symbol-session from the raw 1-minute export. | |
| 2 | + | |
| 3 | +Every intraday family is computed identically on bars of length 1m, 5m and 30m, obtained by aggregating the *same* raw | |
| 4 | +1-minute bars within the regular session (09:30 -> closing print). Daily features come from the same aggregation (D-003). | |
| 5 | +Zero-volume minutes are absent at the source and are never invented; the share of expected buckets present is a feature. | |
| 6 | +Bar statistics are labelled as such; nothing here is order flow, spread or informed trading. | |
| 7 | +""" | |
| 8 | +from __future__ import annotations | |
| 9 | + | |
| 10 | +import math | |
| 11 | +from pathlib import Path | |
| 12 | + | |
| 13 | +import numpy as np | |
| 14 | +import polars as pl | |
| 15 | + | |
| 16 | +from .calendar import _xnys | |
| 17 | + | |
| 18 | +FREQS = {"1m": 1, "5m": 5, "30m": 30} | |
| 19 | +INTRADAY_FAMILIES = ["rv", "bpv", "rj", "rsneg_share", "sjv", "rskew", "rkurt", "max_share", "ac1", "drift", | |
| 20 | + "vol_first30", "vol_last30", "vol_gini", "down_vol_share", "present_share", "rv_mktdown_share", | |
| 21 | + "sxy", "syy", "n_bars"] | |
| 22 | + | |
| 23 | + | |
| 24 | +def session_bounds_table(start, end) -> pl.DataFrame: | |
| 25 | + """session_date, open_utc (09:30 ET bar open), close_utc (closing-print bar open), close_minute (minutes after 09:30).""" | |
| 26 | + cal = _xnys() | |
| 27 | + s = cal.sessions | |
| 28 | + s = s[(s >= str(start)) & (s <= str(end))] | |
| 29 | + opens = cal.opens.loc[s] if hasattr(cal, "opens") else None | |
| 30 | + rows = [] | |
| 31 | + for d in s: | |
| 32 | + o, c = cal.session_open(d), cal.session_close(d) | |
| 33 | + rows.append({"session_date": d.date(), "open_utc": o.tz_convert("UTC").to_pydatetime(), | |
| 34 | + "close_utc": c.tz_convert("UTC").to_pydatetime(), | |
| 35 | + "close_minute": int((c - o).total_seconds() // 60)}) | |
| 36 | + df = pl.DataFrame(rows) | |
| 37 | + return df.with_columns([pl.col("open_utc").dt.replace_time_zone("UTC"), pl.col("close_utc").dt.replace_time_zone("UTC")]) | |
| 38 | + | |
| 39 | + | |
| 40 | +def load_regular_session(paths: list[Path], bounds: pl.DataFrame) -> pl.DataFrame: | |
| 41 | + """Raw 1m bars restricted to the regular session, with minute-of-session index m (0 = 09:30, close_minute = closing print).""" | |
| 42 | + lf = pl.concat([pl.scan_parquet(str(p)) for p in paths]).select( | |
| 43 | + ["symbol_as_reported", "timestamp_utc", "session_date", "open", "high", "low", "close", "volume", "quality_flags"]) | |
| 44 | + df = lf.collect() | |
| 45 | + if df.height == 0: | |
| 46 | + return df | |
| 47 | + # the lake's base+delta layouts (and chunk boundaries) can publish the same bar twice: keep the latest ingestion | |
| 48 | + df = pl.concat([pl.scan_parquet(str(p)) for p in paths]).select( | |
| 49 | + ["symbol_as_reported", "timestamp_utc", "session_date", "open", "high", "low", "close", "volume", "quality_flags", | |
| 50 | + "ingested_at"]).collect().sort("ingested_at").unique(subset=["timestamp_utc"], keep="last").drop("ingested_at") | |
| 51 | + df = df.with_columns(pl.col("timestamp_utc").dt.convert_time_zone("UTC")) | |
| 52 | + df = df.join(bounds, on="session_date", how="inner") | |
| 53 | + df = df.filter((pl.col("timestamp_utc") >= pl.col("open_utc")) & (pl.col("timestamp_utc") <= pl.col("close_utc"))) | |
| 54 | + df = df.with_columns(((pl.col("timestamp_utc") - pl.col("open_utc")).dt.total_minutes()).cast(pl.Int32).alias("m")) | |
| 55 | + df = df.filter(pl.col("quality_flags").list.len() == 0) # drop rows flagged by FinData (ohlc_inconsistent, price_jump, ...) | |
| 56 | + return df.sort(["session_date", "m"]) | |
| 57 | + | |
| 58 | + | |
| 59 | +def aggregate_buckets(reg: pl.DataFrame, freq_min: int) -> pl.DataFrame: | |
| 60 | + """Aggregate regular-session 1m bars into buckets of `freq_min` minutes; the closing print joins the last bucket.""" | |
| 61 | + if freq_min == 1: | |
| 62 | + b = reg.with_columns(pl.col("m").alias("b")) | |
| 63 | + return b.select(["session_date", "b", "open", "high", "low", "close", "volume", "m", "close_minute"]) | |
| 64 | + nb = (pl.col("close_minute") / freq_min).ceil().cast(pl.Int32) # buckets per session | |
| 65 | + b = reg.with_columns(pl.min_horizontal((pl.col("m") / freq_min).floor().cast(pl.Int32), nb - 1).alias("b")) | |
| 66 | + agg = b.group_by(["session_date", "b"], maintain_order=True).agg([ | |
| 67 | + pl.col("open").first(), pl.col("high").max(), pl.col("low").min(), pl.col("close").last(), pl.col("volume").sum(), | |
| 68 | + pl.col("m").first(), pl.col("close_minute").first(), | |
| 69 | + ]) | |
| 70 | + return agg.sort(["session_date", "b"]) | |
| 71 | + | |
| 72 | + | |
| 73 | +def bucket_returns(bk: pl.DataFrame) -> pl.DataFrame: | |
| 74 | + """Within-session log returns: first bucket open->close, then close->close.""" | |
| 75 | + prev_close = pl.col("close").shift(1).over("session_date") | |
| 76 | + r = pl.when(prev_close.is_null()).then((pl.col("close") / pl.col("open")).log()).otherwise((pl.col("close") / prev_close).log()) | |
| 77 | + return bk.with_columns(r.alias("r")) | |
| 78 | + | |
| 79 | + | |
| 80 | +def _gini_expr() -> pl.Expr: | |
| 81 | + v = pl.col("volume").sort() | |
| 82 | + n = pl.len() | |
| 83 | + i = pl.int_range(1, n + 1).cast(pl.Float64) | |
| 84 | + return ((2.0 * (i * v).sum()) / (n.cast(pl.Float64) * v.sum() + 1e-12) - (n.cast(pl.Float64) + 1.0) / n.cast(pl.Float64)) | |
| 85 | + | |
| 86 | + | |
| 87 | +def session_features(bk: pl.DataFrame, freq_min: int, spy: pl.DataFrame | None) -> pl.DataFrame: | |
| 88 | + """One row per session with the intraday families for this frequency (prefix f{freq}_).""" | |
| 89 | + df = bucket_returns(bk) | |
| 90 | + if spy is not None: | |
| 91 | + df = df.join(spy.select(["session_date", "b", pl.col("r").alias("r_spy")]), on=["session_date", "b"], how="left") | |
| 92 | + else: | |
| 93 | + df = df.with_columns(pl.lit(None, dtype=pl.Float64).alias("r_spy")) | |
| 94 | + r = pl.col("r") | |
| 95 | + r2 = r.pow(2) | |
| 96 | + rv = r2.sum() | |
| 97 | + n = pl.len().cast(pl.Float64) | |
| 98 | + prev = r.shift(1).over("session_date") | |
| 99 | + expected = pl.when(pl.lit(freq_min) == 1).then(pl.col("close_minute").first() + 1).otherwise( | |
| 100 | + (pl.col("close_minute").first() / freq_min).ceil()).cast(pl.Float64) | |
| 101 | + first30 = pl.col("m") < 30 | |
| 102 | + last30 = pl.col("m") > (pl.col("close_minute") - 30) | |
| 103 | + out = df.group_by("session_date", maintain_order=True).agg([ | |
| 104 | + n.alias("n_bars"), | |
| 105 | + rv.alias("rv"), | |
| 106 | + ((math.pi / 2.0) * (r.abs() * prev.abs()).sum()).alias("bpv"), | |
| 107 | + (r2.filter(r < 0).sum()).alias("rs_neg"), | |
| 108 | + (r2.filter(r > 0).sum()).alias("rs_pos"), | |
| 109 | + (n.sqrt() * r.pow(3).sum() / (rv.pow(1.5) + 1e-18)).alias("rskew"), | |
| 110 | + (n * r.pow(4).sum() / (rv.pow(2) + 1e-18)).alias("rkurt"), | |
| 111 | + (r2.max() / (rv + 1e-18)).alias("max_share"), | |
| 112 | + pl.corr(r, prev).alias("ac1"), | |
| 113 | + (pl.col("close").last() / pl.col("open").first()).log().alias("drift"), | |
| 114 | + (pl.col("volume").filter(first30).sum() / (pl.col("volume").sum() + 1e-9)).alias("vol_first30"), | |
| 115 | + (pl.col("volume").filter(last30).sum() / (pl.col("volume").sum() + 1e-9)).alias("vol_last30"), | |
| 116 | + _gini_expr().alias("vol_gini"), | |
| 117 | + (pl.col("volume").filter(r < 0).sum() / (pl.col("volume").sum() + 1e-9)).alias("down_vol_share"), | |
| 118 | + (n / expected).alias("present_share"), | |
| 119 | + (r2.filter(pl.col("r_spy") < 0).sum() / (rv + 1e-18)).alias("rv_mktdown_share"), | |
| 120 | + (r * pl.col("r_spy")).sum().alias("sxy"), | |
| 121 | + pl.col("r_spy").pow(2).sum().alias("syy"), | |
| 122 | + pl.col("volume").sum().alias("volume"), | |
| 123 | + pl.col("open").first().alias("open"), pl.col("close").last().alias("close"), | |
| 124 | + pl.col("high").max().alias("high"), pl.col("low").min().alias("low"), | |
| 125 | + ]) | |
| 126 | + out = out.with_columns([ | |
| 127 | + (pl.max_horizontal(pl.col("rv") - pl.col("bpv"), 0.0) / (pl.col("rv") + 1e-18)).alias("rj"), | |
| 128 | + (pl.col("rs_neg") / (pl.col("rv") + 1e-18)).alias("rsneg_share"), | |
| 129 | + ((pl.col("rs_pos") - pl.col("rs_neg")) / (pl.col("rv") + 1e-18)).alias("sjv"), | |
| 130 | + ]) | |
| 131 | + pre = f"f{freq_min}m_" | |
| 132 | + keep = {c: pre + c for c in INTRADAY_FAMILIES} | |
| 133 | + return out.select(["session_date"] + [pl.col(c).alias(keep[c]) for c in INTRADAY_FAMILIES] + | |
| 134 | + ([pl.col("volume").alias("d_volume"), pl.col("open").alias("d_open"), pl.col("close").alias("d_close"), | |
| 135 | + pl.col("high").alias("d_high"), pl.col("low").alias("d_low")] if freq_min == 1 else [])) | |
| 136 | + | |
| 137 | + | |
| 138 | +def daily_block(day: pl.DataFrame, actions: pl.DataFrame | None, symbol: str) -> pl.DataFrame: | |
| 139 | + """Daily OHLCV features from the 1m aggregation + split/dividend-adjusted close-to-close returns.""" | |
| 140 | + d = day.sort("session_date") | |
| 141 | + if actions is not None and actions.height: | |
| 142 | + a = actions.filter(pl.col("symbol") == symbol).select(["ex_date", "split_ratio", "dividend"]) | |
| 143 | + d = d.join(a, left_on="session_date", right_on="ex_date", how="left").with_columns([ | |
| 144 | + pl.col("split_ratio").fill_null(1.0), pl.col("dividend").fill_null(0.0)]) | |
| 145 | + else: | |
| 146 | + d = d.with_columns([pl.lit(1.0).alias("split_ratio"), pl.lit(0.0).alias("dividend")]) | |
| 147 | + prev_close_adj = (pl.col("d_close").shift(1) / pl.col("split_ratio")) | |
| 148 | + d = d.with_columns([ | |
| 149 | + ((pl.col("d_close") + pl.col("dividend")) / prev_close_adj - 1.0).alias("d_ret_cc"), | |
| 150 | + (pl.col("d_open") / prev_close_adj - 1.0).alias("d_ret_co"), | |
| 151 | + (pl.col("d_close") / pl.col("d_open") - 1.0).alias("d_ret_oc"), | |
| 152 | + ((pl.col("d_high") / pl.col("d_low")).log().pow(2) / (4.0 * math.log(2.0))).alias("d_parkinson"), | |
| 153 | + (pl.col("d_volume") * pl.col("d_close")).alias("d_dollar_vol"), | |
| 154 | + ]) | |
| 155 | + return d | |
| 156 | + | |
| 157 | + | |
| 158 | +def compute_symbol_features(paths: list[Path], symbol: str, bounds: pl.DataFrame, spy_buckets: dict[str, pl.DataFrame] | None, | |
| 159 | + actions: pl.DataFrame | None) -> pl.DataFrame: | |
| 160 | + reg = load_regular_session(paths, bounds) | |
| 161 | + if reg.height == 0: | |
| 162 | + return pl.DataFrame() | |
| 163 | + blocks = [] | |
| 164 | + for name, fmin in FREQS.items(): | |
| 165 | + bk = aggregate_buckets(reg, fmin) | |
| 166 | + spy = spy_buckets.get(name) if spy_buckets else None | |
| 167 | + blocks.append(session_features(bk, fmin, spy)) | |
| 168 | + out = blocks[0] | |
| 169 | + for b in blocks[1:]: | |
| 170 | + out = out.join(b, on="session_date", how="left") | |
| 171 | + out = daily_block(out, actions, symbol) | |
| 172 | + return out.with_columns(pl.lit(symbol).alias("symbol")).sort("session_date") | |
| 173 | + | |
| 174 | + | |
| 175 | +def compute_spy_buckets(paths: list[Path], bounds: pl.DataFrame) -> dict[str, pl.DataFrame]: | |
| 176 | + reg = load_regular_session(paths, bounds) | |
| 177 | + out = {} | |
| 178 | + for name, fmin in FREQS.items(): | |
| 179 | + out[name] = bucket_returns(aggregate_buckets(reg, fmin)).select(["session_date", "b", "r"]).unique( | |
| 180 | + subset=["session_date", "b"], keep="last").sort(["session_date", "b"]) | |
| 181 | + return out | |
added
src/wp39/figures.py
+156 −0
@@ -0,0 +1,156 @@ | ||
| 1 | +"""Journal figures (vector PDF, matplotlib). One axis per panel, thin marks, fixed categorical order, direct labels. | |
| 2 | +Palette validated with the dataviz validator (light surface): blue, orange, aqua, yellow (+ gray for references).""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +from pathlib import Path | |
| 6 | + | |
| 7 | +import matplotlib | |
| 8 | +matplotlib.use("Agg") | |
| 9 | +import matplotlib.pyplot as plt # noqa: E402 | |
| 10 | +import numpy as np # noqa: E402 | |
| 11 | +import pandas as pd # noqa: E402 | |
| 12 | + | |
| 13 | +SERIES = {"daily": "#2a78d6", "30m": "#eb6834", "5m": "#1baf7a", "1m": "#eda100", "fund": "#6250d6", "ref": "#8a8a86", "neg": "#e34948"} | |
| 14 | +MODEL_COLOR = {"B1": SERIES["fund"], "B2": SERIES["daily"], "B3": SERIES["30m"], "B4": SERIES["5m"], "B5": SERIES["1m"], "B5all": "#52514e", | |
| 15 | + "LB1": SERIES["fund"], "LB2": SERIES["daily"], "LB3": SERIES["30m"], "LB4": SERIES["5m"], "LB5": SERIES["1m"], "LB5all": "#52514e"} | |
| 16 | +MODEL_LABEL = {"B1": "B1 fundamentals", "B2": "B2 + daily", "B3": "B3 + 30-min", "B4": "B4 + 5-min", "B5": "B5 + 1-min", "B5all": "all frequencies"} | |
| 17 | + | |
| 18 | +plt.rcParams.update({"font.family": "serif", "font.size": 9, "axes.spines.top": False, "axes.spines.right": False, | |
| 19 | + "axes.edgecolor": "#8a8a86", "axes.linewidth": 0.6, "xtick.color": "#52514e", "ytick.color": "#52514e", | |
| 20 | + "axes.labelcolor": "#0b0b0b", "grid.color": "#e6e5e1", "grid.linewidth": 0.5, "legend.frameon": False, | |
| 21 | + "figure.dpi": 150, "savefig.bbox": "tight"}) | |
| 22 | + | |
| 23 | + | |
| 24 | +def _save(fig, out: Path, name: str): | |
| 25 | + out.mkdir(parents=True, exist_ok=True) | |
| 26 | + fig.savefig(out / f"{name}.pdf"); fig.savefig(out / f"{name}.png", dpi=200); plt.close(fig) | |
| 27 | + | |
| 28 | + | |
| 29 | +def fig_timeline(out: Path, h_list=(40, 20, 10, 5, 1)): | |
| 30 | + """Event-time timeline and the no-leakage rule (schematic drawn from the protocol, labelled as such).""" | |
| 31 | + fig, ax = plt.subplots(figsize=(6.4, 2.2)) | |
| 32 | + ax.set_xlim(-75, 15); ax.set_ylim(0, 3); ax.axis("off") | |
| 33 | + ax.plot([-75, 12], [1, 1], color="#52514e", lw=1) | |
| 34 | + for x, lab, c in ((-63, "$A_{q-1}$\nprevious release", SERIES["ref"]), (0, "$A_q$\nrelease of quarter $q$", SERIES["neg"]), | |
| 35 | + (1, "$F_q$ filing", SERIES["ref"]), (2, "$V_q=F_q+1$\nfacts available", SERIES["ref"])): | |
| 36 | + ax.plot([x], [1], marker="|", ms=14, color=c, mew=1.5) | |
| 37 | + ax.text(x, 0.15 if x not in (1,) else 1.55, lab, ha="center", va="center", fontsize=7, color="#0b0b0b") | |
| 38 | + for i, h in enumerate(h_list): | |
| 39 | + ax.plot([-h], [1], marker="o", ms=4, color=SERIES["1m"], mec="white", mew=0.8) | |
| 40 | + ax.text(-h, 1.35 + 0.28 * (i % 2), f"$c=A_q-{h}$", ha="center", fontsize=6.5, color="#52514e") | |
| 41 | + ax.add_patch(plt.Rectangle((-40, 0.85), 20, 0.3, color=SERIES["1m"], alpha=0.25, lw=0)) | |
| 42 | + ax.text(-30, 2.6, "20-session feature window ends at $c$; bars after $c$ and facts with $V_j>c$ never enter the information set", | |
| 43 | + ha="center", fontsize=7, color="#52514e") | |
| 44 | + ax.set_title("Event-time design (schematic of the protocol, not data)", fontsize=8, loc="left") | |
| 45 | + _save(fig, out, "fig01_timeline") | |
| 46 | + | |
| 47 | + | |
| 48 | +def fig_coverage(out: Path, years: pd.DataFrame, sizes: pd.DataFrame | None = None): | |
| 49 | + """years: columns year, firms, firm_quarters, adverse_rate.""" | |
| 50 | + fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.4)) | |
| 51 | + ax = axes[0] | |
| 52 | + ax.bar(years["year"], years["firms"], color=SERIES["daily"], width=0.8) | |
| 53 | + ax.set_ylabel("Firms in sample"); ax.set_xlabel("Year of cutoff"); ax.grid(axis="y") | |
| 54 | + for x, v in zip(years["year"].iloc[::3], years["firms"].iloc[::3]): | |
| 55 | + ax.text(x, v, f"{int(v):,}", ha="center", va="bottom", fontsize=6, color="#52514e") | |
| 56 | + ax = axes[1] | |
| 57 | + ax.plot(years["year"], years["adverse_rate"], color=SERIES["neg"], lw=2, marker="o", ms=4, mec="white") | |
| 58 | + ax.set_ylabel("Adverse-surprise rate"); ax.set_xlabel("Year of cutoff"); ax.grid(axis="y"); ax.set_ylim(0, None) | |
| 59 | + _save(fig, out, "fig02_coverage") | |
| 60 | + | |
| 61 | + | |
| 62 | +def fig_prevalence(out: Path, prev: pd.DataFrame): | |
| 63 | + """prev: columns group (str), label, rate, lo, hi — conditional adverse prevalence by ex-ante characteristics.""" | |
| 64 | + fig, ax = plt.subplots(figsize=(6.0, 2.6)) | |
| 65 | + y = np.arange(len(prev)) | |
| 66 | + ax.hlines(y, prev["lo"], prev["hi"], color="#c3c2b7", lw=2) | |
| 67 | + ax.plot(prev["rate"], y, "o", color=SERIES["neg"], ms=5, mec="white") | |
| 68 | + ax.set_yticks(y); ax.set_yticklabels(prev["label"]); ax.invert_yaxis() | |
| 69 | + ax.set_xlabel("Share of firm-quarters with an adverse operating-income surprise (test period)"); ax.grid(axis="x") | |
| 70 | + _save(fig, out, "fig03_prevalence") | |
| 71 | + | |
| 72 | + | |
| 73 | +def fig_frequency_gains(out: Path, contrasts: pd.DataFrame): | |
| 74 | + """Paired Δ log loss vs B2 for B3/B4/B5 (logistic and LightGBM), with 95 % block-bootstrap intervals.""" | |
| 75 | + fam = [("logistic", ""), ("lightgbm", "L")] | |
| 76 | + fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.6), sharey=True) | |
| 77 | + for ax, (name, pre) in zip(axes, fam): | |
| 78 | + sub = contrasts[(contrasts.family == name) & (contrasts.b == pre + "B2") & (contrasts.a.isin([pre + "B3", pre + "B4", pre + "B5", pre + "B5all"]))] | |
| 79 | + order = [pre + "B3", pre + "B4", pre + "B5", pre + "B5all"] | |
| 80 | + sub = sub.set_index("a").reindex([o for o in order if o in sub.a.values]).reset_index() | |
| 81 | + x = np.arange(len(sub)) | |
| 82 | + for i, r in sub.iterrows(): | |
| 83 | + c = MODEL_COLOR.get(r["a"].replace("L", "", 1) if r["a"].startswith("L") else r["a"], "#52514e") | |
| 84 | + ax.vlines(i, r["ci_lo"], r["ci_hi"], color=c, lw=2) | |
| 85 | + ax.plot(i, r["d_logloss"], "o", color=c, ms=6, mec="white") | |
| 86 | + ax.text(i, r["ci_hi"], f"{r['d_logloss']:+.4f}", ha="center", va="bottom", fontsize=6.5, color="#52514e") | |
| 87 | + ax.axhline(0, color="#8a8a86", lw=0.8) | |
| 88 | + ax.set_xticks(x); ax.set_xticklabels([MODEL_LABEL[a.replace("L", "", 1) if a.startswith("L") else a].replace("B5all", "all") for a in sub.a], fontsize=7) | |
| 89 | + ax.set_title(f"{name}", fontsize=8, loc="left"); ax.grid(axis="y") | |
| 90 | + axes[0].set_ylabel("Δ log loss vs. B2 (daily + accounting)\nnegative = better") | |
| 91 | + _save(fig, out, "fig04_frequency_gains") | |
| 92 | + | |
| 93 | + | |
| 94 | +def fig_reliability(out: Path, rel: pd.DataFrame): | |
| 95 | + fig, ax = plt.subplots(figsize=(3.6, 3.2)) | |
| 96 | + ax.plot([0, rel[["mean_pred", "obs_rate"]].max().max()], [0, rel[["mean_pred", "obs_rate"]].max().max()], color="#8a8a86", lw=0.8, ls="--") | |
| 97 | + for m in ("LB2", "LB5", "B2", "B5"): | |
| 98 | + s = rel[rel.model == m] | |
| 99 | + if len(s): | |
| 100 | + ax.plot(s["mean_pred"], s["obs_rate"], marker="o", ms=4, mec="white", lw=1.6, color=MODEL_COLOR[m], label=m) | |
| 101 | + ax.set_xlabel("Mean predicted probability (decile)"); ax.set_ylabel("Observed adverse rate"); ax.legend(fontsize=7); ax.grid() | |
| 102 | + _save(fig, out, "fig05_reliability") | |
| 103 | + | |
| 104 | + | |
| 105 | +def fig_signal_evolution(out: Path, ladder: pd.DataFrame): | |
| 106 | + """Mean AUC and Δ log loss by horizon (all firm-quarters, no selection).""" | |
| 107 | + fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.5)) | |
| 108 | + for fam, c in (("logistic", SERIES["daily"]), ("lightgbm", SERIES["1m"])): | |
| 109 | + s = ladder[ladder.family == fam].sort_values("h", ascending=False) | |
| 110 | + axes[0].plot(-s["h"], s["auc_a"], marker="o", ms=4, mec="white", lw=1.6, color=c, label=f"{fam}: B5") | |
| 111 | + axes[0].plot(-s["h"], s["auc_b"], marker="s", ms=3.5, mec="white", lw=1.2, ls="--", color=c, label=f"{fam}: B2") | |
| 112 | + axes[1].errorbar(-s["h"], s["d_logloss"], yerr=[s["d_logloss"] - s["ci_lo"], s["ci_hi"] - s["d_logloss"]], fmt="o-", ms=4, lw=1.4, | |
| 113 | + color=c, capsize=2, label=fam) | |
| 114 | + axes[0].set_xlabel("Trading days before the release (−h)"); axes[0].set_ylabel("Test AUC"); axes[0].legend(fontsize=6); axes[0].grid() | |
| 115 | + axes[1].axhline(0, color="#8a8a86", lw=0.8); axes[1].set_xlabel("Trading days before the release (−h)") | |
| 116 | + axes[1].set_ylabel("Δ log loss B5 − B2"); axes[1].legend(fontsize=7); axes[1].grid() | |
| 117 | + _save(fig, out, "fig06_signal_evolution") | |
| 118 | + | |
| 119 | + | |
| 120 | +def fig_heterogeneity(out: Path, sub: pd.DataFrame, groups: list[str]): | |
| 121 | + s = sub[(sub.family == "lightgbm") & sub.subgroup.isin(groups)].set_index("subgroup").reindex(groups).reset_index() | |
| 122 | + fig, ax = plt.subplots(figsize=(6.0, 2.8)) | |
| 123 | + y = np.arange(len(s)) | |
| 124 | + ax.hlines(y, s["ci_lo"], s["ci_hi"], color="#c3c2b7", lw=2) | |
| 125 | + ax.plot(s["mean"], y, "o", color=SERIES["1m"], ms=5, mec="white") | |
| 126 | + for yi, r in zip(y, s.itertuples()): | |
| 127 | + ax.text(r.ci_hi, yi, f" n={int(r.n):,}", va="center", fontsize=6, color="#52514e") | |
| 128 | + ax.axvline(0, color="#8a8a86", lw=0.8) | |
| 129 | + ax.set_yticks(y); ax.set_yticklabels([g.replace("_", " ") for g in s["subgroup"]]); ax.invert_yaxis() | |
| 130 | + ax.set_xlabel("Δ log loss LB5 − LB2 (test period), 95 % block-bootstrap CI"); ax.grid(axis="x") | |
| 131 | + _save(fig, out, "fig07_heterogeneity") | |
| 132 | + | |
| 133 | + | |
| 134 | +def fig_rolling(out: Path, wf: pd.DataFrame, wfc: pd.DataFrame): | |
| 135 | + fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.5)) | |
| 136 | + for m in ("LB2", "LB5", "B2", "B5"): | |
| 137 | + s = wf[wf.model == m].sort_values("year") | |
| 138 | + if len(s): | |
| 139 | + axes[0].plot(s["year"], s["auc"], marker="o", ms=3.5, mec="white", lw=1.4, color=MODEL_COLOR[m], ls="-" if m.startswith("L") else "--", label=m) | |
| 140 | + axes[0].set_ylabel("Out-of-fold AUC (annual refits)"); axes[0].legend(fontsize=6, ncol=2); axes[0].grid() | |
| 141 | + for fam, c in (("lightgbm", SERIES["1m"]), ("logistic", SERIES["daily"])): | |
| 142 | + s = wfc[(wfc.family == fam) & (wfc.year != "all")].copy(); s["year"] = s["year"].astype(int); s = s.sort_values("year") | |
| 143 | + axes[1].errorbar(s["year"], s["d_logloss"], yerr=[s["d_logloss"] - s["ci_lo"], s["ci_hi"] - s["d_logloss"]], fmt="o-", ms=3.5, lw=1.2, capsize=2, color=c, label=fam) | |
| 144 | + axes[1].axhline(0, color="#8a8a86", lw=0.8); axes[1].set_ylabel("Δ log loss B5 − B2 by year"); axes[1].legend(fontsize=7); axes[1].grid() | |
| 145 | + _save(fig, out, "fig08_rolling") | |
| 146 | + | |
| 147 | + | |
| 148 | +def fig_economic(out: Path, port: pd.DataFrame, alphas: dict): | |
| 149 | + fig, ax = plt.subplots(figsize=(4.2, 2.6)) | |
| 150 | + dm = port.groupby("decile")["ret"].mean() * 100 | |
| 151 | + ax.bar(dm.index, dm.values, color=SERIES["daily"], width=0.8) | |
| 152 | + ax.axhline(0, color="#8a8a86", lw=0.8); ax.set_xlabel("Decile of ex-ante deterioration score (10 = most likely)") | |
| 153 | + ax.set_ylabel("Mean monthly return (%)"); ax.grid(axis="y") | |
| 154 | + ax.set_title(f"L/S 1−10 alpha: {alphas['gross']['alpha_monthly']*100:+.2f}% (t={alphas['gross']['alpha_t']:.1f}) gross, " | |
| 155 | + f"{alphas['net20bps']['alpha_monthly']*100:+.2f}% net of 20 bps", fontsize=7, loc="left") | |
| 156 | + _save(fig, out, "fig09_economic") | |
added
src/wp39/inference.py
+134 −0
@@ -0,0 +1,134 @@ | ||
| 1 | +"""Evaluation metrics, paired block-bootstrap intervals, Diebold-Mariano / Giacomini-White tests, Holm correction, | |
| 2 | +calibration, and two-way clustered panel regressions.""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import numpy as np | |
| 6 | +import pandas as pd | |
| 7 | +import polars as pl | |
| 8 | +from scipy import stats | |
| 9 | +from sklearn.metrics import average_precision_score, brier_score_loss, log_loss, roc_auc_score | |
| 10 | + | |
| 11 | + | |
| 12 | +# --------------------------------------------------------------------- metrics | |
| 13 | +def binary_metrics(y: np.ndarray, p: np.ndarray) -> dict: | |
| 14 | + p = np.clip(p, 1e-6, 1 - 1e-6) | |
| 15 | + out = {"n": int(len(y)), "base_rate": float(y.mean()), "logloss": float(log_loss(y, p)), "brier": float(brier_score_loss(y, p))} | |
| 16 | + if 0 < y.mean() < 1: | |
| 17 | + out["auc"] = float(roc_auc_score(y, p)) | |
| 18 | + out["pr_auc"] = float(average_precision_score(y, p)) | |
| 19 | + # calibration intercept / slope (logistic recalibration of the logit) | |
| 20 | + z = np.log(p / (1 - p)) | |
| 21 | + try: | |
| 22 | + from sklearn.linear_model import LogisticRegression | |
| 23 | + m = LogisticRegression(C=1e6, solver="lbfgs", max_iter=1000).fit(z.reshape(-1, 1), y) | |
| 24 | + out["cal_slope"], out["cal_intercept"] = float(m.coef_[0][0]), float(m.intercept_[0]) | |
| 25 | + except Exception: | |
| 26 | + pass | |
| 27 | + return out | |
| 28 | + | |
| 29 | + | |
| 30 | +def continuous_metrics(y: np.ndarray, yhat: np.ndarray, y_train_mean: float) -> dict: | |
| 31 | + e = y - yhat | |
| 32 | + ss = np.sum((y - y_train_mean) ** 2) | |
| 33 | + return {"n": int(len(y)), "mae": float(np.mean(np.abs(e))), "rmse": float(np.sqrt(np.mean(e ** 2))), | |
| 34 | + "oos_r2": float(1 - np.sum(e ** 2) / ss) if ss > 0 else float("nan")} | |
| 35 | + | |
| 36 | + | |
| 37 | +def pointwise_logloss(y: np.ndarray, p: np.ndarray) -> np.ndarray: | |
| 38 | + p = np.clip(p, 1e-6, 1 - 1e-6) | |
| 39 | + return -(y * np.log(p) + (1 - y) * np.log(1 - p)) | |
| 40 | + | |
| 41 | + | |
| 42 | +def reliability_table(y: np.ndarray, p: np.ndarray, bins: int = 10) -> pd.DataFrame: | |
| 43 | + q = np.quantile(p, np.linspace(0, 1, bins + 1)) | |
| 44 | + q[0], q[-1] = -np.inf, np.inf | |
| 45 | + b = np.digitize(p, q[1:-1]) | |
| 46 | + rows = [] | |
| 47 | + for k in range(bins): | |
| 48 | + m = b == k | |
| 49 | + if m.sum(): | |
| 50 | + rows.append({"decile": k + 1, "n": int(m.sum()), "mean_pred": float(p[m].mean()), "obs_rate": float(y[m].mean())}) | |
| 51 | + return pd.DataFrame(rows) | |
| 52 | + | |
| 53 | + | |
| 54 | +# ------------------------------------------------------------ paired inference | |
| 55 | +def quarter_blocks(dates: np.ndarray) -> np.ndarray: | |
| 56 | + d = pd.to_datetime(dates) | |
| 57 | + return (d.year * 4 + (d.month - 1) // 3).to_numpy() | |
| 58 | + | |
| 59 | + | |
| 60 | +def paired_block_bootstrap(d: np.ndarray, blocks: np.ndarray, n_boot: int = 1000, seed: int = 20261102) -> dict: | |
| 61 | + """Mean of a per-observation loss differential with a cluster (block) bootstrap over calendar quarters. | |
| 62 | + Returns the mean, a percentile 95 % CI and a two-sided bootstrap p-value for H0: mean = 0.""" | |
| 63 | + rng = np.random.default_rng(seed) | |
| 64 | + ub = np.unique(blocks) | |
| 65 | + # per-block sums and counts -> resample blocks with replacement (equal-weight by observation) | |
| 66 | + sums = np.array([d[blocks == b].sum() for b in ub]); cnts = np.array([(blocks == b).sum() for b in ub]) | |
| 67 | + mean = float(d.mean()) | |
| 68 | + draws = np.empty(n_boot) | |
| 69 | + for i in range(n_boot): | |
| 70 | + idx = rng.integers(0, len(ub), len(ub)) | |
| 71 | + draws[i] = sums[idx].sum() / cnts[idx].sum() | |
| 72 | + lo, hi = np.percentile(draws, [2.5, 97.5]) | |
| 73 | + p = float(2 * min((draws - mean <= -abs(mean)).mean(), (draws - mean >= abs(mean)).mean())) | |
| 74 | + return {"mean": mean, "ci_lo": float(lo), "ci_hi": float(hi), "p_boot": p, "n_blocks": int(len(ub)), "n": int(len(d))} | |
| 75 | + | |
| 76 | + | |
| 77 | +def dm_gw_test(d: np.ndarray, blocks: np.ndarray) -> dict: | |
| 78 | + """Unconditional Giacomini-White (= Diebold-Mariano with HAC) on the per-quarter mean loss differential. | |
| 79 | + HAC: Newey-West with lag = floor(4 (T/100)^(2/9)).""" | |
| 80 | + ub = np.unique(blocks) | |
| 81 | + dq = np.array([d[blocks == b].mean() for b in ub]) | |
| 82 | + T = len(dq) | |
| 83 | + if T < 4: | |
| 84 | + return {"t": float("nan"), "p": float("nan"), "T": T} | |
| 85 | + m = dq.mean(); u = dq - m | |
| 86 | + L = int(np.floor(4 * (T / 100) ** (2 / 9))) | |
| 87 | + s2 = u @ u / T | |
| 88 | + for l in range(1, L + 1): | |
| 89 | + w = 1 - l / (L + 1) | |
| 90 | + s2 += 2 * w * (u[l:] @ u[:-l]) / T | |
| 91 | + se = np.sqrt(max(s2, 1e-18) / T) | |
| 92 | + t = m / se | |
| 93 | + return {"t": float(t), "p": float(2 * (1 - stats.t.cdf(abs(t), df=T - 1))), "T": int(T), "mean_quarterly": float(m)} | |
| 94 | + | |
| 95 | + | |
| 96 | +def holm(pvals: dict[str, float]) -> dict[str, float]: | |
| 97 | + items = sorted(pvals.items(), key=lambda kv: kv[1]) | |
| 98 | + m = len(items); out = {}; running = 0.0 | |
| 99 | + for i, (k, p) in enumerate(items): | |
| 100 | + adj = min(1.0, (m - i) * p); running = max(running, adj); out[k] = running | |
| 101 | + return out | |
| 102 | + | |
| 103 | + | |
| 104 | +# ------------------------------------------------------------- panel regression | |
| 105 | +def demean_two_way(df: pd.DataFrame, cols: list[str], fe1: str, fe2: str, iters: int = 25) -> pd.DataFrame: | |
| 106 | + """Alternating projections for two fixed effects (firm, sector x quarter).""" | |
| 107 | + x = df[cols].astype(float).copy() | |
| 108 | + for _ in range(iters): | |
| 109 | + x = x - x.groupby(df[fe1]).transform("mean") | |
| 110 | + x = x - x.groupby(df[fe2]).transform("mean") | |
| 111 | + return x | |
| 112 | + | |
| 113 | + | |
| 114 | +def two_way_cluster_ols(df: pd.DataFrame, y: str, xs: list[str], fe1: str, fe2: str, cl1: str, cl2: str) -> pd.DataFrame: | |
| 115 | + """OLS with two-way FE (within transformation) and Cameron-Gelbach-Miller two-way clustered SEs.""" | |
| 116 | + d = df.dropna(subset=[y] + xs + [fe1, fe2, cl1, cl2]).copy() | |
| 117 | + z = demean_two_way(d, [y] + xs, fe1, fe2) | |
| 118 | + X = z[xs].to_numpy(); Y = z[y].to_numpy() | |
| 119 | + XtX_inv = np.linalg.pinv(X.T @ X) | |
| 120 | + beta = XtX_inv @ X.T @ Y | |
| 121 | + e = Y - X @ beta | |
| 122 | + def meat(groups): | |
| 123 | + S = np.zeros((X.shape[1], X.shape[1])) | |
| 124 | + for _, idx in pd.Series(np.arange(len(groups))).groupby(groups.to_numpy()).groups.items(): | |
| 125 | + idx = np.asarray(list(idx)); s = X[idx].T @ e[idx]; S += np.outer(s, s) | |
| 126 | + return S | |
| 127 | + g12 = (d[cl1].astype(str) + "|" + d[cl2].astype(str)) | |
| 128 | + V = XtX_inv @ (meat(d[cl1]) + meat(d[cl2]) - meat(g12)) @ XtX_inv | |
| 129 | + se = np.sqrt(np.clip(np.diag(V), 0, None)) | |
| 130 | + t = beta / se | |
| 131 | + G = min(d[cl1].nunique(), d[cl2].nunique()) | |
| 132 | + p = 2 * (1 - stats.t.cdf(np.abs(t), df=max(G - 1, 1))) | |
| 133 | + return pd.DataFrame({"var": xs, "coef": beta, "se": se, "t": t, "p": p, "n": len(d), "n_firms": d[cl1].nunique(), | |
| 134 | + "n_quarters": d[cl2].nunique()}) | |
added
src/wp39/ingest.py
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +"""Bounded, idempotent FinData exports of raw 1-minute bars (<= 12 years per request, one symbol per request).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import datetime as dt | |
| 5 | +from pathlib import Path | |
| 6 | + | |
| 7 | +import pyarrow.parquet as pq | |
| 8 | + | |
| 9 | +from .api_client import FinDataClient, export_result_dict | |
| 10 | +from .provenance import append_jsonl, manifest_index | |
| 11 | + | |
| 12 | +BARS_FIELDS = ["instrument_id", "symbol_as_reported", "venue", "asset_class", "interval", "timestamp_utc", "session_date", | |
| 13 | + "open", "high", "low", "close", "volume", "currency", "trade_count", "vwap", "adjustment_type", "source_id", | |
| 14 | + "ingested_at", "quality_flags", "revision_id"] | |
| 15 | + | |
| 16 | + | |
| 17 | +def chunk_ranges(start: dt.date, end: dt.date, max_years: int = 12) -> list[tuple[dt.date, dt.date]]: | |
| 18 | + out, a = [], start | |
| 19 | + while a <= end: | |
| 20 | + b = min(end, dt.date(a.year + max_years - 1, 12, 31)) | |
| 21 | + out.append((a, b)) | |
| 22 | + a = dt.date(b.year + 1, 1, 1) | |
| 23 | + return out | |
| 24 | + | |
| 25 | + | |
| 26 | +def validate_bars_parquet(path: Path) -> dict: | |
| 27 | + meta = pq.read_metadata(path) | |
| 28 | + schema = pq.read_schema(path) | |
| 29 | + names = schema.names | |
| 30 | + if names != BARS_FIELDS: | |
| 31 | + raise ValueError(f"unexpected bars schema in {path.name}: {names}") | |
| 32 | + md = {k.decode(): v.decode() for k, v in (schema.metadata or {}).items()} | |
| 33 | + if md.get("findata.schema") != "bars" or md.get("findata.timestamp_convention") != "bar_open": | |
| 34 | + raise ValueError(f"missing/unexpected FinData schema metadata in {path.name}: {md}") | |
| 35 | + return {"rows": meta.num_rows, "schema_version": md.get("findata.schema_version")} | |
| 36 | + | |
| 37 | + | |
| 38 | +def export_symbol(client: FinDataClient, symbol: str, start: dt.date, end: dt.date, dataset_id: str, cache_dir: Path, | |
| 39 | + manifest: Path, keep: bool = False) -> list[Path]: | |
| 40 | + """Export all chunks of one symbol; skip chunks already in the manifest when the file is still present (or keep=False | |
| 41 | + and the chunk was already consumed). Returns the list of Parquet paths available on disk.""" | |
| 42 | + idx = manifest_index(manifest, ["symbol", "start", "end", "dataset_id"]) | |
| 43 | + paths = [] | |
| 44 | + for a, b in chunk_ranges(start, end): | |
| 45 | + dest = cache_dir / f"{symbol}__{a}__{b}.parquet" | |
| 46 | + key = (symbol, str(a), str(b), dataset_id) | |
| 47 | + if dest.exists(): | |
| 48 | + paths.append(dest) | |
| 49 | + continue | |
| 50 | + if key in idx and idx[key].get("consumed") and not keep: | |
| 51 | + # already processed in a previous run; downstream will find the feature rows | |
| 52 | + continue | |
| 53 | + res = client.export(dataset_id, [symbol], str(a), str(b), dest) | |
| 54 | + info = validate_bars_parquet(dest) | |
| 55 | + rec = export_result_dict(res) | |
| 56 | + rec.update({"symbol": symbol, "rows_in_file": info["rows"], "schema_version": info["schema_version"], "consumed": False}) | |
| 57 | + append_jsonl(manifest, rec) | |
| 58 | + paths.append(dest) | |
| 59 | + return paths | |
| 60 | + | |
| 61 | + | |
| 62 | +def mark_consumed(manifest: Path, symbol: str, dataset_id: str) -> None: | |
| 63 | + append_jsonl(manifest, {"symbol": symbol, "dataset_id": dataset_id, "consumed": True, "event": "features_written"}) | |
added
src/wp39/instruments.py
+119 −0
@@ -0,0 +1,119 @@ | ||
| 1 | +"""Instrument master and identifier crosswalk: FinData entity/instrument ↔ ticker ↔ SEC CIK/SIC. | |
| 2 | + | |
| 3 | +SEC data are public; requests carry the fair-access User-Agent (no secret) and are throttled to <= 8 req/s. | |
| 4 | +""" | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import gzip | |
| 8 | +import json | |
| 9 | +import os | |
| 10 | +import time | |
| 11 | +from pathlib import Path | |
| 12 | + | |
| 13 | +import httpx | |
| 14 | +import polars as pl | |
| 15 | + | |
| 16 | +SEC_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json" | |
| 17 | +SEC_SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik:010d}.json" | |
| 18 | + | |
| 19 | +# SIC divisions (US Department of Labor): used as the coarse "sector" for fixed effects and demeaning. | |
| 20 | +SIC_DIVISIONS = [ | |
| 21 | + (100, 999, "A_agriculture"), (1000, 1499, "B_mining"), (1500, 1799, "C_construction"), (2000, 3999, "D_manufacturing"), | |
| 22 | + (4000, 4999, "E_transport_utilities"), (5000, 5199, "F_wholesale"), (5200, 5999, "G_retail"), | |
| 23 | + (6000, 6799, "H_finance_insurance_realestate"), (7000, 8999, "I_services"), (9100, 9729, "J_public_admin"), | |
| 24 | +] | |
| 25 | + | |
| 26 | + | |
| 27 | +def sic_division(sic: int | None) -> str: | |
| 28 | + if sic is None: | |
| 29 | + return "unknown" | |
| 30 | + for lo, hi, name in SIC_DIVISIONS: | |
| 31 | + if lo <= sic <= hi: | |
| 32 | + return name | |
| 33 | + return "unknown" | |
| 34 | + | |
| 35 | + | |
| 36 | +def _ua() -> str: | |
| 37 | + return os.environ.get("SEC_USER_AGENT", "UQO WP39 research (mailto:simon-pierre.boucher@uqo.ca)") | |
| 38 | + | |
| 39 | + | |
| 40 | +def _get_json(client: httpx.Client, url: str) -> dict: | |
| 41 | + r = client.get(url, headers={"User-Agent": _ua(), "Accept-Encoding": "gzip, deflate"}) | |
| 42 | + r.raise_for_status() | |
| 43 | + raw = r.content | |
| 44 | + if raw[:2] == b"\x1f\x8b": | |
| 45 | + raw = gzip.decompress(raw) | |
| 46 | + return json.loads(raw) | |
| 47 | + | |
| 48 | + | |
| 49 | +def fetch_sec_tickers(cache_dir: Path) -> pl.DataFrame: | |
| 50 | + """Current SEC ticker→CIK table (snapshot cached with its download date).""" | |
| 51 | + cache_dir.mkdir(parents=True, exist_ok=True) | |
| 52 | + f = cache_dir / "sec_company_tickers.json" | |
| 53 | + if not f.exists(): | |
| 54 | + with httpx.Client(timeout=60) as c: | |
| 55 | + d = _get_json(c, SEC_TICKERS_URL) | |
| 56 | + f.write_text(json.dumps({"downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "data": d})) | |
| 57 | + d = json.loads(f.read_text())["data"] | |
| 58 | + rows = [{"ticker": v["ticker"].upper(), "cik": int(v["cik_str"]), "sec_name": v["title"]} for v in d.values()] | |
| 59 | + return pl.DataFrame(rows) | |
| 60 | + | |
| 61 | + | |
| 62 | +def fetch_sec_submissions(ciks: list[int], cache_dir: Path, rate_per_s: float = 8.0, log=None, workers: int = 6) -> pl.DataFrame: | |
| 63 | + """SIC code, exchanges, fiscal year end and former names per CIK (cached per CIK; global token-bucket rate limit).""" | |
| 64 | + import threading | |
| 65 | + from concurrent.futures import ThreadPoolExecutor | |
| 66 | + | |
| 67 | + out_dir = cache_dir / "sec_submissions" | |
| 68 | + out_dir.mkdir(parents=True, exist_ok=True) | |
| 69 | + lock, state = threading.Lock(), {"last": 0.0} | |
| 70 | + min_gap = 1.0 / rate_per_s | |
| 71 | + | |
| 72 | + def fetch_one(cik: int) -> None: | |
| 73 | + f = out_dir / f"CIK{cik:010d}.json" | |
| 74 | + if f.exists(): | |
| 75 | + return | |
| 76 | + with lock: | |
| 77 | + wait = min_gap - (time.time() - state["last"]) | |
| 78 | + if wait > 0: | |
| 79 | + time.sleep(wait) | |
| 80 | + state["last"] = time.time() | |
| 81 | + with httpx.Client(timeout=60) as c: | |
| 82 | + try: | |
| 83 | + d = _get_json(c, SEC_SUBMISSIONS_URL.format(cik=cik)) | |
| 84 | + except httpx.HTTPStatusError as exc: | |
| 85 | + d = {"error": exc.response.status_code} | |
| 86 | + except Exception as exc: # noqa: BLE001 | |
| 87 | + d = {"error": type(exc).__name__} | |
| 88 | + slim = {k: d.get(k) for k in ("cik", "sic", "sicDescription", "tickers", "exchanges", "fiscalYearEnd", | |
| 89 | + "formerNames", "name", "stateOfIncorporation", "error")} | |
| 90 | + f.write_text(json.dumps(slim)) | |
| 91 | + | |
| 92 | + with ThreadPoolExecutor(max_workers=workers) as ex: | |
| 93 | + for i, _ in enumerate(ex.map(fetch_one, ciks)): | |
| 94 | + if log and i % 500 == 0: | |
| 95 | + log.info("SEC submissions %d/%d", i, len(ciks)) | |
| 96 | + rows = [] | |
| 97 | + for cik in ciks: | |
| 98 | + d = json.loads((out_dir / f"CIK{cik:010d}.json").read_text()) | |
| 99 | + if d.get("error") and not d.get("sic"): | |
| 100 | + (out_dir / f"CIK{cik:010d}.json").unlink(missing_ok=True) if d.get("error") not in (404,) else None | |
| 101 | + sic = int(d["sic"]) if d.get("sic") not in (None, "") else None | |
| 102 | + rows.append({"cik": cik, "sic": sic, "sic_description": d.get("sicDescription"), | |
| 103 | + "sic_division": sic_division(sic), "sec_tickers": d.get("tickers") or [], | |
| 104 | + "exchanges": d.get("exchanges") or [], "fiscal_year_end": d.get("fiscalYearEnd"), | |
| 105 | + "n_former_names": len(d.get("formerNames") or []), "sec_error": d.get("error")}) | |
| 106 | + return pl.DataFrame(rows) | |
| 107 | + | |
| 108 | + | |
| 109 | +def build_crosswalk(entities: pl.DataFrame, sec_tickers: pl.DataFrame, submissions: pl.DataFrame | None) -> pl.DataFrame: | |
| 110 | + """entities: entity_id, security_id, ticker (from FinData). Effective-date ranges are the fundamentals span.""" | |
| 111 | + xw = entities.join(sec_tickers.unique(subset=["ticker"]), on="ticker", how="left") | |
| 112 | + if submissions is not None: | |
| 113 | + xw = xw.join(submissions, on="cik", how="left") | |
| 114 | + xw = xw.with_columns([ | |
| 115 | + pl.col("cik").is_not_null().alias("has_cik"), | |
| 116 | + pl.when(pl.col("cik").is_null()).then(pl.lit("unknown")).otherwise(pl.col("sic_division").fill_null("unknown")).alias("sic_division") | |
| 117 | + if submissions is not None else pl.lit("unknown").alias("sic_division"), | |
| 118 | + ]) | |
| 119 | + return xw | |
added
src/wp39/labels.py
+104 −0
@@ -0,0 +1,104 @@ | ||
| 1 | +"""Firm-quarter outcomes: reveal dates, seasonal-random-walk expectations and adverse-outcome labels (PIT_PROTOCOL §4). | |
| 2 | + | |
| 3 | +Nothing here uses prices. The expectation E_c[X_q] uses quarters q-1..q-8 *restricted to those available at the cutoff*; | |
| 4 | +samples.py passes the cutoff-specific availability mask. The quarterly series X_q is the first (only) revision. | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import datetime as dt | |
| 9 | + | |
| 10 | +import polars as pl | |
| 11 | + | |
| 12 | +TARGET_VARS = ["op_inc", "net_inc", "cash_op", "rev"] | |
| 13 | + | |
| 14 | + | |
| 15 | +def match_release_dates(wide: pl.DataFrame, events: pl.DataFrame, max_lag_days: int = 30) -> pl.DataFrame: | |
| 16 | + """Attach A_q (first release in [E_q, F_q + 10 d]) and its timing to each firm-quarter; flag exclusions.""" | |
| 17 | + e = events.select(["entity_id", pl.col("event_date").cast(pl.Date), "release_time", "eps_estimate", "eps_reported"]).sort( | |
| 18 | + ["entity_id", "event_date"]) | |
| 19 | + w = wide.select(["entity_id", "fiscal_period_end", "filing_date"]).with_columns( | |
| 20 | + pl.col("fiscal_period_end").alias("_key")).sort(["entity_id", "_key"]) | |
| 21 | + # forward as-of: first event on/after the fiscal period end | |
| 22 | + j = w.join_asof(e, left_on="_key", right_on="event_date", by="entity_id", strategy="forward") | |
| 23 | + j = j.with_columns([ | |
| 24 | + pl.col("event_date").alias("A_q"), | |
| 25 | + (pl.col("event_date") <= pl.col("filing_date") + pl.duration(days=10)).alias("_within"), | |
| 26 | + ]) | |
| 27 | + j = j.with_columns([ | |
| 28 | + pl.when(pl.col("_within")).then(pl.col("A_q")).otherwise(None).alias("A_q"), | |
| 29 | + pl.when(pl.col("_within")).then(pl.col("release_time")).otherwise(None).alias("release_time"), | |
| 30 | + pl.when(pl.col("_within")).then(pl.col("eps_estimate")).otherwise(None).alias("eps_estimate"), | |
| 31 | + pl.when(pl.col("_within")).then(pl.col("eps_reported")).otherwise(None).alias("eps_reported"), | |
| 32 | + ]) | |
| 33 | + j = j.with_columns([ | |
| 34 | + (pl.col("filing_date") - pl.col("A_q")).dt.total_days().alias("filing_minus_release_days"), | |
| 35 | + ]) | |
| 36 | + j = j.with_columns([ | |
| 37 | + pl.when(pl.col("A_q").is_null()).then(pl.lit("no_release")) | |
| 38 | + .when(pl.col("filing_minus_release_days") < 0).then(pl.lit("filing_before_release")) | |
| 39 | + .when(pl.col("filing_minus_release_days") > max_lag_days).then(pl.lit("release_lag_gt_30d")) | |
| 40 | + .otherwise(pl.lit("ok")).alias("release_match"), | |
| 41 | + ]) | |
| 42 | + return wide.join(j.select(["entity_id", "fiscal_period_end", "A_q", "release_time", "eps_estimate", "eps_reported", | |
| 43 | + "filing_minus_release_days", "release_match"]), on=["entity_id", "fiscal_period_end"], how="left") | |
| 44 | + | |
| 45 | + | |
| 46 | +def seasonal_expectation(w: pl.DataFrame, var: str) -> pl.DataFrame: | |
| 47 | + """E[X_q] = X_{q-4} + mean_{k=0..3}(X_{q-1-k} - X_{q-5-k}); requires q-1..q-8 (by quarter index, regular calendar).""" | |
| 48 | + x = pl.col(var) | |
| 49 | + lags = {k: x.shift(k).over("entity_id") for k in range(1, 9)} | |
| 50 | + drift = sum((lags[1 + k] - lags[5 + k]) for k in range(4)) / 4.0 | |
| 51 | + return w.with_columns([ | |
| 52 | + (lags[4] + drift).alias(f"E_{var}"), | |
| 53 | + lags[4].alias(f"{var}_lag4"), | |
| 54 | + lags[1].alias(f"{var}_lag1"), | |
| 55 | + (x - (lags[4] + drift)).alias(f"surprise_{var}"), | |
| 56 | + ]) | |
| 57 | + | |
| 58 | + | |
| 59 | +def build_quarter_table(wide: pl.DataFrame) -> pl.DataFrame: | |
| 60 | + """Quarter-level target ingredients (unscaled). Regularity: consecutive quarters 60-120 days apart.""" | |
| 61 | + w = wide.sort(["entity_id", "fiscal_period_end"]) | |
| 62 | + w = w.with_columns(pl.col("fiscal_period_end").diff().over("entity_id").dt.total_days().alias("days_since_prev_q")) | |
| 63 | + # a quarter is "regular" if the previous 8 gaps are all within [60, 120] | |
| 64 | + ok = (pl.col("days_since_prev_q") >= 60) & (pl.col("days_since_prev_q") <= 120) | |
| 65 | + w = w.with_columns(ok.cast(pl.Int8).alias("_gap_ok")) | |
| 66 | + w = w.with_columns(pl.col("_gap_ok").rolling_sum(window_size=8, min_samples=8).over("entity_id").alias("_gap_ok8")) | |
| 67 | + for v in TARGET_VARS: | |
| 68 | + w = seasonal_expectation(w, v) | |
| 69 | + # secondary ingredients | |
| 70 | + w = w.with_columns([ | |
| 71 | + pl.col("roa").shift(4).over("entity_id").alias("roa_lag4"), | |
| 72 | + (pl.col("assets_c") / (pl.col("liab_c").abs() + 1e-9)).alias("current_ratio"), | |
| 73 | + ]) | |
| 74 | + w = w.with_columns([ | |
| 75 | + pl.col("current_ratio").shift(4).over("entity_id").alias("current_ratio_lag4"), | |
| 76 | + pl.col("intr_cover").shift(4).over("entity_id").alias("intr_cover_lag4"), | |
| 77 | + pl.col("assets_tot").shift(1).over("entity_id").alias("assets_tot_lag1"), | |
| 78 | + ]) | |
| 79 | + return w | |
| 80 | + | |
| 81 | + | |
| 82 | +def finalize_labels(q: pl.DataFrame, threshold: float = -0.005) -> pl.DataFrame: | |
| 83 | + """Scale by the latest *available* total assets (lag-1 quarter) and build binary targets.""" | |
| 84 | + ta = pl.col("assets_tot_lag1").abs() + 1e-9 | |
| 85 | + q = q.with_columns([ | |
| 86 | + (pl.col("surprise_op_inc") / ta).alias("S_opinc"), | |
| 87 | + (pl.col("surprise_net_inc") / ta).alias("S_netinc"), | |
| 88 | + (pl.col("surprise_cash_op") / ta).alias("S_ocf"), | |
| 89 | + (pl.col("surprise_rev") / ta).alias("S_rev"), | |
| 90 | + ((pl.col("cash_op") - pl.col("cash_op_lag4")) / ta).alias("ocf_innov"), | |
| 91 | + (pl.col("roa") - pl.col("roa_lag4")).alias("roa_yoy"), | |
| 92 | + ]) | |
| 93 | + q = q.with_columns([ | |
| 94 | + (pl.col("S_opinc") <= threshold).cast(pl.Int8).alias("y_adverse_opinc"), | |
| 95 | + (pl.col("S_netinc") <= threshold).cast(pl.Int8).alias("y_adverse_netinc"), | |
| 96 | + (pl.col("roa_yoy") <= -0.01).cast(pl.Int8).alias("y_roa_decline"), | |
| 97 | + (pl.col("ocf_innov") <= -0.01).cast(pl.Int8).alias("y_ocf_negative"), | |
| 98 | + (((pl.col("current_ratio") / (pl.col("current_ratio_lag4") + 1e-9)) <= 0.9) | | |
| 99 | + ((pl.col("intr_cover") < 1.0) & (pl.col("intr_cover_lag4") >= 1.0))).cast(pl.Int8).alias("y_liquidity_deterioration"), | |
| 100 | + pl.when(pl.col("eps_estimate").is_not_null() & pl.col("eps_reported").is_not_null()) | |
| 101 | + .then((pl.col("eps_reported") < pl.col("eps_estimate")).cast(pl.Int8)).otherwise(None).alias("y_eps_miss"), | |
| 102 | + (pl.col("S_opinc").abs() >= 0.005).cast(pl.Int8).alias("y_abs_surprise"), # symmetric placebo target (volatility channel) | |
| 103 | + ]) | |
| 104 | + return q | |
added
src/wp39/models.py
+137 −0
@@ -0,0 +1,137 @@ | ||
| 1 | +"""Baseline ladder B0-B6 with train-only preprocessing and an equal tuning budget per information set.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import json | |
| 5 | +import random | |
| 6 | +from dataclasses import dataclass, field | |
| 7 | + | |
| 8 | +import numpy as np | |
| 9 | +import polars as pl | |
| 10 | +from sklearn.linear_model import LogisticRegression, Ridge | |
| 11 | +from sklearn.metrics import log_loss | |
| 12 | + | |
| 13 | +INFO_SETS = { | |
| 14 | + "B1": ["fundamentals"], | |
| 15 | + "B2": ["fundamentals", "daily"], | |
| 16 | + "B3": ["fundamentals", "daily", "intraday_30m"], | |
| 17 | + "B4": ["fundamentals", "daily", "intraday_5m"], | |
| 18 | + "B5": ["fundamentals", "daily", "intraday_1m"], | |
| 19 | + "B5all": ["fundamentals", "daily", "intraday_30m", "intraday_5m", "intraday_1m"], | |
| 20 | +} | |
| 21 | + | |
| 22 | + | |
| 23 | +@dataclass | |
| 24 | +class Preprocessor: | |
| 25 | + """Winsorise (1/99 %), impute (median) and standardise using TRAINING statistics only.""" | |
| 26 | + cols: list[str] | |
| 27 | + lo: dict = field(default_factory=dict) | |
| 28 | + hi: dict = field(default_factory=dict) | |
| 29 | + med: dict = field(default_factory=dict) | |
| 30 | + mu: dict = field(default_factory=dict) | |
| 31 | + sd: dict = field(default_factory=dict) | |
| 32 | + fitted_on_rows: int = 0 | |
| 33 | + | |
| 34 | + def fit(self, X: pl.DataFrame) -> "Preprocessor": | |
| 35 | + for c in self.cols: | |
| 36 | + s = X[c].cast(pl.Float64) | |
| 37 | + s = s.filter(s.is_finite()) | |
| 38 | + if s.len() == 0: | |
| 39 | + self.lo[c], self.hi[c], self.med[c], self.mu[c], self.sd[c] = 0.0, 0.0, 0.0, 0.0, 1.0 | |
| 40 | + continue | |
| 41 | + lo, hi = float(s.quantile(0.01)), float(s.quantile(0.99)) | |
| 42 | + w = s.clip(lo, hi) | |
| 43 | + self.lo[c], self.hi[c], self.med[c] = lo, hi, float(w.median()) | |
| 44 | + self.mu[c] = float(w.mean()) | |
| 45 | + sd = float(w.std()) if w.len() > 1 else 1.0 | |
| 46 | + self.sd[c] = sd if sd and sd > 1e-12 else 1.0 | |
| 47 | + self.fitted_on_rows = X.height | |
| 48 | + return self | |
| 49 | + | |
| 50 | + def transform(self, X: pl.DataFrame) -> np.ndarray: | |
| 51 | + out = np.empty((X.height, len(self.cols)), dtype=np.float64) | |
| 52 | + for j, c in enumerate(self.cols): | |
| 53 | + v = X[c].cast(pl.Float64).to_numpy().astype(np.float64) | |
| 54 | + v = np.where(np.isfinite(v), v, np.nan) | |
| 55 | + v = np.clip(v, self.lo[c], self.hi[c]) | |
| 56 | + v = np.where(np.isnan(v), self.med[c], v) | |
| 57 | + out[:, j] = (v - self.mu[c]) / self.sd[c] | |
| 58 | + return out | |
| 59 | + | |
| 60 | + def to_dict(self) -> dict: | |
| 61 | + return {"cols": self.cols, "lo": self.lo, "hi": self.hi, "med": self.med, "mu": self.mu, "sd": self.sd, | |
| 62 | + "fitted_on_rows": self.fitted_on_rows} | |
| 63 | + | |
| 64 | + | |
| 65 | +def columns_for(info_set: str, fc: dict[str, list[str]]) -> list[str]: | |
| 66 | + cols = [] | |
| 67 | + for fam in INFO_SETS[info_set]: | |
| 68 | + cols += fc[fam] | |
| 69 | + return cols | |
| 70 | + | |
| 71 | + | |
| 72 | +# ----------------------------------------------------------------------------- B0 | |
| 73 | +def base_rate_predict(train_y: np.ndarray, n: int) -> np.ndarray: | |
| 74 | + return np.full(n, float(np.clip(train_y.mean(), 1e-4, 1 - 1e-4))) | |
| 75 | + | |
| 76 | + | |
| 77 | +def firm_history_predict(train: pl.DataFrame, test: pl.DataFrame, y: str) -> np.ndarray: | |
| 78 | + """B0b: trailing firm-level adverse rate (from training rows only, shrunk to the pooled rate with prior weight 8).""" | |
| 79 | + pooled = float(train[y].mean()) | |
| 80 | + g = train.group_by("entity_id").agg([pl.col(y).sum().alias("k"), pl.len().alias("n")]) | |
| 81 | + t = test.select("entity_id").join(g, on="entity_id", how="left").with_columns([pl.col("k").fill_null(0), pl.col("n").fill_null(0)]) | |
| 82 | + k, n = t["k"].to_numpy().astype(float), t["n"].to_numpy().astype(float) | |
| 83 | + return np.clip((k + 8 * pooled) / (n + 8), 1e-4, 1 - 1e-4) | |
| 84 | + | |
| 85 | + | |
| 86 | +# ------------------------------------------------------------------------ logistic | |
| 87 | +def fit_logistic(Xtr: np.ndarray, ytr: np.ndarray, Xva: np.ndarray, yva: np.ndarray, C_grid: list[float], seed: int) -> tuple: | |
| 88 | + best, best_ll, best_C = None, np.inf, None | |
| 89 | + for C in C_grid: | |
| 90 | + m = LogisticRegression(C=C, penalty="l2", solver="lbfgs", max_iter=2000, random_state=seed) | |
| 91 | + m.fit(Xtr, ytr) | |
| 92 | + ll = log_loss(yva, np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6)) | |
| 93 | + if ll < best_ll: | |
| 94 | + best, best_ll, best_C = m, ll, C | |
| 95 | + return best, {"C": best_C, "val_logloss": best_ll} | |
| 96 | + | |
| 97 | + | |
| 98 | +def fit_ridge(Xtr, ytr, Xva, yva, alpha_grid=(0.1, 1, 10, 100, 1000)): | |
| 99 | + best, best_mse, best_a = None, np.inf, None | |
| 100 | + for a in alpha_grid: | |
| 101 | + m = Ridge(alpha=a).fit(Xtr, ytr) | |
| 102 | + mse = float(np.mean((m.predict(Xva) - yva) ** 2)) | |
| 103 | + if mse < best_mse: | |
| 104 | + best, best_mse, best_a = m, mse, a | |
| 105 | + return best, {"alpha": best_a, "val_mse": best_mse} | |
| 106 | + | |
| 107 | + | |
| 108 | +# ------------------------------------------------------------------------ LightGBM | |
| 109 | +def _sample_params(space: dict, rng: random.Random) -> dict: | |
| 110 | + return {k: rng.choice(v) for k, v in space.items()} | |
| 111 | + | |
| 112 | + | |
| 113 | +def fit_lightgbm(Xtr, ytr, Xva, yva, cfg: dict, seed: int, objective: str = "binary", n_jobs: int = 8) -> tuple: | |
| 114 | + import lightgbm as lgb | |
| 115 | + rng = random.Random(seed) | |
| 116 | + best, best_score, best_params, trials = None, np.inf, None, [] | |
| 117 | + for t in range(int(cfg["budget_trials"])): | |
| 118 | + p = _sample_params(cfg["search_space"], rng) | |
| 119 | + params = dict(objective=objective, learning_rate=p["learning_rate"], num_leaves=p["num_leaves"], | |
| 120 | + min_child_samples=p["min_child_samples"], feature_fraction=p["feature_fraction"], | |
| 121 | + bagging_fraction=p["bagging_fraction"], bagging_freq=1 if p["bagging_fraction"] < 1 else 0, | |
| 122 | + lambda_l2=p["lambda_l2"], n_estimators=int(cfg["n_estimators_max"]), random_state=seed + t, | |
| 123 | + n_jobs=n_jobs, verbose=-1) | |
| 124 | + m = lgb.LGBMClassifier(**params) if objective == "binary" else lgb.LGBMRegressor(**params) | |
| 125 | + m.fit(Xtr, ytr, eval_set=[(Xva, yva)], callbacks=[lgb.early_stopping(int(cfg["early_stopping_rounds"]), verbose=False)]) | |
| 126 | + if objective == "binary": | |
| 127 | + score = log_loss(yva, np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6)) | |
| 128 | + else: | |
| 129 | + score = float(np.mean((m.predict(Xva) - yva) ** 2)) | |
| 130 | + trials.append({"trial": t, "params": p, "best_iteration": int(m.best_iteration_ or 0), "val_score": score}) | |
| 131 | + if score < best_score: | |
| 132 | + best, best_score, best_params = m, score, {**p, "best_iteration": int(m.best_iteration_ or 0)} | |
| 133 | + return best, {"val_score": best_score, "params": best_params, "trials": trials} | |
| 134 | + | |
| 135 | + | |
| 136 | +def predict_proba(model, X: np.ndarray) -> np.ndarray: | |
| 137 | + return np.clip(model.predict_proba(X)[:, 1], 1e-6, 1 - 1e-6) | |
added
src/wp39/pipeline.py
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +"""Shared configuration, paths and logging for the WP39 pipeline.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import logging | |
| 5 | +import os | |
| 6 | +from dataclasses import dataclass | |
| 7 | +from pathlib import Path | |
| 8 | + | |
| 9 | +import yaml | |
| 10 | + | |
| 11 | +ROOT = Path(__file__).resolve().parents[2] | |
| 12 | + | |
| 13 | + | |
| 14 | +@dataclass(frozen=True) | |
| 15 | +class Paths: | |
| 16 | + root: Path | |
| 17 | + configs: Path | |
| 18 | + artifacts: Path | |
| 19 | + manifests: Path | |
| 20 | + logs: Path | |
| 21 | + evaluation: Path | |
| 22 | + models: Path | |
| 23 | + processed: Path | |
| 24 | + results: Path | |
| 25 | + figures: Path | |
| 26 | + paper_tables: Path | |
| 27 | + paper_figures: Path | |
| 28 | + cache: Path | |
| 29 | + | |
| 30 | + @classmethod | |
| 31 | + def default(cls) -> "Paths": | |
| 32 | + cache = Path(os.path.expanduser(os.environ.get("WP39_CACHE", "~/wp39-cache"))) | |
| 33 | + p = cls(root=ROOT, configs=ROOT / "configs", artifacts=ROOT / "artifacts", manifests=ROOT / "artifacts/manifests", | |
| 34 | + logs=ROOT / "artifacts/logs", evaluation=ROOT / "artifacts/evaluation", models=ROOT / "artifacts/models", | |
| 35 | + processed=ROOT / "data/processed", results=ROOT / "results", figures=ROOT / "figures", | |
| 36 | + paper_tables=ROOT / "paper/tables", paper_figures=ROOT / "paper/figures", cache=cache) | |
| 37 | + for d in (p.manifests, p.logs, p.evaluation, p.models, p.processed, p.results, p.figures, p.paper_tables, | |
| 38 | + p.paper_figures, p.cache / "bars", p.cache / "fundamentals", p.cache / "features", p.cache / "external"): | |
| 39 | + d.mkdir(parents=True, exist_ok=True) | |
| 40 | + return p | |
| 41 | + | |
| 42 | + | |
| 43 | +def load_config(name: str) -> dict: | |
| 44 | + with open(ROOT / "configs" / f"{name}.yaml") as fh: | |
| 45 | + return yaml.safe_load(fh) | |
| 46 | + | |
| 47 | + | |
| 48 | +def get_logger(name: str, logfile: Path | None = None) -> logging.Logger: | |
| 49 | + log = logging.getLogger(name) | |
| 50 | + if log.handlers: | |
| 51 | + return log | |
| 52 | + log.setLevel(logging.INFO) | |
| 53 | + fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") | |
| 54 | + h = logging.StreamHandler() | |
| 55 | + h.setFormatter(fmt) | |
| 56 | + log.addHandler(h) | |
| 57 | + if logfile: | |
| 58 | + logfile.parent.mkdir(parents=True, exist_ok=True) | |
| 59 | + fh = logging.FileHandler(logfile) | |
| 60 | + fh.setFormatter(fmt) | |
| 61 | + log.addHandler(fh) | |
| 62 | + return log | |
added
src/wp39/pit_fundamentals.py
+115 −0
@@ -0,0 +1,115 @@ | ||
| 1 | +"""Point-in-time fundamentals: extraction, wide quarterly panel, and as-of joins (docs/PIT_PROTOCOL.md). | |
| 2 | + | |
| 3 | +The API's `as_of` semantics are replicated locally on the full extract (available_at <= as_of and revision 1) so that | |
| 4 | +one call per entity suffices; `tests/test_api_contract.py::test_asof_equivalence` checks the equivalence live. | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import datetime as dt | |
| 9 | +from pathlib import Path | |
| 10 | + | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | +from .api_client import FinDataClient | |
| 14 | +from .provenance import append_jsonl, now_iso, sha256_file | |
| 15 | + | |
| 16 | +METRICS = ["rev", "cogs", "gp", "op_inc", "op_ex", "ebit", "ebitda", "net_inc", "net_inc_com", "eps", "eps_d", | |
| 17 | + "cash_op", "capex", "cash_inv", "cash_fin", "cash", "assets_c", "assets_tot", "liab_c", "liab_tot", "debt", | |
| 18 | + "debt_c", "debt_nc", "equity", "receivables", "inventory", "payables", "ppe", "intangibles", "research_dev", | |
| 19 | + "sgna", "interest_exp", "intr_cover", "depr", "shares", "shares_av", "shares_av_diluted", "roa", "roe", | |
| 20 | + "roic", "gm", "om", "ret_earnings", "cash_div", "dps", "share_comp", "tax_expense"] | |
| 21 | + | |
| 22 | +FACT_COLS = ["entity_id", "security_id", "metric", "value", "unit", "fiscal_period_start", "fiscal_period_end", | |
| 23 | + "fiscal_period_type", "fiscal_year", "fiscal_period", "filing_date", "available_at", "observed_at", | |
| 24 | + "revision_id", "restatement_flag", "superseded_at", "source_document"] | |
| 25 | + | |
| 26 | + | |
| 27 | +def fetch_entity_facts(client: FinDataClient, entity_id: str, dest: Path, manifest: Path) -> pl.DataFrame: | |
| 28 | + """All quarterly facts of one entity (no as_of → full history, current revision). Cached as Parquet.""" | |
| 29 | + if dest.exists(): | |
| 30 | + return pl.read_parquet(dest) | |
| 31 | + d = client.fundamentals(entity_id, as_of=None, period_type="q", limit=20000) | |
| 32 | + rows = d.get("data", []) | |
| 33 | + if len(rows) >= 20000: | |
| 34 | + raise RuntimeError(f"{entity_id}: fact count hit the API limit; narrow the metrics") | |
| 35 | + df = pl.DataFrame([{k: r.get(k) for k in FACT_COLS} for r in rows]) if rows else pl.DataFrame(schema={c: pl.Utf8 for c in FACT_COLS}) | |
| 36 | + if rows: | |
| 37 | + df = df.with_columns([ | |
| 38 | + pl.col("value").cast(pl.Float64), pl.col("fiscal_year").cast(pl.Int32), pl.col("revision_id").cast(pl.Int32), | |
| 39 | + pl.col("fiscal_period_end").str.to_date(), pl.col("fiscal_period_start").str.to_date(), | |
| 40 | + pl.col("filing_date").str.to_date(), | |
| 41 | + pl.col("available_at").str.to_datetime(time_zone="UTC", strict=False), | |
| 42 | + pl.col("observed_at").str.to_datetime(time_zone="UTC", strict=False), | |
| 43 | + ]) | |
| 44 | + dest.parent.mkdir(parents=True, exist_ok=True) | |
| 45 | + df.write_parquet(dest) | |
| 46 | + append_jsonl(manifest, {"entity_id": entity_id, "rows": df.height, "sha256": sha256_file(dest), "fetched_at": now_iso(), | |
| 47 | + "pit_policy": d.get("meta", {}).get("pit_policy"), "license": d.get("meta", {}).get("license"), | |
| 48 | + "path": str(dest)}) | |
| 49 | + return df | |
| 50 | + | |
| 51 | + | |
| 52 | +def wide_quarterly(facts: pl.DataFrame) -> pl.DataFrame: | |
| 53 | + """One row per (entity, fiscal_period_end) with metric columns, filing_date, available_at (ET date of availability).""" | |
| 54 | + f = facts.filter((pl.col("fiscal_period_type") == "q") & (pl.col("revision_id") == 1) & pl.col("metric").is_in(METRICS)) | |
| 55 | + if f.height == 0: | |
| 56 | + return pl.DataFrame() | |
| 57 | + wide = f.pivot(on="metric", index=["entity_id", "security_id", "fiscal_period_end", "fiscal_year", "fiscal_period", | |
| 58 | + "filing_date", "available_at"], values="value", aggregate_function="first") | |
| 59 | + for m in METRICS: | |
| 60 | + if m not in wide.columns: | |
| 61 | + wide = wide.with_columns(pl.lit(None, dtype=pl.Float64).alias(m)) | |
| 62 | + wide = wide.with_columns(pl.col("available_at").dt.convert_time_zone("America/New_York").dt.date().alias("available_date")) | |
| 63 | + return wide.sort(["entity_id", "fiscal_period_end"]) | |
| 64 | + | |
| 65 | + | |
| 66 | +def add_quarter_index(wide: pl.DataFrame) -> pl.DataFrame: | |
| 67 | + """Sequential quarter index per entity + gap flags (irregular fiscal calendars).""" | |
| 68 | + w = wide.sort(["entity_id", "fiscal_period_end"]).with_columns([ | |
| 69 | + pl.col("fiscal_period_end").diff().over("entity_id").dt.total_days().alias("days_since_prev_q"), | |
| 70 | + ]) | |
| 71 | + w = w.with_columns(pl.int_range(pl.len()).over("entity_id").alias("qidx")) | |
| 72 | + return w | |
| 73 | + | |
| 74 | + | |
| 75 | +def asof_latest_available(wide: pl.DataFrame, cutoffs: pl.DataFrame) -> pl.DataFrame: | |
| 76 | + """For each (entity_id, cutoff) return the latest quarter with available_date <= cutoff (strict PIT). | |
| 77 | + | |
| 78 | + cutoffs: columns entity_id, cutoff (date). Output keeps all cutoff columns and appends the fundamentals row (suffix _pit). | |
| 79 | + """ | |
| 80 | + cols = [m for m in METRICS if m in wide.columns] | |
| 81 | + w = wide.select(["entity_id", "fiscal_period_end", "available_date", "qidx"] + cols).rename( | |
| 82 | + {"fiscal_period_end": "pit_period_end", "available_date": "pit_available_date", "qidx": "pit_qidx"}) | |
| 83 | + w = w.sort(["entity_id", "pit_available_date"]) | |
| 84 | + c = cutoffs.sort(["entity_id", "cutoff"]) | |
| 85 | + joined = c.join_asof(w, left_on="cutoff", right_on="pit_available_date", by="entity_id", strategy="backward") | |
| 86 | + # join_asof with strategy backward is <=; make the inequality explicit for the leakage test | |
| 87 | + joined = joined.with_columns((pl.col("pit_available_date") <= pl.col("cutoff")).alias("pit_ok")) | |
| 88 | + return joined | |
| 89 | + | |
| 90 | + | |
| 91 | +def pit_feature_block(df: pl.DataFrame) -> pl.DataFrame: | |
| 92 | + """Fundamental ratios from the latest available quarter (columns from asof_latest_available) + lagged growth. | |
| 93 | + | |
| 94 | + Growth rates need the same-quarter-last-year value *as of the cutoff*; they are supplied by `samples.py` through the | |
| 95 | + wide panel (quarters with available_date <= cutoff only). | |
| 96 | + """ | |
| 97 | + eps = 1e-9 | |
| 98 | + ta = pl.col("assets_tot").abs() + eps | |
| 99 | + rev = pl.col("rev").abs() + eps | |
| 100 | + return df.with_columns([ | |
| 101 | + (pl.col("gp") / rev).alias("f_gross_margin"), | |
| 102 | + (pl.col("op_inc") / rev).alias("f_op_margin"), | |
| 103 | + (pl.col("net_inc") / ta).alias("f_ni_assets"), | |
| 104 | + (pl.col("cash_op") / ta).alias("f_ocf_assets"), | |
| 105 | + ((pl.col("net_inc") - pl.col("cash_op")) / ta).alias("f_accruals"), | |
| 106 | + (pl.col("cash") / ta).alias("f_cash_assets"), | |
| 107 | + (pl.col("debt") / ta).alias("f_leverage"), | |
| 108 | + (pl.col("assets_c") / (pl.col("liab_c").abs() + eps)).alias("f_current_ratio"), | |
| 109 | + (pl.col("capex") / ta).alias("f_capex_assets"), | |
| 110 | + (pl.col("research_dev") / rev).alias("f_rd_rev"), | |
| 111 | + pl.col("intr_cover").clip(-50, 50).alias("f_int_cover"), | |
| 112 | + (pl.col("net_inc") < 0).cast(pl.Int8).alias("f_loss"), | |
| 113 | + pl.col("assets_tot").abs().log1p().alias("f_log_assets"), | |
| 114 | + pl.col("roa").alias("f_roa"), | |
| 115 | + ]) | |
added
src/wp39/portfolio.py
+124 −0
@@ -0,0 +1,124 @@ | ||
| 1 | +"""H5: pricing tests. (a) announcement-window abnormal returns regressed on the ex-ante signal; | |
| 2 | +(b) secondary portfolio sorts with factor adjustment, turnover and conservative costs. No fill at the signal close.""" | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import io | |
| 6 | +import zipfile | |
| 7 | +from pathlib import Path | |
| 8 | + | |
| 9 | +import httpx | |
| 10 | +import numpy as np | |
| 11 | +import pandas as pd | |
| 12 | +import polars as pl | |
| 13 | + | |
| 14 | + | |
| 15 | +def download_french(url: str, dest: Path) -> Path: | |
| 16 | + if not dest.exists(): | |
| 17 | + r = httpx.get(url, timeout=120, follow_redirects=True) | |
| 18 | + r.raise_for_status() | |
| 19 | + dest.write_bytes(r.content) | |
| 20 | + return dest | |
| 21 | + | |
| 22 | + | |
| 23 | +def read_french_daily(zip_path: Path) -> pd.DataFrame: | |
| 24 | + with zipfile.ZipFile(zip_path) as z: | |
| 25 | + name = z.namelist()[0] | |
| 26 | + txt = z.read(name).decode("latin1") | |
| 27 | + lines = txt.splitlines() | |
| 28 | + start = next(i for i, l in enumerate(lines) if l.strip()[:8].isdigit()) | |
| 29 | + body = "\n".join(l for l in lines[start:] if l.strip()[:8].isdigit()) | |
| 30 | + df = pd.read_csv(io.StringIO(body), header=None) | |
| 31 | + hdr = [h.strip() for h in lines[start - 1].split(",")] | |
| 32 | + hdr[0] = "date" | |
| 33 | + df.columns = hdr[: df.shape[1]] | |
| 34 | + df["date"] = pd.to_datetime(df["date"].astype(str), format="%Y%m%d") | |
| 35 | + for c in df.columns[1:]: | |
| 36 | + df[c] = pd.to_numeric(df[c], errors="coerce") / 100.0 | |
| 37 | + return df.set_index("date") | |
| 38 | + | |
| 39 | + | |
| 40 | +def car_window(daily: pl.DataFrame, ff: pd.DataFrame, entity_dates: pl.DataFrame) -> pl.DataFrame: | |
| 41 | + """Cumulative market-adjusted return from cutoff+1 to A_q+1 (inclusive) per (symbol, cutoff, A_q).""" | |
| 42 | + d = daily.select(["symbol", "session_date", "d_ret_cc"]).to_pandas() | |
| 43 | + mkt = (ff["Mkt-RF"] + ff["RF"]).rename("mkt") | |
| 44 | + d = d.merge(mkt, left_on="session_date", right_index=True, how="left") | |
| 45 | + d["ar"] = d["d_ret_cc"] - d["mkt"] | |
| 46 | + d = d.sort_values(["symbol", "session_date"]) | |
| 47 | + d["cum_ar"] = d.groupby("symbol")["ar"].cumsum() | |
| 48 | + key = d.set_index(["symbol", "session_date"])["cum_ar"] | |
| 49 | + rows = [] | |
| 50 | + for r in entity_dates.iter_rows(named=True): | |
| 51 | + s = r["symbol"] | |
| 52 | + try: | |
| 53 | + a = key.loc[(s, pd.Timestamp(r["cutoff"]))] | |
| 54 | + b_idx = d[(d.symbol == s) & (d.session_date > pd.Timestamp(r["A_q"]))].head(1) | |
| 55 | + b = float(b_idx["cum_ar"].iloc[0]) if len(b_idx) else np.nan | |
| 56 | + rows.append({**{k: r[k] for k in ("entity_id", "symbol", "cutoff", "A_q", "fiscal_period_end")}, "car": b - float(a)}) | |
| 57 | + except KeyError: | |
| 58 | + continue | |
| 59 | + return pl.DataFrame(rows) | |
| 60 | + | |
| 61 | + | |
| 62 | +def monthly_decile_portfolios(sig: pd.DataFrame, daily: pd.DataFrame, ff: pd.DataFrame, cost_bps: float = 20.0) -> dict: | |
| 63 | + """sig: columns symbol, cutoff (month-end), score (higher = more likely deterioration). Positions formed at the | |
| 64 | + close of the *next* session after the cutoff (no same-close fill), held one month, equal-weighted deciles.""" | |
| 65 | + daily = daily.sort_values(["symbol", "session_date"]) | |
| 66 | + sessions = np.sort(daily["session_date"].unique()) | |
| 67 | + out_rows, prev_hold = [], {} | |
| 68 | + months = sorted(sig["cutoff"].unique()) | |
| 69 | + for i, c in enumerate(months): | |
| 70 | + s = sig[sig.cutoff == c].dropna(subset=["score"]) | |
| 71 | + if len(s) < 50: | |
| 72 | + continue | |
| 73 | + s = s.assign(dec=pd.qcut(s["score"].rank(method="first"), 10, labels=False) + 1) | |
| 74 | + nxt = sessions[sessions > np.datetime64(c)] | |
| 75 | + if len(nxt) < 2: | |
| 76 | + continue | |
| 77 | + entry = nxt[0] | |
| 78 | + exit_ = months[i + 1] if i + 1 < len(months) else sessions[-1] | |
| 79 | + exit_entry = sessions[sessions > np.datetime64(exit_)] | |
| 80 | + exit_day = exit_entry[0] if len(exit_entry) else sessions[-1] | |
| 81 | + w = daily[(daily.session_date > entry) & (daily.session_date <= exit_day)] | |
| 82 | + hold_ret = w.groupby("symbol")["d_ret_cc"].apply(lambda x: float(np.prod(1 + x.fillna(0)) - 1)) | |
| 83 | + s = s.merge(hold_ret.rename("ret"), left_on="symbol", right_index=True, how="left") | |
| 84 | + for dec, g in s.groupby("dec"): | |
| 85 | + names = set(g.symbol) | |
| 86 | + prev = prev_hold.get(dec, set()) | |
| 87 | + turnover = len(names ^ prev) / max(len(names), 1) / 2 | |
| 88 | + prev_hold[dec] = names | |
| 89 | + out_rows.append({"cutoff": c, "entry": entry, "exit": exit_day, "decile": int(dec), "n": len(g), | |
| 90 | + "ret": float(g["ret"].mean()), "turnover": turnover, | |
| 91 | + "ret_net": float(g["ret"].mean()) - 2 * turnover * cost_bps / 1e4}) | |
| 92 | + port = pd.DataFrame(out_rows) | |
| 93 | + return {"portfolios": port} | |
| 94 | + | |
| 95 | + | |
| 96 | +def factor_alpha(port: pd.DataFrame, ff: pd.DataFrame, decile_long: int, decile_short: int, net: bool = False) -> dict: | |
| 97 | + """Monthly long-short (long low-score, short high-score) regressed on FF5 + MOM compounded over the same windows.""" | |
| 98 | + col = "ret_net" if net else "ret" | |
| 99 | + L = port[port.decile == decile_long].set_index("cutoff")[col] | |
| 100 | + S = port[port.decile == decile_short].set_index("cutoff")[col] | |
| 101 | + ls = (L - S).dropna() | |
| 102 | + rows = [] | |
| 103 | + for c in ls.index: | |
| 104 | + w = port[(port.cutoff == c) & (port.decile == decile_long)].iloc[0] | |
| 105 | + f = ff[(ff.index > pd.Timestamp(w["entry"])) & (ff.index <= pd.Timestamp(w["exit"]))] | |
| 106 | + rows.append({"cutoff": c, **{k: float(np.prod(1 + f[k]) - 1) for k in ("Mkt-RF", "SMB", "HML", "RMW", "CMA", "Mom", "RF") if k in f}}) | |
| 107 | + F = pd.DataFrame(rows).set_index("cutoff") | |
| 108 | + y = (ls - F["RF"]).to_numpy() | |
| 109 | + X = np.column_stack([np.ones(len(F))] + [F[k].to_numpy() for k in ("Mkt-RF", "SMB", "HML", "RMW", "CMA", "Mom") if k in F]) | |
| 110 | + beta, *_ = np.linalg.lstsq(X, y, rcond=None) | |
| 111 | + e = y - X @ beta | |
| 112 | + T = len(y); L_ = int(np.floor(4 * (T / 100) ** (2 / 9))) | |
| 113 | + # HAC se for the intercept | |
| 114 | + Xi = np.linalg.pinv(X.T @ X) | |
| 115 | + S0 = sum(np.outer(X[t] * e[t], X[t] * e[t]) for t in range(T)) | |
| 116 | + for l in range(1, L_ + 1): | |
| 117 | + wgt = 1 - l / (L_ + 1) | |
| 118 | + for t in range(l, T): | |
| 119 | + G = np.outer(X[t] * e[t], X[t - l] * e[t - l]); S0 += wgt * (G + G.T) | |
| 120 | + V = Xi @ S0 @ Xi | |
| 121 | + se = np.sqrt(np.diag(V)) | |
| 122 | + return {"alpha_monthly": float(beta[0]), "alpha_se": float(se[0]), "alpha_t": float(beta[0] / se[0]) if se[0] > 0 else float("nan"), | |
| 123 | + "mean_ls": float(ls.mean()), "sd_ls": float(ls.std()), "sharpe_ann": float(ls.mean() / ls.std() * np.sqrt(12)) if ls.std() > 0 else float("nan"), | |
| 124 | + "T": int(T), "net": net} | |
added
src/wp39/provenance.py
+61 −0
@@ -0,0 +1,61 @@ | ||
| 1 | +"""Dataset manifests, hashes and code versions (no raw data, no secrets).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import hashlib | |
| 5 | +import json | |
| 6 | +import os | |
| 7 | +import subprocess | |
| 8 | +import time | |
| 9 | +from pathlib import Path | |
| 10 | +from typing import Any, Iterable | |
| 11 | + | |
| 12 | + | |
| 13 | +def sha256_file(path: str | os.PathLike, chunk: int = 1 << 20) -> str: | |
| 14 | + h = hashlib.sha256() | |
| 15 | + with open(path, "rb") as fh: | |
| 16 | + for block in iter(lambda: fh.read(chunk), b""): | |
| 17 | + h.update(block) | |
| 18 | + return h.hexdigest() | |
| 19 | + | |
| 20 | + | |
| 21 | +def sha256_text(text: str) -> str: | |
| 22 | + return hashlib.sha256(text.encode()).hexdigest() | |
| 23 | + | |
| 24 | + | |
| 25 | +def git_sha(repo: str | os.PathLike | None = None) -> str: | |
| 26 | + try: | |
| 27 | + return subprocess.check_output(["git", "rev-parse", "--short=12", "HEAD"], cwd=repo, text=True, | |
| 28 | + stderr=subprocess.DEVNULL).strip() | |
| 29 | + except Exception: | |
| 30 | + return "nogit" | |
| 31 | + | |
| 32 | + | |
| 33 | +def now_iso() -> str: | |
| 34 | + return time.strftime("%Y-%m-%dT%H:%M:%S%z") | |
| 35 | + | |
| 36 | + | |
| 37 | +def append_jsonl(path: str | os.PathLike, record: dict[str, Any]) -> None: | |
| 38 | + p = Path(path) | |
| 39 | + p.parent.mkdir(parents=True, exist_ok=True) | |
| 40 | + with open(p, "a") as fh: | |
| 41 | + fh.write(json.dumps(record, default=str, sort_keys=True) + "\n") | |
| 42 | + | |
| 43 | + | |
| 44 | +def read_jsonl(path: str | os.PathLike) -> list[dict[str, Any]]: | |
| 45 | + p = Path(path) | |
| 46 | + if not p.exists(): | |
| 47 | + return [] | |
| 48 | + with open(p) as fh: | |
| 49 | + return [json.loads(line) for line in fh if line.strip()] | |
| 50 | + | |
| 51 | + | |
| 52 | +def manifest_index(path: str | os.PathLike, keys: Iterable[str]) -> dict[tuple, dict[str, Any]]: | |
| 53 | + keys = tuple(keys) | |
| 54 | + return {tuple(r.get(k) for k in keys): r for r in read_jsonl(path)} | |
| 55 | + | |
| 56 | + | |
| 57 | +def write_json(path: str | os.PathLike, obj: Any) -> None: | |
| 58 | + p = Path(path) | |
| 59 | + p.parent.mkdir(parents=True, exist_ok=True) | |
| 60 | + with open(p, "w") as fh: | |
| 61 | + json.dump(obj, fh, indent=1, default=str, sort_keys=True) | |
added
src/wp39/robustness.py
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +"""Negative controls, placebos and subgroup stress tests (Phase 5).""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import numpy as np | |
| 5 | +import polars as pl | |
| 6 | + | |
| 7 | + | |
| 8 | +def shuffle_labels_within_quarter(df: pl.DataFrame, y: str, seed: int) -> pl.DataFrame: | |
| 9 | + """Permute the label within calendar quarter of the cutoff: any 'gain' left is overfitting/leakage.""" | |
| 10 | + rng = np.random.default_rng(seed) | |
| 11 | + q = df.with_columns((pl.col("cutoff").dt.year() * 4 + (pl.col("cutoff").dt.month() - 1) // 3).alias("_cq")) | |
| 12 | + parts = [] | |
| 13 | + for _, g in q.group_by("_cq", maintain_order=True): | |
| 14 | + v = g[y].to_numpy().copy(); rng.shuffle(v) | |
| 15 | + parts.append(g.with_columns(pl.Series(y, v))) | |
| 16 | + return pl.concat(parts).drop("_cq") | |
| 17 | + | |
| 18 | + | |
| 19 | +def subgroup_masks(df: pl.DataFrame) -> dict[str, pl.Expr]: | |
| 20 | + """Pre-specified subgroups (terciles are computed on the TRAINING rows and passed as cut-points by the caller).""" | |
| 21 | + return { | |
| 22 | + "all": pl.lit(True), | |
| 23 | + "size_small": pl.col("tercile_size") == 1, "size_large": pl.col("tercile_size") == 3, | |
| 24 | + "illiq_high": pl.col("tercile_illiq") == 3, "illiq_low": pl.col("tercile_illiq") == 1, | |
| 25 | + "lev_high": pl.col("tercile_lev") == 3, "lev_low": pl.col("tercile_lev") == 1, | |
| 26 | + "loss_firms": pl.col("f_loss") == 1, "profit_firms": pl.col("f_loss") == 0, | |
| 27 | + "covid_2020": (pl.col("cutoff").dt.year() == 2020), "ex_covid": (pl.col("cutoff").dt.year() != 2020), | |
| 28 | + "after_close_release": pl.col("release_time") == "after_close", "before_open_release": pl.col("release_time") == "before_open", | |
| 29 | + "sector_finance": pl.col("sic_division") == "H_finance_insurance_realestate", | |
| 30 | + "sector_nonfinance": pl.col("sic_division") != "H_finance_insurance_realestate", | |
| 31 | + } | |
| 32 | + | |
| 33 | + | |
| 34 | +def add_terciles(train: pl.DataFrame, others: list[pl.DataFrame]) -> list[pl.DataFrame]: | |
| 35 | + """Tercile cut-points for size, illiquidity and leverage computed on training rows only.""" | |
| 36 | + cuts = {} | |
| 37 | + for name, col in (("size", "f_log_mcap"), ("illiq", "wd_amihud60"), ("lev", "f_leverage")): | |
| 38 | + s = train[col].drop_nulls().drop_nans() | |
| 39 | + cuts[name] = (float(s.quantile(1 / 3)), float(s.quantile(2 / 3))) | |
| 40 | + out = [] | |
| 41 | + for d in [train] + others: | |
| 42 | + for name, col in (("size", "f_log_mcap"), ("illiq", "wd_amihud60"), ("lev", "f_leverage")): | |
| 43 | + lo, hi = cuts[name] | |
| 44 | + d = d.with_columns(pl.when(pl.col(col) <= lo).then(1).when(pl.col(col) <= hi).then(2).otherwise(3).alias(f"tercile_{name}")) | |
| 45 | + out.append(d) | |
| 46 | + return out | |
added
src/wp39/samples.py
+180 −0
@@ -0,0 +1,180 @@ | ||
| 1 | +"""Cohort construction: cutoffs, window aggregation of session features, PIT fundamentals join, exclusions. | |
| 2 | + | |
| 3 | +Event-time: c = A_q - h trading days (h in horizons). Fixed-calendar: c = month-end sessions, target = next release. | |
| 4 | +All joins are explicit as-of joins on `session_date <= c` (bars) and `available_date <= c` (fundamentals). | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import datetime as dt | |
| 9 | + | |
| 10 | +import numpy as np | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | +from .calendar import session_array, shift_trading_days, month_end_sessions | |
| 14 | +from .features import FREQS, INTRADAY_FAMILIES | |
| 15 | +from .pit_fundamentals import METRICS, pit_feature_block | |
| 16 | + | |
| 17 | +WINDOW_MEAN = ["rv", "bpv", "rj", "rsneg_share", "sjv", "rskew", "rkurt", "max_share", "ac1", "vol_first30", "vol_last30", | |
| 18 | + "vol_gini", "down_vol_share", "present_share", "rv_mktdown_share"] | |
| 19 | + | |
| 20 | + | |
| 21 | +def complete_session_index(feat: pl.DataFrame) -> pl.DataFrame: | |
| 22 | + """Reindex a symbol's session features to every XNYS session between its first and last session (rows for missing | |
| 23 | + sessions have null features) so that rolling windows are measured in trading days.""" | |
| 24 | + arr = session_array() | |
| 25 | + d0, d1 = feat["session_date"].min(), feat["session_date"].max() | |
| 26 | + s = arr[(arr >= np.datetime64(d0)) & (arr <= np.datetime64(d1))] | |
| 27 | + idx = pl.DataFrame({"session_date": pl.Series(s.astype("datetime64[D]")).cast(pl.Date)}) | |
| 28 | + return idx.join(feat, on="session_date", how="left").sort("session_date") | |
| 29 | + | |
| 30 | + | |
| 31 | +def rolling_window_features(feat: pl.DataFrame, short: int = 20, long: int = 60, min_valid: int = 15) -> pl.DataFrame: | |
| 32 | + """Rolling (trailing, inclusive) aggregates per session; the row at session c is the feature vector for cutoff c.""" | |
| 33 | + f = complete_session_index(feat) | |
| 34 | + exprs = [] | |
| 35 | + valid = pl.col("f1m_n_bars").is_not_null() & (pl.col("f1m_n_bars") >= 100) | |
| 36 | + f = f.with_columns(valid.cast(pl.Int8).alias("_valid")) | |
| 37 | + exprs.append(pl.col("_valid").rolling_sum(window_size=short, min_samples=1).alias("w_valid_days")) | |
| 38 | + # intraday families: window means (RV-type quantities averaged -> per-day average, same across frequencies) | |
| 39 | + for name in FREQS: | |
| 40 | + p = f"f{name}_" | |
| 41 | + for fam in WINDOW_MEAN: | |
| 42 | + exprs.append(pl.col(p + fam).rolling_mean(window_size=short, min_samples=min_valid).alias(f"w{name}_{fam}")) | |
| 43 | + exprs.append(pl.col(p + "drift").rolling_sum(window_size=short, min_samples=min_valid).alias(f"w{name}_drift")) | |
| 44 | + exprs.append((pl.col(p + "sxy").rolling_sum(window_size=short, min_samples=min_valid) / | |
| 45 | + (pl.col(p + "syy").rolling_sum(window_size=short, min_samples=min_valid) + 1e-18)).alias(f"w{name}_beta")) | |
| 46 | + exprs.append(pl.col(p + "rv").rolling_std(window_size=short, min_samples=min_valid).alias(f"w{name}_volofvol")) | |
| 47 | + # variance ratio: mean intraday RV vs variance of daily close-to-close returns over the window | |
| 48 | + exprs.append((pl.col(p + "rv").rolling_mean(window_size=short, min_samples=min_valid) / | |
| 49 | + (pl.col("d_ret_cc").rolling_var(window_size=short, min_samples=min_valid) + 1e-18)).alias(f"w{name}_vratio")) | |
| 50 | + # daily block | |
| 51 | + r = pl.col("d_ret_cc") | |
| 52 | + exprs += [ | |
| 53 | + ((1.0 + r).log().rolling_sum(window_size=short, min_samples=min_valid)).alias("wd_ret20"), | |
| 54 | + ((1.0 + r).log().rolling_sum(window_size=long, min_samples=3 * min_valid)).alias("wd_ret60"), | |
| 55 | + r.rolling_std(window_size=short, min_samples=min_valid).alias("wd_vol20"), | |
| 56 | + r.rolling_std(window_size=long, min_samples=3 * min_valid).alias("wd_vol60"), | |
| 57 | + pl.col("d_parkinson").rolling_mean(window_size=short, min_samples=min_valid).sqrt().alias("wd_parkinson20"), | |
| 58 | + r.rolling_skew(window_size=long).alias("wd_skew60"), | |
| 59 | + r.rolling_kurtosis(window_size=long).alias("wd_kurt60"), | |
| 60 | + (pl.col("d_ret_co").rolling_sum(window_size=short, min_samples=min_valid)).alias("wd_overnight20"), | |
| 61 | + (pl.col("d_ret_oc").rolling_sum(window_size=short, min_samples=min_valid)).alias("wd_intraday20"), | |
| 62 | + (pl.col("d_volume").rolling_mean(window_size=short, min_samples=min_valid) / | |
| 63 | + (pl.col("d_volume").rolling_median(window_size=long, min_samples=3 * min_valid) + 1.0)).alias("wd_volratio20"), | |
| 64 | + (pl.col("d_volume").rolling_mean(window_size=5, min_samples=3) / | |
| 65 | + (pl.col("d_volume").rolling_median(window_size=long, min_samples=3 * min_valid) + 1.0)).alias("wd_volratio5"), | |
| 66 | + ((r.abs() / (pl.col("d_dollar_vol") + 1.0)) * 1e6).rolling_mean(window_size=long, min_samples=3 * min_valid).alias("wd_amihud60"), | |
| 67 | + (r == 0).cast(pl.Float64).rolling_mean(window_size=long, min_samples=3 * min_valid).alias("wd_zeroret60"), | |
| 68 | + pl.col("d_dollar_vol").rolling_mean(window_size=long, min_samples=3 * min_valid).log1p().alias("wd_log_dollar_vol60"), | |
| 69 | + pl.col("d_close").alias("wd_close"), | |
| 70 | + pl.col("f1m_n_bars").rolling_mean(window_size=short, min_samples=min_valid).alias("wd_bars20"), | |
| 71 | + ] | |
| 72 | + f = f.with_columns(exprs) | |
| 73 | + # max drawdown over 60 sessions from cumulative log returns | |
| 74 | + f = f.with_columns((1.0 + r).log().fill_null(0.0).cum_sum().alias("_cum")) | |
| 75 | + f = f.with_columns((pl.col("_cum") - pl.col("_cum").rolling_max(window_size=long, min_samples=3 * min_valid)).rolling_min( | |
| 76 | + window_size=long, min_samples=3 * min_valid).alias("wd_maxdd60")) | |
| 77 | + # daily beta / residual vol vs SPY daily (from 1m family sums at daily horizon are not available -> use 5m beta proxy) | |
| 78 | + cols = ["session_date", "symbol", "w_valid_days"] + [c for c in f.columns if c.startswith("w1m_") or c.startswith("w5m_") | |
| 79 | + or c.startswith("w30m_") or c.startswith("wd_")] | |
| 80 | + return f.select(cols) | |
| 81 | + | |
| 82 | + | |
| 83 | +def event_cutoffs(quarters: pl.DataFrame, horizons: list[int]) -> pl.DataFrame: | |
| 84 | + """One row per (entity, quarter, h) with cutoff c = A_q - h trading days (requires release_match == ok).""" | |
| 85 | + q = quarters.filter(pl.col("release_match") == "ok").select( | |
| 86 | + ["entity_id", "fiscal_period_end", "A_q", "filing_date", "available_date", "qidx"]) | |
| 87 | + rows = [] | |
| 88 | + a_prev = q.with_columns(pl.col("A_q").shift(1).over("entity_id").alias("A_prev")) | |
| 89 | + for r in a_prev.iter_rows(named=True): | |
| 90 | + for h in horizons: | |
| 91 | + try: | |
| 92 | + c = shift_trading_days(r["A_q"], -h) | |
| 93 | + except ValueError: | |
| 94 | + continue | |
| 95 | + rows.append({"entity_id": r["entity_id"], "fiscal_period_end": r["fiscal_period_end"], "h": h, "cutoff": c, | |
| 96 | + "A_q": r["A_q"], "F_q": r["filing_date"], "A_prev": r["A_prev"], "qidx": r["qidx"]}) | |
| 97 | + return pl.DataFrame(rows) | |
| 98 | + | |
| 99 | + | |
| 100 | +def calendar_cutoffs(quarters: pl.DataFrame, start: dt.date, end: dt.date, max_td: int = 65) -> pl.DataFrame: | |
| 101 | + """Month-end cutoffs; target = next release after c within max_td trading days.""" | |
| 102 | + me = pl.DataFrame({"cutoff": month_end_sessions(start, end)}) | |
| 103 | + q = quarters.filter(pl.col("release_match") == "ok").select(["entity_id", "fiscal_period_end", "A_q", "filing_date", "qidx"]) | |
| 104 | + ents = q.select("entity_id").unique() | |
| 105 | + grid = ents.join(me, how="cross").sort(["entity_id", "cutoff"]) | |
| 106 | + qq = q.sort(["entity_id", "A_q"]).with_columns(pl.col("A_q").alias("_a")) | |
| 107 | + j = grid.join_asof(qq, left_on="cutoff", right_on="_a", by="entity_id", strategy="forward") | |
| 108 | + j = j.filter(pl.col("A_q") > pl.col("cutoff")) | |
| 109 | + arr = session_array() | |
| 110 | + c = np.searchsorted(arr, j["cutoff"].to_numpy().astype("datetime64[D]"), side="right") | |
| 111 | + a = np.searchsorted(arr, j["A_q"].to_numpy().astype("datetime64[D]"), side="right") | |
| 112 | + j = j.with_columns(pl.Series("h", (a - c).astype(int))).filter(pl.col("h") <= max_td) | |
| 113 | + return j.rename({"filing_date": "F_q"}).with_columns(pl.lit(None, dtype=pl.Date).alias("A_prev")) | |
| 114 | + | |
| 115 | + | |
| 116 | +def apply_admissibility(cut: pl.DataFrame) -> pl.DataFrame: | |
| 117 | + """c < min(A_q, F_q) and c > A_prev (previous reveal already happened).""" | |
| 118 | + return cut.filter((pl.col("cutoff") < pl.col("A_q")) & (pl.col("cutoff") < pl.col("F_q")) & | |
| 119 | + (pl.col("A_prev").is_null() | (pl.col("cutoff") > pl.col("A_prev")))) | |
| 120 | + | |
| 121 | + | |
| 122 | +def join_market(cut: pl.DataFrame, win: pl.DataFrame, symbol_of: pl.DataFrame) -> pl.DataFrame: | |
| 123 | + """As-of join of the rolling window features at session_date <= cutoff (exact session normally).""" | |
| 124 | + c = cut.join(symbol_of, on="entity_id", how="left").sort(["symbol", "cutoff"]) | |
| 125 | + w = win.sort(["symbol", "session_date"]) | |
| 126 | + j = c.join_asof(w, left_on="cutoff", right_on="session_date", by="symbol", strategy="backward", tolerance="5d") | |
| 127 | + return j.rename({"session_date": "market_asof_date"}) | |
| 128 | + | |
| 129 | + | |
| 130 | +def join_fundamentals(j: pl.DataFrame, wide: pl.DataFrame) -> pl.DataFrame: | |
| 131 | + """Latest available quarter at cutoff + yoy growth from the quarter 4 back (also available by construction).""" | |
| 132 | + w = wide.select(["entity_id", "fiscal_period_end", "available_date", "qidx", "A_q"] + METRICS).sort(["entity_id", "available_date"]) | |
| 133 | + w4 = wide.select(["entity_id", pl.col("qidx").alias("pit_qidx4"), pl.col("rev").alias("rev_l4"), pl.col("op_inc").alias("op_inc_l4"), | |
| 134 | + pl.col("shares").alias("shares_l4"), pl.col("available_date").alias("avail_l4")]) | |
| 135 | + wr = w.rename({c: c for c in METRICS}).rename({"fiscal_period_end": "pit_period_end", "available_date": "pit_available_date", | |
| 136 | + "qidx": "pit_qidx", "A_q": "pit_A"}) | |
| 137 | + out = j.sort(["entity_id", "cutoff"]).join_asof(wr, left_on="cutoff", right_on="pit_available_date", by="entity_id", strategy="backward") | |
| 138 | + out = out.with_columns((pl.col("pit_qidx") - 4).alias("pit_qidx4")).join(w4, on=["entity_id", "pit_qidx4"], how="left") | |
| 139 | + out = out.with_columns([ | |
| 140 | + (pl.col("pit_available_date") <= pl.col("cutoff")).alias("pit_ok"), | |
| 141 | + ((pl.col("rev") - pl.col("rev_l4")) / (pl.col("rev_l4").abs() + 1e-9)).alias("f_rev_growth"), | |
| 142 | + ((pl.col("shares") - pl.col("shares_l4")) / (pl.col("shares_l4").abs() + 1e-9)).alias("f_net_issuance"), | |
| 143 | + (pl.col("qidx") - pl.col("pit_qidx")).alias("quarters_ahead"), # 1 = we predict the quarter right after the latest available | |
| 144 | + ]) | |
| 145 | + out = pit_feature_block(out) | |
| 146 | + out = out.with_columns([ | |
| 147 | + (pl.col("wd_close") * pl.col("shares").abs()).log1p().alias("f_log_mcap"), | |
| 148 | + ]) | |
| 149 | + return out | |
| 150 | + | |
| 151 | + | |
| 152 | +def exclusion_cascade(df: pl.DataFrame, min_price: float, min_valid_days: int) -> tuple[pl.DataFrame, pl.DataFrame]: | |
| 153 | + """Apply frozen exclusions in order; return (sample, cascade table).""" | |
| 154 | + steps = [ | |
| 155 | + ("cutoff_admissible", pl.lit(True)), | |
| 156 | + ("market_window_available", pl.col("market_asof_date").is_not_null() & (pl.col("w_valid_days") >= min_valid_days)), | |
| 157 | + ("price_ge_min", pl.col("wd_close") >= min_price), | |
| 158 | + ("fundamentals_available", pl.col("pit_ok").fill_null(False) & pl.col("pit_qidx").is_not_null()), | |
| 159 | + ("predicting_next_quarter", pl.col("quarters_ahead") == 1), | |
| 160 | + ("expectation_defined", pl.col("S_opinc").is_not_null() & pl.col("assets_tot_lag1").is_not_null()), | |
| 161 | + ("regular_fiscal_calendar", pl.col("_gap_ok8") == 8), | |
| 162 | + ] | |
| 163 | + rows, cur = [], df | |
| 164 | + for name, cond in steps: | |
| 165 | + before = cur.height | |
| 166 | + cur = cur.filter(cond) | |
| 167 | + rows.append({"step": name, "remaining": cur.height, "dropped": before - cur.height}) | |
| 168 | + return cur, pl.DataFrame(rows) | |
| 169 | + | |
| 170 | + | |
| 171 | +def feature_columns(df: pl.DataFrame) -> dict[str, list[str]]: | |
| 172 | + fund = [c for c in df.columns if c.startswith("f_")] + ["info_age_td", "last_surprise", "mean_surprise4", "hist_adverse_rate"] | |
| 173 | + daily = [c for c in df.columns if c.startswith("wd_") and c != "wd_close"] | |
| 174 | + return { | |
| 175 | + "fundamentals": [c for c in fund if c in df.columns], | |
| 176 | + "daily": daily, | |
| 177 | + "intraday_30m": [c for c in df.columns if c.startswith("w30m_")], | |
| 178 | + "intraday_5m": [c for c in df.columns if c.startswith("w5m_")], | |
| 179 | + "intraday_1m": [c for c in df.columns if c.startswith("w1m_")], | |
| 180 | + } | |
added
src/wp39/splits.py
+65 −0
@@ -0,0 +1,65 @@ | ||
| 1 | +"""Chronological, walk-forward and firm-held-out splits with purge and embargo (RESEARCH_DESIGN §6). | |
| 2 | + | |
| 3 | +An observation is identified by (entity_id, fiscal_period_end, cutoff, A_q). The *target* is revealed at A_q, so a | |
| 4 | +training observation is purged if its A_q falls after (split_start - embargo); test observations start at split_start. | |
| 5 | +""" | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import datetime as dt | |
| 9 | +import hashlib | |
| 10 | + | |
| 11 | +import polars as pl | |
| 12 | + | |
| 13 | + | |
| 14 | +def _q_start(d: dt.date) -> dt.date: | |
| 15 | + return dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) | |
| 16 | + | |
| 17 | + | |
| 18 | +def _shift_quarters(d: dt.date, n: int) -> dt.date: | |
| 19 | + m = d.month - 1 + 3 * n | |
| 20 | + return dt.date(d.year + m // 12, m % 12 + 1, 1) | |
| 21 | + | |
| 22 | + | |
| 23 | +def purge_train(df: pl.DataFrame, boundary: dt.date, embargo_quarters: int = 1) -> pl.DataFrame: | |
| 24 | + """Keep training rows whose outcome was revealed before boundary - embargo (quarter-aligned).""" | |
| 25 | + limit = _shift_quarters(_q_start(boundary), -embargo_quarters) | |
| 26 | + return df.filter(pl.col("A_q") < limit) | |
| 27 | + | |
| 28 | + | |
| 29 | +def chronological(df: pl.DataFrame, cfg: dict) -> dict[str, pl.DataFrame]: | |
| 30 | + tr0, tr1 = [dt.date.fromisoformat(str(x)) for x in cfg["train"]] | |
| 31 | + va0, va1 = [dt.date.fromisoformat(str(x)) for x in cfg["validation"]] | |
| 32 | + te0, te1 = [dt.date.fromisoformat(str(x)) for x in cfg["test"]] | |
| 33 | + emb = cfg["walk_forward"]["embargo_quarters"] | |
| 34 | + train = purge_train(df.filter((pl.col("cutoff") >= tr0) & (pl.col("cutoff") <= tr1)), va0, emb) | |
| 35 | + valid = purge_train(df.filter((pl.col("cutoff") >= va0) & (pl.col("cutoff") <= va1)), te0, emb) | |
| 36 | + test = df.filter((pl.col("cutoff") >= te0) & (pl.col("cutoff") <= te1)) | |
| 37 | + return {"train": train, "validation": valid, "test": test} | |
| 38 | + | |
| 39 | + | |
| 40 | +def walk_forward(df: pl.DataFrame, cfg: dict, last_year: int) -> list[tuple[int, pl.DataFrame, pl.DataFrame]]: | |
| 41 | + """Expanding annual folds: train on cutoffs < Jan 1 of year y (purged, embargo), test on cutoffs in year y.""" | |
| 42 | + out = [] | |
| 43 | + emb = cfg["walk_forward"]["embargo_quarters"] | |
| 44 | + for y in range(cfg["walk_forward"]["first_test_year"], last_year + 1): | |
| 45 | + b = dt.date(y, 1, 1) | |
| 46 | + train = purge_train(df.filter(pl.col("cutoff") < b), b, emb) | |
| 47 | + test = df.filter((pl.col("cutoff") >= b) & (pl.col("cutoff") < dt.date(y + 1, 1, 1))) | |
| 48 | + if train.height and test.height: | |
| 49 | + out.append((y, train, test)) | |
| 50 | + return out | |
| 51 | + | |
| 52 | + | |
| 53 | +def firm_holdout_mask(entity_ids: pl.Series, fraction: float = 0.2, salt: str = "wp39") -> pl.Series: | |
| 54 | + def h(e: str) -> bool: | |
| 55 | + return int(hashlib.sha256(f"{salt}:{e}".encode()).hexdigest()[:8], 16) / 0xFFFFFFFF < fraction | |
| 56 | + return pl.Series([h(e) for e in entity_ids.to_list()], dtype=pl.Boolean) | |
| 57 | + | |
| 58 | + | |
| 59 | +def overlap_check(train: pl.DataFrame, test: pl.DataFrame) -> dict: | |
| 60 | + """Diagnostics used by tests: no training outcome revealed on/after the first test cutoff; no shared observation ids.""" | |
| 61 | + first_test_cutoff = test["cutoff"].min() | |
| 62 | + leak = train.filter(pl.col("A_q") >= first_test_cutoff).height | |
| 63 | + key = ["entity_id", "fiscal_period_end", "h"] if "h" in train.columns else ["entity_id", "fiscal_period_end"] | |
| 64 | + shared = train.select(key).join(test.select(key), on=key, how="inner").height | |
| 65 | + return {"train_outcomes_revealed_in_test": leak, "shared_observations": shared} | |
added
tests/__init__.py
+0 −0
added
tests/conftest.py
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +import datetime as dt | |
| 2 | +import os | |
| 3 | +import sys | |
| 4 | +from pathlib import Path | |
| 5 | + | |
| 6 | +import polars as pl | |
| 7 | +import pytest | |
| 8 | + | |
| 9 | +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| 10 | + | |
| 11 | +LIVE = os.environ.get("WP39_LIVE") == "1" | |
| 12 | +live = pytest.mark.skipif(not LIVE, reason="set WP39_LIVE=1 to run live FinData tests") | |
| 13 | + | |
| 14 | + | |
| 15 | +@pytest.fixture | |
| 16 | +def toy_wide(): | |
| 17 | + """Two firms, regular quarterly calendar, filing 40 d after period end.""" | |
| 18 | + rows = [] | |
| 19 | + for e in ("frd:AAA", "frd:BBB"): | |
| 20 | + base = 100.0 if e == "frd:AAA" else 50.0 | |
| 21 | + for i in range(14): | |
| 22 | + pe = dt.date(2015, 3, 31) + dt.timedelta(days=int(91.3 * i)) | |
| 23 | + fd = pe + dt.timedelta(days=40) | |
| 24 | + rows.append({"entity_id": e, "security_id": f"firstrate:stock:{e[4:]}", "fiscal_period_end": pe, | |
| 25 | + "fiscal_year": pe.year, "fiscal_period": f"q{(pe.month - 1) // 3 + 1}", "filing_date": fd, | |
| 26 | + "available_date": fd + dt.timedelta(days=1), "op_inc": base + i + (5 if (i % 4 == 0) else 0), | |
| 27 | + "net_inc": base * 0.7 + i, "cash_op": base * 1.1 + i, "rev": base * 10 + 10 * i, | |
| 28 | + "assets_tot": base * 20, "assets_c": base * 5, "liab_c": base * 3, "roa": 0.02 + 0.001 * i, | |
| 29 | + "intr_cover": 5.0, "shares": 1000.0, "gp": base * 4, "debt": base * 6, "cash": base, | |
| 30 | + "capex": base * 0.2, "research_dev": base * 0.1, "qidx": i}) | |
| 31 | + return pl.DataFrame(rows) | |
| 32 | + | |
| 33 | + | |
| 34 | +@pytest.fixture | |
| 35 | +def toy_events(toy_wide): | |
| 36 | + ev = toy_wide.select(["entity_id", "filing_date"]).with_columns( | |
| 37 | + (pl.col("filing_date") - pl.duration(days=1)).alias("event_date"), pl.lit("after_close").alias("release_time"), | |
| 38 | + pl.lit(None, dtype=pl.Float64).alias("eps_estimate"), pl.lit(None, dtype=pl.Float64).alias("eps_reported")).drop("filing_date") | |
| 39 | + return ev | |
added
tests/test_api_contract.py
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +import datetime as dt | |
| 2 | +import random | |
| 3 | + | |
| 4 | +import polars as pl | |
| 5 | +import pytest | |
| 6 | + | |
| 7 | +from tests.conftest import live | |
| 8 | +from wp39.api_client import FinDataClient, _redact | |
| 9 | + | |
| 10 | + | |
| 11 | +def test_redaction(): | |
| 12 | + assert "fd_abc.def" not in _redact("token fd_abc.def leaked") | |
| 13 | + | |
| 14 | + | |
| 15 | +@live | |
| 16 | +def test_schema_and_datasets(): | |
| 17 | + c = FinDataClient() | |
| 18 | + s = c.schema_bars() | |
| 19 | + assert s["timestamp_convention"] == "bar_open" and s["schema_version"] == 1 | |
| 20 | + ids = {d["id"] for d in c.datasets()} | |
| 21 | + assert {"market_bars.stock.1m.raw", "market_bars.etf.1m.raw"} <= ids | |
| 22 | + | |
| 23 | + | |
| 24 | +@live | |
| 25 | +def test_asof_equivalence(): | |
| 26 | + """Local filter available_at <= as_of on the full extract == API as_of result (metric, period) set.""" | |
| 27 | + c = FinDataClient() | |
| 28 | + full = pl.DataFrame(c.fundamentals("frd:MSFT", period_type="q", limit=20000)["data"]) | |
| 29 | + full = full.with_columns(pl.col("available_at").str.to_datetime(time_zone="UTC", strict=False)) | |
| 30 | + for as_of in ("2016-03-01T00:00:00Z", "2019-11-15T00:00:00Z", "2023-08-01T00:00:00Z"): | |
| 31 | + api = pl.DataFrame(c.fundamentals("frd:MSFT", as_of=as_of, period_type="q", limit=20000)["data"]) | |
| 32 | + local = full.filter(pl.col("available_at") <= pl.lit(as_of).str.to_datetime(time_zone="UTC")) | |
| 33 | + a = set(zip(api["metric"], api["fiscal_period_end"])); b = set(zip(local["metric"], local["fiscal_period_end"])) | |
| 34 | + assert a == b, (as_of, len(a), len(b)) | |
| 35 | + | |
| 36 | + | |
| 37 | +@live | |
| 38 | +def test_bars_limits_are_enforced(): | |
| 39 | + from wp39.api_client import FinDataError | |
| 40 | + c = FinDataClient() | |
| 41 | + with pytest.raises(FinDataError): | |
| 42 | + c.get("/market/bars", symbol="AAPL", interval="1m", start="2024-01-01", end="2024-03-15", limit=10) | |
added
tests/test_corporate_actions.py
+22 −0
@@ -0,0 +1,22 @@ | ||
| 1 | +import datetime as dt | |
| 2 | + | |
| 3 | +import polars as pl | |
| 4 | + | |
| 5 | +from wp39.features import daily_block | |
| 6 | + | |
| 7 | + | |
| 8 | +def test_split_adjusted_overnight_return(): | |
| 9 | + day = pl.DataFrame({"session_date": [dt.date(2020, 8, 28), dt.date(2020, 8, 31)], "d_open": [500.0, 127.0], | |
| 10 | + "d_close": [499.0, 129.0], "d_high": [505.0, 131.0], "d_low": [495.0, 126.0], "d_volume": [1e6, 4e6]}) | |
| 11 | + acts = pl.DataFrame({"symbol": ["AAPL"], "ex_date": [dt.date(2020, 8, 31)], "split_ratio": [4.0], "dividend": [0.0]}) | |
| 12 | + d = daily_block(day, acts, "AAPL") | |
| 13 | + assert abs(d["d_ret_cc"][1] - (129.0 / (499.0 / 4.0) - 1)) < 1e-9 | |
| 14 | + assert abs(d["d_ret_co"][1] - (127.0 / (499.0 / 4.0) - 1)) < 1e-9 | |
| 15 | + | |
| 16 | + | |
| 17 | +def test_dividend_adjusted_return(): | |
| 18 | + day = pl.DataFrame({"session_date": [dt.date(2021, 2, 4), dt.date(2021, 2, 5)], "d_open": [100.0, 99.0], | |
| 19 | + "d_close": [100.0, 99.5], "d_high": [101.0, 100.0], "d_low": [99.0, 98.0], "d_volume": [1e6, 1e6]}) | |
| 20 | + acts = pl.DataFrame({"symbol": ["X"], "ex_date": [dt.date(2021, 2, 5)], "split_ratio": [1.0], "dividend": [0.5]}) | |
| 21 | + d = daily_block(day, acts, "X") | |
| 22 | + assert abs(d["d_ret_cc"][1] - ((99.5 + 0.5) / 100.0 - 1)) < 1e-12 | |
added
tests/test_features.py
+37 −0
@@ -0,0 +1,37 @@ | ||
| 1 | +import datetime as dt | |
| 2 | +import math | |
| 3 | + | |
| 4 | +import numpy as np | |
| 5 | +import polars as pl | |
| 6 | + | |
| 7 | +from wp39.features import aggregate_buckets, bucket_returns, session_features | |
| 8 | + | |
| 9 | + | |
| 10 | +def _reg_day(n=391, seed=0): | |
| 11 | + rng = np.random.default_rng(seed) | |
| 12 | + r = rng.normal(0, 1e-3, n) | |
| 13 | + px = 100 * np.exp(np.cumsum(r)) | |
| 14 | + return pl.DataFrame({"session_date": [dt.date(2024, 3, 5)] * n, "m": list(range(n)), "open": np.r_[100.0, px[:-1]], | |
| 15 | + "high": px * 1.0005, "low": px * 0.9995, "close": px, "volume": rng.integers(100, 1000, n).astype(float), | |
| 16 | + "close_minute": [390] * n}) | |
| 17 | + | |
| 18 | + | |
| 19 | +def test_bucket_aggregation_counts_and_close_print(): | |
| 20 | + reg = _reg_day() | |
| 21 | + b5 = aggregate_buckets(reg, 5); b30 = aggregate_buckets(reg, 30) | |
| 22 | + assert b5.height == 78 and b30.height == 13 | |
| 23 | + assert b5["close"][-1] == reg["close"][-1] and b30["close"][-1] == reg["close"][-1] | |
| 24 | + assert abs(b5["volume"].sum() - reg["volume"].sum()) < 1e-9 | |
| 25 | + | |
| 26 | + | |
| 27 | +def test_rv_identities_and_shares(): | |
| 28 | + reg = _reg_day() | |
| 29 | + for f in (1, 5, 30): | |
| 30 | + out = session_features(aggregate_buckets(reg, f), f, None) | |
| 31 | + r = out.row(0, named=True); p = f"f{f}m_" | |
| 32 | + assert 0 <= r[p + "rsneg_share"] <= 1 and 0 <= r[p + "rj"] <= 1 and 0 < r[p + "present_share"] <= 1.0 + 1e-9 | |
| 33 | + assert 0 <= r[p + "vol_first30"] <= 1 and 0 <= r[p + "down_vol_share"] <= 1 and 0 <= r[p + "vol_gini"] <= 1 | |
| 34 | + o1 = session_features(aggregate_buckets(reg, 1), 1, None).row(0, named=True) | |
| 35 | + rr = bucket_returns(aggregate_buckets(reg, 1))["r"].to_numpy() | |
| 36 | + assert abs(o1["f1m_rv"] - np.sum(rr ** 2)) < 1e-12 | |
| 37 | + assert abs(o1["f1m_drift"] - math.log(reg["close"][-1] / reg["open"][0])) < 1e-12 | |
added
tests/test_instrument_ids.py
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +import polars as pl | |
| 2 | + | |
| 3 | +from wp39.instruments import build_crosswalk, sic_division | |
| 4 | + | |
| 5 | + | |
| 6 | +def test_sic_divisions(): | |
| 7 | + assert sic_division(3571) == "D_manufacturing" and sic_division(6021) == "H_finance_insurance_realestate" | |
| 8 | + assert sic_division(None) == "unknown" | |
| 9 | + | |
| 10 | + | |
| 11 | +def test_crosswalk_keeps_all_entities_and_flags_missing_cik(): | |
| 12 | + ents = pl.DataFrame({"entity_id": ["frd:AAA", "frd:ZZZ"], "security_id": ["firstrate:stock:AAA", "firstrate:stock:ZZZ"], | |
| 13 | + "ticker": ["AAA", "ZZZ"]}) | |
| 14 | + sec = pl.DataFrame({"ticker": ["AAA"], "cik": [1], "sec_name": ["AAA Corp"]}) | |
| 15 | + subs = pl.DataFrame({"cik": [1], "sic": [3571], "sic_division": ["D_manufacturing"]}) | |
| 16 | + xw = build_crosswalk(ents, sec, subs) | |
| 17 | + assert xw.height == 2 and xw.filter(pl.col("ticker") == "ZZZ")["has_cik"][0] is False | |
added
tests/test_labels.py
+26 −0
@@ -0,0 +1,26 @@ | ||
| 1 | +import datetime as dt | |
| 2 | + | |
| 3 | +import polars as pl | |
| 4 | + | |
| 5 | +from wp39.labels import build_quarter_table, finalize_labels, match_release_dates | |
| 6 | + | |
| 7 | + | |
| 8 | +def test_release_match_rules(toy_wide, toy_events): | |
| 9 | + ev = toy_events.with_columns(pl.when(pl.col("entity_id") == "frd:BBB").then(pl.col("event_date") - pl.duration(days=60)) | |
| 10 | + .otherwise(pl.col("event_date")).alias("event_date")) | |
| 11 | + w = match_release_dates(toy_wide, ev) | |
| 12 | + assert (w.filter(pl.col("entity_id") == "frd:AAA")["release_match"] == "ok").all() | |
| 13 | + assert (w.filter(pl.col("entity_id") == "frd:BBB")["release_match"] != "ok").all() | |
| 14 | + | |
| 15 | + | |
| 16 | +def test_labels_are_not_price_based(toy_wide, toy_events): | |
| 17 | + q = finalize_labels(build_quarter_table(match_release_dates(toy_wide, toy_events))) | |
| 18 | + assert not any(c.startswith("w") or c.startswith("d_") for c in q.columns) | |
| 19 | + assert set(q["y_adverse_opinc"].drop_nulls().unique().to_list()) <= {0, 1} | |
| 20 | + | |
| 21 | + | |
| 22 | +def test_release_day_excluded(toy_wide, toy_events): | |
| 23 | + from wp39.samples import apply_admissibility, event_cutoffs | |
| 24 | + w = match_release_dates(toy_wide, toy_events) | |
| 25 | + c = apply_admissibility(event_cutoffs(w, [1])) | |
| 26 | + assert (c["cutoff"] < c["A_q"]).all() | |
added
tests/test_pit_leakage.py
+45 −0
@@ -0,0 +1,45 @@ | ||
| 1 | +import datetime as dt | |
| 2 | + | |
| 3 | +import polars as pl | |
| 4 | + | |
| 5 | +from wp39.labels import build_quarter_table, finalize_labels, match_release_dates | |
| 6 | +from wp39.pit_fundamentals import asof_latest_available | |
| 7 | +from wp39.samples import apply_admissibility, event_cutoffs | |
| 8 | + | |
| 9 | + | |
| 10 | +def test_no_future_fundamentals(toy_wide): | |
| 11 | + cut = pl.DataFrame({"entity_id": ["frd:AAA"] * 3, "cutoff": [dt.date(2015, 5, 1), dt.date(2015, 5, 11), dt.date(2015, 5, 12)]}) | |
| 12 | + j = asof_latest_available(toy_wide, cut) | |
| 13 | + # period end 2015-03-31, filing 2015-05-10, available 2015-05-11 | |
| 14 | + assert j["pit_period_end"].to_list() == [None, dt.date(2015, 3, 31), dt.date(2015, 3, 31)] | |
| 15 | + assert j.filter(pl.col("pit_period_end").is_not_null())["pit_ok"].all() | |
| 16 | + | |
| 17 | + | |
| 18 | +def test_join_key_is_available_at_not_period_end(toy_wide): | |
| 19 | + cut = pl.DataFrame({"entity_id": ["frd:AAA"], "cutoff": [dt.date(2015, 4, 15)]}) # after period end, before filing | |
| 20 | + j = asof_latest_available(toy_wide, cut) | |
| 21 | + assert j["pit_period_end"][0] is None | |
| 22 | + | |
| 23 | + | |
| 24 | +def test_cutoff_precedes_release_and_filing(toy_wide, toy_events): | |
| 25 | + w = match_release_dates(toy_wide, toy_events) | |
| 26 | + cut = apply_admissibility(event_cutoffs(w, [20, 5, 1])) | |
| 27 | + assert (cut["cutoff"] < cut["A_q"]).all() and (cut["cutoff"] < cut["F_q"]).all() | |
| 28 | + assert cut.filter(pl.col("A_prev").is_not_null()).select((pl.col("cutoff") > pl.col("A_prev")).all()).item() | |
| 29 | + | |
| 30 | + | |
| 31 | +def test_observed_at_not_used(): | |
| 32 | + import inspect | |
| 33 | + from wp39 import labels, pit_fundamentals, samples | |
| 34 | + src = inspect.getsource(samples) + inspect.getsource(labels) + inspect.getsource(pit_fundamentals.asof_latest_available) | |
| 35 | + assert "observed_at" not in src.replace("observed_at\",", "") # only in the column list of the extract | |
| 36 | + | |
| 37 | + | |
| 38 | +def test_expectation_uses_only_prior_quarters(toy_wide, toy_events): | |
| 39 | + w = match_release_dates(toy_wide, toy_events) | |
| 40 | + q = finalize_labels(build_quarter_table(w)) | |
| 41 | + r = q.filter((pl.col("entity_id") == "frd:AAA") & (pl.col("qidx") == 9)).row(0, named=True) | |
| 42 | + x = toy_wide.filter(pl.col("entity_id") == "frd:AAA").sort("qidx")["op_inc"].to_list() | |
| 43 | + drift = sum(x[9 - 1 - k] - x[9 - 5 - k] for k in range(4)) / 4 | |
| 44 | + assert abs(r["E_op_inc"] - (x[5] + drift)) < 1e-9 | |
| 45 | + assert q.filter(pl.col("qidx") < 8)["E_op_inc"].null_count() == q.filter(pl.col("qidx") < 8).height | |
added
tests/test_reproducibility.py
+23 −0
@@ -0,0 +1,23 @@ | ||
| 1 | +import numpy as np | |
| 2 | + | |
| 3 | +from wp39.inference import holm, paired_block_bootstrap, dm_gw_test | |
| 4 | + | |
| 5 | + | |
| 6 | +def test_bootstrap_is_seeded_and_centered(): | |
| 7 | + rng = np.random.default_rng(0) | |
| 8 | + d = rng.normal(0.01, 0.1, 2000) | |
| 9 | + blocks = np.repeat(np.arange(20), 100) | |
| 10 | + a = paired_block_bootstrap(d, blocks, n_boot=300, seed=1) | |
| 11 | + b = paired_block_bootstrap(d, blocks, n_boot=300, seed=1) | |
| 12 | + assert a == b and a["ci_lo"] < a["mean"] < a["ci_hi"] | |
| 13 | + | |
| 14 | + | |
| 15 | +def test_gw_test_rejects_clear_difference(): | |
| 16 | + d = np.full(2000, 0.05) + np.random.default_rng(1).normal(0, 0.02, 2000) | |
| 17 | + blocks = np.repeat(np.arange(20), 100) | |
| 18 | + assert dm_gw_test(d, blocks)["p"] < 0.001 | |
| 19 | + | |
| 20 | + | |
| 21 | +def test_holm_monotone(): | |
| 22 | + adj = holm({"a": 0.01, "b": 0.04, "c": 0.5}) | |
| 23 | + assert adj["a"] <= adj["b"] <= adj["c"] and adj["a"] == 0.03 | |
added
tests/test_splits.py
+37 −0
@@ -0,0 +1,37 @@ | ||
| 1 | +import datetime as dt | |
| 2 | + | |
| 3 | +import polars as pl | |
| 4 | + | |
| 5 | +from wp39.splits import chronological, firm_holdout_mask, overlap_check, walk_forward | |
| 6 | + | |
| 7 | +CFG = {"train": ["2012-01-01", "2018-12-31"], "validation": ["2019-01-01", "2020-12-31"], "test": ["2021-01-01", "2026-10-30"], | |
| 8 | + "walk_forward": {"first_test_year": 2016, "embargo_quarters": 1}, "firm_holdout_fraction": 0.2} | |
| 9 | + | |
| 10 | + | |
| 11 | +def _panel(): | |
| 12 | + rows = [] | |
| 13 | + d = dt.date(2012, 1, 15) | |
| 14 | + for i in range(140): | |
| 15 | + c = d + dt.timedelta(days=int(i * 36.5)) | |
| 16 | + rows.append({"entity_id": f"frd:E{i % 7}", "fiscal_period_end": c - dt.timedelta(days=30), "h": 20, "cutoff": c, | |
| 17 | + "A_q": c + dt.timedelta(days=28), "y": i % 3 == 0}) | |
| 18 | + return pl.DataFrame(rows) | |
| 19 | + | |
| 20 | + | |
| 21 | +def test_chronological_purge_and_embargo(): | |
| 22 | + s = chronological(_panel(), CFG) | |
| 23 | + assert s["train"]["A_q"].max() < dt.date(2018, 10, 1) # validation starts 2019-01-01, embargo one quarter | |
| 24 | + assert s["validation"]["A_q"].max() < dt.date(2020, 10, 1) | |
| 25 | + assert overlap_check(s["train"], s["test"])["shared_observations"] == 0 | |
| 26 | + | |
| 27 | + | |
| 28 | +def test_walk_forward_folds(): | |
| 29 | + folds = walk_forward(_panel(), CFG, 2025) | |
| 30 | + for y, tr, te in folds: | |
| 31 | + assert tr["A_q"].max() < dt.date(y - 1, 10, 1) | |
| 32 | + assert te["cutoff"].min().year == y | |
| 33 | + | |
| 34 | + | |
| 35 | +def test_firm_holdout_is_deterministic(): | |
| 36 | + e = pl.Series(["frd:A", "frd:B", "frd:C"]) | |
| 37 | + assert firm_holdout_mask(e).to_list() == firm_holdout_mask(e).to_list() | |
added
tests/test_timestamp_dsts.py
+30 −0
@@ -0,0 +1,30 @@ | ||
| 1 | +import datetime as dt | |
| 2 | + | |
| 3 | +import polars as pl | |
| 4 | + | |
| 5 | +from wp39.calendar import close_time_utc, regular_session_bounds_utc, shift_trading_days, trading_days_between | |
| 6 | +from wp39.features import session_bounds_table | |
| 7 | + | |
| 8 | + | |
| 9 | +def test_regular_session_bounds_in_both_dst_regimes(): | |
| 10 | + o, c = regular_session_bounds_utc(dt.date(2024, 1, 10)) # EST: 09:30 ET = 14:30 UTC | |
| 11 | + assert (o.hour, o.minute) == (14, 30) and (c.hour, c.minute) == (21, 0) | |
| 12 | + o, c = regular_session_bounds_utc(dt.date(2024, 7, 10)) # EDT: 09:30 ET = 13:30 UTC | |
| 13 | + assert (o.hour, o.minute) == (13, 30) and (c.hour, c.minute) == (20, 0) | |
| 14 | + | |
| 15 | + | |
| 16 | +def test_early_close_day(): | |
| 17 | + b = session_bounds_table(dt.date(2023, 11, 20), dt.date(2023, 11, 30)) | |
| 18 | + r = b.filter(pl.col("session_date") == dt.date(2023, 11, 24)).row(0, named=True) | |
| 19 | + assert r["close_minute"] == 210 # 13:00 ET close the day after Thanksgiving | |
| 20 | + | |
| 21 | + | |
| 22 | +def test_shift_trading_days_skips_weekends_and_holidays(): | |
| 23 | + assert shift_trading_days(dt.date(2024, 7, 5), -1) == dt.date(2024, 7, 3) # July 4 holiday | |
| 24 | + assert shift_trading_days(dt.date(2024, 7, 8), -1) == dt.date(2024, 7, 5) | |
| 25 | + assert trading_days_between(dt.date(2024, 7, 3), dt.date(2024, 7, 8)) == 2 | |
| 26 | + | |
| 27 | + | |
| 28 | +def test_close_print_is_at_1600_et(): | |
| 29 | + c = close_time_utc(dt.date(2022, 3, 14)) # first session after the spring-forward | |
| 30 | + assert (c.hour, c.minute) == (20, 0) | |
added
tests/test_train_only_transforms.py
+15 −0
@@ -0,0 +1,15 @@ | ||
| 1 | +import numpy as np | |
| 2 | +import polars as pl | |
| 3 | + | |
| 4 | +from wp39.models import Preprocessor | |
| 5 | + | |
| 6 | + | |
| 7 | +def test_preprocessor_statistics_come_from_train_only(): | |
| 8 | + tr = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0, 100.0], "z": [0.0, 0.0, 1.0, 1.0, 1.0]}) | |
| 9 | + te = pl.DataFrame({"x": [1000.0, None, -5.0], "z": [5.0, 5.0, 5.0]}) | |
| 10 | + pp = Preprocessor(["x", "z"]).fit(tr) | |
| 11 | + Xte = pp.transform(te) | |
| 12 | + assert pp.fitted_on_rows == 5 | |
| 13 | + assert Xte[0, 0] <= (pp.hi["x"] - pp.mu["x"]) / pp.sd["x"] + 1e-9 # clipped at the train 99th percentile | |
| 14 | + assert abs(Xte[1, 0] - (pp.med["x"] - pp.mu["x"]) / pp.sd["x"]) < 1e-9 # imputed with the train median | |
| 15 | + assert np.all(np.isfinite(Xte)) | |
| 16 | ||