spb/wp39_uqo

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

added .env.example +11 −0

				@@ -0,0 +1,11 @@
		1	+# WP39 UQO — environment-based authorization. Never commit a real .env.
		2	+# FinData OS (mandatory primary data source). Key file is read if FINDATA_API_KEY is unset.
		3	+FINDATA_BASE_URL=https://findata.spboucher.ai/api/v1
		4	+FINDATA_API_KEY=
		5	+FINDATA_API_KEY_FILE=~/findata-secrets/api-key-wp39.txt
		6	+# Scratch cache for licensed raw extracts (outside the repository, never committed)
		7	+WP39_CACHE=~/wp39-cache
		8	+# SEC EDGAR fair-access User-Agent (public data, no secret) for ticker->CIK->SIC crosswalk
		9	+SEC_USER_AGENT="UQO WP39 research (mailto:simon-pierre.boucher@uqo.ca)"
		10	+# Set to 1 to run the live API contract tests
		11	+WP39_LIVE=0

added .gitignore +35 −0

				@@ -0,0 +1,35 @@
		1	+# Raw & processed data — never committed (see data/raw/README.md for provenance)
		2	+data/raw/*
		3	+!data/raw/README.md
		4	+data/processed/*
		5	+!data/processed/.gitkeep
		6	+
		7	+# Python
		8	+__pycache__/
		9	+*.pyc
		10	+.pytest_cache/
		11	+.venv/
		12	+*.egg-info/
		13	+
		14	+# LaTeX build artifacts (paper/main.pdf IS committed)
		15	+paper/*.aux
		16	+paper/*.log
		17	+paper/*.out
		18	+paper/*.toc
		19	+paper/*.lof
		20	+paper/*.lot
		21	+paper/*.fls
		22	+paper/*.fdb_latexmk
		23	+paper/*.bbl
		24	+paper/*.blg
		25	+paper/*.spl
		26	+paper/*.synctex.gz
		27	+paper/*.run.xml
		28	+paper/*.bcf
		29	+
		30	+# Logs
		31	+logs/
		32	+*.log
		33	+
		34	+# macOS
		35	+.DS_Store

added CLAUDE.md +43 −0

				@@ -0,0 +1,43 @@
		1	+# CLAUDE.md — UQO Research WP39
		2	+
		3	+## THE INVISIBLE BALANCE SHEET
		4	+### Can Intraday Financial Markets Detect Corporate Deterioration Before Accounting Disclosures?
		5	+
		6	+**Project ID:** `wp39_uqo` · **Institution:** Université du Québec en Outaouais (UQO) · **PI:** Simon-Pierre Boucher
		7	+**Status:** research project built 2026-11-02 under the owner's specification (reproduced below in condensed form; the
		8	+owner's original brief is the authoritative text). All findings are hypotheses until reproducibly estimated.
		9	+
		10	+Owner instructions added during the build (2026-11-02): (1) the FinData **fundamentals API** is the accounting source and
		11	+must be used; (2) **minimise AI usage** — no LLM features, no neural/text models, no AI-generated statistics.
		12	+
		13	+## Non-negotiable execution rules
		14	+1. FinData OS (`https://findata.spboucher.ai/api/v1`) is the primary source of prices, volumes, earnings events and
		15	+ fundamentals. No silent substitution; every exception is documented in `DATA_AUDIT.md` / `docs/API_INTEGRATION.md`.
		16	+2. Inspect before changing; preserve all other working papers; this repository is its own git repo.
		17	+3. Credentials from the protected store only (`~/findata-secrets/api-key-wp39.txt` on the node); never printed, never
		18	+ in URLs, git, artifacts, notebooks, figures or manuscript. No destructive ingestion, no unapproved backfills.
		19	+4. FirstRate Data is privately licensed: no raw records in git, public endpoints, replication packages or logs.
		20	+5. Hypotheses, variable definitions, PIT rules, baselines, splits and significance plan are frozen in
		21	+ `docs/RESEARCH_DESIGN.md` and `docs/PIT_PROTOCOL.md` before interpreting results. Prediction ≠ causation.
		22	+6. Every claim traces to a saved table/figure/log/test. No fabricated statistics or references.
		23	+7. Phases with gates; `PROGRESS.md`, `DECISIONS.md`, `DATA_AUDIT.md`, `LIMITATIONS.md` are kept current.
		24	+8. English everywhere; LaTeX article compiled with `latexmk` (`paper/`).
		25	+
		26	+## Where things are
		27	+| Item | Location |
		28	+|---|---|
		29	+| Design, PIT rules, novelty | `docs/RESEARCH_DESIGN.md`, `docs/PIT_PROTOCOL.md`, `docs/NOVELTY_MEMO.md` |
		30	+| Data audit, limitations, decisions, progress | root `*.md` files |
		31	+| Code | `src/wp39/` (client, provenance, calendar, instruments, PIT fundamentals, corporate actions, ingest, features, labels, samples, splits, models, inference, portfolio, robustness, figures, pipeline) |
		32	+| Pipeline | `scripts/00_…09_` in order; `scripts/01a_extract_node_tables.sh` runs on M3U96a |
		33	+| Tests | `tests/` (`pytest`; live API tests need `WP39_LIVE=1`) |
		34	+| Artifacts | `artifacts/manifests` (hashes, versions), `artifacts/evaluation`, `artifacts/logs`, `results/`, `figures/` |
		35	+| Paper | `paper/main.tex` + `sections/` + generated `tables/`, `figures/` |
		36	+
		37	+## How to run (compute node M3U96a, API on localhost)
		38	+```bash
		39	+rsync -a --exclude .venv --exclude artifacts/logs ~/Desktop/UQO/Recherche/Recherche-UQO/finance/invisible-balance-sheet/ M3U96a:~/wp39_uqo/
		40	+ssh M3U96a 'cd ~/wp39_uqo && uv venv -p 3.12 .venv && uv pip install -e ".[dev]" && export FINDATA_BASE_URL=http://127.0.0.1:8600/api/v1 && \
		41	+ bash scripts/01a_extract_node_tables.sh && for s in 00 01 02 03 04 05 06 07 08; do .venv/bin/python scripts/${s}_*.py; done'
		42	+```
		43	+Then `rsync` `results/ figures/ artifacts/ paper/tables paper/figures` back and `cd paper && make`.

added DATA_AUDIT.md +167 −0

				@@ -0,0 +1,167 @@
		1	+# DATA_AUDIT.md — WP39 `wp39_uqo`
		2	+
		3	+Phase 0 discovery and feasibility audit of the **FinData OS** API (`https://findata.spboucher.ai/api/v1`) as the primary
		4	+source for *The Invisible Balance Sheet*. All numbers below were measured on **2026-11-02** (America/Toronto) with the
		5	+owner's Claude Code key (admin scope, laptop) and then with a dedicated least-privilege key `wp39_uqo`
		6	+(scopes `read:market, read:fundamentals, query, export`; stored outside git in `~/findata-secrets/api-key-wp39.txt` on
		7	+M3U96a and `~/Desktop/Cluster/secrets/findata/api-key-wp39.txt` on the laptop, mode 600). No credential appears in
		8	+this repository, in logs or in artifacts. Probe transcripts: `artifacts/logs/phase0_probe.json` (produced by
		9	+`scripts/00_probe_findata.py`).
		10	+
		11	+## 0. Viability conclusion
		12	+
		13	+**YES — viable, with two structural caveats that shape the paper.**
		14	+
		15	+1. A licensed, point-in-time usable cohort exists: **5,451 US-listed entities** with quarterly fundamentals
		16	+ (**67 metrics**, `filing_date` 2012-01-03 → 2026-10-30, median 45 quarterly periods per firm), **373,326 earnings-release
		17	+ events** with release date and (for 65 %) before-open/after-close timing, and **6.70 billion raw 1-minute bars** for
		18	+ 16,195 stock symbols (2000-01-03 → 2026-10-30) including 8,069 `-DELISTED` symbols. Exports of 1-minute history are
		19	+ fast (≈ 1 s per symbol-decade, see §6).
		20	+2. **Caveat A — survivorship in fundamentals, not in prices.** The fundamentals/earnings tables contain **zero delisted
		21	+ entities** (ingestion ran for *active* tickers only). Minute bars do cover delisted names, so the price side is
		22	+ survivorship-free, but the accounting panel is conditioned on being listed on 2026-11-01. The vendor does serve
		23	+ fundamentals for at least some delisted tickers (probe: `FMD` → HTTP 200 with data; `CWZ` → 404), so a scoped
		24	+ delisted-fundamentals backfill is *possible* but requires owner approval and a capacity plan (not run; see §8).
		25	+3. **Caveat B — PIT is a proxy.** `available_at = filing_date + 1 day` is a policy, `revision_id = 1` for every fact
		26	+ (`observed_at` = first ingestion, 2026-11-01/02). Values are the vendor's *current* history, so restatements that
		27	+ happened before ingestion are invisible. The design therefore (i) uses only prior-quarter fundamentals as inputs,
		28	+ (ii) uses the **earnings-release date** (not the filing date) as the reveal date, and (iii) reserves a stringent
		29	+ specification using SEC-filing-level reconstruction for a subset (Phase 5, conditional on time).
		30	+
		31	+## 1. API contract — documented vs observed
		32	+
		33	+| Purpose | Route (observed in `openapi.json`, OpenAPI 3.1.0, API 1.0.0) | Observed behaviour |
		34	+|---|---|---|
		35	+| Datasets | `GET /catalog/datasets` | 64 datasets; 62 `active`, `fundamentals.sec_companyfacts` = `pending_configuration`, `market_ticks` = `unsupported` |
		36	+| Dataset detail | `GET /catalog/datasets/{id}` | version, `license_class = frd-private` |
		37	+| Bars | `GET /market/bars` | `interval ∈ {1m, 5m, 1d}` only (no 30m/1h on REST); `adjustment ∈ {raw, split, split_div}`; `limit ≤ 10 000`; `format ∈ {json, arrow, csv}`; `version` pin |
		38	+| Bars schema | `GET /catalog/schema/bars` | schema v1, `timestamp_convention = bar_open`, `timezone_storage = UTC`, 20 fields (see §3) |
		39	+| Fundamentals | `GET /fundamentals/{entity}` | `as_of`, `metrics`, `period_type ∈ {q,a}`, `include_superseded`, **`limit ≤ 20 000`** (default 2 000) |
		40	+| Fundamental periods | `GET /fundamentals/{entity}/periods` | one row per fiscal period with `filing_date`, `available_at`, `max_revision`, `restated` |
		41	+| SQL | `POST /query` | one read-only statement, `max_rows ≤ 100 000`, 30 s; views = dataset ids with `.`→`_` |
		42	+| Exports | `POST /exports` → `GET /exports/{id}` → `/download` | body `{dataset_id, symbols ≤ 500, start, end, adjustment, format}`; **range ≤ 12 years per request (HTTP 413 otherwise)**; Parquet with schema metadata and pinned `X-Dataset-Version` |
		43	+| Coverage | `GET /coverage`, `/coverage/symbols` (`limit ≤ 5 000`, cursor), `/coverage/heatmap` | per-session bar counts vs `expected_bars` (391 regular) |
		44	+| Instruments | `GET /instruments`, `/instruments/{symbol}` | status `active`/`delisted`, first/last date |
		45	+| Quality | `GET /quality`, `/quality/anomalies` | row-level flags (`price_jump`, `outside_session`, …) |
		46	+| **Earnings events** | **no route** | dataset `fundamentals.earnings` is listed (0 rows reported) but rows live in Postgres `earnings_events` → read-only extraction on the node (documented exception, §7) |
		47	+| Corporate actions | **no route** | Postgres `corporate_actions`: 6,131 splits (2000-01-03 → 2026-10-15), 243,209 cash dividends; applied at query time via `adjustment=` |
		48	+
		49	+Reconciliation notes: the catalog reports `row_count = 0` for Postgres-backed datasets (`fundamentals.*`, `corporate_actions`,
		50	+`reference.instruments`); that is a catalog limitation, not an absence of data. The REST `interval` enum is narrower than the
		51	+lake (30m/1h exist only as vendor-adjusted datasets); WP39 rebuilds 5m/30m/1d bars **from the raw 1m export** so that every
		52	+frequency shares one source and one adjustment path.
		53	+
		54	+## 2. Market bars — coverage
		55	+
		56	+Counts of distinct stock symbols with at least one bar on a mid-June session (SQL playground, `session_date =` predicate):
		57	+
		58	+| Session | 1m symbols | 1m bars | 1d symbols | of which `-DELISTED` in 1d |
		59	+|---|---:|---:|---:|---:|
		60	+| 2001-06-15 | — | — | 1,088 | 0 |
		61	+| 2005-06-15 | 3,498 (340,665 bars delisted) | 631,260 | 1,504 | 0 |
		62	+| 2010-06-15 | — | — | 2,309 | 0 |
		63	+| 2015-06-15 | 6,489 | 1,207,920 | 2,985 | 0 |
		64	+| 2020-06-15 | — | — | 4,144 | 0 |
		65	+| 2025-06-16 | 6,918 | 1,317,343 | 6,485 | 0 |
		66	+
		67	+Finding: the vendor **1d dataset is incomplete** relative to the 1m dataset (about half the symbols in 2005–2015 and no
		68	+delisted names). Decision D-003: daily bars are *derived* from the 1m export (regular session 09:30–16:00 ET) — never taken
		69	+from `market_bars.stock.1d.raw`. Delisted symbols are present in the 1m lake (e.g. `AA.B-DELISTED` 531 sessions
		70	+2014–2016; `CWZ-DELISTED` 2000–2014).
		71	+
		72	+Bar conventions verified on `XOM` and `SPY` exports: timestamps are bar **open** in UTC (`2012-01-03 09:01Z` = 04:01 ET),
		73	+extended hours 04:00–19:59 ET are present, zero-volume minutes are absent (bar counts per session 405–798 vs 391 expected
		74	+regular bars), `quality_flags` list is empty on normal rows, `revision_id = 1`.
		75	+
		76	+## 3. Bars schema (Arrow, schema v1)
		77	+
		78	+`instrument_id, symbol_as_reported, venue (null), asset_class, interval, timestamp_utc (timestamp[us, UTC], bar open),
		79	+session_date (date32, exchange tz), open, high, low, close, volume (double), currency, trade_count (int32?, derived only),
		80	+vwap (double?, derived only), adjustment_type, source_id, ingested_at, quality_flags (list<string>), revision_id (int32)`.
		81	+Parquet exports carry metadata `findata.schema = bars`, `findata.schema_version = 1`, `findata.timestamp_convention = bar_open`.
		82	+
		83	+## 4. Fundamentals — coverage and PIT fields
		84	+
		85	+- Entities: **5,451** (`frd:<TICKER>`, `security_id = firstrate:stock:<TICKER>`, none delisted, none with `security_id` null).
		86	+- Metrics observed per period: **67** (the vendor documents 76; 9 are never populated in the API output). Names:
		87	+ `acc_oci, assets_c, assets_nc, assets_tot, capex, cash, cash_business_acq, cash_change, cash_debt_sec, cash_div,
		88	+ cash_equity, cash_fin, cash_inv, cash_inv_acq, cash_op, cogs, cos_inc, debt, debt_c, debt_nc, def_rev, depr, dps, ebit,
		89	+ ebitda, ebt, eps, eps_d, equity, fx_impact_cash, gm, gp, intangibles, interest_exp, intr_cover, inv_c, inv_nc, inventory,
		90	+ investments, liab_c, liab_nc, liab_tot, net_inc, net_inc_com, net_inc_non_control, net_loss_discontinued, om, op_ex,
		91	+ op_inc, payables, ppe, pref_divs, receivables, research_dev, ret_earnings, rev, roa, roe, roic, sgna, share_comp, shares,
		92	+ shares_av, shares_av_diluted, tax_asset, tax_expense, tax_liability`.
		93	+- Quarterly `rev` facts by fiscal year (entities / facts): 2011: 1,798 / 1,812 · 2012: 2,461 / 8,725 · 2013: 2,623 / 9,868 ·
		94	+ 2014: 2,771 / 10,560 · 2015: 2,916 / 11,145 · 2016: 3,076 / 11,656 · 2017: 3,276 / 12,340 · 2018: 3,505 / 13,209 ·
		95	+ 2019: 3,763 / 14,034 · 2020: 4,309 / 15,488 · 2021: 4,663 / 17,458 · 2022: 4,851 / 18,289 · 2023: 5,043 / 18,904 ·
		96	+ 2024: 5,244 / 19,467 · 2025: 5,370 / 19,625 · 2026: 4,928 / 10,915. `filing_date` is never null (no `pit_fallback`).
		97	+- Firms by number of quarterly periods: ≥ 8: 5,018 · ≥ 20: 4,331 · ≥ 40: 2,891. First quarter year: 2011: 2,141 ·
		98	+ 2012: 317 · 2013–2019: 148–256 per year · 2020: 551 · 2021: 353 · 2022–2025: 164–215 per year.
		99	+- Filing lag (filing_date − fiscal_period_end, quarterly): p10 = 30 d, median = 40 d, p90 = 77 d.
		100	+- Q4 values are genuine quarterly flows (AAPL FY2019 revenue 260.17 bn = 84.31 + 58.02 + 53.81 + 64.04 bn).
		101	+- PIT fields per fact: `filing_date`, `published_at` (null), `available_at = filing_date + 1 d 00:00 ET`, `observed_at`
		102	+ (= 2026-11-01 18:24 ET for the first ingestion), `revision_id = 1`, `restatement_flag = false`, `superseded_at = null`.
		103	+ The API `as_of` semantics (`available_at ≤ as_of` and (`revision_id = 1` or `observed_at ≤ as_of`)) were exercised:
		104	+ AAPL `as_of = 2020-01-01` returns 2,000 facts (limit) and reports `facts_hidden_by_as_of = 2211`.
		105	+- Entity master: `cik`, `sector`, `industry` are **null for all 5,451 entities** → external SEC crosswalk needed (§7).
		106	+
		107	+## 5. Earnings-release events (reveal dates)
		108	+
		109	+Postgres `earnings_events` (`entity_id, event_date, eps_estimate, eps_reported, release_time, source_id, observed_at`):
		110	+373,326 rows, 5,451 entities, 2000-01-01 → 2027-04-29, one row per (entity, date). `release_time`: `after_close` 124,832 ·
		111	+`before_open` 120,586 · null 127,908. For 2012–2024 the null share is 35–36 %; `eps_estimate` is null for 24–27 % of
		112	+events (vendor estimate; vintage unknown → exploratory only). Row counts for 2025–2026 are inflated by forward-dated
		113	+placeholders (2026: 35,361 events), which are dropped at the sample end (2026-10-30).
		114	+
		115	+Matching quarterly filings to the first release event in `[period_end, filing_date + 10 d]` (213,954 filings 2012→):
		116	+96.3 % matched; 1,493 (0.7 %) have `filing_date < event_date`; 16,868 (7.9 %) have a lag > 30 d (likely mismatches or
		117	+late filers); median `filing_date − event_date` = **1 day**. Rule (PIT_PROTOCOL §3): reveal date `A_q` = matched release
		118	+date; a firm-quarter is **excluded** when no release is matched, when `filing_date < event_date`, or when the lag exceeds
		119	+30 days; the prediction cutoff always satisfies `c < min(A_q, F_q)` in trading days.
		120	+
		121	+## 6. Throughput and capacity (measured on M3U96a, API on localhost)
		122	+
		123	+| Operation | Measured |
		124	+|---|---|
		125	+| Export 1m raw, 1 symbol, 2012-01-01→2019-12-31 (`XOM`) | ready in 1.0 s, 859,354 rows, 13.6 MB Parquet, dataset version 17375 |
		126	+| Export 1m raw, 1 symbol, 2020-01-01→2026-10-30 | ready in 1.0 s, 884,983 rows, 15.4 MB |
		127	+| Full-range export (2012→2026, 14.8 years) | **HTTP 413** — limit 12 years per request → two requests per symbol |
		128	+| SQL cross-section (`session_date =` on 1m) | ≈ 3 s |
		129	+| Coverage per symbol (`/coverage/symbols`) | ms, cursor-paged |
		130	+| Fundamentals, one entity, all quarterly facts | one call (≤ 20 000 rows) |
		131	+
		132	+Capacity model for the main sample (≈ 4,300 firms with ≥ 8 quarters and 1m coverage): 2 exports × ~1.5 s ≈ 3.6 h of
		133	+export wall-time (sequential; 4 concurrent streams bring it under 1 h), ≈ 30 MB raw Parquet per firm (≈ 130 GB if kept).
		134	+Decision D-004: raw extracts are processed **streaming** (export → daily feature rows at 1m/5m/30m/1d → delete raw file,
		135	+keep SHA-256 + row count in the manifest), so steady-state scratch usage stays below 5 GB. The storage node M3U96a
		136	+(28 cores, 96 GB RAM, 319 GB free on the internal SSD, hard floor 60 GB for FinData) hosts the computation; the feature
		137	+panel (≈ 4,300 firms × ~3,700 sessions × 4 frequencies) is < 2 GB.
		138	+
		139	+## 7. External sources and exceptions
		140	+
		141	+| Need | Source | Status | Why not FinData |
		142	+|---|---|---|---|
		143	+| Reveal dates / release timing | FinData Postgres `earnings_events` (read-only `psql` on M3U96a) | available | not exposed by the API (gap logged for the FinData backlog) |
		144	+| CIK, SIC industry | SEC `company_tickers.json` (10,423 rows) + `submissions/CIK##########.json` (`sic`, `sicDescription`, `exchanges`, `formerNames`) | verified reachable (fair-access UA) | entity master has no CIK/sector |
		145	+| Daily factors (H5 only) | Kenneth French library, FF5 2×3 daily + momentum daily | verified (HTTP 200, updated 2026-10-26) | not a FinData dataset |
		146	+| Analyst consensus | vendor `eps_estimate` in `earnings_events` (24–27 % missing, vintage unknown) | exploratory only | no point-in-time consensus history available |
		147	+| TAQ / quotes / order flow | none | **not available** (vendor supplies bars only) | bar-based proxies are *never* called order flow, spreads or informed trading |
		148	+
		149	+## 8. Gaps, risks and recommendations to the owner
		150	+
		151	+1. **Delisted fundamentals backfill (recommended, not run).** A one-off `fundamentals` job for the 8,069 `-DELISTED`
		152	+ tickers (vendor returns data for at least part of them) would remove the survivorship conditioning of the accounting
		153	+ panel. Estimated cost: ≤ 8,069 vendor calls (1 req/s policy ≈ 2.3 h), a few hundred MB in Postgres. Requires: owner
		154	+ approval, `findata ingest fundamentals --delisted` (does not exist yet; connector change on M3U96a `~/apps/findata`).
		155	+2. **Earnings events API route** — expose `fundamentals.earnings` through `/api/v1/earnings/{entity}` so that WP39 (and
		156	+ future projects) stop reading Postgres directly.
		157	+3. **SEC EDGAR connector** (`fundamentals.sec_companyfacts`, pending a User-Agent setting) would give filing-level
		158	+ as-originally-reported values for the stringent PIT specification.
		159	+4. API bug found: `POST /admin/keys` returns HTTP 500 (`api_keys_user_id_fkey` violation) when the caller is an API key
		160	+ whose `user_id` is not in `users`; the CLI `findata create-key` works. Logged for the FinData backlog, not fixed here.
		161	+
		162	+## 9. License and privacy constraints applied
		163	+
		164	+FirstRate Data: perpetual private use, research and derived statistics allowed with attribution, **no redistribution of raw
		165	+records**. Consequently: no raw bars or fundamentals are committed; `data/raw` stays empty; the replication package ships
		166	+code, manifests (hashes, versions, counts), derived aggregate tables and figures only; the manuscript attributes FirstRate
		167	+Data via FinData OS. SEC and Kenneth French data are public.

added DECISIONS.md +14 −0

				@@ -0,0 +1,14 @@
		1	+# DECISIONS.md — dated design and engineering decisions (WP39)
		2	+
		3	+| ID | Date (ET) | Decision | Reason | Consequence |
		4	+|---|---|---|---|---|
		5	+| D-001 | 2026-11-02 13:00 | Compute on **M3U96a** (FinData host) with a dedicated least-privilege key `wp39_uqo`; repository lives in the UQO Research workspace (`Recherche-UQO/finance/invisible-balance-sheet`, dir name follows the series convention; project id `wp39_uqo`) and is rsync'd to `M3U96a:~/wp39_uqo` for runs | API on localhost, lake co-located, 28 cores / 96 GB; secrets never leave `~/findata-secrets` / `~/Desktop/Cluster/secrets` | results and manifests synced back; raw cache stays on the node |
		6	+| D-002 | 2026-11-02 13:05 | **Earnings release dates** (`earnings_events.event_date`) define the reveal date `A_q`; the filing date `F_q` defines structured-fact availability (`available_at = F_q + 1`) | press releases precede 10-Q filings by a median of 1 day; cutoffs must precede the *first* public reveal | cutoff rule `c < min(A_q, F_q)`; release rows read from Postgres (no API route) — documented exception |
		7	+| D-003 | 2026-11-02 13:10 | Daily bars are **rebuilt from the 1-minute raw export** (regular session), not taken from `market_bars.stock.1d.raw` | vendor 1d dataset covers about half the symbols before 2016 and no delisted names | one source, one adjustment path for all four frequencies |
		8	+| D-004 | 2026-11-02 13:30 | Streaming feature extraction (export → features → delete raw Parquet; manifest keeps SHA-256) | disk budget on the FinData host (hard floor 60 GB free) | re-runs re-export from the pinned dataset version |
		9	+| D-005 | 2026-11-02 14:00 | Universe = all FinData fundamentals entities (survivor-conditioned); delisted-fundamentals backfill **proposed, not run** | vendor ingestion covered active tickers only; backfill needs owner approval + capacity plan (rule 3 of the brief) | survivorship caveat stated in abstract, data section, limitations; price side is survivorship-free |
		10	+| D-006 | 2026-11-02 14:05 | **No neural (B7) nor text/multimodal (B8) models; no LLM anywhere in the pipeline** | owner instruction received mid-session on 2026-11-02 ("limite le plus possible l'utilisation AI"); also keeps the frequency comparison about information sets rather than capacity | ladder is B0–B6; LightGBM provides the nonlinear benchmark with equal budgets |
		11	+| D-007 | 2026-11-02 14:10 | Primary target = adverse **operating-income** surprise vs a seasonal random walk with drift, scaled by total assets, threshold −0.5 % | no point-in-time consensus exists in FinData; operating income is less affected by one-off items than net income; Foster (1977) model is the classic honest expectation | consensus-based miss (vendor `eps_estimate`) is exploratory only |
		12	+| D-008 | 2026-11-02 14:15 | Frequency sets B3/B4/B5 use **identical feature families** (one column per family) computed from bars of 30 m / 5 m / 1 m | compare information sets, not dimensionality | a supplementary "all frequencies" model is run in Phase 4 |
		13	+| D-009 | 2026-11-02 14:20 | Literature verified with **Crossref metadata** (no AI summarisation); references without a verifiable DOI are cited without one (Grossman–Stiglitz 1980, Holm 1979) | rule 7 of the brief (no fabricated references) | `artifacts/logs/crossref_batch*.json` are the evidence |
		14	+| D-010 | 2026-11-02 14:30 | Phase gates that require "owner approval" are recorded as **self-approved under the autonomous mandate** with the memo attached; the owner can veto any of them before Phase 6 | the owner asked for the whole project to be built in one autonomous session and is not available mid-task | each gate entry in PROGRESS.md names the memo and the date |

added LIMITATIONS.md +31 −0

				@@ -0,0 +1,31 @@
		1	+# LIMITATIONS.md — living list (WP39)
		2	+
		3	+## Data
		4	+- **Survivor-conditioned accounting panel.** Fundamentals and earnings-release rows exist only for the 5,451 tickers active
		5	+ on 2026-11-01. Firms that delisted during 2012–2026 (including those that deteriorated and failed) are absent from the
		6	+ firm-quarter sample, although their minute bars exist. Base rates of severe deterioration are therefore understated and
		7	+ H3/H5 effects for the most vulnerable firms are not identified. A delisted-fundamentals backfill is proposed (DATA_AUDIT §8).
		8	+- **PIT is a documented proxy.** `available_at = filing_date + 1 day` is a policy; all facts are `revision_id = 1` observed
		9	+ on 2026-11-01/02, i.e. the vendor's current (possibly restated) history. Original-as-reported values are not available.
		10	+ Inputs use prior quarters only, so restatements affect the *measurement of inputs and targets*, not the direction of time.
		11	+- **OHLCV bars, not ticks.** No quotes, no trades, no order flow. Bar-based statistics (down-bar volume share, bar
		12	+ autocorrelation) are descriptive proxies and are never interpreted as informed trading or spreads.
		13	+- Vendor merges ticker changes under the latest ticker (fully since 2022); CIK/SIC come from the SEC's *current*
		14	+ `company_tickers.json`, so historical ticker reuse before 2022 is only partially auditable (`formerNames` check).
		15	+- `release_time` is unknown for ~35 % of release events; we never use release-day bars, so this affects only the
		16	+ descriptive timing exhibit.
		17	+- The vendor `eps_estimate` has an unknown vintage and is missing for a quarter of events → exploratory only.
		18	+- 1-minute coverage is thinner before ~2005 and for illiquid names; the sample starts in 2012 and requires ≥ 100 regular
		19	+ bars per session, which excludes the most illiquid firm-days (a documented selection on liquidity).
		20	+
		21	+## Design
		22	+- The event-time cohort conditions on the *existence* of a release in the next 65 trading days; a hypothetical observer
		23	+ knows the approximate schedule but not the exact date. The fixed-calendar design is reported as the robustness check.
		24	+- Prediction, not causation: a positive H1 shows that bar-level information is statistically informative; it does not
		25	+ establish who traded on private information.
		26	+- No B7/B8 (neural/text) models by owner instruction; the frequency comparison is therefore within linear and
		27	+ gradient-boosted families only.
		28	+
		29	+## To be completed as results arrive
		30	+- (Phase 3) achieved sample sizes, base rates, coverage gaps by year/size.
		31	+- (Phase 5) which placebo/stress tests failed or were inconclusive.

added PROGRESS.md +22 −0

				@@ -0,0 +1,22 @@
		1	+# PROGRESS.md — WP39 `wp39_uqo`
		2	+
		3	+## Phase 0 — Discovery and feasibility (2026-11-02) — GATE PASSED
		4	+- Inspected the UQO Research workspace (`Recherche-UQO/`, 39 papers, harmonisation standard); no `wp39` existed; created
		5	+ `finance/invisible-balance-sheet/` from the template; git initialised on a dedicated repository (no other paper touched).
		6	+- Probed FinData: `/health` ok, 64 datasets, OpenAPI 3.1 (45 routes), bars schema v1, fundamentals (67 metrics, 5,451
		7	+ entities, 2012→2026), earnings events (Postgres), coverage endpoints, export throughput (≈ 1 s per symbol-decade;
		8	+ 12-year cap per request). Created least-privilege key `wp39_uqo`.
		9	+- Verified external public sources: SEC `company_tickers.json` + `submissions` (CIK/SIC), Kenneth French daily factors.
		10	+- Verified 94 bibliography entries on Crossref; wrote `docs/NOVELTY_MEMO.md`, `literature/RELATED_WORK_MATRIX.csv`.
		11	+- Outputs: `DATA_AUDIT.md` (viability = YES with survivorship + PIT caveats), `configs/*.yaml`, `docs/API_INTEGRATION.md`,
		12	+ `docs/COMPUTE_BUDGET.md`, `docs/PIT_PROTOCOL.md`, `docs/RESEARCH_DESIGN.md` (frozen), `DECISIONS.md` D-001…D-010.
		13	+- Owner instructions received during the session: (1) the fundamentals API must be used — it is the accounting source;
		14	+ (2) minimise AI usage — no LLM features, no neural models (D-006).
		15	+- Blockers: none. Next safe step: Phase 1 gate memo, then code + tests (Phase 2).
		16	+
		17	+## Phase 1 — Research design and novelty — GATE (self-approved under the autonomous mandate, D-010)
		18	+- `docs/RESEARCH_DESIGN.md` frozen 2026-11-02 14:40 ET; hypotheses H1–H5 with null/alternative, primary endpoint (H1,
		19	+ h = 20, log loss, binary adverse operating-income surprise), decision rules, splits, inference, power plan.
		20	+- Owner review pending; nothing in Phases 2–5 depends on test-period data until the Phase 3 gate.
		21	+
		22	+## Phase 2 — Data acquisition and verification — IN PROGRESS

added configs/data_contract.yaml +40 −0

				@@ -0,0 +1,40 @@
		1	+# Data contract — verified against the live FinData OS API on 2026-11-02 (see DATA_AUDIT.md)
		2	+findata:
		3	+ base_url: https://findata.spboucher.ai/api/v1
		4	+ openapi: /openapi.json
		5	+ datasets:
		6	+ bars_1m: market_bars.stock.1m.raw # raw, bar-open timestamps in UTC, 04:00-20:00 ET, zero-volume minutes absent
		7	+ bars_1m_etf: market_bars.etf.1m.raw # SPY used as market proxy
		8	+ bars_1d: market_bars.stock.1d.raw # INCOMPLETE vs 1m (fewer symbols) -> daily bars are rebuilt from 1m
		9	+ fundamentals: fundamentals.financials # Postgres-backed; GET /fundamentals/{entity}
		10	+ earnings: fundamentals.earnings # Postgres table earnings_events (no API route yet) -> documented exception
		11	+ limits:
		12	+ bars_1m_days_per_call: 31
		13	+ rows_per_page: 10000
		14	+ export_years_per_request: 12
		15	+ export_symbols_per_request: 500
		16	+ query_timeout_s: 30
		17	+ query_max_rows: 100000
		18	+ rate_limit_per_min: 600
		19	+ fundamentals_limit_max: 20000
		20	+ bars_schema_version: 1
		21	+ timestamp_convention: bar_open
		22	+ timezone_storage: UTC
		23	+ exchange_tz: America/New_York
		24	+ regular_session: ["09:30", "16:00"] # the 16:00 bar is the closing print
		25	+ extended_session: ["04:00", "20:00"]
		26	+ license_class: frd-private
		27	+pit:
		28	+ available_at_policy: "filing_date + 1 day 00:00 America/New_York (vendor proxy; fallback period_end + 91 d flagged pit_fallback)"
		29	+ revision_semantics: "revision_id=1 observed 2026-11-01/02; no pre-ingestion restatement history (vendor limitation)"
		30	+ reveal_date: "earnings_events.event_date (press release), release_time in {before_open, after_close, null}"
		31	+universe:
		32	+ entity_source: fundamentals entities (5,451 active tickers as of 2026-11-01; NO delisted entities -> survivorship caveat)
		33	+ min_quarters: 8
		34	+ first_filing_year: 2012
		35	+ last_session: 2026-10-30
		36	+external:
		37	+ sec_company_tickers: https://www.sec.gov/files/company_tickers.json
		38	+ sec_submissions: https://data.sec.gov/submissions/CIK{cik:010d}.json
		39	+ ff5_daily: https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/F-F_Research_Data_5_Factors_2x3_daily_CSV.zip
		40	+ mom_daily: https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/F-F_Momentum_Factor_daily_CSV.zip

added configs/models.yaml +27 −0

				@@ -0,0 +1,27 @@
		1	+# Equal tuning budget across information sets. Hyper-parameters are tuned ONLY on the validation period.
		2	+baselines:
		3	+ B0: {type: base_rate}
		4	+ B1: {type: logistic, inputs: [fundamentals, controls]}
		5	+ B2: {type: logistic, inputs: [fundamentals, controls, daily]}
		6	+ B3: {type: logistic, inputs: [fundamentals, controls, daily, intraday_30m]}
		7	+ B4: {type: logistic, inputs: [fundamentals, controls, daily, intraday_5m]}
		8	+ B5: {type: logistic, inputs: [fundamentals, controls, daily, intraday_1m]}
		9	+logistic:
		10	+ penalty: l2
		11	+ C_grid: [0.01, 0.03, 0.1, 0.3, 1.0, 3.0]
		12	+ standardize: train_only
		13	+ impute: train_median
		14	+lightgbm:
		15	+ information_sets: [B1, B2, B3, B4, B5]
		16	+ budget_trials: 24 # identical random-search budget for every information set
		17	+ seed: 20261102
		18	+ search_space:
		19	+ num_leaves: [7, 15, 31, 63]
		20	+ learning_rate: [0.01, 0.03, 0.05]
		21	+ min_child_samples: [50, 100, 200]
		22	+ feature_fraction: [0.5, 0.7, 0.9]
		23	+ bagging_fraction: [0.6, 0.8, 1.0]
		24	+ lambda_l2: [0.0, 1.0, 10.0]
		25	+ n_estimators_max: 2000
		26	+ early_stopping_rounds: 100
		27	+neural: {enabled: false, reason: "owner instruction 2026-11-02: minimise AI usage; B7/B8 not run"}

added configs/research.yaml +42 −0

				@@ -0,0 +1,42 @@
		1	+# Frozen research design parameters (see docs/RESEARCH_DESIGN.md). Changing a value requires a DECISIONS.md entry.
		2	+seed: 20261102
		3	+sample:
		4	+ start: 2012-01-01
		5	+ end: 2026-10-30
		6	+ min_quarters_per_firm: 8
		7	+ min_price_usd: 1.0 # at cutoff, raw close
		8	+ min_regular_bars_per_day: 100 # sessions with fewer 1m regular-session bars are "sparse"
		9	+ min_valid_days_in_window: 15 # out of the 20-trading-day feature window
		10	+cutoffs:
		11	+ event_time_horizons_td: [40, 20, 10, 5, 1] # trading days before the earnings release date A_q
		12	+ primary_horizon_td: 20
		13	+ fixed_calendar: month_end # last trading day of each month
		14	+ embargo_rule: "cutoff c < min(A_q, F_q); bars with session_date <= c only; fundamentals with available_at <= c only"
		15	+feature_window:
		16	+ short_td: 20
		17	+ long_td: 60
		18	+frequencies: [1d, 30m, 5m, 1m]
		19	+targets:
		20	+ primary:
		21	+ name: adverse_opinc_surprise
		22	+ variable: op_inc
		23	+ expectation: seasonal_random_walk_with_drift # E[X_q] = X_{q-4} + mean(X_{q-1-k} - X_{q-5-k}, k=0..3), PIT
		24	+ scale: assets_tot (latest available)
		25	+ threshold: -0.005 # surprise <= -0.5% of total assets
		26	+ continuous_primary: scaled_opinc_surprise
		27	+ secondary:
		28	+ - {name: roa_yoy_decline, rule: "roa_q - roa_{q-4} <= -0.01"}
		29	+ - {name: ocf_innovation_negative, rule: "(cash_op_q - cash_op_{q-4})/assets <= -0.01"}
		30	+ - {name: liquidity_deterioration, rule: "current ratio falls >= 10% yoy OR interest coverage falls below 1 from >= 1"}
		31	+ - {name: eps_miss, rule: "eps_reported < eps_estimate (vendor estimate, exploratory)"}
		32	+splits:
		33	+ train: [2012-01-01, 2018-12-31]
		34	+ validation: [2019-01-01, 2020-12-31]
		35	+ test: [2021-01-01, 2026-10-30]
		36	+ walk_forward: {first_test_year: 2016, embargo_quarters: 1}
		37	+ firm_holdout_fraction: 0.2
		38	+inference:
		39	+ cluster: [entity_id, calendar_quarter]
		40	+ bootstrap_blocks: calendar_quarter
		41	+ n_bootstrap: 1000
		42	+ multiple_testing: holm

added data/README.md +17 −0

				@@ -0,0 +1,17 @@
		1	+# data/
		2	+
		3	+No licensed raw records are stored in this repository.
		4	+
		5	+| Path | Content | Committed? |
		6	+|---|---|---|
		7	+| `data/raw/` | nothing (raw FirstRate bars live only in the ignored scratch cache `$WP39_CACHE` on the compute node) | no |
		8	+| `data/processed/` | derived firm-quarter panel, feature tables and labels (Parquet). Derived aggregates only; still **not** committed (size, license caution) | no |
		9	+| `artifacts/manifests/` | hashed dataset manifests: dataset ids, `X-Dataset-Version`, extraction dates, row counts, SHA-256 of extracts, code SHA | yes |
		10	+
		11	+Provenance rules (see `DATA_AUDIT.md`, `docs/API_INTEGRATION.md`):
		12	+- Market bars and fundamentals come from the owner's FinData OS API (`https://findata.spboucher.ai/api/v1`), a private
		13	+ lakehouse of FirstRate Data (license: private use, no redistribution of raw records).
		14	+- Earnings-release calendar rows are read from the FinData Postgres table `earnings_events` on the storage node because the
		15	+ API does not expose that dataset yet (documented exception).
		16	+- External public augmentations: SEC EDGAR `company_tickers.json` and `submissions` (CIK, SIC), Kenneth French data library
		17	+ (daily factors). Both are public, versioned by download date in the manifests.

added data/processed/.gitkeep +0 −0

added data/raw/README.md +4 −0

				@@ -0,0 +1,4 @@
		1	+# data/raw — intentionally empty
		2	+Raw FirstRate Data records (via FinData OS) are licensed for private use and are never stored in this repository.
		3	+They live only in the ignored scratch cache on the compute node (`$WP39_CACHE`), are deleted after feature extraction,
		4	+and are described by hashed manifests in `artifacts/manifests/`.

added docs/API_INTEGRATION.md +41 −0

				@@ -0,0 +1,41 @@
		1	+# API_INTEGRATION.md — how WP39 talks to FinData OS
		2	+
		3	+## Authorization
		4	+- Header `Authorization: Bearer <key>`; the key is read from `FINDATA_API_KEY` or the file `FINDATA_API_KEY_FILE`
		5	+ (default `~/findata-secrets/api-key-wp39.txt`, mode 600). The client never logs headers, never puts the key in a URL,
		6	+ and redacts any string starting with `fd_` in exceptions (`src/wp39/api_client.py::_redact`).
		7	+- Dedicated key `wp39_uqo` (scopes `read:market, read:fundamentals, query, export`; **no admin**), created 2026-11-02 with
		8	+ the node CLI `findata create-key` because `POST /admin/keys` returned HTTP 500 for API-key callers (bug logged in
		9	+ DATA_AUDIT §8).
		10	+- Compute runs on **M3U96a** against `http://127.0.0.1:8600/api/v1` (same host as the lake; the public URL
		11	+ `https://findata.spboucher.ai/api/v1` is used from the laptop for probes only).
		12	+
		13	+## Client (`src/wp39/api_client.py`)
		14	+- `httpx.Client` with 60 s timeout, retry with bounded exponential backoff (5 attempts, 0.5 → 8 s, jitter) on 429/5xx and
		15	+ network errors; idempotent GETs only are retried automatically; export creation is retried only when no export id was
		16	+ received.
		17	+- Every response's `X-Request-ID`, `X-Dataset-Id`, `X-Dataset-Version` are recorded in the provenance log
		18	+ (`artifacts/logs/api_calls.jsonl`, one line per call, no payloads).
		19	+- Cursor iteration for `/market/bars` and `/coverage/symbols`; export polling (`/exports/{id}`) every 1 s up to 10 min.
		20	+- Arrow/Parquet schema validation against `GET /catalog/schema/bars` (schema v1) before any file is accepted.
		21	+
		22	+## Calls used by the pipeline
		23	+| Step | Call | Scope | Volume |
		24	+|---|---|---|---|
		25	+| 00 probe | `/health`, `/catalog/datasets`, `/catalog/schema/bars`, `/openapi.json` | read | 4 |
		26	+| 01 universe | `/coverage/symbols?dataset_id=market_bars.stock.1m.raw` (cursor, 5,000/page), `/instruments/{symbol}` | read | ≈ 4 + N |
		27	+| 02 PIT panel | `/fundamentals/{entity}?period_type=q&limit=20000` (all facts, local as-of join), `/fundamentals/{entity}/periods` | read:fundamentals | 2 × N entities |
		28	+| 03 features | `POST /exports` (1m raw, ≤ 12 y, 1 symbol) ×2 per symbol, `/exports/{id}`, `/download`; SPY from `market_bars.etf.1m.raw` | export | 2 × N + 2 |
		29	+| checks | `POST /query` (cross-section counts, spot checks ≤ 100 k rows) | query | few |
		30	+
		31	+Exception (documented): earnings-release rows are extracted once on M3U96a with
		32	+`psql -d findata -c "\copy (select ... from earnings_events) to '…'"` (read-only). The extract's SHA-256, row count and
		33	+timestamp are in `artifacts/manifests/earnings_events.json`.
		34	+
		35	+## Idempotent partitioning and manifests
		36	+- Exports are cached per `(symbol, chunk)` as `$WP39_CACHE/bars/<symbol>__<start>__<end>.parquet` and deleted after the
		37	+ feature rows are written; `artifacts/manifests/bars_manifest.jsonl` keeps `symbol, start, end, dataset_id,
		38	+ dataset_version, row_count, bytes, sha256, exported_at, request_id`.
		39	+- Fundamentals extracts: `$WP39_CACHE/fundamentals/<entity>.parquet` with the same manifest fields.
		40	+- Re-running a script skips completed partitions (manifest lookup) and never overwrites a partition with a different
		41	+ dataset version without logging a `version_drift` warning.

added docs/COMPUTE_BUDGET.md +16 −0

				@@ -0,0 +1,16 @@
		1	+# COMPUTE_BUDGET.md
		2	+
		3	+| Resource | Plan | Measured / status |
		4	+|---|---|---|
		5	+| Compute node | M3U96a (Mac Studio M3 Ultra, 28 cores, 96 GB RAM) — FinData API on localhost | load 1.5–1.9 before start |
		6	+| Scratch | `~/wp39-cache` on the internal SSD (319 GB free; FinData hard floor 60 GB) | streaming design keeps < 5 GB |
		7	+| API exports | ≈ 2 × 4,300 symbol-chunks, ≈ 1 s each server-side, 4 concurrent | pilot measured 1.0 s / 860 k rows |
		8	+| Fundamentals calls | 5,451 × 2 | ms each |
		9	+| Feature computation | Polars per symbol (≈ 1.7 M rows → 4 frequency tables), 8 worker processes | to be measured in the pilot |
		10	+| Models | logistic ridge (seconds) ; LightGBM 24 trials × 5 information sets × (1 chronological + 11 walk-forward folds) ≈ 1,440 fits, early stopping; ≈ 2–4 h on 28 cores | — |
		11	+| Bootstrap | 1,000 stationary-bootstrap draws over 23 test quarters per contrast — seconds | — |
		12	+| LaTeX | laptop (TeX Live, `latexmk`) | available |
		13	+| AI usage | none in the pipeline (no LLM features, no neural B7/B8); Claude Code used only as the engineer | owner instruction 2026-11-02 |
		14	+
		15	+Pilot (Phase 2 gate): 300 firms stratified by first-fundamentals year and size, full history → validates throughput,
		16	+feature sanity and label base rates before the full run.

added docs/NOVELTY_MEMO.md +53 −0

				@@ -0,0 +1,53 @@
		1	+# NOVELTY_MEMO.md — the three closest papers and why WP39 is not them
		2	+
		3	+Prepared 2026-11-02 from publisher metadata verified on Crossref (`literature/RELATED_WORK_MATRIX.csv`,
		4	+`artifacts/logs/crossref_batch*.json`). Full texts were not machine-summarised; the points of distinction below are
		5	+based on the papers' stated designs as reported in their abstracts and our own reading of their published versions.
		6	+
		7	+## Closest paper 1 — Grégoire & Martineau (2022), *How Is Earnings News Transmitted to Stock Prices?*, JAR 60(1)
		8	+- **What it does.** Decomposes price discovery around earnings announcements using TAQ quotes *and* trades, showing that
		9	+ most of the reaction to after-hours announcements occurs in quotes with little trading.
		10	+- **Strongest objection to WP39 from this paper.** "Information arrives at the announcement; what you call anticipation is
		11	+ the pre-announcement drift already documented, and bars cannot even see quotes."
		12	+- **How WP39 differs.** WP39 does not study the announcement reaction at all: all features end strictly *before* the
		13	+ release and the outcome is the accounting number itself, not the return. We make no claim about quotes versus trades
		14	+ (our data are OHLCV bars, stated plainly) and we ask a forecasting question — whether the *resolution* of pre-release
		15	+ bar data carries incremental information about fundamentals — which their event-window decomposition does not address.
		16	+
		17	+## Closest paper 2 — Chen, Cho, Dou & Lev (2022), *Predicting Future Earnings Changes Using Machine Learning and Detailed Financial Data*, JAR 60(2)
		18	+- **What it does.** Shows that gradient-boosted trees on detailed XBRL financial-statement items predict the direction of
		19	+ next-year earnings changes better than parsimonious accounting models, and that the predictions are priced slowly.
		20	+- **Strongest objection.** "Fundamentals-based ML already predicts earnings changes; adding market data is Ou–Penman with
		21	+ a new model, and daily returns are known to lead earnings (Beaver–Lambert–Morse 1980; Kothari–Sloan 1992)."
		22	+- **How WP39 differs.** Their inputs are accounting items; ours hold the accounting information set fixed (B1) and
		23	+ measure the *marginal* value of market data at four sampling frequencies with equal model capacity (B2–B5, B6). The
		24	+ stock-prices-lead-earnings literature establishes the daily/annual lead; WP39 tests whether anything remains at the
		25	+ minute scale once daily information is in, with a strict PIT availability clock and a release-date reveal clock.
		26	+
		27	+## Closest paper 3 — Zhang, Zhang, Cucuringu & Qian (2023/2024), *Volatility Forecasting with Machine Learning and Intraday Commonality*, JFEC
		28	+- **What it does.** Pools intraday realized-volatility features across many stocks to forecast volatility with ML, showing
		29	+ that commonality in intraday patterns improves volatility forecasts.
		30	+- **Strongest objection.** "Intraday features forecast volatility, and earnings surprises are volatility events, so your
		31	+ deterioration signal is a volatility forecast in disguise."
		32	+- **How WP39 differs.** Our target is an accounting quantity with a *sign* (adverse surprise), not a variance; the design
		33	+ includes the volatility families explicitly in every frequency set so that the test isolates information beyond
		34	+ predicted volatility (placebo: a symmetric |surprise| target must show the pattern a volatility channel implies, a signed
		35	+ target must not). We also report directional asymmetry (negative semivariance, down-bar volume) as pre-specified
		36	+ mechanism variables, not as the headline.
		37	+
		38	+## Other adjacent work and the recycling check
		39	+- UQO WP15 (*Faster, Wider, More Fragile?*) studies intraday *shock propagation* across stocks on the same vendor data;
		40	+ it never links bars to accounting outcomes. WP39 shares the data vendor and the realized-measure toolkit only; no
		41	+ result or figure is reused.
		42	+- Pre-announcement informed trading (Christophe–Ferri–Angel 2004; Berkman et al. 2009; Kaniel et al. 2012;
		43	+ Hendershott–Livdan–Schürhoff 2015) uses trader-type data we do not have; our contribution is orthogonal: whether
		44	+ *anonymous* OHLCV at fine resolution is informative, and whether the information is attributable to the frequency.
		45	+- Patton & Verardo (2012) and Bollerslev–Li–Xue (2018) model announcement-day betas and volume-volatility; they are
		46	+ announcement-window studies.
		47	+- Earnings-prediction ML (Easton et al. 2024; Cao et al. 2024) uses accounting, analyst and text inputs, not intraday bars.
		48	+
		49	+## Verdict
		50	+No published paper, to our verified knowledge, measures the marginal forecasting value of 1-minute versus 5-, 30-minute
		51	+and daily OHLCV for *forthcoming accounting deterioration* with a point-in-time information set and matched model
		52	+capacity. The question is therefore open. If the empirical answer is a precise null, that is itself informative given
		53	+the practitioner belief that high-frequency activity "sees" fundamentals early.

added docs/PIT_PROTOCOL.md +75 −0

				@@ -0,0 +1,75 @@
		1	+# PIT_PROTOCOL.md — point-in-time rules for WP39
		2	+
		3	+Frozen 2026-11-02 before any model was estimated. Any change is an amendment logged in `DECISIONS.md`.
		4	+
		5	+## 1. Clocks
		6	+
		7	+All instants are converted to **America/New_York** for session logic and kept in UTC in storage. Trading days follow the
		8	+XNYS calendar (`exchange_calendars`). For firm `i` and fiscal quarter `q`:
		9	+
		10	+| Symbol | Meaning | Source |
		11	+|---|---|---|
		12	+| `E_q` | fiscal period end | `fundamentals_facts.fiscal_period_end` |
		13	+| `A_q` | earnings-release date (press release) — the **reveal date** of quarter-`q` results | `earnings_events.event_date`, first event in `[E_q, F_q + 10 d]` |
		14	+| `τ_q` | release timing: `before_open`, `after_close`, or `unknown` | `earnings_events.release_time` |
		15	+| `F_q` | SEC filing date of the 10-Q/10-K | `fundamentals_facts.filing_date` |
		16	+| `V_q` | vendor-proxy availability of the structured facts | `available_at = F_q + 1 d 00:00 ET` |
		17	+| `c` | prediction cutoff (a trading day; features use information up to the **close** of `c`) | design |
		18	+
		19	+## 2. The information set at cutoff `c`
		20	+
		21	+`I(c)` contains exactly:
		22	+
		23	+1. **Market bars** with `session_date ≤ c` (regular session 09:30–16:00 ET for the main specification; the 16:00 bar is the
		24	+ closing print; extended hours 04:00–09:29 and 16:01–19:59 enter only the "extended" variants, never mixed).
		25	+2. **Fundamentals** with `available_at ≤ c 00:00 ET`, i.e. `F_j + 1 ≤ c`. The latest such quarter is `j*(c)`, and the
		26	+ model also receives the *age* `c − A_{j*}` in trading days. Because `V_q > F_q ≥ A_q > c` by construction (§3), quarter-`q`
		27	+ facts can never enter `I(c)`.
		28	+3. **Earnings-release history** with `event_date < c` (prior realized releases, used for the historical base-rate feature B0).
		29	+4. **Static crosswalk** (CIK, SIC) — time-invariant by assumption; `formerNames` are checked for ticker reuse (Phase 2 audit).
		30	+5. **Nothing else.** No consensus estimates for quarter `q` (vintage unknown), no index-membership backfills, no
		31	+ contemporaneous factors.
		32	+
		33	+## 3. Cutoff rules
		34	+
		35	+- **Event-time design** (primary, H1–H4): for horizon `h ∈ {40, 20, 10, 5, 1}` trading days, `c = A_q − h` trading days.
		36	+ Admissible only if `c < min(A_q, F_q)` *and* `c ≥ A_{q−1} + 1` (the previous release has already happened, so the
		37	+ cohort window never straddles two reveals). With `τ_q = before_open`, bars of `A_q` contain the reaction and are excluded
		38	+ (`c ≤ A_q − 1` guarantees it); with `τ_q = after_close` or `unknown` the same rule applies — we never use `A_q` bars.
		39	+- **Fixed-calendar design** (H1 robustness, H5): `c` = last trading day of each month; the target is the *next* release
		40	+ after `c` (first `A_q > c`), restricted to `A_q − c ≤ 65` trading days. The identity of "the next quarter" is known ex
		41	+ ante; its date is not needed by the forecaster.
		42	+- **Firm-quarter exclusions** (frozen): no matched release; `F_q < A_q`; `F_q − A_q > 30` calendar days; `E_q` not within
		43	+ 60–120 days of `E_{q−1}` (irregular fiscal calendars); quarter `q − 4` or `q − 1` not available at `c` (expectation
		44	+ undefined); raw close at `c` < 1 USD; fewer than 15 valid sessions in the 20-day feature window.
		45	+
		46	+## 4. Label construction (no price-based labels)
		47	+
		48	+Primary: `S_q = (X_q − E_c[X_q]) / TA_{j*}`, `X = op_inc` (quarterly operating income), `TA` = total assets of the latest
		49	+available quarter, `E_c[X_q] = X_{q−4} + (1/4) Σ_{k=0..3} (X_{q−1−k} − X_{q−5−k})` using only quarters available at `c`
		50	+(seasonal random walk with drift; Foster 1977). `D_q = 1[S_q ≤ −0.005]`. The secondary targets are in
		51	+`configs/research.yaml`. `X_q` itself is taken from the **first ingested revision** of quarter `q` (the only one available),
		52	+which is the vendor's current value — see LIMITATIONS §PIT.
		53	+
		54	+## 5. Leakage battery (each item is a unit test in `tests/`)
		55	+
		56	+| # | Risk | Guard | Test |
		57	+|---|---|---|---|
		58	+| L1 | quarter-`q` facts used as inputs | as-of join on `available_at ≤ c` | `test_pit_leakage.py::test_no_future_fundamentals` |
		59	+| L2 | selection by fiscal end instead of availability | join key is `available_at`, never `fiscal_period_end` | `test_pit_leakage.py::test_join_key_is_available_at` |
		60	+| L3 | post-release bars in the feature window | `session_date ≤ c < A_q` | `test_pit_leakage.py::test_no_bars_after_cutoff` |
		61	+| L4 | global standardisation / imputation / tuning on test data | transformers fit on train rows only; tuning on validation period only | `test_train_only_transforms.py` |
		62	+| L5 | DST / timezone errors | UTC → ET conversion, 09:30 and 16:00 bars identified on both DST regimes | `test_timestamp_dsts.py` |
		63	+| L6 | ticker reuse / identifier drift | crosswalk with effective dates; SEC `formerNames` audit | `test_instrument_ids.py` |
		64	+| L7 | splits/dividends creating mechanical returns | split-adjusted returns from `corporate_actions` (ex-date rule), raw bars immutable | `test_corporate_actions.py` |
		65	+| L8 | release timing assumed intraday | release-day bars never used; timing enters only as a descriptive variable | `test_labels.py::test_release_day_excluded` |
		66	+| L9 | overlapping target windows across splits | purge + 1-quarter embargo at every split boundary | `test_splits.py` |
		67	+| L10 | survivorship by construction | universe = all fundamentals entities (documented survivor conditioning), delisted bars kept where present | `01_audit_universe.py` report |
		68	+| L11 | vendor ingestion date mistaken for knowledge date | `observed_at` is never used as a knowledge date; `available_at` only | `test_pit_leakage.py::test_observed_at_not_used` |
		69	+| L12 | randomised placebo | labels shuffled within calendar quarter → gain must vanish; cutoffs shifted −45 td → H4 gain must vanish | `08_run_robustness.py` |
		70	+
		71	+## 6. Equivalence check of the local as-of join with the API
		72	+
		73	+For 25 random entities and 10 random `as_of` instants, the set of (metric, period) pairs returned by
		74	+`GET /fundamentals/{entity}?as_of=` must equal the set produced by the local filter `available_at ≤ as_of` on the full
		75	+extract. Implemented in `tests/test_api_contract.py::test_asof_equivalence` (live test, `WP39_LIVE=1`).

added docs/REPLICATION.md +38 −0

				@@ -0,0 +1,38 @@
		1	+# REPLICATION.md — how to reproduce WP39
		2	+
		3	+## Requirements
		4	+- Python ≥ 3.12 with `uv` (`uv venv -p 3.12 .venv && uv pip install -e ".[dev]"`).
		5	+- Access to FinData OS (owner-issued key with scopes `read:market, read:fundamentals, query, export`) exposed as
		6	+ `FINDATA_API_KEY` or a mode-600 file at `FINDATA_API_KEY_FILE`. Without access, Phases 2–5 cannot run; the saved
		7	+ evaluation artifacts in `artifacts/evaluation/` and `results/` allow Phase 6 (tables/figures/paper) to be regenerated.
		8	+- Read-only `psql` access to the FinData Postgres database on the storage node for `scripts/01a_extract_node_tables.sh`
		9	+ (earnings events, entities, corporate actions — not yet exposed by the API).
		10	+- Internet access to `sec.gov` (CIK/SIC) and the Kenneth French library (daily factors).
		11	+- TeX Live with `latexmk` for the paper.
		12	+
		13	+## Order of execution (compute node, API on localhost)
		14	+```bash
		15	+export FINDATA_BASE_URL=http://127.0.0.1:8600/api/v1 WP39_CACHE=~/wp39-cache
		16	+bash scripts/01a_extract_node_tables.sh # node extracts + manifests
		17	+python scripts/00_probe_findata.py # contract reconciliation -> artifacts/logs/phase0_probe.json
		18	+python scripts/01_audit_universe.py # universe.parquet, results/universe_audit.json
		19	+python scripts/02_build_pit_panel.py # quarters.parquet (labels), design freeze hash
		20	+python scripts/03_build_features.py --workers 8 # streaming exports -> session features -> rolling windows
		21	+python scripts/04_build_labels.py # sample_event.parquet, sample_calendar.parquet, cascades
		22	+python scripts/05_train_baselines.py # B0-B5 logistic, ridge, firm hold-out, walk-forward
		23	+python scripts/06_train_advanced.py # LightGBM equal-budget, horizon ladder, OOF signals
		24	+python scripts/07_evaluate.py # metrics, paired contrasts, GW tests, Holm, power
		25	+python scripts/08_run_robustness.py # placebos, subgroups, panel regressions, H5
		26	+python scripts/09_make_manuscript.py # tables, figures, results_macros.tex
		27	+cd paper && make # main.pdf
		28	+pytest -q # 29 tests; WP39_LIVE=1 adds the live API contract tests
		29	+```
		30	+Runtime on a Mac Studio M3 Ultra (28 cores): see `PROGRESS.md` for measured wall-times of each step.
		31	+
		32	+## What is and is not in the package
		33	+- In: code, configs, docs, tests, manifests (hashes/versions/counts), derived evaluation tables, figures, paper.
		34	+- Out: raw bars and fundamentals (FirstRate Data license), the per-symbol feature cache (derived from licensed data;
		35	+ regenerable), model binaries.
		36	+- Determinism: all seeds come from `configs/research.yaml` (`seed: 20261102`); LightGBM uses `n_jobs=12` with
		37	+ deterministic histogram construction at the default settings, so results are reproducible to numerical precision on
		38	+ the same architecture.

added docs/RESEARCH_DESIGN.md +111 −0

				@@ -0,0 +1,111 @@
		1	+# RESEARCH_DESIGN.md — preregistered specification (internal)
		2	+
		3	+Frozen **2026-11-02 14:40 ET**, before the first test-period observation was opened. Hash of this file at freeze is
		4	+recorded in `artifacts/manifests/design_freeze.json` by `scripts/02_build_pit_panel.py`. Later changes are amendments in
		5	+`DECISIONS.md` (dated, with reason) and never alter frozen primary endpoints.
		6	+
		7	+## 1. Question
		8	+
		9	+Does one-minute OHLCV activity contain **incremental** information about a firm's *forthcoming* adverse accounting
		10	+outcome, beyond (i) point-in-time fundamentals, (ii) daily OHLCV, (iii) coarser intraday aggregates of the same bars, and
		11	+(iv) calendar/industry/size controls? Anticipation (prices move before the release) is not the claim; the claim is about
		12	+the marginal forecasting value of **sampling frequency** holding the information set, calendar coverage, universe,
		13	+cutoff, horizon and training budget fixed.
		14	+
		15	+## 2. Hypotheses — null, alternative, primary endpoint, decision rule
		16	+
		17	+| | Null `H0` | Alternative `H1` | Primary endpoint | Decision rule |
		18	+|---|---|---|---|---|
		19	+| **H1** incremental information | OOS log loss of B5 (fundamentals + daily + 1m) ≥ that of B2 (fundamentals + daily) | B5 < B2 | Δ log loss (B5 − B2) on the untouched test period, 2021-01 → 2026-10, horizon h = 20 | 95 % block-bootstrap CI (blocks = calendar quarters, 1,000 draws) excludes 0 **and** Giacomini–White conditional test p < 0.05 (two-sided) |
		20	+| **H2** temporal resolution | Δ(B5 − B2) ≤ Δ(B4 − B2) and ≤ Δ(B3 − B2) | 1m gain strictly exceeds 5m and 30m gains | paired Δ log loss B5 − B4 and B5 − B3 | both CIs exclude 0 after Holm correction over the 3 frequency contrasts |
		21	+| **H3** heterogeneity | the B5 − B2 gain is equal across terciles of size, illiquidity, leverage | gain larger in small, illiquid, levered firms | interaction coefficient `β_HF×tercile` in the panel regression (§5) | Holm-corrected p < 0.05 for the pre-specified ordering |
		22	+| **H4** timing | the B5 − B2 gain does not increase as h shrinks (40→20→10→5→1) | monotone increase | slope of Δ log loss on −h across horizons | sign test on the five ordered horizons + placebo cutoffs (−45 td) showing no gain |
		23	+| **H5** pricing | the LFDS does not predict the announcement-window abnormal return nor post-cutoff factor-adjusted returns | it does | (a) regression of CAR[c+1, A_q+1] on LFDS with controls; (b) decile long-short FF5+MOM alpha with costs | CI excludes 0; economic value reported with turnover and costs regardless of significance |
		24	+
		25	+Primary endpoint for the headline sentence: **H1 at h = 20 on the binary primary target** with log loss. Brier, AUC,
		26	+PR-AUC, calibration slope/intercept, MAE/RMSE/OOS-R² (continuous target) are secondary metrics reported for every model.
		27	+The headline sentence will read: *"At a 20-trading-day horizon, on a 2012–2026 panel of U.S. listed firms with a strictly
		28	+point-in-time information set, one-minute OHLCV features change the out-of-sample log loss of forecasts of an adverse
		29	+operating-income surprise by [Δ, 95 % CI] relative to a daily-market-and-accounting benchmark; the gain is
		30	+[consistent/inconsistent] with [mechanisms] and has [measured/absent] economic value."*
		31	+
		32	+## 3. Sample
		33	+
		34	+Universe: all FinData fundamentals entities (5,451; survivor-conditioned, see LIMITATIONS) with ≥ 8 quarterly periods
		35	+and 1-minute bars; firm-quarters 2012-Q1 → 2026-Q2 with reveal dates ≤ 2026-10-30 and admissible cutoffs (PIT_PROTOCOL
		36	+§3). Sector = SEC SIC division (2-digit), size = log market cap at `c` (raw close × latest available `shares`).
		37	+
		38	+Expected sample (to be reported, not assumed): ~4,300 firms × ~40 quarters ≈ 150–170 k firm-quarters before exclusions.
		39	+
		40	+## 4. Variables
		41	+
		42	+**Targets** — `configs/research.yaml`. Primary binary `D_q = 1[S_q ≤ −0.5 % of assets]`, primary continuous `S_q`
		43	+(winsorised at 1/99 % on the training period).
		44	+
		45	+**PIT fundamentals (B1)** — latest available quarter at `c`: `rev` growth (yoy), gross margin `gp/rev`, operating margin
		46	+`op_inc/rev`, `roa`, `net_inc/assets_tot`, `cash_op/assets_tot`, accruals `(net_inc − cash_op)/assets_tot`,
		47	+`cash/assets_tot`, leverage `debt/assets_tot`, interest coverage `intr_cover` (clipped), current ratio `assets_c/liab_c`,
		48	+net issuance (Δ`shares` yoy), capex/assets, R&D/rev, `S_{j*}` (last surprise), last-four-quarter surprise mean, loss
		49	+indicator, log assets, log market cap, information age `c − A_{j*}`, fiscal-quarter dummies, sector dummies, year-quarter
		50	+calendar dummies (in-sample only; for prediction, calendar effects are replaced by the trailing cross-sectional base rate).
		51	+
		52	+**Daily OHLCV (B2)** — from 1m regular-session bars aggregated to daily: 20- and 60-day cumulative split-adjusted returns,
		53	+daily return volatility (20d), Parkinson range volatility, skewness and kurtosis of daily returns (60d), daily volume
		54	+relative to 60-day median (20d mean and last 5d), Amihud illiquidity (60d), share of zero-return days, overnight
		55	+(close-to-open) vs intraday (open-to-close) return decomposition (20d sums), max drawdown (60d), beta and residual
		56	+volatility against SPY (60d daily).
		57	+
		58	+**Intraday families (B3 = 30m, B4 = 5m, B5 = 1m)** — the *same* families computed on bars of the given length,
		59	+each aggregated per session then averaged (or summed) over the 20-day window:
		60	+realized variance (RV), bipower variation (BPV), relative jump `max(0, RV − BPV)/RV`, negative semivariance share
		61	+`RS⁻/RV`, signed-jump variation `(RS⁺ − RS⁻)/RV`, realized skewness and kurtosis, maximum absolute bar-return share
		62	+of RV (concentration), first-order autocorrelation of bar returns, variance ratio `RV_freq / RV_daily`, intraday drift
		63	+`(close − open)/open`, time-of-day volume profile (share of volume in the first and last 30 minutes), volume dispersion
		64	+across bars (Gini), down-bar volume share (volume on bars with negative return / total — *a bar statistic, not order
		65	+flow*), volume surprise relative to the firm's 60-day time-of-day seasonal profile, fraction of bars present (activity),
		66	+RV on market-down bars vs market-up bars (asymmetry vs SPY), realized beta to SPY at that frequency, vol-of-vol
		67	+(dispersion of daily RV within the window). At 30m the within-day counts are small (13 bars), so skewness/kurtosis and
		68	+autocorrelation are computed on the pooled window.
		69	+
		70	+Note on frequency comparison: B3/B4/B5 have the **same number of feature columns** (one per family); dimensionality does
		71	+not grow with frequency. A supplementary experiment (Phase 4) concatenates all three to test whether 1m dominates when
		72	+coarser aggregates are also present.
		73	+
		74	+## 5. Models and inference
		75	+
		76	+Baseline ladder B0–B6 (`configs/models.yaml`). Logistic ridge for B1–B5 with `C` tuned on the validation period only;
		77	+LightGBM with an identical random-search budget (24 trials, early stopping on validation) for every information set.
		78	+B7/B8 (temporal neural, text) are **not run** (owner instruction to minimise AI use; declared in DECISIONS D-006).
		79	+
		80	+Panel regression for interpretability and H3:
		81	+`S_{i,q} = α_i + δ_{s(i),t(q)} + β·LFDS_{i,c} + γ'X_{i,c} + ε_{i,q}`, where `LFDS` is the B5 out-of-fold predicted
		82	+probability (walk-forward, so it is never fitted on the same quarter), with firm fixed effects and sector×year-quarter
		83	+effects, two-way clustered SEs (firm, calendar quarter). H3 interactions use ex-ante terciles formed at `c`.
		84	+
		85	+Comparisons: Diebold–Mariano / Giacomini–White on per-quarter mean loss differentials with HAC; stationary block
		86	+bootstrap over calendar quarters for paired CIs; Holm within each hypothesis family; model confidence set over the ladder
		87	+as a supplementary exhibit.
		88	+
		89	+## 6. Validation protocol
		90	+
		91	+- Chronological: train 2012-01 → 2018-12, validation 2019-01 → 2020-12, test 2021-01 → 2026-10 (never opened until Phase 3
		92	+ gate). Purge: training observations whose `A_q` falls after the validation start minus one quarter are dropped
		93	+ (embargo = 1 calendar quarter) and likewise at the test boundary.
		94	+- Walk-forward: expanding window, annual refits, first test year 2016, 1-quarter embargo; this provides the out-of-fold
		95	+ LFDS for the panel regressions and the H4/H5 exercises.
		96	+- Firm hold-out: 20 % of firms (hash of entity id) never seen in training, evaluated separately from the time hold-out.
		97	+- Train-only transforms: winsorisation bounds, medians for imputation, standardisation, sector-time demeaning, tercile
		98	+ cut-points — all computed on the training rows of the current fold.
		99	+
		100	+## 7. Power
		101	+
		102	+With ≈ 40 k test firm-quarters, base rate ≈ 20–30 %, and quarter-level clustering (23 test quarters), the minimum
		103	+detectable paired Δ log loss is roughly 0.002–0.004 (to be computed from the validation-period variance of the loss
		104	+differential and reported in `artifacts/evaluation/power.json` before the test set is opened).
		105	+
		106	+## 8. What would make us abandon or pivot
		107	+
		108	+If, on the validation period, B2 already captures all of B5 (Δ within ±0.001 and CI covering 0), H1 is likely null; we
		109	+proceed to the test anyway (a precise null is a result) and devote Phase 5 to characterising *where* intraday information
		110	+does and does not help. If the earnings-release matching proves unreliable (> 10 % irregular lags after cleaning), the
		111	+event-time design is demoted and the fixed-calendar design becomes primary (amendment required).

added literature/RELATED_WORK_MATRIX.csv +25 −0

				@@ -0,0 +1,25 @@
		1	+key,authors,year,journal,doi_or_url,data_source,frequency,sample,research_question,identification,findings,point_of_distinction_for_WP39
		2	+gregoire2022,Grégoire & Martineau,2022,Journal of Accounting Research,10.1111/1475-679X.12394,TAQ quotes and trades,intraday (quotes/trades),US earnings announcements,How is earnings news incorporated into prices?,event-window decomposition of price discovery,most price discovery for after-hours announcements occurs through quote updates with little trading,"WP39 uses only pre-release bars, forecasts the accounting outcome (not the return), never claims quote/trade mechanisms"
		3	+chen2022,Chen; Cho; Dou; Lev,2022,Journal of Accounting Research,10.1111/1475-679X.12429,XBRL detailed financial statement items,annual,US firms,Can ML on detailed financials predict earnings changes?,OOS prediction + return tests,boosted trees beat parsimonious models; slow pricing,"WP39 holds accounting inputs fixed and measures the marginal value of market data by sampling frequency"
		4	+zhang2023volatility,Zhang; Zhang; Cucuringu; Qian,2024,Journal of Financial Econometrics,10.1093/jjfinec/nbad005,intraday equity returns,intraday,US stocks,Does pooling intraday commonality improve ML volatility forecasts?,OOS forecasting,commonality improves volatility forecasts,"WP39 targets a signed accounting quantity, includes volatility families as controls, uses a signed vs absolute placebo"
		5	+beaver1980,Beaver; Lambert; Morse,1980,Journal of Accounting and Economics,10.1016/0165-4101(80)90013-0,CRSP/Compustat,annual,US,Do prices contain information about future earnings?,regression of earnings changes on returns,prices lead earnings,"establishes the daily/annual lead; WP39 asks what remains at minute resolution"
		6	+kothari1992,Kothari & Sloan,1992,Journal of Accounting and Economics,10.1016/0165-4101(92)90016-U,CRSP/Compustat,annual,US,Information in prices about future earnings (ERC),leading returns,ERCs rise with leading returns,"same"
		7	+collins1994,Collins; Kothari; Shanken; Sloan,1994,Journal of Accounting and Economics,10.1016/0165-4101(94)90024-8,CRSP/Compustat,annual,US,Why is the contemporaneous return-earnings association low?,lack of timeliness vs noise,prices anticipate earnings several years ahead,"same"
		8	+foster1977,Foster,1977,The Accounting Review,10.2308/tar-4493420,quarterly earnings,quarterly,US,Time-series properties of quarterly earnings,forecast evaluation,seasonal random walk with drift forecasts well,"WP39's expectation model for the surprise"
		9	+bernard1989,Bernard & Thomas,1989,Journal of Accounting Research,10.2307/2491062,CRSP/Compustat,daily,US,Post-earnings-announcement drift,event study,drift after SUE,"WP39 studies pre-release information, not post-announcement drift"
		10	+ou1989,Ou & Penman,1989,Journal of Accounting and Economics,10.1016/0165-4101(89)90017-7,Compustat,annual,US,Can financial statement analysis predict earnings and returns?,logit on ratios,yes,"WP39 B1 baseline follows this tradition"
		11	+sloan1996,Sloan,1996,The Accounting Review,10.2308/tar-9608042309,Compustat/CRSP,annual,US,Do prices reflect accruals information?,hedge portfolio,accrual anomaly,"accruals are a B1 control"
		12	+christophe2004,Christophe; Ferri; Angel,2004,Journal of Finance,10.1111/j.1540-6261.2004.00681.x,Nasdaq short sales,daily,US,Do short sellers anticipate earnings news?,pre-announcement short selling,yes,"trader-type data unavailable to WP39; WP39 tests anonymous OHLCV"
		13	+berkman2009,Berkman; Dimitrov; Jain; Koch; Tice,2009,Journal of Financial Economics,10.1016/j.jfineco.2008.04.009,CRSP/IBES,daily,US,Differences of opinion and returns around announcements,event study,prices fall around announcements for high-disagreement stocks,"announcement-window return study"
		14	+kaniel2012,Kaniel; Liu; Saar; Titman,2012,Journal of Finance,10.1111/j.1540-6261.2012.01727.x,NYSE CAUD individual trades,daily,US,Do individual investors trade informatively before earnings?,pre-event trading,yes partly,"trader-type data"
		15	+hendershott2015,Hendershott; Livdan; Schürhoff,2015,Journal of Financial Economics,10.1016/j.jfineco.2015.03.007,institutional order flow,daily,US,Are institutions informed about news?,order flow before news,yes,"order flow unavailable; WP39 never labels bar statistics as order flow"
		16	+patton2012,Patton & Verardo,2012,Review of Financial Studies,10.1093/rfs/hhs073,TAQ,intraday,US,Does beta move with news?,announcement-day realized beta,beta rises on announcement days,"announcement-day study"
		17	+bollerslev2018volume,Bollerslev; Li; Xue,2018,Review of Economic Studies,10.1093/restud/rdy003,TAQ,intraday,US,Volume-volatility relation around public news,structural model,announcement effects on volume-volatility elasticity,"announcement-window study"
		18	+weller2018,Weller,2018,Review of Financial Studies,10.1093/rfs/hhx137,TAQ + earnings,daily/intraday,US,Does algorithmic trading reduce information acquisition?,AT and pre-announcement price informativeness,AT reduces pre-announcement price informativeness,"closest mechanism paper: WP39 reports pre-release information by era as a mechanism exhibit"
		19	+lee2012,Lee,2012,Review of Financial Studies,10.1093/rfs/hhr084,TAQ,intraday,US,Jumps and information flow,jump tests around news,jumps cluster at news,"jump proxies are WP39 features"
		20	+easton2024,Easton; Kapons; Monahan; Schütt; Weisbrod,2024,The Accounting Review,10.2308/tar-2021-0478,Compustat,annual,US,Forecast earnings with k-NN,OOS forecasting,kNN competitive with analysts,"accounting inputs only"
		21	+cao2024,Cao; Jiang; Wang; Yang,2024,Journal of Financial Economics,10.1016/j.jfineco.2024.103910,analyst + ML,monthly,US,Man vs machine in stock analysis,OOS forecasts,AI complements analysts,"no intraday data"
		22	+gu2020,Gu; Kelly; Xiu,2020,Review of Financial Studies,10.1093/rfs/hhaa009,CRSP/Compustat characteristics,monthly,US,ML for return prediction,OOS R2,trees and NNs best,"methodological reference for equal-budget comparisons"
		23	+giacomini2006,Giacomini & White,2006,Econometrica,10.1111/j.1468-0262.2006.00718.x,—,—,—,Conditional predictive ability tests,—,—,"inference for H1/H2"
		24	+liu2015,Liu; Patton; Sheppard,2015,Journal of Econometrics,10.1016/j.jeconom.2015.02.008,TAQ,intraday,multi-asset,Does anything beat 5-minute RV?,forecast comparison,5-min RV hard to beat,"frequency comparison for variance measurement; WP39 asks the analogous question for fundamentals"
		25	+boucher2026wp15,Boucher,2026,UQO Working Paper 15,—,FirstRate Data 1-minute bars,1-minute,US 1992-2026,Intraday shock propagation,event studies,shocks propagate narrowly,"same vendor; different question; nothing reused"

added literature/references.bib +1021 −0

				@@ -0,0 +1,1021 @@
		1	+% WP39 UQO bibliography. Journal metadata verified on Crossref 2026-11-02 (artifacts/logs/crossref_batch*.json).
		2	+
		3	+@article{gregoire2022,
		4	+ author = {Grégoire, Vincent and Martineau, Charles},
		5	+ title = {{How is Earnings News Transmitted to Stock Prices?}},
		6	+ journal = {Journal of Accounting Research},
		7	+ year = {2022},
		8	+ volume = {60},
		9	+ number = {1},
		10	+ pages = {261--297},
		11	+ doi = {10.1111/1475-679x.12394}
		12	+}
		13	+
		14	+@article{zhang2023volatility,
		15	+ author = {Zhang, Chao and Zhang, Yihuang and Cucuringu, Mihai and Qian, Zhongmin},
		16	+ title = {{Volatility Forecasting with Machine Learning and Intraday Commonality}},
		17	+ journal = {Journal of Financial Econometrics},
		18	+ year = {2024},
		19	+ volume = {22},
		20	+ number = {2},
		21	+ pages = {492--530},
		22	+ doi = {10.1093/jjfinec/nbad005}
		23	+}
		24	+
		25	+@article{ball1968,
		26	+ author = {Ball, Ray and Brown, Philip},
		27	+ title = {{An Empirical Evaluation of Accounting Income Numbers}},
		28	+ journal = {Journal of Accounting Research},
		29	+ year = {1968},
		30	+ volume = {6},
		31	+ number = {2},
		32	+ pages = {159},
		33	+ doi = {10.2307/2490232}
		34	+}
		35	+
		36	+@article{bernard1989,
		37	+ author = {Bernard, Victor L. and Thomas, Jacob K.},
		38	+ title = {{Post-Earnings-Announcement Drift: Delayed Price Response or Risk Premium?}},
		39	+ journal = {Journal of Accounting Research},
		40	+ year = {1989},
		41	+ volume = {27},
		42	+ pages = {1},
		43	+ doi = {10.2307/2491062}
		44	+}
		45	+
		46	+@article{bernard1990,
		47	+ author = {Bernard, Victor L. and Thomas, Jacob K.},
		48	+ title = {{Evidence that stock prices do not fully reflect the implications of current earnings for future earnings}},
		49	+ journal = {Journal of Accounting and Economics},
		50	+ year = {1990},
		51	+ volume = {13},
		52	+ number = {4},
		53	+ pages = {305--340},
		54	+ doi = {10.1016/0165-4101(90)90008-r}
		55	+}
		56	+
		57	+@article{sloan1996,
		58	+ author = {Sloan, Richard G.},
		59	+ title = {{Do Stock Prices Fully Reflect Information in Accruals and Cash Flows About Future Earnings?}},
		60	+ journal = {The Accounting Review},
		61	+ year = {1996},
		62	+ volume = {71},
		63	+ number = {3},
		64	+ pages = {289--315},
		65	+ doi = {10.2308/tar-9608042309}
		66	+}
		67	+
		68	+@article{ou1989,
		69	+ author = {Ou, Jane A. and Penman, Stephen H.},
		70	+ title = {{Financial statement analysis and the prediction of stock returns}},
		71	+ journal = {Journal of Accounting and Economics},
		72	+ year = {1989},
		73	+ volume = {11},
		74	+ number = {4},
		75	+ pages = {295--329},
		76	+ doi = {10.1016/0165-4101(89)90017-7}
		77	+}
		78	+
		79	+@article{kyle1985,
		80	+ author = {Kyle, Albert S.},
		81	+ title = {{Continuous Auctions and Insider Trading}},
		82	+ journal = {Econometrica},
		83	+ year = {1985},
		84	+ volume = {53},
		85	+ number = {6},
		86	+ pages = {1315},
		87	+ doi = {10.2307/1913210}
		88	+}
		89	+
		90	+@article{easley1987,
		91	+ author = {Easley, David and O'Hara, Maureen},
		92	+ title = {{Price, trade size, and information in securities markets}},
		93	+ journal = {Journal of Financial Economics},
		94	+ year = {1987},
		95	+ volume = {19},
		96	+ number = {1},
		97	+ pages = {69--90},
		98	+ doi = {10.1016/0304-405x(87)90029-8}
		99	+}
		100	+
		101	+@article{andersen2003,
		102	+ author = {Andersen, Torben G. and Bollerslev, Tim and Diebold, Francis X. and Labys, Paul},
		103	+ title = {{Modeling and Forecasting Realized Volatility}},
		104	+ journal = {Econometrica},
		105	+ year = {2003},
		106	+ volume = {71},
		107	+ number = {2},
		108	+ pages = {579--625},
		109	+ doi = {10.1111/1468-0262.00418}
		110	+}
		111	+
		112	+@article{barndorff2004,
		113	+ author = {Barndorff-Nielsen, O. E.},
		114	+ title = {{Power and Bipower Variation with Stochastic Volatility and Jumps}},
		115	+ journal = {Journal of Financial Econometrics},
		116	+ year = {2004},
		117	+ volume = {2},
		118	+ number = {1},
		119	+ pages = {1--37},
		120	+ doi = {10.1093/jjfinec/nbh001}
		121	+}
		122	+
		123	+@article{gu2020,
		124	+ author = {Gu, Shihao and Kelly, Bryan and Xiu, Dacheng},
		125	+ title = {{Empirical Asset Pricing via Machine Learning}},
		126	+ journal = {The Review of Financial Studies},
		127	+ year = {2020},
		128	+ volume = {33},
		129	+ number = {5},
		130	+ pages = {2223--2273},
		131	+ doi = {10.1093/rfs/hhaa009}
		132	+}
		133	+
		134	+@article{giacomini2006,
		135	+ author = {Giacomini, Raffaella and White, Halbert},
		136	+ title = {{Tests of Conditional Predictive Ability}},
		137	+ journal = {Econometrica},
		138	+ year = {2006},
		139	+ volume = {74},
		140	+ number = {6},
		141	+ pages = {1545--1578},
		142	+ doi = {10.1111/j.1468-0262.2006.00718.x}
		143	+}
		144	+
		145	+@article{diebold1995,
		146	+ author = {Diebold, Francis X. and Mariano, Roberto S.},
		147	+ title = {{Comparing Predictive Accuracy}},
		148	+ journal = {Journal of Business \& Economic Statistics},
		149	+ year = {1995},
		150	+ volume = {13},
		151	+ number = {3},
		152	+ pages = {253--263},
		153	+ doi = {10.1080/07350015.1995.10524599}
		154	+}
		155	+
		156	+@article{hansen2005,
		157	+ author = {Hansen, Peter Reinhard},
		158	+ title = {{A Test for Superior Predictive Ability}},
		159	+ journal = {Journal of Business \& Economic Statistics},
		160	+ year = {2005},
		161	+ volume = {23},
		162	+ number = {4},
		163	+ pages = {365--380},
		164	+ doi = {10.1198/073500105000000063}
		165	+}
		166	+
		167	+@article{fama2015,
		168	+ author = {Fama, Eugene F. and French, Kenneth R.},
		169	+ title = {{A five-factor asset pricing model}},
		170	+ journal = {Journal of Financial Economics},
		171	+ year = {2015},
		172	+ volume = {116},
		173	+ number = {1},
		174	+ pages = {1--22},
		175	+ doi = {10.1016/j.jfineco.2014.10.010}
		176	+}
		177	+
		178	+@article{bollerslev2020good,
		179	+ author = {Bollerslev, Tim and Li, Sophia Zhengzi and Zhao, Bingzhi},
		180	+ title = {{Good Volatility, Bad Volatility, and the Cross Section of Stock Returns}},
		181	+ journal = {Journal of Financial and Quantitative Analysis},
		182	+ year = {2020},
		183	+ volume = {55},
		184	+ number = {3},
		185	+ pages = {751--781},
		186	+ doi = {10.1017/s0022109019000097}
		187	+}
		188	+
		189	+@article{patton2015,
		190	+ author = {Patton, Andrew J. and Sheppard, Kevin},
		191	+ title = {{Good Volatility, Bad Volatility: Signed Jumps and The Persistence of Volatility}},
		192	+ journal = {Review of Economics and Statistics},
		193	+ year = {2015},
		194	+ volume = {97},
		195	+ number = {3},
		196	+ pages = {683--697},
		197	+ doi = {10.1162/rest_a_00503}
		198	+}
		199	+
		200	+@article{chen2022,
		201	+ author = {CHEN, XI and CHO, YANG HA (TONY) and DOU, YIWEI and LEV, BARUCH},
		202	+ title = {{Predicting Future Earnings Changes Using Machine Learning and Detailed Financial Data}},
		203	+ journal = {Journal of Accounting Research},
		204	+ year = {2022},
		205	+ volume = {60},
		206	+ number = {2},
		207	+ pages = {467--515},
		208	+ doi = {10.1111/1475-679x.12429}
		209	+}
		210	+
		211	+@article{piotroski2000,
		212	+ author = {Piotroski, Joseph D.},
		213	+ title = {{Value Investing: The Use of Historical Financial Statement Information to Separate Winners from Losers}},
		214	+ journal = {Journal of Accounting Research},
		215	+ year = {2000},
		216	+ volume = {38},
		217	+ pages = {1},
		218	+ doi = {10.2307/2672906}
		219	+}
		220	+
		221	+@article{hirshleifer2009,
		222	+ author = {HIRSHLEIFER, DAVID and LIM, SONYA SEONGYEON and TEOH, SIEW HONG},
		223	+ title = {{Driven to Distraction: Extraneous Events and Underreaction to Earnings News}},
		224	+ journal = {The Journal of Finance},
		225	+ year = {2009},
		226	+ volume = {64},
		227	+ number = {5},
		228	+ pages = {2289--2325},
		229	+ doi = {10.1111/j.1540-6261.2009.01501.x}
		230	+}
		231	+
		232	+@article{dellavigna2009,
		233	+ author = {DELLAVIGNA, STEFANO and POLLET, JOSHUA M.},
		234	+ title = {{Investor Inattention and Friday Earnings Announcements}},
		235	+ journal = {The Journal of Finance},
		236	+ year = {2009},
		237	+ volume = {64},
		238	+ number = {2},
		239	+ pages = {709--749},
		240	+ doi = {10.1111/j.1540-6261.2009.01447.x}
		241	+}
		242	+
		243	+@article{loughran2011,
		244	+ author = {LOUGHRAN, TIM and MCDONALD, BILL},
		245	+ title = {{When Is a Liability Not a Liability? Textual Analysis, Dictionaries, and 10-Ks}},
		246	+ journal = {The Journal of Finance},
		247	+ year = {2011},
		248	+ volume = {66},
		249	+ number = {1},
		250	+ pages = {35--65},
		251	+ doi = {10.1111/j.1540-6261.2010.01625.x}
		252	+}
		253	+
		254	+@article{bajgrowicz2016,
		255	+ author = {Bajgrowicz, Pierre and Scaillet, Olivier and Treccani, Adrien},
		256	+ title = {{Jumps in High-Frequency Data: Spurious Detections, Dynamics, and News}},
		257	+ journal = {Management Science},
		258	+ year = {2016},
		259	+ volume = {62},
		260	+ number = {8},
		261	+ pages = {2198--2217},
		262	+ doi = {10.1287/mnsc.2015.2234}
		263	+}
		264	+
		265	+@article{lee2008,
		266	+ author = {Lee, Suzanne S. and Mykland, Per A.},
		267	+ title = {{Jumps in Financial Markets: A New Nonparametric Test and Jump Dynamics}},
		268	+ journal = {Review of Financial Studies},
		269	+ year = {2008},
		270	+ volume = {21},
		271	+ number = {6},
		272	+ pages = {2535--2563},
		273	+ doi = {10.1093/rfs/hhm056}
		274	+}
		275	+
		276	+@article{bollerslev2018risk,
		277	+ author = {Bollerslev, Tim and Hood, Benjamin and Huss, John and Pedersen, Lasse Heje},
		278	+ title = {{Risk Everywhere: Modeling and Managing Volatility}},
		279	+ journal = {The Review of Financial Studies},
		280	+ year = {2018},
		281	+ volume = {31},
		282	+ number = {7},
		283	+ pages = {2729--2773},
		284	+ doi = {10.1093/rfs/hhy041}
		285	+}
		286	+
		287	+@article{hendershott2011,
		288	+ author = {HENDERSHOTT, TERRENCE and JONES, CHARLES M. and MENKVELD, ALBERT J.},
		289	+ title = {{Does Algorithmic Trading Improve Liquidity?}},
		290	+ journal = {The Journal of Finance},
		291	+ year = {2011},
		292	+ volume = {66},
		293	+ number = {1},
		294	+ pages = {1--33},
		295	+ doi = {10.1111/j.1540-6261.2010.01624.x}
		296	+}
		297	+
		298	+@article{richardson2005,
		299	+ author = {Richardson, Scott A. and Sloan, Richard G. and Soliman, Mark T. and Tuna, İrem},
		300	+ title = {{Accrual reliability, earnings persistence and stock prices}},
		301	+ journal = {Journal of Accounting and Economics},
		302	+ year = {2005},
		303	+ volume = {39},
		304	+ number = {3},
		305	+ pages = {437--485},
		306	+ doi = {10.1016/j.jacceco.2005.04.005}
		307	+}
		308	+
		309	+@article{beneish1999,
		310	+ author = {Beneish, Messod D.},
		311	+ title = {{The Detection of Earnings Manipulation}},
		312	+ journal = {Financial Analysts Journal},
		313	+ year = {1999},
		314	+ volume = {55},
		315	+ number = {5},
		316	+ pages = {24--36},
		317	+ doi = {10.2469/faj.v55.n5.2296}
		318	+}
		319	+
		320	+@article{campbell2008,
		321	+ author = {CAMPBELL, JOHN Y. and HILSCHER, JENS and SZILAGYI, JAN},
		322	+ title = {{In Search of Distress Risk}},
		323	+ journal = {The Journal of Finance},
		324	+ year = {2008},
		325	+ volume = {63},
		326	+ number = {6},
		327	+ pages = {2899--2939},
		328	+ doi = {10.1111/j.1540-6261.2008.01416.x}
		329	+}
		330	+
		331	+@article{fama1973,
		332	+ author = {Fama, Eugene F. and MacBeth, James D.},
		333	+ title = {{Risk, Return, and Equilibrium: Empirical Tests}},
		334	+ journal = {Journal of Political Economy},
		335	+ year = {1973},
		336	+ volume = {81},
		337	+ number = {3},
		338	+ pages = {607--636},
		339	+ doi = {10.1086/260061}
		340	+}
		341	+
		342	+@article{petersen2009,
		343	+ author = {Petersen, Mitchell A.},
		344	+ title = {{Estimating Standard Errors in Finance Panel Data Sets: Comparing Approaches}},
		345	+ journal = {Review of Financial Studies},
		346	+ year = {2009},
		347	+ volume = {22},
		348	+ number = {1},
		349	+ pages = {435--480},
		350	+ doi = {10.1093/rfs/hhn053}
		351	+}
		352	+
		353	+@article{cameron2011,
		354	+ author = {Cameron, A. Colin and Gelbach, Jonah B. and Miller, Douglas L.},
		355	+ title = {{Robust Inference With Multiway Clustering}},
		356	+ journal = {Journal of Business \& Economic Statistics},
		357	+ year = {2011},
		358	+ volume = {29},
		359	+ number = {2},
		360	+ pages = {238--249},
		361	+ doi = {10.1198/jbes.2010.07136}
		362	+}
		363	+
		364	+@article{novymarx2016,
		365	+ author = {Novy-Marx, Robert and Velikov, Mihail},
		366	+ title = {{A Taxonomy of Anomalies and Their Trading Costs}},
		367	+ journal = {Review of Financial Studies},
		368	+ year = {2016},
		369	+ volume = {29},
		370	+ number = {1},
		371	+ pages = {104--147},
		372	+ doi = {10.1093/rfs/hhv063}
		373	+}
		374	+
		375	+@article{bollerslev2016beta,
		376	+ author = {Bollerslev, Tim and Li, Sophia Zhengzi and Todorov, Viktor},
		377	+ title = {{Roughing up beta: Continuous versus discontinuous betas and the cross section of expected stock returns}},
		378	+ journal = {Journal of Financial Economics},
		379	+ year = {2016},
		380	+ volume = {120},
		381	+ number = {3},
		382	+ pages = {464--490},
		383	+ doi = {10.1016/j.jfineco.2016.02.001}
		384	+}
		385	+
		386	+@article{lo1988,
		387	+ author = {Lo, Andrew W. and MacKinlay, A. Craig},
		388	+ title = {{Stock Market Prices Do Not Follow Random Walks: Evidence from a Simple Specification Test}},
		389	+ journal = {Review of Financial Studies},
		390	+ year = {1988},
		391	+ volume = {1},
		392	+ number = {1},
		393	+ pages = {41--66},
		394	+ doi = {10.1093/rfs/1.1.41}
		395	+}
		396	+
		397	+@article{freyberger2020,
		398	+ author = {Freyberger, Joachim and Neuhierl, Andreas and Weber, Michael},
		399	+ title = {{Dissecting Characteristics Nonparametrically}},
		400	+ journal = {The Review of Financial Studies},
		401	+ year = {2020},
		402	+ volume = {33},
		403	+ number = {5},
		404	+ pages = {2326--2377},
		405	+ doi = {10.1093/rfs/hhz123}
		406	+}
		407	+
		408	+@article{bollerslev2016errors,
		409	+ author = {Bollerslev, Tim and Patton, Andrew J. and Quaedvlieg, Rogier},
		410	+ title = {{Exploiting the errors: A simple approach for improved volatility forecasting}},
		411	+ journal = {Journal of Econometrics},
		412	+ year = {2016},
		413	+ volume = {192},
		414	+ number = {1},
		415	+ pages = {1--18},
		416	+ doi = {10.1016/j.jeconom.2015.10.007}
		417	+}
		418	+
		419	+@article{liu2015,
		420	+ author = {Liu, Lily Y. and Patton, Andrew J. and Sheppard, Kevin},
		421	+ title = {{Does anything beat 5-minute RV? A comparison of realized measures across multiple asset classes}},
		422	+ journal = {Journal of Econometrics},
		423	+ year = {2015},
		424	+ volume = {187},
		425	+ number = {1},
		426	+ pages = {293--311},
		427	+ doi = {10.1016/j.jeconom.2015.02.008}
		428	+}
		429	+
		430	+@article{andersen1998,
		431	+ author = {Andersen, Torben G. and Bollerslev, Tim},
		432	+ title = {{Answering the Skeptics: Yes, Standard Volatility Models do Provide Accurate Forecasts}},
		433	+ journal = {International Economic Review},
		434	+ year = {1998},
		435	+ volume = {39},
		436	+ number = {4},
		437	+ pages = {885},
		438	+ doi = {10.2307/2527343}
		439	+}
		440	+
		441	+@article{aboody2010,
		442	+ author = {Aboody, David and Lehavy, Reuven and Trueman, Brett},
		443	+ title = {{Limited attention and the earnings announcement returns of past stock market winners}},
		444	+ journal = {Review of Accounting Studies},
		445	+ year = {2010},
		446	+ volume = {15},
		447	+ number = {2},
		448	+ pages = {317--344},
		449	+ doi = {10.1007/s11142-009-9104-9}
		450	+}
		451	+
		452	+@article{kothari2001,
		453	+ author = {Kothari, S.P},
		454	+ title = {{Capital markets research in accounting}},
		455	+ journal = {Journal of Accounting and Economics},
		456	+ year = {2001},
		457	+ volume = {31},
		458	+ number = {1-3},
		459	+ pages = {105--231},
		460	+ doi = {10.1016/s0165-4101(01)00030-1}
		461	+}
		462	+
		463	+@article{easley2012,
		464	+ author = {Easley, David and López de Prado, Marcos M. and O'Hara, Maureen},
		465	+ title = {{Flow Toxicity and Liquidity in a High-frequency World}},
		466	+ journal = {Review of Financial Studies},
		467	+ year = {2012},
		468	+ volume = {25},
		469	+ number = {5},
		470	+ pages = {1457--1493},
		471	+ doi = {10.1093/rfs/hhs053}
		472	+}
		473	+
		474	+@article{hou2015,
		475	+ author = {Hou, Kewei and Xue, Chen and Zhang, Lu},
		476	+ title = {{Digesting Anomalies: An Investment Approach}},
		477	+ journal = {Review of Financial Studies},
		478	+ year = {2015},
		479	+ volume = {28},
		480	+ number = {3},
		481	+ pages = {650--705},
		482	+ doi = {10.1093/rfs/hhu068}
		483	+}
		484	+
		485	+@article{dechow2010,
		486	+ author = {Dechow, Patricia and Ge, Weili and Schrand, Catherine},
		487	+ title = {{Understanding earnings quality: A review of the proxies, their determinants and their consequences}},
		488	+ journal = {Journal of Accounting and Economics},
		489	+ year = {2010},
		490	+ volume = {50},
		491	+ number = {2-3},
		492	+ pages = {344--401},
		493	+ doi = {10.1016/j.jacceco.2010.09.001}
		494	+}
		495	+
		496	+@article{fama1970,
		497	+ author = {Fama, Eugene F.},
		498	+ title = {{Efficient Capital Markets: A Review of Theory and Empirical Work}},
		499	+ journal = {The Journal of Finance},
		500	+ year = {1970},
		501	+ volume = {25},
		502	+ number = {2},
		503	+ pages = {383},
		504	+ doi = {10.2307/2325486}
		505	+}
		506	+
		507	+@article{bollerslev2011tails,
		508	+ author = {BOLLERSLEV, TIM and TODOROV, VIKTOR},
		509	+ title = {{Tails, Fears, and Risk Premia}},
		510	+ journal = {The Journal of Finance},
		511	+ year = {2011},
		512	+ volume = {66},
		513	+ number = {6},
		514	+ pages = {2165--2211},
		515	+ doi = {10.1111/j.1540-6261.2011.01695.x}
		516	+}
		517	+
		518	+@article{hirshleifer2003,
		519	+ author = {Hirshleifer, David and Teoh, Siew Hong},
		520	+ title = {{Limited attention, information disclosure, and financial reporting}},
		521	+ journal = {Journal of Accounting and Economics},
		522	+ year = {2003},
		523	+ volume = {36},
		524	+ number = {1-3},
		525	+ pages = {337--386},
		526	+ doi = {10.1016/j.jacceco.2003.10.002}
		527	+}
		528	+
		529	+@article{savor2016,
		530	+ author = {SAVOR, PAVEL and WILSON, MUNGO},
		531	+ title = {{Earnings Announcements and Systematic Risk}},
		532	+ journal = {The Journal of Finance},
		533	+ year = {2016},
		534	+ volume = {71},
		535	+ number = {1},
		536	+ pages = {83--138},
		537	+ doi = {10.1111/jofi.12361}
		538	+}
		539	+
		540	+@article{budish2015,
		541	+ author = {Budish, Eric and Cramton, Peter and Shim, John},
		542	+ title = {{The High-Frequency Trading Arms Race: Frequent Batch Auctions as a Market Design Response *}},
		543	+ journal = {The Quarterly Journal of Economics},
		544	+ year = {2015},
		545	+ volume = {130},
		546	+ number = {4},
		547	+ pages = {1547--1621},
		548	+ doi = {10.1093/qje/qjv027}
		549	+}
		550	+
		551	+@article{chen2024deep,
		552	+ author = {Chen, Luyang and Pelger, Markus and Zhu, Jason},
		553	+ title = {{Deep Learning in Asset Pricing}},
		554	+ journal = {Management Science},
		555	+ year = {2024},
		556	+ volume = {70},
		557	+ number = {2},
		558	+ pages = {714--750},
		559	+ doi = {10.1287/mnsc.2023.4695}
		560	+}
		561	+
		562	+@article{kozak2020,
		563	+ author = {Kozak, Serhiy and Nagel, Stefan and Santosh, Shrihari},
		564	+ title = {{Shrinking the cross-section}},
		565	+ journal = {Journal of Financial Economics},
		566	+ year = {2020},
		567	+ volume = {135},
		568	+ number = {2},
		569	+ pages = {271--292},
		570	+ doi = {10.1016/j.jfineco.2019.06.008}
		571	+}
		572	+
		573	+@article{brogaard2019,
		574	+ author = {BROGAARD, JONATHAN and HENDERSHOTT, TERRENCE and RIORDAN, RYAN},
		575	+ title = {{Price Discovery without Trading: Evidence from Limit Orders}},
		576	+ journal = {The Journal of Finance},
		577	+ year = {2019},
		578	+ volume = {74},
		579	+ number = {4},
		580	+ pages = {1621--1658},
		581	+ doi = {10.1111/jofi.12769}
		582	+}
		583	+
		584	+@article{ang2006,
		585	+ author = {ANG, ANDREW and HODRICK, ROBERT J. and XING, YUHANG and ZHANG, XIAOYAN},
		586	+ title = {{The Cross-Section of Volatility and Expected Returns}},
		587	+ journal = {The Journal of Finance},
		588	+ year = {2006},
		589	+ volume = {61},
		590	+ number = {1},
		591	+ pages = {259--299},
		592	+ doi = {10.1111/j.1540-6261.2006.00836.x}
		593	+}
		594	+
		595	+@article{skinner2002,
		596	+ author = {Skinner, Douglas J. and Sloan, Richard G.},
		597	+ title = {{Earnings Surprises, Growth Expectations, and Stock Returns or Don't Let an Earnings Torpedo Sink Your Portfolio}},
		598	+ journal = {Review of Accounting Studies},
		599	+ year = {2002},
		600	+ volume = {7},
		601	+ number = {2-3},
		602	+ pages = {289--312},
		603	+ doi = {10.1023/a:1020294523516}
		604	+}
		605	+
		606	+@article{levi2015,
		607	+ author = {Levi, Shai and Zhang, Xiao-Jun},
		608	+ title = {{Asymmetric decrease in liquidity trading before earnings announcements and the announcement return premium}},
		609	+ journal = {Journal of Financial Economics},
		610	+ year = {2015},
		611	+ volume = {118},
		612	+ number = {2},
		613	+ pages = {383--398},
		614	+ doi = {10.1016/j.jfineco.2015.08.003}
		615	+}
		616	+
		617	+@article{campbell1993,
		618	+ author = {Campbell, J. Y. and Grossman, S. J. and Wang, J.},
		619	+ title = {{Trading Volume and Serial Correlation in Stock Returns}},
		620	+ journal = {The Quarterly Journal of Economics},
		621	+ year = {1993},
		622	+ volume = {108},
		623	+ number = {4},
		624	+ pages = {905--939},
		625	+ doi = {10.2307/2118454}
		626	+}
		627	+
		628	+@article{fang2009,
		629	+ author = {FANG, LILY and PERESS, JOEL},
		630	+ title = {{Media Coverage and the Cross-section of Stock Returns}},
		631	+ journal = {The Journal of Finance},
		632	+ year = {2009},
		633	+ volume = {64},
		634	+ number = {5},
		635	+ pages = {2023--2052},
		636	+ doi = {10.1111/j.1540-6261.2009.01493.x}
		637	+}
		638	+
		639	+@article{beaver1980,
		640	+ author = {Beaver, William and Lambert, Richard and Morse, Dale},
		641	+ title = {{The information content of security prices}},
		642	+ journal = {Journal of Accounting and Economics},
		643	+ year = {1980},
		644	+ volume = {2},
		645	+ number = {1},
		646	+ pages = {3--28},
		647	+ doi = {10.1016/0165-4101(80)90013-0}
		648	+}
		649	+
		650	+@article{kothari1992,
		651	+ author = {Kothari, S.P and Sloan, Richard G},
		652	+ title = {{Information in prices about future earnings}},
		653	+ journal = {Journal of Accounting and Economics},
		654	+ year = {1992},
		655	+ volume = {15},
		656	+ number = {2-3},
		657	+ pages = {143--171},
		658	+ doi = {10.1016/0165-4101(92)90016-u}
		659	+}
		660	+
		661	+@article{collins1994,
		662	+ author = {Collins, Daniel W. and Kothari, S.P. and Shanken, Jay and Sloan, Richard G.},
		663	+ title = {{Lack of timeliness and noise as explanations for the low contemporaneuos return-earnings association}},
		664	+ journal = {Journal of Accounting and Economics},
		665	+ year = {1994},
		666	+ volume = {18},
		667	+ number = {3},
		668	+ pages = {289--324},
		669	+ doi = {10.1016/0165-4101(94)90024-8}
		670	+}
		671	+
		672	+@article{foster1977,
		673	+ author = {Foster, George},
		674	+ title = {{Quarterly Accounting Data: Time-Series Properties and Predicative-Ability Results.}},
		675	+ journal = {The Accounting Review},
		676	+ year = {1977},
		677	+ volume = {52},
		678	+ number = {1},
		679	+ pages = {1--21},
		680	+ doi = {10.2308/tar-4493420}
		681	+}
		682	+
		683	+@article{weller2018,
		684	+ author = {Weller, Brian M.},
		685	+ title = {{Does Algorithmic Trading Reduce Information Acquisition?}},
		686	+ journal = {The Review of Financial Studies},
		687	+ year = {2018},
		688	+ volume = {31},
		689	+ number = {6},
		690	+ pages = {2184--2226},
		691	+ doi = {10.1093/rfs/hhx137}
		692	+}
		693	+
		694	+@article{bollerslev2018volume,
		695	+ author = {Bollerslev, Tim and Li, Jia and Xue, Yuan},
		696	+ title = {{Volume, Volatility, and Public News Announcements}},
		697	+ journal = {The Review of Economic Studies},
		698	+ year = {2018},
		699	+ volume = {85},
		700	+ number = {4},
		701	+ pages = {2005--2041},
		702	+ doi = {10.1093/restud/rdy003}
		703	+}
		704	+
		705	+@article{patton2012,
		706	+ author = {Patton, Andrew J. and Verardo, Michela},
		707	+ title = {{Does Beta Move with News? Firm-Specific Information Flows and Learning about Profitability}},
		708	+ journal = {Review of Financial Studies},
		709	+ year = {2012},
		710	+ volume = {25},
		711	+ number = {9},
		712	+ pages = {2789--2839},
		713	+ doi = {10.1093/rfs/hhs073}
		714	+}
		715	+
		716	+@article{kaniel2012,
		717	+ author = {KANIEL, RON and LIU, SHUMING and SAAR, GIDEON and TITMAN, SHERIDAN},
		718	+ title = {{Individual Investor Trading and Return Patterns around Earnings Announcements}},
		719	+ journal = {The Journal of Finance},
		720	+ year = {2012},
		721	+ volume = {67},
		722	+ number = {2},
		723	+ pages = {639--680},
		724	+ doi = {10.1111/j.1540-6261.2012.01727.x}
		725	+}
		726	+
		727	+@article{christophe2004,
		728	+ author = {Christophe, Stephen E. and Ferri, Michael G. and Angel, James J.},
		729	+ title = {{Short-Selling Prior to Earnings Announcements}},
		730	+ journal = {The Journal of Finance},
		731	+ year = {2004},
		732	+ volume = {59},
		733	+ number = {4},
		734	+ pages = {1845--1876},
		735	+ doi = {10.1111/j.1540-6261.2004.00681.x}
		736	+}
		737	+
		738	+@article{berkman2009,
		739	+ author = {Berkman, Henk and Dimitrov, Valentin and Jain, Prem C. and Koch, Paul D. and Tice, Sheri},
		740	+ title = {{Sell on the news: Differences of opinion, short-sales constraints, and returns around earnings announcements☆}},
		741	+ journal = {Journal of Financial Economics},
		742	+ year = {2009},
		743	+ volume = {92},
		744	+ number = {3},
		745	+ pages = {376--399},
		746	+ doi = {10.1016/j.jfineco.2008.04.009}
		747	+}
		748	+
		749	+@article{hendershott2015,
		750	+ author = {Hendershott, Terrence and Livdan, Dmitry and Schürhoff, Norman},
		751	+ title = {{Are institutions informed about news?}},
		752	+ journal = {Journal of Financial Economics},
		753	+ year = {2015},
		754	+ volume = {117},
		755	+ number = {2},
		756	+ pages = {249--287},
		757	+ doi = {10.1016/j.jfineco.2015.03.007}
		758	+}
		759	+
		760	+@article{lee2012,
		761	+ author = {Lee, Suzanne S.},
		762	+ title = {{Jumps and Information Flow in Financial Markets}},
		763	+ journal = {Review of Financial Studies},
		764	+ year = {2012},
		765	+ volume = {25},
		766	+ number = {2},
		767	+ pages = {439--479},
		768	+ doi = {10.1093/rfs/hhr084}
		769	+}
		770	+
		771	+@article{cao2024,
		772	+ author = {Cao, Sean and Jiang, Wei and Wang, Junbo and Yang, Baozhong},
		773	+ title = {{From Man vs. Machine to Man + Machine: The art and AI of stock analyses}},
		774	+ journal = {Journal of Financial Economics},
		775	+ year = {2024},
		776	+ volume = {160},
		777	+ pages = {103910},
		778	+ doi = {10.1016/j.jfineco.2024.103910}
		779	+}
		780	+
		781	+@article{bali2023,
		782	+ author = {Bali, Turan G and Beckmeyer, Heiner and Mörke, Mathis and Weigert, Florian},
		783	+ title = {{Option Return Predictability with Machine Learning and Big Data}},
		784	+ journal = {The Review of Financial Studies},
		785	+ year = {2023},
		786	+ volume = {36},
		787	+ number = {9},
		788	+ pages = {3548--3602},
		789	+ doi = {10.1093/rfs/hhad017}
		790	+}
		791	+
		792	+@article{easton2024,
		793	+ author = {Easton, Peter D. and Kapons, Martin M. and Monahan, Steven J. and Schütt, Harm H. and Weisbrod, Eric H.},
		794	+ title = {{Forecasting Earnings Using \emph{k}-Nearest Neighbors}},
		795	+ journal = {The Accounting Review},
		796	+ year = {2024},
		797	+ volume = {99},
		798	+ number = {3},
		799	+ pages = {115--140},
		800	+ doi = {10.2308/tar-2021-0478}
		801	+}
		802	+
		803	+@article{so2013,
		804	+ author = {So, Eric C.},
		805	+ title = {{A new approach to predicting analyst forecast errors: Do investors overweight analyst forecasts?}},
		806	+ journal = {Journal of Financial Economics},
		807	+ year = {2013},
		808	+ volume = {108},
		809	+ number = {3},
		810	+ pages = {615--640},
		811	+ doi = {10.1016/j.jfineco.2013.02.002}
		812	+}
		813	+
		814	+@article{hou2012,
		815	+ author = {Hou, Kewei and van Dijk, Mathijs A. and Zhang, Yinglei},
		816	+ title = {{The implied cost of capital: A new approach}},
		817	+ journal = {Journal of Accounting and Economics},
		818	+ year = {2012},
		819	+ volume = {53},
		820	+ number = {3},
		821	+ pages = {504--526},
		822	+ doi = {10.1016/j.jacceco.2011.12.001}
		823	+}
		824	+
		825	+@article{hansen2006,
		826	+ author = {Hansen, Peter R and Lunde, Asger},
		827	+ title = {{Realized Variance and Market Microstructure Noise}},
		828	+ journal = {Journal of Business \& Economic Statistics},
		829	+ year = {2006},
		830	+ volume = {24},
		831	+ number = {2},
		832	+ pages = {127--161},
		833	+ doi = {10.1198/073500106000000071}
		834	+}
		835	+
		836	+@article{barndorff2008,
		837	+ author = {Barndorff-Nielsen, Ole E. and Hansen, Peter Reinhard and Lunde, Asger and Shephard, Neil},
		838	+ title = {{Designing Realized Kernels to Measure the ex post Variation of Equity Prices in the Presence of Noise}},
		839	+ journal = {Econometrica},
		840	+ year = {2008},
		841	+ volume = {76},
		842	+ number = {6},
		843	+ pages = {1481--1536},
		844	+ doi = {10.3982/ecta6495}
		845	+}
		846	+
		847	+@article{vankervel2019,
		848	+ author = {VAN KERVEL, VINCENT and MENKVELD, ALBERT J.},
		849	+ title = {{High-Frequency Trading around Large Institutional Orders}},
		850	+ journal = {The Journal of Finance},
		851	+ year = {2019},
		852	+ volume = {74},
		853	+ number = {3},
		854	+ pages = {1091--1137},
		855	+ doi = {10.1111/jofi.12759}
		856	+}
		857	+
		858	+@article{avramov2023,
		859	+ author = {Avramov, Doron and Cheng, Si and Metzker, Lior},
		860	+ title = {{Machine Learning vs. Economic Restrictions: Evidence from Stock Return Predictability}},
		861	+ journal = {Management Science},
		862	+ year = {2023},
		863	+ volume = {69},
		864	+ number = {5},
		865	+ pages = {2587--2619},
		866	+ doi = {10.1287/mnsc.2022.4449}
		867	+}
		868	+
		869	+@article{chordia2020,
		870	+ author = {Chordia, Tarun and Goyal, Amit and Saretto, Alessio},
		871	+ title = {{Anomalies and False Rejections}},
		872	+ journal = {The Review of Financial Studies},
		873	+ year = {2020},
		874	+ volume = {33},
		875	+ number = {5},
		876	+ pages = {2134--2179},
		877	+ doi = {10.1093/rfs/hhaa018}
		878	+}
		879	+
		880	+@article{barber2013,
		881	+ author = {Barber, Brad M. and De George, Emmanuel T. and Lehavy, Reuven and Trueman, Brett},
		882	+ title = {{The earnings announcement premium around the globe}},
		883	+ journal = {Journal of Financial Economics},
		884	+ year = {2013},
		885	+ volume = {108},
		886	+ number = {1},
		887	+ pages = {118--138},
		888	+ doi = {10.1016/j.jfineco.2012.10.006}
		889	+}
		890	+
		891	+@article{politis1994,
		892	+ author = {Politis, Dimitris N. and Romano, Joseph P.},
		893	+ title = {{The Stationary Bootstrap}},
		894	+ journal = {Journal of the American Statistical Association},
		895	+ year = {1994},
		896	+ volume = {89},
		897	+ number = {428},
		898	+ pages = {1303--1313},
		899	+ doi = {10.1080/01621459.1994.10476870}
		900	+}
		901	+
		902	+@article{hansen2011mcs,
		903	+ author = {Hansen, Peter R. and Lunde, Asger and Nason, James M.},
		904	+ title = {{The Model Confidence Set}},
		905	+ journal = {Econometrica},
		906	+ year = {2011},
		907	+ volume = {79},
		908	+ number = {2},
		909	+ pages = {453--497},
		910	+ doi = {10.3982/ecta5771}
		911	+}
		912	+
		913	+@article{benjamini1995,
		914	+ author = {Benjamini, Yoav and Hochberg, Yosef},
		915	+ title = {{Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing}},
		916	+ journal = {Journal of the Royal Statistical Society Series B: Statistical Methodology},
		917	+ year = {1995},
		918	+ volume = {57},
		919	+ number = {1},
		920	+ pages = {289--300},
		921	+ doi = {10.1111/j.2517-6161.1995.tb02031.x}
		922	+}
		923	+
		924	+@article{amihud2002,
		925	+ author = {Amihud, Yakov},
		926	+ title = {{Illiquidity and stock returns: cross-section and time-series effects}},
		927	+ journal = {Journal of Financial Markets},
		928	+ year = {2002},
		929	+ volume = {5},
		930	+ number = {1},
		931	+ pages = {31--56},
		932	+ doi = {10.1016/s1386-4181(01)00024-6}
		933	+}
		934	+
		935	+@article{fama1993,
		936	+ author = {Fama, Eugene F. and French, Kenneth R.},
		937	+ title = {{Common risk factors in the returns on stocks and bonds}},
		938	+ journal = {Journal of Financial Economics},
		939	+ year = {1993},
		940	+ volume = {33},
		941	+ number = {1},
		942	+ pages = {3--56},
		943	+ doi = {10.1016/0304-405x(93)90023-5}
		944	+}
		945	+
		946	+@article{grossman1980,
		947	+ author = {Grossman, Sanford J. and Stiglitz, Joseph E.},
		948	+ title = {{On the Impossibility of Informationally Efficient Markets}},
		949	+ journal = {American Economic Review},
		950	+ year = {1980},
		951	+ volume = {70},
		952	+ number = {3},
		953	+ pages = {393--408}
		954	+}
		955	+@article{holm1979,
		956	+ author = {Holm, Sture},
		957	+ title = {{A Simple Sequentially Rejective Multiple Test Procedure}},
		958	+ journal = {Scandinavian Journal of Statistics},
		959	+ year = {1979},
		960	+ volume = {6},
		961	+ number = {2},
		962	+ pages = {65--70}
		963	+}
		964	+@article{kunsch1989,
		965	+ author = {K{\"u}nsch, Hans R.},
		966	+ title = {{The Jackknife and the Bootstrap for General Stationary Observations}},
		967	+ journal = {Annals of Statistics},
		968	+ year = {1989},
		969	+ volume = {17},
		970	+ number = {3},
		971	+ pages = {1217--1241},
		972	+ doi = {10.1214/aos/1176347265}
		973	+}
		974	+@inproceedings{ke2017lightgbm,
		975	+ author = {Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan},
		976	+ title = {{LightGBM: A Highly Efficient Gradient Boosting Decision Tree}},
		977	+ booktitle = {Advances in Neural Information Processing Systems 30},
		978	+ year = {2017},
		979	+ pages = {3146--3154}
		980	+}
		981	+@book{lopezdeprado2018,
		982	+ author = {L{\'o}pez de Prado, Marcos},
		983	+ title = {{Advances in Financial Machine Learning}},
		984	+ publisher = {Wiley},
		985	+ address = {Hoboken, NJ},
		986	+ year = {2018}
		987	+}
		988	+@techreport{frazzini2018costs,
		989	+ author = {Frazzini, Andrea and Israel, Ronen and Moskowitz, Tobias J.},
		990	+ title = {{Trading Costs}},
		991	+ institution = {SSRN Working Paper 3229719},
		992	+ year = {2018},
		993	+ doi = {10.2139/ssrn.3229719}
		994	+}
		995	+@incollection{barndorff2010semivariance,
		996	+ author = {Barndorff-Nielsen, Ole E. and Kinnebrock, Silja and Shephard, Neil},
		997	+ title = {{Measuring Downside Risk: Realised Semivariance}},
		998	+ booktitle = {Volatility and Time Series Econometrics: Essays in Honor of Robert F. Engle},
		999	+ editor = {Bollerslev, Tim and Russell, Jeffrey and Watson, Mark},
		1000	+ publisher = {Oxford University Press},
		1001	+ year = {2010},
		1002	+ pages = {117--136},
		1003	+ doi = {10.1093/acprof:oso/9780199549498.003.0007}
		1004	+}
		1005	+@article{boucher2026wp15,
		1006	+ author = {Boucher, Simon-Pierre},
		1007	+ title = {{Faster, Wider, More Fragile? The Evolution of Intraday Shock Propagation in U.S. Equity Markets, 1992--2026}},
		1008	+ journal = {UQO Working Paper No. 15},
		1009	+ year = {2026}
		1010	+}
		1011	+
		1012	+@article{harvey2016,
		1013	+ author = {Harvey, Campbell R. and Liu, Yan and Zhu, Heqing},
		1014	+ title = {{\ldots and the Cross-Section of Expected Returns}},
		1015	+ journal = {The Review of Financial Studies},
		1016	+ year = {2016},
		1017	+ volume = {29},
		1018	+ number = {1},
		1019	+ pages = {5--68},
		1020	+ doi = {10.1093/rfs/hhv059}
		1021	+}

added paper/.latexmkrc +5 −0

				@@ -0,0 +1,5 @@
		1	+$pdf_mode = 1;
		2	+$pdflatex = 'pdflatex -interaction=nonstopmode -halt-on-error -synctex=1 %O %S';
		3	+$bibtex_use = 2;
		4	+$clean_ext = 'synctex.gz run.xml bbl bcf fdb_latexmk fls log aux out toc lof lot blg';
		5	+@default_files = ('main.tex');

added paper/Makefile +11 −0

				@@ -0,0 +1,11 @@
		1	+.PHONY: all clean distclean
		2	+
		3	+all:
		4	+ latexmk
		5	+
		6	+clean:
		7	+ latexmk -c
		8	+
		9	+distclean:
		10	+ latexmk -C
		11	+ rm -f *.synctex.gz *.run.xml *.bbl *.bcf

added paper/appendix/appendix.tex +54 −0

				@@ -0,0 +1,54 @@
		1	+\newpage
		2	+\begin{appendices}
		3	+\section{Variable definitions and data lineage}
		4	+\label{app:variables}
		5	+\begin{table}[H]
		6	+\centering\scriptsize
		7	+\caption{Variable definitions and lineage}
		8	+\begin{tabular}{p{3.4cm}p{8.2cm}p{3.2cm}}
		9	+\toprule
		10	+Variable & Definition & Source / step \\
		11	+\midrule
		12	+$A_{i,q}$ & Earnings-release date: first \texttt{earnings\_events.event\_date} in $[E_q, F_q+10\,\mathrm{d}]$ & FinData Postgres extract (01a) \\
		13	+$F_{i,q}$, $V_{i,q}$ & SEC filing date; availability $=F+1$ day 00:00 ET & \texttt{/fundamentals} (02) \\
		14	+$S_{i,q}$ & $(X_q-\hat X_q)/TA_{q-1}$, $X=$ operating income, $\hat X_q=X_{q-4}+\frac14\sum_{k=0}^3(X_{q-1-k}-X_{q-5-k})$ & 02, \texttt{labels.py} \\
		15	+$D_{i,q}$ & $\mathbb 1[S_{i,q}\le-0.005]$ & 02 \\
		16	+Secondary targets & net-income surprise; $\Delta$ROA$_{yoy}\le-0.01$; $(CFO_q-CFO_{q-4})/TA\le-0.01$; current ratio $-10\%$ or coverage $<1$ from $\ge1$; $|S|\ge0.005$; EPS $<$ estimate & 02 \\
		17	+Intraday families (per bar length $f\in\{1,5,30\}$ min) & RV $=\sum r^2$; BPV $=\frac{\pi}{2}\sum|r_t||r_{t-1}|$; RJ $=\max(0,\mathrm{RV}-\mathrm{BPV})/\mathrm{RV}$; RS$^-$/RV; SJV $=(\mathrm{RS}^+-\mathrm{RS}^-)/\mathrm{RV}$; realized skewness/kurtosis; $\max r^2/\mathrm{RV}$; AC(1); drift $=\ln(C/O)$; opening/closing 30-min volume shares; volume Gini; down-bar volume share; present-bar share; RV share on SPY-down bars; $\beta=\sum r\,r^{SPY}/\sum (r^{SPY})^2$; vol-of-vol; RV/Var(daily) & 03, \texttt{features.py}, \texttt{samples.py} \\
		18	+Daily block & 20/60-day log returns, SD, Parkinson, skewness, kurtosis, overnight/intraday sums, volume ratios (20d, 5d vs 60d median), Amihud (60d), zero-return share, log dollar volume, max drawdown (60d), bars per day & 03 \\
		19	+Accounting block & gross/operating margin, NI/TA, CFO/TA, accruals, cash/TA, debt/TA, current ratio, capex/TA, R\&D/revenue, coverage, loss dummy, log assets, ROA, revenue growth, net issuance, log market cap, last surprise, mean of last four surprises, trailing adverse rate, information age & 02--04 \\
		20	+Returns & split- and dividend-adjusted close-to-close from raw bars and \texttt{corporate\_actions} (ex-date rule) & 03 \\
		21	+\bottomrule
		22	+\end{tabular}
		23	+\end{table}
		24	+
		25	+\section{Leakage battery}
		26	+\label{app:leakage}
		27	+Each guard is a unit test in \texttt{tests/}: (L1) future facts, (L2) fiscal-end joins, (L3) post-cutoff bars,
		28	+(L4) test-period transforms, (L5) DST/timezone, (L6) identifier drift, (L7) splits/dividends, (L8) release-day bars,
		29	+(L9) overlapping windows across splits, (L10) survivorship report, (L11) ingestion dates, (L12) randomised placebos.
		30	+The live test \texttt{test\_asof\_equivalence} verifies that the local as-of join reproduces the API's
		31	+\texttt{as\_of} output on random entities and instants.
		32	+
		33	+\section{Additional tables}
		34	+\label{app:tables}
		35	+\begin{table}[H]\centering\footnotesize
		36	+\caption{Continuous target: ridge forecasts of the winsorised surprise (test period)}
		37	+\input{tables/tab10b_continuous}
		38	+\end{table}
		39	+\begin{table}[H]\centering\footnotesize
		40	+\caption{Transfer to held-out firms (20\% of firms never seen in training)}
		41	+\input{tables/tab09b_firmholdout}
		42	+\end{table}
		43	+\begin{table}[H]\centering\footnotesize
		44	+\caption{Announcement-window abnormal returns on the ex-ante signal}
		45	+\input{tables/tab11b_car}
		46	+\begin{minipage}{\textwidth}\vspace{0.3em}\footnotesize\emph{Notes.} CAR from $c+1$ to $A_q+1$, market-adjusted, winsorised 1/99\%; firm and sector$\times$quarter fixed effects; two-way clustered SEs. $^{*}$, $^{**}$, $^{***}$: 10, 5, 1\%.\end{minipage}
		47	+\end{table}
		48	+
		49	+\section{Reproducibility}
		50	+\label{app:repro}
		51	+The pipeline (\texttt{scripts/00}--\texttt{09}) runs on the FinData storage node against the local API; manifests in
		52	+\texttt{artifacts/manifests} pin dataset versions and hashes; the design freeze hash is recorded before the first
		53	+model fit. Raw extracts are deleted after feature extraction and are reproducible from the pinned versions.
		54	+\end{appendices}

added paper/main.tex +141 −0

				@@ -0,0 +1,141 @@
		1	+% ============================================================================
		2	+% UQO Working Paper Template — Economics / Finance
		3	+% ============================================================================
		4	+\documentclass[12pt,letterpaper]{article}
		5	+
		6	+% --- Encoding & Language ---
		7	+\usepackage[utf8]{inputenc}
		8	+\usepackage[T1]{fontenc}
		9	+\usepackage[english]{babel}
		10	+
		11	+% --- Page Layout ---
		12	+\usepackage[
		13	+ letterpaper,
		14	+ top=1in,
		15	+ bottom=1in,
		16	+ left=1in,
		17	+ right=1in
		18	+]{geometry}
		19	+
		20	+% --- Typography ---
		21	+\usepackage{mathptmx} % Times-like font (text + math)
		22	+\usepackage{microtype} % Improved typographic refinements
		23	+\usepackage{setspace}
		24	+\onehalfspacing % 1.5 line spacing
		25	+
		26	+% --- Math ---
		27	+\usepackage{amsmath,amssymb,amsthm}
		28	+
		29	+% --- Tables ---
		30	+\usepackage{booktabs}
		31	+\usepackage{tabularx}
		32	+\usepackage{multirow}
		33	+\usepackage{threeparttable}
		34	+\usepackage{siunitx}
		35	+\sisetup{
		36	+ input-symbols = {()},
		37	+ table-format = -1.3,
		38	+ parse-numbers = false
		39	+}
		40	+
		41	+% --- Figures ---
		42	+\usepackage{graphicx}
		43	+\usepackage{subcaption}
		44	+\usepackage{float}
		45	+\usepackage{longtable}
		46	+\usepackage{adjustbox}
		47	+
		48	+% --- Colors & Links ---
		49	+\usepackage[dvipsnames]{xcolor}
		50	+\usepackage[
		51	+ pdftitle={The Invisible Balance Sheet},
		52	+ pdfauthor={Simon-Pierre Boucher},
		53	+ colorlinks = true,
		54	+ linkcolor = NavyBlue,
		55	+ citecolor = NavyBlue,
		56	+ urlcolor = NavyBlue,
		57	+ breaklinks = true
		58	+]{hyperref}
		59	+
		60	+% --- Bibliography ---
		61	+\usepackage[authoryear,round,semicolon]{natbib}
		62	+\bibliographystyle{aer}
		63	+
		64	+% --- Captions ---
		65	+\usepackage[
		66	+ font = small,
		67	+ labelfont = bf,
		68	+ labelsep = period,
		69	+ justification = justified,
		70	+ singlelinecheck = false
		71	+]{caption}
		72	+
		73	+% --- Headers & Footers ---
		74	+\usepackage{fancyhdr}
		75	+\pagestyle{fancy}
		76	+\fancyhf{}
		77	+\renewcommand{\headrulewidth}{0pt}
		78	+\fancyfoot[C]{\thepage}
		79	+
		80	+% --- Section Formatting ---
		81	+\usepackage{titlesec}
		82	+\titleformat{\section}{\large\bfseries}{\thesection.}{0.5em}{}
		83	+\titleformat{\subsection}{\normalsize\bfseries}{\thesubsection.}{0.5em}{}
		84	+\titleformat{\subsubsection}{\normalsize\itshape}{\thesubsubsection.}{0.5em}{}
		85	+
		86	+% --- Appendix Support ---
		87	+\usepackage[toc,page]{appendix}
		88	+
		89	+% --- Misc ---
		90	+\usepackage{enumitem}
		91	+\setlist{nosep,leftmargin=*}
		92	+
		93	+% ============================================================================
		94	+% METADATA
		95	+% ============================================================================
		96	+\input{results_macros}
		97	+\newcommand{\WPnumber}{39}
		98	+\newcommand{\WPtitle}{The Invisible Balance Sheet}
		99	+\newcommand{\WPsubtitle}{Can Intraday Financial Markets Detect Corporate Deterioration Before Accounting Disclosures?}
		100	+\newcommand{\WPdate}{November 2026}
		101	+\newcommand{\WPversion}{1.0}
		102	+\newcommand{\WPabstract}{\input{sections/abstract}}
		103	+\newcommand{\WPkeywords}{intraday data, sampling frequency, earnings surprises, fundamental analysis, point-in-time data, machine learning, market efficiency}
		104	+\newcommand{\WPjel}{G14, G12, G17, M41, C53, C58}
		105	+
		106	+% --- Author ---
		107	+\newcommand{\WPauthor}{Simon-Pierre Boucher}
		108	+\newcommand{\WPaffiliation}{%
		109	+ Département des sciences administratives\\
		110	+ Université du Québec en Outaouais%
		111	+}
		112	+\newcommand{\WPemail}{simon-pierre.boucher@uqo.ca}
		113	+\newcommand{\WPaddress}{%
		114	+ Gatineau -- Pavillon Alexandre-Taché\\
		115	+ 283, boulevard Alexandre-Taché\\
		116	+ Gatineau, Québec, Canada J9A 1L8%
		117	+}
		118	+
		119	+% ============================================================================
		120	+% DOCUMENT
		121	+% ============================================================================
		122	+\begin{document}
		123	+
		124	+\input{sections/titlepage}
		125	+
		126	+\input{sections/introduction}
		127	+\input{sections/theory_hypotheses}
		128	+\input{sections/literature}
		129	+\input{sections/data}
		130	+\input{sections/methodology}
		131	+\input{sections/results}
		132	+\input{sections/mechanisms}
		133	+\input{sections/robustness}
		134	+\input{sections/conclusion}
		135	+
		136	+\newpage
		137	+\bibliography{references}
		138	+
		139	+\input{appendix/appendix}
		140	+
		141	+\end{document}

added paper/references.bib +1021 −0

				@@ -0,0 +1,1021 @@
		1	+% WP39 UQO bibliography. Journal metadata verified on Crossref 2026-11-02 (artifacts/logs/crossref_batch*.json).
		2	+
		3	+@article{gregoire2022,
		4	+ author = {Grégoire, Vincent and Martineau, Charles},
		5	+ title = {{How is Earnings News Transmitted to Stock Prices?}},
		6	+ journal = {Journal of Accounting Research},
		7	+ year = {2022},
		8	+ volume = {60},
		9	+ number = {1},
		10	+ pages = {261--297},
		11	+ doi = {10.1111/1475-679x.12394}
		12	+}
		13	+
		14	+@article{zhang2023volatility,
		15	+ author = {Zhang, Chao and Zhang, Yihuang and Cucuringu, Mihai and Qian, Zhongmin},
		16	+ title = {{Volatility Forecasting with Machine Learning and Intraday Commonality}},
		17	+ journal = {Journal of Financial Econometrics},
		18	+ year = {2024},
		19	+ volume = {22},
		20	+ number = {2},
		21	+ pages = {492--530},
		22	+ doi = {10.1093/jjfinec/nbad005}
		23	+}
		24	+
		25	+@article{ball1968,
		26	+ author = {Ball, Ray and Brown, Philip},
		27	+ title = {{An Empirical Evaluation of Accounting Income Numbers}},
		28	+ journal = {Journal of Accounting Research},
		29	+ year = {1968},
		30	+ volume = {6},
		31	+ number = {2},
		32	+ pages = {159},
		33	+ doi = {10.2307/2490232}
		34	+}
		35	+
		36	+@article{bernard1989,
		37	+ author = {Bernard, Victor L. and Thomas, Jacob K.},
		38	+ title = {{Post-Earnings-Announcement Drift: Delayed Price Response or Risk Premium?}},
		39	+ journal = {Journal of Accounting Research},
		40	+ year = {1989},
		41	+ volume = {27},
		42	+ pages = {1},
		43	+ doi = {10.2307/2491062}
		44	+}
		45	+
		46	+@article{bernard1990,
		47	+ author = {Bernard, Victor L. and Thomas, Jacob K.},
		48	+ title = {{Evidence that stock prices do not fully reflect the implications of current earnings for future earnings}},
		49	+ journal = {Journal of Accounting and Economics},
		50	+ year = {1990},
		51	+ volume = {13},
		52	+ number = {4},
		53	+ pages = {305--340},
		54	+ doi = {10.1016/0165-4101(90)90008-r}
		55	+}
		56	+
		57	+@article{sloan1996,
		58	+ author = {Sloan, Richard G.},
		59	+ title = {{Do Stock Prices Fully Reflect Information in Accruals and Cash Flows About Future Earnings?}},
		60	+ journal = {The Accounting Review},
		61	+ year = {1996},
		62	+ volume = {71},
		63	+ number = {3},
		64	+ pages = {289--315},
		65	+ doi = {10.2308/tar-9608042309}
		66	+}
		67	+
		68	+@article{ou1989,
		69	+ author = {Ou, Jane A. and Penman, Stephen H.},
		70	+ title = {{Financial statement analysis and the prediction of stock returns}},
		71	+ journal = {Journal of Accounting and Economics},
		72	+ year = {1989},
		73	+ volume = {11},
		74	+ number = {4},
		75	+ pages = {295--329},
		76	+ doi = {10.1016/0165-4101(89)90017-7}
		77	+}
		78	+
		79	+@article{kyle1985,
		80	+ author = {Kyle, Albert S.},
		81	+ title = {{Continuous Auctions and Insider Trading}},
		82	+ journal = {Econometrica},
		83	+ year = {1985},
		84	+ volume = {53},
		85	+ number = {6},
		86	+ pages = {1315},
		87	+ doi = {10.2307/1913210}
		88	+}
		89	+
		90	+@article{easley1987,
		91	+ author = {Easley, David and O'Hara, Maureen},
		92	+ title = {{Price, trade size, and information in securities markets}},
		93	+ journal = {Journal of Financial Economics},
		94	+ year = {1987},
		95	+ volume = {19},
		96	+ number = {1},
		97	+ pages = {69--90},
		98	+ doi = {10.1016/0304-405x(87)90029-8}
		99	+}
		100	+
		101	+@article{andersen2003,
		102	+ author = {Andersen, Torben G. and Bollerslev, Tim and Diebold, Francis X. and Labys, Paul},
		103	+ title = {{Modeling and Forecasting Realized Volatility}},
		104	+ journal = {Econometrica},
		105	+ year = {2003},
		106	+ volume = {71},
		107	+ number = {2},
		108	+ pages = {579--625},
		109	+ doi = {10.1111/1468-0262.00418}
		110	+}
		111	+
		112	+@article{barndorff2004,
		113	+ author = {Barndorff-Nielsen, O. E.},
		114	+ title = {{Power and Bipower Variation with Stochastic Volatility and Jumps}},
		115	+ journal = {Journal of Financial Econometrics},
		116	+ year = {2004},
		117	+ volume = {2},
		118	+ number = {1},
		119	+ pages = {1--37},
		120	+ doi = {10.1093/jjfinec/nbh001}
		121	+}
		122	+
		123	+@article{gu2020,
		124	+ author = {Gu, Shihao and Kelly, Bryan and Xiu, Dacheng},
		125	+ title = {{Empirical Asset Pricing via Machine Learning}},
		126	+ journal = {The Review of Financial Studies},
		127	+ year = {2020},
		128	+ volume = {33},
		129	+ number = {5},
		130	+ pages = {2223--2273},
		131	+ doi = {10.1093/rfs/hhaa009}
		132	+}
		133	+
		134	+@article{giacomini2006,
		135	+ author = {Giacomini, Raffaella and White, Halbert},
		136	+ title = {{Tests of Conditional Predictive Ability}},
		137	+ journal = {Econometrica},
		138	+ year = {2006},
		139	+ volume = {74},
		140	+ number = {6},
		141	+ pages = {1545--1578},
		142	+ doi = {10.1111/j.1468-0262.2006.00718.x}
		143	+}
		144	+
		145	+@article{diebold1995,
		146	+ author = {Diebold, Francis X. and Mariano, Roberto S.},
		147	+ title = {{Comparing Predictive Accuracy}},
		148	+ journal = {Journal of Business \& Economic Statistics},
		149	+ year = {1995},
		150	+ volume = {13},
		151	+ number = {3},
		152	+ pages = {253--263},
		153	+ doi = {10.1080/07350015.1995.10524599}
		154	+}
		155	+
		156	+@article{hansen2005,
		157	+ author = {Hansen, Peter Reinhard},
		158	+ title = {{A Test for Superior Predictive Ability}},
		159	+ journal = {Journal of Business \& Economic Statistics},
		160	+ year = {2005},
		161	+ volume = {23},
		162	+ number = {4},
		163	+ pages = {365--380},
		164	+ doi = {10.1198/073500105000000063}
		165	+}
		166	+
		167	+@article{fama2015,
		168	+ author = {Fama, Eugene F. and French, Kenneth R.},
		169	+ title = {{A five-factor asset pricing model}},
		170	+ journal = {Journal of Financial Economics},
		171	+ year = {2015},
		172	+ volume = {116},
		173	+ number = {1},
		174	+ pages = {1--22},
		175	+ doi = {10.1016/j.jfineco.2014.10.010}
		176	+}
		177	+
		178	+@article{bollerslev2020good,
		179	+ author = {Bollerslev, Tim and Li, Sophia Zhengzi and Zhao, Bingzhi},
		180	+ title = {{Good Volatility, Bad Volatility, and the Cross Section of Stock Returns}},
		181	+ journal = {Journal of Financial and Quantitative Analysis},
		182	+ year = {2020},
		183	+ volume = {55},
		184	+ number = {3},
		185	+ pages = {751--781},
		186	+ doi = {10.1017/s0022109019000097}
		187	+}
		188	+
		189	+@article{patton2015,
		190	+ author = {Patton, Andrew J. and Sheppard, Kevin},
		191	+ title = {{Good Volatility, Bad Volatility: Signed Jumps and The Persistence of Volatility}},
		192	+ journal = {Review of Economics and Statistics},
		193	+ year = {2015},
		194	+ volume = {97},
		195	+ number = {3},
		196	+ pages = {683--697},
		197	+ doi = {10.1162/rest_a_00503}
		198	+}
		199	+
		200	+@article{chen2022,
		201	+ author = {CHEN, XI and CHO, YANG HA (TONY) and DOU, YIWEI and LEV, BARUCH},
		202	+ title = {{Predicting Future Earnings Changes Using Machine Learning and Detailed Financial Data}},
		203	+ journal = {Journal of Accounting Research},
		204	+ year = {2022},
		205	+ volume = {60},
		206	+ number = {2},
		207	+ pages = {467--515},
		208	+ doi = {10.1111/1475-679x.12429}
		209	+}
		210	+
		211	+@article{piotroski2000,
		212	+ author = {Piotroski, Joseph D.},
		213	+ title = {{Value Investing: The Use of Historical Financial Statement Information to Separate Winners from Losers}},
		214	+ journal = {Journal of Accounting Research},
		215	+ year = {2000},
		216	+ volume = {38},
		217	+ pages = {1},
		218	+ doi = {10.2307/2672906}
		219	+}
		220	+
		221	+@article{hirshleifer2009,
		222	+ author = {HIRSHLEIFER, DAVID and LIM, SONYA SEONGYEON and TEOH, SIEW HONG},
		223	+ title = {{Driven to Distraction: Extraneous Events and Underreaction to Earnings News}},
		224	+ journal = {The Journal of Finance},
		225	+ year = {2009},
		226	+ volume = {64},
		227	+ number = {5},
		228	+ pages = {2289--2325},
		229	+ doi = {10.1111/j.1540-6261.2009.01501.x}
		230	+}
		231	+
		232	+@article{dellavigna2009,
		233	+ author = {DELLAVIGNA, STEFANO and POLLET, JOSHUA M.},
		234	+ title = {{Investor Inattention and Friday Earnings Announcements}},
		235	+ journal = {The Journal of Finance},
		236	+ year = {2009},
		237	+ volume = {64},
		238	+ number = {2},
		239	+ pages = {709--749},
		240	+ doi = {10.1111/j.1540-6261.2009.01447.x}
		241	+}
		242	+
		243	+@article{loughran2011,
		244	+ author = {LOUGHRAN, TIM and MCDONALD, BILL},
		245	+ title = {{When Is a Liability Not a Liability? Textual Analysis, Dictionaries, and 10-Ks}},
		246	+ journal = {The Journal of Finance},
		247	+ year = {2011},
		248	+ volume = {66},
		249	+ number = {1},
		250	+ pages = {35--65},
		251	+ doi = {10.1111/j.1540-6261.2010.01625.x}
		252	+}
		253	+
		254	+@article{bajgrowicz2016,
		255	+ author = {Bajgrowicz, Pierre and Scaillet, Olivier and Treccani, Adrien},
		256	+ title = {{Jumps in High-Frequency Data: Spurious Detections, Dynamics, and News}},
		257	+ journal = {Management Science},
		258	+ year = {2016},
		259	+ volume = {62},
		260	+ number = {8},
		261	+ pages = {2198--2217},
		262	+ doi = {10.1287/mnsc.2015.2234}
		263	+}
		264	+
		265	+@article{lee2008,
		266	+ author = {Lee, Suzanne S. and Mykland, Per A.},
		267	+ title = {{Jumps in Financial Markets: A New Nonparametric Test and Jump Dynamics}},
		268	+ journal = {Review of Financial Studies},
		269	+ year = {2008},
		270	+ volume = {21},
		271	+ number = {6},
		272	+ pages = {2535--2563},
		273	+ doi = {10.1093/rfs/hhm056}
		274	+}
		275	+
		276	+@article{bollerslev2018risk,
		277	+ author = {Bollerslev, Tim and Hood, Benjamin and Huss, John and Pedersen, Lasse Heje},
		278	+ title = {{Risk Everywhere: Modeling and Managing Volatility}},
		279	+ journal = {The Review of Financial Studies},
		280	+ year = {2018},
		281	+ volume = {31},
		282	+ number = {7},
		283	+ pages = {2729--2773},
		284	+ doi = {10.1093/rfs/hhy041}
		285	+}
		286	+
		287	+@article{hendershott2011,
		288	+ author = {HENDERSHOTT, TERRENCE and JONES, CHARLES M. and MENKVELD, ALBERT J.},
		289	+ title = {{Does Algorithmic Trading Improve Liquidity?}},
		290	+ journal = {The Journal of Finance},
		291	+ year = {2011},
		292	+ volume = {66},
		293	+ number = {1},
		294	+ pages = {1--33},
		295	+ doi = {10.1111/j.1540-6261.2010.01624.x}
		296	+}
		297	+
		298	+@article{richardson2005,
		299	+ author = {Richardson, Scott A. and Sloan, Richard G. and Soliman, Mark T. and Tuna, İrem},
		300	+ title = {{Accrual reliability, earnings persistence and stock prices}},
		301	+ journal = {Journal of Accounting and Economics},
		302	+ year = {2005},
		303	+ volume = {39},
		304	+ number = {3},
		305	+ pages = {437--485},
		306	+ doi = {10.1016/j.jacceco.2005.04.005}
		307	+}
		308	+
		309	+@article{beneish1999,
		310	+ author = {Beneish, Messod D.},
		311	+ title = {{The Detection of Earnings Manipulation}},
		312	+ journal = {Financial Analysts Journal},
		313	+ year = {1999},
		314	+ volume = {55},
		315	+ number = {5},
		316	+ pages = {24--36},
		317	+ doi = {10.2469/faj.v55.n5.2296}
		318	+}
		319	+
		320	+@article{campbell2008,
		321	+ author = {CAMPBELL, JOHN Y. and HILSCHER, JENS and SZILAGYI, JAN},
		322	+ title = {{In Search of Distress Risk}},
		323	+ journal = {The Journal of Finance},
		324	+ year = {2008},
		325	+ volume = {63},
		326	+ number = {6},
		327	+ pages = {2899--2939},
		328	+ doi = {10.1111/j.1540-6261.2008.01416.x}
		329	+}
		330	+
		331	+@article{fama1973,
		332	+ author = {Fama, Eugene F. and MacBeth, James D.},
		333	+ title = {{Risk, Return, and Equilibrium: Empirical Tests}},
		334	+ journal = {Journal of Political Economy},
		335	+ year = {1973},
		336	+ volume = {81},
		337	+ number = {3},
		338	+ pages = {607--636},
		339	+ doi = {10.1086/260061}
		340	+}
		341	+
		342	+@article{petersen2009,
		343	+ author = {Petersen, Mitchell A.},
		344	+ title = {{Estimating Standard Errors in Finance Panel Data Sets: Comparing Approaches}},
		345	+ journal = {Review of Financial Studies},
		346	+ year = {2009},
		347	+ volume = {22},
		348	+ number = {1},
		349	+ pages = {435--480},
		350	+ doi = {10.1093/rfs/hhn053}
		351	+}
		352	+
		353	+@article{cameron2011,
		354	+ author = {Cameron, A. Colin and Gelbach, Jonah B. and Miller, Douglas L.},
		355	+ title = {{Robust Inference With Multiway Clustering}},
		356	+ journal = {Journal of Business \& Economic Statistics},
		357	+ year = {2011},
		358	+ volume = {29},
		359	+ number = {2},
		360	+ pages = {238--249},
		361	+ doi = {10.1198/jbes.2010.07136}
		362	+}
		363	+
		364	+@article{novymarx2016,
		365	+ author = {Novy-Marx, Robert and Velikov, Mihail},
		366	+ title = {{A Taxonomy of Anomalies and Their Trading Costs}},
		367	+ journal = {Review of Financial Studies},
		368	+ year = {2016},
		369	+ volume = {29},
		370	+ number = {1},
		371	+ pages = {104--147},
		372	+ doi = {10.1093/rfs/hhv063}
		373	+}
		374	+
		375	+@article{bollerslev2016beta,
		376	+ author = {Bollerslev, Tim and Li, Sophia Zhengzi and Todorov, Viktor},
		377	+ title = {{Roughing up beta: Continuous versus discontinuous betas and the cross section of expected stock returns}},
		378	+ journal = {Journal of Financial Economics},
		379	+ year = {2016},
		380	+ volume = {120},
		381	+ number = {3},
		382	+ pages = {464--490},
		383	+ doi = {10.1016/j.jfineco.2016.02.001}
		384	+}
		385	+
		386	+@article{lo1988,
		387	+ author = {Lo, Andrew W. and MacKinlay, A. Craig},
		388	+ title = {{Stock Market Prices Do Not Follow Random Walks: Evidence from a Simple Specification Test}},
		389	+ journal = {Review of Financial Studies},
		390	+ year = {1988},
		391	+ volume = {1},
		392	+ number = {1},
		393	+ pages = {41--66},
		394	+ doi = {10.1093/rfs/1.1.41}
		395	+}
		396	+
		397	+@article{freyberger2020,
		398	+ author = {Freyberger, Joachim and Neuhierl, Andreas and Weber, Michael},
		399	+ title = {{Dissecting Characteristics Nonparametrically}},
		400	+ journal = {The Review of Financial Studies},
		401	+ year = {2020},
		402	+ volume = {33},
		403	+ number = {5},
		404	+ pages = {2326--2377},
		405	+ doi = {10.1093/rfs/hhz123}
		406	+}
		407	+
		408	+@article{bollerslev2016errors,
		409	+ author = {Bollerslev, Tim and Patton, Andrew J. and Quaedvlieg, Rogier},
		410	+ title = {{Exploiting the errors: A simple approach for improved volatility forecasting}},
		411	+ journal = {Journal of Econometrics},
		412	+ year = {2016},
		413	+ volume = {192},
		414	+ number = {1},
		415	+ pages = {1--18},
		416	+ doi = {10.1016/j.jeconom.2015.10.007}
		417	+}
		418	+
		419	+@article{liu2015,
		420	+ author = {Liu, Lily Y. and Patton, Andrew J. and Sheppard, Kevin},
		421	+ title = {{Does anything beat 5-minute RV? A comparison of realized measures across multiple asset classes}},
		422	+ journal = {Journal of Econometrics},
		423	+ year = {2015},
		424	+ volume = {187},
		425	+ number = {1},
		426	+ pages = {293--311},
		427	+ doi = {10.1016/j.jeconom.2015.02.008}
		428	+}
		429	+
		430	+@article{andersen1998,
		431	+ author = {Andersen, Torben G. and Bollerslev, Tim},
		432	+ title = {{Answering the Skeptics: Yes, Standard Volatility Models do Provide Accurate Forecasts}},
		433	+ journal = {International Economic Review},
		434	+ year = {1998},
		435	+ volume = {39},
		436	+ number = {4},
		437	+ pages = {885},
		438	+ doi = {10.2307/2527343}
		439	+}
		440	+
		441	+@article{aboody2010,
		442	+ author = {Aboody, David and Lehavy, Reuven and Trueman, Brett},
		443	+ title = {{Limited attention and the earnings announcement returns of past stock market winners}},
		444	+ journal = {Review of Accounting Studies},
		445	+ year = {2010},
		446	+ volume = {15},
		447	+ number = {2},
		448	+ pages = {317--344},
		449	+ doi = {10.1007/s11142-009-9104-9}
		450	+}
		451	+
		452	+@article{kothari2001,
		453	+ author = {Kothari, S.P},
		454	+ title = {{Capital markets research in accounting}},
		455	+ journal = {Journal of Accounting and Economics},
		456	+ year = {2001},
		457	+ volume = {31},
		458	+ number = {1-3},
		459	+ pages = {105--231},
		460	+ doi = {10.1016/s0165-4101(01)00030-1}
		461	+}
		462	+
		463	+@article{easley2012,
		464	+ author = {Easley, David and López de Prado, Marcos M. and O'Hara, Maureen},
		465	+ title = {{Flow Toxicity and Liquidity in a High-frequency World}},
		466	+ journal = {Review of Financial Studies},
		467	+ year = {2012},
		468	+ volume = {25},
		469	+ number = {5},
		470	+ pages = {1457--1493},
		471	+ doi = {10.1093/rfs/hhs053}
		472	+}
		473	+
		474	+@article{hou2015,
		475	+ author = {Hou, Kewei and Xue, Chen and Zhang, Lu},
		476	+ title = {{Digesting Anomalies: An Investment Approach}},
		477	+ journal = {Review of Financial Studies},
		478	+ year = {2015},
		479	+ volume = {28},
		480	+ number = {3},
		481	+ pages = {650--705},
		482	+ doi = {10.1093/rfs/hhu068}
		483	+}
		484	+
		485	+@article{dechow2010,
		486	+ author = {Dechow, Patricia and Ge, Weili and Schrand, Catherine},
		487	+ title = {{Understanding earnings quality: A review of the proxies, their determinants and their consequences}},
		488	+ journal = {Journal of Accounting and Economics},
		489	+ year = {2010},
		490	+ volume = {50},
		491	+ number = {2-3},
		492	+ pages = {344--401},
		493	+ doi = {10.1016/j.jacceco.2010.09.001}
		494	+}
		495	+
		496	+@article{fama1970,
		497	+ author = {Fama, Eugene F.},
		498	+ title = {{Efficient Capital Markets: A Review of Theory and Empirical Work}},
		499	+ journal = {The Journal of Finance},
		500	+ year = {1970},
		501	+ volume = {25},
		502	+ number = {2},
		503	+ pages = {383},
		504	+ doi = {10.2307/2325486}
		505	+}
		506	+
		507	+@article{bollerslev2011tails,
		508	+ author = {BOLLERSLEV, TIM and TODOROV, VIKTOR},
		509	+ title = {{Tails, Fears, and Risk Premia}},
		510	+ journal = {The Journal of Finance},
		511	+ year = {2011},
		512	+ volume = {66},
		513	+ number = {6},
		514	+ pages = {2165--2211},
		515	+ doi = {10.1111/j.1540-6261.2011.01695.x}
		516	+}
		517	+
		518	+@article{hirshleifer2003,
		519	+ author = {Hirshleifer, David and Teoh, Siew Hong},
		520	+ title = {{Limited attention, information disclosure, and financial reporting}},
		521	+ journal = {Journal of Accounting and Economics},
		522	+ year = {2003},
		523	+ volume = {36},
		524	+ number = {1-3},
		525	+ pages = {337--386},
		526	+ doi = {10.1016/j.jacceco.2003.10.002}
		527	+}
		528	+
		529	+@article{savor2016,
		530	+ author = {SAVOR, PAVEL and WILSON, MUNGO},
		531	+ title = {{Earnings Announcements and Systematic Risk}},
		532	+ journal = {The Journal of Finance},
		533	+ year = {2016},
		534	+ volume = {71},
		535	+ number = {1},
		536	+ pages = {83--138},
		537	+ doi = {10.1111/jofi.12361}
		538	+}
		539	+
		540	+@article{budish2015,
		541	+ author = {Budish, Eric and Cramton, Peter and Shim, John},
		542	+ title = {{The High-Frequency Trading Arms Race: Frequent Batch Auctions as a Market Design Response *}},
		543	+ journal = {The Quarterly Journal of Economics},
		544	+ year = {2015},
		545	+ volume = {130},
		546	+ number = {4},
		547	+ pages = {1547--1621},
		548	+ doi = {10.1093/qje/qjv027}
		549	+}
		550	+
		551	+@article{chen2024deep,
		552	+ author = {Chen, Luyang and Pelger, Markus and Zhu, Jason},
		553	+ title = {{Deep Learning in Asset Pricing}},
		554	+ journal = {Management Science},
		555	+ year = {2024},
		556	+ volume = {70},
		557	+ number = {2},
		558	+ pages = {714--750},
		559	+ doi = {10.1287/mnsc.2023.4695}
		560	+}
		561	+
		562	+@article{kozak2020,
		563	+ author = {Kozak, Serhiy and Nagel, Stefan and Santosh, Shrihari},
		564	+ title = {{Shrinking the cross-section}},
		565	+ journal = {Journal of Financial Economics},
		566	+ year = {2020},
		567	+ volume = {135},
		568	+ number = {2},
		569	+ pages = {271--292},
		570	+ doi = {10.1016/j.jfineco.2019.06.008}
		571	+}
		572	+
		573	+@article{brogaard2019,
		574	+ author = {BROGAARD, JONATHAN and HENDERSHOTT, TERRENCE and RIORDAN, RYAN},
		575	+ title = {{Price Discovery without Trading: Evidence from Limit Orders}},
		576	+ journal = {The Journal of Finance},
		577	+ year = {2019},
		578	+ volume = {74},
		579	+ number = {4},
		580	+ pages = {1621--1658},
		581	+ doi = {10.1111/jofi.12769}
		582	+}
		583	+
		584	+@article{ang2006,
		585	+ author = {ANG, ANDREW and HODRICK, ROBERT J. and XING, YUHANG and ZHANG, XIAOYAN},
		586	+ title = {{The Cross-Section of Volatility and Expected Returns}},
		587	+ journal = {The Journal of Finance},
		588	+ year = {2006},
		589	+ volume = {61},
		590	+ number = {1},
		591	+ pages = {259--299},
		592	+ doi = {10.1111/j.1540-6261.2006.00836.x}
		593	+}
		594	+
		595	+@article{skinner2002,
		596	+ author = {Skinner, Douglas J. and Sloan, Richard G.},
		597	+ title = {{Earnings Surprises, Growth Expectations, and Stock Returns or Don't Let an Earnings Torpedo Sink Your Portfolio}},
		598	+ journal = {Review of Accounting Studies},
		599	+ year = {2002},
		600	+ volume = {7},
		601	+ number = {2-3},
		602	+ pages = {289--312},
		603	+ doi = {10.1023/a:1020294523516}
		604	+}
		605	+
		606	+@article{levi2015,
		607	+ author = {Levi, Shai and Zhang, Xiao-Jun},
		608	+ title = {{Asymmetric decrease in liquidity trading before earnings announcements and the announcement return premium}},
		609	+ journal = {Journal of Financial Economics},
		610	+ year = {2015},
		611	+ volume = {118},
		612	+ number = {2},
		613	+ pages = {383--398},
		614	+ doi = {10.1016/j.jfineco.2015.08.003}
		615	+}
		616	+
		617	+@article{campbell1993,
		618	+ author = {Campbell, J. Y. and Grossman, S. J. and Wang, J.},
		619	+ title = {{Trading Volume and Serial Correlation in Stock Returns}},
		620	+ journal = {The Quarterly Journal of Economics},
		621	+ year = {1993},
		622	+ volume = {108},
		623	+ number = {4},
		624	+ pages = {905--939},
		625	+ doi = {10.2307/2118454}
		626	+}
		627	+
		628	+@article{fang2009,
		629	+ author = {FANG, LILY and PERESS, JOEL},
		630	+ title = {{Media Coverage and the Cross-section of Stock Returns}},
		631	+ journal = {The Journal of Finance},
		632	+ year = {2009},
		633	+ volume = {64},
		634	+ number = {5},
		635	+ pages = {2023--2052},
		636	+ doi = {10.1111/j.1540-6261.2009.01493.x}
		637	+}
		638	+
		639	+@article{beaver1980,
		640	+ author = {Beaver, William and Lambert, Richard and Morse, Dale},
		641	+ title = {{The information content of security prices}},
		642	+ journal = {Journal of Accounting and Economics},
		643	+ year = {1980},
		644	+ volume = {2},
		645	+ number = {1},
		646	+ pages = {3--28},
		647	+ doi = {10.1016/0165-4101(80)90013-0}
		648	+}
		649	+
		650	+@article{kothari1992,
		651	+ author = {Kothari, S.P and Sloan, Richard G},
		652	+ title = {{Information in prices about future earnings}},
		653	+ journal = {Journal of Accounting and Economics},
		654	+ year = {1992},
		655	+ volume = {15},
		656	+ number = {2-3},
		657	+ pages = {143--171},
		658	+ doi = {10.1016/0165-4101(92)90016-u}
		659	+}
		660	+
		661	+@article{collins1994,
		662	+ author = {Collins, Daniel W. and Kothari, S.P. and Shanken, Jay and Sloan, Richard G.},
		663	+ title = {{Lack of timeliness and noise as explanations for the low contemporaneuos return-earnings association}},
		664	+ journal = {Journal of Accounting and Economics},
		665	+ year = {1994},
		666	+ volume = {18},
		667	+ number = {3},
		668	+ pages = {289--324},
		669	+ doi = {10.1016/0165-4101(94)90024-8}
		670	+}
		671	+
		672	+@article{foster1977,
		673	+ author = {Foster, George},
		674	+ title = {{Quarterly Accounting Data: Time-Series Properties and Predicative-Ability Results.}},
		675	+ journal = {The Accounting Review},
		676	+ year = {1977},
		677	+ volume = {52},
		678	+ number = {1},
		679	+ pages = {1--21},
		680	+ doi = {10.2308/tar-4493420}
		681	+}
		682	+
		683	+@article{weller2018,
		684	+ author = {Weller, Brian M.},
		685	+ title = {{Does Algorithmic Trading Reduce Information Acquisition?}},
		686	+ journal = {The Review of Financial Studies},
		687	+ year = {2018},
		688	+ volume = {31},
		689	+ number = {6},
		690	+ pages = {2184--2226},
		691	+ doi = {10.1093/rfs/hhx137}
		692	+}
		693	+
		694	+@article{bollerslev2018volume,
		695	+ author = {Bollerslev, Tim and Li, Jia and Xue, Yuan},
		696	+ title = {{Volume, Volatility, and Public News Announcements}},
		697	+ journal = {The Review of Economic Studies},
		698	+ year = {2018},
		699	+ volume = {85},
		700	+ number = {4},
		701	+ pages = {2005--2041},
		702	+ doi = {10.1093/restud/rdy003}
		703	+}
		704	+
		705	+@article{patton2012,
		706	+ author = {Patton, Andrew J. and Verardo, Michela},
		707	+ title = {{Does Beta Move with News? Firm-Specific Information Flows and Learning about Profitability}},
		708	+ journal = {Review of Financial Studies},
		709	+ year = {2012},
		710	+ volume = {25},
		711	+ number = {9},
		712	+ pages = {2789--2839},
		713	+ doi = {10.1093/rfs/hhs073}
		714	+}
		715	+
		716	+@article{kaniel2012,
		717	+ author = {KANIEL, RON and LIU, SHUMING and SAAR, GIDEON and TITMAN, SHERIDAN},
		718	+ title = {{Individual Investor Trading and Return Patterns around Earnings Announcements}},
		719	+ journal = {The Journal of Finance},
		720	+ year = {2012},
		721	+ volume = {67},
		722	+ number = {2},
		723	+ pages = {639--680},
		724	+ doi = {10.1111/j.1540-6261.2012.01727.x}
		725	+}
		726	+
		727	+@article{christophe2004,
		728	+ author = {Christophe, Stephen E. and Ferri, Michael G. and Angel, James J.},
		729	+ title = {{Short-Selling Prior to Earnings Announcements}},
		730	+ journal = {The Journal of Finance},
		731	+ year = {2004},
		732	+ volume = {59},
		733	+ number = {4},
		734	+ pages = {1845--1876},
		735	+ doi = {10.1111/j.1540-6261.2004.00681.x}
		736	+}
		737	+
		738	+@article{berkman2009,
		739	+ author = {Berkman, Henk and Dimitrov, Valentin and Jain, Prem C. and Koch, Paul D. and Tice, Sheri},
		740	+ title = {{Sell on the news: Differences of opinion, short-sales constraints, and returns around earnings announcements☆}},
		741	+ journal = {Journal of Financial Economics},
		742	+ year = {2009},
		743	+ volume = {92},
		744	+ number = {3},
		745	+ pages = {376--399},
		746	+ doi = {10.1016/j.jfineco.2008.04.009}
		747	+}
		748	+
		749	+@article{hendershott2015,
		750	+ author = {Hendershott, Terrence and Livdan, Dmitry and Schürhoff, Norman},
		751	+ title = {{Are institutions informed about news?}},
		752	+ journal = {Journal of Financial Economics},
		753	+ year = {2015},
		754	+ volume = {117},
		755	+ number = {2},
		756	+ pages = {249--287},
		757	+ doi = {10.1016/j.jfineco.2015.03.007}
		758	+}
		759	+
		760	+@article{lee2012,
		761	+ author = {Lee, Suzanne S.},
		762	+ title = {{Jumps and Information Flow in Financial Markets}},
		763	+ journal = {Review of Financial Studies},
		764	+ year = {2012},
		765	+ volume = {25},
		766	+ number = {2},
		767	+ pages = {439--479},
		768	+ doi = {10.1093/rfs/hhr084}
		769	+}
		770	+
		771	+@article{cao2024,
		772	+ author = {Cao, Sean and Jiang, Wei and Wang, Junbo and Yang, Baozhong},
		773	+ title = {{From Man vs. Machine to Man + Machine: The art and AI of stock analyses}},
		774	+ journal = {Journal of Financial Economics},
		775	+ year = {2024},
		776	+ volume = {160},
		777	+ pages = {103910},
		778	+ doi = {10.1016/j.jfineco.2024.103910}
		779	+}
		780	+
		781	+@article{bali2023,
		782	+ author = {Bali, Turan G and Beckmeyer, Heiner and Mörke, Mathis and Weigert, Florian},
		783	+ title = {{Option Return Predictability with Machine Learning and Big Data}},
		784	+ journal = {The Review of Financial Studies},
		785	+ year = {2023},
		786	+ volume = {36},
		787	+ number = {9},
		788	+ pages = {3548--3602},
		789	+ doi = {10.1093/rfs/hhad017}
		790	+}
		791	+
		792	+@article{easton2024,
		793	+ author = {Easton, Peter D. and Kapons, Martin M. and Monahan, Steven J. and Schütt, Harm H. and Weisbrod, Eric H.},
		794	+ title = {{Forecasting Earnings Using \emph{k}-Nearest Neighbors}},
		795	+ journal = {The Accounting Review},
		796	+ year = {2024},
		797	+ volume = {99},
		798	+ number = {3},
		799	+ pages = {115--140},
		800	+ doi = {10.2308/tar-2021-0478}
		801	+}
		802	+
		803	+@article{so2013,
		804	+ author = {So, Eric C.},
		805	+ title = {{A new approach to predicting analyst forecast errors: Do investors overweight analyst forecasts?}},
		806	+ journal = {Journal of Financial Economics},
		807	+ year = {2013},
		808	+ volume = {108},
		809	+ number = {3},
		810	+ pages = {615--640},
		811	+ doi = {10.1016/j.jfineco.2013.02.002}
		812	+}
		813	+
		814	+@article{hou2012,
		815	+ author = {Hou, Kewei and van Dijk, Mathijs A. and Zhang, Yinglei},
		816	+ title = {{The implied cost of capital: A new approach}},
		817	+ journal = {Journal of Accounting and Economics},
		818	+ year = {2012},
		819	+ volume = {53},
		820	+ number = {3},
		821	+ pages = {504--526},
		822	+ doi = {10.1016/j.jacceco.2011.12.001}
		823	+}
		824	+
		825	+@article{hansen2006,
		826	+ author = {Hansen, Peter R and Lunde, Asger},
		827	+ title = {{Realized Variance and Market Microstructure Noise}},
		828	+ journal = {Journal of Business \& Economic Statistics},
		829	+ year = {2006},
		830	+ volume = {24},
		831	+ number = {2},
		832	+ pages = {127--161},
		833	+ doi = {10.1198/073500106000000071}
		834	+}
		835	+
		836	+@article{barndorff2008,
		837	+ author = {Barndorff-Nielsen, Ole E. and Hansen, Peter Reinhard and Lunde, Asger and Shephard, Neil},
		838	+ title = {{Designing Realized Kernels to Measure the ex post Variation of Equity Prices in the Presence of Noise}},
		839	+ journal = {Econometrica},
		840	+ year = {2008},
		841	+ volume = {76},
		842	+ number = {6},
		843	+ pages = {1481--1536},
		844	+ doi = {10.3982/ecta6495}
		845	+}
		846	+
		847	+@article{vankervel2019,
		848	+ author = {VAN KERVEL, VINCENT and MENKVELD, ALBERT J.},
		849	+ title = {{High-Frequency Trading around Large Institutional Orders}},
		850	+ journal = {The Journal of Finance},
		851	+ year = {2019},
		852	+ volume = {74},
		853	+ number = {3},
		854	+ pages = {1091--1137},
		855	+ doi = {10.1111/jofi.12759}
		856	+}
		857	+
		858	+@article{avramov2023,
		859	+ author = {Avramov, Doron and Cheng, Si and Metzker, Lior},
		860	+ title = {{Machine Learning vs. Economic Restrictions: Evidence from Stock Return Predictability}},
		861	+ journal = {Management Science},
		862	+ year = {2023},
		863	+ volume = {69},
		864	+ number = {5},
		865	+ pages = {2587--2619},
		866	+ doi = {10.1287/mnsc.2022.4449}
		867	+}
		868	+
		869	+@article{chordia2020,
		870	+ author = {Chordia, Tarun and Goyal, Amit and Saretto, Alessio},
		871	+ title = {{Anomalies and False Rejections}},
		872	+ journal = {The Review of Financial Studies},
		873	+ year = {2020},
		874	+ volume = {33},
		875	+ number = {5},
		876	+ pages = {2134--2179},
		877	+ doi = {10.1093/rfs/hhaa018}
		878	+}
		879	+
		880	+@article{barber2013,
		881	+ author = {Barber, Brad M. and De George, Emmanuel T. and Lehavy, Reuven and Trueman, Brett},
		882	+ title = {{The earnings announcement premium around the globe}},
		883	+ journal = {Journal of Financial Economics},
		884	+ year = {2013},
		885	+ volume = {108},
		886	+ number = {1},
		887	+ pages = {118--138},
		888	+ doi = {10.1016/j.jfineco.2012.10.006}
		889	+}
		890	+
		891	+@article{politis1994,
		892	+ author = {Politis, Dimitris N. and Romano, Joseph P.},
		893	+ title = {{The Stationary Bootstrap}},
		894	+ journal = {Journal of the American Statistical Association},
		895	+ year = {1994},
		896	+ volume = {89},
		897	+ number = {428},
		898	+ pages = {1303--1313},
		899	+ doi = {10.1080/01621459.1994.10476870}
		900	+}
		901	+
		902	+@article{hansen2011mcs,
		903	+ author = {Hansen, Peter R. and Lunde, Asger and Nason, James M.},
		904	+ title = {{The Model Confidence Set}},
		905	+ journal = {Econometrica},
		906	+ year = {2011},
		907	+ volume = {79},
		908	+ number = {2},
		909	+ pages = {453--497},
		910	+ doi = {10.3982/ecta5771}
		911	+}
		912	+
		913	+@article{benjamini1995,
		914	+ author = {Benjamini, Yoav and Hochberg, Yosef},
		915	+ title = {{Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing}},
		916	+ journal = {Journal of the Royal Statistical Society Series B: Statistical Methodology},
		917	+ year = {1995},
		918	+ volume = {57},
		919	+ number = {1},
		920	+ pages = {289--300},
		921	+ doi = {10.1111/j.2517-6161.1995.tb02031.x}
		922	+}
		923	+
		924	+@article{amihud2002,
		925	+ author = {Amihud, Yakov},
		926	+ title = {{Illiquidity and stock returns: cross-section and time-series effects}},
		927	+ journal = {Journal of Financial Markets},
		928	+ year = {2002},
		929	+ volume = {5},
		930	+ number = {1},
		931	+ pages = {31--56},
		932	+ doi = {10.1016/s1386-4181(01)00024-6}
		933	+}
		934	+
		935	+@article{fama1993,
		936	+ author = {Fama, Eugene F. and French, Kenneth R.},
		937	+ title = {{Common risk factors in the returns on stocks and bonds}},
		938	+ journal = {Journal of Financial Economics},
		939	+ year = {1993},
		940	+ volume = {33},
		941	+ number = {1},
		942	+ pages = {3--56},
		943	+ doi = {10.1016/0304-405x(93)90023-5}
		944	+}
		945	+
		946	+@article{grossman1980,
		947	+ author = {Grossman, Sanford J. and Stiglitz, Joseph E.},
		948	+ title = {{On the Impossibility of Informationally Efficient Markets}},
		949	+ journal = {American Economic Review},
		950	+ year = {1980},
		951	+ volume = {70},
		952	+ number = {3},
		953	+ pages = {393--408}
		954	+}
		955	+@article{holm1979,
		956	+ author = {Holm, Sture},
		957	+ title = {{A Simple Sequentially Rejective Multiple Test Procedure}},
		958	+ journal = {Scandinavian Journal of Statistics},
		959	+ year = {1979},
		960	+ volume = {6},
		961	+ number = {2},
		962	+ pages = {65--70}
		963	+}
		964	+@article{kunsch1989,
		965	+ author = {K{\"u}nsch, Hans R.},
		966	+ title = {{The Jackknife and the Bootstrap for General Stationary Observations}},
		967	+ journal = {Annals of Statistics},
		968	+ year = {1989},
		969	+ volume = {17},
		970	+ number = {3},
		971	+ pages = {1217--1241},
		972	+ doi = {10.1214/aos/1176347265}
		973	+}
		974	+@inproceedings{ke2017lightgbm,
		975	+ author = {Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan},
		976	+ title = {{LightGBM: A Highly Efficient Gradient Boosting Decision Tree}},
		977	+ booktitle = {Advances in Neural Information Processing Systems 30},
		978	+ year = {2017},
		979	+ pages = {3146--3154}
		980	+}
		981	+@book{lopezdeprado2018,
		982	+ author = {L{\'o}pez de Prado, Marcos},
		983	+ title = {{Advances in Financial Machine Learning}},
		984	+ publisher = {Wiley},
		985	+ address = {Hoboken, NJ},
		986	+ year = {2018}
		987	+}
		988	+@techreport{frazzini2018costs,
		989	+ author = {Frazzini, Andrea and Israel, Ronen and Moskowitz, Tobias J.},
		990	+ title = {{Trading Costs}},
		991	+ institution = {SSRN Working Paper 3229719},
		992	+ year = {2018},
		993	+ doi = {10.2139/ssrn.3229719}
		994	+}
		995	+@incollection{barndorff2010semivariance,
		996	+ author = {Barndorff-Nielsen, Ole E. and Kinnebrock, Silja and Shephard, Neil},
		997	+ title = {{Measuring Downside Risk: Realised Semivariance}},
		998	+ booktitle = {Volatility and Time Series Econometrics: Essays in Honor of Robert F. Engle},
		999	+ editor = {Bollerslev, Tim and Russell, Jeffrey and Watson, Mark},
		1000	+ publisher = {Oxford University Press},
		1001	+ year = {2010},
		1002	+ pages = {117--136},
		1003	+ doi = {10.1093/acprof:oso/9780199549498.003.0007}
		1004	+}
		1005	+@article{boucher2026wp15,
		1006	+ author = {Boucher, Simon-Pierre},
		1007	+ title = {{Faster, Wider, More Fragile? The Evolution of Intraday Shock Propagation in U.S. Equity Markets, 1992--2026}},
		1008	+ journal = {UQO Working Paper No. 15},
		1009	+ year = {2026}
		1010	+}
		1011	+
		1012	+@article{harvey2016,
		1013	+ author = {Harvey, Campbell R. and Liu, Yan and Zhu, Heqing},
		1014	+ title = {{\ldots and the Cross-Section of Expected Returns}},
		1015	+ journal = {The Review of Financial Studies},
		1016	+ year = {2016},
		1017	+ volume = {29},
		1018	+ number = {1},
		1019	+ pages = {5--68},
		1020	+ doi = {10.1093/rfs/hhv059}
		1021	+}

added paper/sections/data.tex +83 −0

				@@ -0,0 +1,83 @@
		1	+\section{Data}
		2	+\label{sec:data}
		3	+
		4	+\subsection{Source and license}
		5	+
		6	+All market and accounting data come from FinData OS, a private lakehouse of FirstRate Data operated by the author and
		7	+accessed through its versioned API (\texttt{findata.spboucher.ai/api/v1}). The lake holds 6.7 billion raw 1-minute
		8	+bars for 16,195 U.S. stock symbols (2000--2026, including 8,069 delisted symbols), quarterly fundamentals for
		9	+\nEntities{} entities (67 metrics, filing dates 2012--2026) and 373,326 earnings-release events. Bars are timestamped
		10	+at the bar open in UTC and cover 04:00--20:00 Eastern; zero-volume minutes are absent at the source and are never
		11	+imputed. FirstRate Data is licensed for private research use; no raw record is redistributed, and the replication
		12	+package contains code, hashed extraction manifests (dataset identifiers and versions, row counts, SHA-256 digests) and
		13	+derived aggregates only. Public augmentations are the SEC's \texttt{company\_tickers.json} and \texttt{submissions}
		14	+files (CIK, SIC industry), and the Kenneth French daily factor library.
		15	+
		16	+\subsection{Universe and survivorship}
		17	+
		18	+The accounting panel contains the \nEntities{} entities for which the vendor supplies quarterly statements; it has
		19	+\emph{no delisted entity}, because the vendor ingestion covered tickers active on 1 November 2026. The price side is
		20	+survivorship-free (delisted names are present in the bar lake), but the firm-quarter sample is conditioned on survival
		21	+to the end of the sample. This is a structural caveat: firms that deteriorated and delisted are absent, so adverse
		22	+base rates are understated and the most vulnerable firms are under-represented. We state it in the abstract and
		23	+treat it as the first limitation. Table~\ref{tab:universe} reports the construction of the universe: \nEligible{}
		24	+firms with at least eight quarterly periods, at least 250 sessions of 1-minute bars after 2012 and a non-governmental
		25	+SIC division; \nWithCik{} are matched to a CIK. Identifiers are vendor tickers; the vendor merges ticker changes under
		26	+the latest ticker, and we audit reuse with the SEC's former-name history.
		27	+
		28	+\subsection{Reveal dates and point-in-time discipline}
		29	+
		30	+Earnings-release dates are matched to fiscal quarters as the first release in $[E_q, F_q+10\text{ days}]$;
		31	+\releaseMatchedShare\% of firm-quarters are matched with a filing lag of at most 30 days (median lag between release
		32	+and filing: \filingLagMedian{} day). Release timing is known (before-open or after-close) for about two thirds of events.
		33	+Structured facts carry \texttt{filing\_date}, \texttt{available\_at} ($=$ filing date $+1$ day, a documented policy
		34	+rather than a historical vintage) and a revision identifier; because every fact was first ingested in November 2026 and
		35	+carries revision~1, restatements that occurred before ingestion are not reconstructible. Inputs therefore use prior
		36	+quarters only, outcomes use the vendor's current value of quarter $q$, and the stringent as-originally-reported
		37	+specification is left for a future SEC-filing-level reconstruction (Section~\ref{sec:limitations}).
		38	+
		39	+\subsection{Sample}
		40	+
		41	+The quarter panel has \nFirmQuarters{} firm-quarters for \nFirmsPanel{} firms. After the admissibility and exclusion
		42	+cascade of Table~\ref{tab:cascade} (market window of at least 15 valid sessions, raw close of at least one dollar,
		43	+available fundamentals, prediction of the next quarter, defined expectation, regular fiscal calendar), the event-time
		44	+sample has \nSampleEvent{} observations across the five horizons (\nSampleFirms{} firms, \nSampleQuarters{} firm-quarters)
		45	+and the primary $h=20$ cohort has \nObsPrimary{} observations with cutoffs from \firstCutoff{} to \lastCutoff; the
		46	+fixed-calendar sample has \nSampleCalendar{} firm-months. The adverse outcome occurs in \baseRatePrimary\% of
		47	+primary-cohort firm-quarters. Table~\ref{tab:summary} reports summary statistics and Figure~\ref{fig:coverage} the
		48	+coverage by year together with the adverse rate.
		49	+
		50	+\begin{table}[H]
		51	+\centering\footnotesize
		52	+\caption{Construction of the universe and of the primary sample}
		53	+\label{tab:universe}
		54	+\begin{minipage}[t]{0.42\textwidth}\centering
		55	+\input{tables/tab02_universe}
		56	+\end{minipage}\hfill
		57	+\begin{minipage}[t]{0.56\textwidth}\centering
		58	+\label{tab:cascade}
		59	+\input{tables/tab02b_sample_cascade}
		60	+\end{minipage}
		61	+\begin{minipage}{\textwidth}\vspace{0.3em}\footnotesize
		62	+\emph{Notes.} Left: firms. Right: event-time observations (five horizons pooled); steps are applied in the frozen order
		63	+of \texttt{docs/PIT\_PROTOCOL.md}.
		64	+\end{minipage}
		65	+\end{table}
		66	+
		67	+\begin{table}[H]
		68	+\centering\footnotesize
		69	+\caption{Summary statistics, primary cohort ($h=20$)}
		70	+\label{tab:summary}
		71	+\adjustbox{max width=\textwidth}{\input{tables/tab03_summary}}
		72	+\begin{minipage}{\textwidth}\vspace{0.3em}\footnotesize
		73	+\emph{Notes.} Intraday families are daily values averaged over the 20-session window ending at the cutoff; realized
		74	+variances are sums of squared bar log-returns within the regular session. Raw, unwinsorised values.
		75	+\end{minipage}
		76	+\end{table}
		77	+
		78	+\begin{figure}[H]
		79	+\centering
		80	+\includegraphics[width=\textwidth]{figures/fig02_coverage.pdf}
		81	+\caption{Sample coverage and adverse-outcome rate by year of cutoff ($h=20$ cohort).}
		82	+\label{fig:coverage}
		83	+\end{figure}

added paper/sections/literature.tex +46 −0

				@@ -0,0 +1,46 @@
		1	+\section{Related literature}
		2	+\label{sec:literature}
		3	+
		4	+\paragraph{Prices lead earnings.} Since \citet{ball1968}, accounting research has documented that stock prices
		5	+anticipate accounting income: annual returns forecast earnings changes \citep{beaver1980}, leading returns raise
		6	+earnings-response coefficients \citep{kothari1992}, and the low contemporaneous return--earnings association reflects
		7	+prices' timeliness rather than noise \citep{collins1994}. The post-earnings-announcement drift \citep{bernard1989,
		8	+bernard1990} and the accrual anomaly \citep{sloan1996, richardson2005} show that the incorporation of accounting news
		9	+is incomplete, while fundamental analysis \citep{ou1989, piotroski2000} extracts forecasts of future earnings from
		10	+statements alone. Recent work replaces hand-crafted ratios by machine learning on detailed statement items
		11	+\citep{chen2022, easton2024, cao2024}. All of these studies operate at daily or lower frequency and treat the market side
		12	+as a return; our contribution is to fix the accounting side and vary the \emph{resolution} of the market side.
		13	+
		14	+\paragraph{Information in intraday activity.} Market microstructure theory ties the composition of trading to private
		15	+information \citep{kyle1985, easley1987}, and realized-measure econometrics separates continuous variation, jumps and
		16	+their signs from high-frequency bars \citep{andersen1998, andersen2003, barndorff2004, barndorff2010semivariance,
		17	+patton2015, lee2008}. Signed jump variation and downside semivariance price the cross-section
		18	+\citep{bollerslev2020good, bollerslev2016beta}, jumps cluster around news \citep{lee2012, bajgrowicz2016}, and
		19	+announcement days change betas and the volume--volatility relation \citep{patton2012, bollerslev2018volume,
		20	+savor2016}. The question of the best sampling frequency has been settled for \emph{variance measurement} -- five-minute
		21	+realized variance is hard to beat \citep{liu2015, bollerslev2016errors, bollerslev2018risk} -- but not for the
		22	+\emph{forecasting of fundamentals}, which is our object. \citet{zhang2023volatility} show that pooling intraday
		23	+commonality improves machine-learning volatility forecasts; we include volatility families at every frequency and use a
		24	+symmetric placebo target to separate a volatility channel from a signed-information channel.
		25	+
		26	+\paragraph{Pre-announcement trading and price discovery.} Short sellers \citep{christophe2004}, individual investors
		27	+\citep{kaniel2012}, institutions \citep{hendershott2015} and option traders trade informatively before earnings
		28	+announcements; differences of opinion shape announcement returns \citep{berkman2009}; algorithmic trading may reduce
		29	+pre-announcement information acquisition \citep{weller2018}; and the announcement reaction itself occurs largely in
		30	+quotes rather than trades \citep{gregoire2022, brogaard2019}. These studies use trader-type or quote data that bars do
		31	+not contain. We ask a complementary question with anonymous OHLCV only: how much of the forthcoming accounting outcome
		32	+is visible in the shape of pre-release trading, and does that visibility depend on resolution?
		33	+
		34	+\paragraph{Attention and processing.} Limited attention \citep{hirshleifer2003, hirshleifer2009, dellavigna2009,
		35	+aboody2010} and media coverage \citep{fang2009} govern how fast public information is processed; the timing and
		36	+heterogeneity hypotheses H3--H4 are motivated by these channels and by distress-risk measurement \citep{campbell2008,
		37	+beneish1999}.
		38	+
		39	+\paragraph{Forecast evaluation and machine learning in finance.} We follow the evaluation standards of
		40	+\citet{diebold1995}, \citet{giacomini2006}, \citet{hansen2005} and \citet{hansen2011mcs}, with multiple-testing control
		41	+\citep{holm1979, benjamini1995}, cluster-robust inference \citep{petersen2009, cameron2011}, block bootstrap
		42	+\citep{kunsch1989, politis1994} and purged, embargoed time splits \citep{lopezdeprado2018}. Equal-budget comparisons of
		43	+information sets are inspired by the asset-pricing machine-learning literature \citep{gu2020, freyberger2020, kozak2020,
		44	+chen2024deep, avramov2023} and its warnings about false discoveries \citep{harvey2016, chordia2020}. Gradient-boosted
		45	+trees \citep{ke2017lightgbm} provide the nonlinear benchmark. The companion UQO working paper on intraday shock
		46	+propagation \citep{boucher2026wp15} uses the same vendor data for a different question; no result is reused here.

added paper/sections/methodology.tex +78 −0

				@@ -0,0 +1,78 @@
		1	+\section{Research design}
		2	+\label{sec:method}
		3	+
		4	+\subsection{Clocks, cutoffs and the information set}
		5	+
		6	+Figure~\ref{fig:timeline} summarises the event-time design. For firm $i$ and fiscal quarter $q$ we observe the period end
		7	+$E_{i,q}$, the earnings-release date $A_{i,q}$ (the reveal date, with its before-open/after-close timing when known), the
		8	+10-Q/10-K filing date $F_{i,q}$ and the vendor availability instant $V_{i,q}=F_{i,q}+1$ day. The prediction cutoff is
		9	+$c=A_{i,q}-h$ trading days for $h\in\{40,20,10,5,1\}$, with $h=20$ as the preregistered primary horizon. A cutoff is
		10	+admissible only if $c<\min(A_{i,q},F_{i,q})$ and $c>A_{i,q-1}$, so the feature window never contains the release of
		11	+quarter $q$ and never straddles two reveals. The information set $\mathcal{I}(c)$ contains regular-session bars with
		12	+session date $\le c$ and accounting facts with $V_{i,j}\le c$ only; the latest such quarter is $j^*(c)=q-1$ in the
		13	+retained sample (we require that the forecaster predicts the quarter immediately following the latest available one).
		14	+Twelve leakage guards (future facts, fiscal-end joins, post-cutoff bars, test-period transforms, daylight-saving errors,
		15	+identifier drift, split artefacts, release-day bars, overlapping windows across splits, survivorship, ingestion dates,
		16	+randomised placebos) are implemented as unit tests in the replication package; the local as-of join was verified live
		17	+against the FinData API's \texttt{as\_of} semantics on random entities and instants.
		18	+
		19	+A fixed-calendar design (month-end cutoffs, target = the next release within 65 trading days) is used for the pricing
		20	+tests and as a robustness check because it does not condition on knowledge of the release date.
		21	+
		22	+\subsection{Targets}
		23	+
		24	+The primary outcome is the scaled operating-income surprise $S_{i,q}$ of equation~\eqref{eq:surprise}, with
		25	+$\mathbb{E}[X_q]=X_{q-4}+\tfrac14\sum_{k=0}^{3}(X_{q-1-k}-X_{q-5-k})$ computed from quarters available at $c$ and scaled by
		26	+the latest available total assets. The binary adverse outcome is $D_{i,q}=\mathbb{1}[S_{i,q}\le-0.5\%]$, i.e., a
		27	+shortfall of at least half a percent of assets (roughly a fifth of a typical quarterly return on assets). Secondary
		28	+outcomes are the net-income analogue, a year-on-year ROA decline of at least one point, a negative operating-cash-flow
		29	+innovation of at least one percent of assets, a liquidity/coverage deterioration (current ratio down 10\% or interest
		30	+coverage falling below one), a symmetric placebo $|S_{i,q}|\ge0.5\%$ that a pure volatility channel would predict, and an
		31	+exploratory EPS miss against the vendor's estimate (available for \epsEstimateShare\% of releases; vintage unknown).
		32	+The continuous surprise, winsorised at the training-period 1st/99th percentiles, is modelled by ridge regression. No
		33	+target uses prices.
		34	+
		35	+\subsection{Features by sampling frequency}
		36	+
		37	+All market features are derived from the same raw 1-minute bars of the regular session (09:30 to the closing print),
		38	+aggregated to 5-minute and 30-minute buckets and to daily bars. Each intraday \emph{family} is computed per session
		39	+on bars of the given length and averaged (or summed) over the 20-session window ending at $c$: realized variance,
		40	+bipower variation, relative jump, negative-semivariance share, signed-jump variation, realized skewness and kurtosis,
		41	+concentration of variance in the largest bar, first-order autocorrelation, intraday drift, opening and closing volume
		42	+shares, volume dispersion (Gini), down-bar volume share, share of expected bars present, share of variance on
		43	+market-down bars, realized beta to SPY (ratio of summed cross-products), variance ratio against the daily return variance,
		44	+and volatility-of-volatility. The 30-minute, 5-minute and 1-minute sets therefore have the \emph{same} \kIntra{} columns.
		45	+Daily features (\kDaily{} columns) are 20- and 60-day returns, volatilities, Parkinson range, skewness, kurtosis,
		46	+overnight and intraday return components, volume ratios, Amihud illiquidity, zero-return share, dollar volume, maximum
		47	+drawdown and bar activity. Point-in-time accounting features (\kFund{} columns) are margins, ROA, accruals, cash,
		48	+leverage, current ratio, coverage, capex, R\&D, issuance, revenue growth, size, the last available surprise, the trailing
		49	+mean surprise, the trailing adverse rate and the age of the information. Bar statistics are descriptive; nothing is
		50	+interpreted as order flow, spread or informed trading.
		51	+
		52	+\subsection{Baseline ladder and equal budgets}
		53	+
		54	+Table~\ref{tab:infosets} nests the information sets. B0 is the training base rate (and a shrunk firm-history rate);
		55	+B1--B5 are $\ell_2$-penalised logistic regressions whose penalty is tuned on the validation period only; LB1--LB5 are
		56	+gradient-boosted trees \citep{ke2017lightgbm} with an identical random-search budget (24 trials, early stopping on the
		57	+validation period) for every information set, so that the frequency comparison is a comparison of information sets,
		58	+not of capacity. Winsorisation bounds, imputation medians and standardisation constants are estimated on training rows
		59	+only. A supplementary set (B5all) concatenates all three intraday resolutions. Neural and text models are not used.
		60	+
		61	+\subsection{Validation and inference}
		62	+
		63	+The chronological split trains on cutoffs in 2012--2018, tunes on 2019--2020 and evaluates on 2021--2026, with purging
		64	+of training outcomes revealed after the next split's start minus a one-quarter embargo \citep{lopezdeprado2018}.
		65	+Expanding-window annual refits from 2016 provide out-of-fold predictions for the panel regressions and the timing and
		66	+pricing exercises; a 20\% firm hold-out evaluates transfer to unseen firms. The primary metric is log loss; Brier score,
		67	+AUC, PR-AUC and calibration are reported for every model. Paired differences (B5$-$B2, B5$-$B4, B5$-$B3) are assessed
		68	+with a block bootstrap over calendar quarters of the reveal date \citep{kunsch1989, politis1994} and with the
		69	+Giacomini--White test on quarterly mean loss differentials with Newey--West variance \citep{diebold1995,
		70	+giacomini2006}; Holm's correction is applied within each hypothesis family \citep{holm1979}. Panel regressions
		71	+\begin{equation}
		72	+ S_{i,q}=\alpha_i+\delta_{s(i)\times t(q)}+\beta\,\mathrm{LFDS}_{i,c}+\gamma'X_{i,c}+\varepsilon_{i,q}
		73	+ \label{eq:panel}
		74	+\end{equation}
		75	+use the out-of-fold B5 probability as the Latent Fundamental Deterioration Signal (LFDS), firm and sector-by-quarter
		76	+fixed effects, and two-way clustering by firm and calendar quarter \citep{petersen2009, cameron2011}. Mechanism tests
		77	+(H3) interact the \emph{incremental} signal $\mathrm{LFDS}^{\mathrm{B5}}-\mathrm{LFDS}^{\mathrm{B2}}$ with ex-ante
		78	+terciles of size, illiquidity and leverage formed within each calendar quarter.

added paper/sections/theory_hypotheses.tex +64 −0

				@@ -0,0 +1,64 @@
		1	+\section{Economic framework and hypotheses}
		2	+\label{sec:theory}
		3	+
		4	+\subsection{Anticipation versus revelation}
		5	+
		6	+Let $X_{i,q}$ denote an accounting flow of firm $i$ for fiscal quarter $q$ (operating income in our primary
		7	+specification), revealed to the public at the earnings-release date $A_{i,q}$ and structured in the SEC filing of date
		8	+$F_{i,q}\ge A_{i,q}$. A forecaster standing at a trading day $c<A_{i,q}$ holds an information set $\mathcal{I}(c)$ that
		9	+contains, at most, prices and volumes up to the close of $c$ and the accounting statements that were \emph{available} at
		10	+$c$, i.e., quarters $j$ with $F_{i,j}+1\le c$. The quantity of interest is the conditional distribution of the surprise
		11	+\begin{equation}
		12	+ S_{i,q} \;=\; \frac{X_{i,q}-\mathbb{E}\!\left[X_{i,q}\mid \mathcal{A}(c)\right]}{TA_{i,j^*(c)}},
		13	+ \label{eq:surprise}
		14	+\end{equation}
		15	+where $\mathcal{A}(c)\subset\mathcal{I}(c)$ is the accounting information alone, $j^*(c)$ is the latest available quarter,
		16	+and the expectation is a seasonal random walk with drift \citep{foster1977}, the classic honest benchmark when no
		17	+point-in-time consensus exists. Semi-strong efficiency \citep{fama1970} implies that prices at $c$ impound all public
		18	+information, including $\mathcal{A}(c)$; the ``prices lead earnings'' literature \citep{beaver1980, kothari1992,
		19	+collins1994} documents that daily and annual returns indeed forecast accounting changes. None of this is the question
		20	+we ask. Our question is whether the \emph{resolution} at which the market is observed carries incremental information,
		21	+holding the information set, the universe, the calendar coverage, the horizon and the model capacity fixed.
		22	+
		23	+\subsection{Why resolution could matter, and why it might not}
		24	+
		25	+Three mechanisms make fine-grained activity informative beyond daily aggregates. First, in sequential-trade models
		26	+\citep{kyle1985, easley1987}, informed traders split orders and trade when liquidity is deep, so the \emph{composition} of
		27	+a day's volume and return (intraday autocorrelation, concentration of variance in a few minutes, volume on down-bars) can
		28	+reveal private information that the daily net return hides. Second, realized measures built from fine bars separate the
		29	+continuous and jump components of variation \citep{barndorff2004, andersen2003} and their sign
		30	+\citep{barndorff2010semivariance, patton2015}; negative semivariance and signed jumps have been linked to the
		31	+cross-section of expected returns \citep{bollerslev2020good} and may proxy for the arrival of adverse firm-specific news.
		32	+Third, limited attention \citep{hirshleifer2003, hirshleifer2009, dellavigna2009} predicts that the incorporation of
		33	+public but costly-to-process information is gradual, so that the time path of trading within the day (opening and closing
		34	+concentration, overnight versus intraday returns) carries information about ongoing processing.
		35	+
		36	+Against these channels stand two forces. The Epps effect and microstructure noise contaminate 1-minute returns
		37	+\citep{hansen2006, barndorff2008}, so that the finest sampling is not the most informative for variance measurement
		38	+\citep{liu2015}; and whatever information intraday activity contains about fundamentals may be fully summarised by the
		39	+daily close, in which case the frequency premium is zero. Algorithmic trading may further have reduced the amount of
		40	+pre-announcement information acquisition \citep{weller2018}. The sign and size of the frequency premium are therefore
		41	+empirical questions, and a precise null is as informative as a positive estimate.
		42	+
		43	+\subsection{Hypotheses}
		44	+
		45	+All hypotheses, endpoints and decision rules were frozen before the test period was opened (\texttt{docs/RESEARCH\_DESIGN.md}
		46	+in the replication package). Let $\mathcal{L}(M)$ denote the out-of-sample log loss of model $M$ for the binary adverse
		47	+outcome $D_{i,q}=\mathbb{1}[S_{i,q}\le -0.5\%]$, and let B1--B5 be the nested information sets of Table~\ref{tab:infosets}.
		48	+
		49	+\begin{description}[leftmargin=1.5em]
		50	+\item[H1 (incremental information).] $\mathcal{L}(\mathrm{B5})<\mathcal{L}(\mathrm{B2})$ at the 20-trading-day horizon:
		51	+ one-minute features improve forecasts beyond point-in-time fundamentals and daily OHLCV. Null: no improvement.
		52	+\item[H2 (temporal resolution).] The gain of B5 over B2 exceeds the gains of B4 (5-minute) and B3 (30-minute) over B2,
		53	+ with the same feature families and the same number of inputs. Null: the gains are equal.
		54	+\item[H3 (heterogeneity).] The B5$-$B2 gain is larger for small, illiquid and highly levered firms, the groups in which
		55	+ information asymmetry and processing costs are highest. Null: homogeneous gain.
		56	+\item[H4 (timing).] The gain increases as the cutoff approaches the release ($h=40\to1$) and is absent at placebo cutoffs.
		57	+ Null: no gradient.
		58	+\item[H5 (pricing).] The ex-ante deterioration probability predicts the announcement-window abnormal return and
		59	+ post-cutoff factor-adjusted returns. Null: no return predictability after risk adjustment and costs.
		60	+\end{description}
		61	+
		62	+A positive H1 does not identify \emph{who} trades on what: observational prediction is not causal evidence of informed
		63	+trading, and we do not have quotes, trades or trader identities \citep{gregoire2022, hendershott2015}. The paper's claims
		64	+are confined to the forecasting content of anonymous bars and to the frequency at which that content appears.

added paper/sections/titlepage.tex +77 −0

				@@ -0,0 +1,77 @@
		1	+% ============================================================================
		2	+% Title Page
		3	+% ============================================================================
		4	+\thispagestyle{empty}
		5	+
		6	+\begin{center}
		7	+
		8	+% --- Logo ---
		9	+\includegraphics[width=3.5cm]{uq_logo.jpg}
		10	+
		11	+\vspace{0.6cm}
		12	+
		13	+{\footnotesize\textsc{Université du Québec en Outaouais}}\\[0.15cm]
		14	+{\footnotesize\textsc{Département des sciences administratives}}
		15	+
		16	+\vspace{0.8cm}
		17	+
		18	+{\footnotesize\textsc{Working Paper No.~\WPnumber}}
		19	+
		20	+\vspace{1.2cm}
		21	+
		22	+% --- Title ---
		23	+{\LARGE\bfseries \WPtitle\par}
		24	+
		25	+\ifx\WPsubtitle\empty\else
		26	+ \vspace{0.3cm}
		27	+ {\large\itshape \WPsubtitle\par}
		28	+\fi
		29	+
		30	+\vspace{1.2cm}
		31	+
		32	+% --- Author ---
		33	+{\large \WPauthor}\\[0.3cm]
		34	+{\normalsize \WPaffiliation}\\[0.15cm]
		35	+{\normalsize \href{mailto:\WPemail}{\WPemail}}\\[0.15cm]
		36	+{\small \WPaddress}
		37	+
		38	+\vspace{0.8cm}
		39	+
		40	+% --- Date & Version ---
		41	+{\normalsize \WPdate}\\[0.1cm]
		42	+{\small Version~\WPversion}
		43	+
		44	+\end{center}
		45	+
		46	+\vfill
		47	+
		48	+\newpage
		49	+
		50	+% ============================================================================
		51	+% Abstract Page
		52	+% ============================================================================
		53	+\thispagestyle{empty}
		54	+
		55	+\vspace*{1cm}
		56	+
		57	+\noindent\rule{\textwidth}{0.4pt}
		58	+\vspace{0.3cm}
		59	+
		60	+\noindent\textbf{Abstract}
		61	+
		62	+\vspace{0.15cm}
		63	+
		64	+\noindent\WPabstract
		65	+
		66	+\vspace{0.4cm}
		67	+
		68	+\noindent\textbf{Keywords:} \WPkeywords
		69	+
		70	+\vspace{0.15cm}
		71	+
		72	+\noindent\textbf{JEL Classification:} \WPjel
		73	+
		74	+\vspace{0.3cm}
		75	+\noindent\rule{\textwidth}{0.4pt}
		76	+
		77	+\newpage

added paper/sections/titlepage_anon.tex +53 −0

				@@ -0,0 +1,53 @@
		1	+% ============================================================================
		2	+% ANONYMOUS TITLE PAGE — for double-anonymized journal submission
		3	+% Used by paper/main_anon.tex (same macros as main.tex, no author block,
		4	+% no institution, no logo). Abstract/keywords/JEL page identical to titlepage.tex.
		5	+% ============================================================================
		6	+\thispagestyle{empty}
		7	+\begin{center}
		8	+
		9	+\vspace*{3cm}
		10	+
		11	+{\LARGE\bfseries \WPtitle\par}
		12	+
		13	+\ifx\WPsubtitle\empty\else
		14	+ \vspace{0.3cm}
		15	+ {\large\itshape \WPsubtitle\par}
		16	+\fi
		17	+
		18	+\vspace{1.5cm}
		19	+
		20	+{\normalsize [Author identity withheld for review]}
		21	+
		22	+\vspace{0.8cm}
		23	+
		24	+{\normalsize \WPdate}
		25	+
		26	+\end{center}
		27	+
		28	+\vfill
		29	+
		30	+\newpage
		31	+\thispagestyle{empty}
		32	+\vspace*{1cm}
		33	+
		34	+\noindent\rule{\textwidth}{0.4pt}
		35	+\vspace{0.3cm}
		36	+
		37	+\noindent\textbf{Abstract}
		38	+\vspace{0.15cm}
		39	+
		40	+\noindent\WPabstract
		41	+
		42	+\vspace{0.4cm}
		43	+
		44	+\noindent\textbf{Keywords:} \WPkeywords
		45	+
		46	+\vspace{0.15cm}
		47	+
		48	+\noindent\textbf{JEL Classification:} \WPjel
		49	+
		50	+\vspace{0.3cm}
		51	+\noindent\rule{\textwidth}{0.4pt}
		52	+
		53	+\newpage

added paper/uq_logo.jpg +0 −0

Binary file not shown.

added pyproject.toml +42 −0

				@@ -0,0 +1,42 @@
		1	+[project]
		2	+name = "wp39-uqo"
		3	+version = "0.1.0"
		4	+description = "WP39 UQO - The Invisible Balance Sheet: can intraday markets detect corporate deterioration before accounting disclosures?"
		5	+readme = "README.md"
		6	+requires-python = ">=3.12"
		7	+license = { text = "Research code: MIT. Data: FirstRate Data private license (not redistributed)." }
		8	+authors = [{ name = "Simon-Pierre Boucher", email = "simon-pierre.boucher@uqo.ca" }]
		9	+dependencies = [
		10	+ "httpx>=0.27",
		11	+ "pyarrow>=17",
		12	+ "polars>=1.0",
		13	+ "duckdb>=1.0",
		14	+ "numpy>=1.26",
		15	+ "pandas>=2.2",
		16	+ "scikit-learn>=1.5",
		17	+ "lightgbm>=4.3",
		18	+ "statsmodels>=0.14",
		19	+ "scipy>=1.13",
		20	+ "pyyaml>=6",
		21	+ "matplotlib>=3.9",
		22	+ "exchange-calendars>=4.5",
		23	+]
		24	+
		25	+[project.optional-dependencies]
		26	+dev = ["pytest>=8", "pytest-cov", "ruff"]
		27	+
		28	+[build-system]
		29	+requires = ["setuptools>=68", "wheel"]
		30	+build-backend = "setuptools.build_meta"
		31	+
		32	+[tool.setuptools.packages.find]
		33	+where = ["src"]
		34	+
		35	+[tool.pytest.ini_options]
		36	+testpaths = ["tests"]
		37	+addopts = "-q"
		38	+markers = ["live: tests that call the live FinData API (skipped without WP39_LIVE=1)"]
		39	+
		40	+[tool.ruff]
		41	+line-length = 110
		42	+target-version = "py312"

added scripts/00_probe_findata.py +42 −0

				@@ -0,0 +1,42 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 0: authenticated probe of FinData OS — contract reconciliation, no raw data, no credentials printed."""
		3	+from __future__ import annotations
		4	+
		5	+import json
		6	+import sys
		7	+from pathlib import Path
		8	+
		9	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		10	+from wp39.api_client import FinDataClient # noqa: E402
		11	+from wp39.catalog import probe # noqa: E402
		12	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		13	+from wp39.provenance import git_sha, now_iso, write_json # noqa: E402
		14	+
		15	+
		16	+def main() -> int:
		17	+ P = Paths.default()
		18	+ log = get_logger("00_probe", P.logs / "00_probe.log")
		19	+ contract = load_config("data_contract")
		20	+ client = FinDataClient(log_path=P.logs / "api_calls.jsonl")
		21	+ rep = probe(client, contract)
		22	+ rep["probed_at"], rep["code_sha"] = now_iso(), git_sha(P.root)
		23	+ # spot-check fundamentals + export schema on one well-known entity, counts only
		24	+ f = client.fundamentals("frd:AAPL", period_type="q", limit=20000)
		25	+ rep["aapl_quarterly_facts"] = len(f.get("data", []))
		26	+ rep["aapl_metrics"] = sorted({r["metric"] for r in f.get("data", [])})
		27	+ rep["pit_policy"] = f.get("meta", {}).get("pit_policy")
		28	+ periods = client.fundamentals_periods("frd:AAPL")
		29	+ rep["aapl_periods"] = len(periods)
		30	+ rep["aapl_first_filing"] = min(p["filing_date"] for p in periods if p.get("filing_date"))
		31	+ write_json(P.logs / "phase0_probe.json", rep)
		32	+ log.info("health=%s datasets=%d schema_ok=%s earnings_route=%s", rep["health_status"], rep["n_datasets"],
		33	+ rep["bars_fields_ok"], rep["has_earnings_route"])
		34	+ for k, v in rep["required_datasets"].items():
		35	+ log.info("dataset %s: %s", k, v)
		36	+ print(json.dumps({k: rep[k] for k in ("health_status", "n_datasets", "bars_fields_ok", "has_earnings_route",
		37	+ "aapl_quarterly_facts", "aapl_periods", "lake_free_gb")}, indent=1))
		38	+ return 0
		39	+
		40	+
		41	+if __name__ == "__main__":
		42	+ raise SystemExit(main())

added scripts/01_audit_universe.py +83 −0

				@@ -0,0 +1,83 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 2a: universe audit — fundamentals entities x 1-minute coverage x SEC crosswalk; exclusions documented.
		3	+
		4	+Inputs: node extracts produced by scripts/01a_extract_node_tables.sh (entities, earnings_events, corporate_actions —
		5	+read-only Postgres copies, documented exception), FinData /coverage/symbols and /instruments, SEC public files.
		6	+"""
		7	+from __future__ import annotations
		8	+
		9	+import json
		10	+import sys
		11	+from pathlib import Path
		12	+
		13	+import polars as pl
		14	+
		15	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		16	+from wp39.api_client import FinDataClient # noqa: E402
		17	+from wp39.instruments import build_crosswalk, fetch_sec_submissions, fetch_sec_tickers # noqa: E402
		18	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		19	+from wp39.provenance import now_iso, sha256_file, write_json # noqa: E402
		20	+
		21	+
		22	+def main() -> int:
		23	+ P = Paths.default()
		24	+ log = get_logger("01_audit", P.logs / "01_audit.log")
		25	+ cfg = load_config("research"); contract = load_config("data_contract")
		26	+ ext = P.cache / "external"
		27	+ ents = pl.read_csv(ext / "entities.csv")
		28	+ ents = ents.with_columns(pl.col("entity_id").str.replace("^frd:", "").alias("ticker"))
		29	+ log.info("fundamentals entities: %d", ents.height)
		30	+
		31	+ client = FinDataClient(log_path=P.logs / "api_calls.jsonl")
		32	+ cov = pl.DataFrame(list(client.coverage_symbols(contract["findata"]["datasets"]["bars_1m"], min_sessions=1)))
		33	+ cov = cov.rename({"symbol": "ticker", "sessions": "n_sessions_1m", "d0": "first_session_1m", "d1": "last_session_1m", "bars": "n_bars_1m"})
		34	+ cov = cov.with_columns([pl.col("first_session_1m").str.to_date(), pl.col("last_session_1m").str.to_date()])
		35	+ log.info("1m coverage rows: %d (incl. %d delisted)", cov.height, cov.filter(pl.col("ticker").str.ends_with("-DELISTED")).height)
		36	+
		37	+ sec = fetch_sec_tickers(ext)
		38	+ u = ents.join(cov, on="ticker", how="left")
		39	+ u = u.join(sec.unique(subset=["ticker"]), on="ticker", how="left")
		40	+ subs = fetch_sec_submissions(sorted(u["cik"].drop_nulls().unique().to_list()), ext, log=log)
		41	+ u = u.join(subs, on="cik", how="left")
		42	+ u = u.with_columns([
		43	+ pl.col("first_q").str.to_date(), pl.col("last_q").str.to_date(), pl.col("first_filing").str.to_date(),
		44	+ pl.col("cik").is_not_null().alias("has_cik"),
		45	+ pl.col("sic_division").fill_null("unknown"),
		46	+ ])
		47	+ # eligibility (frozen): >= min quarters, 1m coverage overlapping 2012+, SIC not public administration
		48	+ u = u.with_columns([
		49	+ (pl.col("n_q") >= cfg["sample"]["min_quarters_per_firm"]).alias("ok_quarters"),
		50	+ (pl.col("n_sessions_1m").fill_null(0) >= 250).alias("ok_coverage"),
		51	+ (pl.col("last_session_1m") >= pl.date(2012, 6, 30)).fill_null(False).alias("ok_period"),
		52	+ (pl.col("sic_division") != "J_public_admin").alias("ok_sic"),
		53	+ ])
		54	+ u = u.with_columns((pl.col("ok_quarters") & pl.col("ok_coverage") & pl.col("ok_period") & pl.col("ok_sic")).alias("eligible"))
		55	+ u.write_parquet(P.processed / "universe.parquet")
		56	+ cascade = [
		57	+ {"step": "fundamentals_entities", "n": u.height},
		58	+ {"step": ">= 8 quarterly periods", "n": u.filter(pl.col("ok_quarters")).height},
		59	+ {"step": "+ 1m bars (>= 250 sessions)", "n": u.filter(pl.col("ok_quarters") & pl.col("ok_coverage")).height},
		60	+ {"step": "+ bars after 2012-06", "n": u.filter(pl.col("ok_quarters") & pl.col("ok_coverage") & pl.col("ok_period")).height},
		61	+ {"step": "+ not public administration", "n": u.filter(pl.col("eligible")).height},
		62	+ ]
		63	+ rep = {
		64	+ "built_at": now_iso(), "entities": u.height, "eligible": u.filter(pl.col("eligible")).height,
		65	+ "with_cik": u.filter(pl.col("has_cik")).height, "with_sic": u.filter(pl.col("sic").is_not_null()).height,
		66	+ "sic_division_counts": u.filter(pl.col("eligible")).group_by("sic_division").len().sort("len", descending=True).to_dicts(),
		67	+ "first_q_year_counts": u.filter(pl.col("eligible")).with_columns(pl.col("first_q").dt.year().alias("y")).group_by("y").len().sort("y").to_dicts(),
		68	+ "exchanges": u.filter(pl.col("eligible")).explode("exchanges").group_by("exchanges").len().sort("len", descending=True).head(8).to_dicts(),
		69	+ "delisted_symbols_in_1m_lake": int(cov.filter(pl.col("ticker").str.ends_with("-DELISTED")).height),
		70	+ "delisted_entities_with_fundamentals": 0,
		71	+ "cascade": cascade,
		72	+ "n_former_names_gt0": u.filter(pl.col("eligible") & (pl.col("n_former_names") > 0)).height,
		73	+ "sec_tickers_snapshot_sha256": sha256_file(ext / "sec_company_tickers.json"),
		74	+ }
		75	+ write_json(P.results / "universe_audit.json", rep)
		76	+ pl.DataFrame(cascade).write_csv(P.results / "table_universe_cascade.csv")
		77	+ log.info("eligible firms: %d / %d; with CIK %d; with SIC %d", rep["eligible"], rep["entities"], rep["with_cik"], rep["with_sic"])
		78	+ print(json.dumps({k: rep[k] for k in ("entities", "eligible", "with_cik", "with_sic", "cascade")}, indent=1, default=str))
		79	+ return 0
		80	+
		81	+
		82	+if __name__ == "__main__":
		83	+ raise SystemExit(main())

added scripts/01a_extract_node_tables.sh +16 −0

				@@ -0,0 +1,16 @@
		1	+#!/bin/bash
		2	+# Phase 2a (run ON the FinData storage node M3U96a): read-only Postgres extracts of three tables the API does not expose.
		3	+# Documented exception (docs/API_INTEGRATION.md). Outputs go to $WP39_CACHE/external and are hashed into manifests.
		4	+set -euo pipefail
		5	+OUT="${WP39_CACHE:-$HOME/wp39-cache}/external"; mkdir -p "$OUT"
		6	+MAN="$(cd "$(dirname "$0")/.." && pwd)/artifacts/manifests"; mkdir -p "$MAN"
		7	+psql -d findata -Atc "\copy (select entity_id, security_id, min(fiscal_period_end) first_q, max(fiscal_period_end) last_q, count(distinct fiscal_period_end) n_q, min(filing_date) first_filing from fundamentals_facts where fiscal_period_type='q' and metric='rev' group by 1,2 order by 1) to '$OUT/entities.csv' csv header"
		8	+psql -d findata -Atc "\copy (select entity_id, event_date, eps_estimate, eps_reported, release_time, source_id from earnings_events where event_date <= date '2026-10-30' order by 1,2) to '$OUT/earnings_events.csv' csv header"
		9	+psql -d findata -Atc "\copy (select symbol, action_type, ex_date, ratio, amount from corporate_actions order by 1,3) to '$OUT/corporate_actions.csv' csv header" 2>/dev/null || \
		10	+psql -d findata -Atc "\copy (select * from corporate_actions order by 1) to '$OUT/corporate_actions_raw.csv' csv header"
		11	+for f in entities earnings_events corporate_actions corporate_actions_raw; do
		12	+ [ -f "$OUT/$f.csv" ] || continue
		13	+ rows=$(($(wc -l < "$OUT/$f.csv") - 1)); sha=$(shasum -a 256 "$OUT/$f.csv" | cut -d' ' -f1)
		14	+ printf '{"table":"%s","rows":%d,"sha256":"%s","extracted_at":"%s","source":"findata postgres (read-only copy)","license":"frd-private"}\n' "$f" "$rows" "$sha" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$MAN/node_extract_$f.json"
		15	+ echo "$f: $rows rows"
		16	+done

added scripts/02_build_pit_panel.py +75 −0

				@@ -0,0 +1,75 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 2b: fundamentals extraction per eligible entity (API, cached), wide quarterly panel, reveal dates, labels.
		3	+Also freezes the research design (hash of docs/RESEARCH_DESIGN.md + configs) before any model is fitted."""
		4	+from __future__ import annotations
		5	+
		6	+import json
		7	+import sys
		8	+from concurrent.futures import ThreadPoolExecutor
		9	+from pathlib import Path
		10	+
		11	+import polars as pl
		12	+
		13	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		14	+from wp39.api_client import FinDataClient # noqa: E402
		15	+from wp39.labels import build_quarter_table, finalize_labels, match_release_dates # noqa: E402
		16	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		17	+from wp39.pit_fundamentals import add_quarter_index, fetch_entity_facts, wide_quarterly # noqa: E402
		18	+from wp39.provenance import git_sha, now_iso, sha256_file, write_json # noqa: E402
		19	+
		20	+
		21	+def main() -> int:
		22	+ P = Paths.default()
		23	+ log = get_logger("02_pit", P.logs / "02_pit.log")
		24	+ cfg = load_config("research")
		25	+ # design freeze
		26	+ freeze = {"frozen_at": now_iso(), "code_sha": git_sha(P.root),
		27	+ "files": {f: sha256_file(P.root / f) for f in ("docs/RESEARCH_DESIGN.md", "docs/PIT_PROTOCOL.md",
		28	+ "configs/research.yaml", "configs/models.yaml")}}
		29	+ fz = P.manifests / "design_freeze.json"
		30	+ if not fz.exists():
		31	+ write_json(fz, freeze)
		32	+ uni = pl.read_parquet(P.processed / "universe.parquet").filter(pl.col("eligible"))
		33	+ ents = uni["entity_id"].to_list()
		34	+ log.info("fetching quarterly facts for %d entities", len(ents))
		35	+ man = P.manifests / "fundamentals_manifest.jsonl"
		36	+
		37	+ def one(e: str):
		38	+ c = FinDataClient(log_path=P.logs / "api_calls.jsonl")
		39	+ try:
		40	+ return fetch_entity_facts(c, e, P.cache / "fundamentals" / f"{e.replace(':', '_')}.parquet", man)
		41	+ finally:
		42	+ c.close()
		43	+
		44	+ frames = []
		45	+ with ThreadPoolExecutor(max_workers=6) as ex:
		46	+ for i, df in enumerate(ex.map(one, ents)):
		47	+ frames.append(df)
		48	+ if i % 500 == 0:
		49	+ log.info("fundamentals %d/%d", i, len(ents))
		50	+ facts = pl.concat([f for f in frames if f.height], how="diagonal_relaxed")
		51	+ log.info("facts: %d rows, revisions>1: %d", facts.height, facts.filter(pl.col("revision_id") > 1).height)
		52	+ wide = add_quarter_index(wide_quarterly(facts))
		53	+ events = pl.read_csv(P.cache / "external" / "earnings_events.csv", try_parse_dates=True)
		54	+ wide = match_release_dates(wide, events)
		55	+ q = finalize_labels(build_quarter_table(wide), threshold=cfg["targets"]["primary"]["threshold"])
		56	+ q.write_parquet(P.processed / "quarters.parquet")
		57	+ rep = {
		58	+ "built_at": now_iso(), "entities": q["entity_id"].n_unique(), "firm_quarters": q.height,
		59	+ "release_match": q.group_by("release_match").len().sort("len", descending=True).to_dicts(),
		60	+ "release_time": q.group_by("release_time").len().sort("len", descending=True).to_dicts(),
		61	+ "filing_minus_release_days_quantiles": {str(p): q["filing_minus_release_days"].quantile(p) for p in (0.1, 0.5, 0.9)},
		62	+ "label_base_rates_all": {y: float(q[y].mean()) for y in ("y_adverse_opinc", "y_adverse_netinc", "y_roa_decline",
		63	+ "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise")},
		64	+ "eps_miss_rate_where_available": float(q["y_eps_miss"].drop_nulls().mean()) if q["y_eps_miss"].drop_nulls().len() else None,
		65	+ "eps_estimate_available_share": float(q["eps_estimate"].is_not_null().mean()),
		66	+ "S_opinc_quantiles": {str(p): q["S_opinc"].drop_nulls().quantile(p) for p in (0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99)},
		67	+ "regular_calendar_share": float((q["_gap_ok8"] == 8).mean()),
		68	+ }
		69	+ write_json(P.results / "pit_panel_audit.json", rep)
		70	+ print(json.dumps(rep, indent=1, default=str))
		71	+ return 0
		72	+
		73	+
		74	+if __name__ == "__main__":
		75	+ raise SystemExit(main())

added scripts/03_build_features.py +116 −0

				@@ -0,0 +1,116 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 2c: streaming export -> session features at 1m/5m/30m/1d -> rolling windows, per symbol (multiprocess).
		3	+Raw Parquet is deleted after the feature rows are written; manifests keep hashes and dataset versions."""
		4	+from __future__ import annotations
		5	+
		6	+import argparse
		7	+import datetime as dt
		8	+import json
		9	+import os
		10	+import sys
		11	+import time
		12	+from concurrent.futures import ProcessPoolExecutor, as_completed
		13	+from pathlib import Path
		14	+
		15	+import polars as pl
		16	+
		17	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		18	+from wp39.api_client import FinDataClient # noqa: E402
		19	+from wp39.corporate_actions import load_actions # noqa: E402
		20	+from wp39.features import compute_spy_buckets, compute_symbol_features, session_bounds_table # noqa: E402
		21	+from wp39.ingest import export_symbol, mark_consumed # noqa: E402
		22	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		23	+from wp39.provenance import append_jsonl, now_iso, write_json # noqa: E402
		24	+from wp39.samples import rolling_window_features # noqa: E402
		25	+
		26	+START, END = dt.date(2011, 1, 1), dt.date(2026, 10, 30)
		27	+
		28	+
		29	+def work(symbol: str, keep_raw: bool) -> dict:
		30	+ P = Paths.default()
		31	+ contract = load_config("data_contract")
		32	+ ds = contract["findata"]["datasets"]["bars_1m"]
		33	+ out = P.cache / "features" / f"{symbol}.parquet"
		34	+ win_out = P.cache / "windows" / f"{symbol}.parquet"
		35	+ win_out.parent.mkdir(exist_ok=True)
		36	+ if out.exists() and win_out.exists():
		37	+ return {"symbol": symbol, "status": "cached"}
		38	+ t0 = time.time()
		39	+ client = FinDataClient(log_path=P.logs / f"api_calls_worker_{os.getpid()}.jsonl")
		40	+ try:
		41	+ paths = export_symbol(client, symbol, START, END, ds, P.cache / "bars", P.manifests / "bars_manifest.jsonl", keep=keep_raw)
		42	+ finally:
		43	+ client.close()
		44	+ if not paths:
		45	+ return {"symbol": symbol, "status": "no_export"}
		46	+ bounds = pl.read_parquet(P.cache / "external" / "session_bounds.parquet")
		47	+ spy = {k: pl.read_parquet(P.cache / "external" / f"spy_buckets_{k}.parquet") for k in ("1m", "5m", "30m")}
		48	+ actions = pl.read_parquet(P.cache / "external" / "actions.parquet")
		49	+ feat = compute_symbol_features(paths, symbol, bounds, spy, actions)
		50	+ if feat.height == 0:
		51	+ return {"symbol": symbol, "status": "empty"}
		52	+ feat.write_parquet(out)
		53	+ rolling_window_features(feat).write_parquet(win_out)
		54	+ if not keep_raw:
		55	+ for p in paths:
		56	+ p.unlink(missing_ok=True)
		57	+ mark_consumed(P.manifests / "bars_manifest.jsonl", symbol, ds)
		58	+ return {"symbol": symbol, "status": "ok", "sessions": feat.height, "first": str(feat["session_date"].min()),
		59	+ "last": str(feat["session_date"].max()), "secs": round(time.time() - t0, 1)}
		60	+
		61	+
		62	+def main() -> int:
		63	+ ap = argparse.ArgumentParser()
		64	+ ap.add_argument("--workers", type=int, default=6)
		65	+ ap.add_argument("--limit", type=int, default=0, help="pilot: only the first N eligible symbols")
		66	+ ap.add_argument("--symbols", type=str, default="", help="comma-separated explicit list")
		67	+ ap.add_argument("--keep-raw", action="store_true")
		68	+ args = ap.parse_args()
		69	+ P = Paths.default()
		70	+ log = get_logger("03_features", P.logs / "03_features.log")
		71	+ contract = load_config("data_contract")
		72	+ ext = P.cache / "external"
		73	+ # shared inputs: session bounds, SPY buckets, corporate actions
		74	+ if not (ext / "session_bounds.parquet").exists():
		75	+ session_bounds_table(START, END).write_parquet(ext / "session_bounds.parquet")
		76	+ if not (ext / "spy_buckets_1m.parquet").exists():
		77	+ client = FinDataClient(log_path=P.logs / "api_calls.jsonl")
		78	+ paths = export_symbol(client, "SPY", START, END, contract["findata"]["datasets"]["bars_1m_etf"], P.cache / "bars",
		79	+ P.manifests / "bars_manifest.jsonl", keep=True)
		80	+ client.close()
		81	+ bounds = pl.read_parquet(ext / "session_bounds.parquet")
		82	+ for k, v in compute_spy_buckets(paths, bounds).items():
		83	+ v.write_parquet(ext / f"spy_buckets_{k}.parquet")
		84	+ log.info("SPY buckets written")
		85	+ if not (ext / "actions.parquet").exists():
		86	+ load_actions(ext / "corporate_actions_raw.csv").write_parquet(ext / "actions.parquet")
		87	+ if args.symbols:
		88	+ symbols = args.symbols.split(",")
		89	+ else:
		90	+ uni = pl.read_parquet(P.processed / "universe.parquet").filter(pl.col("eligible")).sort("ticker")
		91	+ symbols = uni["ticker"].to_list()
		92	+ if args.limit:
		93	+ symbols = symbols[: args.limit]
		94	+ log.info("symbols to process: %d (workers=%d)", len(symbols), args.workers)
		95	+ status = P.logs / "03_features_status.jsonl"
		96	+ done = {json.loads(l)["symbol"] for l in open(status)} if status.exists() else set()
		97	+ todo = [s for s in symbols if s not in done]
		98	+ t0 = time.time(); n_ok = 0
		99	+ with ProcessPoolExecutor(max_workers=args.workers) as ex:
		100	+ futs = {ex.submit(work, s, args.keep_raw): s for s in todo}
		101	+ for i, f in enumerate(as_completed(futs)):
		102	+ try:
		103	+ r = f.result()
		104	+ except Exception as exc: # noqa: BLE001
		105	+ r = {"symbol": futs[f], "status": "error", "error": str(exc)[:300]}
		106	+ r["ts"] = now_iso()
		107	+ append_jsonl(status, r)
		108	+ n_ok += r["status"] in ("ok", "cached")
		109	+ if i % 50 == 0:
		110	+ log.info("%d/%d done (ok=%d) elapsed=%.0fs last=%s", i + 1, len(todo), n_ok, time.time() - t0, r)
		111	+ log.info("finished: %d symbols, %.0fs", len(todo), time.time() - t0)
		112	+ return 0
		113	+
		114	+
		115	+if __name__ == "__main__":
		116	+ raise SystemExit(main())

added scripts/04_build_labels.py +83 −0

				@@ -0,0 +1,83 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 2d/3a: cohorts (event-time per horizon + fixed calendar), PIT joins, exclusion cascade, base rates."""
		3	+from __future__ import annotations
		4	+
		5	+import datetime as dt
		6	+import json
		7	+import sys
		8	+from pathlib import Path
		9	+
		10	+import polars as pl
		11	+
		12	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		13	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		14	+from wp39.provenance import now_iso, write_json # noqa: E402
		15	+from wp39.samples import (apply_admissibility, calendar_cutoffs, event_cutoffs, exclusion_cascade, join_fundamentals, # noqa: E402
		16	+ join_market)
		17	+
		18	+
		19	+def hist_features(sample: pl.DataFrame, quarters: pl.DataFrame) -> pl.DataFrame:
		20	+ """Information age, last available surprise, mean of last 4 surprises, trailing adverse rate — all from quarters
		21	+ available at the cutoff (pit_qidx and earlier)."""
		22	+ qq = quarters.select(["entity_id", "qidx", "S_opinc", "y_adverse_opinc", "A_q"]).sort(["entity_id", "qidx"])
		23	+ qq = qq.with_columns([
		24	+ pl.col("S_opinc").alias("last_surprise"),
		25	+ pl.col("S_opinc").rolling_mean(window_size=4, min_samples=2).over("entity_id").alias("mean_surprise4"),
		26	+ pl.col("y_adverse_opinc").cast(pl.Float64).rolling_mean(window_size=8, min_samples=2).over("entity_id").alias("hist_adverse_rate"),
		27	+ pl.col("A_q").alias("A_pit"),
		28	+ ]).select(["entity_id", pl.col("qidx").alias("pit_qidx"), "last_surprise", "mean_surprise4", "hist_adverse_rate", "A_pit"])
		29	+ s = sample.join(qq, on=["entity_id", "pit_qidx"], how="left")
		30	+ return s.with_columns((pl.col("cutoff") - pl.col("A_pit")).dt.total_days().alias("info_age_td"))
		31	+
		32	+
		33	+def build(design: str, cfg: dict, P: Paths, quarters: pl.DataFrame, windows: pl.DataFrame, sym: pl.DataFrame, log) -> pl.DataFrame:
		34	+ if design == "event":
		35	+ cut = event_cutoffs(quarters, cfg["cutoffs"]["event_time_horizons_td"])
		36	+ else:
		37	+ s0, s1 = dt.date.fromisoformat(str(cfg["sample"]["start"])), dt.date.fromisoformat(str(cfg["sample"]["end"]))
		38	+ cut = calendar_cutoffs(quarters, s0, s1)
		39	+ cut = apply_admissibility(cut)
		40	+ log.info("%s cutoffs after admissibility: %d", design, cut.height)
		41	+ j = join_market(cut, windows, sym)
		42	+ j = join_fundamentals(j, quarters)
		43	+ # attach the outcome of quarter q (the row keyed by entity/fiscal_period_end)
		44	+ ycols = ["S_opinc", "S_netinc", "S_ocf", "S_rev", "ocf_innov", "roa_yoy", "y_adverse_opinc", "y_adverse_netinc", "y_roa_decline",
		45	+ "y_ocf_negative", "y_liquidity_deterioration", "y_eps_miss", "y_abs_surprise", "assets_tot_lag1", "_gap_ok8",
		46	+ "release_time", "eps_estimate"]
		47	+ j = j.join(quarters.select(["entity_id", "fiscal_period_end"] + ycols), on=["entity_id", "fiscal_period_end"], how="left")
		48	+ j = hist_features(j, quarters)
		49	+ sample, cascade = exclusion_cascade(j, cfg["sample"]["min_price_usd"], cfg["sample"]["min_valid_days_in_window"])
		50	+ cascade.write_csv(P.results / f"table_sample_cascade_{design}.csv")
		51	+ log.info("%s sample: %d rows; cascade=%s", design, sample.height, cascade.to_dicts())
		52	+ return sample
		53	+
		54	+
		55	+def main() -> int:
		56	+ P = Paths.default()
		57	+ log = get_logger("04_labels", P.logs / "04_labels.log")
		58	+ cfg = load_config("research")
		59	+ quarters = pl.read_parquet(P.processed / "quarters.parquet")
		60	+ uni = pl.read_parquet(P.processed / "universe.parquet").filter(pl.col("eligible"))
		61	+ sym = uni.select(["entity_id", "ticker", "sic_division", "sic"]).rename({"ticker": "symbol"})
		62	+ wins = [p for p in (P.cache / "windows").glob("*.parquet") if p.stem in set(sym["symbol"].to_list())]
		63	+ windows = pl.concat([pl.read_parquet(p) for p in wins], how="diagonal_relaxed")
		64	+ log.info("windows: %d rows, %d symbols", windows.height, len(wins))
		65	+ quarters = quarters.filter(pl.col("entity_id").is_in(sym["entity_id"].to_list()))
		66	+ out = {}
		67	+ for design in ("event", "calendar"):
		68	+ s = build(design, cfg, P, quarters, windows, sym, log)
		69	+ s.write_parquet(P.processed / f"sample_{design}.parquet")
		70	+ out[design] = {
		71	+ "rows": s.height, "firms": s["entity_id"].n_unique(), "quarters": s.select(["entity_id", "fiscal_period_end"]).unique().height,
		72	+ "by_h": s.group_by("h").agg([pl.len().alias("n"), pl.col("y_adverse_opinc").mean().alias("base_rate")]).sort("h").to_dicts() if design == "event" else None,
		73	+ "base_rates": {y: float(s[y].mean()) for y in ("y_adverse_opinc", "y_adverse_netinc", "y_roa_decline", "y_ocf_negative",
		74	+ "y_liquidity_deterioration", "y_abs_surprise")},
		75	+ "years": s.with_columns(pl.col("cutoff").dt.year().alias("y")).group_by("y").len().sort("y").to_dicts(),
		76	+ }
		77	+ write_json(P.results / "sample_audit.json", {"built_at": now_iso(), **out})
		78	+ print(json.dumps(out, indent=1, default=str))
		79	+ return 0
		80	+
		81	+
		82	+if __name__ == "__main__":
		83	+ raise SystemExit(main())

added scripts/05_train_baselines.py +108 −0

				@@ -0,0 +1,108 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 3: baseline ladder B0-B5 (base rates, logistic ridge per information set) on the chronological split, every
		3	+horizon, primary + secondary targets; walk-forward out-of-fold predictions; firm hold-out variant.
		4	+Predictions are saved per (design, h, target, model) so that 07_evaluate.py computes every metric from files."""
		5	+from __future__ import annotations
		6	+
		7	+import json
		8	+import sys
		9	+from pathlib import Path
		10	+
		11	+import numpy as np
		12	+import polars as pl
		13	+
		14	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		15	+from wp39.models import INFO_SETS, Preprocessor, base_rate_predict, columns_for, fit_logistic, fit_ridge, firm_history_predict, predict_proba # noqa: E402
		16	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		17	+from wp39.provenance import now_iso, write_json # noqa: E402
		18	+from wp39.samples import feature_columns # noqa: E402
		19	+from wp39.splits import chronological, firm_holdout_mask, overlap_check, walk_forward # noqa: E402
		20	+
		21	+KEY = ["entity_id", "symbol", "fiscal_period_end", "h", "cutoff", "A_q", "sic_division", "release_time"]
		22	+TARGETS = ["y_adverse_opinc", "y_adverse_netinc", "y_roa_decline", "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise"]
		23	+
		24	+
		25	+def save_preds(P: Paths, df: pl.DataFrame, p: np.ndarray, tag: str, design: str, h: int, target: str, model: str, part: str) -> None:
		26	+ out = df.select(KEY + [target]).with_columns([pl.Series("p", p), pl.lit(model).alias("model"), pl.lit(part).alias("part"),
		27	+ pl.lit(tag).alias("tag")])
		28	+ d = P.evaluation / "preds"; d.mkdir(exist_ok=True)
		29	+ out.write_parquet(d / f"{design}_h{h}_{target}_{model}_{part}_{tag}.parquet")
		30	+
		31	+
		32	+def run_split(P, cfg, mcfg, fc, design, h, target, train, valid, test, tag, log, info_sets, record):
		33	+ ytr, yva, yte = (x[target].to_numpy().astype(int) for x in (train, valid, test))
		34	+ # B0
		35	+ save_preds(P, test, base_rate_predict(ytr, test.height), tag, design, h, target, "B0", "test")
		36	+ save_preds(P, valid, base_rate_predict(ytr, valid.height), tag, design, h, target, "B0", "validation")
		37	+ save_preds(P, test, firm_history_predict(train, test, target), tag, design, h, target, "B0b", "test")
		38	+ for name in info_sets:
		39	+ cols = columns_for(name, fc)
		40	+ pp = Preprocessor(cols).fit(train)
		41	+ Xtr, Xva, Xte = pp.transform(train), pp.transform(valid), pp.transform(test)
		42	+ m, info = fit_logistic(Xtr, ytr, Xva, yva, mcfg["logistic"]["C_grid"], cfg["seed"])
		43	+ save_preds(P, valid, predict_proba(m, Xva), tag, design, h, target, name, "validation")
		44	+ save_preds(P, test, predict_proba(m, Xte), tag, design, h, target, name, "test")
		45	+ record.append({"design": design, "h": h, "target": target, "model": name, "tag": tag, "n_train": train.height,
		46	+ "n_valid": valid.height, "n_test": test.height, "n_features": len(cols), **info,
		47	+ "coef": dict(zip(cols, m.coef_[0].round(4).tolist()))})
		48	+ log.info("%s h=%d %s %s %s: C=%s val_ll=%.4f (p=%d)", design, h, target, name, tag, info["C"], info["val_logloss"], len(cols))
		49	+
		50	+
		51	+def main() -> int:
		52	+ P = Paths.default()
		53	+ log = get_logger("05_baselines", P.logs / "05_baselines.log")
		54	+ cfg, mcfg = load_config("research"), load_config("models")
		55	+ record = []
		56	+ for design in ("event", "calendar"):
		57	+ df = pl.read_parquet(P.processed / f"sample_{design}.parquet")
		58	+ if design == "calendar":
		59	+ df = df.with_columns(pl.lit(0).alias("h")) # calendar: h varies; pooled model
		60	+ fc = feature_columns(df)
		61	+ write_json(P.evaluation / f"feature_columns_{design}.json", fc)
		62	+ horizons = sorted(df["h"].unique().to_list()) if design == "event" else [0]
		63	+ for h in horizons:
		64	+ d = df.filter(pl.col("h") == h) if design == "event" else df
		65	+ targets = TARGETS if (design == "event" and h == cfg["cutoffs"]["primary_horizon_td"]) else ["y_adverse_opinc"]
		66	+ for target in targets:
		67	+ dd = d.filter(pl.col(target).is_not_null())
		68	+ sp = chronological(dd, cfg["splits"])
		69	+ chk = overlap_check(sp["train"], sp["test"])
		70	+ assert chk["shared_observations"] == 0 and chk["train_outcomes_revealed_in_test"] == 0, chk
		71	+ sets = list(INFO_SETS) if target == "y_adverse_opinc" else ["B1", "B2", "B5"]
		72	+ run_split(P, cfg, mcfg, fc, design, h, target, sp["train"], sp["validation"], sp["test"], "chrono", log, sets, record)
		73	+ if design == "event" and h == cfg["cutoffs"]["primary_horizon_td"] and target == "y_adverse_opinc":
		74	+ # continuous target (ridge) on the same split
		75	+ for name in ("B2", "B5"):
		76	+ cols = columns_for(name, fc); pp = Preprocessor(cols).fit(sp["train"])
		77	+ ytr = sp["train"]["S_opinc"].to_numpy(); lo, hi = np.quantile(ytr, [0.01, 0.99]); ytr = np.clip(ytr, lo, hi)
		78	+ m, info = fit_ridge(pp.transform(sp["train"]), ytr, pp.transform(sp["validation"]), np.clip(sp["validation"]["S_opinc"].to_numpy(), lo, hi))
		79	+ save_preds(P, sp["test"], m.predict(pp.transform(sp["test"])), "chrono_cont", design, h, "S_opinc", name, "test")
		80	+ record.append({"design": design, "h": h, "target": "S_opinc", "model": name, "tag": "chrono_cont", "winsor": [float(lo), float(hi)],
		81	+ "train_mean": float(ytr.mean()), **info})
		82	+ # firm hold-out: train/validate on non-held-out firms, test on both groups separately
		83	+ hold = firm_holdout_mask(dd["entity_id"], cfg["splits"]["firm_holdout_fraction"])
		84	+ dd2 = dd.with_columns(pl.Series("_hold", hold))
		85	+ sp2 = chronological(dd2.filter(~pl.col("_hold")), cfg["splits"])
		86	+ te_hold = dd2.filter(pl.col("_hold") & (pl.col("cutoff") >= sp2["test"]["cutoff"].min()))
		87	+ run_split(P, cfg, mcfg, fc, design, h, target, sp2["train"], sp2["validation"], te_hold, "firmhold_heldout", log, ["B2", "B5"], record)
		88	+ run_split(P, cfg, mcfg, fc, design, h, target, sp2["train"], sp2["validation"], sp2["test"], "firmhold_seen", log, ["B2", "B5"], record)
		89	+ # walk-forward out-of-fold predictions (logistic, C fixed at the chronological choice per set)
		90	+ for name in ("B1", "B2", "B3", "B4", "B5"):
		91	+ cols = columns_for(name, fc)
		92	+ C = next(r["C"] for r in record if r["model"] == name and r["tag"] == "chrono" and r["h"] == h and r["target"] == target)
		93	+ parts = []
		94	+ for y, tr, te in walk_forward(dd, cfg["splits"], 2026):
		95	+ pp = Preprocessor(cols).fit(tr)
		96	+ from sklearn.linear_model import LogisticRegression
		97	+ m = LogisticRegression(C=C, max_iter=2000, random_state=cfg["seed"]).fit(pp.transform(tr), tr[target].to_numpy().astype(int))
		98	+ parts.append(te.select(KEY + [target]).with_columns([pl.Series("p", predict_proba(m, pp.transform(te))), pl.lit(y).alias("fold")]))
		99	+ oof = pl.concat(parts).with_columns([pl.lit(name).alias("model"), pl.lit("oof").alias("part"), pl.lit("walkforward").alias("tag")])
		100	+ oof.write_parquet(P.evaluation / "preds" / f"{design}_h{h}_{target}_{name}_oof_walkforward.parquet")
		101	+ log.info("walk-forward %s: %d oof rows", name, oof.height)
		102	+ write_json(P.evaluation / "baselines_fit_log.json", {"run_at": now_iso(), "fits": record})
		103	+ print(json.dumps({"fits": len(record)}, indent=1))
		104	+ return 0
		105	+
		106	+
		107	+if __name__ == "__main__":
		108	+ raise SystemExit(main())

added scripts/06_train_advanced.py +116 −0

				@@ -0,0 +1,116 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 4: LightGBM (B6) with an identical random-search budget for every information set on the chronological split
		3	+(primary horizon, primary target + secondary targets), horizon ladder and walk-forward refits with a fixed configuration
		4	+(no tuning inside folds, so earlier folds never see later data). No neural/text models (DECISIONS D-006)."""
		5	+from __future__ import annotations
		6	+
		7	+import json
		8	+import sys
		9	+from pathlib import Path
		10	+
		11	+import numpy as np
		12	+import polars as pl
		13	+
		14	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		15	+from wp39.models import INFO_SETS, Preprocessor, columns_for, fit_lightgbm, predict_proba # noqa: E402
		16	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		17	+from wp39.provenance import now_iso, write_json # noqa: E402
		18	+from wp39.samples import feature_columns # noqa: E402
		19	+from wp39.splits import chronological, firm_holdout_mask, walk_forward # noqa: E402
		20	+
		21	+KEY = ["entity_id", "symbol", "fiscal_period_end", "h", "cutoff", "A_q", "sic_division", "release_time"]
		22	+FIXED = {"num_leaves": 15, "learning_rate": 0.03, "min_child_samples": 100, "feature_fraction": 0.7, "bagging_fraction": 0.8,
		23	+ "bagging_freq": 1, "lambda_l2": 1.0, "n_estimators": 600, "verbose": -1, "n_jobs": 12}
		24	+
		25	+
		26	+def save(P, df, p, design, h, target, model, part, tag, extra=None):
		27	+ out = df.select(KEY + [target]).with_columns([pl.Series("p", p), pl.lit(model).alias("model"), pl.lit(part).alias("part"), pl.lit(tag).alias("tag")])
		28	+ if extra is not None:
		29	+ out = out.with_columns(extra)
		30	+ out.write_parquet(P.evaluation / "preds" / f"{design}_h{h}_{target}_{model}_{part}_{tag}.parquet")
		31	+
		32	+
		33	+def tuned(P, cfg, mcfg, fc, design, h, target, sp, tag, sets, log, record):
		34	+ ytr, yva = sp["train"][target].to_numpy().astype(int), sp["validation"][target].to_numpy().astype(int)
		35	+ for name in sets:
		36	+ cols = columns_for(name, fc); pp = Preprocessor(cols).fit(sp["train"])
		37	+ m, info = fit_lightgbm(pp.transform(sp["train"]), ytr, pp.transform(sp["validation"]), yva, mcfg["lightgbm"], cfg["seed"], n_jobs=12)
		38	+ save(P, sp["validation"], predict_proba(m, pp.transform(sp["validation"])), design, h, target, "L" + name, "validation", tag)
		39	+ save(P, sp["test"], predict_proba(m, pp.transform(sp["test"])), design, h, target, "L" + name, "test", tag)
		40	+ imp = dict(zip(cols, (m.booster_.feature_importance("gain") / max(m.booster_.feature_importance("gain").sum(), 1)).round(4).tolist()))
		41	+ record.append({"design": design, "h": h, "target": target, "model": "L" + name, "tag": tag, "n_features": len(cols),
		42	+ "val_score": info["val_score"], "params": info["params"], "trials": len(info["trials"]), "importance_gain": imp})
		43	+ log.info("%s h=%d %s L%s %s: val_ll=%.4f params=%s", design, h, target, name, tag, info["val_score"], info["params"])
		44	+
		45	+
		46	+def fixed_fit(train, test, cols, target, seed):
		47	+ import lightgbm as lgb
		48	+ pp = Preprocessor(cols).fit(train)
		49	+ # early stopping on the last 4 calendar quarters of the training window (purged by construction: still before test)
		50	+ cut = train["cutoff"].quantile(0.85)
		51	+ tr, va = train.filter(pl.col("cutoff") < cut), train.filter(pl.col("cutoff") >= cut)
		52	+ m = lgb.LGBMClassifier(random_state=seed, **FIXED)
		53	+ m.fit(pp.transform(tr), tr[target].to_numpy().astype(int), eval_set=[(pp.transform(va), va[target].to_numpy().astype(int))],
		54	+ callbacks=[lgb.early_stopping(50, verbose=False)])
		55	+ return predict_proba(m, pp.transform(test)), int(m.best_iteration_ or 0)
		56	+
		57	+
		58	+def main() -> int:
		59	+ P = Paths.default()
		60	+ log = get_logger("06_advanced", P.logs / "06_advanced.log")
		61	+ cfg, mcfg = load_config("research"), load_config("models")
		62	+ hp = cfg["cutoffs"]["primary_horizon_td"]
		63	+ record = []
		64	+ df = pl.read_parquet(P.processed / "sample_event.parquet")
		65	+ fc = feature_columns(df)
		66	+ # 1) primary: tuned LightGBM, all information sets, h = 20, primary target
		67	+ d = df.filter((pl.col("h") == hp) & pl.col("y_adverse_opinc").is_not_null())
		68	+ sp = chronological(d, cfg["splits"])
		69	+ tuned(P, cfg, mcfg, fc, "event", hp, "y_adverse_opinc", sp, "chrono", list(INFO_SETS), log, record)
		70	+ # 2) secondary targets (B2 vs B5 only), tuned
		71	+ for target in ("y_adverse_netinc", "y_roa_decline", "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise"):
		72	+ dt_ = df.filter((pl.col("h") == hp) & pl.col(target).is_not_null())
		73	+ tuned(P, cfg, mcfg, fc, "event", hp, target, chronological(dt_, cfg["splits"]), "chrono", ["B2", "B5"], log, record)
		74	+ # 3) horizon ladder (fixed config): B2 vs B5 at every h on the chronological split
		75	+ for h in sorted(df["h"].unique().to_list()):
		76	+ dh = df.filter((pl.col("h") == h) & pl.col("y_adverse_opinc").is_not_null())
		77	+ sph = chronological(dh, cfg["splits"])
		78	+ for name in ("B2", "B5"):
		79	+ p, it = fixed_fit(pl.concat([sph["train"], sph["validation"]]), sph["test"], columns_for(name, fc), "y_adverse_opinc", cfg["seed"])
		80	+ save(P, sph["test"], p, "event", h, "y_adverse_opinc", "L" + name, "test", "fixed")
		81	+ record.append({"design": "event", "h": h, "target": "y_adverse_opinc", "model": "L" + name, "tag": "fixed", "best_iteration": it})
		82	+ # 4) firm hold-out (fixed config)
		83	+ hold = firm_holdout_mask(d["entity_id"], cfg["splits"]["firm_holdout_fraction"])
		84	+ d2 = d.with_columns(pl.Series("_hold", hold))
		85	+ sp2 = chronological(d2.filter(~pl.col("_hold")), cfg["splits"])
		86	+ te_hold = d2.filter(pl.col("_hold") & (pl.col("cutoff") >= sp2["test"]["cutoff"].min()))
		87	+ for name in ("B2", "B5"):
		88	+ trv = pl.concat([sp2["train"], sp2["validation"]])
		89	+ p, _ = fixed_fit(trv, te_hold, columns_for(name, fc), "y_adverse_opinc", cfg["seed"]); save(P, te_hold, p, "event", hp, "y_adverse_opinc", "L" + name, "test", "firmhold_heldout")
		90	+ p, _ = fixed_fit(trv, sp2["test"], columns_for(name, fc), "y_adverse_opinc", cfg["seed"]); save(P, sp2["test"], p, "event", hp, "y_adverse_opinc", "L" + name, "test", "firmhold_seen")
		91	+ # 5) walk-forward out-of-fold (fixed config) for B1..B5 at h = 20 -> LFDS for the panel regressions / H4 / H5
		92	+ for name in ("B1", "B2", "B3", "B4", "B5"):
		93	+ parts = []
		94	+ for y, tr, te in walk_forward(d, cfg["splits"], 2026):
		95	+ p, it = fixed_fit(tr, te, columns_for(name, fc), "y_adverse_opinc", cfg["seed"])
		96	+ parts.append(te.select(KEY + ["y_adverse_opinc"]).with_columns([pl.Series("p", p), pl.lit(y).alias("fold")]))
		97	+ oof = pl.concat(parts).with_columns([pl.lit("L" + name).alias("model"), pl.lit("oof").alias("part"), pl.lit("walkforward").alias("tag")])
		98	+ oof.write_parquet(P.evaluation / "preds" / f"event_h{hp}_y_adverse_opinc_L{name}_oof_walkforward.parquet")
		99	+ log.info("walk-forward L%s: %d oof rows", name, oof.height)
		100	+ # 6) calendar design: walk-forward OOF for B2/B5 (scores for H5 portfolio sorts)
		101	+ dc = pl.read_parquet(P.processed / "sample_calendar.parquet").with_columns(pl.lit(0).alias("h"))
		102	+ fcc = feature_columns(dc)
		103	+ for name in ("B2", "B5"):
		104	+ parts = []
		105	+ for y, tr, te in walk_forward(dc, cfg["splits"], 2026):
		106	+ p, _ = fixed_fit(tr, te, columns_for(name, fcc), "y_adverse_opinc", cfg["seed"])
		107	+ parts.append(te.select(KEY + ["y_adverse_opinc"]).with_columns([pl.Series("p", p), pl.lit(y).alias("fold")]))
		108	+ oof = pl.concat(parts).with_columns([pl.lit("L" + name).alias("model"), pl.lit("oof").alias("part"), pl.lit("walkforward").alias("tag")])
		109	+ oof.write_parquet(P.evaluation / "preds" / f"calendar_h0_y_adverse_opinc_L{name}_oof_walkforward.parquet")
		110	+ write_json(P.evaluation / "advanced_fit_log.json", {"run_at": now_iso(), "fixed_config": FIXED, "fits": record})
		111	+ print(json.dumps({"fits": len(record)}, indent=1))
		112	+ return 0
		113	+
		114	+
		115	+if __name__ == "__main__":
		116	+ raise SystemExit(main())

added scripts/07_evaluate.py +209 −0

				@@ -0,0 +1,209 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 3/4 evaluation: every metric and every paired contrast is computed from saved prediction files.
		3	+Outputs (artifacts/evaluation + results/): metrics_all.csv, contrasts.csv, horizon_ladder.csv, secondary_targets.csv,
		4	+firm_holdout.csv, walkforward_by_year.csv, reliability.csv, continuous.csv, power.json."""
		5	+from __future__ import annotations
		6	+
		7	+import json
		8	+import sys
		9	+from pathlib import Path
		10	+
		11	+import numpy as np
		12	+import pandas as pd
		13	+import polars as pl
		14	+
		15	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		16	+from wp39.inference import (binary_metrics, continuous_metrics, dm_gw_test, holm, paired_block_bootstrap, # noqa: E402
		17	+ pointwise_logloss, quarter_blocks, reliability_table)
		18	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		19	+from wp39.provenance import now_iso, write_json # noqa: E402
		20	+
		21	+CONTRASTS = [("B5", "B2"), ("B4", "B2"), ("B3", "B2"), ("B5", "B4"), ("B5", "B3"), ("B5all", "B2"), ("B5all", "B5"), ("B2", "B1"), ("B1", "B0")]
		22	+FAMILIES = {"freq_vs_daily": [("B5", "B2"), ("B4", "B2"), ("B3", "B2")], "fine_vs_coarse": [("B5", "B4"), ("B5", "B3")]}
		23	+
		24	+
		25	+def load_preds(P: Paths) -> pl.DataFrame:
		26	+ files = sorted((P.evaluation / "preds").glob("*.parquet"))
		27	+ frames = []
		28	+ for f in files:
		29	+ d = pl.read_parquet(f)
		30	+ target = [c for c in d.columns if c.startswith("y_") or c == "S_opinc"][0]
		31	+ d = d.rename({target: "y"}).with_columns(pl.lit(target).alias("target"))
		32	+ if "fold" not in d.columns:
		33	+ d = d.with_columns(pl.lit(None, dtype=pl.Int64).alias("fold"))
		34	+ frames.append(d.select(["entity_id", "symbol", "fiscal_period_end", "h", "cutoff", "A_q", "sic_division", "release_time",
		35	+ "y", "p", "model", "part", "tag", "target", "fold"]))
		36	+ return pl.concat(frames)
		37	+
		38	+
		39	+def contrast(a: pl.DataFrame, b: pl.DataFrame, n_boot: int, seed: int) -> dict:
		40	+ key = ["entity_id", "fiscal_period_end", "cutoff"]
		41	+ j = a.select(key + ["y", "p", "A_q"]).join(b.select(key + [pl.col("p").alias("pb")]), on=key, how="inner")
		42	+ y = j["y"].to_numpy().astype(int); pa, pb = j["p"].to_numpy(), j["pb"].to_numpy()
		43	+ d_ll = pointwise_logloss(y, pa) - pointwise_logloss(y, pb)
		44	+ d_br = (pa - y) ** 2 - (pb - y) ** 2
		45	+ blocks = quarter_blocks(j["A_q"].to_numpy())
		46	+ bll, bbr = paired_block_bootstrap(d_ll, blocks, n_boot, seed), paired_block_bootstrap(d_br, blocks, n_boot, seed)
		47	+ gw = dm_gw_test(d_ll, blocks)
		48	+ return {"n": int(len(y)), "d_logloss": bll["mean"], "ci_lo": bll["ci_lo"], "ci_hi": bll["ci_hi"], "p_boot": bll["p_boot"],
		49	+ "gw_t": gw["t"], "gw_p": gw["p"], "n_quarters": gw["T"], "d_brier": bbr["mean"], "brier_ci_lo": bbr["ci_lo"], "brier_ci_hi": bbr["ci_hi"]}
		50	+
		51	+
		52	+def main() -> int:
		53	+ P = Paths.default()
		54	+ log = get_logger("07_eval", P.logs / "07_eval.log")
		55	+ cfg = load_config("research")
		56	+ hp, nb, seed = cfg["cutoffs"]["primary_horizon_td"], cfg["inference"]["n_bootstrap"], cfg["seed"]
		57	+ preds = load_preds(P)
		58	+ log.info("prediction rows: %d (%d files)", preds.height, len(list((P.evaluation / "preds").glob("*.parquet"))))
		59	+
		60	+ # 1) metrics for every binary prediction set
		61	+ rows = []
		62	+ grp = preds.filter(pl.col("target") != "S_opinc").with_columns(pl.col("cutoff").dt.year().alias("year"))
		63	+ for keys, g in grp.group_by(["h", "target", "model", "part", "tag"], maintain_order=True):
		64	+ h, target, model, part, tag = keys
		65	+ m = binary_metrics(g["y"].to_numpy().astype(int), g["p"].to_numpy())
		66	+ rows.append({"h": h, "target": target, "model": model, "part": part, "tag": tag, **m})
		67	+ metrics = pd.DataFrame(rows).sort_values(["target", "h", "tag", "part", "model"])
		68	+ metrics.to_csv(P.evaluation / "metrics_all.csv", index=False)
		69	+
		70	+ # 2) primary contrasts (h = hp, primary target, chronological test) for logistic and LightGBM
		71	+ out = []
		72	+ base = preds.filter((pl.col("h") == hp) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "test") & (pl.col("tag") == "chrono"))
		73	+ for prefix in ("", "L"):
		74	+ pv = {}
		75	+ for a, b in CONTRASTS:
		76	+ A, B = base.filter(pl.col("model") == prefix + a), base.filter(pl.col("model") == (prefix + b if b != "B0" else "B0"))
		77	+ if A.height == 0 or B.height == 0:
		78	+ continue
		79	+ c = contrast(A, B, nb, seed); c.update({"family": "logistic" if prefix == "" else "lightgbm", "a": prefix + a, "b": prefix + b})
		80	+ out.append(c); pv[(a, b)] = c["gw_p"]
		81	+ for fam, pairs in FAMILIES.items():
		82	+ adj = holm({f"{a}-{b}": pv[(a, b)] for a, b in pairs if (a, b) in pv})
		83	+ for r in out:
		84	+ if r["family"] == ("logistic" if prefix == "" else "lightgbm") and f"{r['a'][len(prefix):]}-{r['b'][len(prefix):]}" in adj:
		85	+ r["holm_family"], r["gw_p_holm"] = fam, adj[f"{r['a'][len(prefix):]}-{r['b'][len(prefix):]}"]
		86	+ contrasts = pd.DataFrame(out); contrasts.to_csv(P.evaluation / "contrasts.csv", index=False)
		87	+ log.info("primary contrasts:\n%s", contrasts[["family", "a", "b", "n", "d_logloss", "ci_lo", "ci_hi", "gw_p"]].to_string())
		88	+
		89	+ # 3) horizon ladder (B5 vs B2) logistic chrono + LightGBM fixed
		90	+ lad = []
		91	+ for h in sorted(preds["h"].unique().to_list()):
		92	+ for fam, tag, a, b in (("logistic", "chrono", "B5", "B2"), ("lightgbm", "fixed", "LB5", "LB2")):
		93	+ s = preds.filter((pl.col("h") == h) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "test") & (pl.col("tag") == tag))
		94	+ A, B = s.filter(pl.col("model") == a), s.filter(pl.col("model") == b)
		95	+ if A.height and B.height:
		96	+ c = contrast(A, B, nb, seed); c.update({"h": h, "family": fam})
		97	+ c["auc_a"] = binary_metrics(A["y"].to_numpy().astype(int), A["p"].to_numpy()).get("auc")
		98	+ c["auc_b"] = binary_metrics(B["y"].to_numpy().astype(int), B["p"].to_numpy()).get("auc")
		99	+ lad.append(c)
		100	+ ladder = pd.DataFrame(lad); ladder.to_csv(P.evaluation / "horizon_ladder.csv", index=False)
		101	+ h4 = {}
		102	+ for fam, g in ladder.groupby("family"):
		103	+ g = g.sort_values("h", ascending=False)
		104	+ from scipy.stats import spearmanr
		105	+ rho, p = spearmanr(-g["h"], -g["d_logloss"]) # gain (= -d) should rise as h falls
		106	+ h4[fam] = {"spearman_gain_vs_minus_h": float(rho), "p": float(p), "horizons": g["h"].tolist(), "d_logloss": g["d_logloss"].round(5).tolist()}
		107	+ write_json(P.evaluation / "h4_timing.json", h4)
		108	+
		109	+ # 4) secondary targets (B5 vs B2) logistic + lightgbm at hp
		110	+ sec = []
		111	+ for target in ("y_adverse_netinc", "y_roa_decline", "y_ocf_negative", "y_liquidity_deterioration", "y_abs_surprise", "y_adverse_opinc"):
		112	+ for fam, a, b in (("logistic", "B5", "B2"), ("lightgbm", "LB5", "LB2")):
		113	+ s = preds.filter((pl.col("h") == hp) & (pl.col("target") == target) & (pl.col("part") == "test") & (pl.col("tag") == "chrono"))
		114	+ A, B = s.filter(pl.col("model") == a), s.filter(pl.col("model") == b)
		115	+ if A.height and B.height:
		116	+ c = contrast(A, B, nb, seed); c.update({"target": target, "family": fam, "base_rate": float(A["y"].mean()),
		117	+ "auc_a": binary_metrics(A["y"].to_numpy().astype(int), A["p"].to_numpy()).get("auc"),
		118	+ "auc_b": binary_metrics(B["y"].to_numpy().astype(int), B["p"].to_numpy()).get("auc")})
		119	+ sec.append(c)
		120	+ pd.DataFrame(sec).to_csv(P.evaluation / "secondary_targets.csv", index=False)
		121	+
		122	+ # 5) firm hold-out
		123	+ fh = []
		124	+ for tag in ("firmhold_heldout", "firmhold_seen"):
		125	+ for fam, a, b in (("logistic", "B5", "B2"), ("lightgbm", "LB5", "LB2")):
		126	+ s = preds.filter((pl.col("h") == hp) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "test") & (pl.col("tag") == tag))
		127	+ A, B = s.filter(pl.col("model") == a), s.filter(pl.col("model") == b)
		128	+ if A.height and B.height:
		129	+ c = contrast(A, B, nb, seed); c.update({"tag": tag, "family": fam}); fh.append(c)
		130	+ pd.DataFrame(fh).to_csv(P.evaluation / "firm_holdout.csv", index=False)
		131	+
		132	+ # 6) walk-forward by year
		133	+ wf = []
		134	+ oof = preds.filter((pl.col("tag") == "walkforward") & (pl.col("h") == hp)).with_columns(pl.col("cutoff").dt.year().alias("year"))
		135	+ for keys, g in oof.group_by(["model", "year"], maintain_order=True):
		136	+ m = binary_metrics(g["y"].to_numpy().astype(int), g["p"].to_numpy()); wf.append({"model": keys[0], "year": keys[1], **m})
		137	+ wfd = pd.DataFrame(wf).sort_values(["model", "year"]); wfd.to_csv(P.evaluation / "walkforward_by_year.csv", index=False)
		138	+ wfc = []
		139	+ for fam, a, b in (("logistic", "B5", "B2"), ("lightgbm", "LB5", "LB2")):
		140	+ A, B = oof.filter(pl.col("model") == a), oof.filter(pl.col("model") == b)
		141	+ if A.height and B.height:
		142	+ for y in sorted(A["year"].unique().to_list()):
		143	+ c = contrast(A.filter(pl.col("year") == y), B.filter(pl.col("year") == y), 300, seed); c.update({"family": fam, "year": y}); wfc.append(c)
		144	+ c = contrast(A, B, nb, seed); c.update({"family": fam, "year": "all"}); wfc.append(c)
		145	+ pd.DataFrame(wfc).to_csv(P.evaluation / "walkforward_contrasts.csv", index=False)
		146	+
		147	+ # 7) reliability (test, chrono) for B2, B5, LB2, LB5
		148	+ rel = []
		149	+ for model in ("B2", "B5", "LB2", "LB5"):
		150	+ s = base.filter(pl.col("model") == model)
		151	+ if s.height:
		152	+ t = reliability_table(s["y"].to_numpy().astype(int), s["p"].to_numpy()); t["model"] = model; rel.append(t)
		153	+ if rel:
		154	+ pd.concat(rel).to_csv(P.evaluation / "reliability.csv", index=False)
		155	+
		156	+ # 8) continuous target
		157	+ cont = []
		158	+ fitlog = json.load(open(P.evaluation / "baselines_fit_log.json"))["fits"]
		159	+ for model in ("B2", "B5"):
		160	+ s = preds.filter((pl.col("target") == "S_opinc") & (pl.col("model") == model) & (pl.col("tag") == "chrono_cont"))
		161	+ if s.height:
		162	+ rec = next(r for r in fitlog if r.get("tag") == "chrono_cont" and r["model"] == model)
		163	+ lo, hi = rec["winsor"]
		164	+ y = np.clip(s["y"].to_numpy(), lo, hi)
		165	+ cont.append({"model": model, **continuous_metrics(y, s["p"].to_numpy(), rec["train_mean"])})
		166	+ if cont:
		167	+ A = preds.filter((pl.col("target") == "S_opinc") & (pl.col("model") == "B5") & (pl.col("tag") == "chrono_cont"))
		168	+ B = preds.filter((pl.col("target") == "S_opinc") & (pl.col("model") == "B2") & (pl.col("tag") == "chrono_cont"))
		169	+ key = ["entity_id", "fiscal_period_end", "cutoff"]
		170	+ j = A.join(B.select(key + [pl.col("p").alias("pb")]), on=key)
		171	+ lo, hi = next(r for r in fitlog if r.get("tag") == "chrono_cont" and r["model"] == "B5")["winsor"]
		172	+ y = np.clip(j["y"].to_numpy(), lo, hi)
		173	+ d = (j["p"].to_numpy() - y) ** 2 - (j["pb"].to_numpy() - y) ** 2
		174	+ bb = paired_block_bootstrap(d, quarter_blocks(j["A_q"].to_numpy()), nb, seed)
		175	+ cont.append({"model": "B5-B2 d_mse", **bb})
		176	+ pd.DataFrame(cont).to_csv(P.evaluation / "continuous.csv", index=False)
		177	+
		178	+ # 9) power (validation period, logistic B5 vs B2): MDE at 80 % power, alpha 5 %, quarter clustering
		179	+ v = preds.filter((pl.col("h") == hp) & (pl.col("target") == "y_adverse_opinc") & (pl.col("part") == "validation") & (pl.col("tag") == "chrono"))
		180	+ A, B = v.filter(pl.col("model") == "B5"), v.filter(pl.col("model") == "B2")
		181	+ power = {}
		182	+ if A.height and B.height:
		183	+ key = ["entity_id", "fiscal_period_end", "cutoff"]
		184	+ j = A.join(B.select(key + [pl.col("p").alias("pb")]), on=key)
		185	+ y = j["y"].to_numpy().astype(int)
		186	+ d = pointwise_logloss(y, j["p"].to_numpy()) - pointwise_logloss(y, j["pb"].to_numpy())
		187	+ bl = quarter_blocks(j["A_q"].to_numpy()); ub = np.unique(bl)
		188	+ dq = np.array([d[bl == b].mean() for b in ub]); se = dq.std(ddof=1) / np.sqrt(len(ub))
		189	+ n_test_q = preds.filter((pl.col("part") == "test") & (pl.col("tag") == "chrono") & (pl.col("h") == hp))["A_q"].to_numpy()
		190	+ T_test = len(np.unique(quarter_blocks(n_test_q))) if len(n_test_q) else 23
		191	+ se_test = dq.std(ddof=1) / np.sqrt(T_test)
		192	+ power = {"validation_quarters": int(len(ub)), "sd_quarterly_diff": float(dq.std(ddof=1)), "test_quarters_assumed": int(T_test),
		193	+ "mde_logloss_80pct_power": float((1.96 + 0.84) * se_test), "validation_mean_diff": float(d.mean())}
		194	+ write_json(P.evaluation / "power.json", power)
		195	+
		196	+ # copies to results/
		197	+ for f in ("metrics_all", "contrasts", "horizon_ladder", "secondary_targets", "firm_holdout", "walkforward_by_year",
		198	+ "walkforward_contrasts", "reliability", "continuous"):
		199	+ src = P.evaluation / f"{f}.csv"
		200	+ if src.exists():
		201	+ (P.results / f"{f}.csv").write_text(src.read_text())
		202	+ write_json(P.results / "evaluation_summary.json", {"run_at": now_iso(), "power": power, "h4": h4,
		203	+ "primary": contrasts.to_dict(orient="records") if len(contrasts) else []})
		204	+ print(contrasts[["family", "a", "b", "n", "d_logloss", "ci_lo", "ci_hi", "gw_p"]].to_string() if len(contrasts) else "no contrasts")
		205	+ return 0
		206	+
		207	+
		208	+if __name__ == "__main__":
		209	+ raise SystemExit(main())

added scripts/08_run_robustness.py +156 −0

				@@ -0,0 +1,156 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 5: negative controls, subgroup stress tests (H3), panel regressions (H3), pricing tests (H5)."""
		3	+from __future__ import annotations
		4	+
		5	+import json
		6	+import sys
		7	+from pathlib import Path
		8	+
		9	+import numpy as np
		10	+import pandas as pd
		11	+import polars as pl
		12	+
		13	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		14	+from wp39.inference import binary_metrics, paired_block_bootstrap, pointwise_logloss, quarter_blocks, two_way_cluster_ols # noqa: E402
		15	+from wp39.models import Preprocessor, columns_for, predict_proba # noqa: E402
		16	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		17	+from wp39.portfolio import car_window, download_french, factor_alpha, monthly_decile_portfolios, read_french_daily # noqa: E402
		18	+from wp39.provenance import now_iso, write_json # noqa: E402
		19	+from wp39.robustness import add_terciles, shuffle_labels_within_quarter, subgroup_masks # noqa: E402
		20	+from wp39.samples import feature_columns # noqa: E402
		21	+from wp39.splits import chronological # noqa: E402
		22	+
		23	+KEY = ["entity_id", "fiscal_period_end", "cutoff"]
		24	+FIXED = {"num_leaves": 15, "learning_rate": 0.03, "min_child_samples": 100, "feature_fraction": 0.7, "bagging_fraction": 0.8,
		25	+ "bagging_freq": 1, "lambda_l2": 1.0, "n_estimators": 600, "verbose": -1, "n_jobs": 12}
		26	+
		27	+
		28	+def fixed_fit(train, test, cols, target, seed):
		29	+ import lightgbm as lgb
		30	+ pp = Preprocessor(cols).fit(train)
		31	+ cut = train["cutoff"].quantile(0.85)
		32	+ tr, va = train.filter(pl.col("cutoff") < cut), train.filter(pl.col("cutoff") >= cut)
		33	+ m = lgb.LGBMClassifier(random_state=seed, **FIXED)
		34	+ m.fit(pp.transform(tr), tr[target].to_numpy().astype(int), eval_set=[(pp.transform(va), va[target].to_numpy().astype(int))],
		35	+ callbacks=[lgb.early_stopping(50, verbose=False)])
		36	+ return predict_proba(m, pp.transform(test))
		37	+
		38	+
		39	+def paired(A: pl.DataFrame, B: pl.DataFrame, y: str, nb: int, seed: int) -> dict:
		40	+ j = A.select(KEY + [y, "p", "A_q"]).join(B.select(KEY + [pl.col("p").alias("pb")]), on=KEY)
		41	+ yy = j[y].to_numpy().astype(int)
		42	+ d = pointwise_logloss(yy, j["p"].to_numpy()) - pointwise_logloss(yy, j["pb"].to_numpy())
		43	+ r = paired_block_bootstrap(d, quarter_blocks(j["A_q"].to_numpy()), nb, seed)
		44	+ r["auc_a"] = binary_metrics(yy, j["p"].to_numpy()).get("auc"); r["auc_b"] = binary_metrics(yy, j["pb"].to_numpy()).get("auc")
		45	+ r["base_rate"] = float(yy.mean())
		46	+ return r
		47	+
		48	+
		49	+def main() -> int:
		50	+ P = Paths.default()
		51	+ log = get_logger("08_robust", P.logs / "08_robust.log")
		52	+ cfg = load_config("research"); hp, nb, seed = cfg["cutoffs"]["primary_horizon_td"], cfg["inference"]["n_bootstrap"], cfg["seed"]
		53	+ y = "y_adverse_opinc"
		54	+ df = pl.read_parquet(P.processed / "sample_event.parquet").filter((pl.col("h") == hp) & pl.col(y).is_not_null())
		55	+ fc = feature_columns(df)
		56	+ sp = chronological(df, cfg["splits"])
		57	+ trv = pl.concat([sp["train"], sp["validation"]])
		58	+
		59	+ # ---------------------------------------------------------------- placebo 1: labels shuffled within calendar quarter
		60	+ plc = []
		61	+ for s in range(3):
		62	+ shuf_tr = shuffle_labels_within_quarter(trv, y, seed + s); shuf_te = shuffle_labels_within_quarter(sp["test"], y, seed + 100 + s)
		63	+ A = shuf_te.with_columns(pl.Series("p", fixed_fit(shuf_tr, shuf_te, columns_for("B5", fc), y, seed)))
		64	+ B = shuf_te.with_columns(pl.Series("p", fixed_fit(shuf_tr, shuf_te, columns_for("B2", fc), y, seed)))
		65	+ r = paired(A, B, y, nb, seed); r.update({"placebo": "shuffled_labels", "draw": s}); plc.append(r)
		66	+ log.info("placebo shuffled labels draw %d: d=%.5f [%.5f, %.5f] auc5=%.3f", s, r["mean"], r["ci_lo"], r["ci_hi"], r["auc_a"])
		67	+ pd.DataFrame(plc).to_csv(P.evaluation / "placebo_shuffled.csv", index=False)
		68	+
		69	+ # ---------------------------------------------------------------- subgroups on the chronological LightGBM test predictions
		70	+ preds = {m: pl.read_parquet(P.evaluation / "preds" / f"event_h{hp}_{y}_{m}_test_chrono.parquet") for m in ("LB2", "LB5", "B2", "B5")}
		71	+ tr_t, te_t = add_terciles(sp["train"], [sp["test"]])
		72	+ te_t = te_t.with_columns([
		73	+ ((pl.col("cutoff") - pl.col("A_prev")).dt.total_days() >= 35).fill_null(True).alias("window_clean"),
		74	+ (pl.col("w1m_present_share") >= 0.98).alias("dense_bars"),
		75	+ ])
		76	+ extra = {"window_clean": pl.col("window_clean"), "window_straddles_prev_release": ~pl.col("window_clean"),
		77	+ "dense_bars": pl.col("dense_bars"), "sparse_bars": ~pl.col("dense_bars")}
		78	+ sub = []
		79	+ for fam, a, b in (("lightgbm", "LB5", "LB2"), ("logistic", "B5", "B2")):
		80	+ for name, mask in {**subgroup_masks(te_t), **extra}.items():
		81	+ ids = te_t.filter(mask).select(KEY)
		82	+ A, B = preds[a].join(ids, on=KEY), preds[b].join(ids, on=KEY)
		83	+ if A.height < 500:
		84	+ continue
		85	+ r = paired(A, B, y, nb, seed); r.update({"family": fam, "subgroup": name}); sub.append(r)
		86	+ subdf = pd.DataFrame(sub); subdf.to_csv(P.evaluation / "subgroups.csv", index=False)
		87	+ log.info("subgroups:\n%s", subdf[subdf.family == "lightgbm"][["subgroup", "n", "mean", "ci_lo", "ci_hi", "auc_a", "auc_b"]].to_string())
		88	+
		89	+ # ---------------------------------------------------------------- H3 panel regressions on walk-forward OOF signals
		90	+ oof5 = pl.read_parquet(P.evaluation / "preds" / f"event_h{hp}_{y}_LB5_oof_walkforward.parquet").select(KEY + [pl.col("p").alias("lfds5")])
		91	+ oof2 = pl.read_parquet(P.evaluation / "preds" / f"event_h{hp}_{y}_LB2_oof_walkforward.parquet").select(KEY + [pl.col("p").alias("lfds2")])
		92	+ pan = df.join(oof5, on=KEY).join(oof2, on=KEY)
		93	+ pan = pan.with_columns([
		94	+ (pl.col("cutoff").dt.year().cast(pl.Utf8) + "Q" + ((pl.col("cutoff").dt.month() - 1) // 3 + 1).cast(pl.Utf8)).alias("cq"),
		95	+ ])
		96	+ pan = pan.with_columns((pl.col("sic_division") + "|" + pl.col("cq")).alias("sector_q"))
		97	+ # ex-ante cross-sectional terciles within calendar quarter
		98	+ for name, col in (("size", "f_log_mcap"), ("illiq", "wd_amihud60"), ("lev", "f_leverage")):
		99	+ pan = pan.with_columns(((pl.col(col).rank("average").over("cq") - 1) / pl.len().over("cq") * 3).floor().clip(0, 2).cast(pl.Int8).alias(f"t_{name}"))
		100	+ pd_ = pan.to_pandas()
		101	+ lo, hi = np.quantile(pd_["S_opinc"].dropna(), [0.01, 0.99]); pd_["S_w"] = pd_["S_opinc"].clip(lo, hi)
		102	+ pd_["d_lfds"] = pd_["lfds5"] - pd_["lfds2"]
		103	+ for name in ("size", "illiq", "lev"):
		104	+ for k in (0, 2):
		105	+ pd_[f"lfds5_x_{name}{k}"] = pd_["lfds5"] * (pd_[f"t_{name}"] == k)
		106	+ pd_[f"dlfds_x_{name}{k}"] = pd_["d_lfds"] * (pd_[f"t_{name}"] == k)
		107	+ regs = {}
		108	+ controls = ["f_roa", "f_leverage", "f_log_mcap", "wd_ret20", "wd_vol20", "last_surprise", "mean_surprise4"]
		109	+ regs["S_on_lfds5"] = two_way_cluster_ols(pd_, "S_w", ["lfds5"] + controls, "entity_id", "sector_q", "entity_id", "cq")
		110	+ regs["S_on_lfds2_and_dlfds"] = two_way_cluster_ols(pd_, "S_w", ["lfds2", "d_lfds"] + controls, "entity_id", "sector_q", "entity_id", "cq")
		111	+ regs["y_on_lfds5"] = two_way_cluster_ols(pd_.assign(yb=pd_[y].astype(float)), "yb", ["lfds5"] + controls, "entity_id", "sector_q", "entity_id", "cq")
		112	+ for name in ("size", "illiq", "lev"):
		113	+ regs[f"dlfds_x_{name}"] = two_way_cluster_ols(pd_, "S_w", ["lfds2", "d_lfds", f"dlfds_x_{name}0", f"dlfds_x_{name}2"] + controls,
		114	+ "entity_id", "sector_q", "entity_id", "cq")
		115	+ allreg = pd.concat([r.assign(spec=k) for k, r in regs.items()]); allreg.to_csv(P.evaluation / "panel_regressions.csv", index=False)
		116	+ log.info("panel:\n%s", allreg[allreg.spec.isin(["S_on_lfds5", "S_on_lfds2_and_dlfds"])][["spec", "var", "coef", "se", "t", "p", "n"]].head(12).to_string())
		117	+
		118	+ # ---------------------------------------------------------------- H5a: announcement-window CAR on the ex-ante signal
		119	+ ext = P.cache / "external"
		120	+ ff = read_french_daily(download_french(load_config("data_contract")["external"]["ff5_daily"], ext / "ff5_daily.zip"))
		121	+ mom = read_french_daily(download_french(load_config("data_contract")["external"]["mom_daily"], ext / "mom_daily.zip"))
		122	+ ff = ff.join(mom.iloc[:, :1].rename(columns={mom.columns[0]: "Mom"}), how="left")
		123	+ daily = pl.concat([pl.read_parquet(p).select(["symbol", "session_date", "d_ret_cc", "d_close", "d_dollar_vol"]) for p in (P.cache / "features").glob("*.parquet")])
		124	+ syms = set(pan["symbol"].unique().to_list())
		125	+ daily = daily.filter(pl.col("symbol").is_in(list(syms)))
		126	+ car = car_window(daily, ff, pan.select(["entity_id", "symbol", "cutoff", "A_q", "fiscal_period_end"]))
		127	+ pc = pd_.merge(car.to_pandas(), on=["entity_id", "symbol", "cutoff", "A_q", "fiscal_period_end"], how="inner")
		128	+ lo, hi = np.quantile(pc["car"].dropna(), [0.01, 0.99]); pc["car_w"] = pc["car"].clip(lo, hi)
		129	+ h5 = {"car_on_lfds5": two_way_cluster_ols(pc, "car_w", ["lfds5"] + controls, "entity_id", "sector_q", "entity_id", "cq"),
		130	+ "car_on_lfds2_dlfds": two_way_cluster_ols(pc, "car_w", ["lfds2", "d_lfds"] + controls, "entity_id", "sector_q", "entity_id", "cq")}
		131	+ pd.concat([r.assign(spec=k) for k, r in h5.items()]).to_csv(P.evaluation / "h5_car_regressions.csv", index=False)
		132	+ log.info("H5 CAR:\n%s", h5["car_on_lfds2_dlfds"][["var", "coef", "se", "t", "p", "n"]].head(3).to_string())
		133	+
		134	+ # ---------------------------------------------------------------- H5b: monthly decile sorts (calendar design, OOF scores)
		135	+ port_out = {}
		136	+ for m in ("LB5", "LB2"):
		137	+ f = P.evaluation / "preds" / f"calendar_h0_{y}_{m}_oof_walkforward.parquet"
		138	+ if not f.exists():
		139	+ continue
		140	+ sig = pl.read_parquet(f).select(["symbol", "cutoff", pl.col("p").alias("score")]).to_pandas()
		141	+ res = monthly_decile_portfolios(sig, daily.to_pandas(), ff, cost_bps=20.0)
		142	+ port = res["portfolios"]; port.to_csv(P.evaluation / f"portfolios_{m}.csv", index=False)
		143	+ port_out[m] = {"gross": factor_alpha(port, ff, 1, 10, net=False), "net20bps": factor_alpha(port, ff, 1, 10, net=True),
		144	+ "decile_means": port.groupby("decile")["ret"].mean().round(5).to_dict(),
		145	+ "avg_turnover": float(port["turnover"].mean())}
		146	+ log.info("portfolio %s: %s", m, port_out[m]["gross"])
		147	+ write_json(P.evaluation / "h5_portfolios.json", port_out)
		148	+
		149	+ for f in ("placebo_shuffled", "subgroups", "panel_regressions", "h5_car_regressions"):
		150	+ (P.results / f"{f}.csv").write_text((P.evaluation / f"{f}.csv").read_text())
		151	+ write_json(P.results / "robustness_summary.json", {"run_at": now_iso(), "placebo": plc, "portfolios": port_out})
		152	+ return 0
		153	+
		154	+
		155	+if __name__ == "__main__":
		156	+ raise SystemExit(main())

added scripts/09_make_manuscript.py +203 −0

				@@ -0,0 +1,203 @@
		1	+#!/usr/bin/env python
		2	+"""Phase 6: generate LaTeX tables, figures and a results-macro file from saved artifacts (never from memory)."""
		3	+from __future__ import annotations
		4	+
		5	+import json
		6	+import sys
		7	+from pathlib import Path
		8	+
		9	+import numpy as np
		10	+import pandas as pd
		11	+import polars as pl
		12	+
		13	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		14	+from wp39 import figures as F # noqa: E402
		15	+from wp39.pipeline import Paths, get_logger, load_config # noqa: E402
		16	+
		17	+T = {} # macros
		18	+
		19	+
		20	+def macro(name: str, value, fmt: str = "{}"):
		21	+ T[name] = fmt.format(value)
		22	+
		23	+
		24	+def fnum(x, d=4):
		25	+ return "" if x is None or (isinstance(x, float) and np.isnan(x)) else f"{x:.{d}f}"
		26	+
		27	+
		28	+def stars(p):
		29	+ return "***" if p < 0.01 else "**" if p < 0.05 else "*" if p < 0.1 else ""
		30	+
		31	+
		32	+def tex_table(df: pd.DataFrame, path: Path, cols: list[tuple[str, str, str]], note: str = ""):
		33	+ """cols: (column, header, fmt) — fmt is a python format or 'int'/'str'."""
		34	+ lines = ["\\begin{tabular}{l" + "r" * (len(cols) - 1) + "}", "\\toprule", " & ".join(h for _, h, _ in cols) + " \\\\", "\\midrule"]
		35	+ for _, r in df.iterrows():
		36	+ cells = []
		37	+ for c, _, fmt in cols:
		38	+ v = r[c]
		39	+ if fmt == "int":
		40	+ cells.append(f"{int(v):,}" if pd.notna(v) else "")
		41	+ elif fmt == "str":
		42	+ cells.append(str(v).replace("_", " ").replace("%", "\\%"))
		43	+ else:
		44	+ cells.append(fmt.format(v) if pd.notna(v) else "")
		45	+ lines.append(" & ".join(cells) + " \\\\")
		46	+ lines += ["\\bottomrule", "\\end{tabular}"]
		47	+ path.write_text("\n".join(lines) + "\n")
		48	+
		49	+
		50	+def main() -> int:
		51	+ P = Paths.default()
		52	+ log = get_logger("09_manuscript", P.logs / "09_manuscript.log")
		53	+ cfg = load_config("research"); hp = cfg["cutoffs"]["primary_horizon_td"]
		54	+ E, R, TB, FG = P.evaluation, P.results, P.paper_tables, P.paper_figures
		55	+ ua = json.load(open(R / "universe_audit.json")); pa = json.load(open(R / "pit_panel_audit.json")); sa = json.load(open(R / "sample_audit.json"))
		56	+ ev = pd.read_parquet(P.processed / "sample_event.parquet", columns=["entity_id", "fiscal_period_end", "h", "cutoff", "A_q", "y_adverse_opinc",
		57	+ "S_opinc", "f_log_mcap", "wd_amihud60", "f_leverage", "release_time", "sic_division"])
		58	+ e20 = ev[ev.h == hp]
		59	+ # ---------------- macros: sample
		60	+ macro("nEntities", ua["entities"], "{:,}"); macro("nEligible", ua["eligible"], "{:,}"); macro("nWithCik", ua["with_cik"], "{:,}")
		61	+ macro("nFirmQuarters", pa["firm_quarters"], "{:,}"); macro("nFirmsPanel", pa["entities"], "{:,}")
		62	+ macro("nSampleEvent", sa["event"]["rows"], "{:,}"); macro("nSampleFirms", sa["event"]["firms"], "{:,}")
		63	+ macro("nSampleQuarters", sa["event"]["quarters"], "{:,}"); macro("nSampleCalendar", sa["calendar"]["rows"], "{:,}")
		64	+ macro("nObsPrimary", len(e20), "{:,}"); macro("baseRatePrimary", 100 * e20.y_adverse_opinc.mean(), "{:.1f}")
		65	+ macro("firstCutoff", str(e20.cutoff.min())[:10]); macro("lastCutoff", str(e20.cutoff.max())[:10])
		66	+ macro("releaseMatchedShare", 100 * next(r["len"] for r in pa["release_match"] if r["release_match"] == "ok") / pa["firm_quarters"], "{:.1f}")
		67	+ macro("filingLagMedian", pa["filing_minus_release_days_quantiles"]["0.5"], "{:.0f}")
		68	+ macro("epsEstimateShare", 100 * pa["eps_estimate_available_share"], "{:.0f}")
		69	+ # ---------------- Table 2: universe cascade
		70	+ tex_table(pd.read_csv(R / "table_universe_cascade.csv"), TB / "tab02_universe.tex", [("step", "Step", "str"), ("n", "Firms", "int")])
		71	+ casc = pd.read_csv(R / "table_sample_cascade_event.csv")
		72	+ tex_table(casc, TB / "tab02b_sample_cascade.tex", [("step", "Exclusion step", "str"), ("remaining", "Remaining", "int"), ("dropped", "Dropped", "int")])
		73	+ # ---------------- Table 3: summary statistics (h = 20 sample)
		74	+ feat = pd.read_parquet(P.processed / "sample_event.parquet")
		75	+ feat = feat[feat.h == hp]
		76	+ desc_cols = {"S_opinc": "Scaled operating-income surprise $S_q$", "y_adverse_opinc": "Adverse surprise ($S_q \\le -0.5\\%$)", "f_roa": "ROA (latest available)",
		77	+ "f_leverage": "Debt / assets", "f_log_mcap": "log market cap", "wd_ret20": "20-day return", "wd_vol20": "20-day daily volatility",
		78	+ "wd_amihud60": "Amihud illiquidity (60d, $\\times 10^6$)", "w1m_rv": "1-min realized variance (daily mean)", "w1m_rj": "1-min relative jump",
		79	+ "w1m_rsneg_share": "1-min negative semivariance share", "w1m_down_vol_share": "1-min down-bar volume share", "w1m_present_share": "1-min bars present (share)",
		80	+ "w30m_rv": "30-min realized variance", "w5m_rv": "5-min realized variance", "info_age_td": "Days since last release"}
		81	+ rows = []
		82	+ for c, lab in desc_cols.items():
		83	+ s = pd.to_numeric(feat[c], errors="coerce").replace([np.inf, -np.inf], np.nan).dropna()
		84	+ rows.append({"var": lab, "n": len(s), "mean": s.mean(), "sd": s.std(), "p10": s.quantile(0.1), "p50": s.median(), "p90": s.quantile(0.9)})
		85	+ tex_table(pd.DataFrame(rows), TB / "tab03_summary.tex", [("var", "Variable", "str"), ("n", "N", "int"), ("mean", "Mean", "{:.4g}"), ("sd", "SD", "{:.4g}"),
		86	+ ("p10", "P10", "{:.4g}"), ("p50", "Median", "{:.4g}"), ("p90", "P90", "{:.4g}")])
		87	+ # ---------------- Table 4: information sets
		88	+ fc = json.load(open(E / "feature_columns_event.json"))
		89	+ info = pd.DataFrame([{"set": "B1", "desc": "PIT fundamentals + history + size", "k": len(fc["fundamentals"])},
		90	+ {"set": "B2", "desc": "B1 + daily OHLCV (20/60-day)", "k": len(fc["fundamentals"]) + len(fc["daily"])},
		91	+ {"set": "B3", "desc": "B2 + 30-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_30m"])},
		92	+ {"set": "B4", "desc": "B2 + 5-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_5m"])},
		93	+ {"set": "B5", "desc": "B2 + 1-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_1m"])},
		94	+ {"set": "B5all", "desc": "B2 + 30m + 5m + 1m", "k": sum(len(v) for v in fc.values())}])
		95	+ tex_table(info, TB / "tab04_info_sets.tex", [("set", "Set", "str"), ("desc", "Information set", "str"), ("k", "Features", "int")])
		96	+ macro("kFund", len(fc["fundamentals"])); macro("kDaily", len(fc["daily"])); macro("kIntra", len(fc["intraday_1m"]))
		97	+ # ---------------- Table 5: headline OOS metrics (h=20, chrono test)
		98	+ met = pd.read_csv(E / "metrics_all.csv")
		99	+ m20 = met[(met.h == hp) & (met.target == "y_adverse_opinc") & (met.part == "test") & (met.tag == "chrono")].copy()
		100	+ order = ["B0", "B0b", "B1", "B2", "B3", "B4", "B5", "B5all", "LB1", "LB2", "LB3", "LB4", "LB5", "LB5all"]
		101	+ m20["o"] = m20.model.map({m: i for i, m in enumerate(order)}); m20 = m20.sort_values("o")
		102	+ tex_table(m20, TB / "tab05_headline.tex", [("model", "Model", "str"), ("n", "N", "int"), ("logloss", "Log loss", "{:.4f}"), ("brier", "Brier", "{:.4f}"),
		103	+ ("auc", "AUC", "{:.3f}"), ("pr_auc", "PR-AUC", "{:.3f}"), ("cal_slope", "Cal. slope", "{:.2f}"), ("cal_intercept", "Cal. int.", "{:.2f}")])
		104	+ for _, r in m20.iterrows():
		105	+ macro(f"ll{r.model}", r.logloss, "{:.4f}"); macro(f"auc{r.model}", r.auc, "{:.3f}")
		106	+ # ---------------- Table 5b: paired contrasts
		107	+ con = pd.read_csv(E / "contrasts.csv")
		108	+ con["ci"] = con.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)
		109	+ con["gwp"] = con.apply(lambda r: f"{r.gw_p:.3f}{stars(r.gw_p)}", axis=1)
		110	+ con["holm"] = con["gw_p_holm"].map(lambda v: "" if pd.isna(v) else f"{v:.3f}")
		111	+ tex_table(con, TB / "tab05b_contrasts.tex", [("family", "Family", "str"), ("a", "A", "str"), ("b", "B", "str"), ("n", "N", "int"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"),
		112	+ ("ci", "95\\% CI", "str"), ("gwp", "GW $p$", "str"), ("holm", "Holm", "str"), ("d_brier", "$\\Delta$ Brier", "{:+.5f}")])
		113	+ for _, r in con.iterrows():
		114	+ k = f"{r.family[:3]}{r.a}vs{r.b}".replace("B5all", "Ball")
		115	+ macro(f"d{k}", r.d_logloss, "{:+.4f}"); macro(f"lo{k}", r.ci_lo, "{:+.4f}"); macro(f"hi{k}", r.ci_hi, "{:+.4f}"); macro(f"p{k}", r.gw_p, "{:.3f}")
		116	+ macro(f"n{k}", r.n, "{:,}")
		117	+ # ---------------- Table 7: horizon ladder, secondary targets, firm holdout
		118	+ lad = pd.read_csv(E / "horizon_ladder.csv")
		119	+ lad["ci"] = lad.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)
		120	+ tex_table(lad.sort_values(["family", "h"], ascending=[True, False]), TB / "tab07_horizons.tex", [("family", "Family", "str"), ("h", "$h$", "int"), ("n", "N", "int"),
		121	+ ("auc_b", "AUC B2", "{:.3f}"), ("auc_a", "AUC B5", "{:.3f}"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")])
		122	+ h4 = json.load(open(E / "h4_timing.json"))
		123	+ for fam, v in h4.items():
		124	+ macro(f"h4rho{fam}", v["spearman_gain_vs_minus_h"], "{:+.2f}"); macro(f"h4p{fam}", v["p"], "{:.2f}")
		125	+ sec = pd.read_csv(E / "secondary_targets.csv")
		126	+ sec["ci"] = sec.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)
		127	+ tex_table(sec, TB / "tab10_targets.tex", [("target", "Target", "str"), ("family", "Family", "str"), ("n", "N", "int"), ("base_rate", "Base rate", "{:.3f}"),
		128	+ ("auc_b", "AUC B2", "{:.3f}"), ("auc_a", "AUC B5", "{:.3f}"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")])
		129	+ fh = pd.read_csv(E / "firm_holdout.csv"); fh["ci"] = fh.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)
		130	+ tex_table(fh, TB / "tab09b_firmholdout.tex", [("tag", "Firms", "str"), ("family", "Family", "str"), ("n", "N", "int"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")])
		131	+ # ---------------- Table 6: panel regressions
		132	+ pr = pd.read_csv(E / "panel_regressions.csv")
		133	+ pr = pr[pr["var"].isin(["lfds5", "lfds2", "d_lfds", "dlfds_x_size0", "dlfds_x_size2", "dlfds_x_illiq0", "dlfds_x_illiq2", "dlfds_x_lev0", "dlfds_x_lev2"])].copy()
		134	+ pr["coef_s"] = pr.apply(lambda r: f"{r.coef:.4f}{stars(r.p)}", axis=1); pr["se_s"] = pr.se.map(lambda v: f"({v:.4f})")
		135	+ tex_table(pr, TB / "tab06_panel.tex", [("spec", "Specification", "str"), ("var", "Regressor", "str"), ("coef_s", "Coef.", "str"), ("se_s", "SE", "str"), ("n", "N", "int"), ("n_firms", "Firms", "int"), ("n_quarters", "Quarters", "int")])
		136	+ for _, r in pr.iterrows():
		137	+ macro(f"pan{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.coef, "{:+.4f}"); macro(f"pant{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.t, "{:.2f}")
		138	+ # ---------------- Table 7b: mechanisms / subgroups; Table 8: placebos
		139	+ sub = pd.read_csv(E / "subgroups.csv"); sub["ci"] = sub.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)
		140	+ tex_table(sub[sub.family == "lightgbm"], TB / "tab07b_subgroups.tex", [("subgroup", "Subgroup", "str"), ("n", "N", "int"), ("base_rate", "Base rate", "{:.3f}"),
		141	+ ("auc_b", "AUC LB2", "{:.3f}"), ("auc_a", "AUC LB5", "{:.3f}"), ("mean", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str")])
		142	+ plc = pd.read_csv(E / "placebo_shuffled.csv"); plc["ci"] = plc.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)
		143	+ tex_table(plc, TB / "tab08_placebo.tex", [("placebo", "Placebo", "str"), ("draw", "Draw", "int"), ("n", "N", "int"), ("auc_a", "AUC LB5", "{:.3f}"), ("auc_b", "AUC LB2", "{:.3f}"), ("mean", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str")])
		144	+ macro("placeboMeanD", plc["mean"].mean(), "{:+.4f}"); macro("placeboAUC", plc["auc_a"].mean(), "{:.3f}")
		145	+ # ---------------- Table 11: economic value
		146	+ h5 = json.load(open(E / "h5_portfolios.json")); car = pd.read_csv(E / "h5_car_regressions.csv")
		147	+ rows = []
		148	+ for m, v in h5.items():
		149	+ for k in ("gross", "net20bps"):
		150	+ a = v[k]; rows.append({"model": m, "variant": k, "alpha": 100 * a["alpha_monthly"], "t": a["alpha_t"], "mean_ls": 100 * a["mean_ls"], "sharpe": a["sharpe_ann"], "T": a["T"], "turnover": v["avg_turnover"]})
		151	+ tex_table(pd.DataFrame(rows), TB / "tab11_economic.tex", [("model", "Score", "str"), ("variant", "Variant", "str"), ("mean_ls", "Mean L/S (\\%/mo)", "{:.2f}"), ("alpha", "FF5+MOM $\\alpha$ (\\%/mo)", "{:+.2f}"),
		152	+ ("t", "$t$ (HAC)", "{:.2f}"), ("sharpe", "Sharpe (ann.)", "{:.2f}"), ("turnover", "Turnover", "{:.2f}"), ("T", "Months", "int")])
		153	+ for m, v in h5.items():
		154	+ macro(f"alpha{m}gross", 100 * v["gross"]["alpha_monthly"], "{:+.2f}"); macro(f"alphat{m}gross", v["gross"]["alpha_t"], "{:.2f}")
		155	+ macro(f"alpha{m}net", 100 * v["net20bps"]["alpha_monthly"], "{:+.2f}"); macro(f"alphat{m}net", v["net20bps"]["alpha_t"], "{:.2f}")
		156	+ carr = car[car["var"].isin(["lfds5", "lfds2", "d_lfds"])].copy(); carr["coef_s"] = carr.apply(lambda r: f"{r.coef:.4f}{stars(r.p)}", axis=1); carr["se_s"] = carr.se.map(lambda v: f"({v:.4f})")
		157	+ tex_table(carr, TB / "tab11b_car.tex", [("spec", "Specification", "str"), ("var", "Regressor", "str"), ("coef_s", "Coef.", "str"), ("se_s", "SE", "str"), ("n", "N", "int")])
		158	+ for _, r in carr.iterrows():
		159	+ macro(f"car{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.coef, "{:+.4f}"); macro(f"cart{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.t, "{:.2f}")
		160	+ # continuous + power
		161	+ cont = pd.read_csv(E / "continuous.csv")
		162	+ tex_table(cont[cont.model.isin(["B2", "B5"])], TB / "tab10b_continuous.tex", [("model", "Model", "str"), ("n", "N", "int"), ("mae", "MAE", "{:.5f}"), ("rmse", "RMSE", "{:.5f}"), ("oos_r2", "OOS $R^2$", "{:.4f}")])
		163	+ for _, r in cont.iterrows():
		164	+ if r.model in ("B2", "B5"):
		165	+ macro(f"r2{r.model}", r.oos_r2, "{:.4f}")
		166	+ pw = json.load(open(E / "power.json")); macro("mde", pw.get("mde_logloss_80pct_power", float("nan")), "{:.4f}"); macro("valQuarters", pw.get("validation_quarters", 0))
		167	+ # ---------------- figures
		168	+ F.fig_timeline(FG)
		169	+ yrs = e20.assign(year=pd.to_datetime(e20.cutoff).dt.year).groupby("year").agg(firms=("entity_id", "nunique"), firm_quarters=("entity_id", "size"), adverse_rate=("y_adverse_opinc", "mean")).reset_index()
		170	+ F.fig_coverage(FG, yrs)
		171	+ te = e20[pd.to_datetime(e20.cutoff) >= pd.Timestamp(str(cfg["splits"]["test"][0]))].copy()
		172	+ prev_rows = [{"label": "All firm-quarters", "sel": np.ones(len(te), bool)}]
		173	+ for name, col, lab in (("size", "f_log_mcap", "Market cap"), ("illiq", "wd_amihud60", "Illiquidity"), ("lev", "f_leverage", "Leverage")):
		174	+ q = te[col].rank(pct=True)
		175	+ prev_rows += [{"label": f"{lab}: bottom tercile", "sel": (q <= 1 / 3).values}, {"label": f"{lab}: top tercile", "sel": (q > 2 / 3).values}]
		176	+ for rt in ("before_open", "after_close"):
		177	+ prev_rows.append({"label": f"Release {rt.replace('_', ' ')}", "sel": (te.release_time == rt).values})
		178	+ pr_ = []
		179	+ for r in prev_rows:
		180	+ y = te.y_adverse_opinc.values[r["sel"]]; n = len(y); p = y.mean() if n else np.nan; se = np.sqrt(p * (1 - p) / n) if n else np.nan
		181	+ pr_.append({"label": r["label"], "rate": p, "lo": p - 1.96 * se, "hi": p + 1.96 * se})
		182	+ F.fig_prevalence(FG, pd.DataFrame(pr_))
		183	+ F.fig_frequency_gains(FG, con)
		184	+ F.fig_reliability(FG, pd.read_csv(E / "reliability.csv"))
		185	+ F.fig_signal_evolution(FG, lad)
		186	+ F.fig_heterogeneity(FG, sub, ["all", "size_small", "size_large", "illiq_high", "illiq_low", "lev_high", "lev_low", "loss_firms", "profit_firms",
		187	+ "covid_2020", "ex_covid", "window_clean", "window_straddles_prev_release", "dense_bars", "sparse_bars", "sector_finance", "sector_nonfinance"])
		188	+ F.fig_rolling(FG, pd.read_csv(E / "walkforward_by_year.csv"), pd.read_csv(E / "walkforward_contrasts.csv"))
		189	+ if "LB5" in h5:
		190	+ F.fig_economic(FG, pd.read_csv(E / "portfolios_LB5.csv"), h5["LB5"])
		191	+ # ---------------- macros file
		192	+ lines = ["% Auto-generated by scripts/09_make_manuscript.py — do not edit"]
		193	+ for k, v in T.items():
		194	+ words = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}
		195	+ kk = "".join(words.get(ch, ch) if ch.isdigit() else ch for ch in k if ch.isalnum())
		196	+ lines.append(f"\\newcommand{{\\{kk}}}{{{v}}}")
		197	+ (P.root / "paper" / "results_macros.tex").write_text("\n".join(lines) + "\n")
		198	+ log.info("wrote %d macros, %d tables, figures to %s", len(T), len(list(TB.glob('*.tex'))), FG)
		199	+ return 0
		200	+
		201	+
		202	+if __name__ == "__main__":
		203	+ raise SystemExit(main())

added src/wp39/__init__.py +2 −0

				@@ -0,0 +1,2 @@
		1	+"""WP39 UQO — The Invisible Balance Sheet (research package)."""
		2	+__version__ = "0.1.0"

added src/wp39/api_client.py +226 −0

				@@ -0,0 +1,226 @@
		1	+"""Typed, redacting client for the FinData OS API (https://findata.spboucher.ai/api/v1).
		2	+
		3	+Design rules (docs/API_INTEGRATION.md):
		4	+- the bearer token is read from the environment or a mode-600 file, never logged, never placed in a URL;
		5	+- bounded exponential backoff with jitter on 429/5xx and transport errors (idempotent GETs; export creation only when
		6	+ no export id was obtained);
		7	+- every call appends one provenance line (method, path, status, request id, dataset id/version, elapsed) to a JSONL log;
		8	+- Parquet exports are validated against the published bars schema before being accepted.
		9	+"""
		10	+from __future__ import annotations
		11	+
		12	+import hashlib
		13	+import json
		14	+import os
		15	+import random
		16	+import re
		17	+import time
		18	+from dataclasses import dataclass, asdict
		19	+from pathlib import Path
		20	+from typing import Any, Iterator
		21	+
		22	+import httpx
		23	+
		24	+_TOKEN_RE = re.compile(r"fd_[A-Za-z0-9_.\-]+")
		25	+
		26	+
		27	+def _redact(text: str) -> str:
		28	+ return _TOKEN_RE.sub("fd_<REDACTED>", text)
		29	+
		30	+
		31	+class FinDataError(RuntimeError):
		32	+ def __init__(self, status: int, code: str, message: str, request_id: str | None = None):
		33	+ super().__init__(f"HTTP {status} {code}: {_redact(message)} (request_id={request_id})")
		34	+ self.status, self.code, self.request_id = status, code, request_id
		35	+
		36	+
		37	+def load_api_key() -> str:
		38	+ key = os.environ.get("FINDATA_API_KEY")
		39	+ if key:
		40	+ return key.strip()
		41	+ path = os.path.expanduser(os.environ.get("FINDATA_API_KEY_FILE", "~/findata-secrets/api-key-wp39.txt"))
		42	+ if not os.path.exists(path):
		43	+ path = os.path.expanduser("~/Desktop/Cluster/secrets/findata/api-key-wp39.txt")
		44	+ with open(path) as fh:
		45	+ return fh.read().strip()
		46	+
		47	+
		48	+@dataclass
		49	+class ExportResult:
		50	+ export_id: str
		51	+ dataset_id: str
		52	+ dataset_version: int | None
		53	+ symbols: list[str]
		54	+ start: str
		55	+ end: str
		56	+ adjustment: str
		57	+ row_count: int
		58	+ bytes: int
		59	+ sha256: str
		60	+ path: str
		61	+ request_id: str | None
		62	+ exported_at: str
		63	+
		64	+
		65	+class FinDataClient:
		66	+ def __init__(
		67	+ self,
		68	+ base_url: str | None = None,
		69	+ api_key: str | None = None,
		70	+ log_path: str | os.PathLike | None = None,
		71	+ timeout: float = 60.0,
		72	+ max_attempts: int = 5,
		73	+ ):
		74	+ self.base_url = (base_url or os.environ.get("FINDATA_BASE_URL") or "https://findata.spboucher.ai/api/v1").rstrip("/")
		75	+ self._key = api_key or load_api_key()
		76	+ self.max_attempts = max_attempts
		77	+ self.log_path = Path(log_path) if log_path else None
		78	+ self._client = httpx.Client(timeout=timeout, headers={"Authorization": f"Bearer {self._key}", "User-Agent": "wp39-uqo/0.1"})
		79	+
		80	+ # ------------------------------------------------------------------ core
		81	+ def _log(self, rec: dict[str, Any]) -> None:
		82	+ if not self.log_path:
		83	+ return
		84	+ self.log_path.parent.mkdir(parents=True, exist_ok=True)
		85	+ with open(self.log_path, "a") as fh:
		86	+ fh.write(json.dumps(rec, default=str) + "\n")
		87	+
		88	+ def _request(self, method: str, path: str, *, params=None, json_body=None, stream_to: Path | None = None,
		89	+ retry: bool = True) -> httpx.Response:
		90	+ url = path if path.startswith("http") else f"{self.base_url}{path}"
		91	+ attempt, delay = 0, 0.5
		92	+ while True:
		93	+ attempt += 1
		94	+ t0 = time.perf_counter()
		95	+ try:
		96	+ if stream_to is not None:
		97	+ with self._client.stream(method, url, params=params, json=json_body) as r:
		98	+ if r.status_code >= 400:
		99	+ r.read()
		100	+ resp = r
		101	+ else:
		102	+ stream_to.parent.mkdir(parents=True, exist_ok=True)
		103	+ h = hashlib.sha256()
		104	+ with open(stream_to, "wb") as fh:
		105	+ for chunk in r.iter_bytes():
		106	+ fh.write(chunk)
		107	+ h.update(chunk)
		108	+ r.sha256 = h.hexdigest() # type: ignore[attr-defined]
		109	+ resp = r
		110	+ else:
		111	+ resp = self._client.request(method, url, params=params, json=json_body)
		112	+ elapsed = time.perf_counter() - t0
		113	+ self._log({
		114	+ "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "status": resp.status_code,
		115	+ "request_id": resp.headers.get("X-Request-ID"), "dataset_id": resp.headers.get("X-Dataset-Id"),
		116	+ "dataset_version": resp.headers.get("X-Dataset-Version"), "elapsed_s": round(elapsed, 3), "attempt": attempt,
		117	+ })
		118	+ if resp.status_code < 400:
		119	+ return resp
		120	+ retryable = resp.status_code in (429, 500, 502, 503, 504)
		121	+ if not (retry and retryable) or attempt >= self.max_attempts:
		122	+ try:
		123	+ err = resp.json().get("error", {})
		124	+ except Exception:
		125	+ err = {"code": "http_error", "message": resp.text[:300]}
		126	+ raise FinDataError(resp.status_code, err.get("code", "http_error"), err.get("message", ""), err.get("request_id"))
		127	+ except (httpx.TransportError, httpx.TimeoutException) as exc:
		128	+ self._log({"ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "status": None,
		129	+ "error": _redact(type(exc).__name__), "attempt": attempt})
		130	+ if not retry or attempt >= self.max_attempts:
		131	+ raise RuntimeError(f"FinData transport failure after {attempt} attempts on {path}: {_redact(str(exc))}") from None
		132	+ time.sleep(delay + random.uniform(0, delay / 2))
		133	+ delay = min(delay * 2, 8.0)
		134	+
		135	+ def get(self, path: str, **params) -> Any:
		136	+ return self._request("GET", path, params={k: v for k, v in params.items() if v is not None}).json()
		137	+
		138	+ def post(self, path: str, body: dict, retry: bool = False) -> Any:
		139	+ return self._request("POST", path, json_body=body, retry=retry).json()
		140	+
		141	+ # --------------------------------------------------------------- catalog
		142	+ def health(self) -> dict:
		143	+ return self.get("/health")
		144	+
		145	+ def openapi(self) -> dict:
		146	+ return self.get("/openapi.json")
		147	+
		148	+ def datasets(self) -> list[dict]:
		149	+ return self.get("/catalog/datasets")["data"]
		150	+
		151	+ def dataset(self, dataset_id: str) -> dict:
		152	+ return self.get(f"/catalog/datasets/{dataset_id}")
		153	+
		154	+ def schema_bars(self) -> dict:
		155	+ return self.get("/catalog/schema/bars")
		156	+
		157	+ # ----------------------------------------------------------- instruments
		158	+ def instrument(self, symbol: str) -> dict:
		159	+ return self.get(f"/instruments/{symbol}")
		160	+
		161	+ def instruments_search(self, query: str | None = None, asset_class: str | None = None, status: str | None = None,
		162	+ limit: int = 200) -> Iterator[dict]:
		163	+ cursor = None
		164	+ while True:
		165	+ page = self.get("/instruments", query=query, asset_class=asset_class, status=status, limit=limit, cursor=cursor)
		166	+ yield from page.get("data", [])
		167	+ cursor = page.get("next_cursor")
		168	+ if not cursor:
		169	+ return
		170	+
		171	+ def coverage_symbols(self, dataset_id: str, min_sessions: int = 1, limit: int = 5000) -> Iterator[dict]:
		172	+ cursor = None
		173	+ while True:
		174	+ page = self.get("/coverage/symbols", dataset_id=dataset_id, min_sessions=min_sessions, limit=limit, cursor=cursor)
		175	+ yield from page.get("data", [])
		176	+ cursor = page.get("next_cursor")
		177	+ if not cursor:
		178	+ return
		179	+
		180	+ def coverage(self, dataset_id: str, symbol: str, start: str, end: str, limit: int = 5000) -> list[dict]:
		181	+ return self.get("/coverage", dataset_id=dataset_id, symbol=symbol, start=start, end=end, limit=limit)["data"]
		182	+
		183	+ # ---------------------------------------------------------- fundamentals
		184	+ def fundamentals(self, entity_id: str, as_of: str | None = None, metrics: list[str] | None = None,
		185	+ period_type: str | None = "q", limit: int = 20000, include_superseded: bool = False) -> dict:
		186	+ return self.get(f"/fundamentals/{entity_id}", as_of=as_of, metrics=",".join(metrics) if metrics else None,
		187	+ period_type=period_type, limit=limit, include_superseded=str(include_superseded).lower())
		188	+
		189	+ def fundamentals_periods(self, entity_id: str) -> list[dict]:
		190	+ return self.get(f"/fundamentals/{entity_id}/periods")["periods"]
		191	+
		192	+ # ----------------------------------------------------------------- query
		193	+ def query(self, sql: str, max_rows: int = 1000) -> tuple[list[str], list[list[Any]], dict]:
		194	+ d = self.post("/query", {"sql": sql, "max_rows": max_rows}, retry=True)
		195	+ return [c["name"] for c in d["columns"]], d["rows"], d.get("meta", {})
		196	+
		197	+ # --------------------------------------------------------------- exports
		198	+ def export(self, dataset_id: str, symbols: list[str], start: str, end: str, dest: Path, adjustment: str = "raw",
		199	+ poll_s: float = 1.0, max_wait_s: float = 600.0) -> ExportResult:
		200	+ body = {"dataset_id": dataset_id, "symbols": symbols, "start": start, "end": end, "adjustment": adjustment, "format": "parquet"}
		201	+ r = self._request("POST", "/exports", json_body=body, retry=False)
		202	+ created = r.json()
		203	+ export_id, request_id = created["id"], r.headers.get("X-Request-ID")
		204	+ t0 = time.time()
		205	+ while True:
		206	+ st = self.get(f"/exports/{export_id}")
		207	+ if st["status"] == "ready":
		208	+ break
		209	+ if st["status"] in ("failed", "expired"):
		210	+ raise FinDataError(500, "export_" + st["status"], str(st.get("error")), request_id)
		211	+ if time.time() - t0 > max_wait_s:
		212	+ raise FinDataError(504, "export_timeout", f"export {export_id} not ready after {max_wait_s}s", request_id)
		213	+ time.sleep(poll_s)
		214	+ resp = self._request("GET", f"/exports/{export_id}/download", stream_to=dest)
		215	+ snap = st.get("snapshot") or {}
		216	+ return ExportResult(export_id=export_id, dataset_id=dataset_id, dataset_version=snap.get("version"), symbols=symbols,
		217	+ start=start, end=end, adjustment=adjustment, row_count=int(st.get("row_count") or 0),
		218	+ bytes=int(st.get("bytes") or 0), sha256=getattr(resp, "sha256", ""), path=str(dest),
		219	+ request_id=request_id, exported_at=time.strftime("%Y-%m-%dT%H:%M:%S%z"))
		220	+
		221	+ def close(self) -> None:
		222	+ self._client.close()
		223	+
		224	+
		225	+def export_result_dict(res: ExportResult) -> dict:
		226	+ return asdict(res)

added src/wp39/calendar.py +78 −0

				@@ -0,0 +1,78 @@
		1	+"""XNYS trading calendar helpers (America/New_York sessions, UTC storage)."""
		2	+from __future__ import annotations
		3	+
		4	+import datetime as dt
		5	+from functools import lru_cache
		6	+from zoneinfo import ZoneInfo
		7	+
		8	+import numpy as np
		9	+import pandas as pd
		10	+
		11	+ET = ZoneInfo("America/New_York")
		12	+UTC = ZoneInfo("UTC")
		13	+
		14	+
		15	+@lru_cache(maxsize=1)
		16	+def _xnys():
		17	+ import exchange_calendars as xc
		18	+ return xc.get_calendar("XNYS", start="2000-01-01", end="2027-12-31")
		19	+
		20	+
		21	+@lru_cache(maxsize=1)
		22	+def sessions() -> pd.DatetimeIndex:
		23	+ """All XNYS sessions 2000→2027 as tz-naive dates."""
		24	+ return _xnys().sessions.tz_localize(None) if _xnys().sessions.tz is not None else _xnys().sessions
		25	+
		26	+
		27	+def session_array() -> np.ndarray:
		28	+ return sessions().to_numpy().astype("datetime64[D]")
		29	+
		30	+
		31	+def is_session(day: dt.date) -> bool:
		32	+ return np.datetime64(day) in set(session_array().tolist())
		33	+
		34	+
		35	+def shift_trading_days(day: dt.date | np.datetime64 | pd.Timestamp, n: int) -> dt.date:
		36	+ """Shift by n trading days; if `day` is not a session, start from the previous session (n<=0) or next (n>0)."""
		37	+ arr = session_array()
		38	+ d = np.datetime64(pd.Timestamp(day).date())
		39	+ pos = int(np.searchsorted(arr, d))
		40	+ if pos < len(arr) and arr[pos] == d:
		41	+ base = pos
		42	+ else:
		43	+ base = pos - 1 if n <= 0 else pos
		44	+ idx = base + n
		45	+ if idx < 0 or idx >= len(arr):
		46	+ raise ValueError("date out of calendar range")
		47	+ return arr[idx].astype("datetime64[D]").astype(dt.date)
		48	+
		49	+
		50	+def trading_days_between(a: dt.date, b: dt.date) -> int:
		51	+ """Number of sessions strictly after a up to and including b (negative if b<a)."""
		52	+ arr = session_array()
		53	+ ia, ib = np.searchsorted(arr, np.datetime64(a), side="right"), np.searchsorted(arr, np.datetime64(b), side="right")
		54	+ return int(ib - ia)
		55	+
		56	+
		57	+def month_end_sessions(start: dt.date, end: dt.date) -> list[dt.date]:
		58	+ s = pd.Series(sessions())
		59	+ s = s[(s >= pd.Timestamp(start)) & (s <= pd.Timestamp(end))]
		60	+ return [d.date() for d in s.groupby(s.dt.to_period("M")).max()]
		61	+
		62	+
		63	+def close_time_utc(session: dt.date) -> pd.Timestamp:
		64	+ """Open instant (UTC) of the closing-print bar (16:00 ET, or 13:00 ET on early-close days)."""
		65	+ cal = _xnys()
		66	+ ts = pd.Timestamp(session)
		67	+ close = cal.session_close(ts)
		68	+ return pd.Timestamp(close).tz_convert("UTC") if close.tz is not None else pd.Timestamp(close).tz_localize("UTC")
		69	+
		70	+
		71	+def regular_session_bounds_utc(session: dt.date) -> tuple[pd.Timestamp, pd.Timestamp]:
		72	+ """[open 09:30 ET, close bar open] in UTC for the session (inclusive bar-open bounds)."""
		73	+ cal = _xnys()
		74	+ ts = pd.Timestamp(session)
		75	+ o, c = cal.session_open(ts), cal.session_close(ts)
		76	+ o = pd.Timestamp(o).tz_convert("UTC") if o.tz is not None else pd.Timestamp(o).tz_localize("UTC")
		77	+ c = pd.Timestamp(c).tz_convert("UTC") if c.tz is not None else pd.Timestamp(c).tz_localize("UTC")
		78	+ return o, c

added src/wp39/catalog.py +31 −0

				@@ -0,0 +1,31 @@
		1	+"""Catalog reconciliation: documented contract vs observed API."""
		2	+from __future__ import annotations
		3	+
		4	+from .api_client import FinDataClient
		5	+
		6	+EXPECTED_BARS_FIELDS = ["instrument_id", "symbol_as_reported", "venue", "asset_class", "interval", "timestamp_utc",
		7	+ "session_date", "open", "high", "low", "close", "volume", "currency", "trade_count", "vwap",
		8	+ "adjustment_type", "source_id", "ingested_at", "quality_flags", "revision_id"]
		9	+
		10	+
		11	+def probe(client: FinDataClient, contract: dict) -> dict:
		12	+ health = client.health()
		13	+ datasets = {d["id"]: d for d in client.datasets()}
		14	+ schema = client.schema_bars()
		15	+ spec = client.openapi()
		16	+ routes = sorted(f"{m.upper()} {p}" for p, ops in spec["paths"].items() for m in ops)
		17	+ needed = [contract["findata"]["datasets"][k] for k in ("bars_1m", "bars_1m_etf")]
		18	+ report = {
		19	+ "health_status": health.get("status"),
		20	+ "n_datasets": len(datasets),
		21	+ "schema_version": schema.get("schema_version"),
		22	+ "timestamp_convention": schema.get("timestamp_convention"),
		23	+ "bars_fields_ok": [f["name"] for f in schema["fields"]] == EXPECTED_BARS_FIELDS,
		24	+ "required_datasets": {d: {"status": datasets.get(d, {}).get("status"), "rows": datasets.get(d, {}).get("row_count"),
		25	+ "first": datasets.get(d, {}).get("first_session"), "last": datasets.get(d, {}).get("last_session"),
		26	+ "license": datasets.get(d, {}).get("license_class")} for d in needed},
		27	+ "has_earnings_route": any("/earnings" in r for r in routes),
		28	+ "routes": routes,
		29	+ "lake_free_gb": (health.get("components", {}).get("lake") or {}).get("free_gb"),
		30	+ }
		31	+ return report

added src/wp39/corporate_actions.py +36 −0

				@@ -0,0 +1,36 @@
		1	+"""Splits and cash dividends (FinData `corporate_actions`, vendor meta files) -> per-symbol ex-date adjustment table.
		2	+
		3	+The extract is produced once on the storage node (read-only `psql \\copy`, see docs/API_INTEGRATION.md) because the API
		4	+has no corporate-actions route; its SHA-256 is in artifacts/manifests/corporate_actions.json. Raw bars stay unadjusted;
		5	+adjustments are applied explicitly in features.daily_block (ex-date rule).
		6	+"""
		7	+from __future__ import annotations
		8	+
		9	+from pathlib import Path
		10	+
		11	+import polars as pl
		12	+
		13	+
		14	+def load_actions(path: Path) -> pl.DataFrame:
		15	+ """Raw Postgres copy: id, instrument_id, action_type (split|dividend), ex_date, value, ... .
		16	+ Vendor quirk: dividends are keyed `firstrate:stock:<SYM>_DIVS`; `value` = split ratio (new per old) or cash amount."""
		17	+ if not path.exists():
		18	+ return pl.DataFrame(schema={"symbol": pl.Utf8, "ex_date": pl.Date, "split_ratio": pl.Float64, "dividend": pl.Float64})
		19	+ a = pl.read_csv(path, try_parse_dates=True, infer_schema_length=10000)
		20	+ a = a.with_columns([
		21	+ pl.col("instrument_id").str.replace("^firstrate:[a-z]+:", "").str.replace("_DIVS$", "").alias("symbol"),
		22	+ pl.col("ex_date").cast(pl.Date), pl.col("value").cast(pl.Float64),
		23	+ ])
		24	+ splits = a.filter(pl.col("action_type") == "split").group_by(["symbol", "ex_date"]).agg(pl.col("value").product().alias("split_ratio"))
		25	+ divs = a.filter(pl.col("action_type") == "dividend").group_by(["symbol", "ex_date"]).agg(pl.col("value").sum().alias("dividend"))
		26	+ out = splits.join(divs, on=["symbol", "ex_date"], how="full", coalesce=True).with_columns([
		27	+ pl.col("split_ratio").fill_null(1.0), pl.col("dividend").fill_null(0.0)])
		28	+ out = out.filter((pl.col("split_ratio") > 0) & (pl.col("dividend") >= 0))
		29	+ return out.sort(["symbol", "ex_date"])
		30	+
		31	+
		32	+def split_events_in_window(actions: pl.DataFrame, symbol: str, start, end) -> int:
		33	+ if actions.height == 0:
		34	+ return 0
		35	+ return actions.filter((pl.col("symbol") == symbol) & (pl.col("ex_date") >= start) & (pl.col("ex_date") <= end)
		36	+ & (pl.col("split_ratio") != 1.0)).height

added src/wp39/features.py +181 −0

				@@ -0,0 +1,181 @@
		1	+"""Frequency-controlled OHLCV features computed per symbol-session from the raw 1-minute export.
		2	+
		3	+Every intraday family is computed identically on bars of length 1m, 5m and 30m, obtained by aggregating the *same* raw
		4	+1-minute bars within the regular session (09:30 -> closing print). Daily features come from the same aggregation (D-003).
		5	+Zero-volume minutes are absent at the source and are never invented; the share of expected buckets present is a feature.
		6	+Bar statistics are labelled as such; nothing here is order flow, spread or informed trading.
		7	+"""
		8	+from __future__ import annotations
		9	+
		10	+import math
		11	+from pathlib import Path
		12	+
		13	+import numpy as np
		14	+import polars as pl
		15	+
		16	+from .calendar import _xnys
		17	+
		18	+FREQS = {"1m": 1, "5m": 5, "30m": 30}
		19	+INTRADAY_FAMILIES = ["rv", "bpv", "rj", "rsneg_share", "sjv", "rskew", "rkurt", "max_share", "ac1", "drift",
		20	+ "vol_first30", "vol_last30", "vol_gini", "down_vol_share", "present_share", "rv_mktdown_share",
		21	+ "sxy", "syy", "n_bars"]
		22	+
		23	+
		24	+def session_bounds_table(start, end) -> pl.DataFrame:
		25	+ """session_date, open_utc (09:30 ET bar open), close_utc (closing-print bar open), close_minute (minutes after 09:30)."""
		26	+ cal = _xnys()
		27	+ s = cal.sessions
		28	+ s = s[(s >= str(start)) & (s <= str(end))]
		29	+ opens = cal.opens.loc[s] if hasattr(cal, "opens") else None
		30	+ rows = []
		31	+ for d in s:
		32	+ o, c = cal.session_open(d), cal.session_close(d)
		33	+ rows.append({"session_date": d.date(), "open_utc": o.tz_convert("UTC").to_pydatetime(),
		34	+ "close_utc": c.tz_convert("UTC").to_pydatetime(),
		35	+ "close_minute": int((c - o).total_seconds() // 60)})
		36	+ df = pl.DataFrame(rows)
		37	+ return df.with_columns([pl.col("open_utc").dt.replace_time_zone("UTC"), pl.col("close_utc").dt.replace_time_zone("UTC")])
		38	+
		39	+
		40	+def load_regular_session(paths: list[Path], bounds: pl.DataFrame) -> pl.DataFrame:
		41	+ """Raw 1m bars restricted to the regular session, with minute-of-session index m (0 = 09:30, close_minute = closing print)."""
		42	+ lf = pl.concat([pl.scan_parquet(str(p)) for p in paths]).select(
		43	+ ["symbol_as_reported", "timestamp_utc", "session_date", "open", "high", "low", "close", "volume", "quality_flags"])
		44	+ df = lf.collect()
		45	+ if df.height == 0:
		46	+ return df
		47	+ # the lake's base+delta layouts (and chunk boundaries) can publish the same bar twice: keep the latest ingestion
		48	+ df = pl.concat([pl.scan_parquet(str(p)) for p in paths]).select(
		49	+ ["symbol_as_reported", "timestamp_utc", "session_date", "open", "high", "low", "close", "volume", "quality_flags",
		50	+ "ingested_at"]).collect().sort("ingested_at").unique(subset=["timestamp_utc"], keep="last").drop("ingested_at")
		51	+ df = df.with_columns(pl.col("timestamp_utc").dt.convert_time_zone("UTC"))
		52	+ df = df.join(bounds, on="session_date", how="inner")
		53	+ df = df.filter((pl.col("timestamp_utc") >= pl.col("open_utc")) & (pl.col("timestamp_utc") <= pl.col("close_utc")))
		54	+ df = df.with_columns(((pl.col("timestamp_utc") - pl.col("open_utc")).dt.total_minutes()).cast(pl.Int32).alias("m"))
		55	+ df = df.filter(pl.col("quality_flags").list.len() == 0) # drop rows flagged by FinData (ohlc_inconsistent, price_jump, ...)
		56	+ return df.sort(["session_date", "m"])
		57	+
		58	+
		59	+def aggregate_buckets(reg: pl.DataFrame, freq_min: int) -> pl.DataFrame:
		60	+ """Aggregate regular-session 1m bars into buckets of `freq_min` minutes; the closing print joins the last bucket."""
		61	+ if freq_min == 1:
		62	+ b = reg.with_columns(pl.col("m").alias("b"))
		63	+ return b.select(["session_date", "b", "open", "high", "low", "close", "volume", "m", "close_minute"])
		64	+ nb = (pl.col("close_minute") / freq_min).ceil().cast(pl.Int32) # buckets per session
		65	+ b = reg.with_columns(pl.min_horizontal((pl.col("m") / freq_min).floor().cast(pl.Int32), nb - 1).alias("b"))
		66	+ agg = b.group_by(["session_date", "b"], maintain_order=True).agg([
		67	+ pl.col("open").first(), pl.col("high").max(), pl.col("low").min(), pl.col("close").last(), pl.col("volume").sum(),
		68	+ pl.col("m").first(), pl.col("close_minute").first(),
		69	+ ])
		70	+ return agg.sort(["session_date", "b"])
		71	+
		72	+
		73	+def bucket_returns(bk: pl.DataFrame) -> pl.DataFrame:
		74	+ """Within-session log returns: first bucket open->close, then close->close."""
		75	+ prev_close = pl.col("close").shift(1).over("session_date")
		76	+ r = pl.when(prev_close.is_null()).then((pl.col("close") / pl.col("open")).log()).otherwise((pl.col("close") / prev_close).log())
		77	+ return bk.with_columns(r.alias("r"))
		78	+
		79	+
		80	+def _gini_expr() -> pl.Expr:
		81	+ v = pl.col("volume").sort()
		82	+ n = pl.len()
		83	+ i = pl.int_range(1, n + 1).cast(pl.Float64)
		84	+ return ((2.0 * (i * v).sum()) / (n.cast(pl.Float64) * v.sum() + 1e-12) - (n.cast(pl.Float64) + 1.0) / n.cast(pl.Float64))
		85	+
		86	+
		87	+def session_features(bk: pl.DataFrame, freq_min: int, spy: pl.DataFrame | None) -> pl.DataFrame:
		88	+ """One row per session with the intraday families for this frequency (prefix f{freq}_)."""
		89	+ df = bucket_returns(bk)
		90	+ if spy is not None:
		91	+ df = df.join(spy.select(["session_date", "b", pl.col("r").alias("r_spy")]), on=["session_date", "b"], how="left")
		92	+ else:
		93	+ df = df.with_columns(pl.lit(None, dtype=pl.Float64).alias("r_spy"))
		94	+ r = pl.col("r")
		95	+ r2 = r.pow(2)
		96	+ rv = r2.sum()
		97	+ n = pl.len().cast(pl.Float64)
		98	+ prev = r.shift(1).over("session_date")
		99	+ expected = pl.when(pl.lit(freq_min) == 1).then(pl.col("close_minute").first() + 1).otherwise(
		100	+ (pl.col("close_minute").first() / freq_min).ceil()).cast(pl.Float64)
		101	+ first30 = pl.col("m") < 30
		102	+ last30 = pl.col("m") > (pl.col("close_minute") - 30)
		103	+ out = df.group_by("session_date", maintain_order=True).agg([
		104	+ n.alias("n_bars"),
		105	+ rv.alias("rv"),
		106	+ ((math.pi / 2.0) * (r.abs() * prev.abs()).sum()).alias("bpv"),
		107	+ (r2.filter(r < 0).sum()).alias("rs_neg"),
		108	+ (r2.filter(r > 0).sum()).alias("rs_pos"),
		109	+ (n.sqrt() * r.pow(3).sum() / (rv.pow(1.5) + 1e-18)).alias("rskew"),
		110	+ (n * r.pow(4).sum() / (rv.pow(2) + 1e-18)).alias("rkurt"),
		111	+ (r2.max() / (rv + 1e-18)).alias("max_share"),
		112	+ pl.corr(r, prev).alias("ac1"),
		113	+ (pl.col("close").last() / pl.col("open").first()).log().alias("drift"),
		114	+ (pl.col("volume").filter(first30).sum() / (pl.col("volume").sum() + 1e-9)).alias("vol_first30"),
		115	+ (pl.col("volume").filter(last30).sum() / (pl.col("volume").sum() + 1e-9)).alias("vol_last30"),
		116	+ _gini_expr().alias("vol_gini"),
		117	+ (pl.col("volume").filter(r < 0).sum() / (pl.col("volume").sum() + 1e-9)).alias("down_vol_share"),
		118	+ (n / expected).alias("present_share"),
		119	+ (r2.filter(pl.col("r_spy") < 0).sum() / (rv + 1e-18)).alias("rv_mktdown_share"),
		120	+ (r * pl.col("r_spy")).sum().alias("sxy"),
		121	+ pl.col("r_spy").pow(2).sum().alias("syy"),
		122	+ pl.col("volume").sum().alias("volume"),
		123	+ pl.col("open").first().alias("open"), pl.col("close").last().alias("close"),
		124	+ pl.col("high").max().alias("high"), pl.col("low").min().alias("low"),
		125	+ ])
		126	+ out = out.with_columns([
		127	+ (pl.max_horizontal(pl.col("rv") - pl.col("bpv"), 0.0) / (pl.col("rv") + 1e-18)).alias("rj"),
		128	+ (pl.col("rs_neg") / (pl.col("rv") + 1e-18)).alias("rsneg_share"),
		129	+ ((pl.col("rs_pos") - pl.col("rs_neg")) / (pl.col("rv") + 1e-18)).alias("sjv"),
		130	+ ])
		131	+ pre = f"f{freq_min}m_"
		132	+ keep = {c: pre + c for c in INTRADAY_FAMILIES}
		133	+ return out.select(["session_date"] + [pl.col(c).alias(keep[c]) for c in INTRADAY_FAMILIES] +
		134	+ ([pl.col("volume").alias("d_volume"), pl.col("open").alias("d_open"), pl.col("close").alias("d_close"),
		135	+ pl.col("high").alias("d_high"), pl.col("low").alias("d_low")] if freq_min == 1 else []))
		136	+
		137	+
		138	+def daily_block(day: pl.DataFrame, actions: pl.DataFrame | None, symbol: str) -> pl.DataFrame:
		139	+ """Daily OHLCV features from the 1m aggregation + split/dividend-adjusted close-to-close returns."""
		140	+ d = day.sort("session_date")
		141	+ if actions is not None and actions.height:
		142	+ a = actions.filter(pl.col("symbol") == symbol).select(["ex_date", "split_ratio", "dividend"])
		143	+ d = d.join(a, left_on="session_date", right_on="ex_date", how="left").with_columns([
		144	+ pl.col("split_ratio").fill_null(1.0), pl.col("dividend").fill_null(0.0)])
		145	+ else:
		146	+ d = d.with_columns([pl.lit(1.0).alias("split_ratio"), pl.lit(0.0).alias("dividend")])
		147	+ prev_close_adj = (pl.col("d_close").shift(1) / pl.col("split_ratio"))
		148	+ d = d.with_columns([
		149	+ ((pl.col("d_close") + pl.col("dividend")) / prev_close_adj - 1.0).alias("d_ret_cc"),
		150	+ (pl.col("d_open") / prev_close_adj - 1.0).alias("d_ret_co"),
		151	+ (pl.col("d_close") / pl.col("d_open") - 1.0).alias("d_ret_oc"),
		152	+ ((pl.col("d_high") / pl.col("d_low")).log().pow(2) / (4.0 * math.log(2.0))).alias("d_parkinson"),
		153	+ (pl.col("d_volume") * pl.col("d_close")).alias("d_dollar_vol"),
		154	+ ])
		155	+ return d
		156	+
		157	+
		158	+def compute_symbol_features(paths: list[Path], symbol: str, bounds: pl.DataFrame, spy_buckets: dict[str, pl.DataFrame] | None,
		159	+ actions: pl.DataFrame | None) -> pl.DataFrame:
		160	+ reg = load_regular_session(paths, bounds)
		161	+ if reg.height == 0:
		162	+ return pl.DataFrame()
		163	+ blocks = []
		164	+ for name, fmin in FREQS.items():
		165	+ bk = aggregate_buckets(reg, fmin)
		166	+ spy = spy_buckets.get(name) if spy_buckets else None
		167	+ blocks.append(session_features(bk, fmin, spy))
		168	+ out = blocks[0]
		169	+ for b in blocks[1:]:
		170	+ out = out.join(b, on="session_date", how="left")
		171	+ out = daily_block(out, actions, symbol)
		172	+ return out.with_columns(pl.lit(symbol).alias("symbol")).sort("session_date")
		173	+
		174	+
		175	+def compute_spy_buckets(paths: list[Path], bounds: pl.DataFrame) -> dict[str, pl.DataFrame]:
		176	+ reg = load_regular_session(paths, bounds)
		177	+ out = {}
		178	+ for name, fmin in FREQS.items():
		179	+ out[name] = bucket_returns(aggregate_buckets(reg, fmin)).select(["session_date", "b", "r"]).unique(
		180	+ subset=["session_date", "b"], keep="last").sort(["session_date", "b"])
		181	+ return out

added src/wp39/figures.py +156 −0

				@@ -0,0 +1,156 @@
		1	+"""Journal figures (vector PDF, matplotlib). One axis per panel, thin marks, fixed categorical order, direct labels.
		2	+Palette validated with the dataviz validator (light surface): blue, orange, aqua, yellow (+ gray for references)."""
		3	+from __future__ import annotations
		4	+
		5	+from pathlib import Path
		6	+
		7	+import matplotlib
		8	+matplotlib.use("Agg")
		9	+import matplotlib.pyplot as plt # noqa: E402
		10	+import numpy as np # noqa: E402
		11	+import pandas as pd # noqa: E402
		12	+
		13	+SERIES = {"daily": "#2a78d6", "30m": "#eb6834", "5m": "#1baf7a", "1m": "#eda100", "fund": "#6250d6", "ref": "#8a8a86", "neg": "#e34948"}
		14	+MODEL_COLOR = {"B1": SERIES["fund"], "B2": SERIES["daily"], "B3": SERIES["30m"], "B4": SERIES["5m"], "B5": SERIES["1m"], "B5all": "#52514e",
		15	+ "LB1": SERIES["fund"], "LB2": SERIES["daily"], "LB3": SERIES["30m"], "LB4": SERIES["5m"], "LB5": SERIES["1m"], "LB5all": "#52514e"}
		16	+MODEL_LABEL = {"B1": "B1 fundamentals", "B2": "B2 + daily", "B3": "B3 + 30-min", "B4": "B4 + 5-min", "B5": "B5 + 1-min", "B5all": "all frequencies"}
		17	+
		18	+plt.rcParams.update({"font.family": "serif", "font.size": 9, "axes.spines.top": False, "axes.spines.right": False,
		19	+ "axes.edgecolor": "#8a8a86", "axes.linewidth": 0.6, "xtick.color": "#52514e", "ytick.color": "#52514e",
		20	+ "axes.labelcolor": "#0b0b0b", "grid.color": "#e6e5e1", "grid.linewidth": 0.5, "legend.frameon": False,
		21	+ "figure.dpi": 150, "savefig.bbox": "tight"})
		22	+
		23	+
		24	+def _save(fig, out: Path, name: str):
		25	+ out.mkdir(parents=True, exist_ok=True)
		26	+ fig.savefig(out / f"{name}.pdf"); fig.savefig(out / f"{name}.png", dpi=200); plt.close(fig)
		27	+
		28	+
		29	+def fig_timeline(out: Path, h_list=(40, 20, 10, 5, 1)):
		30	+ """Event-time timeline and the no-leakage rule (schematic drawn from the protocol, labelled as such)."""
		31	+ fig, ax = plt.subplots(figsize=(6.4, 2.2))
		32	+ ax.set_xlim(-75, 15); ax.set_ylim(0, 3); ax.axis("off")
		33	+ ax.plot([-75, 12], [1, 1], color="#52514e", lw=1)
		34	+ for x, lab, c in ((-63, "$A_{q-1}$\nprevious release", SERIES["ref"]), (0, "$A_q$\nrelease of quarter $q$", SERIES["neg"]),
		35	+ (1, "$F_q$ filing", SERIES["ref"]), (2, "$V_q=F_q+1$\nfacts available", SERIES["ref"])):
		36	+ ax.plot([x], [1], marker="|", ms=14, color=c, mew=1.5)
		37	+ ax.text(x, 0.15 if x not in (1,) else 1.55, lab, ha="center", va="center", fontsize=7, color="#0b0b0b")
		38	+ for i, h in enumerate(h_list):
		39	+ ax.plot([-h], [1], marker="o", ms=4, color=SERIES["1m"], mec="white", mew=0.8)
		40	+ ax.text(-h, 1.35 + 0.28 * (i % 2), f"$c=A_q-{h}$", ha="center", fontsize=6.5, color="#52514e")
		41	+ ax.add_patch(plt.Rectangle((-40, 0.85), 20, 0.3, color=SERIES["1m"], alpha=0.25, lw=0))
		42	+ ax.text(-30, 2.6, "20-session feature window ends at $c$; bars after $c$ and facts with $V_j>c$ never enter the information set",
		43	+ ha="center", fontsize=7, color="#52514e")
		44	+ ax.set_title("Event-time design (schematic of the protocol, not data)", fontsize=8, loc="left")
		45	+ _save(fig, out, "fig01_timeline")
		46	+
		47	+
		48	+def fig_coverage(out: Path, years: pd.DataFrame, sizes: pd.DataFrame | None = None):
		49	+ """years: columns year, firms, firm_quarters, adverse_rate."""
		50	+ fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.4))
		51	+ ax = axes[0]
		52	+ ax.bar(years["year"], years["firms"], color=SERIES["daily"], width=0.8)
		53	+ ax.set_ylabel("Firms in sample"); ax.set_xlabel("Year of cutoff"); ax.grid(axis="y")
		54	+ for x, v in zip(years["year"].iloc[::3], years["firms"].iloc[::3]):
		55	+ ax.text(x, v, f"{int(v):,}", ha="center", va="bottom", fontsize=6, color="#52514e")
		56	+ ax = axes[1]
		57	+ ax.plot(years["year"], years["adverse_rate"], color=SERIES["neg"], lw=2, marker="o", ms=4, mec="white")
		58	+ ax.set_ylabel("Adverse-surprise rate"); ax.set_xlabel("Year of cutoff"); ax.grid(axis="y"); ax.set_ylim(0, None)
		59	+ _save(fig, out, "fig02_coverage")
		60	+
		61	+
		62	+def fig_prevalence(out: Path, prev: pd.DataFrame):
		63	+ """prev: columns group (str), label, rate, lo, hi — conditional adverse prevalence by ex-ante characteristics."""
		64	+ fig, ax = plt.subplots(figsize=(6.0, 2.6))
		65	+ y = np.arange(len(prev))
		66	+ ax.hlines(y, prev["lo"], prev["hi"], color="#c3c2b7", lw=2)
		67	+ ax.plot(prev["rate"], y, "o", color=SERIES["neg"], ms=5, mec="white")
		68	+ ax.set_yticks(y); ax.set_yticklabels(prev["label"]); ax.invert_yaxis()
		69	+ ax.set_xlabel("Share of firm-quarters with an adverse operating-income surprise (test period)"); ax.grid(axis="x")
		70	+ _save(fig, out, "fig03_prevalence")
		71	+
		72	+
		73	+def fig_frequency_gains(out: Path, contrasts: pd.DataFrame):
		74	+ """Paired Δ log loss vs B2 for B3/B4/B5 (logistic and LightGBM), with 95 % block-bootstrap intervals."""
		75	+ fam = [("logistic", ""), ("lightgbm", "L")]
		76	+ fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.6), sharey=True)
		77	+ for ax, (name, pre) in zip(axes, fam):
		78	+ sub = contrasts[(contrasts.family == name) & (contrasts.b == pre + "B2") & (contrasts.a.isin([pre + "B3", pre + "B4", pre + "B5", pre + "B5all"]))]
		79	+ order = [pre + "B3", pre + "B4", pre + "B5", pre + "B5all"]
		80	+ sub = sub.set_index("a").reindex([o for o in order if o in sub.a.values]).reset_index()
		81	+ x = np.arange(len(sub))
		82	+ for i, r in sub.iterrows():
		83	+ c = MODEL_COLOR.get(r["a"].replace("L", "", 1) if r["a"].startswith("L") else r["a"], "#52514e")
		84	+ ax.vlines(i, r["ci_lo"], r["ci_hi"], color=c, lw=2)
		85	+ ax.plot(i, r["d_logloss"], "o", color=c, ms=6, mec="white")
		86	+ ax.text(i, r["ci_hi"], f"{r['d_logloss']:+.4f}", ha="center", va="bottom", fontsize=6.5, color="#52514e")
		87	+ ax.axhline(0, color="#8a8a86", lw=0.8)
		88	+ ax.set_xticks(x); ax.set_xticklabels([MODEL_LABEL[a.replace("L", "", 1) if a.startswith("L") else a].replace("B5all", "all") for a in sub.a], fontsize=7)
		89	+ ax.set_title(f"{name}", fontsize=8, loc="left"); ax.grid(axis="y")
		90	+ axes[0].set_ylabel("Δ log loss vs. B2 (daily + accounting)\nnegative = better")
		91	+ _save(fig, out, "fig04_frequency_gains")
		92	+
		93	+
		94	+def fig_reliability(out: Path, rel: pd.DataFrame):
		95	+ fig, ax = plt.subplots(figsize=(3.6, 3.2))
		96	+ ax.plot([0, rel[["mean_pred", "obs_rate"]].max().max()], [0, rel[["mean_pred", "obs_rate"]].max().max()], color="#8a8a86", lw=0.8, ls="--")
		97	+ for m in ("LB2", "LB5", "B2", "B5"):
		98	+ s = rel[rel.model == m]
		99	+ if len(s):
		100	+ ax.plot(s["mean_pred"], s["obs_rate"], marker="o", ms=4, mec="white", lw=1.6, color=MODEL_COLOR[m], label=m)
		101	+ ax.set_xlabel("Mean predicted probability (decile)"); ax.set_ylabel("Observed adverse rate"); ax.legend(fontsize=7); ax.grid()
		102	+ _save(fig, out, "fig05_reliability")
		103	+
		104	+
		105	+def fig_signal_evolution(out: Path, ladder: pd.DataFrame):
		106	+ """Mean AUC and Δ log loss by horizon (all firm-quarters, no selection)."""
		107	+ fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.5))
		108	+ for fam, c in (("logistic", SERIES["daily"]), ("lightgbm", SERIES["1m"])):
		109	+ s = ladder[ladder.family == fam].sort_values("h", ascending=False)
		110	+ axes[0].plot(-s["h"], s["auc_a"], marker="o", ms=4, mec="white", lw=1.6, color=c, label=f"{fam}: B5")
		111	+ axes[0].plot(-s["h"], s["auc_b"], marker="s", ms=3.5, mec="white", lw=1.2, ls="--", color=c, label=f"{fam}: B2")
		112	+ axes[1].errorbar(-s["h"], s["d_logloss"], yerr=[s["d_logloss"] - s["ci_lo"], s["ci_hi"] - s["d_logloss"]], fmt="o-", ms=4, lw=1.4,
		113	+ color=c, capsize=2, label=fam)
		114	+ axes[0].set_xlabel("Trading days before the release (−h)"); axes[0].set_ylabel("Test AUC"); axes[0].legend(fontsize=6); axes[0].grid()
		115	+ axes[1].axhline(0, color="#8a8a86", lw=0.8); axes[1].set_xlabel("Trading days before the release (−h)")
		116	+ axes[1].set_ylabel("Δ log loss B5 − B2"); axes[1].legend(fontsize=7); axes[1].grid()
		117	+ _save(fig, out, "fig06_signal_evolution")
		118	+
		119	+
		120	+def fig_heterogeneity(out: Path, sub: pd.DataFrame, groups: list[str]):
		121	+ s = sub[(sub.family == "lightgbm") & sub.subgroup.isin(groups)].set_index("subgroup").reindex(groups).reset_index()
		122	+ fig, ax = plt.subplots(figsize=(6.0, 2.8))
		123	+ y = np.arange(len(s))
		124	+ ax.hlines(y, s["ci_lo"], s["ci_hi"], color="#c3c2b7", lw=2)
		125	+ ax.plot(s["mean"], y, "o", color=SERIES["1m"], ms=5, mec="white")
		126	+ for yi, r in zip(y, s.itertuples()):
		127	+ ax.text(r.ci_hi, yi, f" n={int(r.n):,}", va="center", fontsize=6, color="#52514e")
		128	+ ax.axvline(0, color="#8a8a86", lw=0.8)
		129	+ ax.set_yticks(y); ax.set_yticklabels([g.replace("_", " ") for g in s["subgroup"]]); ax.invert_yaxis()
		130	+ ax.set_xlabel("Δ log loss LB5 − LB2 (test period), 95 % block-bootstrap CI"); ax.grid(axis="x")
		131	+ _save(fig, out, "fig07_heterogeneity")
		132	+
		133	+
		134	+def fig_rolling(out: Path, wf: pd.DataFrame, wfc: pd.DataFrame):
		135	+ fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.5))
		136	+ for m in ("LB2", "LB5", "B2", "B5"):
		137	+ s = wf[wf.model == m].sort_values("year")
		138	+ if len(s):
		139	+ axes[0].plot(s["year"], s["auc"], marker="o", ms=3.5, mec="white", lw=1.4, color=MODEL_COLOR[m], ls="-" if m.startswith("L") else "--", label=m)
		140	+ axes[0].set_ylabel("Out-of-fold AUC (annual refits)"); axes[0].legend(fontsize=6, ncol=2); axes[0].grid()
		141	+ for fam, c in (("lightgbm", SERIES["1m"]), ("logistic", SERIES["daily"])):
		142	+ s = wfc[(wfc.family == fam) & (wfc.year != "all")].copy(); s["year"] = s["year"].astype(int); s = s.sort_values("year")
		143	+ axes[1].errorbar(s["year"], s["d_logloss"], yerr=[s["d_logloss"] - s["ci_lo"], s["ci_hi"] - s["d_logloss"]], fmt="o-", ms=3.5, lw=1.2, capsize=2, color=c, label=fam)
		144	+ axes[1].axhline(0, color="#8a8a86", lw=0.8); axes[1].set_ylabel("Δ log loss B5 − B2 by year"); axes[1].legend(fontsize=7); axes[1].grid()
		145	+ _save(fig, out, "fig08_rolling")
		146	+
		147	+
		148	+def fig_economic(out: Path, port: pd.DataFrame, alphas: dict):
		149	+ fig, ax = plt.subplots(figsize=(4.2, 2.6))
		150	+ dm = port.groupby("decile")["ret"].mean() * 100
		151	+ ax.bar(dm.index, dm.values, color=SERIES["daily"], width=0.8)
		152	+ ax.axhline(0, color="#8a8a86", lw=0.8); ax.set_xlabel("Decile of ex-ante deterioration score (10 = most likely)")
		153	+ ax.set_ylabel("Mean monthly return (%)"); ax.grid(axis="y")
		154	+ ax.set_title(f"L/S 1−10 alpha: {alphas['gross']['alpha_monthly']*100:+.2f}% (t={alphas['gross']['alpha_t']:.1f}) gross, "
		155	+ f"{alphas['net20bps']['alpha_monthly']*100:+.2f}% net of 20 bps", fontsize=7, loc="left")
		156	+ _save(fig, out, "fig09_economic")

added src/wp39/inference.py +134 −0

				@@ -0,0 +1,134 @@
		1	+"""Evaluation metrics, paired block-bootstrap intervals, Diebold-Mariano / Giacomini-White tests, Holm correction,
		2	+calibration, and two-way clustered panel regressions."""
		3	+from __future__ import annotations
		4	+
		5	+import numpy as np
		6	+import pandas as pd
		7	+import polars as pl
		8	+from scipy import stats
		9	+from sklearn.metrics import average_precision_score, brier_score_loss, log_loss, roc_auc_score
		10	+
		11	+
		12	+# --------------------------------------------------------------------- metrics
		13	+def binary_metrics(y: np.ndarray, p: np.ndarray) -> dict:
		14	+ p = np.clip(p, 1e-6, 1 - 1e-6)
		15	+ out = {"n": int(len(y)), "base_rate": float(y.mean()), "logloss": float(log_loss(y, p)), "brier": float(brier_score_loss(y, p))}
		16	+ if 0 < y.mean() < 1:
		17	+ out["auc"] = float(roc_auc_score(y, p))
		18	+ out["pr_auc"] = float(average_precision_score(y, p))
		19	+ # calibration intercept / slope (logistic recalibration of the logit)
		20	+ z = np.log(p / (1 - p))
		21	+ try:
		22	+ from sklearn.linear_model import LogisticRegression
		23	+ m = LogisticRegression(C=1e6, solver="lbfgs", max_iter=1000).fit(z.reshape(-1, 1), y)
		24	+ out["cal_slope"], out["cal_intercept"] = float(m.coef_[0][0]), float(m.intercept_[0])
		25	+ except Exception:
		26	+ pass
		27	+ return out
		28	+
		29	+
		30	+def continuous_metrics(y: np.ndarray, yhat: np.ndarray, y_train_mean: float) -> dict:
		31	+ e = y - yhat
		32	+ ss = np.sum((y - y_train_mean) ** 2)
		33	+ return {"n": int(len(y)), "mae": float(np.mean(np.abs(e))), "rmse": float(np.sqrt(np.mean(e ** 2))),
		34	+ "oos_r2": float(1 - np.sum(e ** 2) / ss) if ss > 0 else float("nan")}
		35	+
		36	+
		37	+def pointwise_logloss(y: np.ndarray, p: np.ndarray) -> np.ndarray:
		38	+ p = np.clip(p, 1e-6, 1 - 1e-6)
		39	+ return -(y * np.log(p) + (1 - y) * np.log(1 - p))
		40	+
		41	+
		42	+def reliability_table(y: np.ndarray, p: np.ndarray, bins: int = 10) -> pd.DataFrame:
		43	+ q = np.quantile(p, np.linspace(0, 1, bins + 1))
		44	+ q[0], q[-1] = -np.inf, np.inf
		45	+ b = np.digitize(p, q[1:-1])
		46	+ rows = []
		47	+ for k in range(bins):
		48	+ m = b == k
		49	+ if m.sum():
		50	+ rows.append({"decile": k + 1, "n": int(m.sum()), "mean_pred": float(p[m].mean()), "obs_rate": float(y[m].mean())})
		51	+ return pd.DataFrame(rows)
		52	+
		53	+
		54	+# ------------------------------------------------------------ paired inference
		55	+def quarter_blocks(dates: np.ndarray) -> np.ndarray:
		56	+ d = pd.to_datetime(dates)
		57	+ return (d.year * 4 + (d.month - 1) // 3).to_numpy()
		58	+
		59	+
		60	+def paired_block_bootstrap(d: np.ndarray, blocks: np.ndarray, n_boot: int = 1000, seed: int = 20261102) -> dict:
		61	+ """Mean of a per-observation loss differential with a cluster (block) bootstrap over calendar quarters.
		62	+ Returns the mean, a percentile 95 % CI and a two-sided bootstrap p-value for H0: mean = 0."""
		63	+ rng = np.random.default_rng(seed)
		64	+ ub = np.unique(blocks)
		65	+ # per-block sums and counts -> resample blocks with replacement (equal-weight by observation)
		66	+ sums = np.array([d[blocks == b].sum() for b in ub]); cnts = np.array([(blocks == b).sum() for b in ub])
		67	+ mean = float(d.mean())
		68	+ draws = np.empty(n_boot)
		69	+ for i in range(n_boot):
		70	+ idx = rng.integers(0, len(ub), len(ub))
		71	+ draws[i] = sums[idx].sum() / cnts[idx].sum()
		72	+ lo, hi = np.percentile(draws, [2.5, 97.5])
		73	+ p = float(2 * min((draws - mean <= -abs(mean)).mean(), (draws - mean >= abs(mean)).mean()))
		74	+ return {"mean": mean, "ci_lo": float(lo), "ci_hi": float(hi), "p_boot": p, "n_blocks": int(len(ub)), "n": int(len(d))}
		75	+
		76	+
		77	+def dm_gw_test(d: np.ndarray, blocks: np.ndarray) -> dict:
		78	+ """Unconditional Giacomini-White (= Diebold-Mariano with HAC) on the per-quarter mean loss differential.
		79	+ HAC: Newey-West with lag = floor(4 (T/100)^(2/9))."""
		80	+ ub = np.unique(blocks)
		81	+ dq = np.array([d[blocks == b].mean() for b in ub])
		82	+ T = len(dq)
		83	+ if T < 4:
		84	+ return {"t": float("nan"), "p": float("nan"), "T": T}
		85	+ m = dq.mean(); u = dq - m
		86	+ L = int(np.floor(4 * (T / 100) ** (2 / 9)))
		87	+ s2 = u @ u / T
		88	+ for l in range(1, L + 1):
		89	+ w = 1 - l / (L + 1)
		90	+ s2 += 2 * w * (u[l:] @ u[:-l]) / T
		91	+ se = np.sqrt(max(s2, 1e-18) / T)
		92	+ t = m / se
		93	+ return {"t": float(t), "p": float(2 * (1 - stats.t.cdf(abs(t), df=T - 1))), "T": int(T), "mean_quarterly": float(m)}
		94	+
		95	+
		96	+def holm(pvals: dict[str, float]) -> dict[str, float]:
		97	+ items = sorted(pvals.items(), key=lambda kv: kv[1])
		98	+ m = len(items); out = {}; running = 0.0
		99	+ for i, (k, p) in enumerate(items):
		100	+ adj = min(1.0, (m - i) * p); running = max(running, adj); out[k] = running
		101	+ return out
		102	+
		103	+
		104	+# ------------------------------------------------------------- panel regression
		105	+def demean_two_way(df: pd.DataFrame, cols: list[str], fe1: str, fe2: str, iters: int = 25) -> pd.DataFrame:
		106	+ """Alternating projections for two fixed effects (firm, sector x quarter)."""
		107	+ x = df[cols].astype(float).copy()
		108	+ for _ in range(iters):
		109	+ x = x - x.groupby(df[fe1]).transform("mean")
		110	+ x = x - x.groupby(df[fe2]).transform("mean")
		111	+ return x
		112	+
		113	+
		114	+def two_way_cluster_ols(df: pd.DataFrame, y: str, xs: list[str], fe1: str, fe2: str, cl1: str, cl2: str) -> pd.DataFrame:
		115	+ """OLS with two-way FE (within transformation) and Cameron-Gelbach-Miller two-way clustered SEs."""
		116	+ d = df.dropna(subset=[y] + xs + [fe1, fe2, cl1, cl2]).copy()
		117	+ z = demean_two_way(d, [y] + xs, fe1, fe2)
		118	+ X = z[xs].to_numpy(); Y = z[y].to_numpy()
		119	+ XtX_inv = np.linalg.pinv(X.T @ X)
		120	+ beta = XtX_inv @ X.T @ Y
		121	+ e = Y - X @ beta
		122	+ def meat(groups):
		123	+ S = np.zeros((X.shape[1], X.shape[1]))
		124	+ for _, idx in pd.Series(np.arange(len(groups))).groupby(groups.to_numpy()).groups.items():
		125	+ idx = np.asarray(list(idx)); s = X[idx].T @ e[idx]; S += np.outer(s, s)
		126	+ return S
		127	+ g12 = (d[cl1].astype(str) + "|" + d[cl2].astype(str))
		128	+ V = XtX_inv @ (meat(d[cl1]) + meat(d[cl2]) - meat(g12)) @ XtX_inv
		129	+ se = np.sqrt(np.clip(np.diag(V), 0, None))
		130	+ t = beta / se
		131	+ G = min(d[cl1].nunique(), d[cl2].nunique())
		132	+ p = 2 * (1 - stats.t.cdf(np.abs(t), df=max(G - 1, 1)))
		133	+ return pd.DataFrame({"var": xs, "coef": beta, "se": se, "t": t, "p": p, "n": len(d), "n_firms": d[cl1].nunique(),
		134	+ "n_quarters": d[cl2].nunique()})

added src/wp39/ingest.py +63 −0

				@@ -0,0 +1,63 @@
		1	+"""Bounded, idempotent FinData exports of raw 1-minute bars (<= 12 years per request, one symbol per request)."""
		2	+from __future__ import annotations
		3	+
		4	+import datetime as dt
		5	+from pathlib import Path
		6	+
		7	+import pyarrow.parquet as pq
		8	+
		9	+from .api_client import FinDataClient, export_result_dict
		10	+from .provenance import append_jsonl, manifest_index
		11	+
		12	+BARS_FIELDS = ["instrument_id", "symbol_as_reported", "venue", "asset_class", "interval", "timestamp_utc", "session_date",
		13	+ "open", "high", "low", "close", "volume", "currency", "trade_count", "vwap", "adjustment_type", "source_id",
		14	+ "ingested_at", "quality_flags", "revision_id"]
		15	+
		16	+
		17	+def chunk_ranges(start: dt.date, end: dt.date, max_years: int = 12) -> list[tuple[dt.date, dt.date]]:
		18	+ out, a = [], start
		19	+ while a <= end:
		20	+ b = min(end, dt.date(a.year + max_years - 1, 12, 31))
		21	+ out.append((a, b))
		22	+ a = dt.date(b.year + 1, 1, 1)
		23	+ return out
		24	+
		25	+
		26	+def validate_bars_parquet(path: Path) -> dict:
		27	+ meta = pq.read_metadata(path)
		28	+ schema = pq.read_schema(path)
		29	+ names = schema.names
		30	+ if names != BARS_FIELDS:
		31	+ raise ValueError(f"unexpected bars schema in {path.name}: {names}")
		32	+ md = {k.decode(): v.decode() for k, v in (schema.metadata or {}).items()}
		33	+ if md.get("findata.schema") != "bars" or md.get("findata.timestamp_convention") != "bar_open":
		34	+ raise ValueError(f"missing/unexpected FinData schema metadata in {path.name}: {md}")
		35	+ return {"rows": meta.num_rows, "schema_version": md.get("findata.schema_version")}
		36	+
		37	+
		38	+def export_symbol(client: FinDataClient, symbol: str, start: dt.date, end: dt.date, dataset_id: str, cache_dir: Path,
		39	+ manifest: Path, keep: bool = False) -> list[Path]:
		40	+ """Export all chunks of one symbol; skip chunks already in the manifest when the file is still present (or keep=False
		41	+ and the chunk was already consumed). Returns the list of Parquet paths available on disk."""
		42	+ idx = manifest_index(manifest, ["symbol", "start", "end", "dataset_id"])
		43	+ paths = []
		44	+ for a, b in chunk_ranges(start, end):
		45	+ dest = cache_dir / f"{symbol}__{a}__{b}.parquet"
		46	+ key = (symbol, str(a), str(b), dataset_id)
		47	+ if dest.exists():
		48	+ paths.append(dest)
		49	+ continue
		50	+ if key in idx and idx[key].get("consumed") and not keep:
		51	+ # already processed in a previous run; downstream will find the feature rows
		52	+ continue
		53	+ res = client.export(dataset_id, [symbol], str(a), str(b), dest)
		54	+ info = validate_bars_parquet(dest)
		55	+ rec = export_result_dict(res)
		56	+ rec.update({"symbol": symbol, "rows_in_file": info["rows"], "schema_version": info["schema_version"], "consumed": False})
		57	+ append_jsonl(manifest, rec)
		58	+ paths.append(dest)
		59	+ return paths
		60	+
		61	+
		62	+def mark_consumed(manifest: Path, symbol: str, dataset_id: str) -> None:
		63	+ append_jsonl(manifest, {"symbol": symbol, "dataset_id": dataset_id, "consumed": True, "event": "features_written"})

added src/wp39/instruments.py +119 −0

				@@ -0,0 +1,119 @@
		1	+"""Instrument master and identifier crosswalk: FinData entity/instrument ↔ ticker ↔ SEC CIK/SIC.
		2	+
		3	+SEC data are public; requests carry the fair-access User-Agent (no secret) and are throttled to <= 8 req/s.
		4	+"""
		5	+from __future__ import annotations
		6	+
		7	+import gzip
		8	+import json
		9	+import os
		10	+import time
		11	+from pathlib import Path
		12	+
		13	+import httpx
		14	+import polars as pl
		15	+
		16	+SEC_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json"
		17	+SEC_SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik:010d}.json"
		18	+
		19	+# SIC divisions (US Department of Labor): used as the coarse "sector" for fixed effects and demeaning.
		20	+SIC_DIVISIONS = [
		21	+ (100, 999, "A_agriculture"), (1000, 1499, "B_mining"), (1500, 1799, "C_construction"), (2000, 3999, "D_manufacturing"),
		22	+ (4000, 4999, "E_transport_utilities"), (5000, 5199, "F_wholesale"), (5200, 5999, "G_retail"),
		23	+ (6000, 6799, "H_finance_insurance_realestate"), (7000, 8999, "I_services"), (9100, 9729, "J_public_admin"),
		24	+]
		25	+
		26	+
		27	+def sic_division(sic: int | None) -> str:
		28	+ if sic is None:
		29	+ return "unknown"
		30	+ for lo, hi, name in SIC_DIVISIONS:
		31	+ if lo <= sic <= hi:
		32	+ return name
		33	+ return "unknown"
		34	+
		35	+
		36	+def _ua() -> str:
		37	+ return os.environ.get("SEC_USER_AGENT", "UQO WP39 research (mailto:simon-pierre.boucher@uqo.ca)")
		38	+
		39	+
		40	+def _get_json(client: httpx.Client, url: str) -> dict:
		41	+ r = client.get(url, headers={"User-Agent": _ua(), "Accept-Encoding": "gzip, deflate"})
		42	+ r.raise_for_status()
		43	+ raw = r.content
		44	+ if raw[:2] == b"\x1f\x8b":
		45	+ raw = gzip.decompress(raw)
		46	+ return json.loads(raw)
		47	+
		48	+
		49	+def fetch_sec_tickers(cache_dir: Path) -> pl.DataFrame:
		50	+ """Current SEC ticker→CIK table (snapshot cached with its download date)."""
		51	+ cache_dir.mkdir(parents=True, exist_ok=True)
		52	+ f = cache_dir / "sec_company_tickers.json"
		53	+ if not f.exists():
		54	+ with httpx.Client(timeout=60) as c:
		55	+ d = _get_json(c, SEC_TICKERS_URL)
		56	+ f.write_text(json.dumps({"downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "data": d}))
		57	+ d = json.loads(f.read_text())["data"]
		58	+ rows = [{"ticker": v["ticker"].upper(), "cik": int(v["cik_str"]), "sec_name": v["title"]} for v in d.values()]
		59	+ return pl.DataFrame(rows)
		60	+
		61	+
		62	+def fetch_sec_submissions(ciks: list[int], cache_dir: Path, rate_per_s: float = 8.0, log=None, workers: int = 6) -> pl.DataFrame:
		63	+ """SIC code, exchanges, fiscal year end and former names per CIK (cached per CIK; global token-bucket rate limit)."""
		64	+ import threading
		65	+ from concurrent.futures import ThreadPoolExecutor
		66	+
		67	+ out_dir = cache_dir / "sec_submissions"
		68	+ out_dir.mkdir(parents=True, exist_ok=True)
		69	+ lock, state = threading.Lock(), {"last": 0.0}
		70	+ min_gap = 1.0 / rate_per_s
		71	+
		72	+ def fetch_one(cik: int) -> None:
		73	+ f = out_dir / f"CIK{cik:010d}.json"
		74	+ if f.exists():
		75	+ return
		76	+ with lock:
		77	+ wait = min_gap - (time.time() - state["last"])
		78	+ if wait > 0:
		79	+ time.sleep(wait)
		80	+ state["last"] = time.time()
		81	+ with httpx.Client(timeout=60) as c:
		82	+ try:
		83	+ d = _get_json(c, SEC_SUBMISSIONS_URL.format(cik=cik))
		84	+ except httpx.HTTPStatusError as exc:
		85	+ d = {"error": exc.response.status_code}
		86	+ except Exception as exc: # noqa: BLE001
		87	+ d = {"error": type(exc).__name__}
		88	+ slim = {k: d.get(k) for k in ("cik", "sic", "sicDescription", "tickers", "exchanges", "fiscalYearEnd",
		89	+ "formerNames", "name", "stateOfIncorporation", "error")}
		90	+ f.write_text(json.dumps(slim))
		91	+
		92	+ with ThreadPoolExecutor(max_workers=workers) as ex:
		93	+ for i, _ in enumerate(ex.map(fetch_one, ciks)):
		94	+ if log and i % 500 == 0:
		95	+ log.info("SEC submissions %d/%d", i, len(ciks))
		96	+ rows = []
		97	+ for cik in ciks:
		98	+ d = json.loads((out_dir / f"CIK{cik:010d}.json").read_text())
		99	+ if d.get("error") and not d.get("sic"):
		100	+ (out_dir / f"CIK{cik:010d}.json").unlink(missing_ok=True) if d.get("error") not in (404,) else None
		101	+ sic = int(d["sic"]) if d.get("sic") not in (None, "") else None
		102	+ rows.append({"cik": cik, "sic": sic, "sic_description": d.get("sicDescription"),
		103	+ "sic_division": sic_division(sic), "sec_tickers": d.get("tickers") or [],
		104	+ "exchanges": d.get("exchanges") or [], "fiscal_year_end": d.get("fiscalYearEnd"),
		105	+ "n_former_names": len(d.get("formerNames") or []), "sec_error": d.get("error")})
		106	+ return pl.DataFrame(rows)
		107	+
		108	+
		109	+def build_crosswalk(entities: pl.DataFrame, sec_tickers: pl.DataFrame, submissions: pl.DataFrame | None) -> pl.DataFrame:
		110	+ """entities: entity_id, security_id, ticker (from FinData). Effective-date ranges are the fundamentals span."""
		111	+ xw = entities.join(sec_tickers.unique(subset=["ticker"]), on="ticker", how="left")
		112	+ if submissions is not None:
		113	+ xw = xw.join(submissions, on="cik", how="left")
		114	+ xw = xw.with_columns([
		115	+ pl.col("cik").is_not_null().alias("has_cik"),
		116	+ pl.when(pl.col("cik").is_null()).then(pl.lit("unknown")).otherwise(pl.col("sic_division").fill_null("unknown")).alias("sic_division")
		117	+ if submissions is not None else pl.lit("unknown").alias("sic_division"),
		118	+ ])
		119	+ return xw

added src/wp39/labels.py +104 −0

				@@ -0,0 +1,104 @@
		1	+"""Firm-quarter outcomes: reveal dates, seasonal-random-walk expectations and adverse-outcome labels (PIT_PROTOCOL §4).
		2	+
		3	+Nothing here uses prices. The expectation E_c[X_q] uses quarters q-1..q-8 *restricted to those available at the cutoff*;
		4	+samples.py passes the cutoff-specific availability mask. The quarterly series X_q is the first (only) revision.
		5	+"""
		6	+from __future__ import annotations
		7	+
		8	+import datetime as dt
		9	+
		10	+import polars as pl
		11	+
		12	+TARGET_VARS = ["op_inc", "net_inc", "cash_op", "rev"]
		13	+
		14	+
		15	+def match_release_dates(wide: pl.DataFrame, events: pl.DataFrame, max_lag_days: int = 30) -> pl.DataFrame:
		16	+ """Attach A_q (first release in [E_q, F_q + 10 d]) and its timing to each firm-quarter; flag exclusions."""
		17	+ e = events.select(["entity_id", pl.col("event_date").cast(pl.Date), "release_time", "eps_estimate", "eps_reported"]).sort(
		18	+ ["entity_id", "event_date"])
		19	+ w = wide.select(["entity_id", "fiscal_period_end", "filing_date"]).with_columns(
		20	+ pl.col("fiscal_period_end").alias("_key")).sort(["entity_id", "_key"])
		21	+ # forward as-of: first event on/after the fiscal period end
		22	+ j = w.join_asof(e, left_on="_key", right_on="event_date", by="entity_id", strategy="forward")
		23	+ j = j.with_columns([
		24	+ pl.col("event_date").alias("A_q"),
		25	+ (pl.col("event_date") <= pl.col("filing_date") + pl.duration(days=10)).alias("_within"),
		26	+ ])
		27	+ j = j.with_columns([
		28	+ pl.when(pl.col("_within")).then(pl.col("A_q")).otherwise(None).alias("A_q"),
		29	+ pl.when(pl.col("_within")).then(pl.col("release_time")).otherwise(None).alias("release_time"),
		30	+ pl.when(pl.col("_within")).then(pl.col("eps_estimate")).otherwise(None).alias("eps_estimate"),
		31	+ pl.when(pl.col("_within")).then(pl.col("eps_reported")).otherwise(None).alias("eps_reported"),
		32	+ ])
		33	+ j = j.with_columns([
		34	+ (pl.col("filing_date") - pl.col("A_q")).dt.total_days().alias("filing_minus_release_days"),
		35	+ ])
		36	+ j = j.with_columns([
		37	+ pl.when(pl.col("A_q").is_null()).then(pl.lit("no_release"))
		38	+ .when(pl.col("filing_minus_release_days") < 0).then(pl.lit("filing_before_release"))
		39	+ .when(pl.col("filing_minus_release_days") > max_lag_days).then(pl.lit("release_lag_gt_30d"))
		40	+ .otherwise(pl.lit("ok")).alias("release_match"),
		41	+ ])
		42	+ return wide.join(j.select(["entity_id", "fiscal_period_end", "A_q", "release_time", "eps_estimate", "eps_reported",
		43	+ "filing_minus_release_days", "release_match"]), on=["entity_id", "fiscal_period_end"], how="left")
		44	+
		45	+
		46	+def seasonal_expectation(w: pl.DataFrame, var: str) -> pl.DataFrame:
		47	+ """E[X_q] = X_{q-4} + mean_{k=0..3}(X_{q-1-k} - X_{q-5-k}); requires q-1..q-8 (by quarter index, regular calendar)."""
		48	+ x = pl.col(var)
		49	+ lags = {k: x.shift(k).over("entity_id") for k in range(1, 9)}
		50	+ drift = sum((lags[1 + k] - lags[5 + k]) for k in range(4)) / 4.0
		51	+ return w.with_columns([
		52	+ (lags[4] + drift).alias(f"E_{var}"),
		53	+ lags[4].alias(f"{var}_lag4"),
		54	+ lags[1].alias(f"{var}_lag1"),
		55	+ (x - (lags[4] + drift)).alias(f"surprise_{var}"),
		56	+ ])
		57	+
		58	+
		59	+def build_quarter_table(wide: pl.DataFrame) -> pl.DataFrame:
		60	+ """Quarter-level target ingredients (unscaled). Regularity: consecutive quarters 60-120 days apart."""
		61	+ w = wide.sort(["entity_id", "fiscal_period_end"])
		62	+ w = w.with_columns(pl.col("fiscal_period_end").diff().over("entity_id").dt.total_days().alias("days_since_prev_q"))
		63	+ # a quarter is "regular" if the previous 8 gaps are all within [60, 120]
		64	+ ok = (pl.col("days_since_prev_q") >= 60) & (pl.col("days_since_prev_q") <= 120)
		65	+ w = w.with_columns(ok.cast(pl.Int8).alias("_gap_ok"))
		66	+ w = w.with_columns(pl.col("_gap_ok").rolling_sum(window_size=8, min_samples=8).over("entity_id").alias("_gap_ok8"))
		67	+ for v in TARGET_VARS:
		68	+ w = seasonal_expectation(w, v)
		69	+ # secondary ingredients
		70	+ w = w.with_columns([
		71	+ pl.col("roa").shift(4).over("entity_id").alias("roa_lag4"),
		72	+ (pl.col("assets_c") / (pl.col("liab_c").abs() + 1e-9)).alias("current_ratio"),
		73	+ ])
		74	+ w = w.with_columns([
		75	+ pl.col("current_ratio").shift(4).over("entity_id").alias("current_ratio_lag4"),
		76	+ pl.col("intr_cover").shift(4).over("entity_id").alias("intr_cover_lag4"),
		77	+ pl.col("assets_tot").shift(1).over("entity_id").alias("assets_tot_lag1"),
		78	+ ])
		79	+ return w
		80	+
		81	+
		82	+def finalize_labels(q: pl.DataFrame, threshold: float = -0.005) -> pl.DataFrame:
		83	+ """Scale by the latest *available* total assets (lag-1 quarter) and build binary targets."""
		84	+ ta = pl.col("assets_tot_lag1").abs() + 1e-9
		85	+ q = q.with_columns([
		86	+ (pl.col("surprise_op_inc") / ta).alias("S_opinc"),
		87	+ (pl.col("surprise_net_inc") / ta).alias("S_netinc"),
		88	+ (pl.col("surprise_cash_op") / ta).alias("S_ocf"),
		89	+ (pl.col("surprise_rev") / ta).alias("S_rev"),
		90	+ ((pl.col("cash_op") - pl.col("cash_op_lag4")) / ta).alias("ocf_innov"),
		91	+ (pl.col("roa") - pl.col("roa_lag4")).alias("roa_yoy"),
		92	+ ])
		93	+ q = q.with_columns([
		94	+ (pl.col("S_opinc") <= threshold).cast(pl.Int8).alias("y_adverse_opinc"),
		95	+ (pl.col("S_netinc") <= threshold).cast(pl.Int8).alias("y_adverse_netinc"),
		96	+ (pl.col("roa_yoy") <= -0.01).cast(pl.Int8).alias("y_roa_decline"),
		97	+ (pl.col("ocf_innov") <= -0.01).cast(pl.Int8).alias("y_ocf_negative"),
		98	+ (((pl.col("current_ratio") / (pl.col("current_ratio_lag4") + 1e-9)) <= 0.9) |
		99	+ ((pl.col("intr_cover") < 1.0) & (pl.col("intr_cover_lag4") >= 1.0))).cast(pl.Int8).alias("y_liquidity_deterioration"),
		100	+ pl.when(pl.col("eps_estimate").is_not_null() & pl.col("eps_reported").is_not_null())
		101	+ .then((pl.col("eps_reported") < pl.col("eps_estimate")).cast(pl.Int8)).otherwise(None).alias("y_eps_miss"),
		102	+ (pl.col("S_opinc").abs() >= 0.005).cast(pl.Int8).alias("y_abs_surprise"), # symmetric placebo target (volatility channel)
		103	+ ])
		104	+ return q

added src/wp39/models.py +137 −0

				@@ -0,0 +1,137 @@
		1	+"""Baseline ladder B0-B6 with train-only preprocessing and an equal tuning budget per information set."""
		2	+from __future__ import annotations
		3	+
		4	+import json
		5	+import random
		6	+from dataclasses import dataclass, field
		7	+
		8	+import numpy as np
		9	+import polars as pl
		10	+from sklearn.linear_model import LogisticRegression, Ridge
		11	+from sklearn.metrics import log_loss
		12	+
		13	+INFO_SETS = {
		14	+ "B1": ["fundamentals"],
		15	+ "B2": ["fundamentals", "daily"],
		16	+ "B3": ["fundamentals", "daily", "intraday_30m"],
		17	+ "B4": ["fundamentals", "daily", "intraday_5m"],
		18	+ "B5": ["fundamentals", "daily", "intraday_1m"],
		19	+ "B5all": ["fundamentals", "daily", "intraday_30m", "intraday_5m", "intraday_1m"],
		20	+}
		21	+
		22	+
		23	+@dataclass
		24	+class Preprocessor:
		25	+ """Winsorise (1/99 %), impute (median) and standardise using TRAINING statistics only."""
		26	+ cols: list[str]
		27	+ lo: dict = field(default_factory=dict)
		28	+ hi: dict = field(default_factory=dict)
		29	+ med: dict = field(default_factory=dict)
		30	+ mu: dict = field(default_factory=dict)
		31	+ sd: dict = field(default_factory=dict)
		32	+ fitted_on_rows: int = 0
		33	+
		34	+ def fit(self, X: pl.DataFrame) -> "Preprocessor":
		35	+ for c in self.cols:
		36	+ s = X[c].cast(pl.Float64)
		37	+ s = s.filter(s.is_finite())
		38	+ if s.len() == 0:
		39	+ self.lo[c], self.hi[c], self.med[c], self.mu[c], self.sd[c] = 0.0, 0.0, 0.0, 0.0, 1.0
		40	+ continue
		41	+ lo, hi = float(s.quantile(0.01)), float(s.quantile(0.99))
		42	+ w = s.clip(lo, hi)
		43	+ self.lo[c], self.hi[c], self.med[c] = lo, hi, float(w.median())
		44	+ self.mu[c] = float(w.mean())
		45	+ sd = float(w.std()) if w.len() > 1 else 1.0
		46	+ self.sd[c] = sd if sd and sd > 1e-12 else 1.0
		47	+ self.fitted_on_rows = X.height
		48	+ return self
		49	+
		50	+ def transform(self, X: pl.DataFrame) -> np.ndarray:
		51	+ out = np.empty((X.height, len(self.cols)), dtype=np.float64)
		52	+ for j, c in enumerate(self.cols):
		53	+ v = X[c].cast(pl.Float64).to_numpy().astype(np.float64)
		54	+ v = np.where(np.isfinite(v), v, np.nan)
		55	+ v = np.clip(v, self.lo[c], self.hi[c])
		56	+ v = np.where(np.isnan(v), self.med[c], v)
		57	+ out[:, j] = (v - self.mu[c]) / self.sd[c]
		58	+ return out
		59	+
		60	+ def to_dict(self) -> dict:
		61	+ return {"cols": self.cols, "lo": self.lo, "hi": self.hi, "med": self.med, "mu": self.mu, "sd": self.sd,
		62	+ "fitted_on_rows": self.fitted_on_rows}
		63	+
		64	+
		65	+def columns_for(info_set: str, fc: dict[str, list[str]]) -> list[str]:
		66	+ cols = []
		67	+ for fam in INFO_SETS[info_set]:
		68	+ cols += fc[fam]
		69	+ return cols
		70	+
		71	+
		72	+# ----------------------------------------------------------------------------- B0
		73	+def base_rate_predict(train_y: np.ndarray, n: int) -> np.ndarray:
		74	+ return np.full(n, float(np.clip(train_y.mean(), 1e-4, 1 - 1e-4)))
		75	+
		76	+
		77	+def firm_history_predict(train: pl.DataFrame, test: pl.DataFrame, y: str) -> np.ndarray:
		78	+ """B0b: trailing firm-level adverse rate (from training rows only, shrunk to the pooled rate with prior weight 8)."""
		79	+ pooled = float(train[y].mean())
		80	+ g = train.group_by("entity_id").agg([pl.col(y).sum().alias("k"), pl.len().alias("n")])
		81	+ t = test.select("entity_id").join(g, on="entity_id", how="left").with_columns([pl.col("k").fill_null(0), pl.col("n").fill_null(0)])
		82	+ k, n = t["k"].to_numpy().astype(float), t["n"].to_numpy().astype(float)
		83	+ return np.clip((k + 8 * pooled) / (n + 8), 1e-4, 1 - 1e-4)
		84	+
		85	+
		86	+# ------------------------------------------------------------------------ logistic
		87	+def fit_logistic(Xtr: np.ndarray, ytr: np.ndarray, Xva: np.ndarray, yva: np.ndarray, C_grid: list[float], seed: int) -> tuple:
		88	+ best, best_ll, best_C = None, np.inf, None
		89	+ for C in C_grid:
		90	+ m = LogisticRegression(C=C, penalty="l2", solver="lbfgs", max_iter=2000, random_state=seed)
		91	+ m.fit(Xtr, ytr)
		92	+ ll = log_loss(yva, np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6))
		93	+ if ll < best_ll:
		94	+ best, best_ll, best_C = m, ll, C
		95	+ return best, {"C": best_C, "val_logloss": best_ll}
		96	+
		97	+
		98	+def fit_ridge(Xtr, ytr, Xva, yva, alpha_grid=(0.1, 1, 10, 100, 1000)):
		99	+ best, best_mse, best_a = None, np.inf, None
		100	+ for a in alpha_grid:
		101	+ m = Ridge(alpha=a).fit(Xtr, ytr)
		102	+ mse = float(np.mean((m.predict(Xva) - yva) ** 2))
		103	+ if mse < best_mse:
		104	+ best, best_mse, best_a = m, mse, a
		105	+ return best, {"alpha": best_a, "val_mse": best_mse}
		106	+
		107	+
		108	+# ------------------------------------------------------------------------ LightGBM
		109	+def _sample_params(space: dict, rng: random.Random) -> dict:
		110	+ return {k: rng.choice(v) for k, v in space.items()}
		111	+
		112	+
		113	+def fit_lightgbm(Xtr, ytr, Xva, yva, cfg: dict, seed: int, objective: str = "binary", n_jobs: int = 8) -> tuple:
		114	+ import lightgbm as lgb
		115	+ rng = random.Random(seed)
		116	+ best, best_score, best_params, trials = None, np.inf, None, []
		117	+ for t in range(int(cfg["budget_trials"])):
		118	+ p = _sample_params(cfg["search_space"], rng)
		119	+ params = dict(objective=objective, learning_rate=p["learning_rate"], num_leaves=p["num_leaves"],
		120	+ min_child_samples=p["min_child_samples"], feature_fraction=p["feature_fraction"],
		121	+ bagging_fraction=p["bagging_fraction"], bagging_freq=1 if p["bagging_fraction"] < 1 else 0,
		122	+ lambda_l2=p["lambda_l2"], n_estimators=int(cfg["n_estimators_max"]), random_state=seed + t,
		123	+ n_jobs=n_jobs, verbose=-1)
		124	+ m = lgb.LGBMClassifier(**params) if objective == "binary" else lgb.LGBMRegressor(**params)
		125	+ m.fit(Xtr, ytr, eval_set=[(Xva, yva)], callbacks=[lgb.early_stopping(int(cfg["early_stopping_rounds"]), verbose=False)])
		126	+ if objective == "binary":
		127	+ score = log_loss(yva, np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6))
		128	+ else:
		129	+ score = float(np.mean((m.predict(Xva) - yva) ** 2))
		130	+ trials.append({"trial": t, "params": p, "best_iteration": int(m.best_iteration_ or 0), "val_score": score})
		131	+ if score < best_score:
		132	+ best, best_score, best_params = m, score, {**p, "best_iteration": int(m.best_iteration_ or 0)}
		133	+ return best, {"val_score": best_score, "params": best_params, "trials": trials}
		134	+
		135	+
		136	+def predict_proba(model, X: np.ndarray) -> np.ndarray:
		137	+ return np.clip(model.predict_proba(X)[:, 1], 1e-6, 1 - 1e-6)

added src/wp39/pipeline.py +62 −0

				@@ -0,0 +1,62 @@
		1	+"""Shared configuration, paths and logging for the WP39 pipeline."""
		2	+from __future__ import annotations
		3	+
		4	+import logging
		5	+import os
		6	+from dataclasses import dataclass
		7	+from pathlib import Path
		8	+
		9	+import yaml
		10	+
		11	+ROOT = Path(__file__).resolve().parents[2]
		12	+
		13	+
		14	+@dataclass(frozen=True)
		15	+class Paths:
		16	+ root: Path
		17	+ configs: Path
		18	+ artifacts: Path
		19	+ manifests: Path
		20	+ logs: Path
		21	+ evaluation: Path
		22	+ models: Path
		23	+ processed: Path
		24	+ results: Path
		25	+ figures: Path
		26	+ paper_tables: Path
		27	+ paper_figures: Path
		28	+ cache: Path
		29	+
		30	+ @classmethod
		31	+ def default(cls) -> "Paths":
		32	+ cache = Path(os.path.expanduser(os.environ.get("WP39_CACHE", "~/wp39-cache")))
		33	+ p = cls(root=ROOT, configs=ROOT / "configs", artifacts=ROOT / "artifacts", manifests=ROOT / "artifacts/manifests",
		34	+ logs=ROOT / "artifacts/logs", evaluation=ROOT / "artifacts/evaluation", models=ROOT / "artifacts/models",
		35	+ processed=ROOT / "data/processed", results=ROOT / "results", figures=ROOT / "figures",
		36	+ paper_tables=ROOT / "paper/tables", paper_figures=ROOT / "paper/figures", cache=cache)
		37	+ for d in (p.manifests, p.logs, p.evaluation, p.models, p.processed, p.results, p.figures, p.paper_tables,
		38	+ p.paper_figures, p.cache / "bars", p.cache / "fundamentals", p.cache / "features", p.cache / "external"):
		39	+ d.mkdir(parents=True, exist_ok=True)
		40	+ return p
		41	+
		42	+
		43	+def load_config(name: str) -> dict:
		44	+ with open(ROOT / "configs" / f"{name}.yaml") as fh:
		45	+ return yaml.safe_load(fh)
		46	+
		47	+
		48	+def get_logger(name: str, logfile: Path | None = None) -> logging.Logger:
		49	+ log = logging.getLogger(name)
		50	+ if log.handlers:
		51	+ return log
		52	+ log.setLevel(logging.INFO)
		53	+ fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
		54	+ h = logging.StreamHandler()
		55	+ h.setFormatter(fmt)
		56	+ log.addHandler(h)
		57	+ if logfile:
		58	+ logfile.parent.mkdir(parents=True, exist_ok=True)
		59	+ fh = logging.FileHandler(logfile)
		60	+ fh.setFormatter(fmt)
		61	+ log.addHandler(fh)
		62	+ return log

added src/wp39/pit_fundamentals.py +115 −0

				@@ -0,0 +1,115 @@
		1	+"""Point-in-time fundamentals: extraction, wide quarterly panel, and as-of joins (docs/PIT_PROTOCOL.md).
		2	+
		3	+The API's `as_of` semantics are replicated locally on the full extract (available_at <= as_of and revision 1) so that
		4	+one call per entity suffices; `tests/test_api_contract.py::test_asof_equivalence` checks the equivalence live.
		5	+"""
		6	+from __future__ import annotations
		7	+
		8	+import datetime as dt
		9	+from pathlib import Path
		10	+
		11	+import polars as pl
		12	+
		13	+from .api_client import FinDataClient
		14	+from .provenance import append_jsonl, now_iso, sha256_file
		15	+
		16	+METRICS = ["rev", "cogs", "gp", "op_inc", "op_ex", "ebit", "ebitda", "net_inc", "net_inc_com", "eps", "eps_d",
		17	+ "cash_op", "capex", "cash_inv", "cash_fin", "cash", "assets_c", "assets_tot", "liab_c", "liab_tot", "debt",
		18	+ "debt_c", "debt_nc", "equity", "receivables", "inventory", "payables", "ppe", "intangibles", "research_dev",
		19	+ "sgna", "interest_exp", "intr_cover", "depr", "shares", "shares_av", "shares_av_diluted", "roa", "roe",
		20	+ "roic", "gm", "om", "ret_earnings", "cash_div", "dps", "share_comp", "tax_expense"]
		21	+
		22	+FACT_COLS = ["entity_id", "security_id", "metric", "value", "unit", "fiscal_period_start", "fiscal_period_end",
		23	+ "fiscal_period_type", "fiscal_year", "fiscal_period", "filing_date", "available_at", "observed_at",
		24	+ "revision_id", "restatement_flag", "superseded_at", "source_document"]
		25	+
		26	+
		27	+def fetch_entity_facts(client: FinDataClient, entity_id: str, dest: Path, manifest: Path) -> pl.DataFrame:
		28	+ """All quarterly facts of one entity (no as_of → full history, current revision). Cached as Parquet."""
		29	+ if dest.exists():
		30	+ return pl.read_parquet(dest)
		31	+ d = client.fundamentals(entity_id, as_of=None, period_type="q", limit=20000)
		32	+ rows = d.get("data", [])
		33	+ if len(rows) >= 20000:
		34	+ raise RuntimeError(f"{entity_id}: fact count hit the API limit; narrow the metrics")
		35	+ df = pl.DataFrame([{k: r.get(k) for k in FACT_COLS} for r in rows]) if rows else pl.DataFrame(schema={c: pl.Utf8 for c in FACT_COLS})
		36	+ if rows:
		37	+ df = df.with_columns([
		38	+ pl.col("value").cast(pl.Float64), pl.col("fiscal_year").cast(pl.Int32), pl.col("revision_id").cast(pl.Int32),
		39	+ pl.col("fiscal_period_end").str.to_date(), pl.col("fiscal_period_start").str.to_date(),
		40	+ pl.col("filing_date").str.to_date(),
		41	+ pl.col("available_at").str.to_datetime(time_zone="UTC", strict=False),
		42	+ pl.col("observed_at").str.to_datetime(time_zone="UTC", strict=False),
		43	+ ])
		44	+ dest.parent.mkdir(parents=True, exist_ok=True)
		45	+ df.write_parquet(dest)
		46	+ append_jsonl(manifest, {"entity_id": entity_id, "rows": df.height, "sha256": sha256_file(dest), "fetched_at": now_iso(),
		47	+ "pit_policy": d.get("meta", {}).get("pit_policy"), "license": d.get("meta", {}).get("license"),
		48	+ "path": str(dest)})
		49	+ return df
		50	+
		51	+
		52	+def wide_quarterly(facts: pl.DataFrame) -> pl.DataFrame:
		53	+ """One row per (entity, fiscal_period_end) with metric columns, filing_date, available_at (ET date of availability)."""
		54	+ f = facts.filter((pl.col("fiscal_period_type") == "q") & (pl.col("revision_id") == 1) & pl.col("metric").is_in(METRICS))
		55	+ if f.height == 0:
		56	+ return pl.DataFrame()
		57	+ wide = f.pivot(on="metric", index=["entity_id", "security_id", "fiscal_period_end", "fiscal_year", "fiscal_period",
		58	+ "filing_date", "available_at"], values="value", aggregate_function="first")
		59	+ for m in METRICS:
		60	+ if m not in wide.columns:
		61	+ wide = wide.with_columns(pl.lit(None, dtype=pl.Float64).alias(m))
		62	+ wide = wide.with_columns(pl.col("available_at").dt.convert_time_zone("America/New_York").dt.date().alias("available_date"))
		63	+ return wide.sort(["entity_id", "fiscal_period_end"])
		64	+
		65	+
		66	+def add_quarter_index(wide: pl.DataFrame) -> pl.DataFrame:
		67	+ """Sequential quarter index per entity + gap flags (irregular fiscal calendars)."""
		68	+ w = wide.sort(["entity_id", "fiscal_period_end"]).with_columns([
		69	+ pl.col("fiscal_period_end").diff().over("entity_id").dt.total_days().alias("days_since_prev_q"),
		70	+ ])
		71	+ w = w.with_columns(pl.int_range(pl.len()).over("entity_id").alias("qidx"))
		72	+ return w
		73	+
		74	+
		75	+def asof_latest_available(wide: pl.DataFrame, cutoffs: pl.DataFrame) -> pl.DataFrame:
		76	+ """For each (entity_id, cutoff) return the latest quarter with available_date <= cutoff (strict PIT).
		77	+
		78	+ cutoffs: columns entity_id, cutoff (date). Output keeps all cutoff columns and appends the fundamentals row (suffix _pit).
		79	+ """
		80	+ cols = [m for m in METRICS if m in wide.columns]
		81	+ w = wide.select(["entity_id", "fiscal_period_end", "available_date", "qidx"] + cols).rename(
		82	+ {"fiscal_period_end": "pit_period_end", "available_date": "pit_available_date", "qidx": "pit_qidx"})
		83	+ w = w.sort(["entity_id", "pit_available_date"])
		84	+ c = cutoffs.sort(["entity_id", "cutoff"])
		85	+ joined = c.join_asof(w, left_on="cutoff", right_on="pit_available_date", by="entity_id", strategy="backward")
		86	+ # join_asof with strategy backward is <=; make the inequality explicit for the leakage test
		87	+ joined = joined.with_columns((pl.col("pit_available_date") <= pl.col("cutoff")).alias("pit_ok"))
		88	+ return joined
		89	+
		90	+
		91	+def pit_feature_block(df: pl.DataFrame) -> pl.DataFrame:
		92	+ """Fundamental ratios from the latest available quarter (columns from asof_latest_available) + lagged growth.
		93	+
		94	+ Growth rates need the same-quarter-last-year value *as of the cutoff*; they are supplied by `samples.py` through the
		95	+ wide panel (quarters with available_date <= cutoff only).
		96	+ """
		97	+ eps = 1e-9
		98	+ ta = pl.col("assets_tot").abs() + eps
		99	+ rev = pl.col("rev").abs() + eps
		100	+ return df.with_columns([
		101	+ (pl.col("gp") / rev).alias("f_gross_margin"),
		102	+ (pl.col("op_inc") / rev).alias("f_op_margin"),
		103	+ (pl.col("net_inc") / ta).alias("f_ni_assets"),
		104	+ (pl.col("cash_op") / ta).alias("f_ocf_assets"),
		105	+ ((pl.col("net_inc") - pl.col("cash_op")) / ta).alias("f_accruals"),
		106	+ (pl.col("cash") / ta).alias("f_cash_assets"),
		107	+ (pl.col("debt") / ta).alias("f_leverage"),
		108	+ (pl.col("assets_c") / (pl.col("liab_c").abs() + eps)).alias("f_current_ratio"),
		109	+ (pl.col("capex") / ta).alias("f_capex_assets"),
		110	+ (pl.col("research_dev") / rev).alias("f_rd_rev"),
		111	+ pl.col("intr_cover").clip(-50, 50).alias("f_int_cover"),
		112	+ (pl.col("net_inc") < 0).cast(pl.Int8).alias("f_loss"),
		113	+ pl.col("assets_tot").abs().log1p().alias("f_log_assets"),
		114	+ pl.col("roa").alias("f_roa"),
		115	+ ])

added src/wp39/portfolio.py +124 −0

				@@ -0,0 +1,124 @@
		1	+"""H5: pricing tests. (a) announcement-window abnormal returns regressed on the ex-ante signal;
		2	+(b) secondary portfolio sorts with factor adjustment, turnover and conservative costs. No fill at the signal close."""
		3	+from __future__ import annotations
		4	+
		5	+import io
		6	+import zipfile
		7	+from pathlib import Path
		8	+
		9	+import httpx
		10	+import numpy as np
		11	+import pandas as pd
		12	+import polars as pl
		13	+
		14	+
		15	+def download_french(url: str, dest: Path) -> Path:
		16	+ if not dest.exists():
		17	+ r = httpx.get(url, timeout=120, follow_redirects=True)
		18	+ r.raise_for_status()
		19	+ dest.write_bytes(r.content)
		20	+ return dest
		21	+
		22	+
		23	+def read_french_daily(zip_path: Path) -> pd.DataFrame:
		24	+ with zipfile.ZipFile(zip_path) as z:
		25	+ name = z.namelist()[0]
		26	+ txt = z.read(name).decode("latin1")
		27	+ lines = txt.splitlines()
		28	+ start = next(i for i, l in enumerate(lines) if l.strip()[:8].isdigit())
		29	+ body = "\n".join(l for l in lines[start:] if l.strip()[:8].isdigit())
		30	+ df = pd.read_csv(io.StringIO(body), header=None)
		31	+ hdr = [h.strip() for h in lines[start - 1].split(",")]
		32	+ hdr[0] = "date"
		33	+ df.columns = hdr[: df.shape[1]]
		34	+ df["date"] = pd.to_datetime(df["date"].astype(str), format="%Y%m%d")
		35	+ for c in df.columns[1:]:
		36	+ df[c] = pd.to_numeric(df[c], errors="coerce") / 100.0
		37	+ return df.set_index("date")
		38	+
		39	+
		40	+def car_window(daily: pl.DataFrame, ff: pd.DataFrame, entity_dates: pl.DataFrame) -> pl.DataFrame:
		41	+ """Cumulative market-adjusted return from cutoff+1 to A_q+1 (inclusive) per (symbol, cutoff, A_q)."""
		42	+ d = daily.select(["symbol", "session_date", "d_ret_cc"]).to_pandas()
		43	+ mkt = (ff["Mkt-RF"] + ff["RF"]).rename("mkt")
		44	+ d = d.merge(mkt, left_on="session_date", right_index=True, how="left")
		45	+ d["ar"] = d["d_ret_cc"] - d["mkt"]
		46	+ d = d.sort_values(["symbol", "session_date"])
		47	+ d["cum_ar"] = d.groupby("symbol")["ar"].cumsum()
		48	+ key = d.set_index(["symbol", "session_date"])["cum_ar"]
		49	+ rows = []
		50	+ for r in entity_dates.iter_rows(named=True):
		51	+ s = r["symbol"]
		52	+ try:
		53	+ a = key.loc[(s, pd.Timestamp(r["cutoff"]))]
		54	+ b_idx = d[(d.symbol == s) & (d.session_date > pd.Timestamp(r["A_q"]))].head(1)
		55	+ b = float(b_idx["cum_ar"].iloc[0]) if len(b_idx) else np.nan
		56	+ rows.append({**{k: r[k] for k in ("entity_id", "symbol", "cutoff", "A_q", "fiscal_period_end")}, "car": b - float(a)})
		57	+ except KeyError:
		58	+ continue
		59	+ return pl.DataFrame(rows)
		60	+
		61	+
		62	+def monthly_decile_portfolios(sig: pd.DataFrame, daily: pd.DataFrame, ff: pd.DataFrame, cost_bps: float = 20.0) -> dict:
		63	+ """sig: columns symbol, cutoff (month-end), score (higher = more likely deterioration). Positions formed at the
		64	+ close of the *next* session after the cutoff (no same-close fill), held one month, equal-weighted deciles."""
		65	+ daily = daily.sort_values(["symbol", "session_date"])
		66	+ sessions = np.sort(daily["session_date"].unique())
		67	+ out_rows, prev_hold = [], {}
		68	+ months = sorted(sig["cutoff"].unique())
		69	+ for i, c in enumerate(months):
		70	+ s = sig[sig.cutoff == c].dropna(subset=["score"])
		71	+ if len(s) < 50:
		72	+ continue
		73	+ s = s.assign(dec=pd.qcut(s["score"].rank(method="first"), 10, labels=False) + 1)
		74	+ nxt = sessions[sessions > np.datetime64(c)]
		75	+ if len(nxt) < 2:
		76	+ continue
		77	+ entry = nxt[0]
		78	+ exit_ = months[i + 1] if i + 1 < len(months) else sessions[-1]
		79	+ exit_entry = sessions[sessions > np.datetime64(exit_)]
		80	+ exit_day = exit_entry[0] if len(exit_entry) else sessions[-1]
		81	+ w = daily[(daily.session_date > entry) & (daily.session_date <= exit_day)]
		82	+ hold_ret = w.groupby("symbol")["d_ret_cc"].apply(lambda x: float(np.prod(1 + x.fillna(0)) - 1))
		83	+ s = s.merge(hold_ret.rename("ret"), left_on="symbol", right_index=True, how="left")
		84	+ for dec, g in s.groupby("dec"):
		85	+ names = set(g.symbol)
		86	+ prev = prev_hold.get(dec, set())
		87	+ turnover = len(names ^ prev) / max(len(names), 1) / 2
		88	+ prev_hold[dec] = names
		89	+ out_rows.append({"cutoff": c, "entry": entry, "exit": exit_day, "decile": int(dec), "n": len(g),
		90	+ "ret": float(g["ret"].mean()), "turnover": turnover,
		91	+ "ret_net": float(g["ret"].mean()) - 2 * turnover * cost_bps / 1e4})
		92	+ port = pd.DataFrame(out_rows)
		93	+ return {"portfolios": port}
		94	+
		95	+
		96	+def factor_alpha(port: pd.DataFrame, ff: pd.DataFrame, decile_long: int, decile_short: int, net: bool = False) -> dict:
		97	+ """Monthly long-short (long low-score, short high-score) regressed on FF5 + MOM compounded over the same windows."""
		98	+ col = "ret_net" if net else "ret"
		99	+ L = port[port.decile == decile_long].set_index("cutoff")[col]
		100	+ S = port[port.decile == decile_short].set_index("cutoff")[col]
		101	+ ls = (L - S).dropna()
		102	+ rows = []
		103	+ for c in ls.index:
		104	+ w = port[(port.cutoff == c) & (port.decile == decile_long)].iloc[0]
		105	+ f = ff[(ff.index > pd.Timestamp(w["entry"])) & (ff.index <= pd.Timestamp(w["exit"]))]
		106	+ rows.append({"cutoff": c, **{k: float(np.prod(1 + f[k]) - 1) for k in ("Mkt-RF", "SMB", "HML", "RMW", "CMA", "Mom", "RF") if k in f}})
		107	+ F = pd.DataFrame(rows).set_index("cutoff")
		108	+ y = (ls - F["RF"]).to_numpy()
		109	+ X = np.column_stack([np.ones(len(F))] + [F[k].to_numpy() for k in ("Mkt-RF", "SMB", "HML", "RMW", "CMA", "Mom") if k in F])
		110	+ beta, *_ = np.linalg.lstsq(X, y, rcond=None)
		111	+ e = y - X @ beta
		112	+ T = len(y); L_ = int(np.floor(4 * (T / 100) ** (2 / 9)))
		113	+ # HAC se for the intercept
		114	+ Xi = np.linalg.pinv(X.T @ X)
		115	+ S0 = sum(np.outer(X[t] * e[t], X[t] * e[t]) for t in range(T))
		116	+ for l in range(1, L_ + 1):
		117	+ wgt = 1 - l / (L_ + 1)
		118	+ for t in range(l, T):
		119	+ G = np.outer(X[t] * e[t], X[t - l] * e[t - l]); S0 += wgt * (G + G.T)
		120	+ V = Xi @ S0 @ Xi
		121	+ se = np.sqrt(np.diag(V))
		122	+ return {"alpha_monthly": float(beta[0]), "alpha_se": float(se[0]), "alpha_t": float(beta[0] / se[0]) if se[0] > 0 else float("nan"),
		123	+ "mean_ls": float(ls.mean()), "sd_ls": float(ls.std()), "sharpe_ann": float(ls.mean() / ls.std() * np.sqrt(12)) if ls.std() > 0 else float("nan"),
		124	+ "T": int(T), "net": net}

added src/wp39/provenance.py +61 −0

				@@ -0,0 +1,61 @@
		1	+"""Dataset manifests, hashes and code versions (no raw data, no secrets)."""
		2	+from __future__ import annotations
		3	+
		4	+import hashlib
		5	+import json
		6	+import os
		7	+import subprocess
		8	+import time
		9	+from pathlib import Path
		10	+from typing import Any, Iterable
		11	+
		12	+
		13	+def sha256_file(path: str | os.PathLike, chunk: int = 1 << 20) -> str:
		14	+ h = hashlib.sha256()
		15	+ with open(path, "rb") as fh:
		16	+ for block in iter(lambda: fh.read(chunk), b""):
		17	+ h.update(block)
		18	+ return h.hexdigest()
		19	+
		20	+
		21	+def sha256_text(text: str) -> str:
		22	+ return hashlib.sha256(text.encode()).hexdigest()
		23	+
		24	+
		25	+def git_sha(repo: str | os.PathLike | None = None) -> str:
		26	+ try:
		27	+ return subprocess.check_output(["git", "rev-parse", "--short=12", "HEAD"], cwd=repo, text=True,
		28	+ stderr=subprocess.DEVNULL).strip()
		29	+ except Exception:
		30	+ return "nogit"
		31	+
		32	+
		33	+def now_iso() -> str:
		34	+ return time.strftime("%Y-%m-%dT%H:%M:%S%z")
		35	+
		36	+
		37	+def append_jsonl(path: str | os.PathLike, record: dict[str, Any]) -> None:
		38	+ p = Path(path)
		39	+ p.parent.mkdir(parents=True, exist_ok=True)
		40	+ with open(p, "a") as fh:
		41	+ fh.write(json.dumps(record, default=str, sort_keys=True) + "\n")
		42	+
		43	+
		44	+def read_jsonl(path: str | os.PathLike) -> list[dict[str, Any]]:
		45	+ p = Path(path)
		46	+ if not p.exists():
		47	+ return []
		48	+ with open(p) as fh:
		49	+ return [json.loads(line) for line in fh if line.strip()]
		50	+
		51	+
		52	+def manifest_index(path: str | os.PathLike, keys: Iterable[str]) -> dict[tuple, dict[str, Any]]:
		53	+ keys = tuple(keys)
		54	+ return {tuple(r.get(k) for k in keys): r for r in read_jsonl(path)}
		55	+
		56	+
		57	+def write_json(path: str | os.PathLike, obj: Any) -> None:
		58	+ p = Path(path)
		59	+ p.parent.mkdir(parents=True, exist_ok=True)
		60	+ with open(p, "w") as fh:
		61	+ json.dump(obj, fh, indent=1, default=str, sort_keys=True)

added src/wp39/robustness.py +46 −0

				@@ -0,0 +1,46 @@
		1	+"""Negative controls, placebos and subgroup stress tests (Phase 5)."""
		2	+from __future__ import annotations
		3	+
		4	+import numpy as np
		5	+import polars as pl
		6	+
		7	+
		8	+def shuffle_labels_within_quarter(df: pl.DataFrame, y: str, seed: int) -> pl.DataFrame:
		9	+ """Permute the label within calendar quarter of the cutoff: any 'gain' left is overfitting/leakage."""
		10	+ rng = np.random.default_rng(seed)
		11	+ q = df.with_columns((pl.col("cutoff").dt.year() * 4 + (pl.col("cutoff").dt.month() - 1) // 3).alias("_cq"))
		12	+ parts = []
		13	+ for _, g in q.group_by("_cq", maintain_order=True):
		14	+ v = g[y].to_numpy().copy(); rng.shuffle(v)
		15	+ parts.append(g.with_columns(pl.Series(y, v)))
		16	+ return pl.concat(parts).drop("_cq")
		17	+
		18	+
		19	+def subgroup_masks(df: pl.DataFrame) -> dict[str, pl.Expr]:
		20	+ """Pre-specified subgroups (terciles are computed on the TRAINING rows and passed as cut-points by the caller)."""
		21	+ return {
		22	+ "all": pl.lit(True),
		23	+ "size_small": pl.col("tercile_size") == 1, "size_large": pl.col("tercile_size") == 3,
		24	+ "illiq_high": pl.col("tercile_illiq") == 3, "illiq_low": pl.col("tercile_illiq") == 1,
		25	+ "lev_high": pl.col("tercile_lev") == 3, "lev_low": pl.col("tercile_lev") == 1,
		26	+ "loss_firms": pl.col("f_loss") == 1, "profit_firms": pl.col("f_loss") == 0,
		27	+ "covid_2020": (pl.col("cutoff").dt.year() == 2020), "ex_covid": (pl.col("cutoff").dt.year() != 2020),
		28	+ "after_close_release": pl.col("release_time") == "after_close", "before_open_release": pl.col("release_time") == "before_open",
		29	+ "sector_finance": pl.col("sic_division") == "H_finance_insurance_realestate",
		30	+ "sector_nonfinance": pl.col("sic_division") != "H_finance_insurance_realestate",
		31	+ }
		32	+
		33	+
		34	+def add_terciles(train: pl.DataFrame, others: list[pl.DataFrame]) -> list[pl.DataFrame]:
		35	+ """Tercile cut-points for size, illiquidity and leverage computed on training rows only."""
		36	+ cuts = {}
		37	+ for name, col in (("size", "f_log_mcap"), ("illiq", "wd_amihud60"), ("lev", "f_leverage")):
		38	+ s = train[col].drop_nulls().drop_nans()
		39	+ cuts[name] = (float(s.quantile(1 / 3)), float(s.quantile(2 / 3)))
		40	+ out = []
		41	+ for d in [train] + others:
		42	+ for name, col in (("size", "f_log_mcap"), ("illiq", "wd_amihud60"), ("lev", "f_leverage")):
		43	+ lo, hi = cuts[name]
		44	+ d = d.with_columns(pl.when(pl.col(col) <= lo).then(1).when(pl.col(col) <= hi).then(2).otherwise(3).alias(f"tercile_{name}"))
		45	+ out.append(d)
		46	+ return out

added src/wp39/samples.py +180 −0

				@@ -0,0 +1,180 @@
		1	+"""Cohort construction: cutoffs, window aggregation of session features, PIT fundamentals join, exclusions.
		2	+
		3	+Event-time: c = A_q - h trading days (h in horizons). Fixed-calendar: c = month-end sessions, target = next release.
		4	+All joins are explicit as-of joins on `session_date <= c` (bars) and `available_date <= c` (fundamentals).
		5	+"""
		6	+from __future__ import annotations
		7	+
		8	+import datetime as dt
		9	+
		10	+import numpy as np
		11	+import polars as pl
		12	+
		13	+from .calendar import session_array, shift_trading_days, month_end_sessions
		14	+from .features import FREQS, INTRADAY_FAMILIES
		15	+from .pit_fundamentals import METRICS, pit_feature_block
		16	+
		17	+WINDOW_MEAN = ["rv", "bpv", "rj", "rsneg_share", "sjv", "rskew", "rkurt", "max_share", "ac1", "vol_first30", "vol_last30",
		18	+ "vol_gini", "down_vol_share", "present_share", "rv_mktdown_share"]
		19	+
		20	+
		21	+def complete_session_index(feat: pl.DataFrame) -> pl.DataFrame:
		22	+ """Reindex a symbol's session features to every XNYS session between its first and last session (rows for missing
		23	+ sessions have null features) so that rolling windows are measured in trading days."""
		24	+ arr = session_array()
		25	+ d0, d1 = feat["session_date"].min(), feat["session_date"].max()
		26	+ s = arr[(arr >= np.datetime64(d0)) & (arr <= np.datetime64(d1))]
		27	+ idx = pl.DataFrame({"session_date": pl.Series(s.astype("datetime64[D]")).cast(pl.Date)})
		28	+ return idx.join(feat, on="session_date", how="left").sort("session_date")
		29	+
		30	+
		31	+def rolling_window_features(feat: pl.DataFrame, short: int = 20, long: int = 60, min_valid: int = 15) -> pl.DataFrame:
		32	+ """Rolling (trailing, inclusive) aggregates per session; the row at session c is the feature vector for cutoff c."""
		33	+ f = complete_session_index(feat)
		34	+ exprs = []
		35	+ valid = pl.col("f1m_n_bars").is_not_null() & (pl.col("f1m_n_bars") >= 100)
		36	+ f = f.with_columns(valid.cast(pl.Int8).alias("_valid"))
		37	+ exprs.append(pl.col("_valid").rolling_sum(window_size=short, min_samples=1).alias("w_valid_days"))
		38	+ # intraday families: window means (RV-type quantities averaged -> per-day average, same across frequencies)
		39	+ for name in FREQS:
		40	+ p = f"f{name}_"
		41	+ for fam in WINDOW_MEAN:
		42	+ exprs.append(pl.col(p + fam).rolling_mean(window_size=short, min_samples=min_valid).alias(f"w{name}_{fam}"))
		43	+ exprs.append(pl.col(p + "drift").rolling_sum(window_size=short, min_samples=min_valid).alias(f"w{name}_drift"))
		44	+ exprs.append((pl.col(p + "sxy").rolling_sum(window_size=short, min_samples=min_valid) /
		45	+ (pl.col(p + "syy").rolling_sum(window_size=short, min_samples=min_valid) + 1e-18)).alias(f"w{name}_beta"))
		46	+ exprs.append(pl.col(p + "rv").rolling_std(window_size=short, min_samples=min_valid).alias(f"w{name}_volofvol"))
		47	+ # variance ratio: mean intraday RV vs variance of daily close-to-close returns over the window
		48	+ exprs.append((pl.col(p + "rv").rolling_mean(window_size=short, min_samples=min_valid) /
		49	+ (pl.col("d_ret_cc").rolling_var(window_size=short, min_samples=min_valid) + 1e-18)).alias(f"w{name}_vratio"))
		50	+ # daily block
		51	+ r = pl.col("d_ret_cc")
		52	+ exprs += [
		53	+ ((1.0 + r).log().rolling_sum(window_size=short, min_samples=min_valid)).alias("wd_ret20"),
		54	+ ((1.0 + r).log().rolling_sum(window_size=long, min_samples=3 * min_valid)).alias("wd_ret60"),
		55	+ r.rolling_std(window_size=short, min_samples=min_valid).alias("wd_vol20"),
		56	+ r.rolling_std(window_size=long, min_samples=3 * min_valid).alias("wd_vol60"),
		57	+ pl.col("d_parkinson").rolling_mean(window_size=short, min_samples=min_valid).sqrt().alias("wd_parkinson20"),
		58	+ r.rolling_skew(window_size=long).alias("wd_skew60"),
		59	+ r.rolling_kurtosis(window_size=long).alias("wd_kurt60"),
		60	+ (pl.col("d_ret_co").rolling_sum(window_size=short, min_samples=min_valid)).alias("wd_overnight20"),
		61	+ (pl.col("d_ret_oc").rolling_sum(window_size=short, min_samples=min_valid)).alias("wd_intraday20"),
		62	+ (pl.col("d_volume").rolling_mean(window_size=short, min_samples=min_valid) /
		63	+ (pl.col("d_volume").rolling_median(window_size=long, min_samples=3 * min_valid) + 1.0)).alias("wd_volratio20"),
		64	+ (pl.col("d_volume").rolling_mean(window_size=5, min_samples=3) /
		65	+ (pl.col("d_volume").rolling_median(window_size=long, min_samples=3 * min_valid) + 1.0)).alias("wd_volratio5"),
		66	+ ((r.abs() / (pl.col("d_dollar_vol") + 1.0)) * 1e6).rolling_mean(window_size=long, min_samples=3 * min_valid).alias("wd_amihud60"),
		67	+ (r == 0).cast(pl.Float64).rolling_mean(window_size=long, min_samples=3 * min_valid).alias("wd_zeroret60"),
		68	+ pl.col("d_dollar_vol").rolling_mean(window_size=long, min_samples=3 * min_valid).log1p().alias("wd_log_dollar_vol60"),
		69	+ pl.col("d_close").alias("wd_close"),
		70	+ pl.col("f1m_n_bars").rolling_mean(window_size=short, min_samples=min_valid).alias("wd_bars20"),
		71	+ ]
		72	+ f = f.with_columns(exprs)
		73	+ # max drawdown over 60 sessions from cumulative log returns
		74	+ f = f.with_columns((1.0 + r).log().fill_null(0.0).cum_sum().alias("_cum"))
		75	+ f = f.with_columns((pl.col("_cum") - pl.col("_cum").rolling_max(window_size=long, min_samples=3 * min_valid)).rolling_min(
		76	+ window_size=long, min_samples=3 * min_valid).alias("wd_maxdd60"))
		77	+ # daily beta / residual vol vs SPY daily (from 1m family sums at daily horizon are not available -> use 5m beta proxy)
		78	+ cols = ["session_date", "symbol", "w_valid_days"] + [c for c in f.columns if c.startswith("w1m_") or c.startswith("w5m_")
		79	+ or c.startswith("w30m_") or c.startswith("wd_")]
		80	+ return f.select(cols)
		81	+
		82	+
		83	+def event_cutoffs(quarters: pl.DataFrame, horizons: list[int]) -> pl.DataFrame:
		84	+ """One row per (entity, quarter, h) with cutoff c = A_q - h trading days (requires release_match == ok)."""
		85	+ q = quarters.filter(pl.col("release_match") == "ok").select(
		86	+ ["entity_id", "fiscal_period_end", "A_q", "filing_date", "available_date", "qidx"])
		87	+ rows = []
		88	+ a_prev = q.with_columns(pl.col("A_q").shift(1).over("entity_id").alias("A_prev"))
		89	+ for r in a_prev.iter_rows(named=True):
		90	+ for h in horizons:
		91	+ try:
		92	+ c = shift_trading_days(r["A_q"], -h)
		93	+ except ValueError:
		94	+ continue
		95	+ rows.append({"entity_id": r["entity_id"], "fiscal_period_end": r["fiscal_period_end"], "h": h, "cutoff": c,
		96	+ "A_q": r["A_q"], "F_q": r["filing_date"], "A_prev": r["A_prev"], "qidx": r["qidx"]})
		97	+ return pl.DataFrame(rows)
		98	+
		99	+
		100	+def calendar_cutoffs(quarters: pl.DataFrame, start: dt.date, end: dt.date, max_td: int = 65) -> pl.DataFrame:
		101	+ """Month-end cutoffs; target = next release after c within max_td trading days."""
		102	+ me = pl.DataFrame({"cutoff": month_end_sessions(start, end)})
		103	+ q = quarters.filter(pl.col("release_match") == "ok").select(["entity_id", "fiscal_period_end", "A_q", "filing_date", "qidx"])
		104	+ ents = q.select("entity_id").unique()
		105	+ grid = ents.join(me, how="cross").sort(["entity_id", "cutoff"])
		106	+ qq = q.sort(["entity_id", "A_q"]).with_columns(pl.col("A_q").alias("_a"))
		107	+ j = grid.join_asof(qq, left_on="cutoff", right_on="_a", by="entity_id", strategy="forward")
		108	+ j = j.filter(pl.col("A_q") > pl.col("cutoff"))
		109	+ arr = session_array()
		110	+ c = np.searchsorted(arr, j["cutoff"].to_numpy().astype("datetime64[D]"), side="right")
		111	+ a = np.searchsorted(arr, j["A_q"].to_numpy().astype("datetime64[D]"), side="right")
		112	+ j = j.with_columns(pl.Series("h", (a - c).astype(int))).filter(pl.col("h") <= max_td)
		113	+ return j.rename({"filing_date": "F_q"}).with_columns(pl.lit(None, dtype=pl.Date).alias("A_prev"))
		114	+
		115	+
		116	+def apply_admissibility(cut: pl.DataFrame) -> pl.DataFrame:
		117	+ """c < min(A_q, F_q) and c > A_prev (previous reveal already happened)."""
		118	+ return cut.filter((pl.col("cutoff") < pl.col("A_q")) & (pl.col("cutoff") < pl.col("F_q")) &
		119	+ (pl.col("A_prev").is_null() | (pl.col("cutoff") > pl.col("A_prev"))))
		120	+
		121	+
		122	+def join_market(cut: pl.DataFrame, win: pl.DataFrame, symbol_of: pl.DataFrame) -> pl.DataFrame:
		123	+ """As-of join of the rolling window features at session_date <= cutoff (exact session normally)."""
		124	+ c = cut.join(symbol_of, on="entity_id", how="left").sort(["symbol", "cutoff"])
		125	+ w = win.sort(["symbol", "session_date"])
		126	+ j = c.join_asof(w, left_on="cutoff", right_on="session_date", by="symbol", strategy="backward", tolerance="5d")
		127	+ return j.rename({"session_date": "market_asof_date"})
		128	+
		129	+
		130	+def join_fundamentals(j: pl.DataFrame, wide: pl.DataFrame) -> pl.DataFrame:
		131	+ """Latest available quarter at cutoff + yoy growth from the quarter 4 back (also available by construction)."""
		132	+ w = wide.select(["entity_id", "fiscal_period_end", "available_date", "qidx", "A_q"] + METRICS).sort(["entity_id", "available_date"])
		133	+ w4 = wide.select(["entity_id", pl.col("qidx").alias("pit_qidx4"), pl.col("rev").alias("rev_l4"), pl.col("op_inc").alias("op_inc_l4"),
		134	+ pl.col("shares").alias("shares_l4"), pl.col("available_date").alias("avail_l4")])
		135	+ wr = w.rename({c: c for c in METRICS}).rename({"fiscal_period_end": "pit_period_end", "available_date": "pit_available_date",
		136	+ "qidx": "pit_qidx", "A_q": "pit_A"})
		137	+ out = j.sort(["entity_id", "cutoff"]).join_asof(wr, left_on="cutoff", right_on="pit_available_date", by="entity_id", strategy="backward")
		138	+ out = out.with_columns((pl.col("pit_qidx") - 4).alias("pit_qidx4")).join(w4, on=["entity_id", "pit_qidx4"], how="left")
		139	+ out = out.with_columns([
		140	+ (pl.col("pit_available_date") <= pl.col("cutoff")).alias("pit_ok"),
		141	+ ((pl.col("rev") - pl.col("rev_l4")) / (pl.col("rev_l4").abs() + 1e-9)).alias("f_rev_growth"),
		142	+ ((pl.col("shares") - pl.col("shares_l4")) / (pl.col("shares_l4").abs() + 1e-9)).alias("f_net_issuance"),
		143	+ (pl.col("qidx") - pl.col("pit_qidx")).alias("quarters_ahead"), # 1 = we predict the quarter right after the latest available
		144	+ ])
		145	+ out = pit_feature_block(out)
		146	+ out = out.with_columns([
		147	+ (pl.col("wd_close") * pl.col("shares").abs()).log1p().alias("f_log_mcap"),
		148	+ ])
		149	+ return out
		150	+
		151	+
		152	+def exclusion_cascade(df: pl.DataFrame, min_price: float, min_valid_days: int) -> tuple[pl.DataFrame, pl.DataFrame]:
		153	+ """Apply frozen exclusions in order; return (sample, cascade table)."""
		154	+ steps = [
		155	+ ("cutoff_admissible", pl.lit(True)),
		156	+ ("market_window_available", pl.col("market_asof_date").is_not_null() & (pl.col("w_valid_days") >= min_valid_days)),
		157	+ ("price_ge_min", pl.col("wd_close") >= min_price),
		158	+ ("fundamentals_available", pl.col("pit_ok").fill_null(False) & pl.col("pit_qidx").is_not_null()),
		159	+ ("predicting_next_quarter", pl.col("quarters_ahead") == 1),
		160	+ ("expectation_defined", pl.col("S_opinc").is_not_null() & pl.col("assets_tot_lag1").is_not_null()),
		161	+ ("regular_fiscal_calendar", pl.col("_gap_ok8") == 8),
		162	+ ]
		163	+ rows, cur = [], df
		164	+ for name, cond in steps:
		165	+ before = cur.height
		166	+ cur = cur.filter(cond)
		167	+ rows.append({"step": name, "remaining": cur.height, "dropped": before - cur.height})
		168	+ return cur, pl.DataFrame(rows)
		169	+
		170	+
		171	+def feature_columns(df: pl.DataFrame) -> dict[str, list[str]]:
		172	+ fund = [c for c in df.columns if c.startswith("f_")] + ["info_age_td", "last_surprise", "mean_surprise4", "hist_adverse_rate"]
		173	+ daily = [c for c in df.columns if c.startswith("wd_") and c != "wd_close"]
		174	+ return {
		175	+ "fundamentals": [c for c in fund if c in df.columns],
		176	+ "daily": daily,
		177	+ "intraday_30m": [c for c in df.columns if c.startswith("w30m_")],
		178	+ "intraday_5m": [c for c in df.columns if c.startswith("w5m_")],
		179	+ "intraday_1m": [c for c in df.columns if c.startswith("w1m_")],
		180	+ }

added src/wp39/splits.py +65 −0

				@@ -0,0 +1,65 @@
		1	+"""Chronological, walk-forward and firm-held-out splits with purge and embargo (RESEARCH_DESIGN §6).
		2	+
		3	+An observation is identified by (entity_id, fiscal_period_end, cutoff, A_q). The *target* is revealed at A_q, so a
		4	+training observation is purged if its A_q falls after (split_start - embargo); test observations start at split_start.
		5	+"""
		6	+from __future__ import annotations
		7	+
		8	+import datetime as dt
		9	+import hashlib
		10	+
		11	+import polars as pl
		12	+
		13	+
		14	+def _q_start(d: dt.date) -> dt.date:
		15	+ return dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
		16	+
		17	+
		18	+def _shift_quarters(d: dt.date, n: int) -> dt.date:
		19	+ m = d.month - 1 + 3 * n
		20	+ return dt.date(d.year + m // 12, m % 12 + 1, 1)
		21	+
		22	+
		23	+def purge_train(df: pl.DataFrame, boundary: dt.date, embargo_quarters: int = 1) -> pl.DataFrame:
		24	+ """Keep training rows whose outcome was revealed before boundary - embargo (quarter-aligned)."""
		25	+ limit = _shift_quarters(_q_start(boundary), -embargo_quarters)
		26	+ return df.filter(pl.col("A_q") < limit)
		27	+
		28	+
		29	+def chronological(df: pl.DataFrame, cfg: dict) -> dict[str, pl.DataFrame]:
		30	+ tr0, tr1 = [dt.date.fromisoformat(str(x)) for x in cfg["train"]]
		31	+ va0, va1 = [dt.date.fromisoformat(str(x)) for x in cfg["validation"]]
		32	+ te0, te1 = [dt.date.fromisoformat(str(x)) for x in cfg["test"]]
		33	+ emb = cfg["walk_forward"]["embargo_quarters"]
		34	+ train = purge_train(df.filter((pl.col("cutoff") >= tr0) & (pl.col("cutoff") <= tr1)), va0, emb)
		35	+ valid = purge_train(df.filter((pl.col("cutoff") >= va0) & (pl.col("cutoff") <= va1)), te0, emb)
		36	+ test = df.filter((pl.col("cutoff") >= te0) & (pl.col("cutoff") <= te1))
		37	+ return {"train": train, "validation": valid, "test": test}
		38	+
		39	+
		40	+def walk_forward(df: pl.DataFrame, cfg: dict, last_year: int) -> list[tuple[int, pl.DataFrame, pl.DataFrame]]:
		41	+ """Expanding annual folds: train on cutoffs < Jan 1 of year y (purged, embargo), test on cutoffs in year y."""
		42	+ out = []
		43	+ emb = cfg["walk_forward"]["embargo_quarters"]
		44	+ for y in range(cfg["walk_forward"]["first_test_year"], last_year + 1):
		45	+ b = dt.date(y, 1, 1)
		46	+ train = purge_train(df.filter(pl.col("cutoff") < b), b, emb)
		47	+ test = df.filter((pl.col("cutoff") >= b) & (pl.col("cutoff") < dt.date(y + 1, 1, 1)))
		48	+ if train.height and test.height:
		49	+ out.append((y, train, test))
		50	+ return out
		51	+
		52	+
		53	+def firm_holdout_mask(entity_ids: pl.Series, fraction: float = 0.2, salt: str = "wp39") -> pl.Series:
		54	+ def h(e: str) -> bool:
		55	+ return int(hashlib.sha256(f"{salt}:{e}".encode()).hexdigest()[:8], 16) / 0xFFFFFFFF < fraction
		56	+ return pl.Series([h(e) for e in entity_ids.to_list()], dtype=pl.Boolean)
		57	+
		58	+
		59	+def overlap_check(train: pl.DataFrame, test: pl.DataFrame) -> dict:
		60	+ """Diagnostics used by tests: no training outcome revealed on/after the first test cutoff; no shared observation ids."""
		61	+ first_test_cutoff = test["cutoff"].min()
		62	+ leak = train.filter(pl.col("A_q") >= first_test_cutoff).height
		63	+ key = ["entity_id", "fiscal_period_end", "h"] if "h" in train.columns else ["entity_id", "fiscal_period_end"]
		64	+ shared = train.select(key).join(test.select(key), on=key, how="inner").height
		65	+ return {"train_outcomes_revealed_in_test": leak, "shared_observations": shared}

added tests/__init__.py +0 −0

added tests/conftest.py +39 −0

				@@ -0,0 +1,39 @@
		1	+import datetime as dt
		2	+import os
		3	+import sys
		4	+from pathlib import Path
		5	+
		6	+import polars as pl
		7	+import pytest
		8	+
		9	+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
		10	+
		11	+LIVE = os.environ.get("WP39_LIVE") == "1"
		12	+live = pytest.mark.skipif(not LIVE, reason="set WP39_LIVE=1 to run live FinData tests")
		13	+
		14	+
		15	+@pytest.fixture
		16	+def toy_wide():
		17	+ """Two firms, regular quarterly calendar, filing 40 d after period end."""
		18	+ rows = []
		19	+ for e in ("frd:AAA", "frd:BBB"):
		20	+ base = 100.0 if e == "frd:AAA" else 50.0
		21	+ for i in range(14):
		22	+ pe = dt.date(2015, 3, 31) + dt.timedelta(days=int(91.3 * i))
		23	+ fd = pe + dt.timedelta(days=40)
		24	+ rows.append({"entity_id": e, "security_id": f"firstrate:stock:{e[4:]}", "fiscal_period_end": pe,
		25	+ "fiscal_year": pe.year, "fiscal_period": f"q{(pe.month - 1) // 3 + 1}", "filing_date": fd,
		26	+ "available_date": fd + dt.timedelta(days=1), "op_inc": base + i + (5 if (i % 4 == 0) else 0),
		27	+ "net_inc": base * 0.7 + i, "cash_op": base * 1.1 + i, "rev": base * 10 + 10 * i,
		28	+ "assets_tot": base * 20, "assets_c": base * 5, "liab_c": base * 3, "roa": 0.02 + 0.001 * i,
		29	+ "intr_cover": 5.0, "shares": 1000.0, "gp": base * 4, "debt": base * 6, "cash": base,
		30	+ "capex": base * 0.2, "research_dev": base * 0.1, "qidx": i})
		31	+ return pl.DataFrame(rows)
		32	+
		33	+
		34	+@pytest.fixture
		35	+def toy_events(toy_wide):
		36	+ ev = toy_wide.select(["entity_id", "filing_date"]).with_columns(
		37	+ (pl.col("filing_date") - pl.duration(days=1)).alias("event_date"), pl.lit("after_close").alias("release_time"),
		38	+ pl.lit(None, dtype=pl.Float64).alias("eps_estimate"), pl.lit(None, dtype=pl.Float64).alias("eps_reported")).drop("filing_date")
		39	+ return ev

added tests/test_api_contract.py +42 −0

				@@ -0,0 +1,42 @@
		1	+import datetime as dt
		2	+import random
		3	+
		4	+import polars as pl
		5	+import pytest
		6	+
		7	+from tests.conftest import live
		8	+from wp39.api_client import FinDataClient, _redact
		9	+
		10	+
		11	+def test_redaction():
		12	+ assert "fd_abc.def" not in _redact("token fd_abc.def leaked")
		13	+
		14	+
		15	+@live
		16	+def test_schema_and_datasets():
		17	+ c = FinDataClient()
		18	+ s = c.schema_bars()
		19	+ assert s["timestamp_convention"] == "bar_open" and s["schema_version"] == 1
		20	+ ids = {d["id"] for d in c.datasets()}
		21	+ assert {"market_bars.stock.1m.raw", "market_bars.etf.1m.raw"} <= ids
		22	+
		23	+
		24	+@live
		25	+def test_asof_equivalence():
		26	+ """Local filter available_at <= as_of on the full extract == API as_of result (metric, period) set."""
		27	+ c = FinDataClient()
		28	+ full = pl.DataFrame(c.fundamentals("frd:MSFT", period_type="q", limit=20000)["data"])
		29	+ full = full.with_columns(pl.col("available_at").str.to_datetime(time_zone="UTC", strict=False))
		30	+ for as_of in ("2016-03-01T00:00:00Z", "2019-11-15T00:00:00Z", "2023-08-01T00:00:00Z"):
		31	+ api = pl.DataFrame(c.fundamentals("frd:MSFT", as_of=as_of, period_type="q", limit=20000)["data"])
		32	+ local = full.filter(pl.col("available_at") <= pl.lit(as_of).str.to_datetime(time_zone="UTC"))
		33	+ a = set(zip(api["metric"], api["fiscal_period_end"])); b = set(zip(local["metric"], local["fiscal_period_end"]))
		34	+ assert a == b, (as_of, len(a), len(b))
		35	+
		36	+
		37	+@live
		38	+def test_bars_limits_are_enforced():
		39	+ from wp39.api_client import FinDataError
		40	+ c = FinDataClient()
		41	+ with pytest.raises(FinDataError):
		42	+ c.get("/market/bars", symbol="AAPL", interval="1m", start="2024-01-01", end="2024-03-15", limit=10)

added tests/test_corporate_actions.py +22 −0

				@@ -0,0 +1,22 @@
		1	+import datetime as dt
		2	+
		3	+import polars as pl
		4	+
		5	+from wp39.features import daily_block
		6	+
		7	+
		8	+def test_split_adjusted_overnight_return():
		9	+ day = pl.DataFrame({"session_date": [dt.date(2020, 8, 28), dt.date(2020, 8, 31)], "d_open": [500.0, 127.0],
		10	+ "d_close": [499.0, 129.0], "d_high": [505.0, 131.0], "d_low": [495.0, 126.0], "d_volume": [1e6, 4e6]})
		11	+ acts = pl.DataFrame({"symbol": ["AAPL"], "ex_date": [dt.date(2020, 8, 31)], "split_ratio": [4.0], "dividend": [0.0]})
		12	+ d = daily_block(day, acts, "AAPL")
		13	+ assert abs(d["d_ret_cc"][1] - (129.0 / (499.0 / 4.0) - 1)) < 1e-9
		14	+ assert abs(d["d_ret_co"][1] - (127.0 / (499.0 / 4.0) - 1)) < 1e-9
		15	+
		16	+
		17	+def test_dividend_adjusted_return():
		18	+ day = pl.DataFrame({"session_date": [dt.date(2021, 2, 4), dt.date(2021, 2, 5)], "d_open": [100.0, 99.0],
		19	+ "d_close": [100.0, 99.5], "d_high": [101.0, 100.0], "d_low": [99.0, 98.0], "d_volume": [1e6, 1e6]})
		20	+ acts = pl.DataFrame({"symbol": ["X"], "ex_date": [dt.date(2021, 2, 5)], "split_ratio": [1.0], "dividend": [0.5]})
		21	+ d = daily_block(day, acts, "X")
		22	+ assert abs(d["d_ret_cc"][1] - ((99.5 + 0.5) / 100.0 - 1)) < 1e-12

added tests/test_features.py +37 −0

				@@ -0,0 +1,37 @@
		1	+import datetime as dt
		2	+import math
		3	+
		4	+import numpy as np
		5	+import polars as pl
		6	+
		7	+from wp39.features import aggregate_buckets, bucket_returns, session_features
		8	+
		9	+
		10	+def _reg_day(n=391, seed=0):
		11	+ rng = np.random.default_rng(seed)
		12	+ r = rng.normal(0, 1e-3, n)
		13	+ px = 100 * np.exp(np.cumsum(r))
		14	+ return pl.DataFrame({"session_date": [dt.date(2024, 3, 5)] * n, "m": list(range(n)), "open": np.r_[100.0, px[:-1]],
		15	+ "high": px * 1.0005, "low": px * 0.9995, "close": px, "volume": rng.integers(100, 1000, n).astype(float),
		16	+ "close_minute": [390] * n})
		17	+
		18	+
		19	+def test_bucket_aggregation_counts_and_close_print():
		20	+ reg = _reg_day()
		21	+ b5 = aggregate_buckets(reg, 5); b30 = aggregate_buckets(reg, 30)
		22	+ assert b5.height == 78 and b30.height == 13
		23	+ assert b5["close"][-1] == reg["close"][-1] and b30["close"][-1] == reg["close"][-1]
		24	+ assert abs(b5["volume"].sum() - reg["volume"].sum()) < 1e-9
		25	+
		26	+
		27	+def test_rv_identities_and_shares():
		28	+ reg = _reg_day()
		29	+ for f in (1, 5, 30):
		30	+ out = session_features(aggregate_buckets(reg, f), f, None)
		31	+ r = out.row(0, named=True); p = f"f{f}m_"
		32	+ assert 0 <= r[p + "rsneg_share"] <= 1 and 0 <= r[p + "rj"] <= 1 and 0 < r[p + "present_share"] <= 1.0 + 1e-9
		33	+ assert 0 <= r[p + "vol_first30"] <= 1 and 0 <= r[p + "down_vol_share"] <= 1 and 0 <= r[p + "vol_gini"] <= 1
		34	+ o1 = session_features(aggregate_buckets(reg, 1), 1, None).row(0, named=True)
		35	+ rr = bucket_returns(aggregate_buckets(reg, 1))["r"].to_numpy()
		36	+ assert abs(o1["f1m_rv"] - np.sum(rr ** 2)) < 1e-12
		37	+ assert abs(o1["f1m_drift"] - math.log(reg["close"][-1] / reg["open"][0])) < 1e-12

added tests/test_instrument_ids.py +17 −0

				@@ -0,0 +1,17 @@
		1	+import polars as pl
		2	+
		3	+from wp39.instruments import build_crosswalk, sic_division
		4	+
		5	+
		6	+def test_sic_divisions():
		7	+ assert sic_division(3571) == "D_manufacturing" and sic_division(6021) == "H_finance_insurance_realestate"
		8	+ assert sic_division(None) == "unknown"
		9	+
		10	+
		11	+def test_crosswalk_keeps_all_entities_and_flags_missing_cik():
		12	+ ents = pl.DataFrame({"entity_id": ["frd:AAA", "frd:ZZZ"], "security_id": ["firstrate:stock:AAA", "firstrate:stock:ZZZ"],
		13	+ "ticker": ["AAA", "ZZZ"]})
		14	+ sec = pl.DataFrame({"ticker": ["AAA"], "cik": [1], "sec_name": ["AAA Corp"]})
		15	+ subs = pl.DataFrame({"cik": [1], "sic": [3571], "sic_division": ["D_manufacturing"]})
		16	+ xw = build_crosswalk(ents, sec, subs)
		17	+ assert xw.height == 2 and xw.filter(pl.col("ticker") == "ZZZ")["has_cik"][0] is False

added tests/test_labels.py +26 −0

				@@ -0,0 +1,26 @@
		1	+import datetime as dt
		2	+
		3	+import polars as pl
		4	+
		5	+from wp39.labels import build_quarter_table, finalize_labels, match_release_dates
		6	+
		7	+
		8	+def test_release_match_rules(toy_wide, toy_events):
		9	+ ev = toy_events.with_columns(pl.when(pl.col("entity_id") == "frd:BBB").then(pl.col("event_date") - pl.duration(days=60))
		10	+ .otherwise(pl.col("event_date")).alias("event_date"))
		11	+ w = match_release_dates(toy_wide, ev)
		12	+ assert (w.filter(pl.col("entity_id") == "frd:AAA")["release_match"] == "ok").all()
		13	+ assert (w.filter(pl.col("entity_id") == "frd:BBB")["release_match"] != "ok").all()
		14	+
		15	+
		16	+def test_labels_are_not_price_based(toy_wide, toy_events):
		17	+ q = finalize_labels(build_quarter_table(match_release_dates(toy_wide, toy_events)))
		18	+ assert not any(c.startswith("w") or c.startswith("d_") for c in q.columns)
		19	+ assert set(q["y_adverse_opinc"].drop_nulls().unique().to_list()) <= {0, 1}
		20	+
		21	+
		22	+def test_release_day_excluded(toy_wide, toy_events):
		23	+ from wp39.samples import apply_admissibility, event_cutoffs
		24	+ w = match_release_dates(toy_wide, toy_events)
		25	+ c = apply_admissibility(event_cutoffs(w, [1]))
		26	+ assert (c["cutoff"] < c["A_q"]).all()

added tests/test_pit_leakage.py +45 −0

				@@ -0,0 +1,45 @@
		1	+import datetime as dt
		2	+
		3	+import polars as pl
		4	+
		5	+from wp39.labels import build_quarter_table, finalize_labels, match_release_dates
		6	+from wp39.pit_fundamentals import asof_latest_available
		7	+from wp39.samples import apply_admissibility, event_cutoffs
		8	+
		9	+
		10	+def test_no_future_fundamentals(toy_wide):
		11	+ cut = pl.DataFrame({"entity_id": ["frd:AAA"] * 3, "cutoff": [dt.date(2015, 5, 1), dt.date(2015, 5, 11), dt.date(2015, 5, 12)]})
		12	+ j = asof_latest_available(toy_wide, cut)
		13	+ # period end 2015-03-31, filing 2015-05-10, available 2015-05-11
		14	+ assert j["pit_period_end"].to_list() == [None, dt.date(2015, 3, 31), dt.date(2015, 3, 31)]
		15	+ assert j.filter(pl.col("pit_period_end").is_not_null())["pit_ok"].all()
		16	+
		17	+
		18	+def test_join_key_is_available_at_not_period_end(toy_wide):
		19	+ cut = pl.DataFrame({"entity_id": ["frd:AAA"], "cutoff": [dt.date(2015, 4, 15)]}) # after period end, before filing
		20	+ j = asof_latest_available(toy_wide, cut)
		21	+ assert j["pit_period_end"][0] is None
		22	+
		23	+
		24	+def test_cutoff_precedes_release_and_filing(toy_wide, toy_events):
		25	+ w = match_release_dates(toy_wide, toy_events)
		26	+ cut = apply_admissibility(event_cutoffs(w, [20, 5, 1]))
		27	+ assert (cut["cutoff"] < cut["A_q"]).all() and (cut["cutoff"] < cut["F_q"]).all()
		28	+ assert cut.filter(pl.col("A_prev").is_not_null()).select((pl.col("cutoff") > pl.col("A_prev")).all()).item()
		29	+
		30	+
		31	+def test_observed_at_not_used():
		32	+ import inspect
		33	+ from wp39 import labels, pit_fundamentals, samples
		34	+ src = inspect.getsource(samples) + inspect.getsource(labels) + inspect.getsource(pit_fundamentals.asof_latest_available)
		35	+ assert "observed_at" not in src.replace("observed_at\",", "") # only in the column list of the extract
		36	+
		37	+
		38	+def test_expectation_uses_only_prior_quarters(toy_wide, toy_events):
		39	+ w = match_release_dates(toy_wide, toy_events)
		40	+ q = finalize_labels(build_quarter_table(w))
		41	+ r = q.filter((pl.col("entity_id") == "frd:AAA") & (pl.col("qidx") == 9)).row(0, named=True)
		42	+ x = toy_wide.filter(pl.col("entity_id") == "frd:AAA").sort("qidx")["op_inc"].to_list()
		43	+ drift = sum(x[9 - 1 - k] - x[9 - 5 - k] for k in range(4)) / 4
		44	+ assert abs(r["E_op_inc"] - (x[5] + drift)) < 1e-9
		45	+ assert q.filter(pl.col("qidx") < 8)["E_op_inc"].null_count() == q.filter(pl.col("qidx") < 8).height

added tests/test_reproducibility.py +23 −0

				@@ -0,0 +1,23 @@
		1	+import numpy as np
		2	+
		3	+from wp39.inference import holm, paired_block_bootstrap, dm_gw_test
		4	+
		5	+
		6	+def test_bootstrap_is_seeded_and_centered():
		7	+ rng = np.random.default_rng(0)
		8	+ d = rng.normal(0.01, 0.1, 2000)
		9	+ blocks = np.repeat(np.arange(20), 100)
		10	+ a = paired_block_bootstrap(d, blocks, n_boot=300, seed=1)
		11	+ b = paired_block_bootstrap(d, blocks, n_boot=300, seed=1)
		12	+ assert a == b and a["ci_lo"] < a["mean"] < a["ci_hi"]
		13	+
		14	+
		15	+def test_gw_test_rejects_clear_difference():
		16	+ d = np.full(2000, 0.05) + np.random.default_rng(1).normal(0, 0.02, 2000)
		17	+ blocks = np.repeat(np.arange(20), 100)
		18	+ assert dm_gw_test(d, blocks)["p"] < 0.001
		19	+
		20	+
		21	+def test_holm_monotone():
		22	+ adj = holm({"a": 0.01, "b": 0.04, "c": 0.5})
		23	+ assert adj["a"] <= adj["b"] <= adj["c"] and adj["a"] == 0.03

added tests/test_splits.py +37 −0

				@@ -0,0 +1,37 @@
		1	+import datetime as dt
		2	+
		3	+import polars as pl
		4	+
		5	+from wp39.splits import chronological, firm_holdout_mask, overlap_check, walk_forward
		6	+
		7	+CFG = {"train": ["2012-01-01", "2018-12-31"], "validation": ["2019-01-01", "2020-12-31"], "test": ["2021-01-01", "2026-10-30"],
		8	+ "walk_forward": {"first_test_year": 2016, "embargo_quarters": 1}, "firm_holdout_fraction": 0.2}
		9	+
		10	+
		11	+def _panel():
		12	+ rows = []
		13	+ d = dt.date(2012, 1, 15)
		14	+ for i in range(140):
		15	+ c = d + dt.timedelta(days=int(i * 36.5))
		16	+ rows.append({"entity_id": f"frd:E{i % 7}", "fiscal_period_end": c - dt.timedelta(days=30), "h": 20, "cutoff": c,
		17	+ "A_q": c + dt.timedelta(days=28), "y": i % 3 == 0})
		18	+ return pl.DataFrame(rows)
		19	+
		20	+
		21	+def test_chronological_purge_and_embargo():
		22	+ s = chronological(_panel(), CFG)
		23	+ assert s["train"]["A_q"].max() < dt.date(2018, 10, 1) # validation starts 2019-01-01, embargo one quarter
		24	+ assert s["validation"]["A_q"].max() < dt.date(2020, 10, 1)
		25	+ assert overlap_check(s["train"], s["test"])["shared_observations"] == 0
		26	+
		27	+
		28	+def test_walk_forward_folds():
		29	+ folds = walk_forward(_panel(), CFG, 2025)
		30	+ for y, tr, te in folds:
		31	+ assert tr["A_q"].max() < dt.date(y - 1, 10, 1)
		32	+ assert te["cutoff"].min().year == y
		33	+
		34	+
		35	+def test_firm_holdout_is_deterministic():
		36	+ e = pl.Series(["frd:A", "frd:B", "frd:C"])
		37	+ assert firm_holdout_mask(e).to_list() == firm_holdout_mask(e).to_list()

added tests/test_timestamp_dsts.py +30 −0

				@@ -0,0 +1,30 @@
		1	+import datetime as dt
		2	+
		3	+import polars as pl
		4	+
		5	+from wp39.calendar import close_time_utc, regular_session_bounds_utc, shift_trading_days, trading_days_between
		6	+from wp39.features import session_bounds_table
		7	+
		8	+
		9	+def test_regular_session_bounds_in_both_dst_regimes():
		10	+ o, c = regular_session_bounds_utc(dt.date(2024, 1, 10)) # EST: 09:30 ET = 14:30 UTC
		11	+ assert (o.hour, o.minute) == (14, 30) and (c.hour, c.minute) == (21, 0)
		12	+ o, c = regular_session_bounds_utc(dt.date(2024, 7, 10)) # EDT: 09:30 ET = 13:30 UTC
		13	+ assert (o.hour, o.minute) == (13, 30) and (c.hour, c.minute) == (20, 0)
		14	+
		15	+
		16	+def test_early_close_day():
		17	+ b = session_bounds_table(dt.date(2023, 11, 20), dt.date(2023, 11, 30))
		18	+ r = b.filter(pl.col("session_date") == dt.date(2023, 11, 24)).row(0, named=True)
		19	+ assert r["close_minute"] == 210 # 13:00 ET close the day after Thanksgiving
		20	+
		21	+
		22	+def test_shift_trading_days_skips_weekends_and_holidays():
		23	+ assert shift_trading_days(dt.date(2024, 7, 5), -1) == dt.date(2024, 7, 3) # July 4 holiday
		24	+ assert shift_trading_days(dt.date(2024, 7, 8), -1) == dt.date(2024, 7, 5)
		25	+ assert trading_days_between(dt.date(2024, 7, 3), dt.date(2024, 7, 8)) == 2
		26	+
		27	+
		28	+def test_close_print_is_at_1600_et():
		29	+ c = close_time_utc(dt.date(2022, 3, 14)) # first session after the spring-forward
		30	+ assert (c.hour, c.minute) == (20, 0)

added tests/test_train_only_transforms.py +15 −0

				@@ -0,0 +1,15 @@
		1	+import numpy as np
		2	+import polars as pl
		3	+
		4	+from wp39.models import Preprocessor
		5	+
		6	+
		7	+def test_preprocessor_statistics_come_from_train_only():
		8	+ tr = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0, 100.0], "z": [0.0, 0.0, 1.0, 1.0, 1.0]})
		9	+ te = pl.DataFrame({"x": [1000.0, None, -5.0], "z": [5.0, 5.0, 5.0]})
		10	+ pp = Preprocessor(["x", "z"]).fit(tr)
		11	+ Xte = pp.transform(te)
		12	+ assert pp.fitted_on_rows == 5
		13	+ assert Xte[0, 0] <= (pp.hi["x"] - pp.mu["x"]) / pp.sd["x"] + 1e-9 # clipped at the train 99th percentile
		14	+ assert abs(Xte[1, 0] - (pp.med["x"] - pp.mu["x"]) / pp.sd["x"]) < 1e-9 # imputed with the train median
		15	+ assert np.all(np.isfinite(Xte))
		16