fundamentals: ratios (45 formules documentées), service point-in-time, screener, ingestion et routes /v1/fundamentals

- ratios.py : formules avec docstring, null + raison, render_docs()
- service.py : statements/facts/ratios/ratios daily (ASOF join DuckDB)/filings/coverage/frames, as_of partout
- screener.py : grammaire de filtres (< > <= >= = != plages, suffixes k/m/b/t/%) sur fund_latest précalculée
- ingest.py : univers CIK↔tickers ∩ lac de prix (classes d'actions, historique, radiations), lac de faits Parquet, remplacement des états, couverture, fund_latest, événements Redis, poll Atom + index quotidien
- routes.py : 8 endpoints + _mapping + _health, formats json/csv/parquet, curseurs, OpenAPI avec exemples Apple réels

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 20 days ago (Sep 5, 2026) parent 20f98bb

6 changed files +1,999 −0

added hfmarketdata/api/fundamentals/ingest.py +540 −0

		`@@ -0,0 +1,540 @@`
	1	+"""Ingestion: EDGAR → facts lake (Parquet) → standardized statements (SQLite) → coverage → screener table → events.
	2	+
	3	+Entry points (used by scripts/edgar_backfill.py, scripts/edgar_incremental.py, tests):
	4	+
	5	+* `sync_universe(client)` CIK↔ticker mapping restricted to the price lake (share classes, history, delistings)
	6	+* `ingest_company(client, cik)` companyfacts + submissions → lake + statements + coverage + fund_latest
	7	+* `refresh_latest(cik)` recompute the screener row of one company (TTM + ratios × last close)
	8	+* `build_latest_all()` rebuild `fund_latest` for every company
	9	+* `poll_new_filings(client)` Atom feed + daily index → re-ingest affected CIKs, publish `filings` events
	10	+* `publish_filing_event(...)` Redis pub/sub + stream buffer consumed by the WebSocket
	11	+
	12	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	13	+"""
	14	+from __future__ import annotations
	15	+
	16	+import logging
	17	+import math
	18	+import time
	19	+from dataclasses import dataclass, field
	20	+from datetime import date, datetime, timedelta
	21	+from pathlib import Path
	22	+from typing import Any
	23	+
	24	+import pandas as pd
	25	+from sqlalchemy import delete, insert, select
	26	+
	27	+from core.config import settings
	28	+from core.db import session
	29	+
	30	+from . import mapping as M
	31	+from . import utcnow
	32	+from . import normalize as N
	33	+from . import prices
	34	+from .edgar_client import FINANCIAL_FORMS, TRACKED_FORMS, EdgarClient, parse_atom, primary_doc_url
	35	+from .models import (EdgarCompany, EdgarFiling, FundCoverage, FundIngestState, FundMappingLog, fund_latest,
	36	+ fund_statements, init_db)
	37	+from .ratios import RATIO_NAMES, compute_all
	38	+from .service import ratio_inputs
	39	+
	40	+log = logging.getLogger("hfmarketdata.fundamentals.ingest")
	41	+
	42	+EVENT_FORMS = ("10-K", "10-Q", "8-K", "20-F", "10-K/A", "10-Q/A", "20-F/A", "40-F", "6-K")
	43	+
	44	+
	45	+def facts_dir(cik: int) -> Path:
	46	+ return settings.data_root / "edgar" / "facts" / f"cik={int(cik)}"
	47	+
	48	+
	49	+# ------------------------------------------------------------------------------------------- universe
	50	+@dataclass
	51	+class UniverseStats:
	52	+ lake_tickers: int = 0
	53	+ sec_tickers: int = 0
	54	+ companies: int = 0
	55	+ added: int = 0
	56	+ updated: int = 0
	57	+ delisted: int = 0
	58	+ unmatched: list[str] = field(default_factory=list)
	59	+
	60	+
	61	+def sync_universe(client: EdgarClient, *, tickers_filter: set[str] \| None = None) -> UniverseStats:
	62	+ """Build/refresh `edgar_companies` from company_tickers(.json\|_exchange.json) ∩ price lake.
	63	+
	64	+ * one CIK, several tickers (GOOGL/GOOG, BRK-A/BRK-B): one row, `tickers` lists every class, `ticker` = the
	65	+ SEC's first listing (used as the canonical key);
	66	+ * a ticker that moves to another CIK or disappears from the SEC list is appended to `ticker_history` and
	67	+ the company is marked `delisted` when none of its tickers is listed anymore (data keeps being served)."""
	68	+ init_db()
	69	+ st = UniverseStats()
	70	+ lake = tickers_filter if tickers_filter is not None else prices.lake_tickers()
	71	+ st.lake_tickers = len(lake)
	72	+ raw = client.company_tickers() or {}
	73	+ exch = client.company_tickers_exchange() or {}
	74	+ exchange_by_cik: dict[int, str] = {}
	75	+ if exch.get("data"):
	76	+ fields = exch.get("fields", [])
	77	+ i_cik, i_ex = fields.index("cik"), fields.index("exchange")
	78	+ for row in exch["data"]:
	79	+ if row[i_ex]:
	80	+ exchange_by_cik.setdefault(int(row[i_cik]), row[i_ex])
	81	+ by_cik: dict[int, dict[str, Any]] = {}
	82	+ for v in raw.values():
	83	+ tk = str(v["ticker"]).upper().replace(".", "-")
	84	+ cik = int(v["cik_str"])
	85	+ st.sec_tickers += 1
	86	+ if tk not in lake and tk.replace("-", ".") not in lake:
	87	+ continue
	88	+ e = by_cik.setdefault(cik, {"tickers": [], "name": v.get("title", "")})
	89	+ if tk not in e["tickers"]:
	90	+ e["tickers"].append(tk)
	91	+ today = date.today().isoformat()
	92	+ with session() as s:
	93	+ existing = {c.cik: c for c in s.scalars(select(EdgarCompany))}
	94	+ listed_tickers = {t for e in by_cik.values() for t in e["tickers"]}
	95	+ for cik, e in by_cik.items():
	96	+ c = existing.get(cik)
	97	+ if c is None:
	98	+ c = EdgarCompany(cik=cik, ticker=e["tickers"][0], tickers=e["tickers"], name=e["name"],
	99	+ exchange=exchange_by_cik.get(cik), status="active", ticker_history=[])
	100	+ s.add(c)
	101	+ st.added += 1
	102	+ else:
	103	+ hist = list(c.ticker_history or [])
	104	+ for old in (c.tickers or []):
	105	+ if old not in e["tickers"]:
	106	+ hist.append({"ticker": old, "from": None, "to": today, "note": "no longer listed for this CIK"})
	107	+ for new in e["tickers"]:
	108	+ if new not in (c.tickers or []) and c.tickers:
	109	+ hist.append({"ticker": new, "from": today, "to": None, "note": "new listing"})
	110	+ changed = (c.tickers != e["tickers"]) or (c.name != e["name"]) or (c.status != "active")
	111	+ c.tickers, c.name, c.status = e["tickers"], e["name"], "active"
	112	+ c.ticker = e["tickers"][0] if c.ticker not in e["tickers"] else c.ticker
	113	+ c.exchange = exchange_by_cik.get(cik, c.exchange)
	114	+ c.ticker_history = hist
	115	+ st.updated += int(changed)
	116	+ st.companies += 1
	117	+ for cik, c in existing.items():
	118	+ if cik not in by_cik and c.status == "active":
	119	+ c.status = "delisted"
	120	+ c.ticker_history = list(c.ticker_history or []) + [{"ticker": c.ticker, "from": None, "to": today,
	121	+ "note": "removed from SEC ticker list"}]
	122	+ st.delisted += 1
	123	+ st.unmatched = sorted(lake - listed_tickers)[:50]
	124	+ return st
	125	+
	126	+
	127	+def tracked_ciks() -> list[int]:
	128	+ with session() as s:
	129	+ return [c for c in s.scalars(select(EdgarCompany.cik).order_by(EdgarCompany.cik))]
	130	+
	131	+
	132	+# ---------------------------------------------------------------------------------------- per company
	133	+@dataclass
	134	+class IngestResult:
	135	+ cik: int
	136	+ ticker: str
	137	+ facts: int = 0
	138	+ filings: int = 0
	139	+ rows: int = 0
	140	+ derived_rows: int = 0
	141	+ restated_rows: int = 0
	142	+ extensions: int = 0
	143	+ unmapped_tags: int = 0
	144	+ completeness: float \| None = None
	145	+ seconds: float = 0.0
	146	+ error: str \| None = None
	147	+ new_filings: list[str] = field(default_factory=list)
	148	+
	149	+
	150	+def ingest_company(client: EdgarClient \| None, cik: int, *, refresh: bool = False, with_metalinks: bool = True,
	151	+ companyfacts: dict \| None = None, submissions: dict \| None = None, publish: bool = False) -> IngestResult:
	152	+ """Fetch (or reuse cached / provided) companyfacts + submissions and rebuild everything for one company."""
	153	+ t0 = time.time()
	154	+ with session() as s:
	155	+ c = s.get(EdgarCompany, int(cik))
	156	+ ticker = c.ticker if c else str(cik)
	157	+ fye = c.fiscal_year_end if c else None
	158	+ res = IngestResult(cik=int(cik), ticker=ticker)
	159	+ try:
	160	+ cf = companyfacts if companyfacts is not None else client.companyfacts(cik, refresh=refresh)
	161	+ sub = submissions if submissions is not None else client.submissions(cik, refresh=refresh)
	162	+ if cf is None:
	163	+ res.error = "companyfacts_404"
	164	+ _mark_company(cik, sub, None)
	165	+ return res
	166	+ facts = N.facts_frame(cf)
	167	+ res.facts = len(facts)
	168	+ _write_facts(cik, facts)
	169	+ filings = N.filings_from_submissions(sub)
	170	+ known_before = _known_accns(cik)
	171	+ _upsert_filings(cik, filings, facts)
	172	+ fye = (sub or {}).get("fiscalYearEnd") or fye
	173	+ metalinks = None
	174	+ if with_metalinks and client is not None:
	175	+ latest_10k = next((f for f in sorted(filings.values(), key=lambda f: f.filed, reverse=True) if f.form in ("10-K", "20-F", "40-F")), None)
	176	+ if latest_10k is not None:
	177	+ try:
	178	+ metalinks = client.metalinks(cik, latest_10k.accn)
	179	+ except Exception as e: # pragma: no cover
	180	+ log.warning("metalinks %s/%s: %s", cik, latest_10k.accn, e)
	181	+ norm = N.normalize_company(int(cik), ticker, facts, filings, fye, metalinks)
	182	+ _replace_statements(cik, norm.rows)
	183	+ _upsert_mapping_log(cik, norm.mapping_log)
	184	+ cov = compute_coverage(cik, ticker, norm.rows, len(filings), norm.stats)
	185	+ _mark_company(cik, sub, fye)
	186	+ refresh_latest(cik)
	187	+ res.filings, res.rows = len(filings), len(norm.rows)
	188	+ res.derived_rows, res.restated_rows = norm.stats["derived_rows"], norm.stats["restated_rows"]
	189	+ res.extensions, res.unmapped_tags = norm.stats["extensions"], norm.stats["unmapped_tags"]
	190	+ res.completeness = cov.get("completeness")
	191	+ res.new_filings = sorted(set(a for a, f in filings.items() if f.form in EVENT_FORMS) - known_before)
	192	+ if publish and res.new_filings:
	193	+ for accn in res.new_filings:
	194	+ f = filings[accn]
	195	+ publish_filing_event(cik, ticker, f.form, f.filed, f.report_date, accn, f.primary_doc)
	196	+ except Exception as e:
	197	+ log.exception("ingest %s failed", cik)
	198	+ res.error = f"{type(e).__name__}: {e}"
	199	+ res.seconds = time.time() - t0
	200	+ return res
	201	+
	202	+
	203	+def _write_facts(cik: int, facts: pd.DataFrame) -> None:
	204	+ d = facts_dir(cik)
	205	+ d.mkdir(parents=True, exist_ok=True)
	206	+ df = facts.copy()
	207	+ for c in ("start", "end", "filed"):
	208	+ df[c] = pd.to_datetime(df[c])
	209	+ df["fy"] = df["fy"].astype("Int64")
	210	+ tmp = d / "facts.parquet.tmp"
	211	+ df.to_parquet(tmp, index=False, compression="zstd")
	212	+ tmp.replace(d / "facts.parquet")
	213	+
	214	+
	215	+def _known_accns(cik: int) -> set[str]:
	216	+ with session() as s:
	217	+ return set(s.scalars(select(EdgarFiling.accn).where(EdgarFiling.cik == int(cik))))
	218	+
	219	+
	220	+def _upsert_filings(cik: int, filings: dict[str, N.Filing], facts: pd.DataFrame) -> None:
	221	+ xbrl_accns = set(facts["accn"].unique()) if not facts.empty else set()
	222	+ now = utcnow()
	223	+ with session() as s:
	224	+ existing = {f.accn: f for f in s.scalars(select(EdgarFiling).where(EdgarFiling.cik == int(cik)))}
	225	+ for accn, f in filings.items():
	226	+ if f.form not in TRACKED_FORMS and accn not in xbrl_accns:
	227	+ continue
	228	+ row = existing.get(accn)
	229	+ if row is None:
	230	+ s.add(EdgarFiling(accn=accn, cik=int(cik), form=f.form, filed_date=f.filed, period_of_report=f.report_date,
	231	+ primary_doc=primary_doc_url(int(cik), accn, f.primary_doc), is_amendment=f.is_amendment,
	232	+ is_xbrl=f.is_xbrl or accn in xbrl_accns, parsed_at=now if accn in xbrl_accns else None))
	233	+ else:
	234	+ row.is_xbrl = row.is_xbrl or accn in xbrl_accns
	235	+ row.period_of_report = row.period_of_report or f.report_date
	236	+ if accn in xbrl_accns:
	237	+ row.parsed_at = now
	238	+
	239	+
	240	+def _replace_statements(cik: int, rows: list[dict[str, Any]]) -> None:
	241	+ cols = [c for c in fund_statements.c.keys() if c != "id"]
	242	+ clean = []
	243	+ for r in rows:
	244	+ rec = {k: r.get(k) for k in cols} # executemany needs identical keys on every row
	245	+ for k, v in rec.items():
	246	+ if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
	247	+ rec[k] = None
	248	+ clean.append(rec)
	249	+ with session() as s:
	250	+ s.execute(delete(fund_statements).where(fund_statements.c.cik == int(cik)))
	251	+ for i in range(0, len(clean), 500):
	252	+ s.execute(insert(fund_statements), clean[i:i + 500])
	253	+
	254	+
	255	+def _upsert_mapping_log(cik: int, entries: list[dict[str, Any]]) -> None:
	256	+ with session() as s:
	257	+ existing = {(m.taxonomy, m.tag): m for m in s.scalars(select(FundMappingLog).where(FundMappingLog.cik == int(cik)))}
	258	+ for e in entries:
	259	+ m = existing.get((e["taxonomy"], e["tag"]))
	260	+ if m is None:
	261	+ s.add(FundMappingLog(**e))
	262	+ else:
	263	+ m.occurrences = e["occurrences"]
	264	+ m.last_seen = e["last_seen"] or m.last_seen
	265	+ m.sample_accn = e["sample_accn"] or m.sample_accn
	266	+ m.hint_account = e["hint_account"] or m.hint_account
	267	+
	268	+
	269	+def _mark_company(cik: int, sub: dict \| None, fye: str \| None) -> None:
	270	+ with session() as s:
	271	+ c = s.get(EdgarCompany, int(cik))
	272	+ if c is None:
	273	+ return
	274	+ now = utcnow()
	275	+ c.facts_updated_at = now
	276	+ c.normalized_at = now
	277	+ if sub:
	278	+ c.sic = str(sub.get("sic") or c.sic or "") or None
	279	+ c.sic_description = sub.get("sicDescription") or c.sic_description
	280	+ c.fiscal_year_end = fye or c.fiscal_year_end
	281	+ c.state_of_incorporation = sub.get("stateOfIncorporation") or c.state_of_incorporation
	282	+ if not c.exchange and sub.get("exchanges"):
	283	+ c.exchange = sub["exchanges"][0]
	284	+ if not c.name:
	285	+ c.name = sub.get("name", "")
	286	+
	287	+
	288	+# ------------------------------------------------------------------------------------------- coverage
	289	+def compute_coverage(cik: int, ticker: str, rows: list[dict[str, Any]], n_filings: int, stats: dict[str, Any]) -> dict[str, Any]:
	290	+ latest = N.select_as_of(rows, None)
	291	+ quarters = {(r["fiscal_year"], r["fiscal_quarter"]) for r in latest if r["fiscal_quarter"] in (1, 2, 3, 4)}
	292	+ annuals = {r["fiscal_year"] for r in latest if r["fiscal_quarter"] == 0}
	293	+ public = [a for a in M.ACCOUNTS if not a.auxiliary]
	294	+ per_stmt: dict[str, list[float]] = {st: [] for st in M.STATEMENTS}
	295	+ missing: dict[str, dict[str, Any]] = {}
	296	+ for r in latest:
	297	+ accs = [a for a in public if a.statement == r["statement"]]
	298	+ if not accs:
	299	+ continue
	300	+ non_null = sum(1 for a in accs if r.get(a.name) is not None)
	301	+ per_stmt[r["statement"]].append(non_null / len(accs))
	302	+ for a in accs:
	303	+ if r.get(a.name) is None:
	304	+ m = missing.setdefault(a.name, {"reason": (r.get("coverage") or {}).get(a.name, {}).get("reason", "no_mapped_tag"),
	305	+ "periods": 0})
	306	+ m["periods"] += 1
	307	+ total_cells = sum(len(v) for v in per_stmt.values())
	308	+ completeness = (100.0 * sum(sum(v) for v in per_stmt.values()) / total_cells) if total_cells else None
	309	+ by_stmt = {st: (round(100.0 * sum(v) / len(v), 1) if v else None) for st, v in per_stmt.items()}
	310	+ # gaps: fiscal quarters missing between first and last quarter
	311	+ gaps: list[str] = []
	312	+ if quarters:
	313	+ fy0, q0 = min(quarters)
	314	+ fy1, q1 = max(quarters)
	315	+ fy, q = fy0, q0
	316	+ while (fy, q) <= (fy1, q1):
	317	+ if (fy, q) not in quarters:
	318	+ gaps.append(f"{fy}Q{q}")
	319	+ fy, q = (fy, q + 1) if q < 4 else (fy + 1, 1)
	320	+ ends = [N._d(r["period_end"]) for r in latest]
	321	+ cov = {"ticker": ticker, "cik": int(cik), "first_period_end": min(ends) if ends else None,
	322	+ "last_period_end": max(ends) if ends else None, "quarters": len(quarters), "annuals": len(annuals),
	323	+ "filings": n_filings, "completeness": round(completeness, 1) if completeness is not None else None,
	324	+ "completeness_by_statement": by_stmt,
	325	+ "missing_accounts": dict(sorted(missing.items(), key=lambda kv: -kv[1]["periods"])),
	326	+ "gaps": gaps[:100], "derived_quarters": stats.get("derived_rows", 0), "restated_periods": stats.get("restated_rows", 0),
	327	+ "extensions_logged": stats.get("extensions", 0),
	328	+ "last_filed_date": max((N._d(r["filed_date"]) for r in rows), default=None)}
	329	+ with session() as s:
	330	+ row = s.get(FundCoverage, ticker)
	331	+ if row is None:
	332	+ s.add(FundCoverage(**cov))
	333	+ else:
	334	+ for k, v in cov.items():
	335	+ setattr(row, k, v)
	336	+ return cov
	337	+
	338	+
	339	+# ------------------------------------------------------------------------------------ screener table
	340	+def refresh_latest(cik: int) -> dict[str, Any] \| None:
	341	+ """Recompute the `fund_latest` row (TTM + ratios × last close) of one company."""
	342	+ with session() as s:
	343	+ c = s.get(EdgarCompany, int(cik))
	344	+ if c is None:
	345	+ return None
	346	+ rows = [dict(r._mapping) for r in s.execute(select(fund_statements).where(fund_statements.c.cik == int(cik)))]
	347	+ latest = N.select_as_of(rows, None)
	348	+ if not latest:
	349	+ return None
	350	+ px = prices.close_at(c.ticker, None)
	351	+ f, info = ratio_inputs(latest, price=px[0] if px else None)
	352	+ values, reasons = compute_all(f)
	353	+ rec: dict[str, Any] = {"ticker": c.ticker, "cik": c.cik, "name": c.name, "sic": c.sic, "exchange": c.exchange,
	354	+ "period_end": N._d(info.get("balance_period_end") or info.get("period_end")),
	355	+ "fiscal_year": info.get("fiscal_year"), "fiscal_quarter": info.get("fiscal_quarter"),
	356	+ "filed_date": N._d(info.get("filed_date")), "price": px[0] if px else None,
	357	+ "price_date": px[1] if px else None, "shares_source": info.get("shares_source"),
	358	+ "reasons": reasons, "updated_at": utcnow()}
	359	+ for a in M.PUBLIC_ACCOUNTS:
	360	+ rec[a] = _clean(f.get(a))
	361	+ for r in RATIO_NAMES:
	362	+ if r not in rec:
	363	+ rec[r] = _clean(values.get(r))
	364	+ with session() as s:
	365	+ s.execute(delete(fund_latest).where(fund_latest.c.cik == int(cik)))
	366	+ s.execute(insert(fund_latest).values(**rec))
	367	+ return rec
	368	+
	369	+
	370	+def _clean(v: Any) -> float \| None:
	371	+ if v is None:
	372	+ return None
	373	+ try:
	374	+ x = float(v)
	375	+ except (TypeError, ValueError):
	376	+ return None
	377	+ return None if math.isnan(x) or math.isinf(x) else x
	378	+
	379	+
	380	+def build_latest_all() -> int:
	381	+ n = 0
	382	+ for cik in tracked_ciks():
	383	+ if refresh_latest(cik):
	384	+ n += 1
	385	+ return n
	386	+
	387	+
	388	+# ------------------------------------------------------------------------------------------- events
	389	+def publish_filing_event(cik: int, ticker: str, form: str, filed: date, period: date \| None, accn: str,
	390	+ primary_doc: str \| None) -> dict[str, Any] \| None:
	391	+ """Build the `filing` event (with a statement summary when the filing carries statements) and push it
	392	+ to Redis (`filings` pub/sub channel + `filings:stream` buffer). Returns the event or None (Redis down)."""
	393	+ summary: dict[str, Any] \| None = None
	394	+ if form in FINANCIAL_FORMS:
	395	+ with session() as s:
	396	+ rows = [dict(r._mapping) for r in s.execute(select(fund_statements).where(fund_statements.c.cik == int(cik),
	397	+ fund_statements.c.accn == accn))]
	398	+ inc = [r for r in rows if r["statement"] == M.INCOME and (r["fiscal_quarter"] != 0 or form.startswith(("10-K", "20-F", "40-F")))]
	399	+ if inc:
	400	+ cur = max(inc, key=lambda r: N._d(r["period_end"]))
	401	+ with session() as s:
	402	+ allrows = [dict(r._mapping) for r in s.execute(select(fund_statements).where(fund_statements.c.cik == int(cik)))]
	403	+ latest = N.select_as_of(allrows, N._d(filed))
	404	+ bal = [r for r in latest if r["statement"] == M.BALANCE and r["fiscal_year"] == cur["fiscal_year"] and r["fiscal_quarter"] == cur["fiscal_quarter"]]
	405	+ cfr = [r for r in latest if r["statement"] == M.CASHFLOW and r["fiscal_year"] == cur["fiscal_year"] and r["fiscal_quarter"] == cur["fiscal_quarter"]]
	406	+ prev = [r for r in latest if r["statement"] == M.INCOME and r["fiscal_year"] == cur["fiscal_year"] - 1 and r["fiscal_quarter"] == cur["fiscal_quarter"]]
	407	+ yoy = {}
	408	+ if prev:
	409	+ for k in ("revenue", "net_income", "eps_diluted"):
	410	+ a, b = cur.get(k), prev[0].get(k)
	411	+ yoy[k] = (a / b - 1) if a is not None and b not in (None, 0) and b > 0 else None
	412	+ summary = {"revenue": cur.get("revenue"), "net_income": cur.get("net_income"), "eps_diluted": cur.get("eps_diluted"),
	413	+ "total_assets": bal[0].get("total_assets") if bal else None,
	414	+ "operating_cash_flow": cfr[0].get("operating_cash_flow") if cfr else None,
	415	+ "fiscal_year": cur["fiscal_year"], "fiscal_quarter": cur["fiscal_quarter"], "yoy": yoy}
	416	+ event = {"type": "filing", "ticker": ticker, "cik": int(cik), "form": form, "period": period.isoformat() if period else None,
	417	+ "filed_date": filed.isoformat() if filed else None, "url": primary_doc_url(int(cik), accn, primary_doc), "accn": accn,
	418	+ "summary": summary}
	419	+ try:
	420	+ from stream.broker import publish
	421	+ seq = publish(event)
	422	+ event["seq"] = seq
	423	+ _bump_state("incremental", events_published=1)
	424	+ return event
	425	+ except Exception as e: # Redis unavailable: the API keeps serving, the stream just misses the event
	426	+ log.warning("publish filing event failed: %s", e)
	427	+ return None
	428	+
	429	+
	430	+# ------------------------------------------------------------------------------------ incremental poll
	431	+def poll_new_filings(client: EdgarClient, *, since: datetime \| None = None, forms: tuple[str, ...] = ("10-K", "10-Q", "8-K", "20-F"),
	432	+ include_daily_index: bool = True) -> dict[str, Any]:
	433	+ """One polling cycle: Atom `getcurrent` per form (+ yesterday/today master index as a safety net) →
	434	+ filter tracked CIKs → re-ingest them (fresh companyfacts) → publish events. Returns a summary."""
	435	+ tracked = set(tracked_ciks())
	436	+ seen: dict[str, dict[str, Any]] = {}
	437	+ for form in forms:
	438	+ try:
	439	+ for e in parse_atom(client.atom_current(form)):
	440	+ if e["cik"] in tracked and e["accn"]:
	441	+ seen[e["accn"]] = e
	442	+ except Exception as ex: # pragma: no cover
	443	+ log.warning("atom %s: %s", form, ex)
	444	+ if include_daily_index:
	445	+ for back in (0, 1):
	446	+ d = date.today() - timedelta(days=back)
	447	+ try:
	448	+ for e in parse_master_index(client.get_text(master_index_url(d))):
	449	+ if e["cik"] in tracked and e["form"] in TRACKED_FORMS and e["accn"] not in seen:
	450	+ seen[e["accn"]] = e
	451	+ except Exception:
	452	+ continue
	453	+ known = _all_known_accns()
	454	+ new = {a: e for a, e in seen.items() if a not in known}
	455	+ affected = sorted({e["cik"] for e in new.values()})
	456	+ results = []
	457	+ for cik in affected:
	458	+ results.append(ingest_company(client, cik, refresh=True, with_metalinks=False, publish=True))
	459	+ now = utcnow()
	460	+ newest = max((datetime.fromisoformat(e["filed"]) for e in seen.values() if e.get("filed")), default=None)
	461	+ lag = (now - newest).total_seconds() if newest else None
	462	+ _set_state("incremental", last_run_at=now, last_success_at=now, last_rss_check_at=now, lag_seconds=lag,
	463	+ requests_made=client.stats.requests, last_filing_seen=newest,
	464	+ failures_add=sum(1 for r in results if r.error), failure_samples=[r.error for r in results if r.error][:5])
	465	+ _update_mapping_failure_rate()
	466	+ return {"seen": len(seen), "new": len(new), "affected_ciks": affected,
	467	+ "events": sum(len(r.new_filings) for r in results), "errors": [r.error for r in results if r.error]}
	468	+
	469	+
	470	+def master_index_url(d: date) -> str:
	471	+ q = (d.month - 1) // 3 + 1
	472	+ return f"https://www.sec.gov/Archives/edgar/daily-index/{d.year}/QTR{q}/master.{d:%Y%m%d}.idx"
	473	+
	474	+
	475	+def parse_master_index(text: str) -> list[dict[str, Any]]:
	476	+ """`CIK\|Company Name\|Form Type\|Date Filed\|File Name` rows → [{cik, name, form, filed, accn}]."""
	477	+ out = []
	478	+ for line in text.splitlines():
	479	+ parts = line.split("\|")
	480	+ if len(parts) != 5 or not parts[0].strip().isdigit():
	481	+ continue
	482	+ cik, name, form, filed, fname = parts
	483	+ accn = fname.rsplit("/", 1)[-1].replace(".txt", "")
	484	+ try:
	485	+ fd = datetime.strptime(filed.strip(), "%Y%m%d").date().isoformat()
	486	+ except ValueError:
	487	+ fd = None
	488	+ out.append({"cik": int(cik), "name": name.strip(), "form": form.strip(), "filed": fd, "accn": accn})
	489	+ return out
	490	+
	491	+
	492	+def _all_known_accns() -> set[str]:
	493	+ with session() as s:
	494	+ return set(s.scalars(select(EdgarFiling.accn)))
	495	+
	496	+
	497	+# ------------------------------------------------------------------------------------------- state
	498	+def _set_state(key: str, , failures_add: int = 0, *fields: Any) -> None:
	499	+ with session() as s:
	500	+ st = s.get(FundIngestState, key)
	501	+ if st is None:
	502	+ st = FundIngestState(key=key)
	503	+ s.add(st)
	504	+ for k, v in fields.items():
	505	+ setattr(st, k, v)
	506	+ st.failures = (st.failures or 0) + failures_add
	507	+
	508	+
	509	+def _bump_state(key: str, **counters: int) -> None:
	510	+ with session() as s:
	511	+ st = s.get(FundIngestState, key)
	512	+ if st is None:
	513	+ st = FundIngestState(key=key)
	514	+ s.add(st)
	515	+ for k, v in counters.items():
	516	+ setattr(st, k, (getattr(st, k) or 0) + v)
	517	+
	518	+
	519	+def _update_mapping_failure_rate() -> None:
	520	+ """Share of public accounts that are null across the latest versions (screener table)."""
	521	+ with session() as s:
	522	+ rows = [dict(r._mapping) for r in s.execute(select(*[fund_latest.c[a] for a in M.PUBLIC_ACCOUNTS]))]
	523	+ if not rows:
	524	+ return
	525	+ total = len(rows) * len(M.PUBLIC_ACCOUNTS)
	526	+ nulls = sum(1 for r in rows for a in M.PUBLIC_ACCOUNTS if r.get(a) is None)
	527	+ rate = nulls / total if total else None
	528	+ _set_state("incremental", mapping_failure_rate=rate)
	529	+ _set_state("backfill", mapping_failure_rate=rate)
	530	+
	531	+
	532	+def state_snapshot() -> dict[str, Any]:
	533	+ with session() as s:
	534	+ return {st.key: {"last_run_at": st.last_run_at, "lag_seconds": st.lag_seconds, "failures": st.failures,
	535	+ "companies_done": st.companies_done, "companies_total": st.companies_total}
	536	+ for st in s.scalars(select(FundIngestState))}
	537	+
	538	+
	539	+__all__ = ["sync_universe", "ingest_company", "refresh_latest", "build_latest_all", "poll_new_filings",
	540	+ "publish_filing_event", "compute_coverage", "tracked_ciks", "parse_master_index", "master_index_url"]

added hfmarketdata/api/fundamentals/prices.py +75 −0

		`@@ -0,0 +1,75 @@`
	1	+"""Daily close prices from the Parquet lake, used to join valuation ratios with fundamentals.
	2	+
	3	+Preference order for the adjustment folder: UNADJUSTED (raw prints match the reported share counts),
	4	+then adj_split, then adj_splitdiv. The chosen source is reported in `meta.price_source`.
	5	+
	6	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	7	+"""
	8	+from __future__ import annotations
	9	+
	10	+from datetime import date
	11	+from pathlib import Path
	12	+
	13	+import pandas as pd
	14	+
	15	+from core import duck
	16	+from core.config import settings
	17	+
	18	+ADJ_PREFERENCE = ("UNADJUSTED", "adj_split", "adj_splitdiv")
	19	+
	20	+
	21	+def price_file(ticker: str) -> tuple[Path, str] \| None:
	22	+ """(path, adjustment) of the daily bars of `ticker` in the stock or ETF lake, or None."""
	23	+ t = ticker.upper()
	24	+ for asset in ("stock", "etf"):
	25	+ for adj in ADJ_PREFERENCE:
	26	+ p = settings.parquet / asset / "1day" / adj / f"{t}_1day.parquet"
	27	+ if p.is_file():
	28	+ return p, f"{asset}/1day/{adj}"
	29	+ return None
	30	+
	31	+
	32	+def lake_tickers() -> set[str]:
	33	+ """Every ticker with daily bars in the stock or ETF lake (the fundamentals universe)."""
	34	+ out: set[str] = set()
	35	+ for asset in ("stock", "etf"):
	36	+ for adj in ADJ_PREFERENCE:
	37	+ d = settings.parquet / asset / "1day" / adj
	38	+ if d.is_dir():
	39	+ out \|= {p.stem.split("_")[0].upper() for p in d.iterdir() if p.suffix == ".parquet"}
	40	+ break
	41	+ return out
	42	+
	43	+
	44	+def close_at(ticker: str, on: date \| None = None) -> tuple[float, date] \| None:
	45	+ """Last close at or before `on` (default: latest bar)."""
	46	+ pf = price_file(ticker)
	47	+ if pf is None:
	48	+ return None
	49	+ con = duck.con()
	50	+ if on is None:
	51	+ row = con.execute("SELECT datetime, close FROM read_parquet(?) ORDER BY datetime DESC LIMIT 1", [str(pf[0])]).fetchone()
	52	+ else:
	53	+ row = con.execute("SELECT datetime, close FROM read_parquet(?) WHERE datetime <= ? ORDER BY datetime DESC LIMIT 1",
	54	+ [str(pf[0]), pd.Timestamp(on) + pd.Timedelta(hours=23, minutes=59)]).fetchone()
	55	+ if row is None or row[1] is None:
	56	+ return None
	57	+ return float(row[1]), pd.Timestamp(row[0]).date()
	58	+
	59	+
	60	+def closes(ticker: str, start: date \| None, end: date \| None) -> pd.DataFrame:
	61	+ """Daily closes in [start, end] as a DataFrame (date, close)."""
	62	+ pf = price_file(ticker)
	63	+ if pf is None:
	64	+ return pd.DataFrame(columns=["date", "close"])
	65	+ conds, params = [], [str(pf[0])]
	66	+ if start:
	67	+ conds.append("datetime >= ?")
	68	+ params.append(pd.Timestamp(start))
	69	+ if end:
	70	+ conds.append("datetime <= ?")
	71	+ params.append(pd.Timestamp(end) + pd.Timedelta(hours=23, minutes=59))
	72	+ where = ("WHERE " + " AND ".join(conds)) if conds else ""
	73	+ df = duck.con().execute(f"SELECT CAST(datetime AS DATE) AS date, close FROM read_parquet(?) {where} ORDER BY datetime",
	74	+ params).df()
	75	+ return df

added hfmarketdata/api/fundamentals/ratios.py +480 −0

		`@@ -0,0 +1,480 @@`
	1	+"""Financial ratio formulas — one function per ratio, docstring = the formula served in the docs.
	2	+
	3	+Inputs are a flat dict `f` built by `service.ratio_inputs()`:
	4	+
	5	+* flows (`revenue`, `net_income`, `operating_cash_flow`, …) are trailing twelve months (sum of the
	6	+ last four discrete quarters, or the fiscal year when `period=annual`);
	7	+* balances (`total_assets`, `total_equity`, `cash_and_equivalents`, …) are the latest balance sheet;
	8	+* `price` is the last close known at the valuation date, `shares_outstanding` the cover-page share
	9	+ count of the latest filing (falls back to weighted diluted shares, flagged);
	10	+* `_prev_year` / `_prev_quarter` / `_3y` / `_5y` / `*_10y` are the same aggregates shifted in time
	11	+ (used by growth ratios).
	12	+
	13	+Every formula returns `None` when an input is missing or a denominator is zero/negative where that
	14	+makes the ratio meaningless — nothing is ever invented. `compute_all()` also returns the reason for
	15	+each `None`. `render_docs()` produces docs/fundamentals-ratios.md from these docstrings.
	16	+
	17	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	18	+"""
	19	+from __future__ import annotations
	20	+
	21	+import inspect
	22	+import math
	23	+from collections.abc import Callable
	24	+from dataclasses import dataclass
	25	+from typing import Any
	26	+
	27	+Inputs = dict[str, Any]
	28	+
	29	+
	30	+@dataclass(frozen=True)
	31	+class Ratio:
	32	+ name: str
	33	+ group: str
	34	+ inputs: tuple[str, ...]
	35	+ fn: Callable[[Inputs], float \| None]
	36	+
	37	+ @property
	38	+ def doc(self) -> str:
	39	+ return inspect.getdoc(self.fn) or ""
	40	+
	41	+
	42	+RATIOS: list[Ratio] = []
	43	+GROUPS = ("valuation", "profitability", "liquidity", "solvency", "efficiency", "growth", "per_share")
	44	+
	45	+
	46	+def ratio(group: str, *inputs: str):
	47	+ def deco(fn):
	48	+ RATIOS.append(Ratio(fn.__name__, group, inputs, fn))
	49	+ return fn
	50	+ return deco
	51	+
	52	+
	53	+def _num(x: Any) -> float \| None:
	54	+ if x is None:
	55	+ return None
	56	+ try:
	57	+ v = float(x)
	58	+ except (TypeError, ValueError):
	59	+ return None
	60	+ return None if math.isnan(v) or math.isinf(v) else v
	61	+
	62	+
	63	+def _div(a: Any, b: Any, *, positive_denominator: bool = False) -> float \| None:
	64	+ a, b = _num(a), _num(b)
	65	+ if a is None or b is None or b == 0 or (positive_denominator and b <= 0):
	66	+ return None
	67	+ return a / b
	68	+
	69	+
	70	+def _growth(cur: Any, prev: Any) -> float \| None:
	71	+ """(cur / prev) − 1, only when the base is strictly positive (a negative base makes growth meaningless)."""
	72	+ cur, prev = _num(cur), _num(prev)
	73	+ if cur is None or prev is None or prev <= 0:
	74	+ return None
	75	+ return cur / prev - 1
	76	+
	77	+
	78	+def _cagr(cur: Any, base: Any, years: int) -> float \| None:
	79	+ cur, base = _num(cur), _num(base)
	80	+ if cur is None or base is None or base <= 0 or cur <= 0:
	81	+ return None
	82	+ return (cur / base) ** (1 / years) - 1
	83	+
	84	+
	85	+# ------------------------------------------------------------------------------------------ valuation
	86	+@ratio("valuation", "price", "shares_outstanding")
	87	+def market_cap(f: Inputs) -> float \| None:
	88	+ """market_cap = price × shares_outstanding
	89	+
	90	+ `shares_outstanding` is the cover-page share count (dei:EntityCommonStockSharesOutstanding, all
	91	+ classes summed) of the latest filing known at the valuation date; falls back to weighted-average
	92	+ diluted shares (flagged `shares_source=weighted_diluted`)."""
	93	+ p, s = _num(f.get("price")), _num(f.get("shares_outstanding"))
	94	+ return None if p is None or s is None else p * s
	95	+
	96	+
	97	+@ratio("valuation", "market_cap", "total_debt", "cash_and_equivalents", "short_term_investments")
	98	+def enterprise_value(f: Inputs) -> float \| None:
	99	+ """enterprise_value = market_cap + total_debt − cash_and_equivalents − short_term_investments
	100	+
	101	+ Missing `short_term_investments` is treated as 0 (flagged); missing `total_debt` is treated as 0 only
	102	+ when the balance sheet has no debt account at all (flagged `total_debt_assumed_zero`)."""
	103	+ mc = _num(f.get("market_cap"))
	104	+ if mc is None:
	105	+ return None
	106	+ debt = _num(f.get("total_debt")) or 0.0
	107	+ cash = _num(f.get("cash_and_equivalents"))
	108	+ if cash is None:
	109	+ return None
	110	+ return mc + debt - cash - (_num(f.get("short_term_investments")) or 0.0)
	111	+
	112	+
	113	+@ratio("valuation", "price", "eps_diluted")
	114	+def pe(f: Inputs) -> float \| None:
	115	+ """pe = price / eps_diluted (TTM)
	116	+
	117	+ Null when TTM diluted EPS ≤ 0 (a negative P/E is not meaningful)."""
	118	+ return _div(f.get("price"), f.get("eps_diluted"), positive_denominator=True)
	119	+
	120	+
	121	+@ratio("valuation", "forward_eps")
	122	+def forward_pe(f: Inputs) -> float \| None:
	123	+ """forward_pe = price / forward_eps
	124	+
	125	+ HF Market Data does not carry analyst consensus estimates, so `forward_eps` is never available
	126	+ and `forward_pe` is always null with reason `no_estimates` — we do not extrapolate."""
	127	+ return _div(f.get("price"), f.get("forward_eps"), positive_denominator=True)
	128	+
	129	+
	130	+@ratio("valuation", "market_cap", "total_equity")
	131	+def pb(f: Inputs) -> float \| None:
	132	+ """pb = market_cap / total_equity
	133	+
	134	+ Null when book equity ≤ 0."""
	135	+ return _div(f.get("market_cap"), f.get("total_equity"), positive_denominator=True)
	136	+
	137	+
	138	+@ratio("valuation", "market_cap", "revenue")
	139	+def ps(f: Inputs) -> float \| None:
	140	+ """ps = market_cap / revenue (TTM)"""
	141	+ return _div(f.get("market_cap"), f.get("revenue"), positive_denominator=True)
	142	+
	143	+
	144	+@ratio("valuation", "enterprise_value", "ebitda")
	145	+def ev_ebitda(f: Inputs) -> float \| None:
	146	+ """ev_ebitda = enterprise_value / ebitda (TTM)
	147	+
	148	+ Null when EBITDA ≤ 0."""
	149	+ return _div(f.get("enterprise_value"), f.get("ebitda"), positive_denominator=True)
	150	+
	151	+
	152	+@ratio("valuation", "enterprise_value", "revenue")
	153	+def ev_sales(f: Inputs) -> float \| None:
	154	+ """ev_sales = enterprise_value / revenue (TTM)"""
	155	+ return _div(f.get("enterprise_value"), f.get("revenue"), positive_denominator=True)
	156	+
	157	+
	158	+@ratio("valuation", "enterprise_value", "free_cash_flow")
	159	+def ev_fcf(f: Inputs) -> float \| None:
	160	+ """ev_fcf = enterprise_value / free_cash_flow (TTM)
	161	+
	162	+ Null when FCF ≤ 0."""
	163	+ return _div(f.get("enterprise_value"), f.get("free_cash_flow"), positive_denominator=True)
	164	+
	165	+
	166	+@ratio("valuation", "net_income", "market_cap")
	167	+def earnings_yield(f: Inputs) -> float \| None:
	168	+ """earnings_yield = net_income (TTM) / market_cap"""
	169	+ return _div(f.get("net_income"), f.get("market_cap"), positive_denominator=True)
	170	+
	171	+
	172	+@ratio("valuation", "free_cash_flow", "market_cap")
	173	+def fcf_yield(f: Inputs) -> float \| None:
	174	+ """fcf_yield = free_cash_flow (TTM) / market_cap"""
	175	+ return _div(f.get("free_cash_flow"), f.get("market_cap"), positive_denominator=True)
	176	+
	177	+
	178	+@ratio("valuation", "dividends", "market_cap")
	179	+def dividend_yield(f: Inputs) -> float \| None:
	180	+ """dividend_yield = dividends paid (TTM, cash flow statement) / market_cap"""
	181	+ return _div(f.get("dividends"), f.get("market_cap"), positive_denominator=True)
	182	+
	183	+
	184	+@ratio("valuation", "buybacks", "market_cap")
	185	+def buyback_yield(f: Inputs) -> float \| None:
	186	+ """buyback_yield = buybacks (TTM, cash paid for repurchases) / market_cap"""
	187	+ return _div(f.get("buybacks"), f.get("market_cap"), positive_denominator=True)
	188	+
	189	+
	190	+# -------------------------------------------------------------------------------------- profitability
	191	+@ratio("profitability", "gross_profit", "revenue")
	192	+def gross_margin(f: Inputs) -> float \| None:
	193	+ """gross_margin = gross_profit / revenue (TTM)"""
	194	+ return _div(f.get("gross_profit"), f.get("revenue"), positive_denominator=True)
	195	+
	196	+
	197	+@ratio("profitability", "operating_income", "revenue")
	198	+def operating_margin(f: Inputs) -> float \| None:
	199	+ """operating_margin = operating_income / revenue (TTM)"""
	200	+ return _div(f.get("operating_income"), f.get("revenue"), positive_denominator=True)
	201	+
	202	+
	203	+@ratio("profitability", "net_income", "revenue")
	204	+def net_margin(f: Inputs) -> float \| None:
	205	+ """net_margin = net_income / revenue (TTM)"""
	206	+ return _div(f.get("net_income"), f.get("revenue"), positive_denominator=True)
	207	+
	208	+
	209	+@ratio("profitability", "ebitda", "revenue")
	210	+def ebitda_margin(f: Inputs) -> float \| None:
	211	+ """ebitda_margin = ebitda / revenue (TTM)"""
	212	+ return _div(f.get("ebitda"), f.get("revenue"), positive_denominator=True)
	213	+
	214	+
	215	+@ratio("profitability", "free_cash_flow", "revenue")
	216	+def fcf_margin(f: Inputs) -> float \| None:
	217	+ """fcf_margin = free_cash_flow / revenue (TTM)"""
	218	+ return _div(f.get("free_cash_flow"), f.get("revenue"), positive_denominator=True)
	219	+
	220	+
	221	+@ratio("profitability", "net_income", "total_equity")
	222	+def roe(f: Inputs) -> float \| None:
	223	+ """roe = net_income (TTM) / total_equity (latest)
	224	+
	225	+ Uses the latest book equity, not the average — simpler and point-in-time consistent. Null when
	226	+ equity ≤ 0."""
	227	+ return _div(f.get("net_income"), f.get("total_equity"), positive_denominator=True)
	228	+
	229	+
	230	+@ratio("profitability", "net_income", "total_assets")
	231	+def roa(f: Inputs) -> float \| None:
	232	+ """roa = net_income (TTM) / total_assets (latest)"""
	233	+ return _div(f.get("net_income"), f.get("total_assets"), positive_denominator=True)
	234	+
	235	+
	236	+@ratio("profitability", "operating_income", "income_tax", "pretax_income", "total_debt", "total_equity", "cash_and_equivalents")
	237	+def roic(f: Inputs) -> float \| None:
	238	+ """roic = operating_income × (1 − tax_rate) / (total_debt + total_equity − cash_and_equivalents)
	239	+
	240	+ `tax_rate` = income_tax / pretax_income (TTM), only accepted in [0, 1]; when the effective rate is
	241	+ not computable the ratio is null (`tax_rate_unavailable`) — no statutory rate is assumed. Null when
	242	+ invested capital ≤ 0."""
	243	+ oi = _num(f.get("operating_income"))
	244	+ tax_rate = _div(f.get("income_tax"), f.get("pretax_income"), positive_denominator=True)
	245	+ if oi is None or tax_rate is None or not (0 <= tax_rate <= 1):
	246	+ return None
	247	+ eq, cash = _num(f.get("total_equity")), _num(f.get("cash_and_equivalents"))
	248	+ if eq is None or cash is None:
	249	+ return None
	250	+ invested = (_num(f.get("total_debt")) or 0.0) + eq - cash
	251	+ return None if invested <= 0 else oi * (1 - tax_rate) / invested
	252	+
	253	+
	254	+# ------------------------------------------------------------------------------------------ liquidity
	255	+@ratio("liquidity", "total_current_assets", "total_current_liabilities")
	256	+def current_ratio(f: Inputs) -> float \| None:
	257	+ """current_ratio = total_current_assets / total_current_liabilities"""
	258	+ return _div(f.get("total_current_assets"), f.get("total_current_liabilities"), positive_denominator=True)
	259	+
	260	+
	261	+@ratio("liquidity", "cash_and_equivalents", "short_term_investments", "receivables", "total_current_liabilities")
	262	+def quick_ratio(f: Inputs) -> float \| None:
	263	+ """quick_ratio = (cash_and_equivalents + short_term_investments + receivables) / total_current_liabilities
	264	+
	265	+ Missing short_term_investments or receivables are treated as 0 (flagged)."""
	266	+ cash = _num(f.get("cash_and_equivalents"))
	267	+ if cash is None:
	268	+ return None
	269	+ num = cash + (_num(f.get("short_term_investments")) or 0.0) + (_num(f.get("receivables")) or 0.0)
	270	+ return _div(num, f.get("total_current_liabilities"), positive_denominator=True)
	271	+
	272	+
	273	+@ratio("liquidity", "cash_and_equivalents", "short_term_investments", "total_current_liabilities")
	274	+def cash_ratio(f: Inputs) -> float \| None:
	275	+ """cash_ratio = (cash_and_equivalents + short_term_investments) / total_current_liabilities"""
	276	+ cash = _num(f.get("cash_and_equivalents"))
	277	+ if cash is None:
	278	+ return None
	279	+ return _div(cash + (_num(f.get("short_term_investments")) or 0.0), f.get("total_current_liabilities"),
	280	+ positive_denominator=True)
	281	+
	282	+
	283	+# ------------------------------------------------------------------------------------------- solvency
	284	+@ratio("solvency", "total_debt", "total_equity")
	285	+def debt_to_equity(f: Inputs) -> float \| None:
	286	+ """debt_to_equity = total_debt / total_equity
	287	+
	288	+ Null when equity ≤ 0."""
	289	+ return _div(f.get("total_debt"), f.get("total_equity"), positive_denominator=True)
	290	+
	291	+
	292	+@ratio("solvency", "total_debt", "total_assets")
	293	+def debt_to_assets(f: Inputs) -> float \| None:
	294	+ """debt_to_assets = total_debt / total_assets"""
	295	+ return _div(f.get("total_debt"), f.get("total_assets"), positive_denominator=True)
	296	+
	297	+
	298	+@ratio("solvency", "net_debt", "ebitda")
	299	+def net_debt_to_ebitda(f: Inputs) -> float \| None:
	300	+ """net_debt_to_ebitda = net_debt / ebitda (TTM)
	301	+
	302	+ Null when EBITDA ≤ 0."""
	303	+ return _div(f.get("net_debt"), f.get("ebitda"), positive_denominator=True)
	304	+
	305	+
	306	+@ratio("solvency", "operating_income", "interest_expense")
	307	+def interest_coverage(f: Inputs) -> float \| None:
	308	+ """interest_coverage = operating_income / interest_expense (TTM)
	309	+
	310	+ Null when interest expense is 0 or not reported."""
	311	+ return _div(f.get("operating_income"), f.get("interest_expense"), positive_denominator=True)
	312	+
	313	+
	314	+# ----------------------------------------------------------------------------------------- efficiency
	315	+@ratio("efficiency", "revenue", "total_assets")
	316	+def asset_turnover(f: Inputs) -> float \| None:
	317	+ """asset_turnover = revenue (TTM) / total_assets (latest)"""
	318	+ return _div(f.get("revenue"), f.get("total_assets"), positive_denominator=True)
	319	+
	320	+
	321	+@ratio("efficiency", "cost_of_revenue", "inventory")
	322	+def inventory_turnover(f: Inputs) -> float \| None:
	323	+ """inventory_turnover = cost_of_revenue (TTM) / inventory (latest)"""
	324	+ return _div(f.get("cost_of_revenue"), f.get("inventory"), positive_denominator=True)
	325	+
	326	+
	327	+@ratio("efficiency", "revenue", "receivables")
	328	+def receivables_turnover(f: Inputs) -> float \| None:
	329	+ """receivables_turnover = revenue (TTM) / receivables (latest)"""
	330	+ return _div(f.get("revenue"), f.get("receivables"), positive_denominator=True)
	331	+
	332	+
	333	+@ratio("efficiency", "receivables", "revenue")
	334	+def days_sales_outstanding(f: Inputs) -> float \| None:
	335	+ """days_sales_outstanding = 365 × receivables / revenue (TTM)"""
	336	+ r = _div(f.get("receivables"), f.get("revenue"), positive_denominator=True)
	337	+ return None if r is None else 365 * r
	338	+
	339	+
	340	+@ratio("efficiency", "receivables", "revenue", "inventory", "accounts_payable", "cost_of_revenue")
	341	+def cash_conversion_cycle(f: Inputs) -> float \| None:
	342	+ """cash_conversion_cycle = DSO + DIO − DPO
	343	+
	344	+ DSO = 365 × receivables / revenue · DIO = 365 × inventory / cost_of_revenue ·
	345	+ DPO = 365 × accounts_payable / cost_of_revenue (all flows TTM, balances latest)."""
	346	+ dso = _div(f.get("receivables"), f.get("revenue"), positive_denominator=True)
	347	+ dio = _div(f.get("inventory"), f.get("cost_of_revenue"), positive_denominator=True)
	348	+ dpo = _div(f.get("accounts_payable"), f.get("cost_of_revenue"), positive_denominator=True)
	349	+ if dso is None or dio is None or dpo is None:
	350	+ return None
	351	+ return 365 * (dso + dio - dpo)
	352	+
	353	+
	354	+# --------------------------------------------------------------------------------------------- growth
	355	+@ratio("growth", "revenue", "revenue_prev_year")
	356	+def revenue_growth_yoy(f: Inputs) -> float \| None:
	357	+ """revenue_growth_yoy = revenue (TTM) / revenue (TTM one year earlier) − 1"""
	358	+ return _growth(f.get("revenue"), f.get("revenue_prev_year"))
	359	+
	360	+
	361	+@ratio("growth", "revenue_q", "revenue_prev_quarter")
	362	+def revenue_growth_qoq(f: Inputs) -> float \| None:
	363	+ """revenue_growth_qoq = revenue (latest quarter) / revenue (previous quarter) − 1"""
	364	+ return _growth(f.get("revenue_q"), f.get("revenue_prev_quarter"))
	365	+
	366	+
	367	+@ratio("growth", "eps_diluted", "eps_diluted_prev_year")
	368	+def eps_growth_yoy(f: Inputs) -> float \| None:
	369	+ """eps_growth_yoy = eps_diluted (TTM) / eps_diluted (TTM one year earlier) − 1
	370	+
	371	+ Null when the base EPS ≤ 0."""
	372	+ return _growth(f.get("eps_diluted"), f.get("eps_diluted_prev_year"))
	373	+
	374	+
	375	+@ratio("growth", "free_cash_flow", "free_cash_flow_prev_year")
	376	+def fcf_growth_yoy(f: Inputs) -> float \| None:
	377	+ """fcf_growth_yoy = free_cash_flow (TTM) / free_cash_flow (TTM one year earlier) − 1"""
	378	+ return _growth(f.get("free_cash_flow"), f.get("free_cash_flow_prev_year"))
	379	+
	380	+
	381	+@ratio("growth", "revenue", "revenue_3y")
	382	+def revenue_cagr_3y(f: Inputs) -> float \| None:
	383	+ """revenue_cagr_3y = (revenue TTM / revenue TTM 3 years earlier)^(1/3) − 1"""
	384	+ return _cagr(f.get("revenue"), f.get("revenue_3y"), 3)
	385	+
	386	+
	387	+@ratio("growth", "revenue", "revenue_5y")
	388	+def revenue_cagr_5y(f: Inputs) -> float \| None:
	389	+ """revenue_cagr_5y = (revenue TTM / revenue TTM 5 years earlier)^(1/5) − 1"""
	390	+ return _cagr(f.get("revenue"), f.get("revenue_5y"), 5)
	391	+
	392	+
	393	+@ratio("growth", "revenue", "revenue_10y")
	394	+def revenue_cagr_10y(f: Inputs) -> float \| None:
	395	+ """revenue_cagr_10y = (revenue TTM / revenue TTM 10 years earlier)^(1/10) − 1"""
	396	+ return _cagr(f.get("revenue"), f.get("revenue_10y"), 10)
	397	+
	398	+
	399	+@ratio("growth", "eps_diluted", "eps_diluted_5y")
	400	+def eps_cagr_5y(f: Inputs) -> float \| None:
	401	+ """eps_cagr_5y = (eps_diluted TTM / eps_diluted TTM 5 years earlier)^(1/5) − 1"""
	402	+ return _cagr(f.get("eps_diluted"), f.get("eps_diluted_5y"), 5)
	403	+
	404	+
	405	+# ------------------------------------------------------------------------------------------ per share
	406	+@ratio("per_share", "revenue", "shares_diluted")
	407	+def revenue_ps(f: Inputs) -> float \| None:
	408	+ """revenue_ps = revenue (TTM) / shares_diluted (weighted average, latest quarter)"""
	409	+ return _div(f.get("revenue"), f.get("shares_diluted"), positive_denominator=True)
	410	+
	411	+
	412	+@ratio("per_share", "total_equity", "shares_outstanding")
	413	+def book_value_ps(f: Inputs) -> float \| None:
	414	+ """book_value_ps = total_equity / shares_outstanding"""
	415	+ return _div(f.get("total_equity"), f.get("shares_outstanding"), positive_denominator=True)
	416	+
	417	+
	418	+@ratio("per_share", "free_cash_flow", "shares_diluted")
	419	+def fcf_ps(f: Inputs) -> float \| None:
	420	+ """fcf_ps = free_cash_flow (TTM) / shares_diluted"""
	421	+ return _div(f.get("free_cash_flow"), f.get("shares_diluted"), positive_denominator=True)
	422	+
	423	+
	424	+@ratio("per_share", "cash_and_equivalents", "short_term_investments", "shares_outstanding")
	425	+def cash_ps(f: Inputs) -> float \| None:
	426	+ """cash_ps = (cash_and_equivalents + short_term_investments) / shares_outstanding"""
	427	+ cash = _num(f.get("cash_and_equivalents"))
	428	+ if cash is None:
	429	+ return None
	430	+ return _div(cash + (_num(f.get("short_term_investments")) or 0.0), f.get("shares_outstanding"),
	431	+ positive_denominator=True)
	432	+
	433	+
	434	+RATIO_NAMES: tuple[str, ...] = tuple(r.name for r in RATIOS)
	435	+RATIO_BY_NAME: dict[str, Ratio] = {r.name: r for r in RATIOS}
	436	+# ratios whose result feeds other ratios — computed first, in this order
	437	+_ORDER = ["market_cap", "enterprise_value"] + [r.name for r in RATIOS if r.name not in ("market_cap", "enterprise_value")]
	438	+
	439	+
	440	+def compute_all(inputs: Inputs) -> tuple[dict[str, float \| None], dict[str, str]]:
	441	+ """Compute every ratio. Returns (values, reasons) where `reasons[name]` explains each null."""
	442	+ f = dict(inputs)
	443	+ values: dict[str, float \| None] = {}
	444	+ reasons: dict[str, str] = {}
	445	+ for name in _ORDER:
	446	+ r = RATIO_BY_NAME[name]
	447	+ v = r.fn(f)
	448	+ values[name] = v
	449	+ f[name] = v
	450	+ if v is None:
	451	+ missing = [k for k in r.inputs if _num(f.get(k)) is None]
	452	+ if name == "forward_pe":
	453	+ reasons[name] = "no_estimates"
	454	+ elif missing:
	455	+ reasons[name] = "missing:" + ",".join(missing)
	456	+ else:
	457	+ reasons[name] = "denominator_not_positive"
	458	+ return values, reasons
	459	+
	460	+
	461	+def render_docs() -> str:
	462	+ """Markdown reference of every formula (written to docs/fundamentals-ratios.md)."""
	463	+ lines = ["# Fundamentals — ratio formulas", "",
	464	+ "Generated from `hfmarketdata/api/fundamentals/ratios.py` (`render_docs()`); do not edit by hand.", "",
	465	+ "Conventions: flows are trailing twelve months (sum of the last four discrete quarters — or the fiscal year "
	466	+ "with `period=annual`), balances are the latest balance sheet, `price` is the last close known at the "
	467	+ "valuation date. A ratio is `null` (with a reason in `meta.reasons`) whenever an input is missing or the "
	468	+ "denominator is not positive — nothing is ever invented.", ""]
	469	+ for g in GROUPS:
	470	+ lines += [f"## {g.replace('_', ' ').title()}", ""]
	471	+ for r in RATIOS:
	472	+ if r.group != g:
	473	+ continue
	474	+ doc = r.doc.splitlines()
	475	+ lines += [f"### `{r.name}`", "", f"`{doc[0]}`", ""]
	476	+ rest = "\n".join(doc[1:]).strip()
	477	+ if rest:
	478	+ lines += [rest, ""]
	479	+ lines += [f"Inputs: {', '.join(f'`{i}`' for i in r.inputs)}", ""]
	480	+ return "\n".join(lines)

added hfmarketdata/api/fundamentals/routes.py +274 −0

		`@@ -0,0 +1,274 @@`
	1	+"""`/v1/fundamentals/*` — SEC EDGAR standardized statements, ratios, screener, frames (point-in-time).
	2	+
	3	+Every tabular endpoint supports `format=json\|csv\|parquet`, cursor pagination (`cursor` = opaque offset) and
	4	+`as_of` (only what was public at that date is used — no look-ahead). Errors use the uniform envelope with the
	5	+codes FUNDAMENTALS_NOT_AVAILABLE, CONCEPT_NOT_FOUND, INVALID_FILTER, INVALID_PARAMETER.
	6	+
	7	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	8	+"""
	9	+from __future__ import annotations
	10	+
	11	+from typing import Any
	12	+
	13	+from fastapi import APIRouter, Path, Query, Request
	14	+
	15	+from core.errors import ApiError
	16	+from core.responses import clamp_limit, decode_cursor, encode_cursor, frame_response, json_response, parse_format
	17	+
	18	+from . import mapping as M
	19	+from . import screener as SC
	20	+from . import service as S
	21	+from .models import init_db
	22	+from .ratios import RATIO_NAMES
	23	+
	24	+router = APIRouter(prefix="/v1/fundamentals", tags=["fundamentals"])
	25	+init_db()
	26	+
	27	+_AAPL_Q2_2024 = {"ticker": "AAPL", "cik": 320193, "statement": "income", "fiscal_year": 2024, "fiscal_quarter": 2,
	28	+ "period_start": "2023-12-31", "period_end": "2024-03-30", "calendar_quarter": "2024Q1", "form": "10-Q",
	29	+ "accn": "0000320193-24-000069", "filed_date": "2024-05-03", "derived": False, "restated": False,
	30	+ "currency": "USD", "revenue": 90753000000.0, "cost_of_revenue": 48482000000.0, "gross_profit": 42271000000.0,
	31	+ "rnd_expense": 7903000000.0, "sga_expense": 6468000000.0, "operating_income": 27900000000.0,
	32	+ "interest_expense": None, "pretax_income": 28058000000.0, "income_tax": 4422000000.0,
	33	+ "net_income": 23636000000.0, "eps_basic": 1.53, "eps_diluted": 1.53, "shares_basic": 15405856000.0,
	34	+ "shares_diluted": 15464709000.0, "ebitda": 30736000000.0, "dividends_paid": None,
	35	+ "coverage": {"interest_expense": {"reason": "no_mapped_tag"}, "dividends_paid": {"reason": "no_mapped_tag"},
	36	+ "ebitda": {"computed": "operating_income + depreciation_amortization"}}}
	37	+
	38	+_PAGE_NOTE = ("\n\nPagination: `limit` + opaque `cursor` (from `meta.next_cursor`). Formats: `format=json\|csv\|parquet` "
	39	+ "(Parquet is charged half the rows). Point-in-time: `as_of=YYYY-MM-DD` hides every filing made after that "
	40	+ "date and serves the version that was public then.")
	41	+
	42	+
	43	+def _paginate(df, request: Request, limit: int \| None, cursor: str \| None, default: int, hard_max: int):
	44	+ lim = clamp_limit(limit, default, hard_max, request)
	45	+ off = decode_cursor(cursor) or 0
	46	+ if not isinstance(off, int) or off < 0:
	47	+ raise ApiError(400, "INVALID_PARAMETER", "cursor is not valid")
	48	+ page = df.iloc[off:off + lim]
	49	+ nxt = encode_cursor(off + lim) if off + lim < len(df) else None
	50	+ return page, nxt, lim
	51	+
	52	+
	53	+@router.get("/{ticker}/statements", summary="Standardized financial statements (quarterly, annual, TTM)",
	54	+ description="Income statement, balance sheet and cash flow of an SEC filer, standardized on a fixed chart of "
	55	+ "accounts (see `/v1/fundamentals/_mapping`) from the XBRL facts of its 10-K/10-Q/20-F filings since "
	56	+ "2010.\n\n* `period=quarterly` returns discrete quarters — Q4 (and cash-flow Q2/Q3 that filers only "
	57	+ "report year-to-date) are derived (`derived=true`, formula in `coverage`); `annual` = fiscal "
	58	+ "years (`fiscal_quarter=0`); `ttm` = rolling four quarters (balance sheet = latest).\n* Values are raw "
	59	+ "USD (not thousands), shares raw, EPS in USD/share; outflows (capex, buybacks, dividends…) are "
	60	+ "positive.\n* A null account is never guessed: `coverage[account].reason` says why "
	61	+ "(`no_mapped_tag`, `missing_quarters`…), computed accounts carry their formula.\n* `view=as_reported` "
	62	+ "returns the XBRL facts (tag, value, unit, accession) behind each standardized row." + _PAGE_NOTE,
	63	+ responses={200: {"description": "Statements", "content": {"application/json": {"example": {
	64	+ "data": [_AAPL_Q2_2024], "meta": {"count": 1, "ticker": "AAPL", "cik": 320193, "statement": "income",
	65	+ "period": "quarterly", "as_of": None, "mapping_version": M.MAPPING_VERSION,
	66	+ "next_cursor": None}}}}}},
	67	+ openapi_extra={"x-errors": ["FUNDAMENTALS_NOT_AVAILABLE", "INVALID_PARAMETER", "ROW_LIMIT_EXCEEDED"]})
	68	+def get_statements(request: Request, ticker: str = Path(..., description="Ticker (any share class, e.g. GOOG or GOOGL)"),
	69	+ statement: str = Query("all", description="income \| balance \| cashflow \| all"),
	70	+ period: str = Query("quarterly", description="quarterly \| annual \| ttm"),
	71	+ from_: str \| None = Query(None, alias="from", description="Min period_end (YYYY-MM-DD)"),
	72	+ to: str \| None = Query(None, description="Max period_end (YYYY-MM-DD)"),
	73	+ as_of: str \| None = Query(None, description="Point-in-time date: only filings made on or before"),
	74	+ view: str = Query("standardized", description="standardized \| as_reported"),
	75	+ limit: int \| None = Query(None, ge=1, description="Rows per page (default 40, max 2000)"),
	76	+ cursor: str \| None = Query(None), format: str = Query("json")):
	77	+ fmt = parse_format(format)
	78	+ st, per = S.parse_statement(statement), S.parse_period(period)
	79	+ if view not in S.VIEWS:
	80	+ raise ApiError(400, "INVALID_PARAMETER", "view must be standardized or as_reported")
	81	+ df, meta = S.statements(ticker, statement=st, period=per, date_from=S.parse_date(from_, "from"),
	82	+ date_to=S.parse_date(to, "to"), as_of=S.parse_date(as_of, "as_of"), view=view)
	83	+ page, nxt, _ = _paginate(df, request, limit, cursor, 40, 2000)
	84	+ return frame_response(page, fmt, meta={**meta, "next_cursor": nxt, "total": int(len(df))}, request=request,
	85	+ filename=f"{ticker.upper()}_{st}_{per}")
	86	+
	87	+
	88	+@router.get("/{ticker}/facts/{concept}", summary="Time series of one account or raw XBRL concept",
	89	+ description="`concept` is either a standardized account (`revenue`, `total_assets`, `free_cash_flow`… → the "
	90	+ "point-in-time standardized series with provenance) or a raw XBRL concept "
	91	+ "(`us-gaap:Revenues`, `Revenues`, `dei:EntityCommonStockSharesOutstanding` → every fact instance "
	92	+ "from the Parquet facts lake, one per filing that reported it, with `frame`, `accn`, `filed`). "
	93	+ "Raw facts are what EDGAR published — `fy`/`fp` describe the filing, not the fact's period." + _PAGE_NOTE,
	94	+ responses={200: {"description": "Series", "content": {"application/json": {"example": {
	95	+ "data": [{"ticker": "AAPL", "concept": "revenue", "fiscal_year": 2024, "fiscal_quarter": 2,
	96	+ "period_start": "2023-12-31", "period_end": "2024-03-30", "calendar_quarter": "2024Q1",
	97	+ "value": 90753000000.0, "unit": "USD", "derived": False, "restated": False, "form": "10-Q",
	98	+ "accn": "0000320193-24-000069", "filed_date": "2024-05-03",
	99	+ "coverage": {"tag": "us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax", "priority": 2}}],
	100	+ "meta": {"count": 1, "concept": "revenue", "kind": "standardized", "next_cursor": None}}}}}},
	101	+ openapi_extra={"x-errors": ["FUNDAMENTALS_NOT_AVAILABLE", "CONCEPT_NOT_FOUND", "INVALID_PARAMETER"]})
	102	+def get_facts(request: Request, ticker: str, concept: str,
	103	+ from_: str \| None = Query(None, alias="from"), to: str \| None = Query(None),
	104	+ as_of: str \| None = Query(None), limit: int \| None = Query(None, ge=1), cursor: str \| None = Query(None),
	105	+ format: str = Query("json")):
	106	+ fmt = parse_format(format)
	107	+ df, meta = S.facts_for_concept(ticker, concept, as_of=S.parse_date(as_of, "as_of"), date_from=S.parse_date(from_, "from"),
	108	+ date_to=S.parse_date(to, "to"))
	109	+ page, nxt, _ = _paginate(df, request, limit, cursor, 200, 5000)
	110	+ return frame_response(page, fmt, meta={**meta, "next_cursor": nxt, "total": int(len(df))}, request=request,
	111	+ filename=f"{ticker.upper()}_{concept.replace(':', '_')}")
	112	+
	113	+
	114	+@router.get("/{ticker}/ratios", summary="Valuation, profitability, liquidity, solvency, efficiency, growth ratios",
	115	+ description="Ratios computed from the latest fundamentals known at `as_of` (TTM flows, latest balance sheet) "
	116	+ "and the last close of the price lake at that date. Every formula is documented in "
	117	+ "`docs/fundamentals-ratios.md`; a ratio is `null` with a reason in `meta.reasons` whenever an input "
	118	+ "is missing or a denominator is not positive. `forward_pe` is always null (`no_estimates`): we do not "
	119	+ "carry consensus estimates and never extrapolate. `period=annual` uses fiscal-year flows instead of TTM.",
	120	+ responses={200: {"description": "Ratios", "content": {"application/json": {"example": {
	121	+ "data": {"ticker": "AAPL", "cik": 320193, "as_of": "2024-05-03", "price": 183.38, "price_date": "2024-05-03",
	122	+ "fundamentals_period_end": "2024-03-30", "fiscal_year": 2024, "fiscal_quarter": 2,
	123	+ "valuation": {"market_cap": 2811600000000.0, "pe": 28.5, "forward_pe": None, "pb": 37.9},
	124	+ "profitability": {"gross_margin": 0.456, "net_margin": 0.263, "roe": 1.35},
	125	+ "liquidity": {"current_ratio": 1.04}, "solvency": {"debt_to_equity": 1.41},
	126	+ "efficiency": {"asset_turnover": 1.13}, "growth": {"revenue_growth_yoy": -0.003},
	127	+ "per_share": {"book_value_ps": 4.84}},
	128	+ "meta": {"count": 1, "period": "ttm", "shares_source": "dei:EntityCommonStockSharesOutstanding",
	129	+ "reasons": {"forward_pe": "no_estimates"}}}}}}},
	130	+ openapi_extra={"x-errors": ["FUNDAMENTALS_NOT_AVAILABLE", "INVALID_PARAMETER"]})
	131	+def get_ratios(request: Request, ticker: str, as_of: str \| None = Query(None),
	132	+ period: str = Query("ttm", description="ttm \| annual")):
	133	+ if period not in ("ttm", "annual"):
	134	+ raise ApiError(400, "INVALID_PARAMETER", "period must be ttm or annual")
	135	+ data, meta = S.ratios(ticker, as_of=S.parse_date(as_of, "as_of"), period=period)
	136	+ return json_response(data, meta=meta, rows=1)
	137	+
	138	+
	139	+@router.get("/{ticker}/ratios/daily", summary="Daily point-in-time ratios (close × fundamentals known that day)",
	140	+ description="For every trading day in the range: the close and the ratios computed with the fundamentals that "
	141	+ "were public on that day (as-of join on `filed_date`, DuckDB ASOF). This is the series to use in "
	142	+ "backtests — no look-ahead: a 10-Q filed on May 3 only affects May 3 onwards. `fields` selects the "
	143	+ "ratio columns (default: all)." + _PAGE_NOTE,
	144	+ responses={200: {"description": "Daily ratios", "content": {"application/json": {"example": {
	145	+ "data": [{"date": "2024-05-03", "close": 183.38, "fundamentals_as_of": "2024-05-03",
	146	+ "fundamentals_period_end": "2024-03-30", "pe": 28.5, "pb": 37.9, "fcf_yield": 0.036}],
	147	+ "meta": {"count": 1, "ticker": "AAPL", "point_in_time": True, "snapshots": 3}}}}}},
	148	+ openapi_extra={"x-errors": ["FUNDAMENTALS_NOT_AVAILABLE", "TICKER_NOT_FOUND", "INVALID_PARAMETER", "ROW_LIMIT_EXCEEDED"]})
	149	+def get_ratios_daily(request: Request, ticker: str, from_: str \| None = Query(None, alias="from"),
	150	+ to: str \| None = Query(None), fields: str \| None = Query(None, description="Comma-separated ratio names"),
	151	+ limit: int \| None = Query(None, ge=1), cursor: str \| None = Query(None), format: str = Query("json")):
	152	+ fmt = parse_format(format)
	153	+ want = [f.strip() for f in fields.split(",") if f.strip()] if fields else None
	154	+ df, meta = S.ratios_daily(ticker, date_from=S.parse_date(from_, "from"), date_to=S.parse_date(to, "to"), fields=want)
	155	+ page, nxt, _ = _paginate(df, request, limit, cursor, 1000, 20000)
	156	+ return frame_response(page, fmt, meta={**meta, "next_cursor": nxt, "total": int(len(df))}, request=request,
	157	+ filename=f"{ticker.upper()}_ratios_daily")
	158	+
	159	+
	160	+@router.get("/{ticker}/filings", summary="SEC filings of a company with EDGAR links",
	161	+ description="Filings tracked for the company (10-K, 10-Q, 8-K, 20-F, amendments…) with filing date, period of "
	162	+ "report, XBRL flag and direct EDGAR links (`primary_doc_url`, `index_url`). `form=10-K,10-Q` filters." + _PAGE_NOTE,
	163	+ responses={200: {"description": "Filings", "content": {"application/json": {"example": {
	164	+ "data": [{"ticker": "AAPL", "cik": 320193, "accn": "0000320193-24-000069", "form": "10-Q", "filed_date": "2024-05-03",
	165	+ "period_of_report": "2024-03-30", "is_amendment": False, "is_xbrl": True,
	166	+ "primary_doc_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000069/aapl-20240330.htm",
	167	+ "index_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000069/0000320193-24-000069-index.htm"}],
	168	+ "meta": {"count": 1, "ticker": "AAPL", "next_cursor": None}}}}}},
	169	+ openapi_extra={"x-errors": ["FUNDAMENTALS_NOT_AVAILABLE", "INVALID_PARAMETER"]})
	170	+def get_filings(request: Request, ticker: str, form: str \| None = Query(None), from_: str \| None = Query(None, alias="from"),
	171	+ to: str \| None = Query(None), limit: int \| None = Query(None, ge=1), cursor: str \| None = Query(None),
	172	+ format: str = Query("json")):
	173	+ fmt = parse_format(format)
	174	+ df, meta = S.filings(ticker, form=form, date_from=S.parse_date(from_, "from"), date_to=S.parse_date(to, "to"))
	175	+ page, nxt, _ = _paginate(df, request, limit, cursor, 100, 5000)
	176	+ return frame_response(page, fmt, meta={**meta, "next_cursor": nxt, "total": int(len(df))}, request=request,
	177	+ filename=f"{ticker.upper()}_filings")
	178	+
	179	+
	180	+@router.get("/{ticker}/coverage", summary="Data coverage and quality of a company",
	181	+ description="What we have for the company: identity (CIK, share classes, ticker history, SIC, fiscal year end), "
	182	+ "period range, number of quarters/annuals/filings, completeness (% of standardized accounts filled) "
	183	+ "overall and per statement, accounts that are missing and why, fiscal-quarter gaps, derived and "
	184	+ "restated counts, and the company's custom XBRL extensions found in its statements (logged, never "
	185	+ "mapped automatically — e.g. `shak:OperatingMaterialsExpense`).",
	186	+ responses={200: {"description": "Coverage", "content": {"application/json": {"example": {
	187	+ "data": {"ticker": "AAPL", "cik": 320193, "name": "Apple Inc.", "tickers": ["AAPL"], "status": "active",
	188	+ "fiscal_year_end": "0927", "quarters": 60, "annuals": 15, "completeness": 88.4,
	189	+ "missing_accounts": {"interest_expense": {"reason": "no_mapped_tag", "periods": 12}},
	190	+ "gaps": [], "custom_extensions": []}, "meta": {"count": 1}}}}}},
	191	+ openapi_extra={"x-errors": ["FUNDAMENTALS_NOT_AVAILABLE"]})
	192	+def get_coverage(ticker: str):
	193	+ return json_response(S.coverage(ticker), rows=1)
	194	+
	195	+
	196	+@router.get("/screener", summary="Screen the universe on fundamentals and ratios",
	197	+ description="Filter and rank every covered company on a precomputed table (`fund_latest`: TTM accounts, latest "
	198	+ "balance sheet, ratios × last close), refreshed after each ingestion.\n\n`filters` grammar: comma-"
	199	+ "separated `field<op>value` with op `< <= > >= = !=` and ranges `pe=10..20`; numbers accept `k m b t` "
	200	+ "and `%` (`market_cap>10b`, `roe>15%`); text fields `ticker`, `sic`, `exchange`, `name` accept `\|` lists "
	201	+ "(`exchange=Nasdaq\|NYSE`). `sort=field:asc\|desc` (nulls last). `columns` chooses the returned "
	202	+ "fields. Requires an API key; costs 2 requests." + _PAGE_NOTE,
	203	+ responses={200: {"description": "Matching companies", "content": {"application/json": {"example": {
	204	+ "data": [{"ticker": "AAPL", "cik": 320193, "name": "Apple Inc.", "price": 183.38, "market_cap": 2.81e12,
	205	+ "pe": 28.5, "roe": 1.35, "fcf_yield": 0.036}],
	206	+ "meta": {"count": 1, "total": 1, "filters": "pe<30,roe>0.15", "sort": "fcf_yield:desc", "next_cursor": None}}}}}},
	207	+ openapi_extra={"x-errors": ["INVALID_FILTER", "INVALID_PARAMETER", "AUTH_REQUIRED", "ROW_LIMIT_EXCEEDED"]})
	208	+def get_screener(request: Request, filters: str \| None = Query(None, description="e.g. pe<15,roe>0.15,market_cap>1b"),
	209	+ sort: str \| None = Query("market_cap:desc"), columns: str \| None = Query(None),
	210	+ limit: int \| None = Query(None, ge=1), cursor: str \| None = Query(None), format: str = Query("json")):
	211	+ request.state.request_cost = 2
	212	+ request.state.requires_key = True
	213	+ fmt = parse_format(format)
	214	+ flt = SC.parse_filters(filters)
	215	+ srt = SC.parse_sort(sort)
	216	+ cols = [c.strip() for c in columns.split(",") if c.strip()] if columns else None
	217	+ lim = clamp_limit(limit, 100, 5000, request)
	218	+ off = decode_cursor(cursor) or 0
	219	+ if not isinstance(off, int) or off < 0:
	220	+ raise ApiError(400, "INVALID_PARAMETER", "cursor is not valid")
	221	+ df, total = SC.run(flt, srt, limit=lim, offset=off, columns=cols)
	222	+ nxt = encode_cursor(off + lim) if off + lim < total else None
	223	+ return frame_response(df, fmt, meta={"total": total, "filters": filters, "sort": f"{srt[0]}:{'desc' if srt[1] else 'asc'}",
	224	+ "next_cursor": nxt, "fields": list(SC.SCREENER_FIELDS)}, request=request, filename="screener")
	225	+
	226	+
	227	+@router.get("/frames/{concept}", summary="Cross-section of one account across all companies",
	228	+ description="One value per company for a standardized account at a calendar quarter "
	229	+ "(`calendar_quarter=2024Q1`: every fiscal quarter ending in Jan–Mar 2024, whatever the fiscal "
	230	+ "calendar) or a fiscal period (`fiscal_year=2024&fiscal_quarter=2`; `fiscal_quarter=0` = fiscal "
	231	+ "year). Latest version known at `as_of`. Sorted by value descending, nulls last. Costs 2 requests." + _PAGE_NOTE,
	232	+ responses={200: {"description": "Frame", "content": {"application/json": {"example": {
	233	+ "data": [{"cik": 320193, "ticker": "AAPL", "concept": "revenue", "fiscal_year": 2024, "fiscal_quarter": 2,
	234	+ "period_end": "2024-03-30", "calendar_quarter": "2024Q1", "filed_date": "2024-05-03",
	235	+ "accn": "0000320193-24-000069", "derived": False, "restated": False, "currency": "USD",
	236	+ "value": 90753000000.0}],
	237	+ "meta": {"count": 1, "concept": "revenue", "calendar_quarter": "2024Q1", "next_cursor": None}}}}}},
	238	+ openapi_extra={"x-errors": ["CONCEPT_NOT_FOUND", "INVALID_PARAMETER", "ROW_LIMIT_EXCEEDED"]})
	239	+def get_frames(request: Request, concept: str, calendar_quarter: str \| None = Query(None, description="e.g. 2024Q1"),
	240	+ fiscal_year: int \| None = Query(None), fiscal_quarter: int \| None = Query(None, ge=0, le=4),
	241	+ as_of: str \| None = Query(None), limit: int \| None = Query(None, ge=1), cursor: str \| None = Query(None),
	242	+ format: str = Query("json")):
	243	+ request.state.request_cost = 2
	244	+ fmt = parse_format(format)
	245	+ df, meta = S.frames(concept, calendar_quarter=calendar_quarter, fiscal_year=fiscal_year, fiscal_quarter=fiscal_quarter,
	246	+ as_of=S.parse_date(as_of, "as_of"))
	247	+ page, nxt, _ = _paginate(df, request, limit, cursor, 500, 10000)
	248	+ return frame_response(page, fmt, meta={**meta, "next_cursor": nxt, "total": int(len(df))}, request=request,
	249	+ filename=f"frame_{concept}")
	250	+
	251	+
	252	+@router.get("/_mapping", summary="Chart of accounts and prioritized tag mapping",
	253	+ description="The standardized chart of accounts (statement, kind, unit, computed formula) and, for each account, "
	254	+ "the ordered list of XBRL tags it is read from with the reason of each fallback. This is the exact table "
	255	+ "seeded in `fund_mapping` (version in `meta.version`).",
	256	+ responses={200: {"description": "Mapping", "content": {"application/json": {"example": {
	257	+ "data": [{"account": "revenue", "statement": "income", "kind": "duration", "unit": "USD", "computed": False,
	258	+ "tags": [{"taxonomy": "us-gaap", "tag": "Revenues", "priority": 1, "notes": "aggregate revenue"}]}],
	259	+ "meta": {"count": 1, "version": M.MAPPING_VERSION}}}}}},
	260	+ openapi_extra={"x-errors": []})
	261	+def get_mapping():
	262	+ data: list[dict[str, Any]] = []
	263	+ for a in M.ACCOUNTS:
	264	+ data.append({"account": a.name, "statement": a.statement, "kind": a.kind, "unit": a.unit, "computed": a.computed,
	265	+ "auxiliary": a.auxiliary, "formula": a.formula or None, "notes": a.notes or None,
	266	+ "tags": [{"taxonomy": t.taxonomy, "tag": t.tag, "priority": i, "notes": t.notes}
	267	+ for i, t in enumerate(a.tags, start=1)]})
	268	+ return json_response(data, meta={"version": M.MAPPING_VERSION, "ratios": list(RATIO_NAMES), "sign_convention": M.SIGN_CONVENTION})
	269	+
	270	+
	271	+@router.get("/_health", include_in_schema=False)
	272	+def get_health():
	273	+ """Internal: ingestion lag, failures, mapping failure rate (also in `fund_ingest_state`)."""
	274	+ return json_response(S.health(), rows=1)

added hfmarketdata/api/fundamentals/screener.py +131 −0

		`@@ -0,0 +1,131 @@`
	1	+"""Screener over the precomputed `fund_latest` table — filter grammar + query builder.
	2	+
	3	+Grammar (comma-separated, one expression per field):
	4	+
	5	+ pe<15 pe<=15 roe>0.15 market_cap>=1b exchange=Nasdaq
	6	+ pe=10..20 (inclusive range) sic=3571 ticker=AAPL,MSFT (list, `\|` separated: AAPL\|MSFT)
	7	+
	8	+Numbers accept the suffixes k, m, b, t (1e3 … 1e12) and percents (`15%` = 0.15). Fields are the
	9	+standardized accounts (TTM / latest balance), the ratios (see docs/fundamentals-ratios.md) plus
	10	+`price`, `ticker`, `sic`, `exchange`, `name`. Sort: `sort=fcf_yield:desc` (nulls last).
	11	+
	12	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	13	+"""
	14	+from __future__ import annotations
	15	+
	16	+import re
	17	+from dataclasses import dataclass
	18	+from typing import Any
	19	+
	20	+import pandas as pd
	21	+from sqlalchemy import and_, or_, select
	22	+
	23	+from core.db import session
	24	+from core.errors import ApiError
	25	+
	26	+from .models import SCREENER_FIELDS, SCREENER_TEXT_FIELDS, fund_latest
	27	+
	28	+OPS = ("<=", ">=", "!=", "<", ">", "=")
	29	+_NUM = re.compile(r"^([-+]?\d+(?:\.\d+)?(?:e[-+]?\d+)?)\s*([kmbt%]?)$", re.I)
	30	+_SUFFIX = {"": 1.0, "k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12, "%": 0.01}
	31	+
	32	+
	33	+@dataclass(frozen=True)
	34	+class Filter:
	35	+ field: str
	36	+ op: str
	37	+ value: Any # float \| str \| (lo, hi) \| list[str]
	38	+
	39	+
	40	+def parse_number(text: str) -> float:
	41	+ m = _NUM.match(text.strip())
	42	+ if not m:
	43	+ raise ApiError(400, "INVALID_FILTER", f"'{text}' is not a number (use 15, 1.5e9, 1b, 250m, 15%)")
	44	+ return float(m.group(1)) * _SUFFIX[m.group(2).lower()]
	45	+
	46	+
	47	+def parse_filters(expr: str \| None) -> list[Filter]:
	48	+ out: list[Filter] = []
	49	+ if not expr or not expr.strip():
	50	+ return out
	51	+ for raw in expr.split(","):
	52	+ part = raw.strip()
	53	+ if not part:
	54	+ continue
	55	+ for op in OPS:
	56	+ if op in part:
	57	+ field, value = part.split(op, 1)
	58	+ break
	59	+ else:
	60	+ raise ApiError(400, "INVALID_FILTER", f"'{part}': expected <field><op><value> with op in {' '.join(OPS)}")
	61	+ field, value = field.strip().lower(), value.strip()
	62	+ if field not in SCREENER_FIELDS and field not in SCREENER_TEXT_FIELDS:
	63	+ raise ApiError(400, "INVALID_FILTER", f"unknown field '{field}'",
	64	+ details={"numeric_fields": list(SCREENER_FIELDS), "text_fields": list(SCREENER_TEXT_FIELDS)})
	65	+ if not value:
	66	+ raise ApiError(400, "INVALID_FILTER", f"'{part}': missing value")
	67	+ if field in SCREENER_TEXT_FIELDS:
	68	+ if op not in ("=", "!="):
	69	+ raise ApiError(400, "INVALID_FILTER", f"'{field}' only supports = and !=")
	70	+ vals = [v.strip().upper() if field != "name" else v.strip() for v in value.split("\|") if v.strip()]
	71	+ out.append(Filter(field, op, vals))
	72	+ continue
	73	+ if ".." in value:
	74	+ if op != "=":
	75	+ raise ApiError(400, "INVALID_FILTER", f"'{part}': ranges use '=' (pe=10..20)")
	76	+ lo, hi = value.split("..", 1)
	77	+ lo_v, hi_v = parse_number(lo), parse_number(hi)
	78	+ if lo_v > hi_v:
	79	+ raise ApiError(400, "INVALID_FILTER", f"'{part}': range lower bound above upper bound")
	80	+ out.append(Filter(field, "range", (lo_v, hi_v)))
	81	+ else:
	82	+ out.append(Filter(field, op, parse_number(value)))
	83	+ return out
	84	+
	85	+
	86	+def parse_sort(sort: str \| None, default: str = "market_cap:desc") -> tuple[str, bool]:
	87	+ s = (sort or default).strip()
	88	+ field, _, direction = s.partition(":")
	89	+ field = field.lower()
	90	+ if field not in SCREENER_FIELDS and field not in SCREENER_TEXT_FIELDS:
	91	+ raise ApiError(400, "INVALID_FILTER", f"cannot sort on unknown field '{field}'")
	92	+ direction = (direction or "desc").lower()
	93	+ if direction not in ("asc", "desc"):
	94	+ raise ApiError(400, "INVALID_FILTER", "sort direction must be asc or desc")
	95	+ return field, direction == "desc"
	96	+
	97	+
	98	+def _clause(f: Filter):
	99	+ col = fund_latest.c[f.field]
	100	+ if f.op == "range":
	101	+ return and_(col >= f.value[0], col <= f.value[1])
	102	+ if isinstance(f.value, list):
	103	+ if f.field == "name":
	104	+ conds = [col.ilike(f"%{v}%") for v in f.value]
	105	+ return or_(conds) if f.op == "=" else and_([~c for c in conds])
	106	+ return col.in_(f.value) if f.op == "=" else col.notin_(f.value)
	107	+ return {"<": col < f.value, "<=": col <= f.value, ">": col > f.value, ">=": col >= f.value,
	108	+ "=": col == f.value, "!=": col != f.value}[f.op]
	109	+
	110	+
	111	+def run(filters: list[Filter], sort: tuple[str, bool], *, limit: int, offset: int = 0,
	112	+ columns: list[str] \| None = None) -> tuple[pd.DataFrame, int]:
	113	+ """Execute the screener; returns (rows, total matching)."""
	114	+ from sqlalchemy import func
	115	+ base_cols = ["ticker", "cik", "name", "sic", "exchange", "price", "price_date", "period_end", "fiscal_year",
	116	+ "fiscal_quarter", "filed_date"]
	117	+ wanted = list(dict.fromkeys(base_cols + (columns or ["market_cap", "pe", "pb", "ps", "ev_ebitda", "roe", "roa",
	118	+ "gross_margin", "operating_margin", "net_margin", "fcf_yield",
	119	+ "dividend_yield", "debt_to_equity", "revenue_growth_yoy"])))
	120	+ bad = [c for c in wanted if c not in fund_latest.c]
	121	+ if bad:
	122	+ raise ApiError(400, "INVALID_PARAMETER", f"unknown column(s): {', '.join(bad)}")
	123	+ where = [_clause(f) for f in filters]
	124	+ sort_col = fund_latest.c[sort[0]]
	125	+ order = (sort_col.desc().nulls_last() if sort[1] else sort_col.asc().nulls_last(), fund_latest.c.ticker.asc())
	126	+ q = select([fund_latest.c[c] for c in wanted]).where(and_(where) if where else True).order_by(*order).limit(limit).offset(offset)
	127	+ cnt = select(func.count()).select_from(fund_latest).where(and_(*where) if where else True)
	128	+ with session() as s:
	129	+ total = int(s.scalar(cnt) or 0)
	130	+ df = pd.DataFrame([dict(r._mapping) for r in s.execute(q)], columns=wanted)
	131	+ return df, total

added hfmarketdata/api/fundamentals/service.py +499 −0

		`@@ -0,0 +1,499 @@`
	1	+"""Read side of the fundamentals module: point-in-time queries over fund_statements + price lake joins.
	2	+
	3	+Every public function takes `as_of` (date \| None): rows filed after `as_of` are invisible (see
	4	+normalize.select_as_of) and the price used for valuation ratios is the last close at or before `as_of`.
	5	+
	6	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	7	+"""
	8	+from __future__ import annotations
	9	+
	10	+from datetime import date
	11	+from typing import Any
	12	+
	13	+import pandas as pd
	14	+from sqlalchemy import and_, func, select
	15	+
	16	+from core import duck
	17	+from core.db import session
	18	+from core.errors import ApiError
	19	+
	20	+from . import mapping as M
	21	+from . import prices
	22	+from .models import (EdgarCompany, EdgarFiling, FundCoverage, FundMappingLog, fund_latest, fund_statements)
	23	+from .normalize import _d, select_as_of, ttm
	24	+from .ratios import RATIO_NAMES, compute_all
	25	+
	26	+STATEMENT_ALIASES = {"income": M.INCOME, "is": M.INCOME, "pnl": M.INCOME, "balance": M.BALANCE, "bs": M.BALANCE,
	27	+ "cashflow": M.CASHFLOW, "cf": M.CASHFLOW, "cash_flow": M.CASHFLOW, "all": "all"}
	28	+PERIODS = ("quarterly", "annual", "ttm")
	29	+VIEWS = ("standardized", "as_reported")
	30	+BASE_COLUMNS = ["ticker", "cik", "statement", "fiscal_year", "fiscal_quarter", "period_start", "period_end",
	31	+ "calendar_quarter", "form", "accn", "filed_date", "derived", "restated", "currency"]
	32	+
	33	+
	34	+# ------------------------------------------------------------------------------------------ helpers
	35	+def parse_statement(value: str \| None) -> str:
	36	+ v = (value or "all").lower()
	37	+ if v not in STATEMENT_ALIASES:
	38	+ raise ApiError(400, "INVALID_PARAMETER", "statement must be one of income, balance, cashflow, all")
	39	+ return STATEMENT_ALIASES[v]
	40	+
	41	+
	42	+def parse_period(value: str \| None) -> str:
	43	+ v = (value or "quarterly").lower()
	44	+ if v not in PERIODS:
	45	+ raise ApiError(400, "INVALID_PARAMETER", "period must be one of quarterly, annual, ttm")
	46	+ return v
	47	+
	48	+
	49	+def parse_date(value: str \| None, name: str) -> date \| None:
	50	+ if value in (None, ""):
	51	+ return None
	52	+ try:
	53	+ return date.fromisoformat(str(value)[:10])
	54	+ except ValueError:
	55	+ raise ApiError(400, "INVALID_PARAMETER", f"{name} must be an ISO date (YYYY-MM-DD)")
	56	+
	57	+
	58	+def resolve_company(ticker: str) -> EdgarCompany:
	59	+ """Ticker (any share class, current or historical) → company row, or FUNDAMENTALS_NOT_AVAILABLE."""
	60	+ t = ticker.upper().strip()
	61	+ with session() as s:
	62	+ c = s.scalar(select(EdgarCompany).where(EdgarCompany.ticker == t))
	63	+ if c is None:
	64	+ for row in s.scalars(select(EdgarCompany)):
	65	+ if t in (row.tickers or []) or any(h.get("ticker") == t for h in (row.ticker_history or [])):
	66	+ c = row
	67	+ break
	68	+ if c is None:
	69	+ raise ApiError(404, "FUNDAMENTALS_NOT_AVAILABLE",
	70	+ f"No SEC EDGAR fundamentals for '{t}'. The ticker is either not an SEC filer (ETF, foreign "
	71	+ "listing), not in the price lake, or not ingested yet.", details={"ticker": t})
	72	+ s.expunge(c)
	73	+ return c
	74	+
	75	+
	76	+def _rows(cik: int, statements: list[str] \| None = None) -> list[dict[str, Any]]:
	77	+ q = select(fund_statements).where(fund_statements.c.cik == cik)
	78	+ if statements:
	79	+ q = q.where(fund_statements.c.statement.in_(statements))
	80	+ with session() as s:
	81	+ return [dict(r._mapping) for r in s.execute(q)]
	82	+
	83	+
	84	+def company_rows(cik: int, as_of: date \| None, statements: list[str] \| None = None,
	85	+ all_versions: bool = False) -> list[dict[str, Any]]:
	86	+ rows = _rows(cik, statements)
	87	+ if all_versions:
	88	+ return [r for r in rows if as_of is None or _d(r["filed_date"]) <= as_of]
	89	+ return select_as_of(rows, as_of)
	90	+
	91	+
	92	+def _filter_period(rows: list[dict], period: str) -> list[dict]:
	93	+ if period == "annual":
	94	+ return [r for r in rows if int(r["fiscal_quarter"]) == 0]
	95	+ return [r for r in rows if int(r["fiscal_quarter"]) in (1, 2, 3, 4)]
	96	+
	97	+
	98	+def _project(rows: list[dict], statement: str) -> pd.DataFrame:
	99	+ cols = list(BASE_COLUMNS)
	100	+ if statement == "all":
	101	+ acc = [a.name for a in M.ACCOUNTS if not a.auxiliary]
	102	+ else:
	103	+ acc = [a.name for a in M.accounts_for(statement) if not a.auxiliary]
	104	+ cols += acc
	105	+ if rows and rows[0].get("ttm"):
	106	+ cols.insert(cols.index("derived"), "ttm")
	107	+ cols.append("coverage")
	108	+ out = []
	109	+ for r in rows:
	110	+ cov = r.get("coverage") or {}
	111	+ # only the reasons for nulls + provenance flags of computed/derived values are served
	112	+ slim = {k: v for k, v in cov.items() if k in acc and isinstance(v, dict)
	113	+ and ("reason" in v or "derived" in v or "computed" in v or "components" in v or "approx" in v or "currency" in v)}
	114	+ out.append({**{c: r.get(c) for c in cols if c != "coverage"}, "coverage": slim})
	115	+ return pd.DataFrame(out, columns=cols)
	116	+
	117	+
	118	+# ------------------------------------------------------------------------------------------ statements
	119	+def statements(ticker: str, *, statement: str = "all", period: str = "quarterly", date_from: date \| None = None,
	120	+ date_to: date \| None = None, as_of: date \| None = None, view: str = "standardized") -> tuple[pd.DataFrame, dict]:
	121	+ c = resolve_company(ticker)
	122	+ stmts = M.STATEMENTS if statement == "all" else [statement]
	123	+ rows = company_rows(c.cik, as_of, list(stmts))
	124	+ if not rows:
	125	+ raise ApiError(404, "FUNDAMENTALS_NOT_AVAILABLE", f"No standardized statements for '{c.ticker}'"
	126	+ + (f" as of {as_of}" if as_of else ""), details={"ticker": c.ticker, "cik": c.cik})
	127	+ if period == "ttm":
	128	+ out: list[dict] = []
	129	+ for st in stmts:
	130	+ out += ttm(rows, st)
	131	+ else:
	132	+ out = _filter_period(rows, period)
	133	+ if date_from:
	134	+ out = [r for r in out if _d(r["period_end"]) >= date_from]
	135	+ if date_to:
	136	+ out = [r for r in out if _d(r["period_end"]) <= date_to]
	137	+ out.sort(key=lambda r: (_d(r["period_end"]), r["statement"], int(r["fiscal_quarter"])), reverse=True)
	138	+ meta = {"ticker": c.ticker, "cik": c.cik, "name": c.name, "statement": statement, "period": period, "view": view,
	139	+ "as_of": as_of.isoformat() if as_of else None, "mapping_version": M.MAPPING_VERSION,
	140	+ "units": {"USD": "raw (not thousands)", "shares": "raw", "eps": "USD/share"},
	141	+ "sign_convention": M.SIGN_CONVENTION}
	142	+ if view == "as_reported":
	143	+ return as_reported(c, out, as_of), {**meta, "note": "as_reported = the XBRL facts behind each standardized row"}
	144	+ return _project(out, statement), meta
	145	+
	146	+
	147	+def as_reported(c: EdgarCompany, rows: list[dict], as_of: date \| None) -> pd.DataFrame:
	148	+ """The raw facts (tag, value, unit, accn) behind each standardized row — from the Parquet facts lake."""
	149	+ facts = facts_lake(c.cik, as_of=as_of)
	150	+ if facts.empty or not rows:
	151	+ return pd.DataFrame(columns=["ticker", "statement", "fiscal_year", "fiscal_quarter", "period_end", "account",
	152	+ "taxonomy", "tag", "unit", "val", "accn", "filed"])
	153	+ out = []
	154	+ for r in rows:
	155	+ for acc, cov in (r.get("coverage") or {}).items():
	156	+ if not isinstance(cov, dict) or "tag" not in cov or cov.get("components"):
	157	+ continue
	158	+ tax, tag = cov["tag"].split(":", 1) if ":" in cov["tag"] else ("us-gaap", cov["tag"])
	159	+ sub = facts[(facts["tag"] == tag) & (facts["taxonomy"] == tax) & (facts["accn"] == r["accn"]) &
	160	+ (facts["end"] == pd.Timestamp(r["period_end"]))]
	161	+ for f in sub.itertuples(index=False):
	162	+ out.append({"ticker": c.ticker, "statement": r["statement"], "fiscal_year": r["fiscal_year"],
	163	+ "fiscal_quarter": r["fiscal_quarter"], "period_end": r["period_end"], "account": acc,
	164	+ "taxonomy": f.taxonomy, "tag": f.tag, "unit": f.unit, "val": f.val, "accn": f.accn, "filed": f.filed})
	165	+ return pd.DataFrame(out)
	166	+
	167	+
	168	+# ------------------------------------------------------------------------------------------ facts lake
	169	+def facts_path(cik: int) -> str:
	170	+ from core.config import settings
	171	+ return str(settings.data_root / "edgar" / "facts" / f"cik={int(cik)}" / "facts.parquet")
	172	+
	173	+
	174	+def facts_lake(cik: int, *, taxonomy: str \| None = None, tag: str \| None = None, as_of: date \| None = None,
	175	+ date_from: date \| None = None, date_to: date \| None = None) -> pd.DataFrame:
	176	+ import os
	177	+ p = facts_path(cik)
	178	+ if not os.path.isfile(p):
	179	+ return pd.DataFrame()
	180	+ conds, params = [], [p]
	181	+ if taxonomy:
	182	+ conds.append("taxonomy = ?"); params.append(taxonomy)
	183	+ if tag:
	184	+ conds.append("tag = ?"); params.append(tag)
	185	+ if as_of:
	186	+ conds.append("filed <= ?"); params.append(as_of)
	187	+ if date_from:
	188	+ conds.append("\"end\" >= ?"); params.append(date_from)
	189	+ if date_to:
	190	+ conds.append("\"end\" <= ?"); params.append(date_to)
	191	+ where = ("WHERE " + " AND ".join(conds)) if conds else ""
	192	+ return duck.con().execute(f"SELECT * FROM read_parquet(?) {where} ORDER BY \"end\", filed", params).df()
	193	+
	194	+
	195	+def facts_for_concept(ticker: str, concept: str, *, as_of: date \| None, date_from: date \| None,
	196	+ date_to: date \| None) -> tuple[pd.DataFrame, dict]:
	197	+ """`concept` = a standardized account (→ point-in-time standardized series) or a raw XBRL tag
	198	+ (`us-gaap:Revenues`, `Revenues`, `dei:EntityCommonStockSharesOutstanding`) → raw facts."""
	199	+ c = resolve_company(ticker)
	200	+ name = concept.strip()
	201	+ if name in M.ACCOUNT_BY_NAME:
	202	+ acc = M.ACCOUNT_BY_NAME[name]
	203	+ rows = company_rows(c.cik, as_of, [acc.statement])
	204	+ if date_from:
	205	+ rows = [r for r in rows if _d(r["period_end"]) >= date_from]
	206	+ if date_to:
	207	+ rows = [r for r in rows if _d(r["period_end"]) <= date_to]
	208	+ df = pd.DataFrame([{"ticker": c.ticker, "concept": name, "fiscal_year": r["fiscal_year"],
	209	+ "fiscal_quarter": r["fiscal_quarter"], "period_start": r["period_start"], "period_end": r["period_end"],
	210	+ "calendar_quarter": r["calendar_quarter"], "value": r.get(name), "unit": acc.unit if acc.unit != "USD" else r.get("currency", "USD"),
	211	+ "derived": r["derived"], "restated": r["restated"], "form": r["form"], "accn": r["accn"],
	212	+ "filed_date": r["filed_date"],
	213	+ "coverage": (r.get("coverage") or {}).get(name)} for r in rows])
	214	+ if not df.empty:
	215	+ df = df.sort_values(["period_end", "fiscal_quarter"], ascending=[False, True])
	216	+ meta = {"ticker": c.ticker, "cik": c.cik, "concept": name, "kind": "standardized", "statement": acc.statement,
	217	+ "tags": [f"{t.taxonomy}:{t.tag}" for t in acc.tags], "computed": acc.computed, "formula": acc.formula or None,
	218	+ "as_of": as_of.isoformat() if as_of else None}
	219	+ return df, meta
	220	+ tax, tag = name.split(":", 1) if ":" in name else ("us-gaap", name)
	221	+ df = facts_lake(c.cik, taxonomy=tax, tag=tag, as_of=as_of, date_from=date_from, date_to=date_to)
	222	+ if df.empty:
	223	+ exists = not facts_lake(c.cik, taxonomy=tax, tag=tag).empty
	224	+ if not exists:
	225	+ raise ApiError(404, "CONCEPT_NOT_FOUND", f"'{concept}' is neither a standardized account nor an XBRL concept "
	226	+ f"reported by {c.ticker}.", details={"ticker": c.ticker, "hint": "see /v1/fundamentals/{ticker}/coverage"})
	227	+ df = pd.DataFrame(columns=["ticker", "taxonomy", "tag", "unit", "fy", "fp", "form", "start", "end", "val", "accn", "filed", "frame"])
	228	+ else:
	229	+ df = df.drop(columns=["cik"]).sort_values(["end", "filed"], ascending=[False, False])
	230	+ df.insert(0, "ticker", c.ticker)
	231	+ meta = {"ticker": c.ticker, "cik": c.cik, "concept": f"{tax}:{tag}", "kind": "xbrl_fact",
	232	+ "note": "fy/fp describe the filing, not the fact's own period; `frame` marks the canonical calendar period",
	233	+ "as_of": as_of.isoformat() if as_of else None}
	234	+ return df, meta
	235	+
	236	+
	237	+# ------------------------------------------------------------------------------------------ ratios
	238	+def _latest_quarter(rows: list[dict], statement: str) -> dict \| None:
	239	+ q = [r for r in rows if r["statement"] == statement and int(r["fiscal_quarter"]) in (1, 2, 3, 4)]
	240	+ return max(q, key=lambda r: (_d(r["period_end"]), _d(r["filed_date"]))) if q else None
	241	+
	242	+
	243	+def _annuals(rows: list[dict], statement: str) -> list[dict]:
	244	+ return sorted([r for r in rows if r["statement"] == statement and int(r["fiscal_quarter"]) == 0],
	245	+ key=lambda r: int(r["fiscal_year"]))
	246	+
	247	+
	248	+def ratio_inputs(rows: list[dict], *, price: float \| None, period: str = "ttm") -> tuple[dict[str, Any], dict[str, Any]]:
	249	+ """Build the flat input dict of ratios.compute_all from as-of selected rows (see ratios.py header)."""
	250	+ f: dict[str, Any] = {"price": price}
	251	+ info: dict[str, Any] = {"period": period}
	252	+ if period == "annual":
	253	+ inc, cf, bal = _annuals(rows, M.INCOME), _annuals(rows, M.CASHFLOW), _annuals(rows, M.BALANCE)
	254	+ cur_inc, cur_cf, cur_bal = (inc[-1] if inc else None), (cf[-1] if cf else None), (bal[-1] if bal else None)
	255	+ def back(lst, n): # noqa: E306
	256	+ return lst[-1 - n] if len(lst) > n else None
	257	+ prev_inc, prev_cf = back(inc, 1), back(cf, 1)
	258	+ inc3, inc5, inc10 = back(inc, 3), back(inc, 5), back(inc, 10)
	259	+ else:
	260	+ inc_t, cf_t = ttm(rows, M.INCOME), ttm(rows, M.CASHFLOW)
	261	+ cur_inc, cur_cf = (inc_t[-1] if inc_t else None), (cf_t[-1] if cf_t else None)
	262	+ cur_bal = _latest_quarter(rows, M.BALANCE)
	263	+ def back(lst, n): # noqa: E306
	264	+ return lst[-1 - n] if len(lst) > n else None
	265	+ prev_inc, prev_cf = back(inc_t, 4), back(cf_t, 4)
	266	+ inc3, inc5, inc10 = back(inc_t, 12), back(inc_t, 20), back(inc_t, 40)
	267	+ lq = _latest_quarter(rows, M.INCOME)
	268	+ if lq is not None:
	269	+ f["revenue_q"] = lq.get("revenue")
	270	+ qs = sorted([r for r in rows if r["statement"] == M.INCOME and int(r["fiscal_quarter"]) in (1, 2, 3, 4)],
	271	+ key=lambda r: _d(r["period_end"]))
	272	+ f["revenue_prev_quarter"] = qs[-2].get("revenue") if len(qs) > 1 else None
	273	+ for src in (cur_inc, cur_cf, cur_bal):
	274	+ if src:
	275	+ for a in M.ACCOUNTS:
	276	+ if a.statement == src["statement"]:
	277	+ f[a.name] = src.get(a.name)
	278	+ if cur_inc:
	279	+ info["period_end"] = cur_inc["period_end"]
	280	+ info["fiscal_year"], info["fiscal_quarter"] = cur_inc["fiscal_year"], cur_inc["fiscal_quarter"]
	281	+ info["filed_date"] = max(_d(x["filed_date"]) for x in (cur_inc, cur_cf, cur_bal) if x)
	282	+ elif cur_bal:
	283	+ info["period_end"], info["fiscal_year"], info["fiscal_quarter"] = cur_bal["period_end"], cur_bal["fiscal_year"], cur_bal["fiscal_quarter"]
	284	+ info["filed_date"] = _d(cur_bal["filed_date"])
	285	+ if cur_bal:
	286	+ info["balance_period_end"] = cur_bal["period_end"]
	287	+ f["revenue_prev_year"] = prev_inc.get("revenue") if prev_inc else None
	288	+ f["eps_diluted_prev_year"] = prev_inc.get("eps_diluted") if prev_inc else None
	289	+ f["free_cash_flow_prev_year"] = prev_cf.get("free_cash_flow") if prev_cf else None
	290	+ f["revenue_3y"] = inc3.get("revenue") if inc3 else None
	291	+ f["revenue_5y"] = inc5.get("revenue") if inc5 else None
	292	+ f["revenue_10y"] = inc10.get("revenue") if inc10 else None
	293	+ f["eps_diluted_5y"] = inc5.get("eps_diluted") if inc5 else None
	294	+ # shares for market cap: cover-page shares, else weighted diluted (flagged)
	295	+ if f.get("shares_outstanding") is not None:
	296	+ info["shares_source"] = "dei:EntityCommonStockSharesOutstanding"
	297	+ elif f.get("shares_diluted") is not None:
	298	+ f["shares_outstanding"] = f["shares_diluted"]
	299	+ info["shares_source"] = "weighted_diluted"
	300	+ else:
	301	+ info["shares_source"] = None
	302	+ f["forward_eps"] = None
	303	+ return f, info
	304	+
	305	+
	306	+def ratios(ticker: str, *, as_of: date \| None = None, period: str = "ttm") -> tuple[dict[str, Any], dict[str, Any]]:
	307	+ c = resolve_company(ticker)
	308	+ rows = company_rows(c.cik, as_of)
	309	+ if not rows:
	310	+ raise ApiError(404, "FUNDAMENTALS_NOT_AVAILABLE", f"No fundamentals for '{c.ticker}'"
	311	+ + (f" as of {as_of}" if as_of else ""), details={"ticker": c.ticker})
	312	+ px = prices.close_at(c.ticker, as_of)
	313	+ f, info = ratio_inputs(rows, price=px[0] if px else None, period=period)
	314	+ values, reasons = compute_all(f)
	315	+ if px is None:
	316	+ reasons.setdefault("price", "no_price_in_lake")
	317	+ groups: dict[str, dict[str, float \| None]] = {}
	318	+ from .ratios import RATIOS
	319	+ for r in RATIOS:
	320	+ groups.setdefault(r.group, {})[r.name] = values[r.name]
	321	+ data = {"ticker": c.ticker, "cik": c.cik, "as_of": as_of.isoformat() if as_of else None,
	322	+ "price": px[0] if px else None, "price_date": px[1].isoformat() if px else None,
	323	+ "fundamentals_period_end": _iso(info.get("period_end")), "fiscal_year": info.get("fiscal_year"),
	324	+ "fiscal_quarter": info.get("fiscal_quarter"), "filed_date": _iso(info.get("filed_date")),
	325	+ "inputs": {k: f.get(k) for k in ("revenue", "net_income", "eps_diluted", "ebitda", "free_cash_flow", "total_assets",
	326	+ "total_equity", "total_debt", "cash_and_equivalents", "shares_outstanding")},
	327	+ **groups}
	328	+ pf = prices.price_file(c.ticker)
	329	+ meta = {"period": period, "shares_source": info.get("shares_source"), "price_source": pf[1] if pf else None,
	330	+ "reasons": reasons, "formulas": "https://www.hfmarketdata.io/docs/fundamentals-ratios"}
	331	+ return data, meta
	332	+
	333	+
	334	+def _iso(v: Any) -> str \| None:
	335	+ d = _d(v)
	336	+ return d.isoformat() if d else None
	337	+
	338	+
	339	+def ratios_daily(ticker: str, *, date_from: date \| None, date_to: date \| None, fields: list[str] \| None = None) -> tuple[pd.DataFrame, dict]:
	340	+ """Daily close × the fundamentals known at each date (point-in-time): snapshots are built at every
	341	+ filed_date of the company and ASOF-joined to the price series in DuckDB."""
	342	+ c = resolve_company(ticker)
	343	+ all_rows = _rows(c.cik)
	344	+ if not all_rows:
	345	+ raise ApiError(404, "FUNDAMENTALS_NOT_AVAILABLE", f"No fundamentals for '{c.ticker}'", details={"ticker": c.ticker})
	346	+ px = prices.closes(c.ticker, date_from, date_to)
	347	+ if px.empty:
	348	+ raise ApiError(404, "TICKER_NOT_FOUND", f"No daily prices for '{c.ticker}' in the requested range", details={"ticker": c.ticker})
	349	+ px["date"] = pd.to_datetime(px["date"]).dt.date
	350	+ filed_dates = sorted({_d(r["filed_date"]) for r in all_rows})
	351	+ lo = px["date"].min()
	352	+ # snapshots: one per filed_date ≤ last price date, plus the state just before the window
	353	+ snaps = []
	354	+ relevant = [d for d in filed_dates if d <= px["date"].max()]
	355	+ before = [d for d in relevant if d <= lo]
	356	+ starts = ([before[-1]] if before else []) + [d for d in relevant if d > lo]
	357	+ for d in starts:
	358	+ rows = select_as_of(all_rows, d)
	359	+ f, info = ratio_inputs(rows, price=None)
	360	+ snap = {k: f.get(k) for k in f if k != "price"}
	361	+ snap["valid_from"] = d
	362	+ snap["fundamentals_period_end"] = _d(info.get("period_end"))
	363	+ snaps.append(snap)
	364	+ if not snaps:
	365	+ raise ApiError(404, "FUNDAMENTALS_NOT_AVAILABLE", f"No fundamentals filed before {px['date'].max()} for '{c.ticker}'")
	366	+ sdf = pd.DataFrame(snaps)
	367	+ con = duck.con()
	368	+ con.register("px_daily", px)
	369	+ con.register("fund_snaps", sdf)
	370	+ joined = con.execute("SELECT p.date, p.close, s.* FROM px_daily p ASOF JOIN fund_snaps s ON p.date >= s.valid_from "
	371	+ "ORDER BY p.date").df()
	372	+ con.unregister("px_daily"); con.unregister("fund_snaps")
	373	+ want = fields or list(RATIO_NAMES)
	374	+ bad = [w for w in want if w not in RATIO_NAMES]
	375	+ if bad:
	376	+ raise ApiError(400, "INVALID_PARAMETER", f"unknown ratio field(s): {', '.join(bad)}", details={"available": list(RATIO_NAMES)})
	377	+ out_rows = []
	378	+ for rec in joined.to_dict(orient="records"):
	379	+ f = {k: (None if (isinstance(v, float) and pd.isna(v)) else v) for k, v in rec.items()}
	380	+ f["price"] = f.pop("close")
	381	+ values, _ = compute_all(f)
	382	+ out_rows.append({"date": rec["date"], "close": f["price"], "fundamentals_as_of": f.get("valid_from"),
	383	+ "fundamentals_period_end": f.get("fundamentals_period_end"), **{w: values[w] for w in want}})
	384	+ df = pd.DataFrame(out_rows)
	385	+ pf = prices.price_file(c.ticker)
	386	+ meta = {"ticker": c.ticker, "cik": c.cik, "price_source": pf[1] if pf else None, "point_in_time": True,
	387	+ "snapshots": len(snaps), "fields": want}
	388	+ return df, meta
	389	+
	390	+
	391	+# ------------------------------------------------------------------------------------------ filings / coverage
	392	+def filings(ticker: str, *, form: str \| None, date_from: date \| None, date_to: date \| None) -> tuple[pd.DataFrame, dict]:
	393	+ from .edgar_client import filing_index_url, primary_doc_url
	394	+ c = resolve_company(ticker)
	395	+ q = select(EdgarFiling).where(EdgarFiling.cik == c.cik)
	396	+ if form:
	397	+ forms = [x.strip().upper() for x in form.split(",") if x.strip()]
	398	+ q = q.where(EdgarFiling.form.in_(forms))
	399	+ if date_from:
	400	+ q = q.where(EdgarFiling.filed_date >= date_from)
	401	+ if date_to:
	402	+ q = q.where(EdgarFiling.filed_date <= date_to)
	403	+ q = q.order_by(EdgarFiling.filed_date.desc(), EdgarFiling.accn.desc())
	404	+ with session() as s:
	405	+ rows = [{"ticker": c.ticker, "cik": c.cik, "accn": f.accn, "form": f.form, "filed_date": f.filed_date,
	406	+ "period_of_report": f.period_of_report, "is_amendment": f.is_amendment, "is_xbrl": f.is_xbrl,
	407	+ "primary_doc_url": f.primary_doc if (f.primary_doc or "").startswith("http") else primary_doc_url(c.cik, f.accn, f.primary_doc),
	408	+ "index_url": filing_index_url(c.cik, f.accn)}
	409	+ for f in s.scalars(q)]
	410	+ return pd.DataFrame(rows, columns=["ticker", "cik", "accn", "form", "filed_date", "period_of_report", "is_amendment",
	411	+ "is_xbrl", "primary_doc_url", "index_url"]), {"ticker": c.ticker, "cik": c.cik}
	412	+
	413	+
	414	+def coverage(ticker: str) -> dict[str, Any]:
	415	+ c = resolve_company(ticker)
	416	+ with session() as s:
	417	+ cov = s.get(FundCoverage, c.ticker)
	418	+ logs = s.scalars(select(FundMappingLog).where(FundMappingLog.cik == c.cik, FundMappingLog.is_extension.is_(True))
	419	+ .order_by(FundMappingLog.tag)).all()
	420	+ data: dict[str, Any] = {"ticker": c.ticker, "cik": c.cik, "name": c.name, "tickers": c.tickers, "status": c.status,
	421	+ "sic": c.sic, "sic_description": c.sic_description, "exchange": c.exchange,
	422	+ "fiscal_year_end": c.fiscal_year_end, "ticker_history": c.ticker_history,
	423	+ "mapping_version": M.MAPPING_VERSION}
	424	+ if cov is None:
	425	+ data.update({"periods": None, "completeness": None, "note": "not normalized yet"})
	426	+ return data
	427	+ data.update({"first_period_end": _iso(cov.first_period_end), "last_period_end": _iso(cov.last_period_end),
	428	+ "quarters": cov.quarters, "annuals": cov.annuals, "filings": cov.filings, "completeness": cov.completeness,
	429	+ "completeness_by_statement": cov.completeness_by_statement, "missing_accounts": cov.missing_accounts,
	430	+ "gaps": cov.gaps, "derived_quarters": cov.derived_quarters, "restated_periods": cov.restated_periods,
	431	+ "last_filed_date": _iso(cov.last_filed_date),
	432	+ "custom_extensions": [{"tag": f"{l.taxonomy}:{l.tag}", "hint_account": l.hint_account} for l in logs],
	433	+ "updated_at": cov.updated_at.isoformat() + "Z" if cov.updated_at else None})
	434	+ return data
	435	+
	436	+
	437	+# ------------------------------------------------------------------------------------------ frames
	438	+def frames(concept: str, *, calendar_quarter: str \| None, fiscal_year: int \| None, fiscal_quarter: int \| None,
	439	+ as_of: date \| None) -> tuple[pd.DataFrame, dict]:
	440	+ """Cross-section of one standardized account across the universe for a calendar quarter (`2024Q1`) or a
	441	+ fiscal period (`fiscal_year=2024&fiscal_quarter=2`, quarter 0 = annual)."""
	442	+ if concept not in M.ACCOUNT_BY_NAME:
	443	+ raise ApiError(404, "CONCEPT_NOT_FOUND", f"'{concept}' is not a standardized account",
	444	+ details={"available": list(M.PUBLIC_ACCOUNTS)})
	445	+ acc = M.ACCOUNT_BY_NAME[concept]
	446	+ t = fund_statements
	447	+ conds = [t.c.statement == acc.statement]
	448	+ if calendar_quarter:
	449	+ cq = calendar_quarter.upper().replace("-", "")
	450	+ if len(cq) != 6 or cq[4] != "Q":
	451	+ raise ApiError(400, "INVALID_PARAMETER", "calendar_quarter must look like 2024Q1")
	452	+ conds += [t.c.calendar_quarter == cq, t.c.fiscal_quarter.in_([1, 2, 3, 4])]
	453	+ elif fiscal_year is not None:
	454	+ conds.append(t.c.fiscal_year == fiscal_year)
	455	+ conds.append(t.c.fiscal_quarter == (fiscal_quarter if fiscal_quarter is not None else 0))
	456	+ else:
	457	+ raise ApiError(400, "INVALID_PARAMETER", "give calendar_quarter=2024Q1 or fiscal_year (+ fiscal_quarter)")
	458	+ if as_of:
	459	+ conds.append(t.c.filed_date <= as_of)
	460	+ q = select(t.c.cik, t.c.ticker, t.c.fiscal_year, t.c.fiscal_quarter, t.c.period_end, t.c.calendar_quarter, t.c.filed_date,
	461	+ t.c.accn, t.c.derived, t.c.restated, t.c.currency, t.c[concept].label("value"),
	462	+ func.row_number().over(partition_by=[t.c.cik, t.c.fiscal_year, t.c.fiscal_quarter],
	463	+ order_by=[t.c.filed_date.desc(), t.c.id.desc()]).label("rn")).where(and_(*conds))
	464	+ sub = q.subquery()
	465	+ outer = select(sub).where(sub.c.rn == 1).order_by(sub.c.value.desc().nulls_last(), sub.c.ticker)
	466	+ with session() as s:
	467	+ df = pd.DataFrame([dict(r._mapping) for r in s.execute(outer)])
	468	+ if not df.empty:
	469	+ df = df.drop(columns=["rn"])
	470	+ df.insert(2, "concept", concept)
	471	+ meta = {"concept": concept, "statement": acc.statement, "unit": acc.unit, "calendar_quarter": calendar_quarter,
	472	+ "fiscal_year": fiscal_year, "fiscal_quarter": fiscal_quarter, "as_of": as_of.isoformat() if as_of else None,
	473	+ "note": "one row per company: latest version known at as_of; value null = not reported (see coverage)"}
	474	+ return df, meta
	475	+
	476	+
	477	+def health() -> dict[str, Any]:
	478	+ from .models import FundIngestState
	479	+ with session() as s:
	480	+ states = {st.key: {"last_run_at": _ts(st.last_run_at), "last_success_at": _ts(st.last_success_at),
	481	+ "last_rss_check_at": _ts(st.last_rss_check_at), "lag_seconds": st.lag_seconds,
	482	+ "companies_total": st.companies_total, "companies_done": st.companies_done, "failures": st.failures,
	483	+ "mapping_failure_rate": st.mapping_failure_rate, "requests_made": st.requests_made,
	484	+ "events_published": st.events_published, "extra": st.extra}
	485	+ for st in s.scalars(select(FundIngestState))}
	486	+ companies = s.scalar(select(func.count()).select_from(EdgarCompany)) or 0
	487	+ statements_n = s.scalar(select(func.count()).select_from(fund_statements)) or 0
	488	+ latest_n = s.scalar(select(func.count()).select_from(fund_latest)) or 0
	489	+ last_filed = s.scalar(select(func.max(fund_statements.c.filed_date)))
	490	+ return {"companies": companies, "statement_versions": statements_n, "screener_rows": latest_n,
	491	+ "last_filed_date": _iso(last_filed), "mapping_version": M.MAPPING_VERSION, "jobs": states}
	492	+
	493	+
	494	+def _ts(v) -> str \| None:
	495	+ return (v.isoformat() + "Z") if v else None
	496	+
	497	+
	498	+__all__ = ["statements", "facts_for_concept", "ratios", "ratios_daily", "filings", "coverage", "frames", "health",
	499	+ "ratio_inputs", "resolve_company", "company_rows", "parse_statement", "parse_period", "parse_date"]
	500