fundamentals: client EDGAR poli (token bucket 10 req/s, backoff 429/503, cache gzip) + moteur de normalisation

- edgar_client.py : companyfacts/submissions/MetaLinks, flux Atom getcurrent, index quotidien, efts ; jamais proxifié aux utilisateurs
- normalize.py : calendrier fiscal (fy/fp = le dépôt, pas le fait ; snap 52/53 semaines), résolution par priorité, sommes de composantes, identités comptables, versions par filed_date (restatements), dé-cumul YTD (Q2/Q3/Q4 dérivés), comptes calculés, TTM, select_as_of anti look-ahead, journal des extensions

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 20 days ago (Sep 5, 2026) parent cbb707c

2 changed files +1,134 −0

added hfmarketdata/api/fundamentals/edgar_client.py +259 −0

		`@@ -0,0 +1,259 @@`
	1	+"""Polite SEC EDGAR client — token bucket (≤ 10 req/s), exponential backoff, gzip disk cache.
	2	+
	3	+SEC fair-access rules: declare a real User-Agent (`settings.sec_user_agent`), stay under 10 requests per
	4	+second, back off on 429/503. Every JSON document fetched is cached under `data_root/edgar/raw/` so a
	5	+re-run never hits EDGAR twice for the same content and normalisation can be replayed offline.
	6	+
	7	+EDGAR is never proxied live to API users: the client is only used by the ingestion jobs.
	8	+
	9	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	10	+"""
	11	+from __future__ import annotations
	12	+
	13	+import gzip
	14	+import json
	15	+import logging
	16	+import random
	17	+import threading
	18	+import time
	19	+from dataclasses import dataclass, field
	20	+from datetime import date
	21	+from pathlib import Path
	22	+from typing import Any
	23	+
	24	+import httpx
	25	+
	26	+from core.config import settings
	27	+
	28	+log = logging.getLogger("hfmarketdata.edgar")
	29	+
	30	+DATA_BASE = "https://data.sec.gov"
	31	+WWW_BASE = "https://www.sec.gov"
	32	+EFTS_BASE = "https://efts.sec.gov"
	33	+
	34	+URL_COMPANY_TICKERS = f"{WWW_BASE}/files/company_tickers.json"
	35	+URL_COMPANY_TICKERS_EXCHANGE = f"{WWW_BASE}/files/company_tickers_exchange.json"
	36	+URL_ATOM_CURRENT = WWW_BASE + "/cgi-bin/browse-edgar?action=getcurrent&type={form}&owner=include&count=100&output=atom"
	37	+
	38	+TRACKED_FORMS = ("10-K", "10-Q", "8-K", "20-F", "10-K/A", "10-Q/A", "20-F/A", "40-F", "6-K")
	39	+FINANCIAL_FORMS = ("10-K", "10-Q", "20-F", "40-F", "10-K/A", "10-Q/A", "20-F/A", "10-KT", "10-QT")
	40	+
	41	+
	42	+def cik10(cik: int \| str) -> str:
	43	+ return f"{int(cik):010d}"
	44	+
	45	+
	46	+def accn_nodash(accn: str) -> str:
	47	+ return accn.replace("-", "")
	48	+
	49	+
	50	+def filing_index_url(cik: int, accn: str) -> str:
	51	+ return f"{WWW_BASE}/Archives/edgar/data/{int(cik)}/{accn_nodash(accn)}/{accn}-index.htm"
	52	+
	53	+
	54	+def primary_doc_url(cik: int, accn: str, doc: str \| None) -> str:
	55	+ if not doc:
	56	+ return filing_index_url(cik, accn)
	57	+ return f"{WWW_BASE}/Archives/edgar/data/{int(cik)}/{accn_nodash(accn)}/{doc}"
	58	+
	59	+
	60	+def metalinks_url(cik: int, accn: str) -> str:
	61	+ return f"{WWW_BASE}/Archives/edgar/data/{int(cik)}/{accn_nodash(accn)}/MetaLinks.json"
	62	+
	63	+
	64	+class TokenBucket:
	65	+ """Thread-safe token bucket: `rate` tokens per second, burst `capacity`."""
	66	+
	67	+ def __init__(self, rate: float = 10.0, capacity: int = 10):
	68	+ self.rate, self.capacity = rate, capacity
	69	+ self._tokens = float(capacity)
	70	+ self._ts = time.monotonic()
	71	+ self._lock = threading.Lock()
	72	+
	73	+ def acquire(self) -> None:
	74	+ while True:
	75	+ with self._lock:
	76	+ now = time.monotonic()
	77	+ self._tokens = min(self.capacity, self._tokens + (now - self._ts) * self.rate)
	78	+ self._ts = now
	79	+ if self._tokens >= 1:
	80	+ self._tokens -= 1
	81	+ return
	82	+ wait = (1 - self._tokens) / self.rate
	83	+ time.sleep(wait)
	84	+
	85	+
	86	+@dataclass
	87	+class ClientStats:
	88	+ requests: int = 0
	89	+ cache_hits: int = 0
	90	+ retries: int = 0
	91	+ bytes: int = 0
	92	+ errors: list[str] = field(default_factory=list)
	93	+
	94	+
	95	+class EdgarClient:
	96	+ """Sync client used by the ingestion scripts (one instance per process)."""
	97	+
	98	+ def __init__(self, *, rate: float = 10.0, raw_dir: Path \| None = None, timeout: float = 60.0,
	99	+ max_retries: int = 6, http: httpx.Client \| None = None):
	100	+ self.bucket = TokenBucket(rate=rate, capacity=int(rate))
	101	+ self.raw_dir = raw_dir or (settings.data_root / "edgar" / "raw")
	102	+ self.max_retries = max_retries
	103	+ self.stats = ClientStats()
	104	+ self._http = http or httpx.Client(timeout=timeout, follow_redirects=True, headers={
	105	+ "User-Agent": settings.sec_user_agent, "Accept-Encoding": "gzip, deflate", "Accept": "application/json,/"})
	106	+
	107	+ # ------------------------------------------------------------------------------------ low level
	108	+ def _request(self, url: str, *, accept_404: bool = False) -> httpx.Response \| None:
	109	+ delay = 0.5
	110	+ for attempt in range(self.max_retries + 1):
	111	+ self.bucket.acquire()
	112	+ self.stats.requests += 1
	113	+ try:
	114	+ r = self._http.get(url)
	115	+ except httpx.HTTPError as e:
	116	+ if attempt == self.max_retries:
	117	+ self.stats.errors.append(f"{url}: {e}")
	118	+ raise
	119	+ self.stats.retries += 1
	120	+ time.sleep(delay + random.uniform(0, 0.25))
	121	+ delay = min(delay * 2, 30)
	122	+ continue
	123	+ if r.status_code == 404 and accept_404:
	124	+ return None
	125	+ if r.status_code in (429, 503, 500, 502, 504):
	126	+ if attempt == self.max_retries:
	127	+ self.stats.errors.append(f"{url}: HTTP {r.status_code}")
	128	+ r.raise_for_status()
	129	+ retry_after = r.headers.get("Retry-After")
	130	+ wait = float(retry_after) if retry_after and retry_after.isdigit() else delay
	131	+ log.warning("EDGAR %s on %s — backing off %.1fs", r.status_code, url, wait)
	132	+ self.stats.retries += 1
	133	+ time.sleep(wait + random.uniform(0, 0.25))
	134	+ delay = min(delay * 2, 60)
	135	+ continue
	136	+ r.raise_for_status()
	137	+ self.stats.bytes += len(r.content)
	138	+ return r
	139	+ return None # pragma: no cover
	140	+
	141	+ def get_json(self, url: str, cache_path: Path \| None = None, *, max_age_hours: float \| None = None,
	142	+ accept_404: bool = False) -> Any \| None:
	143	+ """GET a JSON document; served from the gzip cache when present (and fresh when `max_age_hours`)."""
	144	+ if cache_path is not None and cache_path.exists():
	145	+ age_h = (time.time() - cache_path.stat().st_mtime) / 3600
	146	+ if max_age_hours is None or age_h <= max_age_hours:
	147	+ self.stats.cache_hits += 1
	148	+ with gzip.open(cache_path, "rt", encoding="utf-8") as fh:
	149	+ return json.load(fh)
	150	+ r = self._request(url, accept_404=accept_404)
	151	+ if r is None:
	152	+ return None
	153	+ data = r.json()
	154	+ if cache_path is not None:
	155	+ cache_path.parent.mkdir(parents=True, exist_ok=True)
	156	+ tmp = cache_path.with_suffix(cache_path.suffix + ".tmp")
	157	+ with gzip.open(tmp, "wt", encoding="utf-8", compresslevel=6) as fh:
	158	+ json.dump(data, fh, separators=(",", ":"))
	159	+ tmp.replace(cache_path)
	160	+ return data
	161	+
	162	+ def get_text(self, url: str) -> str:
	163	+ r = self._request(url)
	164	+ return r.text if r is not None else ""
	165	+
	166	+ # ------------------------------------------------------------------------------------ documents
	167	+ def company_tickers(self, max_age_hours: float = 24) -> dict[str, Any]:
	168	+ return self.get_json(URL_COMPANY_TICKERS, self.raw_dir / "company_tickers.json.gz", max_age_hours=max_age_hours)
	169	+
	170	+ def company_tickers_exchange(self, max_age_hours: float = 24) -> dict[str, Any]:
	171	+ return self.get_json(URL_COMPANY_TICKERS_EXCHANGE, self.raw_dir / "company_tickers_exchange.json.gz",
	172	+ max_age_hours=max_age_hours)
	173	+
	174	+ def companyfacts(self, cik: int, *, refresh: bool = False) -> dict[str, Any] \| None:
	175	+ p = self.raw_dir / "companyfacts" / f"CIK{cik10(cik)}.json.gz"
	176	+ if refresh and p.exists():
	177	+ p.unlink()
	178	+ return self.get_json(f"{DATA_BASE}/api/xbrl/companyfacts/CIK{cik10(cik)}.json", p, accept_404=True)
	179	+
	180	+ def submissions(self, cik: int, *, refresh: bool = False, include_older: bool = True) -> dict[str, Any] \| None:
	181	+ """Submissions JSON with the paginated older files merged into `filings.recent`."""
	182	+ p = self.raw_dir / "submissions" / f"CIK{cik10(cik)}.json.gz"
	183	+ if refresh and p.exists():
	184	+ p.unlink()
	185	+ data = self.get_json(f"{DATA_BASE}/submissions/CIK{cik10(cik)}.json", p, accept_404=True)
	186	+ if not data:
	187	+ return None
	188	+ if include_older:
	189	+ recent = data.get("filings", {}).get("recent", {})
	190	+ for extra in data.get("filings", {}).get("files", []) or []:
	191	+ name = extra.get("name")
	192	+ if not name:
	193	+ continue
	194	+ more = self.get_json(f"{DATA_BASE}/submissions/{name}", self.raw_dir / "submissions" / (name + ".gz"),
	195	+ accept_404=True)
	196	+ if more:
	197	+ for k, v in more.items():
	198	+ if isinstance(v, list) and k in recent:
	199	+ recent[k] = list(recent[k]) + v
	200	+ return data
	201	+
	202	+ def metalinks(self, cik: int, accn: str) -> dict[str, Any] \| None:
	203	+ p = self.raw_dir / "metalinks" / f"CIK{cik10(cik)}" / f"{accn_nodash(accn)}.json.gz"
	204	+ return self.get_json(metalinks_url(cik, accn), p, accept_404=True)
	205	+
	206	+ def search_filings(self, start: date, end: date, forms: tuple[str, ...] = TRACKED_FORMS, *,
	207	+ page_size: int = 100, max_pages: int = 20) -> list[dict[str, Any]]:
	208	+ """EDGAR full-text search index (efts) — every filing of the given forms filed in [start, end].
	209	+
	210	+ Returns dicts: {cik, ciks, form, filed, accn, period, file_date}. Never cached (it is the live feed)."""
	211	+ out: list[dict[str, Any]] = []
	212	+ forms_q = ",".join(forms)
	213	+ for page in range(max_pages):
	214	+ url = (f"{EFTS_BASE}/LATEST/search-index?q=%22*%22&dateRange=custom&startdt={start.isoformat()}"
	215	+ f"&enddt={end.isoformat()}&forms={forms_q}&from={page * page_size}&size={page_size}")
	216	+ data = self.get_json(url)
	217	+ hits = (data or {}).get("hits", {}).get("hits", [])
	218	+ for h in hits:
	219	+ src = h.get("_source", {})
	220	+ ciks = [int(c) for c in src.get("ciks", []) if str(c).isdigit()]
	221	+ accn = src.get("adsh") or (h.get("_id", "").split(":")[0])
	222	+ out.append({"cik": ciks[0] if ciks else None, "ciks": ciks, "form": src.get("form") or src.get("file_type"),
	223	+ "filed": src.get("file_date"), "accn": accn, "period": src.get("period_ending"),
	224	+ "names": src.get("display_names", [])})
	225	+ if len(hits) < page_size:
	226	+ break
	227	+ return out
	228	+
	229	+ def atom_current(self, form: str = "") -> str:
	230	+ """Fallback feed (Atom) of the latest filings on EDGAR, optionally filtered by form type."""
	231	+ return self.get_text(URL_ATOM_CURRENT.format(form=form))
	232	+
	233	+ def close(self) -> None:
	234	+ self._http.close()
	235	+
	236	+
	237	+def parse_atom(xml_text: str) -> list[dict[str, Any]]:
	238	+ """Minimal parser of the EDGAR `getcurrent` Atom feed → [{accn, cik, form, filed, title, link}]."""
	239	+ import re
	240	+ import xml.etree.ElementTree as ET
	241	+ out: list[dict[str, Any]] = []
	242	+ if not xml_text.strip():
	243	+ return out
	244	+ ns = {"a": "http://www.w3.org/2005/Atom"}
	245	+ try:
	246	+ root = ET.fromstring(xml_text)
	247	+ except ET.ParseError:
	248	+ return out
	249	+ for e in root.findall("a:entry", ns):
	250	+ title = e.findtext("a:title", default="", namespaces=ns)
	251	+ link = (e.find("a:link", ns).attrib.get("href") if e.find("a:link", ns) is not None else "") or ""
	252	+ updated = e.findtext("a:updated", default="", namespaces=ns)
	253	+ m_form = re.match(r"^\s*([0-9A-Z\-/]+)\s+-\s+", title)
	254	+ m_cik = re.search(r"$(\d{6,10})$", title)
	255	+ m_accn = re.search(r"(\d{10}-\d{2}-\d{6})", link)
	256	+ out.append({"accn": m_accn.group(1) if m_accn else None, "cik": int(m_cik.group(1)) if m_cik else None,
	257	+ "form": m_form.group(1) if m_form else None, "filed": updated[:10] if updated else None,
	258	+ "title": title, "link": link})
	259	+ return out

added hfmarketdata/api/fundamentals/normalize.py +875 −0

		`@@ -0,0 +1,875 @@`
	1	+"""XBRL facts → standardized, versioned statements.
	2	+
	3	+Pipeline for one company (`normalize_company`):
	4	+
	5	+1. Facts frame — `facts_frame()` flattens companyfacts JSON to rows (cik, taxonomy, tag, unit, fy, fp,
	6	+ form, start, end, val, accn, filed, frame), de-duplicated on (taxonomy, tag, unit, start, end, accn)
	7	+ keeping the latest `filed`.
	8	+2. Fiscal calendar — `FiscalCalendar` maps any period end to (fiscal_year, fiscal_quarter) from the
	9	+ 10-K report dates (+ `fiscalYearEnd` MMDD extrapolation, ±7 days snap for 52/53-week filers). The
	10	+ `fy`/`fp` fields of a fact describe the filing, not the fact's own period (a Q2 FY2025 10-Q re-reports
	11	+ Q2 FY2024 with fy=2025/fp=Q2) — so they are only used as a sanity check on the filing's own period.
	12	+3. Resolution — per filing (accn) and period, each standard account takes the first tag of its
	13	+ priority list that has a fact with a compatible unit; component sums (SG&A split, current debt pieces)
	14	+ and accounting identities are flagged in `coverage`.
	15	+4. Versioning — filings are replayed in `filed` order; a period gets a new version only when a
	16	+ filing changes (or completes) its values. `restated=True` when a previously served number changed.
	17	+ Serving "latest" = max filed_date per period; `as_of=D` = max filed_date ≤ D (no look-ahead).
	18	+5. Derivation — quarters missing as discrete facts are derived from year-to-date facts
	19	+ (Q2 = YTD6 − Q1, Q3 = YTD9 − YTD6, Q4 = FY − YTD9 or FY − (Q1+Q2+Q3)) only when all inputs belong to
	20	+ the same fiscal year, same unit and are known at that filing date; rows are flagged `derived=True`.
	21	+6. Computed accounts — ebitda, total_debt, net_debt, working_capital, free_cash_flow (and identity
	22	+ fallbacks for gross_profit / total_liabilities / operating_income) with their formula in `coverage`.
	23	+
	24	+`ttm()` and `select_as_of()` are shared with the service layer.
	25	+
	26	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	27	+"""
	28	+from __future__ import annotations
	29	+
	30	+import calendar
	31	+import logging
	32	+import math
	33	+import re
	34	+from collections import defaultdict
	35	+from dataclasses import dataclass, field
	36	+from datetime import date, timedelta
	37	+from typing import Any
	38	+
	39	+import pandas as pd
	40	+
	41	+from . import mapping as M
	42	+
	43	+log = logging.getLogger("hfmarketdata.fundamentals.normalize")
	44	+
	45	+FACT_COLUMNS = ["cik", "taxonomy", "tag", "unit", "fy", "fp", "form", "start", "end", "val", "accn", "filed", "frame"]
	46	+FORMS_10K = ("10-K", "10-K/A", "10-KT", "20-F", "20-F/A", "40-F", "40-F/A")
	47	+FORMS_10Q = ("10-Q", "10-Q/A", "10-QT", "6-K")
	48	+REL_TOL = 1e-6
	49	+
	50	+# spans (months) -> label
	51	+SPAN_Q, SPAN_H, SPAN_9M, SPAN_FY = "Q", "H", "9M", "FY"
	52	+
	53	+# component-sum fallbacks: account -> list of alternate groups (first present tag of each group is summed)
	54	+COMPONENT_SUMS: dict[str, list[list[str]]] = {
	55	+ "sga_expense": [["GeneralAndAdministrativeExpense"], ["SellingAndMarketingExpense", "SellingExpense"]],
	56	+ "short_term_debt": [["LongTermDebtAndCapitalLeaseObligationsCurrent", "LongTermDebtCurrent"], ["ShortTermBorrowings"],
	57	+ ["CommercialPaper"], ["NotesPayableCurrent"]],
	58	+}
	59	+# tags that unlock component sums only when the primary tag is absent
	60	+COMPONENT_PRIMARY: dict[str, tuple[str, ...]] = {
	61	+ "sga_expense": ("SellingGeneralAndAdministrativeExpense",),
	62	+ "short_term_debt": ("DebtCurrent",),
	63	+}
	64	+# helper (identity) tags, resolved like accounts but never stored
	65	+HELPERS: dict[str, tuple[str, str, str]] = { # name -> (taxonomy, tag, kind)
	66	+ "_costs_and_expenses": ("us-gaap", "CostsAndExpenses", M.DURATION),
	67	+ "_liabilities_and_equity": ("us-gaap", "LiabilitiesAndStockholdersEquity", M.INSTANT),
	68	+ "_operating_expenses": ("us-gaap", "OperatingExpenses", M.DURATION),
	69	+}
	70	+UNIT_CCY = re.compile(r"^[A-Z]{3}$")
	71	+UNIT_PER_SHARE = re.compile(r"^([A-Z]{3})/shares$")
	72	+
	73	+
	74	+# ------------------------------------------------------------------------------------------------ facts
	75	+def facts_frame(cf: dict[str, Any]) -> pd.DataFrame:
	76	+ """Flatten a companyfacts document into the raw facts table (see FACT_COLUMNS)."""
	77	+ cik = int(cf.get("cik", 0))
	78	+ rows: list[tuple] = []
	79	+ for taxonomy, tags in (cf.get("facts") or {}).items():
	80	+ for tag, body in tags.items():
	81	+ for unit, facts in (body.get("units") or {}).items():
	82	+ for f in facts:
	83	+ rows.append((cik, taxonomy, tag, unit, f.get("fy"), f.get("fp"), f.get("form"), f.get("start"),
	84	+ f.get("end"), f.get("val"), f.get("accn"), f.get("filed"), f.get("frame")))
	85	+ df = pd.DataFrame(rows, columns=FACT_COLUMNS)
	86	+ if df.empty:
	87	+ return df
	88	+ for c in ("start", "end", "filed"):
	89	+ df[c] = pd.to_datetime(df[c], errors="coerce").dt.date
	90	+ df["val"] = pd.to_numeric(df["val"], errors="coerce")
	91	+ df["fy"] = pd.to_numeric(df["fy"], errors="coerce").astype("Int64")
	92	+ df = df.dropna(subset=["end", "val", "accn"])
	93	+ # dedup: same fact reported twice in one filing → keep the latest filed
	94	+ df = df.sort_values("filed").drop_duplicates(["taxonomy", "tag", "unit", "start", "end", "accn"], keep="last")
	95	+ return df.reset_index(drop=True)
	96	+
	97	+
	98	+@dataclass
	99	+class Filing:
	100	+ accn: str
	101	+ form: str
	102	+ filed: date
	103	+ report_date: date \| None = None
	104	+ primary_doc: str \| None = None
	105	+ is_amendment: bool = False
	106	+ is_xbrl: bool = False
	107	+
	108	+
	109	+def filings_from_submissions(sub: dict[str, Any] \| None) -> dict[str, Filing]:
	110	+ out: dict[str, Filing] = {}
	111	+ if not sub:
	112	+ return out
	113	+ rec = (sub.get("filings") or {}).get("recent") or {}
	114	+ n = len(rec.get("accessionNumber", []))
	115	+ for i in range(n):
	116	+ form = rec["form"][i]
	117	+ accn = rec["accessionNumber"][i]
	118	+ filed = _d(rec["filingDate"][i])
	119	+ if filed is None:
	120	+ continue
	121	+ out[accn] = Filing(accn=accn, form=form, filed=filed, report_date=_d(rec.get("reportDate", [None] * n)[i]),
	122	+ primary_doc=(rec.get("primaryDocument") or [None] * n)[i], is_amendment=form.endswith("/A"),
	123	+ is_xbrl=bool((rec.get("isXBRL") or [0] * n)[i]))
	124	+ return out
	125	+
	126	+
	127	+def filings_from_facts(facts: pd.DataFrame, known: dict[str, Filing]) -> dict[str, Filing]:
	128	+ """Complete the filings map with every accn present in the facts (report_date = the filing's own period)."""
	129	+ out = dict(known)
	130	+ if facts.empty:
	131	+ return out
	132	+ for accn, g in facts.groupby("accn"):
	133	+ form = str(g["form"].iloc[0])
	134	+ filed = max(x for x in g["filed"] if x is not None) if g["filed"].notna().any() else None
	135	+ if filed is None:
	136	+ continue
	137	+ own = g[g["fp"].notna()]
	138	+ report = max(own["end"]) if not own.empty else max(g["end"])
	139	+ f = out.get(accn)
	140	+ if f is None:
	141	+ out[accn] = Filing(accn=accn, form=form, filed=filed, report_date=report, is_amendment=form.endswith("/A"), is_xbrl=True)
	142	+ elif f.report_date is None:
	143	+ f.report_date = report
	144	+ return out
	145	+
	146	+
	147	+def _d(v: Any) -> date \| None:
	148	+ if v is None or v == "" or (isinstance(v, float) and math.isnan(v)):
	149	+ return None
	150	+ if isinstance(v, date):
	151	+ return v
	152	+ try:
	153	+ return date.fromisoformat(str(v)[:10])
	154	+ except ValueError:
	155	+ return None
	156	+
	157	+
	158	+# --------------------------------------------------------------------------------------- fiscal calendar
	159	+class FiscalCalendar:
	160	+ """Map period ends to (fiscal_year, fiscal_quarter).
	161	+
	162	+ `fy_ends` = {fiscal_year: fiscal year end date} learnt from the 10-K filings (own period of each 10-K,
	163	+ with the filing's `fy`). Years without a 10-K are extrapolated from the `fiscalYearEnd` MMDD, snapping
	164	+ to the nearest ±7 days so 52/53-week filers (Apple: 2023-09-30, 2024-09-28) are handled."""
	165	+
	166	+ SNAP_DAYS = 7
	167	+
	168	+ def __init__(self, fye_mmdd: str \| None, fy_ends: dict[int, date] \| None = None):
	169	+ self.fy_ends: dict[int, date] = dict(fy_ends or {})
	170	+ self.mm, self.dd = None, None
	171	+ if fye_mmdd and len(fye_mmdd) == 4 and fye_mmdd.isdigit():
	172	+ self.mm, self.dd = int(fye_mmdd[:2]), int(fye_mmdd[2:])
	173	+ elif self.fy_ends:
	174	+ last = self.fy_ends[max(self.fy_ends)]
	175	+ self.mm, self.dd = last.month, last.day
	176	+
	177	+ def _nominal_end(self, year: int) -> date \| None:
	178	+ if not self.mm:
	179	+ return None
	180	+ return date(year, self.mm, min(self.dd, calendar.monthrange(year, self.mm)[1]))
	181	+
	182	+ def fy_end(self, fy: int) -> date \| None:
	183	+ if fy in self.fy_ends:
	184	+ return self.fy_ends[fy]
	185	+ # the fiscal year label = calendar year in which it ends for most filers; when the fiscal year end
	186	+ # falls in Jan–Mar some filers label by the previous year — the learnt table decides, else nominal
	187	+ if self.fy_ends:
	188	+ # infer the label convention from the closest learnt year
	189	+ ref = min(self.fy_ends, key=lambda y: abs(y - fy))
	190	+ offset = self.fy_ends[ref].year - ref
	191	+ return self._nominal_end(fy + offset)
	192	+ return self._nominal_end(fy)
	193	+
	194	+ def locate(self, end: date) -> tuple[int, int] \| None:
	195	+ """(fiscal_year, fiscal_quarter) of a period ending at `end` (quarter 4 = fiscal year end)."""
	196	+ best: tuple[int, date] \| None = None
	197	+ candidates: list[tuple[int, date]] = list(self.fy_ends.items())
	198	+ for y in (end.year - 1, end.year, end.year + 1):
	199	+ for fy in (y - 1, y, y + 1):
	200	+ fe = self.fy_end(fy)
	201	+ if fe is not None:
	202	+ candidates.append((fy, fe))
	203	+ for fy, fe in candidates:
	204	+ if fe + timedelta(days=self.SNAP_DAYS) >= end and (fe - end).days < 372:
	205	+ if best is None or fe < best[1]:
	206	+ best = (fy, fe)
	207	+ if best is None:
	208	+ return None
	209	+ fy, fe = best
	210	+ days = (fe - end).days
	211	+ if days <= self.SNAP_DAYS:
	212	+ return fy, 4
	213	+ q = 4 - int(round(days / 91.3125))
	214	+ if q < 1 or q > 4:
	215	+ return None
	216	+ return fy, q
	217	+
	218	+ def quarter_end(self, fy: int, fq: int) -> date \| None:
	219	+ fe = self.fy_end(fy)
	220	+ if fe is None:
	221	+ return None
	222	+ return fe - timedelta(days=round(91.3125 * (4 - fq)))
	223	+
	224	+
	225	+def span_type(start: date \| None, end: date) -> str \| None:
	226	+ if start is None:
	227	+ return None
	228	+ days = (end - start).days
	229	+ if 75 <= days <= 105:
	230	+ return SPAN_Q
	231	+ if 165 <= days <= 195:
	232	+ return SPAN_H
	233	+ if 255 <= days <= 290:
	234	+ return SPAN_9M
	235	+ if 340 <= days <= 380:
	236	+ return SPAN_FY
	237	+ return None
	238	+
	239	+
	240	+def calendar_quarter(end: date) -> str:
	241	+ """Calendar quarter of a period end; ends in the first 7 days of a month belong to the previous month
	242	+ (52/53-week years: a quarter ending 2025-01-03 is calendar 2024Q4)."""
	243	+ d = end
	244	+ if d.day <= 7:
	245	+ d = (d.replace(day=1) - timedelta(days=1))
	246	+ return f"{d.year}Q{(d.month - 1) // 3 + 1}"
	247	+
	248	+
	249	+def learn_fy_ends(facts: pd.DataFrame, filings: dict[str, Filing]) -> dict[int, date]:
	250	+ """fiscal_year -> fiscal year end date, from the own period of each 10-K (facts with fp == FY)."""
	251	+ out: dict[int, date] = {}
	252	+ if facts.empty:
	253	+ return out
	254	+ k = facts[(facts["fp"] == "FY") & facts["form"].isin(FORMS_10K) & facts["fy"].notna()]
	255	+ for (accn, fy), g in k.groupby(["accn", "fy"]):
	256	+ f = filings.get(accn)
	257	+ end = f.report_date if f and f.report_date else max(g["end"])
	258	+ dur = g[g["start"].notna()]
	259	+ if not dur.empty:
	260	+ # the filing's own annual period ends at the report date; sanity: an annual span must end there
	261	+ ok = dur[(dur["end"] == end) & dur.apply(lambda r: span_type(r["start"], r["end"]) == SPAN_FY, axis=1)]
	262	+ if ok.empty:
	263	+ continue
	264	+ fy = int(fy)
	265	+ if abs(end.year - fy) > 1:
	266	+ continue
	267	+ if fy not in out or end > out[fy]:
	268	+ out[fy] = end
	269	+ return out
	270	+
	271	+
	272	+# ------------------------------------------------------------------------------------------ resolution
	273	+@dataclass
	274	+class Resolved:
	275	+ """Values of one statement for one period as reported by one filing."""
	276	+ values: dict[str, float] = field(default_factory=dict)
	277	+ coverage: dict[str, dict[str, Any]] = field(default_factory=dict)
	278	+ currency: str \| None = None
	279	+ start: date \| None = None
	280	+ end: date \| None = None
	281	+
	282	+
	283	+def _unit_ok(account: M.Account, unit: str) -> str \| None:
	284	+ """Return the currency (or 'shares') when the unit fits the account, else None."""
	285	+ if account.unit == M.SHARES:
	286	+ return "shares" if unit == "shares" else None
	287	+ if account.unit == M.PER_SHARE:
	288	+ m = UNIT_PER_SHARE.match(unit)
	289	+ return m.group(1) if m else None
	290	+ return unit if UNIT_CCY.match(unit) else None
	291	+
	292	+
	293	+def _period_key(cal: FiscalCalendar, account_kind: str, start: date \| None, end: date) -> tuple[int, int, str] \| None:
	294	+ loc = cal.locate(end)
	295	+ if loc is None:
	296	+ return None
	297	+ fy, fq = loc
	298	+ if account_kind == M.INSTANT:
	299	+ return fy, fq, "I"
	300	+ sp = span_type(start, end)
	301	+ if sp is None:
	302	+ return None
	303	+ if sp == SPAN_FY:
	304	+ return fy, 0, SPAN_FY
	305	+ if sp == SPAN_Q:
	306	+ return fy, fq, SPAN_Q
	307	+ if sp == SPAN_H:
	308	+ return (fy, 2, SPAN_H) if fq == 2 else None
	309	+ if sp == SPAN_9M:
	310	+ return (fy, 3, SPAN_9M) if fq == 3 else None
	311	+ return None
	312	+
	313	+
	314	+def resolve_filing(g: pd.DataFrame, cal: FiscalCalendar, filing: Filing) -> tuple[dict, dict, list[str]]:
	315	+ """Resolve every account for every period reported by one filing.
	316	+
	317	+ Returns (periods, helpers, mismatches):
	318	+ periods[(fy, fq, span)][statement] = Resolved
	319	+ helpers[(fy, fq, span)][helper_name] = value
	320	+ """
	321	+ periods: dict[tuple, dict[str, Resolved]] = defaultdict(dict)
	322	+ helpers: dict[tuple, dict[str, float]] = defaultdict(dict)
	323	+ mismatches: list[str] = []
	324	+ # index facts of this filing by (taxonomy, tag)
	325	+ by_tag: dict[tuple[str, str], pd.DataFrame] = {k: v for k, v in g.groupby(["taxonomy", "tag"])}
	326	+
	327	+ own_key: tuple \| None = None
	328	+ if filing.report_date is not None:
	329	+ loc = cal.locate(filing.report_date)
	330	+ if loc is not None:
	331	+ own_key = (loc[0], loc[1], "I")
	332	+
	333	+ def candidates(account: M.Account, tag: M.Tag) -> list[tuple[tuple, float, str, date \| None, date]]:
	334	+ """(period key, value, currency, start, end) for every fact of `tag` compatible with `account`."""
	335	+ df = by_tag.get((tag.taxonomy, tag.tag))
	336	+ if df is None:
	337	+ return []
	338	+ out: list[tuple[tuple, float, str, date \| None, date]] = []
	339	+ if account.name == "shares_outstanding" and tag.taxonomy == "dei":
	340	+ # cover-page shares are dated at the cover date and reported per share class: sum the classes
	341	+ # and attach the total to the filing's own fiscal period
	342	+ if own_key is None:
	343	+ return []
	344	+ total, end = 0.0, None
	345	+ for r in df.itertuples(index=False):
	346	+ if _unit_ok(account, r.unit) is None:
	347	+ continue
	348	+ total += float(r.val)
	349	+ end = r.end if end is None or r.end > end else end
	350	+ return [(own_key, total, "shares", None, end)] if end is not None else []
	351	+ for r in df.itertuples(index=False):
	352	+ ccy = _unit_ok(account, r.unit)
	353	+ if ccy is None:
	354	+ continue
	355	+ key = _period_key(cal, account.kind, r.start, r.end)
	356	+ if key is None:
	357	+ continue
	358	+ out.append((key, float(r.val), ccy, r.start, r.end))
	359	+ return out
	360	+
	361	+ for account in M.ACCOUNTS:
	362	+ if account.computed:
	363	+ continue
	364	+ chosen: dict[tuple, tuple[float, str, M.Tag, int, date \| None, date]] = {}
	365	+ for prio, tag in enumerate(account.tags, start=1):
	366	+ for key, val, ccy, start, end in candidates(account, tag):
	367	+ cur = chosen.get(key)
	368	+ if cur is None or (ccy == "USD" and cur[1] != "USD"):
	369	+ chosen[key] = (val, ccy, tag, prio, start, end)
	370	+ # component sums when the primary tag is missing for a period
	371	+ if account.name in COMPONENT_SUMS:
	372	+ primaries = COMPONENT_PRIMARY[account.name]
	373	+ comp_vals: dict[tuple, tuple[float, list[str], str, date \| None, date]] = {}
	374	+ for group in COMPONENT_SUMS[account.name]:
	375	+ for tname in group:
	376	+ tag = M.Tag(tname)
	377	+ hit_keys: set[tuple] = set()
	378	+ for key, val, ccy, start, end in candidates(account, tag):
	379	+ if key in hit_keys:
	380	+ continue
	381	+ hit_keys.add(key)
	382	+ v, tags, c, s, e = comp_vals.get(key, (0.0, [], ccy, start, end))
	383	+ comp_vals[key] = (v + val, tags + [tname], c, s, e)
	384	+ if hit_keys:
	385	+ break # first present alternate of the group
	386	+ for key, (v, tags, ccy, start, end) in comp_vals.items():
	387	+ cur = chosen.get(key)
	388	+ if cur is None or cur[2].tag not in primaries:
	389	+ if len(tags) > 1 or cur is None:
	390	+ chosen[key] = (v, ccy, M.Tag("+".join(tags), notes="components"), 99, start, end)
	391	+ for key, (val, ccy, tag, prio, start, end) in chosen.items():
	392	+ res = periods[key].setdefault(account.statement, Resolved())
	393	+ res.values[account.name] = val
	394	+ cov: dict[str, Any] = {"tag": f"{tag.taxonomy}:{tag.tag}", "priority": prio}
	395	+ if tag.notes == "components":
	396	+ cov = {"tag": tag.tag, "components": True}
	397	+ if ccy not in ("USD", "shares") and account.unit != M.SHARES:
	398	+ cov["currency"] = ccy
	399	+ res.coverage[account.name] = cov
	400	+ if account.unit == M.USD and ccy != "shares":
	401	+ res.currency = res.currency or ccy
	402	+ if account.name != "shares_outstanding": # cover date is not the balance sheet date
	403	+ res.end = end if res.end is None else max(res.end, end)
	404	+ if start is not None:
	405	+ res.start = start if res.start is None else min(res.start, start)
	406	+ for name, (tax, tag, kind) in HELPERS.items():
	407	+ df = by_tag.get((tax, tag))
	408	+ if df is None:
	409	+ continue
	410	+ for r in df.itertuples(index=False):
	411	+ if not UNIT_CCY.match(r.unit):
	412	+ continue
	413	+ key = _period_key(cal, kind, r.start, r.end)
	414	+ if key is not None:
	415	+ helpers[key][name] = float(r.val)
	416	+ # sanity check fy/fp of the filing's own period against the calendar
	417	+ if filing.report_date is not None:
	418	+ own = g[(g["end"] == filing.report_date) & g["fp"].notna() & g["fy"].notna()]
	419	+ if not own.empty:
	420	+ fp, fy = str(own["fp"].iloc[0]), int(own["fy"].iloc[0])
	421	+ loc = cal.locate(filing.report_date)
	422	+ if loc is not None:
	423	+ exp_fq = 4 if fp == "FY" else (int(fp[1:]) if fp.startswith("Q") and fp[1:].isdigit() else None)
	424	+ if loc[0] != fy or (exp_fq is not None and loc[1] != exp_fq):
	425	+ mismatches.append(f"{filing.accn}: fy/fp={fy}/{fp} vs calendar {loc[0]}/Q{loc[1]}")
	426	+ return periods, helpers, mismatches
	427	+
	428	+
	429	+# ------------------------------------------------------------------------------------------ versioning
	430	+@dataclass
	431	+class Version:
	432	+ cik: int
	433	+ ticker: str
	434	+ statement: str
	435	+ fiscal_year: int
	436	+ fiscal_quarter: int # 0 = annual
	437	+ period_start: date \| None
	438	+ period_end: date
	439	+ form: str
	440	+ accn: str
	441	+ filed_date: date
	442	+ values: dict[str, float \| None]
	443	+ coverage: dict[str, Any]
	444	+ derived: bool = False
	445	+ restated: bool = False
	446	+ currency: str = "USD"
	447	+
	448	+ def row(self) -> dict[str, Any]:
	449	+ r = {"cik": self.cik, "ticker": self.ticker, "statement": self.statement, "fiscal_year": self.fiscal_year,
	450	+ "fiscal_quarter": self.fiscal_quarter, "period_start": self.period_start, "period_end": self.period_end,
	451	+ "calendar_quarter": calendar_quarter(self.period_end), "form": self.form, "accn": self.accn,
	452	+ "filed_date": self.filed_date, "derived": self.derived, "restated": self.restated, "currency": self.currency,
	453	+ "coverage": self.coverage, "mapping_version": M.MAPPING_VERSION}
	454	+ for a in M.ACCOUNTS:
	455	+ if a.statement == self.statement:
	456	+ r[a.name] = self.values.get(a.name)
	457	+ return r
	458	+
	459	+
	460	+def _changed(old: float \| None, new: float \| None) -> bool:
	461	+ if old is None and new is None:
	462	+ return False
	463	+ if old is None or new is None:
	464	+ return True
	465	+ return not math.isclose(old, new, rel_tol=REL_TOL, abs_tol=0.5)
	466	+
	467	+
	468	+def _rank(cov: dict[str, Any] \| None) -> int:
	469	+ """Provenance rank of a value (lower is better): mapped tag priority < components < identity < derived."""
	470	+ if not cov:
	471	+ return 2000
	472	+ if "priority" in cov:
	473	+ return int(cov["priority"])
	474	+ if cov.get("components"):
	475	+ return 99
	476	+ if cov.get("identity") or cov.get("computed"):
	477	+ return 500
	478	+ if cov.get("derived"):
	479	+ return 1000
	480	+ return 2000
	481	+
	482	+
	483	+@dataclass
	484	+class NormalizeResult:
	485	+ rows: list[dict[str, Any]]
	486	+ mapping_log: list[dict[str, Any]]
	487	+ stats: dict[str, Any]
	488	+ fy_ends: dict[int, date]
	489	+
	490	+
	491	+class _State:
	492	+ """Latest known version per (statement, fy, fq) while replaying filings chronologically."""
	493	+
	494	+ def __init__(self):
	495	+ self.latest: dict[tuple[str, int, int], Version] = {}
	496	+ self.ytd: dict[tuple[int, int], dict[str, dict[str, float]]] = defaultdict(dict) # (fy, fq) -> statement -> values
	497	+ self.out: list[Version] = []
	498	+
	499	+ def get(self, statement: str, fy: int, fq: int) -> Version \| None:
	500	+ return self.latest.get((statement, fy, fq))
	501	+
	502	+ def values(self, statement: str, fy: int, fq: int) -> dict[str, float \| None]:
	503	+ v = self.get(statement, fy, fq)
	504	+ return dict(v.values) if v else {}
	505	+
	506	+ def publish(self, v: Version) -> bool:
	507	+ """Insert as a new version when it changes what is known; returns True when inserted."""
	508	+ key = (v.statement, v.fiscal_year, v.fiscal_quarter)
	509	+ prev = self.latest.get(key)
	510	+ if prev is not None:
	511	+ merged = dict(prev.values)
	512	+ cov = dict(prev.coverage)
	513	+ changed_any = False
	514	+ restated = False
	515	+ for k, val in v.values.items():
	516	+ if val is None:
	517	+ continue
	518	+ new_rank, old_rank = _rank(v.coverage.get(k)), _rank(cov.get(k))
	519	+ if new_rank > old_rank and merged.get(k) is not None:
	520	+ # a later filing re-reports the period with a lower-priority concept (e.g. cash incl.
	521	+ # restricted cash in a comparative column): the better concept already known wins
	522	+ continue
	523	+ if _changed(merged.get(k), val):
	524	+ changed_any = True
	525	+ if merged.get(k) is not None:
	526	+ restated = True
	527	+ merged[k] = val
	528	+ cov[k] = v.coverage.get(k, {})
	529	+ elif k in v.coverage and new_rank < old_rank:
	530	+ cov[k] = v.coverage[k] # same number, better provenance (reported replaces derived)
	531	+ changed_any = True
	532	+ if not changed_any:
	533	+ return False
	534	+ for k, c in v.coverage.items():
	535	+ if k not in cov:
	536	+ cov[k] = c
	537	+ v.values = merged
	538	+ v.coverage = cov
	539	+ v.restated = (restated or prev.restated) if prev.accn == v.accn else restated
	540	+ v.derived = any(isinstance(c, dict) and c.get("derived") for c in cov.values())
	541	+ if prev.accn == v.accn:
	542	+ # same filing (reported + derived pieces): one version per (period, filing)
	543	+ prev.values, prev.coverage, prev.restated, prev.derived = v.values, v.coverage, v.restated, v.derived
	544	+ prev.period_start = prev.period_start or v.period_start
	545	+ return True
	546	+ self.latest[key] = v
	547	+ self.out.append(v)
	548	+ return True
	549	+
	550	+
	551	+def normalize_company(cik: int, ticker: str, facts: pd.DataFrame, filings: dict[str, Filing],
	552	+ fye_mmdd: str \| None, metalinks: dict[str, Any] \| None = None) -> NormalizeResult:
	553	+ """Full pipeline for one company; see module docstring."""
	554	+ stats: dict[str, Any] = {"facts": int(len(facts)), "filings": 0, "versions": 0, "derived_rows": 0,
	555	+ "restated_rows": 0, "fiscal_mismatches": [], "unmapped_tags": 0, "extensions": 0}
	556	+ if facts.empty:
	557	+ return NormalizeResult([], [], stats, {})
	558	+ all_facts = facts
	559	+ # only periodic financial reports carry statements (proxy statements, S-1… also embed XBRL facts)
	560	+ facts = facts[facts["form"].isin(FORMS_10K + FORMS_10Q)]
	561	+ if facts.empty:
	562	+ return NormalizeResult([], _mapping_log(cik, all_facts, metalinks), stats, {})
	563	+ filings = filings_from_facts(facts, filings)
	564	+ fy_ends = learn_fy_ends(facts, filings)
	565	+ cal = FiscalCalendar(fye_mmdd, fy_ends)
	566	+ state = _State()
	567	+ accns = sorted({a for a in facts["accn"].unique() if a in filings}, key=lambda a: (filings[a].filed, a))
	568	+ stats["filings"] = len(accns)
	569	+ for accn in accns:
	570	+ filing = filings[accn]
	571	+ g = facts[facts["accn"] == accn]
	572	+ periods, helpers, mism = resolve_filing(g, cal, filing)
	573	+ stats["fiscal_mismatches"] += mism
	574	+ # 1) reported periods
	575	+ for (fy, fq, span), by_stmt in sorted(periods.items()):
	576	+ for statement, res in by_stmt.items():
	577	+ if span in (SPAN_H, SPAN_9M):
	578	+ state.ytd[(fy, fq)][statement] = dict(res.values)
	579	+ state.ytd[(fy, fq)].setdefault("_filed", {})[statement] = filing.filed
	580	+ continue
	581	+ p_end = res.end or filing.report_date
	582	+ if p_end is None:
	583	+ continue
	584	+ if span == "I":
	585	+ # balance sheet: quarter row + annual row at fiscal year end
	586	+ targets = [(fy, fq)] + ([(fy, 0)] if fq == 4 else [])
	587	+ p_start = None
	588	+ else:
	589	+ targets = [(fy, fq)]
	590	+ p_start = res.start
	591	+ for tfy, tfq in targets:
	592	+ v = Version(cik, ticker, statement, tfy, tfq, p_start, p_end, filing.form, accn, filing.filed,
	593	+ dict(res.values), dict(res.coverage), currency=res.currency or "USD")
	594	+ _apply_helpers(v, helpers.get((fy, fq, span), {}))
	595	+ state.publish(v)
	596	+ # 2) derivations for the fiscal years touched by this filing (using what is known now)
	597	+ touched = {fy for (fy, _, _) in periods}
	598	+ for fy in sorted(touched):
	599	+ _derive_quarters(state, cik, ticker, fy, filing, cal)
	600	+ # 3) computed accounts for every period touched (cross-statement)
	601	+ for (fy, fq, _) in set(periods):
	602	+ for tfq in ({fq, 0} if fq == 4 else {fq}):
	603	+ _compute_derived_accounts(state, fy, tfq, filing)
	604	+ rows = [v.row() for v in state.out]
	605	+ stats["versions"] = len(rows)
	606	+ stats["derived_rows"] = sum(1 for v in state.out if v.derived)
	607	+ stats["restated_rows"] = sum(1 for v in state.out if v.restated)
	608	+ mapping_log = _mapping_log(cik, facts, metalinks)
	609	+ stats["unmapped_tags"] = sum(1 for m in mapping_log if not m["is_extension"])
	610	+ stats["extensions"] = sum(1 for m in mapping_log if m["is_extension"])
	611	+ return NormalizeResult(rows, mapping_log, stats, fy_ends)
	612	+
	613	+
	614	+def _apply_helpers(v: Version, helpers: dict[str, float]) -> None:
	615	+ """Identity fallbacks that need helper tags (CostsAndExpenses, LiabilitiesAndStockholdersEquity)."""
	616	+ if v.statement == M.INCOME and v.values.get("operating_income") is None:
	617	+ rev, ce = v.values.get("revenue"), helpers.get("_costs_and_expenses")
	618	+ if rev is not None and ce is not None:
	619	+ v.values["operating_income"] = rev - ce
	620	+ v.coverage["operating_income"] = {"computed": "revenue - us-gaap:CostsAndExpenses", "identity": True}
	621	+ if v.statement == M.BALANCE and v.values.get("total_liabilities") is None:
	622	+ lse, eq = helpers.get("_liabilities_and_equity"), v.values.get("total_equity")
	623	+ if lse is not None and eq is not None:
	624	+ v.values["total_liabilities"] = lse - eq
	625	+ v.coverage["total_liabilities"] = {"computed": "us-gaap:LiabilitiesAndStockholdersEquity - total_equity",
	626	+ "identity": True}
	627	+
	628	+
	629	+def _derive_quarters(state: _State, cik: int, ticker: str, fy: int, filing: Filing, cal: FiscalCalendar) -> None:
	630	+ """De-cumulate YTD facts and derive Q4 = FY − (Q1+Q2+Q3) for the flow statements (same fy, same unit)."""
	631	+ for statement in (M.INCOME, M.CASHFLOW):
	632	+ accounts = [a for a in M.accounts_for(statement) if not a.computed]
	633	+ q = {i: state.values(statement, fy, i) for i in (1, 2, 3, 4)}
	634	+ fyv = state.values(statement, fy, 0)
	635	+ ytd2 = state.ytd.get((fy, 2), {}).get(statement, {})
	636	+ ytd3 = state.ytd.get((fy, 3), {}).get(statement, {})
	637	+ derived: dict[int, tuple[dict, dict, date \| None, date \| None]] = {}
	638	+ for fq in (2, 3, 4):
	639	+ vals: dict[str, float \| None] = {}
	640	+ cov: dict[str, Any] = {}
	641	+ for a in accounts:
	642	+ if q[fq].get(a.name) is not None and not state.get(statement, fy, fq).coverage.get(a.name, {}).get("derived"):
	643	+ continue # reported discrete value exists
	644	+ val, how = None, None
	645	+ if a.unit == M.SHARES:
	646	+ # weighted averages: Q4 ≈ 4×FY − (Q1+Q2+Q3); YTD shares cannot be de-cumulated
	647	+ if fq == 4 and all(q[i].get(a.name) is not None for i in (1, 2, 3)) and fyv.get(a.name) is not None:
	648	+ val, how = 4 * fyv[a.name] - sum(q[i][a.name] for i in (1, 2, 3)), "4*FY-(Q1+Q2+Q3)"
	649	+ elif fq == 2:
	650	+ if ytd2.get(a.name) is not None and q[1].get(a.name) is not None:
	651	+ val, how = ytd2[a.name] - q[1][a.name], "YTD6-Q1"
	652	+ elif fq == 3:
	653	+ if ytd3.get(a.name) is not None and ytd2.get(a.name) is not None:
	654	+ val, how = ytd3[a.name] - ytd2[a.name], "YTD9-YTD6"
	655	+ elif ytd3.get(a.name) is not None and q[1].get(a.name) is not None and q[2].get(a.name) is not None:
	656	+ val, how = ytd3[a.name] - q[1][a.name] - q[2][a.name], "YTD9-Q1-Q2"
	657	+ else:
	658	+ if fyv.get(a.name) is not None:
	659	+ if ytd3.get(a.name) is not None:
	660	+ val, how = fyv[a.name] - ytd3[a.name], "FY-YTD9"
	661	+ elif all(q[i].get(a.name) is not None for i in (1, 2, 3)):
	662	+ val, how = fyv[a.name] - sum(q[i][a.name] for i in (1, 2, 3)), "FY-(Q1+Q2+Q3)"
	663	+ if val is not None:
	664	+ vals[a.name] = val
	665	+ cov[a.name] = {"derived": how, "approx": a.unit != M.USD}
	666	+ if vals:
	667	+ p_end = cal.quarter_end(fy, fq) if fq < 4 else cal.fy_end(fy)
	668	+ existing = state.get(statement, fy, fq)
	669	+ if existing is not None:
	670	+ p_end = existing.period_end
	671	+ if p_end is None:
	672	+ continue
	673	+ p_start = (existing.period_start if existing and existing.period_start else
	674	+ (cal.quarter_end(fy, fq - 1) + timedelta(days=1) if cal.quarter_end(fy, fq - 1) else None))
	675	+ derived[fq] = (vals, cov, p_start, p_end)
	676	+ for fq, (vals, cov, p_start, p_end) in derived.items():
	677	+ v = Version(cik, ticker, statement, fy, fq, p_start, p_end, filing.form, filing.accn, filing.filed, vals, cov,
	678	+ derived=True, currency=(state.get(statement, fy, 0) or state.get(statement, fy, 1) or
	679	+ Version(0, "", "", 0, 0, None, p_end, "", "", filing.filed, {}, {})).currency)
	680	+ if state.publish(v):
	681	+ q[fq] = state.values(statement, fy, fq)
	682	+
	683	+
	684	+def _compute_derived_accounts(state: _State, fy: int, fq: int, filing: Filing) -> None:
	685	+ inc, bal, cf = state.get(M.INCOME, fy, fq), state.get(M.BALANCE, fy, fq), state.get(M.CASHFLOW, fy, fq)
	686	+ if inc is not None:
	687	+ vals, cov = dict(inc.values), {}
	688	+ if vals.get("gross_profit") is None and vals.get("revenue") is not None and vals.get("cost_of_revenue") is not None:
	689	+ vals["gross_profit"] = vals["revenue"] - vals["cost_of_revenue"]
	690	+ cov["gross_profit"] = {"computed": "revenue - cost_of_revenue", "identity": True}
	691	+ if vals.get("sga_expense") is None and vals.get("operating_income") is None:
	692	+ pass
	693	+ da = cf.values.get("depreciation_amortization") if cf is not None else None
	694	+ if vals.get("operating_income") is not None and da is not None:
	695	+ vals["ebitda"] = vals["operating_income"] + da
	696	+ cov["ebitda"] = {"computed": "operating_income + depreciation_amortization"}
	697	+ else:
	698	+ cov["ebitda"] = {"reason": "missing:" + ",".join(k for k, ok in (("operating_income", vals.get("operating_income") is not None),
	699	+ ("depreciation_amortization", da is not None)) if not ok)}
	700	+ _publish_computed(state, inc, vals, cov, filing)
	701	+ if bal is not None:
	702	+ vals, cov = dict(bal.values), {}
	703	+ if vals.get("total_liabilities") is None and vals.get("total_assets") is not None and vals.get("total_equity") is not None:
	704	+ vals["total_liabilities"] = vals["total_assets"] - vals["total_equity"]
	705	+ cov["total_liabilities"] = {"computed": "total_assets - total_equity", "identity": True,
	706	+ "note": "may include mezzanine equity / NCI presented outside equity"}
	707	+ ltd, std = vals.get("long_term_debt"), vals.get("short_term_debt")
	708	+ if ltd is not None or std is not None:
	709	+ vals["total_debt"] = (ltd or 0.0) + (std or 0.0)
	710	+ c: dict[str, Any] = {"computed": "short_term_debt + long_term_debt"}
	711	+ if std is None:
	712	+ c["short_term_debt_assumed_zero"] = True
	713	+ if ltd is None:
	714	+ c["long_term_debt_assumed_zero"] = True
	715	+ cov["total_debt"] = c
	716	+ else:
	717	+ cov["total_debt"] = {"reason": "missing:short_term_debt,long_term_debt"}
	718	+ cash = vals.get("cash_and_equivalents")
	719	+ if vals.get("total_debt") is not None and cash is not None:
	720	+ sti = vals.get("short_term_investments")
	721	+ vals["net_debt"] = vals["total_debt"] - cash - (sti or 0.0)
	722	+ c = {"computed": "total_debt - cash_and_equivalents - short_term_investments"}
	723	+ if sti is None:
	724	+ c["short_term_investments_assumed_zero"] = True
	725	+ cov["net_debt"] = c
	726	+ else:
	727	+ cov["net_debt"] = {"reason": "missing:" + ",".join(k for k, ok in (("total_debt", vals.get("total_debt") is not None),
	728	+ ("cash_and_equivalents", cash is not None)) if not ok)}
	729	+ ca, cl = vals.get("total_current_assets"), vals.get("total_current_liabilities")
	730	+ if ca is not None and cl is not None:
	731	+ vals["working_capital"] = ca - cl
	732	+ cov["working_capital"] = {"computed": "total_current_assets - total_current_liabilities"}
	733	+ else:
	734	+ cov["working_capital"] = {"reason": "missing:" + ",".join(k for k, ok in (("total_current_assets", ca is not None),
	735	+ ("total_current_liabilities", cl is not None)) if not ok)}
	736	+ _publish_computed(state, bal, vals, cov, filing)
	737	+ if cf is not None:
	738	+ vals, cov = dict(cf.values), {}
	739	+ ocf, capex = vals.get("operating_cash_flow"), vals.get("capex")
	740	+ if ocf is not None and capex is not None:
	741	+ vals["free_cash_flow"] = ocf - capex
	742	+ cov["free_cash_flow"] = {"computed": "operating_cash_flow - capex"}
	743	+ else:
	744	+ cov["free_cash_flow"] = {"reason": "missing:" + ",".join(k for k, ok in (("operating_cash_flow", ocf is not None),
	745	+ ("capex", capex is not None)) if not ok)}
	746	+ _publish_computed(state, cf, vals, cov, filing)
	747	+
	748	+
	749	+def _publish_computed(state: _State, base: Version, vals: dict, cov: dict, filing: Filing) -> None:
	750	+ """Attach computed accounts to the latest version in place (same accn) — they are deterministic
	751	+ functions of that version, so they do not create a new version."""
	752	+ changed = False
	753	+ for k, v in vals.items():
	754	+ if _changed(base.values.get(k), v):
	755	+ base.values[k] = v
	756	+ changed = True
	757	+ for k, c in cov.items():
	758	+ if base.coverage.get(k) != c:
	759	+ base.coverage[k] = c
	760	+ # fill reasons for every public account still null
	761	+ for a in M.accounts_for(base.statement):
	762	+ if base.values.get(a.name) is None and a.name not in base.coverage:
	763	+ base.coverage[a.name] = {"reason": "no_mapped_tag"}
	764	+ _ = changed
	765	+
	766	+
	767	+def _mapping_log(cik: int, facts: pd.DataFrame, metalinks: dict[str, Any] \| None) -> list[dict[str, Any]]:
	768	+ out: list[dict[str, Any]] = []
	769	+ std = facts[facts["taxonomy"].isin(M.STANDARD_TAXONOMIES)]
	770	+ for (tax, tag), g in std.groupby(["taxonomy", "tag"]):
	771	+ if M.is_mapped(tax, tag) or (tax, tag) in M.IDENTITY_TAGS or tax == "dei":
	772	+ continue
	773	+ out.append({"cik": cik, "taxonomy": tax, "tag": tag, "is_extension": False, "occurrences": int(len(g)),
	774	+ "first_seen": min(g["filed"]), "last_seen": max(g["filed"]), "sample_accn": str(g["accn"].iloc[-1]),
	775	+ "hint_account": None})
	776	+ if metalinks:
	777	+ inst = next(iter((metalinks.get("instance") or {}).values()), {})
	778	+ prefix = inst.get("nsprefix")
	779	+ roles = {r.get("role"): r.get("shortName", "") for r in (inst.get("report") or {}).values()
	780	+ if r.get("menuCat") == "Statements"}
	781	+ for name, info in (inst.get("tag") or {}).items():
	782	+ if not prefix or not name.startswith(prefix + "_"):
	783	+ continue
	784	+ if name.endswith(("Member", "Axis", "Domain", "Abstract", "Table", "LineItems")):
	785	+ continue
	786	+ pres = [roles[p] for p in (info.get("presentation") or []) if p in roles]
	787	+ if not pres:
	788	+ continue
	789	+ out.append({"cik": cik, "taxonomy": prefix, "tag": name[len(prefix) + 1:], "is_extension": True,
	790	+ "occurrences": 1, "first_seen": None, "last_seen": None, "sample_accn": None,
	791	+ "hint_account": _hint(name, pres)})
	792	+ return out
	793	+
	794	+
	795	+def _hint(name: str, statements: list[str]) -> str \| None:
	796	+ n = name.lower()
	797	+ stmt = " ".join(statements).upper()
	798	+ if "INCOME" in stmt or "OPERATIONS" in stmt:
	799	+ if "cost" in n or "materials" in n:
	800	+ return "cost_of_revenue"
	801	+ if "revenue" in n or "sales" in n:
	802	+ return "revenue"
	803	+ if "BALANCE" in stmt and ("debt" in n or "borrow" in n or "notes" in n):
	804	+ return "long_term_debt"
	805	+ if "CASH" in stmt and ("purchase" in n or "capital" in n):
	806	+ return "capex"
	807	+ return None
	808	+
	809	+
	810	+# ------------------------------------------------------------------------------------------ queries
	811	+def select_as_of(rows: list[dict[str, Any]], as_of: date \| None) -> list[dict[str, Any]]:
	812	+ """Latest version of every (statement, fiscal_year, fiscal_quarter) known at `as_of` (inclusive).
	813	+
	814	+ This is the anti look-ahead primitive: a row is visible only if `filed_date <= as_of`."""
	815	+ best: dict[tuple, dict[str, Any]] = {}
	816	+ for r in rows:
	817	+ fd = _d(r["filed_date"])
	818	+ if as_of is not None and fd is not None and fd > as_of:
	819	+ continue
	820	+ key = (r["statement"], int(r["fiscal_year"]), int(r["fiscal_quarter"]))
	821	+ cur = best.get(key)
	822	+ if cur is None or (_d(cur["filed_date"]), cur.get("id", 0)) < (fd, r.get("id", 0)):
	823	+ best[key] = r
	824	+ return sorted(best.values(), key=lambda r: (r["statement"], r["period_end"], r["fiscal_quarter"]))
	825	+
	826	+
	827	+def ttm(rows: list[dict[str, Any]], statement: str, n_quarters: int = 4) -> list[dict[str, Any]]:
	828	+ """Trailing-twelve-month rows from discrete quarterly rows (one statement, already as-of selected).
	829	+
	830	+ Flows: sum of the last 4 consecutive fiscal quarters (period_end of the window = latest quarter);
	831	+ balance sheet: the latest quarter (no summing); shares_*: latest quarter; EPS: sum of the four quarterly
	832	+ EPS (flagged `approx`). Windows with a missing quarter are skipped."""
	833	+ qs = sorted([r for r in rows if r["statement"] == statement and int(r["fiscal_quarter"]) in (1, 2, 3, 4)],
	834	+ key=lambda r: (int(r["fiscal_year"]), int(r["fiscal_quarter"])))
	835	+ out: list[dict[str, Any]] = []
	836	+ accounts = M.accounts_for(statement)
	837	+ for i in range(n_quarters - 1, len(qs)):
	838	+ window = qs[i - n_quarters + 1:i + 1]
	839	+ if not _consecutive(window):
	840	+ continue
	841	+ last = window[-1]
	842	+ row = {k: last[k] for k in ("cik", "ticker", "statement", "fiscal_year", "fiscal_quarter", "period_end",
	843	+ "calendar_quarter", "form", "accn", "filed_date", "currency", "mapping_version")}
	844	+ row["period_start"] = window[0].get("period_start")
	845	+ row["filed_date"] = max(_d(w["filed_date"]) for w in window)
	846	+ row["derived"] = any(w.get("derived") for w in window)
	847	+ row["restated"] = any(w.get("restated") for w in window)
	848	+ row["ttm"] = True
	849	+ cov: dict[str, Any] = {}
	850	+ for a in accounts:
	851	+ vals = [w.get(a.name) for w in window]
	852	+ if statement == M.BALANCE or a.unit == M.SHARES:
	853	+ row[a.name] = last.get(a.name)
	854	+ if last.get(a.name) is None:
	855	+ cov[a.name] = (last.get("coverage") or {}).get(a.name, {"reason": "no_mapped_tag"})
	856	+ elif any(v is None for v in vals):
	857	+ row[a.name] = None
	858	+ cov[a.name] = {"reason": "missing_quarters", "quarters": [f"{w['fiscal_year']}Q{w['fiscal_quarter']}"
	859	+ for w, v in zip(window, vals) if v is None]}
	860	+ else:
	861	+ row[a.name] = float(sum(vals))
	862	+ if a.unit == M.PER_SHARE:
	863	+ cov[a.name] = {"approx": "sum of 4 quarterly EPS"}
	864	+ row["coverage"] = cov
	865	+ out.append(row)
	866	+ return out
	867	+
	868	+
	869	+def _consecutive(window: list[dict[str, Any]]) -> bool:
	870	+ for a, b in zip(window, window[1:]):
	871	+ fy, fq = int(a["fiscal_year"]), int(a["fiscal_quarter"])
	872	+ nfy, nfq = (fy, fq + 1) if fq < 4 else (fy + 1, 1)
	873	+ if (int(b["fiscal_year"]), int(b["fiscal_quarter"])) != (nfy, nfq):
	874	+ return False
	875	+ return True
	876