"""SEC EDGAR XBRL `companyfacts` connector — independent public-domain source. Status: **pending configuration** until `SEC_EDGAR_USER_AGENT` (name + contact e-mail, per SEC fair-access policy) is set. Parsing is implemented and fixture-tested; live fetches refuse to run without the User-Agent and respect the 10 requests/second limit. PIT: `companyfacts` provides `filed` (filing date), `accn` (accession), `form`, `fy`, `fp`, `start`, `end`, `frame`. `available_at = filed + 1 day 00:00 ET`. Facts re-reported in later filings appear as distinct (accn, filed) rows → FinData keeps each as a revision ordered by filing date, so as-of queries naturally pick the latest filing *available at* the query time, never a later restatement. """ from __future__ import annotations import time from collections.abc import Iterator from datetime import UTC, date, datetime, timedelta from typing import Any from zoneinfo import ZoneInfo import httpx from findata_core.config import get_settings SOURCE_ID = "sec_edgar" ET = ZoneInfo("America/New_York") BASE = "https://data.sec.gov" # us-gaap concept -> FinData metric name (subset; others keep the concept name prefixed `xbrl:`) CONCEPT_MAP = { "Revenues": "rev", "RevenueFromContractWithCustomerExcludingAssessedTax": "rev", "CostOfRevenue": "cogs", "GrossProfit": "gp", "OperatingIncomeLoss": "op_inc", "NetIncomeLoss": "net_inc", "EarningsPerShareBasic": "eps", "EarningsPerShareDiluted": "eps_d", "Assets": "assets_tot", "AssetsCurrent": "assets_c", "Liabilities": "liab_tot", "LiabilitiesCurrent": "liab_c", "StockholdersEquity": "equity", "CashAndCashEquivalentsAtCarryingValue": "cash", "NetCashProvidedByUsedInOperatingActivities": "cash_op", "NetCashProvidedByUsedInInvestingActivities": "cash_inv", "NetCashProvidedByUsedInFinancingActivities": "cash_fin", "PaymentsToAcquirePropertyPlantAndEquipment": "capex", "CommonStockSharesOutstanding": "shares", "WeightedAverageNumberOfDilutedSharesOutstanding": "shares_av_diluted", "WeightedAverageNumberOfSharesOutstandingBasic": "shares_av", "ResearchAndDevelopmentExpense": "research_dev", "LongTermDebt": "debt", "RetainedEarningsAccumulatedDeficit": "ret_earnings", } def facts_from_companyfacts(payload: dict[str, Any], observed_at: datetime, only_mapped: bool = True) -> Iterator[dict[str, Any]]: cik = int(payload.get("cik", 0)) entity_id = f"sec:CIK{cik:010d}" name = payload.get("entityName") for taxonomy, concepts in (payload.get("facts") or {}).items(): if taxonomy not in ("us-gaap", "dei"): continue for concept, body in concepts.items(): metric = CONCEPT_MAP.get(concept) if metric is None: if only_mapped: continue metric = f"xbrl:{concept}" for unit, vals in (body.get("units") or {}).items(): for v in vals: end = date.fromisoformat(v["end"]) start = date.fromisoformat(v["start"]) if v.get("start") else None filed = date.fromisoformat(v["filed"]) fp = (v.get("fp") or "").upper() ptype = "a" if fp == "FY" or (start and (end - start).days > 300) else "q" if start and (end - start).days > 120 and ptype == "q": continue # year-to-date cumulative values are not quarterly facts yield { "entity_id": entity_id, "security_id": None, "metric": metric, "value": float(v["val"]), "unit": unit, "currency": "USD" if unit == "USD" else None, "fiscal_period_start": start, "fiscal_period_end": end, "fiscal_period_type": ptype, "fiscal_year": v.get("fy"), "fiscal_period": fp.lower() or None, "filing_date": filed, "published_at": None, "available_at": datetime.combine(filed + timedelta(days=1), datetime.min.time(), tzinfo=ET).astimezone(UTC), "observed_at": observed_at, "source_document": f"{v.get('form')} {v.get('accn')} ({name})", "source_id": SOURCE_ID, "accession": v.get("accn"), "form": v.get("form"), "frame": v.get("frame"), } class SecEdgarConnector: source_id = SOURCE_ID def __init__(self, user_agent: str | None = None, transport: httpx.BaseTransport | None = None): self.user_agent = user_agent if user_agent is not None else get_settings().sec_edgar_user_agent self._client = httpx.Client(timeout=60.0, transport=transport, headers={"User-Agent": self.user_agent or "unset", "Accept-Encoding": "gzip"}) self._last = 0.0 @property def configured(self) -> bool: return bool(self.user_agent and "@" in self.user_agent) def status(self) -> dict[str, Any]: return {"source_id": SOURCE_ID, "status": "active" if self.configured else "pending_configuration", "reason": None if self.configured else "SEC_EDGAR_USER_AGENT (name + e-mail) not set"} def _get(self, url: str) -> dict[str, Any]: if not self.configured: raise RuntimeError("SEC EDGAR connector pending configuration: set SEC_EDGAR_USER_AGENT") wait = 0.11 - (time.monotonic() - self._last) if wait > 0: time.sleep(wait) self._last = time.monotonic() r = self._client.get(url) r.raise_for_status() return r.json() def company_tickers(self) -> dict[str, Any]: return self._get("https://www.sec.gov/files/company_tickers.json") def companyfacts(self, cik: int) -> dict[str, Any]: return self._get(f"{BASE}/api/xbrl/companyfacts/CIK{cik:010d}.json")