SPB Git forge
spboucher.ai ↗

spb/hfmarketdata

Public

Open high-frequency market data platform — FirstRate full-history downloader, DuckDB/Parquet lake, open REST API and React docs platform (www.hfmarketdata.io)

127commits 1branches 0releases
24.7 MBsize
maindefault branch
11 days agolast push
JavaScript 53.7% Python 38.3% CSS 4.6% TypeScript 3.1%

fundamentals: schéma SQLite (facts versionnés, états standardisés, mapping versionné) + plan de comptes et mapping US-GAAP priorisé

- models.py : edgar_companies, edgar_filings, fund_statements (large, versionné par filed_date, index point-in-time), fund_mapping, fund_mapping_log, fund_coverage, fund_ingest_state, fund_latest (screener)
- mapping.py : 49 comptes (16 income, 19 balance, 12 cash flow + 2 auxiliaires), 183 (compte, tag) priorisés et documentés, version 2026.09.1
- core/errors.py : codes FUNDAMENTALS_NOT_AVAILABLE, INVALID_FILTER, CONCEPT_NOT_FOUND, STREAM_CONNECTION_LIMIT (+ statuts OpenAPI)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 20 days ago (Sep 5, 2026) parent 7ee2ef9

5 changed files +627 −1

modified hfmarketdata/api/core/errors.py +5 −0
@@ -40,6 +40,11 @@ CODES: dict[str, str] = {
40 40 "CONFLICT": "The resource already exists or is in a conflicting state.",
41 41 "INTERNAL_ERROR": "Unexpected server error.",
42 42 "SERVICE_UNAVAILABLE": "A dependency (Redis, data lake) is unavailable.",
43 + # fundamentals / stream / bulk
44 + "FUNDAMENTALS_NOT_AVAILABLE": "No SEC EDGAR fundamentals for this ticker (not an SEC filer, not yet ingested, or no data for the requested period).",
45 + "INVALID_FILTER": "A screener filter expression is malformed or references an unknown field.",
46 + "CONCEPT_NOT_FOUND": "Unknown standardized account or XBRL concept.",
47 + "STREAM_CONNECTION_LIMIT": "Too many concurrent WebSocket connections for this API key.",
43 48 }
44 49
45 50
added hfmarketdata/api/fundamentals/__init__.py +12 −0
@@ -0,0 +1,12 @@
1 +"""SEC EDGAR fundamentals — standardized financial statements, ratios, screener (point-in-time).
2 +
3 +Author: Simon-Pierre Boucher <contact@spboucher.ai>
4 +"""
5 +from __future__ import annotations
6 +
7 +from datetime import datetime, timezone
8 +
9 +
10 +def utcnow() -> datetime:
11 + """Naive UTC timestamp (what SQLite stores) without the deprecated `datetime.utcnow()`."""
12 + return datetime.now(timezone.utc).replace(tzinfo=None)
added hfmarketdata/api/fundamentals/mapping.py +404 −0
@@ -0,0 +1,404 @@
1 +"""Standard chart of accounts and the prioritized XBRL tag mapping.
2 +
3 +Every standardized account lists the XBRL tags that may carry it, **in priority order**: the first
4 +tag that has a fact for the period wins, the others are documented fallbacks (see `notes`). The
5 +mapping is versioned (`MAPPING_VERSION`) and seeded into the `fund_mapping` table so a served value
6 +can always be traced back to the exact (taxonomy, tag) it came from.
7 +
8 +Rules honoured here (details in docs/fundamentals.md):
9 +
10 +* duration accounts (income statement, cash flow) come from facts with `start`+`end`; instant
11 + accounts (balance sheet) come from facts with `end` only;
12 +* units are normalised to raw USD, shares and USD/share (`unit` column); non-USD filers keep their
13 + currency (see normalize.py);
14 +* accounts marked `computed=True` are never read from a tag — they are derived from other standard
15 + accounts and flagged in `coverage`;
16 +* company extension tags (`aapl:…`, `msft:…`) are **never guessed**: they are logged in
17 + `fund_mapping_log` so the mapping can be extended by a human.
18 +
19 +Author: Simon-Pierre Boucher <contact@spboucher.ai>
20 +"""
21 +from __future__ import annotations
22 +
23 +from dataclasses import dataclass
24 +
25 +MAPPING_VERSION = "2026.09.1"
26 +
27 +INCOME, BALANCE, CASHFLOW = "income", "balance", "cashflow"
28 +DURATION, INSTANT = "duration", "instant"
29 +USD, SHARES, PER_SHARE = "USD", "shares", "USD/share"
30 +
31 +STANDARD_TAXONOMIES = ("us-gaap", "ifrs-full", "dei", "srt")
32 +
33 +
34 +@dataclass(frozen=True)
35 +class Tag:
36 + tag: str
37 + taxonomy: str = "us-gaap"
38 + notes: str = ""
39 +
40 +
41 +@dataclass(frozen=True)
42 +class Account:
43 + name: str
44 + statement: str
45 + kind: str # duration | instant
46 + unit: str # USD | shares | USD/share
47 + tags: tuple[Tag, ...] = ()
48 + computed: bool = False # derived from other accounts, never read from a tag
49 + auxiliary: bool = False # not part of the public chart of accounts (input for computed ones)
50 + formula: str = "" # human-readable formula for computed accounts
51 + notes: str = ""
52 +
53 + @property
54 + def is_flow(self) -> bool:
55 + return self.kind == DURATION
56 +
57 +
58 +def _t(*items: str | tuple[str, str] | tuple[str, str, str]) -> tuple[Tag, ...]:
59 + out: list[Tag] = []
60 + for it in items:
61 + if isinstance(it, str):
62 + out.append(Tag(it))
63 + elif len(it) == 2:
64 + out.append(Tag(it[0], notes=it[1]))
65 + else:
66 + out.append(Tag(it[0], taxonomy=it[1], notes=it[2]))
67 + return tuple(out)
68 +
69 +
70 +ACCOUNTS: tuple[Account, ...] = (
71 + # ------------------------------------------------------------------ income statement (duration)
72 + Account("revenue", INCOME, DURATION, USD, _t(
73 + ("Revenues", "aggregate revenue, the most generic concept"),
74 + ("RevenueFromContractWithCustomerExcludingAssessedTax", "ASC 606 revenue (2018+), used by most filers today"),
75 + ("SalesRevenueNet", "pre-2018 concept, deprecated by the FASB in 2018"),
76 + ("RevenueFromContractWithCustomerIncludingAssessedTax", "ASC 606 revenue gross of sales taxes"),
77 + ("SalesRevenueGoodsNet", "goods-only revenue (pre-2018)"),
78 + ("SalesRevenueServicesNet", "services-only revenue (pre-2018)"),
79 + ("RevenuesNetOfInterestExpense", "banks / brokers: net revenue after interest expense"),
80 + ("InterestAndDividendIncomeOperating", "banks: interest income as top line when nothing else is reported"),
81 + ("RegulatedAndUnregulatedOperatingRevenue", "utilities"),
82 + ("OperatingLeasesIncomeStatementLeaseRevenue", "REITs / lessors"),
83 + ("Revenue", "ifrs-full", "IFRS filers (20-F)"),
84 + ), notes="Top line. Gross of returns/allowances only when nothing else is reported."),
85 + Account("cost_of_revenue", INCOME, DURATION, USD, _t(
86 + ("CostOfRevenue", "aggregate cost of revenue"),
87 + ("CostOfGoodsAndServicesSold", "most common concept since 2018"),
88 + ("CostOfGoodsSold", "goods only (pre-2018)"),
89 + ("CostOfServices", "services only"),
90 + ("CostOfGoodsAndServicesSoldExcludingDepreciationDepletionAndAmortization", "ex-D&A variant (flagged: excludes D&A)"),
91 + ("CostOfGoodsSoldExcludingDepreciationDepletionAndAmortization", "ex-D&A variant (flagged: excludes D&A)"),
92 + ("CostOfSales", "ifrs-full", "IFRS filers"),
93 + )),
94 + Account("gross_profit", INCOME, DURATION, USD, _t(
95 + ("GrossProfit", "reported gross profit"),
96 + ("GrossProfit", "ifrs-full", "IFRS filers"),
97 + ), formula="revenue - cost_of_revenue (identity fallback when GrossProfit is not reported, flagged `identity`)"),
98 + Account("rnd_expense", INCOME, DURATION, USD, _t(
99 + ("ResearchAndDevelopmentExpense", "R&D expense"),
100 + ("ResearchAndDevelopmentExpenseExcludingAcquiredInProcessCost", "R&D excluding acquired IPR&D"),
101 + ("ResearchAndDevelopmentExpenseSoftwareExcludingAcquiredInProcessCost", "software R&D"),
102 + ("ResearchAndDevelopmentInProcess", "acquired IPR&D only — last resort"),
103 + )),
104 + Account("sga_expense", INCOME, DURATION, USD, _t(
105 + ("SellingGeneralAndAdministrativeExpense", "combined SG&A"),
106 + ("GeneralAndAdministrativeExpense", "G&A only — summed with SellingAndMarketingExpense when both exist (see normalize)"),
107 + ("SellingAndMarketingExpense", "S&M only"),
108 + ("SellingExpense", "selling only"),
109 + ), notes="When SG&A is split, normalize.py sums the available components and flags `components`."),
110 + Account("operating_income", INCOME, DURATION, USD, _t(
111 + ("OperatingIncomeLoss", "operating income"),
112 + ("IncomeLossFromContinuingOperationsBeforeInterestExpenseInterestIncomeIncomeTaxesExtraordinaryItemsNoncontrollingInterest",
113 + "EBIT-like concept used by some filers"),
114 + ), formula="revenue - CostsAndExpenses (identity fallback when the filer reports total costs, flagged)"),
115 + Account("interest_expense", INCOME, DURATION, USD, _t(
116 + ("InterestExpense", "total interest expense"),
117 + ("InterestExpenseNonoperating", "non-operating interest expense (2023+ taxonomy)"),
118 + ("InterestExpenseDebt", "interest on debt only"),
119 + ("InterestAndDebtExpense", "interest and debt-related expense"),
120 + ("InterestExpenseBorrowings", "banks: interest on borrowings"),
121 + ("FinanceCosts", "ifrs-full", "IFRS filers"),
122 + )),
123 + Account("pretax_income", INCOME, DURATION, USD, _t(
124 + ("IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest", "pre-tax income"),
125 + ("IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments",
126 + "pre-tax income before equity-method investees"),
127 + ("IncomeLossFromContinuingOperationsBeforeIncomeTaxesDomestic", "domestic part only — last resort, flagged partial"),
128 + ("ProfitLossBeforeTax", "ifrs-full", "IFRS filers"),
129 + )),
130 + Account("income_tax", INCOME, DURATION, USD, _t(
131 + ("IncomeTaxExpenseBenefit", "total income tax expense"),
132 + ("IncomeTaxesPaidNet", "cash taxes paid — last resort, flagged (cash basis)"),
133 + ("IncomeTaxExpenseContinuingOperations", "ifrs-full", "IFRS filers"),
134 + )),
135 + Account("net_income", INCOME, DURATION, USD, _t(
136 + ("NetIncomeLoss", "net income attributable to the parent"),
137 + ("NetIncomeLossAvailableToCommonStockholdersBasic", "after preferred dividends"),
138 + ("ProfitLoss", "net income including non-controlling interests"),
139 + ("IncomeLossFromContinuingOperations", "continuing operations only"),
140 + ("ProfitLossAttributableToOwnersOfParent", "ifrs-full", "IFRS filers"),
141 + ("ProfitLoss", "ifrs-full", "IFRS filers, incl. NCI"),
142 + )),
143 + Account("eps_basic", INCOME, DURATION, PER_SHARE, _t(
144 + ("EarningsPerShareBasic", "basic EPS"),
145 + ("EarningsPerShareBasicAndDiluted", "single EPS figure when basic = diluted"),
146 + ("IncomeLossFromContinuingOperationsPerBasicShare", "continuing operations only"),
147 + ("BasicEarningsLossPerShare", "ifrs-full", "IFRS filers"),
148 + )),
149 + Account("eps_diluted", INCOME, DURATION, PER_SHARE, _t(
150 + ("EarningsPerShareDiluted", "diluted EPS"),
151 + ("EarningsPerShareBasicAndDiluted", "single EPS figure when basic = diluted"),
152 + ("IncomeLossFromContinuingOperationsPerDilutedShare", "continuing operations only"),
153 + ("DilutedEarningsLossPerShare", "ifrs-full", "IFRS filers"),
154 + )),
155 + Account("shares_basic", INCOME, DURATION, SHARES, _t(
156 + ("WeightedAverageNumberOfSharesOutstandingBasic", "weighted average basic shares"),
157 + ("WeightedAverageNumberOfShareOutstandingBasicAndDiluted", "single figure when basic = diluted"),
158 + )),
159 + Account("shares_diluted", INCOME, DURATION, SHARES, _t(
160 + ("WeightedAverageNumberOfDilutedSharesOutstanding", "weighted average diluted shares"),
161 + ("WeightedAverageNumberOfShareOutstandingBasicAndDiluted", "single figure when basic = diluted"),
162 + )),
163 + Account("ebitda", INCOME, DURATION, USD, computed=True,
164 + formula="operating_income + depreciation_amortization"),
165 + Account("dividends_paid", INCOME, DURATION, USD, _t(
166 + ("DividendsCommonStockCash", "cash dividends declared on common stock"),
167 + ("DividendsCommonStock", "dividends declared on common stock (cash + stock)"),
168 + ("DividendsCash", "cash dividends declared, all classes"),
169 + ("Dividends", "dividends declared, all classes"),
170 + ), notes="Dividends *declared* (equity statement). Cash paid is `dividends` in the cash flow statement."),
171 + # ------------------------------------------------------------------ balance sheet (instant)
172 + Account("cash_and_equivalents", BALANCE, INSTANT, USD, _t(
173 + ("CashAndCashEquivalentsAtCarryingValue", "cash & equivalents"),
174 + ("CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents", "incl. restricted cash (ASU 2016-18 presentation)"),
175 + ("Cash", "cash only"),
176 + ("CashAndDueFromBanks", "banks"),
177 + ("CashCashEquivalentsAndShortTermInvestments", "incl. short-term investments — last resort, flagged"),
178 + ("CashAndCashEquivalents", "ifrs-full", "IFRS filers"),
179 + )),
180 + Account("short_term_investments", BALANCE, INSTANT, USD, _t(
181 + ("ShortTermInvestments", "short-term investments"),
182 + ("MarketableSecuritiesCurrent", "current marketable securities (Apple, Microsoft…)"),
183 + ("AvailableForSaleSecuritiesDebtSecuritiesCurrent", "AFS debt securities, current"),
184 + ("AvailableForSaleSecuritiesCurrent", "AFS securities, current (pre-2018)"),
185 + ("HeldToMaturitySecuritiesCurrent", "HTM securities, current"),
186 + ("TradingSecuritiesCurrent", "trading securities, current"),
187 + )),
188 + Account("receivables", BALANCE, INSTANT, USD, _t(
189 + ("AccountsReceivableNetCurrent", "trade receivables, net"),
190 + ("ReceivablesNetCurrent", "all current receivables, net"),
191 + ("AccountsNotesAndLoansReceivableNetCurrent", "accounts + notes + loans receivable"),
192 + ("AccountsReceivableNet", "receivables without current/non-current split"),
193 + ("TradeAndOtherCurrentReceivables", "ifrs-full", "IFRS filers"),
194 + )),
195 + Account("inventory", BALANCE, INSTANT, USD, _t(
196 + ("InventoryNet", "inventories, net"),
197 + ("InventoryGross", "gross inventories — last resort, flagged"),
198 + ("Inventories", "ifrs-full", "IFRS filers"),
199 + )),
200 + Account("total_current_assets", BALANCE, INSTANT, USD, _t(
201 + ("AssetsCurrent", "total current assets"),
202 + ("CurrentAssets", "ifrs-full", "IFRS filers"),
203 + )),
204 + Account("ppe_net", BALANCE, INSTANT, USD, _t(
205 + ("PropertyPlantAndEquipmentNet", "PP&E net"),
206 + ("PropertyPlantAndEquipmentAndFinanceLeaseRightOfUseAssetAfterAccumulatedDepreciationAndAmortization",
207 + "PP&E incl. finance-lease ROU assets"),
208 + ("PropertyPlantAndEquipment", "ifrs-full", "IFRS filers"),
209 + )),
210 + Account("goodwill", BALANCE, INSTANT, USD, _t(
211 + ("Goodwill", "goodwill"),
212 + ("Goodwill", "ifrs-full", "IFRS filers"),
213 + )),
214 + Account("intangibles", BALANCE, INSTANT, USD, _t(
215 + ("IntangibleAssetsNetExcludingGoodwill", "intangibles ex-goodwill"),
216 + ("FiniteLivedIntangibleAssetsNet", "finite-lived intangibles only"),
217 + ("IndefiniteLivedIntangibleAssetsExcludingGoodwill", "indefinite-lived only — partial"),
218 + ("IntangibleAssetsOtherThanGoodwill", "ifrs-full", "IFRS filers"),
219 + )),
220 + Account("total_assets", BALANCE, INSTANT, USD, _t(
221 + ("Assets", "total assets"),
222 + ("Assets", "ifrs-full", "IFRS filers"),
223 + )),
224 + Account("accounts_payable", BALANCE, INSTANT, USD, _t(
225 + ("AccountsPayableCurrent", "trade payables"),
226 + ("AccountsPayableTradeCurrent", "trade payables (explicit)"),
227 + ("AccountsPayableAndAccruedLiabilitiesCurrent", "payables + accrued liabilities — flagged broader"),
228 + ("TradeAndOtherCurrentPayables", "ifrs-full", "IFRS filers"),
229 + )),
230 + Account("short_term_debt", BALANCE, INSTANT, USD, _t(
231 + ("DebtCurrent", "total debt due within one year"),
232 + ("LongTermDebtAndCapitalLeaseObligationsCurrent", "current portion of LTD incl. finance leases"),
233 + ("LongTermDebtCurrent", "current portion of long-term debt (component)"),
234 + ("ShortTermBorrowings", "short-term borrowings (component)"),
235 + ("CommercialPaper", "commercial paper (component)"),
236 + ("NotesPayableCurrent", "notes payable, current (component)"),
237 + ("ShorttermBorrowings", "ifrs-full", "IFRS filers"),
238 + ), notes="When DebtCurrent is absent normalize.py sums the disjoint components (LongTermDebtCurrent or "
239 + "LongTermDebtAndCapitalLeaseObligationsCurrent, ShortTermBorrowings, CommercialPaper, NotesPayableCurrent) "
240 + "and flags `components`."),
241 + Account("total_current_liabilities", BALANCE, INSTANT, USD, _t(
242 + ("LiabilitiesCurrent", "total current liabilities"),
243 + ("CurrentLiabilities", "ifrs-full", "IFRS filers"),
244 + )),
245 + Account("long_term_debt", BALANCE, INSTANT, USD, _t(
246 + ("LongTermDebtNoncurrent", "long-term debt, non-current portion"),
247 + ("LongTermDebtAndCapitalLeaseObligations", "non-current LTD incl. finance leases"),
248 + ("LongTermNotesPayable", "long-term notes"),
249 + ("SeniorLongTermNotes", "senior notes"),
250 + ("ConvertibleLongTermNotesPayable", "convertible notes"),
251 + ("OtherLongTermDebtNoncurrent", "other LTD"),
252 + ("LongTermDebt", "total LTD incl. current portion — last resort, flagged `includes_current`"),
253 + ("NoncurrentPortionOfNoncurrentBorrowings", "ifrs-full", "IFRS filers"),
254 + )),
255 + Account("total_liabilities", BALANCE, INSTANT, USD, _t(
256 + ("Liabilities", "total liabilities"),
257 + ("Liabilities", "ifrs-full", "IFRS filers"),
258 + ), formula="LiabilitiesAndStockholdersEquity - total_equity (identity fallback, flagged) — many filers do not tag Liabilities"),
259 + Account("retained_earnings", BALANCE, INSTANT, USD, _t(
260 + ("RetainedEarningsAccumulatedDeficit", "retained earnings / accumulated deficit"),
261 + ("RetainedEarnings", "ifrs-full", "IFRS filers"),
262 + )),
263 + Account("total_equity", BALANCE, INSTANT, USD, _t(
264 + ("StockholdersEquity", "equity attributable to the parent"),
265 + ("StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest", "total equity incl. NCI"),
266 + ("PartnersCapital", "partnerships (MLPs)"),
267 + ("MembersEquity", "LLCs"),
268 + ("EquityAttributableToOwnersOfParent", "ifrs-full", "IFRS filers"),
269 + ("Equity", "ifrs-full", "IFRS filers, incl. NCI"),
270 + )),
271 + Account("total_debt", BALANCE, INSTANT, USD, computed=True,
272 + formula="short_term_debt + long_term_debt (short_term_debt null → 0, flagged `short_term_debt_assumed_zero`)"),
273 + Account("net_debt", BALANCE, INSTANT, USD, computed=True,
274 + formula="total_debt - cash_and_equivalents - short_term_investments (missing STI → 0, flagged)"),
275 + Account("working_capital", BALANCE, INSTANT, USD, computed=True,
276 + formula="total_current_assets - total_current_liabilities"),
277 + # ------------------------------------------------------------------ cash flow (duration)
278 + Account("operating_cash_flow", CASHFLOW, DURATION, USD, _t(
279 + ("NetCashProvidedByUsedInOperatingActivities", "cash from operations"),
280 + ("NetCashProvidedByUsedInOperatingActivitiesContinuingOperations", "continuing operations only"),
281 + ("CashFlowsFromUsedInOperatingActivities", "ifrs-full", "IFRS filers"),
282 + )),
283 + Account("capex", CASHFLOW, DURATION, USD, _t(
284 + ("PaymentsToAcquirePropertyPlantAndEquipment", "purchases of PP&E (positive = outflow)"),
285 + ("PaymentsToAcquireProductiveAssets", "productive assets (PP&E + intangibles)"),
286 + ("PaymentsForCapitalImprovements", "capital improvements (REITs)"),
287 + ("PaymentsToAcquireOtherPropertyPlantAndEquipment", "other PP&E"),
288 + ("PurchaseOfPropertyPlantAndEquipmentClassifiedAsInvestingActivities", "ifrs-full", "IFRS filers"),
289 + ), notes="Reported as a positive outflow (EDGAR convention). free_cash_flow subtracts it."),
290 + Account("free_cash_flow", CASHFLOW, DURATION, USD, computed=True, formula="operating_cash_flow - capex"),
291 + Account("acquisitions", CASHFLOW, DURATION, USD, _t(
292 + ("PaymentsToAcquireBusinessesNetOfCashAcquired", "acquisitions net of cash acquired"),
293 + ("PaymentsToAcquireBusinessesGross", "acquisitions gross"),
294 + ("PaymentsToAcquireBusinessesAndInterestInAffiliates", "businesses + affiliates"),
295 + ), notes="Positive = outflow."),
296 + Account("investing_cash_flow", CASHFLOW, DURATION, USD, _t(
297 + ("NetCashProvidedByUsedInInvestingActivities", "cash from investing"),
298 + ("NetCashProvidedByUsedInInvestingActivitiesContinuingOperations", "continuing operations only"),
299 + ("CashFlowsFromUsedInInvestingActivities", "ifrs-full", "IFRS filers"),
300 + )),
301 + Account("debt_issued", CASHFLOW, DURATION, USD, _t(
302 + ("ProceedsFromIssuanceOfLongTermDebt", "LTD issued"),
303 + ("ProceedsFromIssuanceOfDebt", "all debt issued"),
304 + ("ProceedsFromIssuanceOfSeniorLongTermDebt", "senior notes issued"),
305 + ("ProceedsFromDebtNetOfIssuanceCosts", "net of issuance costs"),
306 + ("ProceedsFromNotesPayable", "notes payable"),
307 + ("ProceedsFromBorrowingsClassifiedAsFinancingActivities", "ifrs-full", "IFRS filers"),
308 + )),
309 + Account("debt_repaid", CASHFLOW, DURATION, USD, _t(
310 + ("RepaymentsOfLongTermDebt", "LTD repaid"),
311 + ("RepaymentsOfDebt", "all debt repaid"),
312 + ("RepaymentsOfLongTermDebtAndCapitalSecurities", "LTD + capital securities"),
313 + ("RepaymentsOfDebtAndCapitalLeaseObligations", "debt + finance leases"),
314 + ("RepaymentsOfSeniorDebt", "senior notes"),
315 + ("RepaymentsOfNotesPayable", "notes payable"),
316 + ("RepaymentsOfBorrowingsClassifiedAsFinancingActivities", "ifrs-full", "IFRS filers"),
317 + ), notes="Positive = outflow."),
318 + Account("buybacks", CASHFLOW, DURATION, USD, _t(
319 + ("PaymentsForRepurchaseOfCommonStock", "common stock repurchased"),
320 + ("PaymentsForRepurchaseOfEquity", "all equity repurchased"),
321 + ("StockRepurchasedDuringPeriodValue", "equity statement figure — last resort, flagged (accrual basis)"),
322 + ("PaymentsToAcquireOrRedeemEntitysShares", "ifrs-full", "IFRS filers"),
323 + ), notes="Positive = outflow."),
324 + Account("dividends", CASHFLOW, DURATION, USD, _t(
325 + ("PaymentsOfDividends", "all dividends paid"),
326 + ("PaymentsOfDividendsCommonStock", "common dividends paid"),
327 + ("PaymentsOfOrdinaryDividends", "ordinary dividends paid"),
328 + ("DividendsPaidClassifiedAsFinancingActivities", "ifrs-full", "IFRS filers"),
329 + ), notes="Cash dividends paid (positive = outflow)."),
330 + Account("financing_cash_flow", CASHFLOW, DURATION, USD, _t(
331 + ("NetCashProvidedByUsedInFinancingActivities", "cash from financing"),
332 + ("NetCashProvidedByUsedInFinancingActivitiesContinuingOperations", "continuing operations only"),
333 + ("CashFlowsFromUsedInFinancingActivities", "ifrs-full", "IFRS filers"),
334 + )),
335 + Account("net_change_in_cash", CASHFLOW, DURATION, USD, _t(
336 + ("CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect",
337 + "ASU 2016-18 presentation incl. FX effect"),
338 + ("CashAndCashEquivalentsPeriodIncreaseDecrease", "pre-2018 presentation"),
339 + ("CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseExcludingExchangeRateEffect",
340 + "ex-FX effect"),
341 + ("CashAndCashEquivalentsPeriodIncreaseDecreaseExcludingExchangeRateEffect", "pre-2018, ex-FX"),
342 + ("IncreaseDecreaseInCashAndCashEquivalents", "ifrs-full", "IFRS filers"),
343 + )),
344 + Account("stock_based_compensation", CASHFLOW, DURATION, USD, _t(
345 + ("ShareBasedCompensation", "SBC add-back in the cash flow statement"),
346 + ("AllocatedShareBasedCompensationExpense", "SBC expense recognised (income statement note)"),
347 + ("StockOptionPlanExpense", "option plans only"),
348 + )),
349 + # ------------------------------------------------------------------ auxiliary inputs (not in the public chart)
350 + Account("depreciation_amortization", CASHFLOW, DURATION, USD, _t(
351 + ("DepreciationDepletionAndAmortization", "D&A add-back"),
352 + ("DepreciationAndAmortization", "D&A"),
353 + ("DepreciationAmortizationAndAccretionNet", "D&A + accretion"),
354 + ("Depreciation", "depreciation only — flagged partial"),
355 + ("DepreciationAmortizationAndOther", "D&A and other"),
356 + ("DepreciationAndAmortisationExpense", "ifrs-full", "IFRS filers"),
357 + ), auxiliary=True, notes="Input for ebitda."),
358 + Account("shares_outstanding", BALANCE, INSTANT, SHARES, _t(
359 + ("EntityCommonStockSharesOutstanding", "dei", "cover-page shares outstanding (all classes summed)"),
360 + ("CommonStockSharesOutstanding", "balance-sheet shares outstanding"),
361 + ), auxiliary=True, notes="Input for market_cap. dei facts are dated at the cover date and attached to the filing's fiscal period."),
362 +)
363 +
364 +ACCOUNT_BY_NAME: dict[str, Account] = {a.name: a for a in ACCOUNTS}
365 +PUBLIC_ACCOUNTS: tuple[str, ...] = tuple(a.name for a in ACCOUNTS if not a.auxiliary)
366 +ALL_ACCOUNT_NAMES: tuple[str, ...] = tuple(a.name for a in ACCOUNTS)
367 +COMPUTED_ACCOUNTS: tuple[str, ...] = tuple(a.name for a in ACCOUNTS if a.computed)
368 +STATEMENTS = (INCOME, BALANCE, CASHFLOW)
369 +
370 +
371 +def accounts_for(statement: str) -> list[Account]:
372 + return [a for a in ACCOUNTS if a.statement == statement]
373 +
374 +
375 +def mapping_rows() -> list[dict]:
376 + """Flat rows for the `fund_mapping` table (one per (account, tag))."""
377 + rows: list[dict] = []
378 + for a in ACCOUNTS:
379 + for prio, t in enumerate(a.tags, start=1):
380 + rows.append({"version": MAPPING_VERSION, "standard_account": a.name, "statement": a.statement,
381 + "priority": prio, "taxonomy": t.taxonomy, "tag": t.tag, "kind": a.kind, "unit": a.unit,
382 + "notes": t.notes})
383 + return rows
384 +
385 +
386 +# tag -> [(account, priority)] reverse index, used to spot unmapped tags quickly
387 +TAG_INDEX: dict[tuple[str, str], list[tuple[str, int]]] = {}
388 +for _a in ACCOUNTS:
389 + for _p, _t in enumerate(_a.tags, start=1):
390 + TAG_INDEX.setdefault((_t.taxonomy, _t.tag), []).append((_a.name, _p))
391 +
392 +
393 +def is_mapped(taxonomy: str, tag: str) -> bool:
394 + return (taxonomy, tag) in TAG_INDEX
395 +
396 +
397 +# Tags that are useful to keep in the raw lake even though they are not mapped (identity fallbacks).
398 +IDENTITY_TAGS: tuple[tuple[str, str], ...] = (
399 + ("us-gaap", "CostsAndExpenses"),
400 + ("us-gaap", "LiabilitiesAndStockholdersEquity"),
401 + ("us-gaap", "OperatingExpenses"),
402 +)
403 +
404 +SIGN_CONVENTION = "Outflows (capex, acquisitions, buybacks, dividends, debt_repaid) are positive numbers."
added hfmarketdata/api/fundamentals/models.py +204 −0
@@ -0,0 +1,204 @@
1 +"""SQLite schema of the fundamentals module (metadata + standardized statements).
2 +
3 +Raw XBRL facts are NOT here: they live in the Parquet lake `data_root/edgar/facts/cik={cik}/facts.parquet`
4 +(queried with DuckDB). SQLite holds what needs indexes and point-in-time lookups:
5 +
6 +* `edgar_companies` CIK ↔ ticker(s) (share classes, history), SIC, exchange, fiscal year end, status
7 +* `edgar_filings` one row per filing (accession number), amendments flagged
8 +* `fund_statements` wide standardized statements, **versioned by filed_date** (point-in-time index on
9 + (ticker, period_end, filed_date)); `coverage` JSON explains every null
10 +* `fund_mapping` the prioritized tag mapping, versioned (seeded from mapping.py)
11 +* `fund_mapping_log` unmapped / custom-extension tags seen per CIK (never guessed)
12 +* `fund_coverage` per ticker: periods, completeness %, gaps
13 +* `fund_ingest_state` lag, last checks, failures (exposed on /v1/fundamentals/_health)
14 +* `fund_latest` precomputed screener table (latest TTM values + ratios per ticker)
15 +
16 +Author: Simon-Pierre Boucher <contact@spboucher.ai>
17 +"""
18 +from __future__ import annotations
19 +
20 +from datetime import date, datetime
21 +
22 +from sqlalchemy import (JSON, Boolean, Column, Date, DateTime, Float, Index, Integer, String, Table, Text,
23 + UniqueConstraint)
24 +from sqlalchemy.orm import Mapped, mapped_column
25 +
26 +from core.db import Base, create_all
27 +
28 +from . import utcnow
29 +from .mapping import ALL_ACCOUNT_NAMES, PUBLIC_ACCOUNTS
30 +from .ratios import RATIO_NAMES
31 +
32 +
33 +class EdgarCompany(Base):
34 + __tablename__ = "edgar_companies"
35 + cik: Mapped[int] = mapped_column(Integer, primary_key=True)
36 + ticker: Mapped[str] = mapped_column(String(16), index=True) # primary ticker (first SEC listing)
37 + tickers: Mapped[list] = mapped_column(JSON, default=list) # all share classes, e.g. ["GOOGL","GOOG"]
38 + name: Mapped[str] = mapped_column(String(255), default="")
39 + sic: Mapped[str | None] = mapped_column(String(8), nullable=True)
40 + sic_description: Mapped[str | None] = mapped_column(String(255), nullable=True)
41 + exchange: Mapped[str | None] = mapped_column(String(32), nullable=True)
42 + fiscal_year_end: Mapped[str | None] = mapped_column(String(4), nullable=True) # MMDD, e.g. "0930"
43 + status: Mapped[str] = mapped_column(String(16), default="active") # active | delisted
44 + ticker_history: Mapped[list] = mapped_column(JSON, default=list) # [{"ticker","from","to"}]
45 + state_of_incorporation: Mapped[str | None] = mapped_column(String(8), nullable=True)
46 + facts_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
47 + normalized_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
48 + updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow)
49 +
50 +
51 +class EdgarFiling(Base):
52 + __tablename__ = "edgar_filings"
53 + accn: Mapped[str] = mapped_column(String(24), primary_key=True) # 0000320193-24-000069
54 + cik: Mapped[int] = mapped_column(Integer, index=True)
55 + form: Mapped[str] = mapped_column(String(16), index=True) # 10-K, 10-Q, 8-K, 20-F, 10-K/A…
56 + filed_date: Mapped[date] = mapped_column(Date, index=True)
57 + period_of_report: Mapped[date | None] = mapped_column(Date, nullable=True)
58 + primary_doc: Mapped[str | None] = mapped_column(String(512), nullable=True) # full EDGAR URL
59 + is_amendment: Mapped[bool] = mapped_column(Boolean, default=False)
60 + is_xbrl: Mapped[bool] = mapped_column(Boolean, default=False)
61 + parsed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
62 + __table_args__ = (Index("ix_edgar_filings_cik_filed", "cik", "filed_date"),)
63 +
64 +
65 +class FundMapping(Base):
66 + __tablename__ = "fund_mapping"
67 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
68 + version: Mapped[str] = mapped_column(String(16), index=True)
69 + standard_account: Mapped[str] = mapped_column(String(64), index=True)
70 + statement: Mapped[str] = mapped_column(String(16))
71 + priority: Mapped[int] = mapped_column(Integer)
72 + taxonomy: Mapped[str] = mapped_column(String(16))
73 + tag: Mapped[str] = mapped_column(String(255))
74 + kind: Mapped[str] = mapped_column(String(8)) # duration | instant
75 + unit: Mapped[str] = mapped_column(String(16))
76 + notes: Mapped[str] = mapped_column(Text, default="")
77 + __table_args__ = (UniqueConstraint("version", "standard_account", "taxonomy", "tag", name="uq_fund_mapping"),)
78 +
79 +
80 +class FundMappingLog(Base):
81 + """Tags we saw but did not map: standard-taxonomy tags outside the mapping and company extensions."""
82 + __tablename__ = "fund_mapping_log"
83 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
84 + cik: Mapped[int] = mapped_column(Integer, index=True)
85 + taxonomy: Mapped[str] = mapped_column(String(64)) # us-gaap | aapl | msft | …
86 + tag: Mapped[str] = mapped_column(String(255))
87 + is_extension: Mapped[bool] = mapped_column(Boolean, default=False)
88 + occurrences: Mapped[int] = mapped_column(Integer, default=0)
89 + first_seen: Mapped[date | None] = mapped_column(Date, nullable=True)
90 + last_seen: Mapped[date | None] = mapped_column(Date, nullable=True)
91 + sample_accn: Mapped[str | None] = mapped_column(String(24), nullable=True)
92 + hint_account: Mapped[str | None] = mapped_column(String(64), nullable=True) # account left null when this tag appeared
93 + __table_args__ = (UniqueConstraint("cik", "taxonomy", "tag", name="uq_fund_mapping_log"),)
94 +
95 +
96 +class FundCoverage(Base):
97 + __tablename__ = "fund_coverage"
98 + ticker: Mapped[str] = mapped_column(String(16), primary_key=True)
99 + cik: Mapped[int] = mapped_column(Integer, index=True)
100 + first_period_end: Mapped[date | None] = mapped_column(Date, nullable=True)
101 + last_period_end: Mapped[date | None] = mapped_column(Date, nullable=True)
102 + quarters: Mapped[int] = mapped_column(Integer, default=0)
103 + annuals: Mapped[int] = mapped_column(Integer, default=0)
104 + filings: Mapped[int] = mapped_column(Integer, default=0)
105 + completeness: Mapped[float | None] = mapped_column(Float, nullable=True) # % of public accounts non-null
106 + completeness_by_statement: Mapped[dict] = mapped_column(JSON, default=dict)
107 + missing_accounts: Mapped[dict] = mapped_column(JSON, default=dict) # account -> {"reason", "periods"}
108 + gaps: Mapped[list] = mapped_column(JSON, default=list) # missing fiscal quarters
109 + derived_quarters: Mapped[int] = mapped_column(Integer, default=0)
110 + restated_periods: Mapped[int] = mapped_column(Integer, default=0)
111 + extensions_logged: Mapped[int] = mapped_column(Integer, default=0)
112 + last_filed_date: Mapped[date | None] = mapped_column(Date, nullable=True)
113 + updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow)
114 +
115 +
116 +class FundIngestState(Base):
117 + __tablename__ = "fund_ingest_state"
118 + key: Mapped[str] = mapped_column(String(32), primary_key=True) # backfill | incremental | reconcile
119 + last_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
120 + last_success_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
121 + last_rss_check_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
122 + last_filing_seen: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) # newest EDGAR filing acceptance
123 + lag_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
124 + companies_total: Mapped[int] = mapped_column(Integer, default=0)
125 + companies_done: Mapped[int] = mapped_column(Integer, default=0)
126 + failures: Mapped[int] = mapped_column(Integer, default=0)
127 + failure_samples: Mapped[list] = mapped_column(JSON, default=list)
128 + mapping_failure_rate: Mapped[float | None] = mapped_column(Float, nullable=True) # share of null public accounts
129 + requests_made: Mapped[int] = mapped_column(Integer, default=0)
130 + events_published: Mapped[int] = mapped_column(Integer, default=0)
131 + extra: Mapped[dict] = mapped_column(JSON, default=dict)
132 + updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow)
133 +
134 +
135 +# ------------------------------------------------------------------------------------------- wide tables
136 +STATEMENT_KEY_COLUMNS = ("cik", "ticker", "statement", "fiscal_year", "fiscal_quarter", "period_start", "period_end",
137 + "calendar_quarter", "form", "accn", "filed_date", "derived", "restated", "currency", "coverage",
138 + "mapping_version")
139 +
140 +fund_statements = Table(
141 + "fund_statements", Base.metadata,
142 + Column("id", Integer, primary_key=True, autoincrement=True),
143 + Column("cik", Integer, nullable=False, index=True),
144 + Column("ticker", String(16), nullable=False),
145 + Column("statement", String(16), nullable=False), # income | balance | cashflow
146 + Column("fiscal_year", Integer, nullable=False),
147 + Column("fiscal_quarter", Integer, nullable=False), # 1..4 ; 0 = fiscal year (annual)
148 + Column("period_start", Date, nullable=True), # null for balance sheet (instant)
149 + Column("period_end", Date, nullable=False),
150 + Column("calendar_quarter", String(6), nullable=False), # 2024Q1
151 + Column("form", String(16), nullable=False),
152 + Column("accn", String(24), nullable=False), # filing that made this version known
153 + Column("filed_date", Date, nullable=False),
154 + Column("derived", Boolean, default=False), # Q4 / de-cumulated quarter
155 + Column("restated", Boolean, default=False), # differs from an earlier version of the same period
156 + Column("currency", String(3), default="USD"),
157 + Column("coverage", JSON, default=dict), # {account: {"reason": …, "tag": …, "computed": …}}
158 + Column("mapping_version", String(16)),
159 + *[Column(name, Float, nullable=True) for name in ALL_ACCOUNT_NAMES],
160 + UniqueConstraint("cik", "statement", "fiscal_year", "fiscal_quarter", "accn", name="uq_fund_statements_version"),
161 + Index("ix_fund_statements_pit", "ticker", "period_end", "filed_date"),
162 + Index("ix_fund_statements_period", "cik", "statement", "fiscal_year", "fiscal_quarter", "filed_date"),
163 + Index("ix_fund_statements_cal", "statement", "fiscal_quarter", "calendar_quarter"),
164 +)
165 +
166 +LATEST_KEY_COLUMNS = ("ticker", "cik", "name", "sic", "exchange", "period_end", "fiscal_year", "fiscal_quarter",
167 + "filed_date", "price", "price_date", "shares_source", "reasons", "updated_at")
168 +
169 +fund_latest = Table(
170 + "fund_latest", Base.metadata,
171 + Column("ticker", String(16), primary_key=True),
172 + Column("cik", Integer, index=True),
173 + Column("name", String(255)),
174 + Column("sic", String(8)),
175 + Column("exchange", String(32)),
176 + Column("period_end", Date), # latest balance sheet date
177 + Column("fiscal_year", Integer),
178 + Column("fiscal_quarter", Integer),
179 + Column("filed_date", Date), # latest filing used
180 + Column("price", Float),
181 + Column("price_date", Date),
182 + Column("shares_source", String(24)),
183 + Column("reasons", JSON, default=dict),
184 + Column("updated_at", DateTime, default=utcnow),
185 + *[Column(name, Float, nullable=True) for name in PUBLIC_ACCOUNTS], # TTM flows / latest balances
186 + *[Column(name, Float, nullable=True) for name in RATIO_NAMES if name not in PUBLIC_ACCOUNTS],
187 +)
188 +
189 +SCREENER_FIELDS: tuple[str, ...] = tuple(
190 + ["price", *PUBLIC_ACCOUNTS, *[n for n in RATIO_NAMES if n not in PUBLIC_ACCOUNTS]])
191 +SCREENER_TEXT_FIELDS: tuple[str, ...] = ("ticker", "sic", "exchange", "name")
192 +
193 +
194 +def init_db() -> None:
195 + """Idempotent create_all + mapping seed (called at module import by routes.py and by the scripts)."""
196 + create_all()
197 + from .mapping import MAPPING_VERSION, mapping_rows
198 + from core.db import session
199 + from sqlalchemy import select
200 + with session() as s:
201 + have = s.scalar(select(FundMapping.id).where(FundMapping.version == MAPPING_VERSION).limit(1))
202 + if have is None:
203 + for r in mapping_rows():
204 + s.add(FundMapping(**r))
modified hfmarketdata/api/openapi.py +2 −1
@@ -56,7 +56,8 @@ STATUS_FOR = {"INVALID_PARAMETER": 400, "VALIDATION_ERROR": 422, "NOT_FOUND": 40
56 56 "ASSET_NOT_FOUND": 404, "INVALID_CONTRACT_SYMBOL": 400, "CONTRACT_NOT_FOUND": 404, "ROOT_NOT_FOUND": 404,
57 57 "OPTIONS_UNAVAILABLE": 503, "INVALID_API_KEY": 401, "AUTH_REQUIRED": 401, "FORBIDDEN": 403,
58 58 "RATE_LIMIT_EXCEEDED": 429, "ROW_LIMIT_EXCEEDED": 400, "CONFLICT": 409, "INTERNAL_ERROR": 500,
59 − "SERVICE_UNAVAILABLE": 503}
59 + "SERVICE_UNAVAILABLE": 503, "FUNDAMENTALS_NOT_AVAILABLE": 404, "INVALID_FILTER": 400,
60 + "CONCEPT_NOT_FOUND": 404, "STREAM_CONNECTION_LIMIT": 429}
60 61
61 62
62 63 def build(app: FastAPI) -> dict:
63 64