1"use client";2import Link from "next/link";3import { ExternalLink } from "lucide-react";4import { Banner, Card, PageHeader } from "@/components/ui";56const BAR_COLUMNS: [string, string, string][] = [7 ["instrument_id", "string", "Stable id, e.g. firstrate:stock:AAPL"],8 ["symbol_as_reported", "string", "Ticker as delivered by the vendor at that date"],9 ["venue", "string?", "Exchange / venue when known"],10 ["asset_class", "string", "stock, etf, index, crypto, fx, futures"],11 ["interval", "string", "1m, 5m, 1d"],12 ["timestamp_utc", "timestamp[us, UTC]", "Bar OPEN instant, UTC"],13 ["session_date", "date", "US trading session (America/New_York)"],14 ["open / high / low / close", "float64", "Raw (as traded) prices"],15 ["volume", "float64", "Shares/contracts; zero-volume bars are absent"],16 ["currency", "string", "ISO 4217 (USD)"],17 ["trade_count", "int32?", "When supplied by the vendor"],18 ["vwap", "float64?", "When supplied by the vendor"],19 ["adjustment_type", "string", "Always raw in storage"],20 ["source_id", "string", "Source / product that produced the row"],21 ["ingested_at", "timestamp[us, UTC]", "Load instant"],22 ["quality_flags", "list<string>", "e.g. ohlc_inconsistent, price_jump, outside_session"],23 ["revision_id", "int32", "Increments when the vendor re-delivers a bar"],24];2526export default function DocsPage() {27 return (28 <>29 <PageHeader30 title="Documentation"31 subtitle="What FinData OS is, the data contract, licensing and how to use the API."32 actions={33 <>34 <a className="btn btn-sm" href="/api/docs" target="_blank" rel="noopener">35 Swagger UI <ExternalLink className="h-3.5 w-3.5" />36 </a>37 <a className="btn btn-sm" href="/api/v1/openapi.json" target="_blank" rel="noopener">38 OpenAPI JSON <ExternalLink className="h-3.5 w-3.5" />39 </a>40 </>41 }42 />43 <div className="prose-sm grid gap-4 lg:grid-cols-[220px_minmax(0,1fr)]">44 <nav className="card hidden h-fit lg:sticky lg:top-6 lg:block" aria-label="On this page">45 <ul className="flex flex-col gap-1 text-sm">46 {[["#platform", "The platform"], ["#contract", "Data contract"], ["#bars", "Bars columns"], ["#adjustments", "Adjustments"], ["#fundamentals", "Fundamentals (PIT)"], ["#license", "License"], ["#api", "API quick start"], ["#sdk", "Python SDK"], ["#limits", "Limits"], ["#limitations", "Limitations"]].map(([h, l]) => (47 <li key={h}>48 <a href={h} className="nav-link">49 {l}50 </a>51 </li>52 ))}53 </ul>54 </nav>55 <div className="flex min-w-0 flex-col gap-4">56 <Card title="The platform" id="platform">57 <p className="text-sm">58 <strong>FinData OS</strong> is a private, research-grade financial data lakehouse. It ingests vendor files (FirstRate Data intraday bars, corporate actions and fundamentals; SEC EDGAR when configured) into an immutable Parquet lake with a59 Postgres catalog, versions every dataset, checks quality at load time, and serves the result through one HTTP API that this console, the Python SDK and ad-hoc SQL all share. One writer (the ingest worker) owns the lake; the API is read-only60 except for job control, keys and settings.61 </p>62 <ul className="mt-2 list-disc pl-5 text-sm text-ink-2">63 <li>Curated layer: Parquet files per dataset, <code>base</code> files plus <code>delta</code> appends, compacted into new versions.</li>64 <li>Raw layer: vendor archives kept for re-processing (hash-verified).</li>65 <li>Catalog: datasets, versions, files, coverage per (symbol, session), instruments, corporate actions, jobs, exports, audit.</li>66 </ul>67 </Card>6869 <Card title="Data contract" id="contract">70 <dl className="grid gap-x-6 gap-y-2 text-sm sm:grid-cols-[auto_1fr]">71 <dt className="font-medium">Timestamp convention</dt>72 <dd>73 Every bar is stamped at its <strong>open</strong>: the 1-minute bar labelled 13:30:00Z covers 13:30:00 → 13:30:59.999 UTC.74 </dd>75 <dt className="font-medium">Storage timezone</dt>76 <dd>77 <code>timestamp_utc</code> is UTC. <code>session_date</code> is the US trading session in America/New_York; it is the key for coverage and expected-bar counts.78 </dd>79 <dt className="font-medium">Session span</dt>80 <dd>81 Bars are retained for the extended session <strong>04:00 – 20:00 ET</strong>; regular hours are 09:30 – 16:00 ET. Completeness = bars ÷ expected regular-hours bars, capped at 100 %.82 </dd>83 <dt className="font-medium">Zero-volume bars</dt>84 <dd>Minutes without a trade are <strong>absent</strong>, not stored as zero rows. A gap is not necessarily a vendor outage — check the coverage page.</dd>85 <dt className="font-medium">Adjustments</dt>86 <dd>87 Stored prices are <strong>raw</strong>. <code>split</code> and <code>split_div</code> are computed at query time from the corporate-actions table (ex-date based, backward factor), so a new split re-adjusts history without rewriting88 files. The API echoes <code>adjustment_notes</code> explaining what was applied.89 </dd>90 <dt className="font-medium">Versioning</dt>91 <dd>92 Each dataset has a <code>current_version</code> with a snapshot id. Exports pin the version at submission; <code>GET /market/bars?version=</code> can read an earlier one.93 </dd>94 <dt className="font-medium">Quality flags</dt>95 <dd>Rows are never silently dropped; suspicious bars carry <code>quality_flags</code> and are also recorded as anomalies.</dd>96 </dl>97 </Card>9899 <Card title="Bars schema (findata.schema = bars, version 1)" id="bars">100 <div className="table-wrap">101 <table className="table">102 <thead>103 <tr>104 <th>Column</th>105 <th>Arrow type</th>106 <th>Meaning</th>107 </tr>108 </thead>109 <tbody>110 {BAR_COLUMNS.map(([c, t, m]) => (111 <tr key={c}>112 <td className="font-mono text-xs">{c}</td>113 <td className="font-mono text-xs text-ink-2">{t}</td>114 <td className="text-xs">{m}</td>115 </tr>116 ))}117 </tbody>118 </table>119 </div>120 <p className="mt-2 text-xs text-ink-3">121 The authoritative schema is served by <code>GET /api/v1/catalog/schema/bars</code>. In the SQL playground each dataset is a DuckDB view named after its id with dots replaced by underscores, e.g. <code>market_bars_stock_1m_raw</code>.122 </p>123 </Card>124125 <Card title="Adjustments in practice" id="adjustments">126 <ul className="list-disc pl-5 text-sm">127 <li>128 <code>raw</code> — prices as traded; volumes as reported. Use for microstructure work and when you join your own actions.129 </li>130 <li>131 <code>split</code> — prices divided and volumes multiplied by the cumulative split factor of all splits with ex-date after the bar.132 </li>133 <li>134 <code>split_div</code> — additionally applies the cash-dividend backward factor <em>(1 − D / P<sub>close, prev</sub>)</em>. Total-return style; not suitable for tick-size studies.135 </li>136 </ul>137 </Card>138139 <Card title="Fundamentals — point in time" id="fundamentals">140 <p className="text-sm">141 Every fact carries <code>available_at</code> (= filing date + 1 day 00:00 America/New_York, a conservative knowability instant) and <code>observed_at</code> (when this platform first saw the value). A request with <code>as_of</code> returns142 only facts with <code>available_at ≤ as_of</code>; among revisions, the first observed value is used unless a later revision was observed before <code>as_of</code>. The response counts <code>facts_hidden_by_as_of</code> so you can tell how much of the143 record you are not seeing.144 </p>145 </Card>146147 <Card title="License and redistribution" id="license">148 <Banner tone="warn">149 <strong>FirstRate Data — private use only, no redistribution.</strong>150 </Banner>151 <ul className="mt-3 list-disc pl-5 text-sm">152 <li>Raw bars, exports, SQL results and tables derived row-for-row from them may be used by the subscriber only. They may <strong>not</strong> be published, shared, re-hosted or served through a public API.</li>153 <li>Aggregated results (charts, statistics, model outputs) may be published with attribution to FirstRate Data.</li>154 <li>SEC EDGAR content is U.S. public domain; the SEC fair-access policy (declared User-Agent, request rate) applies to retrieval.</li>155 <li>API keys are personal. Every download, export and query is written to the audit log with the principal and request id.</li>156 </ul>157 </Card>158159 <Card title="API quick start" id="api">160 <p className="text-sm">All endpoints live under <code>/api/v1</code>. Authenticate with the console session (cookie + <code>X-CSRF-Token</code> on writes) or with an API key: <code>Authorization: Bearer fd_…</code>. Errors are JSON: <code>{`{"error":{"code","message","request_id"}}`}</code>.</p>161 <pre className="mt-2 overflow-auto rounded-md border border-hairline bg-surface-2 p-3 text-xs leading-relaxed">{`export FINDATA_URL="https://findata.spboucher.ai"162export FINDATA_API_KEY="fd_xxx.yyy"163164# health + catalog165curl -s -H "Authorization: Bearer $FINDATA_API_KEY" "$FINDATA_URL/api/v1/health" | jq .status166curl -s -H "Authorization: Bearer $FINDATA_API_KEY" "$FINDATA_URL/api/v1/catalog/datasets" | jq '.data[].id'167168# one week of 1-minute bars, split-adjusted, as CSV (≤ 31 days per call for 1m)169curl -s -H "Authorization: Bearer $FINDATA_API_KEY" \\170 "$FINDATA_URL/api/v1/market/bars?symbol=AAPL&interval=1m&start=2026-10-20&end=2026-10-24&adjustment=split&format=csv&limit=10000" -o aapl.csv171172# bigger ranges: async export → poll → download173curl -s -X POST -H "Authorization: Bearer $FINDATA_API_KEY" -H "Content-Type: application/json" \\174 -d '{"dataset_id":"market_bars.stock.1m.raw","symbols":["AAPL","MSFT"],"start":"2024-01-01","end":"2026-10-30","format":"parquet"}' \\175 "$FINDATA_URL/api/v1/exports"176# → {"id":"…","status":"queued","poll":"/api/v1/exports/…"}; when status=ready: GET /api/v1/exports/{id}/download177178# point-in-time fundamentals179curl -s -H "Authorization: Bearer $FINDATA_API_KEY" \\180 "$FINDATA_URL/api/v1/fundamentals/AAPL?as_of=2025-01-01T00:00:00Z&metrics=rev,net_inc&period_type=q"181182# read-only SQL (DuckDB)183curl -s -X POST -H "Authorization: Bearer $FINDATA_API_KEY" -H "Content-Type: application/json" \\184 -d '{"sql":"SELECT session_date, sum(volume) v FROM market_bars_stock_1m_raw WHERE symbol_as_reported='"'"'AAPL'"'"' GROUP BY 1 ORDER BY 1 DESC LIMIT 5","max_rows":100}' \\185 "$FINDATA_URL/api/v1/query"`}</pre>186 </Card>187188 <Card title="Python SDK" id="sdk">189 <p className="text-sm">190 Install from the repo (<code>packages/python-sdk</code>): <code>pip install -e packages/python-sdk</code>. The client pages through Arrow IPC automatically.191 </p>192 <pre className="mt-2 overflow-auto rounded-md border border-hairline bg-surface-2 p-3 text-xs leading-relaxed">{`from findata_sdk import FinData193194fd = FinData("https://findata.spboucher.ai", api_key="fd_xxx.yyy") # or FINDATA_URL / FINDATA_API_KEY env vars195196bars = fd.bars("AAPL", "1m", "2026-10-26", "2026-10-30", adjustment="split") # pyarrow.Table, all pages197df = bars.to_pandas() # or .to_polars()198199path = fd.export(["AAPL", "MSFT"], "2024-01-01", "2026-10-30", format="parquet") # waits, downloads, returns local path200201facts = fd.fundamentals("AAPL", as_of="2025-01-01T00:00:00Z", metrics=["rev", "net_inc"])202print(fd.datasets()[0]["id"], fd.health()["status"])`}</pre>203 </Card>204205 <Card title="Limits" id="limits">206 <ul className="list-disc pl-5 text-sm">207 <li>208 <code>GET /market/bars</code>: span ≤ <strong>31 days</strong> for 1m, 186 days for 5m, 3 660 days for 1d; ≤ 10 000 rows per page (cursor pagination). HTTP 413 <code>range_too_large</code> otherwise → use{" "}209 <Link href="/exports/" className="text-accent-ink hover:underline">210 Exports211 </Link>212 .213 </li>214 <li>Exports: ≤ 500 symbols, ≤ 12 years, 3 concurrent per principal, files expire after ~72 h.</li>215 <li>SQL: one read-only statement, server-side timeout, ≤ 100 000 rows; the console displays at most 1 000 rows (download the CSV for the rest).</li>216 <li>Rate limit per key/session (default 600 requests/min, HTTP 429 with <code>Retry-After</code>).</li>217 </ul>218 </Card>219220 <Card title="Known limitations" id="limitations">221 <ul className="list-disc pl-5 text-sm">222 <li>223 <strong>Fundamentals revision history starts at first ingestion.</strong> Values observed before the platform existed are a single revision (proxy for “as originally filed”); true as-filed history for older periods is not reconstructed.224 </li>225 <li>226 <strong>SEC EDGAR is “pending configuration”</strong> until a fair-access User-Agent is set on the server (<code>SEC_EDGAR_USER_AGENT</code>). Until then the dataset is listed but empty.227 </li>228 <li>229 <strong>Unsupported:</strong> tick data, quotes/NBBO, order-book depth, options chains. Those datasets appear as <em>unsupported</em> in the catalog on purpose so nobody plans on them.230 </li>231 <li>Vendor files arrive with a lag (typically next day); the overview shows <em>vendor lag</em> and <em>days since last session</em> per dataset.</li>232 <li>The console is a static bundle served by the API; it needs JavaScript and a modern browser. No telemetry, no third-party assets.</li>233 </ul>234 </Card>235 </div>236 </div>237 </>238 );239}240