SPB Git

spb/findata

Public

docs: API quick start, STATUS skeleton

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent 0bdc298 Browse files

3 changed files +168 −0

added apps/web/scripts/mock-server.mjs +101 −0
@@ -0,0 +1,101 @@
1 +#!/usr/bin/env node
2 +/**
3 + * Local smoke-test server: serves `out/` exactly like the Python API does (directory index.html fallback,
4 + * 404.html, the same strict CSP) and answers /api/v1/* with fixtures so the console can be exercised
5 + * without Postgres or the lake. Never used in production.
6 + *
7 + * node scripts/mock-server.mjs [port=8787] [--signed-out] [--mfa]
8 + */
9 +import { createServer } from "node:http";
10 +import { existsSync, readFileSync, statSync } from "node:fs";
11 +import { extname, join, normalize } from "node:path";
12 +import { dirname } from "node:path";
13 +import { fileURLToPath } from "node:url";
14 +
15 +const root = join(dirname(fileURLToPath(import.meta.url)), "..", "out");
16 +const port = Number(process.argv.find((a) => /^\d+$/.test(a)) ?? 8787);
17 +let signedIn = !process.argv.includes("--signed-out");
18 +let mfaPassed = !process.argv.includes("--mfa");
19 +const CSP = "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'";
20 +const MIME = { ".html": "text/html; charset=utf-8", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".txt": "text/plain", ".json": "application/json", ".woff2": "font/woff2", ".ico": "image/x-icon" };
21 +
22 +const day = (n) => { const d = new Date(); d.setUTCDate(d.getUTCDate() - n); return d.toISOString().slice(0, 10); };
23 +const ts = (h) => new Date(Date.now() - h * 3600e3).toISOString();
24 +const health = {
25 + status: "warning", checked_at: ts(0),
26 + alerts: [{ severity: "warning", code: "stale_dataset", message: "market_bars.stock.1m.raw is 1 day(s) behind the vendor" }, { severity: "info", code: "no_backup_yet", message: "no backup recorded yet" }],
27 + components: { postgres: "ok", lake: { root: "/lake", free_gb: 412.3, curated_gb: 118.4, raw_gb: 61.2, soft_limit_gb: 250, hard_min_free_gb: 60 }, ingest_worker: "running", backup: { last: ts(30), age_hours: 30, verified: true }, ssd_smart: "Verified" },
28 + datasets: [{ dataset_id: "market_bars.stock.1m.raw", version: 7, last_session: day(1), days_since_last_session: 1, vendor_last_update: day(0), vendor_lag_days: 1 }, { dataset_id: "market_bars.stock.1d.raw", version: 3, last_session: day(1), days_since_last_session: 1, vendor_last_update: null, vendor_lag_days: null }],
29 +};
30 +const jobs = ["succeeded", "running", "queued", "failed", "dead", "succeeded"].map((status, i) => ({ id: `0000000${i}-aaaa-bbbb-cccc-00000000000${i}`, kind: ["incremental", "derive", "quality", "backfill", "fundamentals", "metadata"][i], dataset_id: "market_bars.stock.1m.raw", params: { period: "day" }, priority: 50, status, attempts: 1, max_attempts: 3, created_at: ts(i + 1), started_at: status === "queued" ? null : ts(i + 0.9), finished_at: status === "running" || status === "queued" ? null : ts(i + 0.5), last_error: status === "failed" || status === "dead" ? "HTTPError 503 vendor unavailable (retry 3/3)" : null, progress: status === "running" ? { files: 12, of: 40, symbol: "MSFT" } : null, result: status === "succeeded" ? { rows: 1_234_567 } : null, created_by: "owner@example.com" }));
31 +const datasets = [
32 + { id: "market_bars.stock.1m.raw", name: "US stocks — 1-minute bars", description: "Intraday OHLCV bars for US equities and ETFs from FirstRate Data; extended session 04:00–20:00 ET.", kind: "market_bars", asset_class: "stock", interval: "1m", adjustment: "raw", license_class: "frd-private", license_text: "FirstRate Data license: private use by the subscriber only.", status: "active", timestamp_convention: "bar_open", timezone: "UTC", current_version: 7, snapshot_id: "snap_7f3a", file_count: 2210, row_count: 9_143_221_004, bytes: 118_400_000_000, version_created_at: ts(20), first_session: "2007-01-03", last_session: day(1), symbols: 8123, source_name: "FirstRate Data", vendor: "firstrate", notes: null },
33 + { id: "market_bars.stock.5m.raw", name: "US stocks — 5-minute bars (derived)", description: "Derived from 1-minute bars.", kind: "derived", asset_class: "stock", interval: "5m", adjustment: "raw", license_class: "frd-private", license_text: "FirstRate Data license.", status: "active", timestamp_convention: "bar_open", timezone: "UTC", current_version: 3, snapshot_id: "snap_12aa", file_count: 410, row_count: 1_832_000_000, bytes: 24_000_000_000, first_session: "2007-01-03", last_session: day(1), symbols: 8123, source_name: "FinData derive", vendor: "findata" },
34 + { id: "fundamentals.frd.pit", name: "Fundamentals (FirstRate)", description: "Point-in-time quarterly and annual fundamentals.", kind: "fundamentals", asset_class: null, interval: null, adjustment: null, license_class: "frd-private", license_text: "FirstRate Data license.", status: "active", timestamp_convention: null, timezone: "UTC", current_version: 2, row_count: 2_410_000, bytes: 410_000_000, file_count: 12, symbols: 2410, source_name: "FirstRate Data", vendor: "firstrate" },
35 + { id: "fundamentals.sec.edgar", name: "SEC EDGAR XBRL facts", description: "Company facts from SEC EDGAR.", kind: "fundamentals", license_class: "public-domain", license_text: "U.S. government work.", status: "pending_configuration", timezone: "UTC", current_version: null, source_name: "SEC EDGAR", vendor: "sec" },
36 + { id: "market_bars.futures.1m.raw", name: "Futures — 1-minute bars", description: "Planned.", kind: "market_bars", asset_class: "futures", interval: "1m", adjustment: "raw", license_class: "pending", license_text: "License not yet determined.", status: "planned", timezone: "UTC", current_version: null, source_name: "FirstRate Data", vendor: "firstrate" },
37 + { id: "market_ticks.stock", name: "Tick data", description: "Not supported by this platform.", kind: "ticks", asset_class: "stock", license_class: "pending", license_text: "", status: "unsupported", timezone: "UTC", current_version: null, source_name: "—", vendor: "—" },
38 +];
39 +const bars = (n, iv) => Array.from({ length: n }, (_, i) => { const t = new Date(Date.UTC(2026, 9, 26, 13, 30) + i * (iv === "1d" ? 864e5 : 60e3)); const c = 230 + Math.sin(i / 9) * 4 + (i % 7) * 0.1; return { timestamp_utc: t.toISOString().replace(".000Z", "Z"), session_date: t.toISOString().slice(0, 10), open: c - 0.2, high: c + 0.5, low: c - 0.6, close: c, volume: 10000 + ((i * 7919) % 50000), vwap: c + 0.05, trade_count: 120, quality_flags: i % 37 === 0 ? ["price_jump"] : [], adjustment_type: "raw", revision_id: 1 }; });
40 +const sessions = Array.from({ length: 120 }, (_, i) => day(i)).filter((d) => ![0, 6].includes(new Date(d).getUTCDay())).reverse().map((d, i) => ({ session_date: d, symbols: 8000 + (i % 100), bars: 2_900_000 + (i % 13) * 10000, completeness: 0.8 + ((i * 37) % 20) / 100, non_trading_rows: 0 }));
41 +
42 +function api(method, path, q, body) {
43 + const p = path.replace(/^\/api\/v1/, "");
44 + if (p === "/auth/login" && method === "POST") { signedIn = true; mfaPassed = !process.argv.includes("--mfa"); return [200, { ok: true, mfa_required: !mfaPassed, csrf_token: "csrf-mock", email: body.email, role: "owner" }]; }
45 + if (p === "/auth/mfa" && method === "POST") { if (body.code !== "123456") return [401, { error: { code: "invalid_code", message: "Invalid one-time code", request_id: "rq1" } }]; mfaPassed = true; return [200, { ok: true, csrf_token: "csrf-mock" }]; }
46 + if (p === "/auth/logout") { signedIn = false; return [200, { ok: true }]; }
47 + if (p === "/auth/me") return signedIn ? [200, { kind: "user", email: "owner@example.com", role: "owner", scopes: ["admin", "read:market", "read:fundamentals", "query", "export"], mfa_passed: mfaPassed, totp_enabled: false, csrf_token: "csrf-mock" }] : [401, { error: { code: "unauthenticated", message: "Not signed in", request_id: "rq0" } }];
48 + if (!signedIn) return [401, { error: { code: "unauthenticated", message: "Not signed in", request_id: "rq0" } }];
49 + if (p === "/auth/totp/setup") return [200, { otpauth_uri: "otpauth://totp/x", qr_png_data_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=", secret: "JBSWY3DPEHPK3PXP" }];
50 + if (p === "/health") return [200, health];
51 + if (p === "/health/summary") return [200, { datasets: { active: 3, planned: 1, unsupported: 1, pending: 1 }, instruments: { n: 11240, active: 8123 }, bars_1m: { symbols: 8123, sessions: 4970, bars: 9_143_221_004, last: day(1) }, jobs: { succeeded: 412, running: 1, queued: 1, failed: 1, dead: 1 }, recent_jobs: jobs, fundamentals: { entities: 2410, facts: 2_410_000 }, anomalies_7d: { warning: 42, error: 3 }, health }];
52 + if (p === "/catalog/datasets") return [200, { data: datasets, meta: { count: datasets.length } }];
53 + if (p.startsWith("/catalog/datasets/")) { const d = datasets.find((x) => x.id === decodeURIComponent(p.split("/")[3])); return d ? [200, { data: d, versions: [7, 6, 5].map((v) => ({ version: v, snapshot_id: `snap_${v}`, file_count: 2000 + v, row_count: 9e9 - v * 1e8, bytes: 1.1e11, created_at: ts(24 * (8 - v)), note: v === 7 ? "compaction" : null })), files: [{ layout: "base", n: 2100, rows: 9e9, bytes: 1.1e11, d0: "2007-01-03", d1: day(20) }, { layout: "delta", n: 110, rows: 1.4e8, bytes: 2e9, d0: day(19), d1: day(1) }], checkpoints: [{ key: "last_vendor_update:day", value: { vendor_last_update: day(0) }, updated_at: ts(2) }] }] : [404, { error: { code: "not_found", message: "unknown dataset", request_id: "rq2" } }]; }
54 + if (p === "/instruments") return [200, { data: ["AAPL", "AAL", "AAON", "AAP", "AAPU"].filter((s) => !q.query || s.startsWith(q.query.toUpperCase())).map((s) => ({ instrument_id: `firstrate:stock:${s}`, symbol: s, name: `${s} Inc.`, asset_class: "stock", status: "active", first_date: "2007-01-03", last_date: day(1), currency: "USD" })), meta: { total_instruments: 11240, active: 8123 }, next_cursor: null }];
55 + if (p.startsWith("/instruments/")) { const s = p.split("/")[2]; return [200, { instrument: { instrument_id: `firstrate:stock:${s}`, symbol: s, name: `${s} Inc.`, asset_class: "stock", status: "active", first_date: "2007-01-03", last_date: day(1), currency: "USD" }, aliases: [], corporate_actions: [{ action_type: "split", ex_date: "2020-08-31", value: { ratio: 4 } }, { action_type: "dividend", ex_date: "2026-08-11", value: { amount: 0.26 } }], coverage: [{ dataset_id: "market_bars.stock.1m.raw", sessions: 4970, d0: "2007-01-03", d1: day(1), bars: 3_812_000 }], fundamentals: { facts: 1240, last_period: "2026-06-27", last_filing: "2026-08-01" } }]; }
56 + if (p === "/market/bars") { const span = (new Date(q.end) - new Date(q.start)) / 864e5; if (q.interval === "1m" && span > 31) return [413, { error: { code: "range_too_large", message: "1m requests are limited to 31 days per call; use POST /api/v1/exports for larger ranges (async Parquet).", request_id: "rq3", max_days: 31 } }]; const n = Math.min(Number(q.limit ?? 5000), 390); return [200, { data: bars(n, q.interval), meta: { version: 7, files_scanned: 3, symbol: q.symbol, interval: q.interval, adjustment: q.adjustment, start: q.start, end: q.end, rows: n, rows_matching: 1950, adjustment_notes: q.adjustment !== "raw" ? ["applied split 4:1 ex 2020-08-31"] : [], timestamp_convention: "bar_open", license: "frd-private: no redistribution" }, next_cursor: q.cursor ? null : "Y3Vyc29y" }]; }
57 + if (p === "/market/sessions") return [200, { data: sessions.slice(-40).reverse().map((s) => ({ session_date: s.session_date, bar_count: 930, expected_bars: 390, first_ts: `${s.session_date}T08:00:00Z`, last_ts: `${s.session_date}T23:59:00Z`, flags: [] })), meta: { dataset_id: "market_bars.stock.1m.raw", symbol: q.symbol } }];
58 + if (p === "/coverage/heatmap") return [200, { sessions, top_symbols: ["SPY", "AAPL", "TSLA", "NVDA", "QQQ", "AMD"].map((s, i) => ({ symbol: s, sessions: 84, bars: 80000 - i * 3000, completeness: 0.99 - i * 0.01 })), meta: { dataset_id: q.dataset_id, start: q.start, end: q.end } }];
59 + if (p === "/quality") return [200, { latest: { id: "qr1", created_at: ts(5), rows_checked: 2_900_000, anomalies: 45, summary: { ohlc_inconsistent: 3, price_jump: 42 } }, anomalies_by_code_30d: [{ code: "price_jump", severity: "warning", n: 420, rows: 420 }, { code: "ohlc_inconsistent", severity: "error", n: 12, rows: 12 }, { code: "outside_session", severity: "info", n: 7, rows: 7 }], reports: [{ id: "qr1", created_at: ts(5), rows_checked: 2_900_000, anomalies: 45, summary: {} }], codes: { ohlc_inconsistent: "high < max(open,close) or low > min(open,close)", price_jump: ">25 % move between consecutive bars", outside_session: "outside 04:00–20:00 ET" } }];
60 + if (p === "/quality/anomalies") return [200, { data: Array.from({ length: 25 }, (_, i) => ({ id: `a${i}`, code: i % 5 ? "price_jump" : "ohlc_inconsistent", severity: i % 5 ? "warning" : "error", symbol: ["AAPL", "TSLA", "XYZ"][i % 3], session_date: day(i), detail: { rows: 1, ts: ts(i) }, created_at: ts(i) })) }];
61 + if (p === "/query/views") return [200, { views: [{ name: "market_bars_stock_1m_raw", files: 2210 }, { name: "market_bars_stock_5m_raw", files: 410 }], examples: ["SELECT symbol_as_reported, count(*) bars FROM market_bars_stock_1m_raw GROUP BY 1 ORDER BY 2 DESC LIMIT 20"] }];
62 + if (p === "/query") return [200, { columns: [{ name: "symbol_as_reported", type: "string" }, { name: "bars", type: "int64" }], rows: [{ symbol_as_reported: "SPY", bars: 4_120_000 }, { symbol_as_reported: "AAPL", bars: 3_812_000 }], meta: { truncated: false, rows: 2, elapsed_ms: 184, views: ["market_bars_stock_1m_raw"], engine: "duckdb (read-only, sandboxed, 1 statement)" } }];
63 + if (p === "/exports" && method === "POST") return [202, { id: "e-new", status: "queued", poll: "/api/v1/exports/e-new" }];
64 + if (p === "/exports") return [200, { data: [{ id: "e1", status: "ready", request: { dataset_id: "market_bars.stock.1m.raw", symbols: ["AAPL", "MSFT"], start: "2024-01-01", end: day(1), adjustment: "raw", format: "parquet", version: 7 }, format: "parquet", bytes: 812_000_000, row_count: 61_000_000, created_at: ts(3), finished_at: ts(2.5), expires_at: ts(-69), error: null, snapshot: {} }, { id: "e2", status: "failed", request: { dataset_id: "market_bars.stock.1m.raw", symbols: ["ZZZZ"], start: "2024-01-01", end: day(1), adjustment: "raw", format: "csv" }, format: "csv", bytes: null, row_count: null, created_at: ts(4), finished_at: ts(3.9), expires_at: null, error: "no files for symbols", snapshot: null }] }];
65 + if (/^\/fundamentals\/[^/]+\/periods$/.test(p)) return [200, { periods: [{ fiscal_period_end: "2026-06-27", fiscal_period_type: "q", fiscal_year: 2026, fiscal_period: "Q3", filing_date: "2026-08-01", available_at: ts(100), metrics: 40, max_revision: 1, restated: false }], earnings: [{ event_date: "2026-07-31", eps_estimate: 1.42, eps_reported: 1.57, release_time: "amc" }] }];
66 + if (p.startsWith("/fundamentals/")) { const per = ["2026-06-27", "2026-03-28", "2025-12-27", "2025-09-27"]; const met = ["rev", "net_inc", "eps", "assets", "equity", "op_cf", "capex"]; const data = []; for (const pe of per) for (const m of met) data.push({ entity_id: "frd:AAPL", metric: m, value: m === "eps" ? 1.5 : 9e10 * (1 + per.indexOf(pe) / 10), unit: m === "eps" ? "usd_per_share" : "usd", currency: "USD", fiscal_period_start: null, fiscal_period_end: pe, fiscal_period_type: "q", fiscal_year: 2026, fiscal_period: "Q3", filing_date: pe, published_at: null, available_at: ts(100), observed_at: ts(90), source_document: "10-Q", source_id: "firstrate", revision_id: m === "rev" && pe === per[0] ? 2 : 1, restatement_flag: false, superseded_at: null }); return [200, { data, entity: { entity_id: "frd:AAPL", name: "Apple Inc." }, meta: { as_of: q.as_of ?? ts(0), rows: data.length, facts_hidden_by_as_of: 14, pit_policy: "available_at = filing_date + 1 day 00:00 America/New_York; revisions observed after as_of are not shown", license: "frd-private" } }]; }
67 + if (p === "/admin/jobs" && method === "GET") return [200, { data: jobs.filter((j) => !q.status || j.status === q.status).filter((j) => !q.kind || j.kind === q.kind) }];
68 + if (p === "/admin/jobs" && method === "POST") return [201, { id: "job-new" }];
69 + if (/^\/admin\/jobs\/.+\/(retry|cancel)$/.test(p)) return [200, { ok: true }];
70 + if (p === "/admin/keys" && method === "GET") return [200, { data: [{ id: "k1", name: "research-laptop", prefix: "fd_a1b2", scopes: ["read:market", "query"], rate_limit_per_minute: 600, expires_at: null, last_used_at: ts(1), revoked_at: null, created_at: ts(400), email: "owner@example.com" }], usage: [{ principal: "key:fd_a1b2", day: day(0), requests: 1200, rows_returned: 4_000_000, bytes_exported: 0 }] }];
71 + if (p === "/admin/keys" && method === "POST") return [201, { id: "k2", token: "fd_newkey.secretsecretsecret" }];
72 + if (p.startsWith("/admin/keys/") && method === "DELETE") return [200, { ok: true }];
73 + if (p === "/admin/audit") return [200, { data: Array.from({ length: 30 }, (_, i) => ({ id: `au${i}`, at: ts(i), action: ["auth.login", "admin.enqueue", "query.sql", "export.create"][i % 4], actor: "owner@example.com", target: i % 2 ? "0000000-id" : null, outcome: i % 9 === 0 ? "denied" : "ok", ip: "192.168.2.10", request_id: `rq${i}`, detail: { rows: i } })) }];
74 + if (p === "/admin/storage") return [200, { lake_root: "/Users/x/findata-lake", volume: { total_gb: 1000, free_gb: 412.3 }, budget: { soft_limit_gb: 250, hard_min_free_gb: 60 }, curated_bytes: 118.4e9, raw_bytes: 61.2e9, retired_bytes: 8e9, exports_bytes: 2.1e9, per_dataset: [{ dataset_id: "market_bars.stock.1m.raw", layout: "base", files: 2100, bytes: 1.1e11, rows: 9e9 }, { dataset_id: "market_bars.stock.1m.raw", layout: "delta", files: 110, bytes: 2e9, rows: 1.4e8 }, { dataset_id: "market_bars.stock.5m.raw", layout: "base", files: 410, bytes: 2.4e10, rows: 1.8e9 }], raw_archives: [{ status: "verified", n: 2200, bytes: 6e10 }, { status: "pending", n: 3, bytes: 1.2e9 }], exports: [{ status: "ready", n: 1, bytes: 8.1e8 }, { status: "failed", n: 1, bytes: 0 }], retired_files: { n: 41, bytes: 8e9 }, postgres_bytes: 3.2e9, last_backup: { value: { verified: true, target: "M4M64a:~/backups/findata" }, updated_at: ts(30) }, capacity_model: { bytes_per_1m_bar_curated: 13, note: "see docs/DISCOVERY.md §5" } }];
75 + if (p === "/admin/settings" && method === "GET") return [200, { settings: [{ key: "fundamentals_tracked_tickers", value: ["AAPL", "MSFT", "NVDA"], updated_at: ts(50) }, { key: "last_backup", value: { verified: true }, updated_at: ts(30) }], env: { env: "production", lake: "/Users/x/findata-lake", public_url: "https://findata.spboucher.ai", backup_target: "M4M64a:~/backups/findata", vendor_configured: true, sec_edgar_configured: false, ai_providers: { anthropic: false, openai: false } } }];
76 + if (p === "/admin/settings/tracked-tickers") return [200, { tickers: body.tickers }];
77 + if (p === "/admin/discover") return [200, { firstrate: { reachable: true, last_update: day(0), products: 3 }, sec_edgar: { status: "pending_configuration" } }];
78 + return [404, { error: { code: "not_found", message: `no mock for ${method} ${p}`, request_id: "rq404" } }];
79 +}
80 +
81 +createServer((req, res) => {
82 + const url = new URL(req.url, "http://x");
83 + let body = "";
84 + req.on("data", (c) => (body += c));
85 + req.on("end", () => {
86 + res.setHeader("Content-Security-Policy", CSP);
87 + res.setHeader("X-Content-Type-Options", "nosniff");
88 + if (url.pathname.startsWith("/api/")) {
89 + if (req.method !== "GET" && url.pathname !== "/api/v1/auth/login" && req.headers["x-csrf-token"] !== "csrf-mock") { res.writeHead(403, { "content-type": "application/json" }); return res.end(JSON.stringify({ error: { code: "csrf", message: "Missing or invalid CSRF token", request_id: "rq403" } })); }
90 + const [status, payload] = api(req.method, url.pathname, Object.fromEntries(url.searchParams), body ? JSON.parse(body) : {});
91 + res.writeHead(status, { "content-type": "application/json" });
92 + return res.end(JSON.stringify(payload));
93 + }
94 + let p = normalize(decodeURIComponent(url.pathname)).replace(/^(\.\.[/\\])+/, "");
95 + let file = join(root, p);
96 + if (existsSync(file) && statSync(file).isDirectory()) file = join(file, "index.html");
97 + if (!existsSync(file)) { file = join(root, "404.html"); res.statusCode = 404; }
98 + res.setHeader("content-type", MIME[extname(file)] ?? "application/octet-stream");
99 + res.end(readFileSync(file));
100 + });
101 +}).listen(port, "127.0.0.1", () => console.log(`mock console on http://127.0.0.1:${port} (signedIn=${signedIn}, mfaPassed=${mfaPassed})`));
added docs/API.md +26 −0
@@ -0,0 +1,26 @@
1 +# FinData OS — API quick start
2 +
3 +Base URL: `https://findata.spboucher.ai/api/v1` · OpenAPI: `/api/v1/openapi.json` · Swagger UI: `/api/docs`
4 +Auth: `Authorization: Bearer fd_<prefix>.<secret>` (create keys in the console → API keys). Browser sessions use the cookie + `X-CSRF-Token`.
5 +Conventions: UTC ISO-8601 (`Z`), bars stamped at **bar open**, cursor pagination (`next_cursor`), structured errors `{"error":{"code","message","request_id"}}`, response headers `X-Request-ID`, `X-Dataset-Id`, `X-Dataset-Version`.
6 +
7 +```bash
8 +K=fd_xxx.yyy
9 +curl -s -H "Authorization: Bearer $K" https://findata.spboucher.ai/api/v1/catalog/datasets | jq '.data[] | {id,status,license_class}'
10 +curl -s -H "Authorization: Bearer $K" "https://findata.spboucher.ai/api/v1/market/bars?symbol=AAPL&interval=1m&start=2026-10-30&end=2026-10-30&limit=5" | jq .
11 +curl -s -H "Authorization: Bearer $K" "https://findata.spboucher.ai/api/v1/fundamentals/AAPL?as_of=2025-01-01T00:00:00Z&metrics=rev,net_inc" | jq '.data[0], .meta'
12 +curl -s -H "Authorization: Bearer $K" -H 'Content-Type: application/json' -d '{"sql":"SELECT symbol_as_reported, count(*) FROM market_bars_stock_1m_raw GROUP BY 1 ORDER BY 2 DESC LIMIT 5"}' https://findata.spboucher.ai/api/v1/query | jq .rows
13 +curl -s -H "Authorization: Bearer $K" -H 'Content-Type: application/json' -d '{"dataset_id":"market_bars.stock.1m.raw","symbols":["AAPL","MSFT"],"start":"2026-10-01","end":"2026-10-30"}' https://findata.spboucher.ai/api/v1/exports # 202 → poll /exports/{id} → /download
14 +```
15 +
16 +Python SDK (`packages/python-sdk`, `pip install -e packages/python-sdk`):
17 +```python
18 +from findata_sdk import FinData
19 +fd = FinData("https://findata.spboucher.ai", api_key="fd_xxx.yyy")
20 +bars = fd.bars("AAPL", "1m", "2026-10-26", "2026-10-30") # pyarrow.Table (Arrow IPC pages)
21 +df = bars.to_pandas()
22 +path = fd.export(["AAPL", "MSFT"], "2024-01-01", "2026-10-30") # async Parquet export → local file
23 +facts = fd.fundamentals("AAPL", as_of="2025-01-01T00:00:00Z", metrics=["rev", "eps_d"])
24 +```
25 +
26 +Limits: 1m bars ≤ 31 days per call (5m ≤ 186 d, 1d ≤ 10 y), ≤ 10 000 rows per page → use exports for larger ranges. Query playground: one read-only statement, 30 s, ≤ 100 000 rows. Rate limit 600 req/min per key (configurable per key). License: FirstRate Data private use — do not redistribute raw data.
added docs/STATUS.md +41 −0
@@ -0,0 +1,41 @@
1 +# FinData OS — STATUS (updated 2026-11-01, build session)
2 +
3 +Legend: ✅ done and verified · 🟡 done, not yet verified in production · ⛔ blocked · ⬜ not started
4 +
5 +## Phase 0 — Discover ✅
6 +- ✅ Cluster/SSD inventory, node choice M3U96a with rationale (`docs/DISCOVERY.md` §1)
7 +- ✅ Vendor identified (FirstRate Data), auth model, capability matrix, license summary, sanitized probes (`docs/evidence/phase0-probes.md`)
8 +- ✅ Minimal authorized test requests (last_update ×7 types, ticker listings, one 15.8 MB daily 1-min archive, splits/dividends, fundamentals JSON)
9 +- ✅ Capacity model and disk budget (≤ 250 GB curated, 60 GB hard stop)
10 +
11 +## Phase 1 — Foundation
12 +- ✅ Monorepo (uv workspace, 8 packages), migrations (`findata migrate`), secrets in `.env` only, redaction filter
13 +- ✅ Authentication: owner password (argon2) + TOTP MFA, sessions, CSRF, scoped API keys; tests green
14 +- 🟡 Deployment via `mld` to M3U96a, TLS via MacLustr Tunnel, health checks — see "Production" below
15 +- ✅ CI-equivalent: `make test` (37 tests), `make lint` (ruff clean), `pip-audit` (no known vulnerabilities, 2026-11-01)
16 +
17 +## Phase 2 — Market pipeline (US stocks, 1-minute)
18 +- ✅ Connector (discover/capabilities/fetch_range/normalize/validate/checkpoint), immutable raw archives, delta Parquet per session, atomic versioned publication, supersede without duplicates (tested), 5-minute derivation (tested vs manual aggregation and DST), coverage + quality anomalies
19 +- 🟡 Real incremental run on the node (see Production)
20 +
21 +## Phase 3 — Fundamentals
22 +- ✅ FirstRate fundamentals connector (verified live on AAPL during discovery), long-format PIT facts, `available_at` policy, revision/restatement tracking, as-of query hides later filings and later-observed restatements (tested), earnings events
23 +- ⬜ SEC EDGAR connector: parsing implemented + fixture-tested; **pending configuration** (`SEC_EDGAR_USER_AGENT`)
24 +- ⚠️ Limitation: vendor gives no revision history → restatements before first ingestion are unknown
25 +
26 +## Phase 4 — Platform
27 +- ✅ REST API `/api/v1` (38+ routes, OpenAPI), cursor pagination, Arrow/CSV/JSON, async Parquet exports, sandboxed SQL, coverage/quality/health/admin, Prometheus metrics
28 +- ✅ Python SDK (bars via Arrow pages, exports, fundamentals, SQL) — tested against a live uvicorn thread
29 +- 🟡 Web console (Next.js static export) — built by a sub-agent; see Production for verification
30 +
31 +## Phase 5 — Scale
32 +- 🟡 Representative multi-year backfill (bounded letter range) — to run on the node
33 +- 🟡 Backup + restore drill — to run on the node
34 +- ⬜ Load benchmark
35 +
36 +## Phase 6 — AI and polish
37 +- ✅ Assistant endpoint implemented (Anthropic SDK, model proposes SQL only; FinData validates/executes/cites versions) — **pending configuration** until `ANTHROPIC_API_KEY` is set and a capability/cost test is recorded here; OpenAI provider not implemented
38 +- ⬜ Security review pass on production
39 +
40 +## Production (to be filled with real outputs)
41 +(see bottom of this file after deployment)
42
Menu

Theme