1#!/usr/bin/env node2/**3 * Local smoke-test server: serves `out/` exactly like the Python API does (directory index.html fallback,4 * 404.html, the same strict CSP) and answers /api/v1/* with fixtures so the console can be exercised5 * without Postgres or the lake. Never used in production.6 *7 * node scripts/mock-server.mjs [port=8787] [--signed-out] [--mfa]8 */9import { createServer } from "node:http";10import { existsSync, readFileSync, statSync } from "node:fs";11import { extname, join, normalize } from "node:path";12import { dirname } from "node:path";13import { fileURLToPath } from "node:url";1415const root = join(dirname(fileURLToPath(import.meta.url)), "..", "out");16const port = Number(process.argv.find((a) => /^\d+$/.test(a)) ?? 8787);17let signedIn = !process.argv.includes("--signed-out");18let mfaPassed = !process.argv.includes("--mfa");19const CSP = "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'";20const MIME = { ".html": "text/html; charset=utf-8", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".txt": "text/plain", ".json": "application/json", ".woff2": "font/woff2", ".ico": "image/x-icon" };2122const day = (n) => { const d = new Date(); d.setUTCDate(d.getUTCDate() - n); return d.toISOString().slice(0, 10); };23const ts = (h) => new Date(Date.now() - h * 3600e3).toISOString();24const health = {25 status: "warning", checked_at: ts(0),26 alerts: [{ severity: "warning", code: "stale_dataset", message: "market_bars.stock.1m.raw is 1 day(s) behind the vendor" }, { severity: "info", code: "no_backup_yet", message: "no backup recorded yet" }],27 components: { postgres: "ok", lake: { root: "/lake", free_gb: 412.3, curated_gb: 118.4, raw_gb: 61.2, soft_limit_gb: 250, hard_min_free_gb: 60 }, ingest_worker: "running", backup: { last: ts(30), age_hours: 30, verified: true }, ssd_smart: "Verified" },28 datasets: [{ dataset_id: "market_bars.stock.1m.raw", version: 7, last_session: day(1), days_since_last_session: 1, vendor_last_update: day(0), vendor_lag_days: 1 }, { dataset_id: "market_bars.stock.1d.raw", version: 3, last_session: day(1), days_since_last_session: 1, vendor_last_update: null, vendor_lag_days: null }],29};30const jobs = ["succeeded", "running", "queued", "failed", "dead", "succeeded"].map((status, i) => ({ id: `0000000${i}-aaaa-bbbb-cccc-00000000000${i}`, kind: ["incremental", "derive", "quality", "backfill", "fundamentals", "metadata"][i], dataset_id: "market_bars.stock.1m.raw", params: { period: "day" }, priority: 50, status, attempts: 1, max_attempts: 3, created_at: ts(i + 1), started_at: status === "queued" ? null : ts(i + 0.9), finished_at: status === "running" || status === "queued" ? null : ts(i + 0.5), last_error: status === "failed" || status === "dead" ? "HTTPError 503 vendor unavailable (retry 3/3)" : null, progress: status === "running" ? { files: 12, of: 40, symbol: "MSFT" } : null, result: status === "succeeded" ? { rows: 1_234_567 } : null, created_by: "owner@example.com" }));31const datasets = [32 { id: "market_bars.stock.1m.raw", name: "US stocks — 1-minute bars", description: "Intraday OHLCV bars for US equities and ETFs from FirstRate Data; extended session 04:00–20:00 ET.", kind: "market_bars", asset_class: "stock", interval: "1m", adjustment: "raw", license_class: "frd-private", license_text: "FirstRate Data license: private use by the subscriber only.", status: "active", timestamp_convention: "bar_open", timezone: "UTC", current_version: 7, snapshot_id: "snap_7f3a", file_count: 2210, row_count: 9_143_221_004, bytes: 118_400_000_000, version_created_at: ts(20), first_session: "2007-01-03", last_session: day(1), symbols: 8123, source_name: "FirstRate Data", vendor: "firstrate", notes: null },33 { id: "market_bars.stock.5m.raw", name: "US stocks — 5-minute bars (derived)", description: "Derived from 1-minute bars.", kind: "derived", asset_class: "stock", interval: "5m", adjustment: "raw", license_class: "frd-private", license_text: "FirstRate Data license.", status: "active", timestamp_convention: "bar_open", timezone: "UTC", current_version: 3, snapshot_id: "snap_12aa", file_count: 410, row_count: 1_832_000_000, bytes: 24_000_000_000, first_session: "2007-01-03", last_session: day(1), symbols: 8123, source_name: "FinData derive", vendor: "findata" },34 { id: "fundamentals.frd.pit", name: "Fundamentals (FirstRate)", description: "Point-in-time quarterly and annual fundamentals.", kind: "fundamentals", asset_class: null, interval: null, adjustment: null, license_class: "frd-private", license_text: "FirstRate Data license.", status: "active", timestamp_convention: null, timezone: "UTC", current_version: 2, row_count: 2_410_000, bytes: 410_000_000, file_count: 12, symbols: 2410, source_name: "FirstRate Data", vendor: "firstrate" },35 { id: "fundamentals.sec.edgar", name: "SEC EDGAR XBRL facts", description: "Company facts from SEC EDGAR.", kind: "fundamentals", license_class: "public-domain", license_text: "U.S. government work.", status: "pending_configuration", timezone: "UTC", current_version: null, source_name: "SEC EDGAR", vendor: "sec" },36 { id: "market_bars.futures.1m.raw", name: "Futures — 1-minute bars", description: "Planned.", kind: "market_bars", asset_class: "futures", interval: "1m", adjustment: "raw", license_class: "pending", license_text: "License not yet determined.", status: "planned", timezone: "UTC", current_version: null, source_name: "FirstRate Data", vendor: "firstrate" },37 { id: "market_ticks.stock", name: "Tick data", description: "Not supported by this platform.", kind: "ticks", asset_class: "stock", license_class: "pending", license_text: "", status: "unsupported", timezone: "UTC", current_version: null, source_name: "—", vendor: "—" },38];39const bars = (n, iv) => Array.from({ length: n }, (_, i) => { const t = new Date(Date.UTC(2026, 9, 26, 13, 30) + i * (iv === "1d" ? 864e5 : 60e3)); const c = 230 + Math.sin(i / 9) * 4 + (i % 7) * 0.1; return { timestamp_utc: t.toISOString().replace(".000Z", "Z"), session_date: t.toISOString().slice(0, 10), open: c - 0.2, high: c + 0.5, low: c - 0.6, close: c, volume: 10000 + ((i * 7919) % 50000), vwap: c + 0.05, trade_count: 120, quality_flags: i % 37 === 0 ? ["price_jump"] : [], adjustment_type: "raw", revision_id: 1 }; });40const sessions = Array.from({ length: 120 }, (_, i) => day(i)).filter((d) => ![0, 6].includes(new Date(d).getUTCDay())).reverse().map((d, i) => ({ session_date: d, symbols: 8000 + (i % 100), bars: 2_900_000 + (i % 13) * 10000, completeness: 0.8 + ((i * 37) % 20) / 100, non_trading_rows: 0 }));4142function api(method, path, q, body) {43 const p = path.replace(/^\/api\/v1/, "");44 if (p === "/auth/login" && method === "POST") { signedIn = true; mfaPassed = !process.argv.includes("--mfa"); return [200, { ok: true, mfa_required: !mfaPassed, csrf_token: "csrf-mock", email: body.email, role: "owner" }]; }45 if (p === "/auth/mfa" && method === "POST") { if (body.code !== "123456") return [401, { error: { code: "invalid_code", message: "Invalid one-time code", request_id: "rq1" } }]; mfaPassed = true; return [200, { ok: true, csrf_token: "csrf-mock" }]; }46 if (p === "/auth/logout") { signedIn = false; return [200, { ok: true }]; }47 if (p === "/auth/me") return signedIn ? [200, { kind: "user", email: "owner@example.com", role: "owner", scopes: ["admin", "read:market", "read:fundamentals", "query", "export"], mfa_passed: mfaPassed, totp_enabled: false, csrf_token: "csrf-mock" }] : [401, { error: { code: "unauthenticated", message: "Not signed in", request_id: "rq0" } }];48 if (!signedIn) return [401, { error: { code: "unauthenticated", message: "Not signed in", request_id: "rq0" } }];49 if (p === "/auth/totp/setup") return [200, { otpauth_uri: "otpauth://totp/x", qr_png_data_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=", secret: "JBSWY3DPEHPK3PXP" }];50 if (p === "/health") return [200, health];51 if (p === "/health/summary") return [200, { datasets: { active: 3, planned: 1, unsupported: 1, pending: 1 }, instruments: { n: 11240, active: 8123 }, bars_1m: { symbols: 8123, sessions: 4970, bars: 9_143_221_004, last: day(1) }, jobs: { succeeded: 412, running: 1, queued: 1, failed: 1, dead: 1 }, recent_jobs: jobs, fundamentals: { entities: 2410, facts: 2_410_000 }, anomalies_7d: { warning: 42, error: 3 }, health }];52 if (p === "/catalog/datasets") return [200, { data: datasets, meta: { count: datasets.length } }];53 if (p.startsWith("/catalog/datasets/")) { const d = datasets.find((x) => x.id === decodeURIComponent(p.split("/")[3])); return d ? [200, { data: d, versions: [7, 6, 5].map((v) => ({ version: v, snapshot_id: `snap_${v}`, file_count: 2000 + v, row_count: 9e9 - v * 1e8, bytes: 1.1e11, created_at: ts(24 * (8 - v)), note: v === 7 ? "compaction" : null })), files: [{ layout: "base", n: 2100, rows: 9e9, bytes: 1.1e11, d0: "2007-01-03", d1: day(20) }, { layout: "delta", n: 110, rows: 1.4e8, bytes: 2e9, d0: day(19), d1: day(1) }], checkpoints: [{ key: "last_vendor_update:day", value: { vendor_last_update: day(0) }, updated_at: ts(2) }] }] : [404, { error: { code: "not_found", message: "unknown dataset", request_id: "rq2" } }]; }54 if (p === "/instruments") return [200, { data: ["AAPL", "AAL", "AAON", "AAP", "AAPU"].filter((s) => !q.query || s.startsWith(q.query.toUpperCase())).map((s) => ({ instrument_id: `firstrate:stock:${s}`, symbol: s, name: `${s} Inc.`, asset_class: "stock", status: "active", first_date: "2007-01-03", last_date: day(1), currency: "USD" })), meta: { total_instruments: 11240, active: 8123 }, next_cursor: null }];55 if (p.startsWith("/instruments/")) { const s = p.split("/")[2]; return [200, { instrument: { instrument_id: `firstrate:stock:${s}`, symbol: s, name: `${s} Inc.`, asset_class: "stock", status: "active", first_date: "2007-01-03", last_date: day(1), currency: "USD" }, aliases: [], corporate_actions: [{ action_type: "split", ex_date: "2020-08-31", value: { ratio: 4 } }, { action_type: "dividend", ex_date: "2026-08-11", value: { amount: 0.26 } }], coverage: [{ dataset_id: "market_bars.stock.1m.raw", sessions: 4970, d0: "2007-01-03", d1: day(1), bars: 3_812_000 }], fundamentals: { facts: 1240, last_period: "2026-06-27", last_filing: "2026-08-01" } }]; }56 if (p === "/market/bars") { const span = (new Date(q.end) - new Date(q.start)) / 864e5; if (q.interval === "1m" && span > 31) return [413, { error: { code: "range_too_large", message: "1m requests are limited to 31 days per call; use POST /api/v1/exports for larger ranges (async Parquet).", request_id: "rq3", max_days: 31 } }]; const n = Math.min(Number(q.limit ?? 5000), 390); return [200, { data: bars(n, q.interval), meta: { version: 7, files_scanned: 3, symbol: q.symbol, interval: q.interval, adjustment: q.adjustment, start: q.start, end: q.end, rows: n, rows_matching: 1950, adjustment_notes: q.adjustment !== "raw" ? ["applied split 4:1 ex 2020-08-31"] : [], timestamp_convention: "bar_open", license: "frd-private: no redistribution" }, next_cursor: q.cursor ? null : "Y3Vyc29y" }]; }57 if (p === "/market/sessions") return [200, { data: sessions.slice(-40).reverse().map((s) => ({ session_date: s.session_date, bar_count: 930, expected_bars: 390, first_ts: `${s.session_date}T08:00:00Z`, last_ts: `${s.session_date}T23:59:00Z`, flags: [] })), meta: { dataset_id: "market_bars.stock.1m.raw", symbol: q.symbol } }];58 if (p === "/coverage/heatmap") return [200, { sessions, top_symbols: ["SPY", "AAPL", "TSLA", "NVDA", "QQQ", "AMD"].map((s, i) => ({ symbol: s, sessions: 84, bars: 80000 - i * 3000, completeness: 0.99 - i * 0.01 })), meta: { dataset_id: q.dataset_id, start: q.start, end: q.end } }];59 if (p === "/quality") return [200, { latest: { id: "qr1", created_at: ts(5), rows_checked: 2_900_000, anomalies: 45, summary: { ohlc_inconsistent: 3, price_jump: 42 } }, anomalies_by_code_30d: [{ code: "price_jump", severity: "warning", n: 420, rows: 420 }, { code: "ohlc_inconsistent", severity: "error", n: 12, rows: 12 }, { code: "outside_session", severity: "info", n: 7, rows: 7 }], reports: [{ id: "qr1", created_at: ts(5), rows_checked: 2_900_000, anomalies: 45, summary: {} }], codes: { ohlc_inconsistent: "high < max(open,close) or low > min(open,close)", price_jump: ">25 % move between consecutive bars", outside_session: "outside 04:00–20:00 ET" } }];60 if (p === "/quality/anomalies") return [200, { data: Array.from({ length: 25 }, (_, i) => ({ id: `a${i}`, code: i % 5 ? "price_jump" : "ohlc_inconsistent", severity: i % 5 ? "warning" : "error", symbol: ["AAPL", "TSLA", "XYZ"][i % 3], session_date: day(i), detail: { rows: 1, ts: ts(i) }, created_at: ts(i) })) }];61 if (p === "/query/views") return [200, { views: [{ name: "market_bars_stock_1m_raw", files: 2210 }, { name: "market_bars_stock_5m_raw", files: 410 }], examples: ["SELECT symbol_as_reported, count(*) bars FROM market_bars_stock_1m_raw GROUP BY 1 ORDER BY 2 DESC LIMIT 20"] }];62 if (p === "/query") return [200, { columns: [{ name: "symbol_as_reported", type: "string" }, { name: "bars", type: "int64" }], rows: [{ symbol_as_reported: "SPY", bars: 4_120_000 }, { symbol_as_reported: "AAPL", bars: 3_812_000 }], meta: { truncated: false, rows: 2, elapsed_ms: 184, views: ["market_bars_stock_1m_raw"], engine: "duckdb (read-only, sandboxed, 1 statement)" } }];63 if (p === "/exports" && method === "POST") return [202, { id: "e-new", status: "queued", poll: "/api/v1/exports/e-new" }];64 if (p === "/exports") return [200, { data: [{ id: "e1", status: "ready", request: { dataset_id: "market_bars.stock.1m.raw", symbols: ["AAPL", "MSFT"], start: "2024-01-01", end: day(1), adjustment: "raw", format: "parquet", version: 7 }, format: "parquet", bytes: 812_000_000, row_count: 61_000_000, created_at: ts(3), finished_at: ts(2.5), expires_at: ts(-69), error: null, snapshot: {} }, { id: "e2", status: "failed", request: { dataset_id: "market_bars.stock.1m.raw", symbols: ["ZZZZ"], start: "2024-01-01", end: day(1), adjustment: "raw", format: "csv" }, format: "csv", bytes: null, row_count: null, created_at: ts(4), finished_at: ts(3.9), expires_at: null, error: "no files for symbols", snapshot: null }] }];65 if (/^\/fundamentals\/[^/]+\/periods$/.test(p)) return [200, { periods: [{ fiscal_period_end: "2026-06-27", fiscal_period_type: "q", fiscal_year: 2026, fiscal_period: "Q3", filing_date: "2026-08-01", available_at: ts(100), metrics: 40, max_revision: 1, restated: false }], earnings: [{ event_date: "2026-07-31", eps_estimate: 1.42, eps_reported: 1.57, release_time: "amc" }] }];66 if (p.startsWith("/fundamentals/")) { const per = ["2026-06-27", "2026-03-28", "2025-12-27", "2025-09-27"]; const met = ["rev", "net_inc", "eps", "assets", "equity", "op_cf", "capex"]; const data = []; for (const pe of per) for (const m of met) data.push({ entity_id: "frd:AAPL", metric: m, value: m === "eps" ? 1.5 : 9e10 * (1 + per.indexOf(pe) / 10), unit: m === "eps" ? "usd_per_share" : "usd", currency: "USD", fiscal_period_start: null, fiscal_period_end: pe, fiscal_period_type: "q", fiscal_year: 2026, fiscal_period: "Q3", filing_date: pe, published_at: null, available_at: ts(100), observed_at: ts(90), source_document: "10-Q", source_id: "firstrate", revision_id: m === "rev" && pe === per[0] ? 2 : 1, restatement_flag: false, superseded_at: null }); return [200, { data, entity: { entity_id: "frd:AAPL", name: "Apple Inc." }, meta: { as_of: q.as_of ?? ts(0), rows: data.length, facts_hidden_by_as_of: 14, pit_policy: "available_at = filing_date + 1 day 00:00 America/New_York; revisions observed after as_of are not shown", license: "frd-private" } }]; }67 if (p === "/admin/jobs" && method === "GET") return [200, { data: jobs.filter((j) => !q.status || j.status === q.status).filter((j) => !q.kind || j.kind === q.kind) }];68 if (p === "/admin/jobs" && method === "POST") return [201, { id: "job-new" }];69 if (/^\/admin\/jobs\/.+\/(retry|cancel)$/.test(p)) return [200, { ok: true }];70 if (p === "/admin/keys" && method === "GET") return [200, { data: [{ id: "k1", name: "research-laptop", prefix: "fd_a1b2", scopes: ["read:market", "query"], rate_limit_per_minute: 600, expires_at: null, last_used_at: ts(1), revoked_at: null, created_at: ts(400), email: "owner@example.com" }], usage: [{ principal: "key:fd_a1b2", day: day(0), requests: 1200, rows_returned: 4_000_000, bytes_exported: 0 }] }];71 if (p === "/admin/keys" && method === "POST") return [201, { id: "k2", token: "fd_newkey.secretsecretsecret" }];72 if (p.startsWith("/admin/keys/") && method === "DELETE") return [200, { ok: true }];73 if (p === "/admin/audit") return [200, { data: Array.from({ length: 30 }, (_, i) => ({ id: `au${i}`, at: ts(i), action: ["auth.login", "admin.enqueue", "query.sql", "export.create"][i % 4], actor: "owner@example.com", target: i % 2 ? "0000000-id" : null, outcome: i % 9 === 0 ? "denied" : "ok", ip: "192.168.2.10", request_id: `rq${i}`, detail: { rows: i } })) }];74 if (p === "/admin/storage") return [200, { lake_root: "/Users/x/findata-lake", volume: { total_gb: 1000, free_gb: 412.3 }, budget: { soft_limit_gb: 250, hard_min_free_gb: 60 }, curated_bytes: 118.4e9, raw_bytes: 61.2e9, retired_bytes: 8e9, exports_bytes: 2.1e9, per_dataset: [{ dataset_id: "market_bars.stock.1m.raw", layout: "base", files: 2100, bytes: 1.1e11, rows: 9e9 }, { dataset_id: "market_bars.stock.1m.raw", layout: "delta", files: 110, bytes: 2e9, rows: 1.4e8 }, { dataset_id: "market_bars.stock.5m.raw", layout: "base", files: 410, bytes: 2.4e10, rows: 1.8e9 }], raw_archives: [{ status: "verified", n: 2200, bytes: 6e10 }, { status: "pending", n: 3, bytes: 1.2e9 }], exports: [{ status: "ready", n: 1, bytes: 8.1e8 }, { status: "failed", n: 1, bytes: 0 }], retired_files: { n: 41, bytes: 8e9 }, postgres_bytes: 3.2e9, last_backup: { value: { verified: true, target: "M4M64a:~/backups/findata" }, updated_at: ts(30) }, capacity_model: { bytes_per_1m_bar_curated: 13, note: "see docs/DISCOVERY.md §5" } }];75 if (p === "/admin/settings" && method === "GET") return [200, { settings: [{ key: "fundamentals_tracked_tickers", value: ["AAPL", "MSFT", "NVDA"], updated_at: ts(50) }, { key: "last_backup", value: { verified: true }, updated_at: ts(30) }], env: { env: "production", lake: "/Users/x/findata-lake", public_url: "https://findata.spboucher.ai", backup_target: "M4M64a:~/backups/findata", vendor_configured: true, sec_edgar_configured: false, ai_providers: { anthropic: false, openai: false } } }];76 if (p === "/admin/settings/tracked-tickers") return [200, { tickers: body.tickers }];77 if (p === "/admin/discover") return [200, { firstrate: { reachable: true, last_update: day(0), products: 3 }, sec_edgar: { status: "pending_configuration" } }];78 return [404, { error: { code: "not_found", message: `no mock for ${method} ${p}`, request_id: "rq404" } }];79}8081createServer((req, res) => {82 const url = new URL(req.url, "http://x");83 let body = "";84 req.on("data", (c) => (body += c));85 req.on("end", () => {86 res.setHeader("Content-Security-Policy", CSP);87 res.setHeader("X-Content-Type-Options", "nosniff");88 if (url.pathname.startsWith("/api/")) {89 if (req.method !== "GET" && url.pathname !== "/api/v1/auth/login" && req.headers["x-csrf-token"] !== "csrf-mock") { res.writeHead(403, { "content-type": "application/json" }); return res.end(JSON.stringify({ error: { code: "csrf", message: "Missing or invalid CSRF token", request_id: "rq403" } })); }90 const [status, payload] = api(req.method, url.pathname, Object.fromEntries(url.searchParams), body ? JSON.parse(body) : {});91 res.writeHead(status, { "content-type": "application/json" });92 return res.end(JSON.stringify(payload));93 }94 let p = normalize(decodeURIComponent(url.pathname)).replace(/^(\.\.[/\\])+/, "");95 let file = join(root, p);96 if (existsSync(file) && statSync(file).isDirectory()) file = join(file, "index.html");97 if (!existsSync(file)) { file = join(root, "404.html"); res.statusCode = 404; }98 res.setHeader("content-type", MIME[extname(file)] ?? "application/octet-stream");99 res.end(readFileSync(file));100 });101}).listen(port, "127.0.0.1", () => console.log(`mock console on http://127.0.0.1:${port} (signedIn=${signedIn}, mfaPassed=${mfaPassed})`));102