public landing: anonymous overview metrics (/api/v1/public/overview) and docs; data pages stay behind sign-in
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 changed files +283 −8
modified
apps/web/src/app/page.tsx
+15 −4
@@ -8,9 +8,20 @@ import { Banner, Card, ErrorState, KpiTile, PageHeader, SeverityBadge, Skeleton, | ||
| 8 | 8 | import { DataTable } from "@/components/DataTable"; |
| 9 | 9 | import { Job } from "@/lib/api"; |
| 10 | 10 | import { useAuth } from "@/lib/auth"; |
| 11 | +import { PublicOverview } from "@/components/PublicOverview"; | |
| 11 | 12 | |
| 12 | 13 | export default function OverviewPage() { |
| 13 | − const { isOwner } = useAuth(); | |
| 14 | + const { isOwner, me } = useAuth(); | |
| 15 | + const signedIn = !!me && me.mfa_passed; | |
| 16 | + return ( | |
| 17 | + <> | |
| 18 | + <PublicOverview signedIn={signedIn} /> | |
| 19 | + {signedIn && <OperatorOverview isOwner={isOwner} />} | |
| 20 | + </> | |
| 21 | + ); | |
| 22 | +} | |
| 23 | + | |
| 24 | +function OperatorOverview({ isOwner }: { isOwner: boolean }) { | |
| 14 | 25 | const { data, error, loading, reload } = useAsync((signal) => api.get<HealthSummary>("/health/summary", undefined, signal), []); |
| 15 | 26 | useInterval(reload, 60_000); |
| 16 | 27 | |
@@ -24,9 +35,9 @@ export default function OverviewPage() { | ||
| 24 | 35 | const freeTone = lake ? (lake.free_gb < lake.hard_min_free_gb ? "crit" : lake.free_gb < lake.hard_min_free_gb * 2 ? "warn" : undefined) : undefined; |
| 25 | 36 | |
| 26 | 37 | return ( |
| 27 | − <> | |
| 38 | + <div className="mt-6"> | |
| 28 | 39 | <PageHeader |
| 29 | − title="Overview" | |
| 40 | + title="Operations" | |
| 30 | 41 | subtitle={h ? <>Health checked {fmtRelative(h.checked_at)} · auto-refresh every 60 s</> : "Lakehouse status at a glance"} |
| 31 | 42 | actions={ |
| 32 | 43 | <> |
@@ -151,6 +162,6 @@ export default function OverviewPage() { | ||
| 151 | 162 | </div> |
| 152 | 163 | </> |
| 153 | 164 | )} |
| 154 | − </> | |
| 165 | + </div> | |
| 155 | 166 | ); |
| 156 | 167 | } |
added
apps/web/src/components/PublicOverview.tsx
+158 −0
@@ -0,0 +1,158 @@ | ||
| 1 | +"use client"; | |
| 2 | +import Link from "next/link"; | |
| 3 | +import { RefreshCw } from "lucide-react"; | |
| 4 | +import { api } from "@/lib/api"; | |
| 5 | +import { useAsync, useInterval } from "@/lib/hooks"; | |
| 6 | +import { fmtBytes, fmtCompact, fmtGB, fmtInt, fmtRelative } from "@/lib/format"; | |
| 7 | +import { Card, ErrorState, KpiTile, PageHeader, Skeleton, StatusBadge } from "@/components/ui"; | |
| 8 | +import { DataTable } from "@/components/DataTable"; | |
| 9 | + | |
| 10 | +export interface PublicOverviewData { | |
| 11 | + generated_at: string; | |
| 12 | + status: string | null; | |
| 13 | + totals: { files: number; rows: number; bytes: number; datasets_active: number; dataset_versions: number; raw_archives: number; raw_bytes: number }; | |
| 14 | + families: { family: string; datasets: number; rows: number; bytes: number }[]; | |
| 15 | + datasets: { id: string; name: string; kind: string; asset_class: string | null; interval: string | null; adjustment: string | null; status: string; current_version: number; row_count: number | null; bytes: number | null; file_count: number | null }[]; | |
| 16 | + coverage: { dataset_id: string; symbols: number; first_session: string | null; last_session: string | null; bars: number | null }[]; | |
| 17 | + options: { underlyings: number | null; first_day: string | null; last_day: string | null; rows: number | null }; | |
| 18 | + fundamentals: { entities: number; facts: number; first_period: string | null; last_period: string | null; earnings_events: number }; | |
| 19 | + instruments: { n: number; active: number; delisted: number }; | |
| 20 | + freshness: { vendor_last_update: string | null; last_successful_job: string | null; jobs_24h: Record<string, number> }; | |
| 21 | + storage: { hot_free_gb: number | null; cold_free_gb: number | null; curated_gb: number | null; raw_gb: number | null }; | |
| 22 | + backup: { last: string | null; verified: boolean | null }; | |
| 23 | + license: string; | |
| 24 | +} | |
| 25 | + | |
| 26 | +const FAMILY_LABEL: Record<string, string> = { | |
| 27 | + "market_bars.stock": "US stocks", | |
| 28 | + "market_bars.etf": "US ETFs", | |
| 29 | + "options.eod": "Options (EOD chains)", | |
| 30 | + "market_bars.futures": "Futures (continuous)", | |
| 31 | + "market_bars.futures_contracts": "Futures (contracts)", | |
| 32 | + "market_bars.fx": "FX", | |
| 33 | + "market_bars.crypto": "Crypto", | |
| 34 | + "market_bars.index": "Indices", | |
| 35 | +}; | |
| 36 | + | |
| 37 | +export function PublicOverview({ signedIn }: { signedIn: boolean }) { | |
| 38 | + const { data, error, loading, reload } = useAsync((signal) => api.get<PublicOverviewData>("/public/overview", undefined, signal), []); | |
| 39 | + useInterval(reload, 120_000); | |
| 40 | + const stocks = data?.coverage.find((c) => c.dataset_id === "market_bars.stock.1m.raw"); | |
| 41 | + const etfs = data?.coverage.find((c) => c.dataset_id === "market_bars.etf.1m.raw"); | |
| 42 | + const indices = data?.coverage.find((c) => c.dataset_id === "market_bars.index.1m.raw"); | |
| 43 | + const jobs = data?.freshness.jobs_24h ?? {}; | |
| 44 | + | |
| 45 | + return ( | |
| 46 | + <> | |
| 47 | + <PageHeader | |
| 48 | + title="FinData OS — the lake at a glance" | |
| 49 | + subtitle={data ? <>Catalog figures refreshed {fmtRelative(data.generated_at)} · public metrics only, data access requires sign-in</> : "Research-grade market data lakehouse (FirstRate Data, private use)"} | |
| 50 | + actions={ | |
| 51 | + <> | |
| 52 | + {data?.status && <StatusBadge status={data.status} label={data.status === "ok" ? "all systems ok" : data.status} />} | |
| 53 | + <button className="btn btn-sm" onClick={reload} disabled={loading}> | |
| 54 | + <RefreshCw className={loading ? "h-3.5 w-3.5 animate-spin" : "h-3.5 w-3.5"} /> Refresh | |
| 55 | + </button> | |
| 56 | + {!signedIn && ( | |
| 57 | + <Link href="/login/" className="btn btn-primary btn-sm"> | |
| 58 | + Sign in | |
| 59 | + </Link> | |
| 60 | + )} | |
| 61 | + </> | |
| 62 | + } | |
| 63 | + /> | |
| 64 | + | |
| 65 | + {error && <ErrorState error={error} retry={reload} />} | |
| 66 | + {!data && !error && <Skeleton rows={6} />} | |
| 67 | + | |
| 68 | + {data && ( | |
| 69 | + <> | |
| 70 | + <div className="mb-4 grid grid-cols-2 gap-3 md:grid-cols-3 xl:grid-cols-6"> | |
| 71 | + <KpiTile label="Rows in the lake" value={fmtCompact(data.totals.rows)} sub={fmtInt(data.totals.rows)} /> | |
| 72 | + <KpiTile label="Curated Parquet" value={fmtBytes(data.totals.bytes, 0)} sub={`${fmtInt(data.totals.files)} files · ${fmtInt(data.totals.datasets_active)} active datasets`} href={signedIn ? "/datasets/" : undefined} /> | |
| 73 | + <KpiTile label="Stocks (1-min)" value={fmtInt(stocks?.symbols)} sub={stocks ? `${stocks.first_session} → ${stocks.last_session}` : "—"} href={signedIn ? "/symbols/" : undefined} /> | |
| 74 | + <KpiTile label="ETFs (1-min)" value={fmtInt(etfs?.symbols)} sub={etfs ? `${etfs.first_session} → ${etfs.last_session}` : "—"} /> | |
| 75 | + <KpiTile label="Option underlyings" value={fmtInt(data.options.underlyings)} sub={data.options.first_day ? `${fmtCompact(data.options.rows)} rows · ${data.options.first_day} → ${data.options.last_day}` : "—"} /> | |
| 76 | + <KpiTile label="Fundamentals" value={fmtInt(data.fundamentals.entities)} sub={`${fmtCompact(data.fundamentals.facts)} PIT facts · ${fmtCompact(data.fundamentals.earnings_events)} earnings`} href={signedIn ? "/fundamentals/" : undefined} /> | |
| 77 | + <KpiTile label="Indices (1-min)" value={fmtInt(indices?.symbols)} sub={indices ? `${indices.first_session} → ${indices.last_session}` : "—"} /> | |
| 78 | + <KpiTile label="Instruments" value={fmtInt(data.instruments.n)} sub={`${fmtInt(data.instruments.active)} active · ${fmtInt(data.instruments.delisted)} delisted`} /> | |
| 79 | + <KpiTile label="Vendor last update" value={data.freshness.vendor_last_update ?? "—"} sub={data.freshness.last_successful_job ? `last job ${fmtRelative(data.freshness.last_successful_job)}` : undefined} /> | |
| 80 | + <KpiTile label="Jobs (24 h)" value={`${fmtInt(jobs.succeeded ?? 0)} ok`} sub={`${fmtInt(jobs.queued ?? 0)} queued · ${fmtInt(jobs.running ?? 0)} running · ${fmtInt(jobs.dead ?? 0)} dead`} tone={jobs.dead ? "crit" : undefined} /> | |
| 81 | + <KpiTile label="Raw archives kept" value={fmtBytes(data.totals.raw_bytes, 0)} sub={`${fmtInt(data.totals.raw_archives)} vendor archives (sha256 recorded)`} /> | |
| 82 | + <KpiTile | |
| 83 | + label="Free space" | |
| 84 | + value={data.storage.hot_free_gb !== null ? fmtGB(data.storage.hot_free_gb, 0) : "—"} | |
| 85 | + sub={data.storage.cold_free_gb !== null ? `internal · external ${fmtGB(data.storage.cold_free_gb, 0)}` : "internal SSD"} | |
| 86 | + /> | |
| 87 | + </div> | |
| 88 | + | |
| 89 | + <div className="grid gap-4 lg:grid-cols-[1fr_minmax(280px,0.7fr)]"> | |
| 90 | + <Card title="Data families"> | |
| 91 | + <DataTable | |
| 92 | + rows={data.families} | |
| 93 | + rowKey={(r) => r.family} | |
| 94 | + mobile="scroll" | |
| 95 | + columns={[ | |
| 96 | + { key: "family", header: "Family", render: (r) => <span className="font-medium">{FAMILY_LABEL[r.family] ?? r.family}</span> }, | |
| 97 | + { key: "datasets", header: "Datasets", render: (r) => <span className="tnum">{fmtInt(r.datasets)}</span> }, | |
| 98 | + { key: "rows", header: "Rows", render: (r) => <span className="tnum">{fmtInt(r.rows)}</span> }, | |
| 99 | + { key: "bytes", header: "Size", render: (r) => <span className="tnum">{fmtBytes(r.bytes, 1)}</span> }, | |
| 100 | + ]} | |
| 101 | + /> | |
| 102 | + </Card> | |
| 103 | + <Card title="Platform"> | |
| 104 | + <ul className="flex flex-col gap-2 text-sm"> | |
| 105 | + <li className="flex items-center justify-between gap-2"> | |
| 106 | + <span>Status</span> | |
| 107 | + <StatusBadge status={data.status} label={data.status ?? "unknown"} /> | |
| 108 | + </li> | |
| 109 | + <li className="flex items-center justify-between gap-2"> | |
| 110 | + <span>Dataset versions published</span> | |
| 111 | + <span className="tnum">{fmtInt(data.totals.dataset_versions)}</span> | |
| 112 | + </li> | |
| 113 | + <li className="flex items-center justify-between gap-2"> | |
| 114 | + <span>Fundamentals periods</span> | |
| 115 | + <span className="tnum text-xs">{data.fundamentals.first_period ?? "—"} → {data.fundamentals.last_period ?? "—"}</span> | |
| 116 | + </li> | |
| 117 | + <li className="flex items-center justify-between gap-2"> | |
| 118 | + <span>Last backup</span> | |
| 119 | + <span className="text-right text-xs"> | |
| 120 | + {data.backup.last ? ( | |
| 121 | + <> | |
| 122 | + <StatusBadge status={data.backup.verified ? "verified" : "warning"} label={data.backup.verified ? "verified" : "unverified"} />{" "} | |
| 123 | + <span className="text-ink-2">{fmtRelative(data.backup.last)}</span> | |
| 124 | + </> | |
| 125 | + ) : ( | |
| 126 | + <StatusBadge status="warning" label="none yet" /> | |
| 127 | + )} | |
| 128 | + </span> | |
| 129 | + </li> | |
| 130 | + </ul> | |
| 131 | + <p className="mt-4 text-xs text-ink-2">{data.license}</p> | |
| 132 | + <p className="mt-2 text-xs text-ink-2"> | |
| 133 | + API reference: <a className="text-accent-ink hover:underline" href="/api/docs">/api/docs</a> · <Link className="text-accent-ink hover:underline" href="/docs/">Documentation</Link> | |
| 134 | + </p> | |
| 135 | + </Card> | |
| 136 | + </div> | |
| 137 | + | |
| 138 | + <Card title="Datasets" className="mt-4"> | |
| 139 | + <DataTable | |
| 140 | + rows={data.datasets.filter((d) => d.status === "active" && (d.row_count ?? 0) > 0)} | |
| 141 | + rowKey={(r) => r.id} | |
| 142 | + mobile="scroll" | |
| 143 | + columns={[ | |
| 144 | + { key: "id", header: "Dataset", mono: true, render: (r) => (signedIn ? <Link href={`/datasets/detail/?id=${encodeURIComponent(r.id)}`} className="text-accent-ink hover:underline">{r.id}</Link> : r.id) }, | |
| 145 | + { key: "asset_class", header: "Asset", hideSm: true, render: (r) => r.asset_class ?? "—" }, | |
| 146 | + { key: "interval", header: "Interval", hideSm: true, render: (r) => r.interval ?? "—" }, | |
| 147 | + { key: "adjustment", header: "Adjustment", hideSm: true, render: (r) => r.adjustment ?? "—" }, | |
| 148 | + { key: "row_count", header: "Rows", render: (r) => <span className="tnum">{fmtInt(r.row_count)}</span> }, | |
| 149 | + { key: "bytes", header: "Size", render: (r) => <span className="tnum">{fmtBytes(r.bytes, 1)}</span> }, | |
| 150 | + { key: "current_version", header: "Version", hideSm: true, render: (r) => <span className="tnum">v{r.current_version}</span> }, | |
| 151 | + ]} | |
| 152 | + /> | |
| 153 | + </Card> | |
| 154 | + </> | |
| 155 | + )} | |
| 156 | + </> | |
| 157 | + ); | |
| 158 | +} | |
modified
apps/web/src/components/Shell.tsx
+23 −0
@@ -28,6 +28,29 @@ export function Shell({ children }: { children: ReactNode }) { | ||
| 28 | 28 | |
| 29 | 29 | if (isLogin) return <main className="min-h-dvh">{children}</main>; |
| 30 | 30 | |
| 31 | + const isPublicPage = pathname === "/" || pathname === "/docs/"; | |
| 32 | + if (!loading && (!me || !me.mfa_passed) && isPublicPage) { | |
| 33 | + return ( | |
| 34 | + <div className="min-h-dvh"> | |
| 35 | + <header className="sticky top-0 z-30 flex items-center justify-between border-b border-hairline bg-surface-1/95 px-3 py-2 backdrop-blur sm:px-5"> | |
| 36 | + <Link href="/" className="flex items-center gap-2"> | |
| 37 | + <Logo /> | |
| 38 | + <span className="text-sm font-semibold">FinData OS</span> | |
| 39 | + </Link> | |
| 40 | + <nav className="flex items-center gap-1" aria-label="Public"> | |
| 41 | + <Link href="/" className={clsx("btn btn-ghost btn-sm", pathname === "/" && "text-accent-ink")}>Overview</Link> | |
| 42 | + <Link href="/docs/" className={clsx("btn btn-ghost btn-sm", pathname === "/docs/" && "text-accent-ink")}>Docs</Link> | |
| 43 | + <a href="/api/docs" className="btn btn-ghost btn-sm">API</a> | |
| 44 | + <Link href="/login/" className="btn btn-primary btn-sm">Sign in</Link> | |
| 45 | + </nav> | |
| 46 | + </header> | |
| 47 | + <main className="min-w-0 px-3 pb-10 pt-3 sm:px-5 sm:pt-4 lg:px-8 lg:pt-6"> | |
| 48 | + <div className="mx-auto max-w-[1400px]">{children}</div> | |
| 49 | + </main> | |
| 50 | + </div> | |
| 51 | + ); | |
| 52 | + } | |
| 53 | + | |
| 31 | 54 | if (loading || !me || !me.mfa_passed) { |
| 32 | 55 | return ( |
| 33 | 56 | <main className="flex min-h-dvh items-center justify-center p-6"> |
modified
apps/web/src/lib/auth.tsx
+4 −3
@@ -14,7 +14,7 @@ interface AuthState { | ||
| 14 | 14 | |
| 15 | 15 | const Ctx = createContext<AuthState | null>(null); |
| 16 | 16 | |
| 17 | −const PUBLIC_PATHS = ["/login"]; | |
| 17 | +const PUBLIC_PATHS = ["/login", "/", "/docs"]; // "/" and "/docs" show public metrics; data pages need a session | |
| 18 | 18 | |
| 19 | 19 | function isPublic(path: string | null): boolean { |
| 20 | 20 | if (!path) return false; |
@@ -56,12 +56,13 @@ export function AuthProvider({ children }: { children: ReactNode }) { | ||
| 56 | 56 | useEffect(() => { |
| 57 | 57 | if (loading) return; |
| 58 | 58 | const pub = isPublic(pathname); |
| 59 | + const onLogin = (pathname ?? "").replace(/\/+$/, "") === "/login"; | |
| 59 | 60 | if (!me && !pub) { |
| 60 | 61 | const next = pathname && pathname !== "/" ? `?next=${encodeURIComponent(pathname)}` : ""; |
| 61 | 62 | router.replace(`/login/${next}`); |
| 62 | − } else if (me && !me.mfa_passed && !pub) { | |
| 63 | + } else if (me && !me.mfa_passed && !onLogin) { | |
| 63 | 64 | router.replace("/login/"); |
| 64 | − } else if (me && me.mfa_passed && pub) { | |
| 65 | + } else if (me && me.mfa_passed && onLogin) { | |
| 65 | 66 | const next = typeof window !== "undefined" ? new URLSearchParams(window.location.search).get("next") : null; |
| 66 | 67 | router.replace(next && next.startsWith("/") ? next : "/"); |
| 67 | 68 | } |
modified
services/api/src/findata_api/main.py
+4 −1
@@ -28,6 +28,7 @@ from findata_api.routers import ( | ||
| 28 | 28 | health, |
| 29 | 29 | instruments, |
| 30 | 30 | market, |
| 31 | + public, | |
| 31 | 32 | quality, |
| 32 | 33 | query, |
| 33 | 34 | ) |
@@ -96,6 +97,8 @@ class RateLimit(BaseHTTPMiddleware): | ||
| 96 | 97 | if request.url.path.startswith(API_PREFIX) and not request.url.path.endswith("/health"): |
| 97 | 98 | auth_h = request.headers.get("authorization", "") |
| 98 | 99 | key = auth_h[-24:] if auth_h else (request.cookies.get(A.SESSION_COOKIE, "")[-24:] or (A.client_ip(request) or "anon")) |
| 100 | + if request.url.path.startswith(f"{API_PREFIX}/public/"): | |
| 101 | + key = f"public:{A.client_ip(request) or 'anon'}" | |
| 99 | 102 | ok, remaining = bucket.allow(f"rl:{key}", get_settings().rate_limit_per_minute) |
| 100 | 103 | if not ok: |
| 101 | 104 | return JSONResponse({"error": {"code": "rate_limited", "message": "Too many requests", "request_id": getattr(request.state, "request_id", None)}}, |
@@ -132,7 +135,7 @@ def create_app() -> FastAPI: | ||
| 132 | 135 | return JSONResponse({"error": {"code": "internal_error", "message": "Internal error", "request_id": getattr(request.state, "request_id", None)}}, status_code=500) |
| 133 | 136 | |
| 134 | 137 | for r in (auth.router, catalog.router, instruments.router, market.router, fundamentals.router, query.router, exports.router, |
| 135 | − coverage.router, quality.router, health.router, admin.router, assistant.router): | |
| 138 | + coverage.router, quality.router, health.router, admin.router, assistant.router, public.router): | |
| 136 | 139 | app.include_router(r, prefix=API_PREFIX) |
| 137 | 140 | |
| 138 | 141 | @app.get("/metrics", include_in_schema=False) |
added
services/api/src/findata_api/routers/public.py
+79 −0
@@ -0,0 +1,79 @@ | ||
| 1 | +"""Anonymous overview metrics for the public landing page. No data rows, no paths, no secrets — only catalog-level | |
| 2 | +figures (what exists, how much, how fresh). Everything else stays behind authentication (FirstRate license: | |
| 3 | +raw bars/tables are private).""" | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import threading | |
| 7 | +import time | |
| 8 | +from datetime import UTC, datetime | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +from fastapi import APIRouter | |
| 12 | +from findata_core.db import connection | |
| 13 | + | |
| 14 | +from findata_api.health import cached_evaluate | |
| 15 | + | |
| 16 | +router = APIRouter(prefix="/public", tags=["public"]) | |
| 17 | +_cache: dict[str, Any] = {"at": 0.0, "value": None} | |
| 18 | +_lock = threading.Lock() | |
| 19 | + | |
| 20 | + | |
| 21 | +def _compute() -> dict[str, Any]: | |
| 22 | + with connection() as conn: | |
| 23 | + totals = conn.execute("SELECT count(*) files, coalesce(sum(row_count),0) rows, coalesce(sum(bytes),0) bytes FROM lake_files WHERE version_removed IS NULL").fetchone() | |
| 24 | + fam = conn.execute(""" | |
| 25 | + SELECT split_part(dataset_id,'.',1)||'.'||split_part(dataset_id,'.',2) AS family, count(DISTINCT dataset_id) datasets, | |
| 26 | + sum(row_count) rows, sum(bytes) bytes | |
| 27 | + FROM lake_files WHERE version_removed IS NULL GROUP BY 1 ORDER BY sum(bytes) DESC""").fetchall() | |
| 28 | + ds = conn.execute(""" | |
| 29 | + SELECT d.id, d.name, d.kind, d.asset_class, d.interval, d.adjustment, d.status, d.license_class, d.current_version, | |
| 30 | + v.row_count, v.bytes, v.file_count, v.created_at AS version_created_at | |
| 31 | + FROM datasets d LEFT JOIN dataset_versions v ON v.dataset_id=d.id AND v.version=d.current_version ORDER BY d.id""").fetchall() | |
| 32 | + cov = conn.execute(""" | |
| 33 | + SELECT dataset_id, count(DISTINCT symbol) symbols, min(session_date) first_session, max(session_date) last_session, sum(bar_count) bars | |
| 34 | + FROM coverage WHERE dataset_id IN ('market_bars.stock.1m.raw','market_bars.etf.1m.raw','market_bars.index.1m.raw') GROUP BY 1""").fetchall() | |
| 35 | + opt = conn.execute("SELECT count(DISTINCT symbol_min) underlyings, min(date_min) first_day, max(date_max) last_day, sum(row_count) rows FROM lake_files WHERE dataset_id='options.eod' AND version_removed IS NULL").fetchone() | |
| 36 | + fund = conn.execute("SELECT count(DISTINCT entity_id) entities, count(*) facts, min(fiscal_period_end) first_period, max(fiscal_period_end) last_period FROM fundamentals_facts WHERE superseded_at IS NULL").fetchone() | |
| 37 | + earn = conn.execute("SELECT count(*) n FROM earnings_events").fetchone() | |
| 38 | + inst = conn.execute("SELECT count(*) n, count(*) FILTER (WHERE status='active') active, count(*) FILTER (WHERE status='delisted') delisted FROM instruments").fetchone() | |
| 39 | + raw = conn.execute("SELECT count(*) n, coalesce(sum(bytes),0) bytes FROM raw_archives WHERE path IS NOT NULL").fetchone() | |
| 40 | + jobs = conn.execute("SELECT status, count(*) n FROM ingest_jobs WHERE created_at > now() - interval '24 hours' GROUP BY 1").fetchall() | |
| 41 | + lastjob = conn.execute("SELECT max(finished_at) t FROM ingest_jobs WHERE status='succeeded'").fetchone() | |
| 42 | + ck = conn.execute("SELECT value FROM checkpoints WHERE scope='dataset:market_bars.stock.1m.raw' AND key='last_vendor_update:day'").fetchone() | |
| 43 | + versions = conn.execute("SELECT count(*) n FROM dataset_versions").fetchone() | |
| 44 | + backup = conn.execute("SELECT updated_at, value->>'verified' AS verified FROM settings_kv WHERE key='last_backup'").fetchone() | |
| 45 | + h = cached_evaluate() | |
| 46 | + lake = h.get("components", {}).get("lake", {}) if isinstance(h, dict) else {} | |
| 47 | + return { | |
| 48 | + "generated_at": datetime.now(UTC).isoformat(), | |
| 49 | + "status": h.get("status"), | |
| 50 | + "totals": {"files": int(totals["files"]), "rows": int(totals["rows"]), "bytes": int(totals["bytes"]), | |
| 51 | + "datasets_active": sum(1 for d in ds if d["status"] == "active"), "dataset_versions": int(versions["n"]), | |
| 52 | + "raw_archives": int(raw["n"]), "raw_bytes": int(raw["bytes"])}, | |
| 53 | + "families": [{"family": f["family"], "datasets": f["datasets"], "rows": int(f["rows"]), "bytes": int(f["bytes"])} for f in fam], | |
| 54 | + "datasets": [{k: (v.isoformat() if hasattr(v, "isoformat") else v) for k, v in dict(d).items()} for d in ds], | |
| 55 | + "coverage": [{k: (v.isoformat() if hasattr(v, "isoformat") else v) for k, v in dict(c).items()} for c in cov], | |
| 56 | + "options": {k: (v.isoformat() if hasattr(v, "isoformat") else v) for k, v in dict(opt).items()}, | |
| 57 | + "fundamentals": {**{k: (v.isoformat() if hasattr(v, "isoformat") else v) for k, v in dict(fund).items()}, "earnings_events": int(earn["n"])}, | |
| 58 | + "instruments": dict(inst), | |
| 59 | + "freshness": {"vendor_last_update": (ck["value"] or {}).get("vendor_last_update") if ck else None, | |
| 60 | + "last_successful_job": lastjob["t"].isoformat() if lastjob and lastjob["t"] else None, | |
| 61 | + "jobs_24h": {j["status"]: j["n"] for j in jobs}}, | |
| 62 | + "storage": {"hot_free_gb": lake.get("free_gb"), "cold_free_gb": lake.get("cold_free_gb"), "curated_gb": lake.get("curated_gb"), | |
| 63 | + "raw_gb": lake.get("raw_gb")}, | |
| 64 | + "backup": {"last": backup["updated_at"].isoformat() if backup else None, "verified": (backup["verified"] == "true") if backup else None}, | |
| 65 | + "license": "FirstRate Data — private research use; raw bars and tables are not redistributed. Sign in for data access.", | |
| 66 | + } | |
| 67 | + | |
| 68 | + | |
| 69 | +@router.get("/overview") | |
| 70 | +def overview(): | |
| 71 | + now = time.monotonic() | |
| 72 | + if _cache["value"] is not None and now - _cache["at"] < 120: | |
| 73 | + return _cache["value"] | |
| 74 | + with _lock: | |
| 75 | + if _cache["value"] is not None and time.monotonic() - _cache["at"] < 120: | |
| 76 | + return _cache["value"] | |
| 77 | + v = _compute() | |
| 78 | + _cache.update(at=time.monotonic(), value=v) | |
| 79 | + return v | |
| 80 | ||