from __future__ import annotations import json from datetime import datetime from typing import Any from fastapi import APIRouter, Depends, Query, Request from findata_core.config import get_settings from findata_core.db import connection, transaction from findata_ingest import queue from findata_ingest.lake import Lake from pydantic import BaseModel, Field from findata_api import auth as A from findata_api.errors import ApiError router = APIRouter(prefix="/admin", tags=["admin"]) class JobIn(BaseModel): kind: str dataset_id: str | None = None params: dict[str, Any] = Field(default_factory=dict) priority: int = 50 class KeyIn(BaseModel): name: str = Field(..., min_length=1, max_length=60) scopes: list[str] rate_limit_per_minute: int = Field(600, ge=1, le=10000) expires_at: datetime | None = None class TrackedIn(BaseModel): tickers: list[str] @router.get("/jobs") def jobs(status: str | None = None, kind: str | None = None, limit: int = Query(100, le=1000), p: A.Principal = Depends(A.require("admin"))): with connection() as conn: rows = conn.execute("SELECT id, kind, dataset_id, params, priority, status, attempts, max_attempts, next_run_at, created_at, started_at, finished_at, last_error, progress, result, created_by " "FROM ingest_jobs WHERE (%s::text IS NULL OR status=%s) AND (%s::text IS NULL OR kind=%s) ORDER BY created_at DESC LIMIT %s", (status, status, kind, kind, limit)).fetchall() return {"data": rows} @router.post("/jobs", status_code=201) def enqueue(body: JobIn, request: Request, p: A.Principal = Depends(A.require("admin"))): from findata_ingest.tasks import REGISTRY if body.kind not in REGISTRY: raise ApiError(400, "bad_kind", f"kind must be one of {sorted(REGISTRY)}") if body.kind == "backfill": lake = Lake() if lake.free_gb() < get_settings().lake_hard_min_free_gb * 1.5: raise ApiError(409, "disk_budget", "not enough free space to start a backfill") jid = queue.enqueue(body.kind, body.dataset_id, body.params, body.priority, created_by=p.email or p.label) A.audit(request, "admin.enqueue", jid, kind=body.kind, dataset=body.dataset_id, params=body.params) return {"id": jid} @router.post("/jobs/{job_id}/retry") def retry(job_id: str, request: Request, p: A.Principal = Depends(A.require("admin"))): with transaction() as conn: r = conn.execute("UPDATE ingest_jobs SET status='queued', attempts=0, next_run_at=now(), last_error=NULL WHERE id=%s AND status IN ('failed','dead','cancelled') RETURNING id", (job_id,)).fetchone() if not r: raise ApiError(409, "not_retryable", "job is not in a retryable state") A.audit(request, "admin.retry", job_id) return {"ok": True} @router.post("/jobs/{job_id}/cancel") def cancel(job_id: str, request: Request, p: A.Principal = Depends(A.require("admin"))): with transaction() as conn: r = conn.execute("UPDATE ingest_jobs SET status='cancelled', finished_at=now() WHERE id=%s AND status='queued' RETURNING id", (job_id,)).fetchone() if not r: raise ApiError(409, "not_cancellable", "only queued jobs can be cancelled") A.audit(request, "admin.cancel", job_id) return {"ok": True} @router.get("/keys") def keys(p: A.Principal = Depends(A.require("admin"))): with connection() as conn: rows = conn.execute("SELECT k.id, k.name, k.prefix, k.scopes, k.rate_limit_per_minute, k.expires_at, k.last_used_at, k.revoked_at, k.created_at, u.email " "FROM api_keys k JOIN users u ON u.id=k.user_id ORDER BY k.created_at DESC").fetchall() usage = conn.execute("SELECT principal, day, requests, rows_returned, bytes_exported FROM usage_daily WHERE day > current_date - 14 ORDER BY day DESC").fetchall() return {"data": rows, "usage": usage} @router.post("/keys", status_code=201) def create_key(body: KeyIn, request: Request, p: A.Principal = Depends(A.require("admin"))): try: out = A.new_api_key(p.id, body.name, body.scopes, body.rate_limit_per_minute, body.expires_at) except ValueError as e: raise ApiError(400, "bad_scopes", str(e)) from None A.audit(request, "admin.key_create", out["id"], name=body.name, scopes=body.scopes) return out @router.delete("/keys/{key_id}") def revoke_key(key_id: str, request: Request, p: A.Principal = Depends(A.require("admin"))): with transaction() as conn: conn.execute("UPDATE api_keys SET revoked_at=now() WHERE id=%s", (key_id,)) A.audit(request, "admin.key_revoke", key_id) return {"ok": True} @router.get("/audit") def audit_log(limit: int = Query(200, le=2000), action: str | None = None, p: A.Principal = Depends(A.require("admin"))): with connection() as conn: rows = conn.execute("SELECT * FROM audit_log WHERE (%s::text IS NULL OR action LIKE %s) ORDER BY at DESC LIMIT %s", (action, (action or "") + "%", limit)).fetchall() return {"data": rows} @router.get("/storage") def storage(p: A.Principal = Depends(A.require("admin"))): lake = Lake() s = get_settings() with connection() as conn: per_ds = conn.execute("SELECT dataset_id, layout, count(*) files, sum(bytes) bytes, sum(row_count) rows FROM lake_files WHERE version_removed IS NULL GROUP BY 1,2 ORDER BY 1,2").fetchall() raw = conn.execute("SELECT status, count(*) n, sum(bytes) bytes FROM raw_archives GROUP BY status").fetchall() exports = conn.execute("SELECT status, count(*) n, coalesce(sum(bytes),0) bytes FROM exports GROUP BY status").fetchall() retired = conn.execute("SELECT count(*) n, coalesce(sum(bytes),0) bytes FROM lake_files WHERE version_removed IS NOT NULL").fetchone() db = conn.execute("SELECT pg_database_size(current_database()) bytes").fetchone() backup = conn.execute("SELECT value, updated_at FROM settings_kv WHERE key='last_backup'").fetchone() from findata_api.health import lake_usage_bytes usage = lake_usage_bytes(conn) import os st = os.statvfs(lake.root) return {"lake_root": str(lake.root), "volume": {"total_gb": round(st.f_blocks * st.f_frsize / 1e9, 1), "free_gb": round(st.f_bavail * st.f_frsize / 1e9, 1)}, "budget": {"soft_limit_gb": s.lake_soft_limit_gb, "hard_min_free_gb": s.lake_hard_min_free_gb}, "curated_bytes": usage["curated"], "raw_bytes": usage["raw"], "retired_bytes": usage["retired"], "exports_bytes": sum(int(e["bytes"] or 0) for e in exports), "per_dataset": per_ds, "cold_root": str(lake.cold) if lake.cold else None, "cold_free_gb": round(lake.free_gb(lake.cold), 1) if lake.cold else None, "raw_archives": raw, "exports": exports, "retired_files": retired, "postgres_bytes": db["bytes"], "last_backup": backup, "capacity_model": {"bytes_per_1m_bar_curated": 13, "note": "see docs/DISCOVERY.md §5 — measured on the prior lake; recomputed from per_dataset above"}} @router.get("/settings") def get_settings_(p: A.Principal = Depends(A.require("admin"))): with connection() as conn: rows = conn.execute("SELECT key, value, updated_at FROM settings_kv WHERE key NOT LIKE 'secret%'").fetchall() s = get_settings() return {"settings": rows, "env": {"env": s.env, "lake": str(s.lake), "public_url": s.public_url, "backup_target": s.backup_target, "vendor_configured": bool(s.frd_userid.get_secret_value()), "sec_edgar_configured": bool(s.sec_edgar_user_agent), "ai_providers": {"anthropic": bool(s.anthropic_api_key.get_secret_value()), "openai": bool(s.openai_api_key.get_secret_value())}}} @router.put("/settings/tracked-tickers") def tracked(body: TrackedIn, request: Request, p: A.Principal = Depends(A.require("admin"))): tk = sorted({t.upper().strip() for t in body.tickers if t.strip()})[:2000] with transaction() as conn: conn.execute("INSERT INTO settings_kv(key, value) VALUES ('fundamentals_tracked_tickers', %s) ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value, updated_at=now()", (json.dumps(tk),)) A.audit(request, "admin.tracked_tickers", None, count=len(tk)) return {"tickers": tk} @router.post("/discover") def discover(request: Request, p: A.Principal = Depends(A.require("admin"))): """Sanitized live vendor probe (no bulk download).""" from findata_fundamentals import SecEdgarConnector from findata_market_firstrate import FirstRateMarketConnector out = {"firstrate": FirstRateMarketConnector().discover(), "sec_edgar": SecEdgarConnector().status()} A.audit(request, "admin.discover") return out