"""LLM API application: startup sequence, routers, shutdown.""" from __future__ import annotations import logging import logging.handlers import os import sys import time from contextlib import asynccontextmanager import httpx from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from . import __version__ from .api import admin_routes, openai_routes from .api.admin_routes import public_health from .auth import Auth from .config import Settings, get_settings from .db import Database from .downloads import Downloader from .errors import APIError, api_error_handler from .events import bus from .hardware import detect_hardware from .harvester import Harvester from .jobs import JobRunner from .manager import ModelManager from .metrics import MetricsCollector from .models.registry import Registry from .proxy_ui import router as ui_router log = logging.getLogger("llm_api") def setup_logging(settings: Settings) -> None: settings.logs_path.mkdir(parents=True, exist_ok=True) fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") root = logging.getLogger() root.setLevel(logging.INFO) if not any(isinstance(h, logging.StreamHandler) for h in root.handlers): sh = logging.StreamHandler(sys.stdout) sh.setFormatter(fmt) root.addHandler(sh) fh = logging.handlers.RotatingFileHandler(settings.logs_path / "llm-api.log", maxBytes=20_000_000, backupCount=5) fh.setFormatter(fmt) root.addHandler(fh) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) logging.getLogger("uvicorn.access").setLevel(logging.WARNING) @asynccontextmanager async def lifespan(app: FastAPI): settings: Settings = app.state.settings t0 = time.time() # 1. config + dirs settings.ensure_dirs() # 2. database db = Database(settings.db_path) await db.connect() app.state.db = db # 3. hardware hw = detect_hardware(settings.models_dir) log.info("hardware: %s, %.0f GB, %s CPU cores, %s GPU cores, %s %s", hw.chip, hw.memory_gb, hw.cpu_cores, hw.gpu_cores, hw.os, hw.os_version) if settings.max_model_memory_gb > hw.memory_gb - settings.macos_reserve_gb: log.warning("MAX_MODEL_MEMORY_GB=%s is above memory minus the macOS reserve (%.0f GB)", settings.max_model_memory_gb, hw.memory_gb - settings.macos_reserve_gb) # 4. auth auth = Auth(db, settings) app.state.auth = auth if await auth.user_count() == 0 and settings.admin_email and settings.admin_password: try: await auth.create_user(settings.admin_email, settings.admin_password) log.info("seeded admin user %s", settings.admin_email) except Exception as e: log.warning("could not seed admin user: %s", e) # 5. registry + manager (clears stale state) registry = Registry(db, settings) manager = ModelManager(settings, db, registry) app.state.manager = manager app.state.registry = registry jobs = JobRunner(db) await jobs.start() app.state.jobs = jobs downloader = Downloader(settings, db, registry, jobs) app.state.downloader = downloader app.state.harvester = Harvester(settings, db, registry, jobs, downloader) metrics = MetricsCollector(settings, db, manager) app.state.metrics = metrics app.state.ui_client = httpx.AsyncClient(timeout=httpx.Timeout(30.0, read=120.0), follow_redirects=False) # 6. scan registry try: summary = await registry.rescan() log.info("registry: %s", summary) except Exception: log.exception("initial rescan failed") # 7. start manager loops (kills stale workers, preload) await manager.start() await metrics.start() await db.audit("server.start", actor="system", detail={"version": __version__, "seconds": round(time.time() - t0, 2)}) log.info("LLM API %s ready on %s:%s (%.1fs)", __version__, settings.host, settings.port, time.time() - t0) bus.publish("server", {"event": "started", "version": __version__}) try: yield finally: log.info("shutting down: draining requests, unloading models") await metrics.stop() await manager.stop() await app.state.ui_client.aclose() await db.audit("server.stop", actor="system") await db.close() def create_app(settings: Settings | None = None) -> FastAPI: settings = settings or get_settings() setup_logging(settings) app = FastAPI(title="LLM API", version=__version__, docs_url="/openapi", redoc_url=None, openapi_url="/openapi.json", lifespan=lifespan) app.state.settings = settings app.add_exception_handler(APIError, api_error_handler) # type: ignore[arg-type] @app.exception_handler(RequestValidationError) async def _validation(_: Request, exc: RequestValidationError): errs = exc.errors() msg = "; ".join(f"{'.'.join(str(x) for x in e.get('loc', []))}: {e.get('msg')}" for e in errs[:3]) return JSONResponse(status_code=400, content={"error": {"message": msg or "invalid request", "type": "invalid_request_error", "code": "INVALID_REQUEST", "param": None}}) @app.exception_handler(Exception) async def _unhandled(_: Request, exc: Exception): log.exception("unhandled error: %s", exc) return JSONResponse(status_code=500, content={"error": {"message": "Internal server error.", "type": "server_error", "code": "INTERNAL", "param": None}}) @app.middleware("http") async def _security_headers(request: Request, call_next): # Body size guard for JSON APIs cl = request.headers.get("content-length") if cl and cl.isdigit() and int(cl) > settings.max_body_bytes and request.url.path.startswith(("/v1", "/api")): return JSONResponse(status_code=413, content={"error": {"message": "Request body too large.", "type": "invalid_request_error", "code": "BODY_TOO_LARGE", "param": None}}) resp = await call_next(request) resp.headers.setdefault("X-Content-Type-Options", "nosniff") resp.headers.setdefault("Referrer-Policy", "same-origin") resp.headers.setdefault("X-Frame-Options", "DENY") if request.url.path.startswith(("/v1", "/api")): resp.headers.setdefault("Cache-Control", "no-store") return resp @app.get("/health", include_in_schema=False) async def health(request: Request): return await public_health(request) @app.get("/api/status", include_in_schema=False) async def status(request: Request): return await public_health(request) app.include_router(openai_routes.router) app.include_router(admin_routes.router) app.include_router(ui_router) # must be last (catch-all) return app app = create_app()