"use client"; import { useSearchParams } from "next/navigation"; import { Suspense, useEffect, useMemo, useRef, useState } from "react"; import { api, ApiError, streamChat } from "@/lib/api"; import { useLive } from "@/lib/events"; import { fmtMs } from "@/lib/format"; import type { Model } from "@/lib/types"; import { Code, Modal, Pill, StatusPill } from "@/components/ui"; interface Msg { role: "user" | "assistant" | "system"; content: string; reasoning?: string; stats?: { tps?: number; ttft?: number; tokens?: number; prompt?: number; total?: number } } function Playground() { const params = useSearchParams(); const live = useLive(); const [models, setModels] = useState([]); const [model, setModel] = useState(params.get("model") || ""); const [system, setSystem] = useState(""); const [temp, setTemp] = useState(0.7); const [topP, setTopP] = useState(0.95); const [maxTokens, setMaxTokens] = useState(1024); const [thinking, setThinking] = useState(true); const [msgs, setMsgs] = useState([]); const [input, setInput] = useState(""); const [busy, setBusy] = useState(false); const [phase, setPhase] = useState(""); const [showRaw, setShowRaw] = useState(false); const [showSettings, setShowSettings] = useState(false); const abortRef = useRef(null); const endRef = useRef(null); useEffect(() => { api.get<{ models: Model[] }>("/api/models").then((r) => { const ok = r.models.filter((m) => m.installed && m.enabled && !m.embedding && !m.reranker); setModels(ok); if (!model && ok.length) { const cur = ok.find((m) => m.status === "ready") || ok.find((m) => m.favorite) || ok[0]; setModel(cur.id); } }).catch(() => {}); // eslint-disable-next-line react-hooks/exhaustive-deps }, [live.version]); useEffect(() => { endRef.current?.scrollIntoView({ block: "end" }); }, [msgs, phase]); const current = models.find((m) => m.id === model); const status = live.manager?.loaded.find((w) => w.model_id === model)?.status || live.manager?.progress[model]?.status || current?.status || "unloaded"; const body = useMemo(() => { const messages: { role: string; content: string }[] = []; if (system.trim()) messages.push({ role: "system", content: system }); for (const m of msgs) if (m.role !== "system") messages.push({ role: m.role, content: m.content }); const b: Record = { model, messages, temperature: temp, top_p: topP, max_tokens: maxTokens, stream: true }; if (current?.thinking) b.chat_template_kwargs = { enable_thinking: thinking }; return b; }, [model, system, msgs, temp, topP, maxTokens, thinking, current]); const send = async (text?: string, regenerate = false) => { const userText = text ?? input.trim(); if (!model || busy) return; let history = msgs; if (regenerate) { history = msgs.slice(0, msgs.length - 1); } else { if (!userText) return; history = [...msgs, { role: "user", content: userText }]; setInput(""); } const assistant: Msg = { role: "assistant", content: "", reasoning: "" }; setMsgs([...history, assistant]); setBusy(true); setPhase(status === "ready" ? "Generating…" : "Loading model…"); const ac = new AbortController(); abortRef.current = ac; const t0 = performance.now(); let first = 0; try { const messages: { role: string; content: string }[] = []; if (system.trim()) messages.push({ role: "system", content: system }); for (const m of history) messages.push({ role: m.role, content: m.content }); const req: Record = { model, messages, temperature: temp, top_p: topP, max_tokens: maxTokens }; if (current?.thinking) req.chat_template_kwargs = { enable_thinking: thinking }; for await (const chunk of streamChat(req, ac.signal)) { if ((chunk as { error?: { message: string } }).error) throw new Error((chunk as { error: { message: string } }).error.message); const choices = (chunk.choices as { delta?: { content?: string; reasoning_content?: string } }[]) || []; const d = choices[0]?.delta; if (d?.content || d?.reasoning_content) { if (!first) { first = performance.now(); setPhase("Generating…"); } assistant.content += d.content || ""; assistant.reasoning = (assistant.reasoning || "") + (d.reasoning_content || ""); setMsgs([...history, { ...assistant }]); } const usage = chunk.usage as { completion_tokens?: number; prompt_tokens?: number } | undefined; const timings = chunk.timings as { generation_tps?: number; ttft_ms?: number } | undefined; if (usage || timings) { assistant.stats = { tokens: usage?.completion_tokens, prompt: usage?.prompt_tokens, tps: timings?.generation_tps, ttft: timings?.ttft_ms ?? (first ? first - t0 : undefined), total: performance.now() - t0, }; setMsgs([...history, { ...assistant }]); } } } catch (e) { if ((e as Error).name !== "AbortError") { assistant.content += `\n\n⚠️ ${e instanceof ApiError ? e.message : (e as Error).message}`; setMsgs([...history, { ...assistant }]); } } finally { if (!assistant.stats) assistant.stats = { total: performance.now() - t0, ttft: first ? first - t0 : undefined }; setMsgs([...history, { ...assistant }]); setBusy(false); setPhase(""); abortRef.current = null; } }; return (
{current && {current.runtime === "mlx" ? "MLX" : "llama.cpp"} · {current.quantization} · ctx {Math.round((current.recommended_context || 0) / 1024)}K}
{!msgs.length && (
λ
Send a message. If {current?.name || "the model"} is not loaded, it is loaded from SSD first — you will see the status change above.
)} {msgs.map((m, i) => (
{m.reasoning && (
Reasoning ({m.reasoning.length} chars)
{m.reasoning}
)} {m.content || (busy && i === msgs.length - 1 ? {phase || "…"} : "")} {m.role === "assistant" && m.stats && !(busy && i === msgs.length - 1) && (
{m.stats.tokens != null && {m.stats.tokens} tokens} {m.stats.tps && {m.stats.tps.toFixed(1)} tok/s} {m.stats.ttft != null && TTFT {fmtMs(m.stats.ttft)}} {m.stats.total != null && {fmtMs(m.stats.total)} total} {i === msgs.length - 1 && }
)}
))}
{ e.preventDefault(); send(); }}>