1"use client";23import { useSearchParams } from "next/navigation";4import { Suspense, useEffect, useMemo, useRef, useState } from "react";5import { api, ApiError, streamChat } from "@/lib/api";6import { useLive } from "@/lib/events";7import { fmtMs } from "@/lib/format";8import type { Model } from "@/lib/types";9import { Code, Modal, Pill, StatusPill } from "@/components/ui";1011interface Msg { role: "user" | "assistant" | "system"; content: string; reasoning?: string; stats?: { tps?: number; ttft?: number; tokens?: number; prompt?: number; total?: number } }1213function Playground() {14 const params = useSearchParams();15 const live = useLive();16 const [models, setModels] = useState<Model[]>([]);17 const [model, setModel] = useState(params.get("model") || "");18 const [system, setSystem] = useState("");19 const [temp, setTemp] = useState(0.7);20 const [topP, setTopP] = useState(0.95);21 const [maxTokens, setMaxTokens] = useState(1024);22 const [thinking, setThinking] = useState(true);23 const [msgs, setMsgs] = useState<Msg[]>([]);24 const [input, setInput] = useState("");25 const [busy, setBusy] = useState(false);26 const [phase, setPhase] = useState<string>("");27 const [showRaw, setShowRaw] = useState(false);28 const [showSettings, setShowSettings] = useState(false);29 const abortRef = useRef<AbortController | null>(null);30 const endRef = useRef<HTMLDivElement>(null);3132 useEffect(() => {33 api.get<{ models: Model[] }>("/api/models").then((r) => {34 const ok = r.models.filter((m) => m.installed && m.enabled && !m.embedding && !m.reranker);35 setModels(ok);36 if (!model && ok.length) {37 const cur = ok.find((m) => m.status === "ready") || ok.find((m) => m.favorite) || ok[0];38 setModel(cur.id);39 }40 }).catch(() => {});41 // eslint-disable-next-line react-hooks/exhaustive-deps42 }, [live.version]);4344 useEffect(() => { endRef.current?.scrollIntoView({ block: "end" }); }, [msgs, phase]);4546 const current = models.find((m) => m.id === model);47 const status = live.manager?.loaded.find((w) => w.model_id === model)?.status || live.manager?.progress[model]?.status || current?.status || "unloaded";4849 const body = useMemo(() => {50 const messages: { role: string; content: string }[] = [];51 if (system.trim()) messages.push({ role: "system", content: system });52 for (const m of msgs) if (m.role !== "system") messages.push({ role: m.role, content: m.content });53 const b: Record<string, unknown> = { model, messages, temperature: temp, top_p: topP, max_tokens: maxTokens, stream: true };54 if (current?.thinking) b.chat_template_kwargs = { enable_thinking: thinking };55 return b;56 }, [model, system, msgs, temp, topP, maxTokens, thinking, current]);5758 const send = async (text?: string, regenerate = false) => {59 const userText = text ?? input.trim();60 if (!model || busy) return;61 let history = msgs;62 if (regenerate) {63 history = msgs.slice(0, msgs.length - 1);64 } else {65 if (!userText) return;66 history = [...msgs, { role: "user", content: userText }];67 setInput("");68 }69 const assistant: Msg = { role: "assistant", content: "", reasoning: "" };70 setMsgs([...history, assistant]);71 setBusy(true);72 setPhase(status === "ready" ? "Generating…" : "Loading model…");73 const ac = new AbortController();74 abortRef.current = ac;75 const t0 = performance.now();76 let first = 0;77 try {78 const messages: { role: string; content: string }[] = [];79 if (system.trim()) messages.push({ role: "system", content: system });80 for (const m of history) messages.push({ role: m.role, content: m.content });81 const req: Record<string, unknown> = { model, messages, temperature: temp, top_p: topP, max_tokens: maxTokens };82 if (current?.thinking) req.chat_template_kwargs = { enable_thinking: thinking };83 for await (const chunk of streamChat(req, ac.signal)) {84 if ((chunk as { error?: { message: string } }).error) throw new Error((chunk as { error: { message: string } }).error.message);85 const choices = (chunk.choices as { delta?: { content?: string; reasoning_content?: string } }[]) || [];86 const d = choices[0]?.delta;87 if (d?.content || d?.reasoning_content) {88 if (!first) { first = performance.now(); setPhase("Generating…"); }89 assistant.content += d.content || "";90 assistant.reasoning = (assistant.reasoning || "") + (d.reasoning_content || "");91 setMsgs([...history, { ...assistant }]);92 }93 const usage = chunk.usage as { completion_tokens?: number; prompt_tokens?: number } | undefined;94 const timings = chunk.timings as { generation_tps?: number; ttft_ms?: number } | undefined;95 if (usage || timings) {96 assistant.stats = {97 tokens: usage?.completion_tokens, prompt: usage?.prompt_tokens,98 tps: timings?.generation_tps, ttft: timings?.ttft_ms ?? (first ? first - t0 : undefined), total: performance.now() - t0,99 };100 setMsgs([...history, { ...assistant }]);101 }102 }103 } catch (e) {104 if ((e as Error).name !== "AbortError") {105 assistant.content += `\n\n⚠️ ${e instanceof ApiError ? e.message : (e as Error).message}`;106 setMsgs([...history, { ...assistant }]);107 }108 } finally {109 if (!assistant.stats) assistant.stats = { total: performance.now() - t0, ttft: first ? first - t0 : undefined };110 setMsgs([...history, { ...assistant }]);111 setBusy(false);112 setPhase("");113 abortRef.current = null;114 }115 };116117 return (118 <div className="flex flex-col h-[calc(100vh-56px)] md:h-[calc(100vh-64px)] -mb-20 md:mb-0">119 <div className="flex flex-wrap items-center gap-2 pb-3 border-b border-border">120 <select className="input w-auto max-w-[280px]" value={model} onChange={(e) => setModel(e.target.value)}>121 {!models.length && <option value="">No models installed</option>}122 {models.map((m) => <option key={m.id} value={m.id}>{m.favorite ? "★ " : ""}{m.name}{m.status === "ready" ? " · loaded" : ""}</option>)}123 </select>124 <StatusPill status={status} />125 {current && <span className="text-xs text-ink-3 hidden sm:inline">{current.runtime === "mlx" ? "MLX" : "llama.cpp"} · {current.quantization} · ctx {Math.round((current.recommended_context || 0) / 1024)}K</span>}126 <div className="ml-auto flex gap-2">127 <button className="btn btn-sm" onClick={() => setShowSettings(true)}>Parameters</button>128 <button className="btn btn-sm" onClick={() => setShowRaw(true)}>Raw request</button>129 <button className="btn btn-sm btn-ghost" onClick={() => setMsgs([])} disabled={busy}>Clear</button>130 </div>131 </div>132133 <div className="flex-1 overflow-auto py-4 flex flex-col gap-4">134 {!msgs.length && (135 <div className="m-auto text-center text-ink-3 text-sm max-w-md">136 <div className="text-2xl mb-2">λ</div>137 Send a message. If <span className="text-ink">{current?.name || "the model"}</span> is not loaded, it is loaded from SSD first — you will see the status change above.138 </div>139 )}140 {msgs.map((m, i) => (141 <div key={i} className={`flex ${m.role === "user" ? "justify-end" : "justify-start"}`}>142 <div className={`max-w-[85%] md:max-w-[75%] rounded-2xl px-4 py-2.5 text-[14px] leading-relaxed whitespace-pre-wrap ${m.role === "user" ? "bg-accent text-white rounded-br-md" : "card rounded-bl-md"}`}>143 {m.reasoning && (144 <details className="mb-2 text-xs text-ink-3">145 <summary className="cursor-pointer select-none">Reasoning ({m.reasoning.length} chars)</summary>146 <div className="mt-1 whitespace-pre-wrap border-l-2 border-border pl-2 max-h-60 overflow-auto">{m.reasoning}</div>147 </details>148 )}149 {m.content || (busy && i === msgs.length - 1 ? <span className="text-ink-3 pulse">{phase || "…"}</span> : "")}150 {m.role === "assistant" && m.stats && !(busy && i === msgs.length - 1) && (151 <div className="mt-2 pt-2 border-t border-border flex flex-wrap gap-x-3 gap-y-1 text-[11px] text-ink-3 num">152 {m.stats.tokens != null && <span>{m.stats.tokens} tokens</span>}153 {m.stats.tps && <span>{m.stats.tps.toFixed(1)} tok/s</span>}154 {m.stats.ttft != null && <span>TTFT {fmtMs(m.stats.ttft)}</span>}155 {m.stats.total != null && <span>{fmtMs(m.stats.total)} total</span>}156 <button className="hover:text-ink" onClick={() => navigator.clipboard.writeText(m.content)}>copy</button>157 {i === msgs.length - 1 && <button className="hover:text-ink" onClick={() => send(undefined, true)}>regenerate</button>}158 </div>159 )}160 </div>161 </div>162 ))}163 <div ref={endRef} />164 </div>165166 <form className="pt-3 border-t border-border flex gap-2 items-end" onSubmit={(e) => { e.preventDefault(); send(); }}>167 <textarea className="input flex-1 min-h-[44px] max-h-40" rows={1} placeholder={`Message ${current?.name || "model"}… (⌘/Ctrl+Enter to send)`} value={input}168 onChange={(e) => setInput(e.target.value)}169 onKeyDown={(e) => { if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { e.preventDefault(); send(); } }} />170 {busy ? (171 <button type="button" className="btn btn-danger h-[44px]" onClick={() => abortRef.current?.abort()}>Stop</button>172 ) : (173 <button className="btn btn-primary h-[44px]" disabled={!input.trim() || !model}>Send</button>174 )}175 </form>176177 <Modal open={showSettings} onClose={() => setShowSettings(false)} title="Parameters">178 <div className="flex flex-col gap-4 text-sm">179 <label className="flex flex-col gap-1"><span className="label">System prompt</span><textarea className="input" rows={3} value={system} onChange={(e) => setSystem(e.target.value)} placeholder="You are a helpful assistant." /></label>180 <label className="flex flex-col gap-1"><span className="label flex justify-between">Temperature <span className="num">{temp.toFixed(2)}</span></span><input type="range" min={0} max={2} step={0.05} value={temp} onChange={(e) => setTemp(Number(e.target.value))} /></label>181 <label className="flex flex-col gap-1"><span className="label flex justify-between">Top-p <span className="num">{topP.toFixed(2)}</span></span><input type="range" min={0.05} max={1} step={0.05} value={topP} onChange={(e) => setTopP(Number(e.target.value))} /></label>182 <label className="flex flex-col gap-1"><span className="label">Max tokens</span><input className="input" type="number" min={1} max={32768} value={maxTokens} onChange={(e) => setMaxTokens(Number(e.target.value))} /></label>183 {current?.thinking && <label className="flex items-center gap-2"><input type="checkbox" checked={thinking} onChange={(e) => setThinking(e.target.checked)} /> Enable thinking (reasoning models)</label>}184 <div className="text-xs text-ink-3">Context window of the loaded worker: {current?.recommended_context ? `${Math.round(current.recommended_context / 1024)}K tokens` : "—"} <Pill>{current?.runtime}</Pill></div>185 </div>186 </Modal>187 <Modal open={showRaw} onClose={() => setShowRaw(false)} title="Raw API request" width={680}>188 <Code>{`curl ${typeof location !== "undefined" ? location.origin : ""}/v1/chat/completions \\189 -H "Authorization: Bearer llm_live_..." \\190 -H "Content-Type: application/json" \\191 -d '${JSON.stringify(body, null, 2)}'`}</Code>192 </Modal>193 </div>194 );195}196197export default function PlaygroundPage() {198 return <Suspense><Playground /></Suspense>;199}200