import { test, expect } from "@playwright/test"; import { TEST_EMAIL, TEST_PASSWORD, lastLinkFromLog, login, noConsoleErrors, readEnv, restoreSession, saveSession, sql } from "./helpers"; /** * Full workflow against the dev server (EMAIL_DRY_RUN=1 EMAIL_DRY_RUN_PRINT=1 so links land in the log): * sign up → verify email → login → add provider key → validate → select model → send → stream → new chat → * reload → persistence → change password → logout → login again. */ test.describe.serial("PolyLLM full flow", () => { const email = TEST_EMAIL; const password = TEST_PASSWORD; let conversationUrl = ""; test("landing renders without console errors", async ({ page }) => { const errors = await noConsoleErrors(page); await page.goto("/"); await expect(page.getByRole("heading", { level: 1 })).toContainText(/every model/i); await expect(page.getByRole("link", { name: /start using polyllm/i }).first()).toBeVisible(); expect(errors).toEqual([]); }); test("sign up sends a verification email", async ({ page }) => { await page.goto("/signup"); await page.getByLabel(/name/i).fill("E2E Tester"); await page.getByLabel(/email/i).fill(email); await page.getByLabel(/^password$/i).fill(password); const confirm = page.getByLabel(/confirm/i); if (await confirm.count()) await confirm.fill(password); await page.getByRole("button", { name: /create account|sign up/i }).click(); await expect(page.getByText(/check your inbox|verify/i).first()).toBeVisible({ timeout: 30_000 }); // the account exists but is unverified await expect.poll(() => sql(`select email_verified from users where email='${email}'`)).toBe("f"); }); test("login before verification is refused", async ({ page }) => { await page.goto("/login"); await page.getByLabel(/email/i).fill(email); await page.getByLabel(/^password$/i).fill(password); await page.getByRole("button", { name: /sign in/i }).click(); await expect(page.getByText(/verify|not verified|check your inbox/i).first()).toBeVisible({ timeout: 20_000 }); }); test("verification link activates the account and signs in", async ({ page }) => { const link = lastLinkFromLog(/http:\/\/localhost:3000\/api\/auth\/verify-email\?token=[^\s"'<>]+/g); expect(link, "verification link should be printed in the dev log").toBeTruthy(); await page.goto(link!); await page.waitForURL(/\/app|\/verify-email/, { timeout: 30_000 }); await expect.poll(() => sql(`select email_verified from users where email='${email}'`)).toBe("t"); if (!/\/app/.test(page.url())) await login(page, email, password); await expect(page).toHaveURL(/\/app\/chat/); }); test("add + validate provider keys (real providers)", async ({ page }) => { await login(page, email, password); await saveSession(page); await page.goto("/app/settings/providers"); const names: Record = { anthropic: /^Anthropic$/, openai: /^OpenAI$/, xai: /^xAI$/, gemini: /^Google Gemini$/, mistral: /^Mistral AI$/, deepseek: /^DeepSeek$/, kimi: /^Kimi/, openrouter: /^OpenRouter$/, cerebras: /^Cerebras$/ }; for (const [provider, envName] of [ ["anthropic", "ANTHROPIC_API_KEY"], ["openai", "OPENAI_API_KEY"], ["xai", "XAI_API_KEY"], ["gemini", "GOOGLE_GEMINI_API_KEY"], ["mistral", "MISTRAL_API_KEY"], ["deepseek", "DEEPSEEK_API_KEY"], ["kimi", "KIMI_API_KEY"], ["openrouter", "OPENROUTER_API_KEY"], ["cerebras", "CEREBRAS_API_KEY"], ] as const) { const key = readEnv(envName); test.skip(!key, `${envName} missing`); const item = page.locator("li", { has: page.getByRole("heading", { level: 3, name: names[provider] }) }).first(); await item.getByRole("button", { name: /add key|replace key/i }).click(); await page.locator("#provider-api-key").fill(key!); await page.getByRole("button", { name: /^connect$|^replace key$/i }).last().click(); await expect(item).toContainText(/connected/i, { timeout: 60_000 }); await expect(page.locator("#provider-api-key")).toHaveCount(0); } // never in the DOM const html = await page.content(); expect(html).not.toContain(readEnv("ANTHROPIC_API_KEY")!.slice(10, 30)); await expect.poll(() => sql(`select count(*) from provider_connections pc join users u on u.id=pc.user_id where u.email='${email}' and pc.status='valid'`)).toBe("9"); // keys are encrypted at rest const stored = sql(`select encrypted_key from provider_connections pc join users u on u.id=pc.user_id where u.email='${email}' and provider='anthropic'`); expect(stored.startsWith("v1.")).toBe(true); expect(stored).not.toContain("sk-ant"); }); test("select a model, send a message and receive a real stream", async ({ page }) => { await restoreSession(page); await page.goto("/app/chat"); await page.getByRole("button", { name: /select model/i }).click(); await page.getByLabel(/search models/i).fill("haiku 4.5"); await page.getByRole("option").first().click(); await expect(page.getByRole("button", { name: /select model/i })).toContainText(/haiku/i); await page.getByLabel(/^message$/i).fill("Reply with exactly the word PONG and nothing else."); await page.getByRole("button", { name: /^send$/i }).click(); await expect(page.getByText(/PONG/).first()).toBeVisible({ timeout: 60_000 }); await page.waitForURL(/\/app\/chat\/cnv_/, { timeout: 30_000 }); conversationUrl = page.url(); // footer meta appears (tokens + latency) await expect(page.getByText(/tok\/s|\bin ·/).first()).toBeVisible({ timeout: 30_000 }); // sidebar shows the conversation with an auto title await expect(page.locator("aside").getByText(/reply with exactly/i).first()).toBeVisible(); }); test("conversation persists after reload and a second model works", async ({ page }) => { await restoreSession(page); await page.goto(conversationUrl); await expect(page.getByText(/PONG/).first()).toBeVisible(); // switch to an OpenAI model in the same conversation and continue await page.getByRole("button", { name: /select model/i }).click(); await page.getByLabel(/search models/i).fill("gpt-5.4-nano"); await page.getByRole("option").first().click(); await page.getByLabel(/^message$/i).fill("Now reply with exactly the word PING."); await page.getByRole("button", { name: /^send$/i }).click(); await expect(page.getByText(/PING/).first()).toBeVisible({ timeout: 60_000 }); await page.reload(); await expect(page.getByText(/PONG/).first()).toBeVisible(); await expect(page.getByText(/PING/).first()).toBeVisible(); }); test("model configuration is capability-driven", async ({ page }) => { await restoreSession(page); await page.goto("/app/chat"); await page.getByRole("button", { name: /select model/i }).click(); // Pin the native OpenAI catalog (OpenRouter also lists openai/* models with different parameter sheets). await page.getByRole("dialog").getByRole("button", { name: /^OpenAI$/ }).click(); await page.getByLabel(/search models/i).fill("gpt-5.5"); await page.getByRole("option").first().click(); await page.getByRole("button", { name: /model configuration/i }).click(); await expect(page.getByText(/reasoning/i).first()).toBeVisible(); await expect(page.getByText(/^Effort$/)).toBeVisible(); // seed/stop are not supported by the Responses API → never shown await expect(page.getByText(/^Seed$/)).toHaveCount(0); await expect(page.getByText(/^Stop sequences$/)).toHaveCount(0); await expect(page.getByText(/not available for this model/i)).toContainText(/seed/i); await page.keyboard.press("Escape"); }); test("usage dashboard shows the requests", async ({ page }) => { await restoreSession(page); await page.goto("/app/usage"); await expect(page.getByText(/requests/i).first()).toBeVisible(); await expect.poll(() => Number(sql(`select count(*) from usage_records ur join users u on u.id=ur.user_id where u.email='${email}'`))).toBeGreaterThanOrEqual(2); }); test("change password, logout, login again", async ({ page }) => { await restoreSession(page); await page.goto("/app/settings/security"); const newPassword = `${password}-2`; await page.getByLabel(/current password/i).fill(password); await page.getByLabel(/^new password$/i).fill(newPassword); const confirm = page.getByLabel(/confirm/i); if (await confirm.count()) await confirm.fill(newPassword); await page.getByRole("button", { name: /change password|update password/i }).click(); await expect(page.getByText(/password (changed|updated)/i).first()).toBeVisible({ timeout: 20_000 }); // sign out via the sidebar menu await page.goto("/app/chat"); await page.getByText(email).first().click(); await page.getByRole("menuitem", { name: /sign out/i }).click(); await page.waitForURL(/\/login/); await login(page, email, newPassword); await expect(page).toHaveURL(/\/app/); }); });