spb/polyllm

1import { test, expect } from "@playwright/test";2import { TEST_EMAIL, TEST_PASSWORD, lastLinkFromLog, login, noConsoleErrors, readEnv, restoreSession, saveSession, sql } from "./helpers";34/**5 * Full workflow against the dev server (EMAIL_DRY_RUN=1 EMAIL_DRY_RUN_PRINT=1 so links land in the log):6 * sign up → verify email → login → add provider key → validate → select model → send → stream → new chat →7 * reload → persistence → change password → logout → login again.8 */9test.describe.serial("PolyLLM full flow", () => {10  const email = TEST_EMAIL;11  const password = TEST_PASSWORD;12  let conversationUrl = "";1314  test("landing renders without console errors", async ({ page }) => {15    const errors = await noConsoleErrors(page);16    await page.goto("/");17    await expect(page.getByRole("heading", { level: 1 })).toContainText(/every model/i);18    await expect(page.getByRole("link", { name: /start using polyllm/i }).first()).toBeVisible();19    expect(errors).toEqual([]);20  });2122  test("sign up sends a verification email", async ({ page }) => {23    await page.goto("/signup");24    await page.getByLabel(/name/i).fill("E2E Tester");25    await page.getByLabel(/email/i).fill(email);26    await page.getByLabel(/^password$/i).fill(password);27    const confirm = page.getByLabel(/confirm/i);28    if (await confirm.count()) await confirm.fill(password);29    await page.getByRole("button", { name: /create account|sign up/i }).click();30    await expect(page.getByText(/check your inbox|verify/i).first()).toBeVisible({ timeout: 30_000 });31    // the account exists but is unverified32    await expect.poll(() => sql(`select email_verified from users where email='${email}'`)).toBe("f");33  });3435  test("login before verification is refused", async ({ page }) => {36    await page.goto("/login");37    await page.getByLabel(/email/i).fill(email);38    await page.getByLabel(/^password$/i).fill(password);39    await page.getByRole("button", { name: /sign in/i }).click();40    await expect(page.getByText(/verify|not verified|check your inbox/i).first()).toBeVisible({ timeout: 20_000 });41  });4243  test("verification link activates the account and signs in", async ({ page }) => {44    const link = lastLinkFromLog(/http:\/\/localhost:3000\/api\/auth\/verify-email\?token=[^\s"'<>]+/g);45    expect(link, "verification link should be printed in the dev log").toBeTruthy();46    await page.goto(link!);47    await page.waitForURL(/\/app|\/verify-email/, { timeout: 30_000 });48    await expect.poll(() => sql(`select email_verified from users where email='${email}'`)).toBe("t");49    if (!/\/app/.test(page.url())) await login(page, email, password);50    await expect(page).toHaveURL(/\/app\/chat/);51  });5253  test("add + validate provider keys (real providers)", async ({ page }) => {54    await login(page, email, password);55    await saveSession(page);56    await page.goto("/app/settings/providers");57    const names: Record<string, RegExp> = { anthropic: /^Anthropic$/, openai: /^OpenAI$/, xai: /^xAI$/, gemini: /^Google Gemini$/, mistral: /^Mistral AI$/, deepseek: /^DeepSeek$/, kimi: /^Kimi/, openrouter: /^OpenRouter$/, cerebras: /^Cerebras$/ };58    for (const [provider, envName] of [59      ["anthropic", "ANTHROPIC_API_KEY"],60      ["openai", "OPENAI_API_KEY"],61      ["xai", "XAI_API_KEY"],62      ["gemini", "GOOGLE_GEMINI_API_KEY"],63      ["mistral", "MISTRAL_API_KEY"],64      ["deepseek", "DEEPSEEK_API_KEY"],65      ["kimi", "KIMI_API_KEY"],66      ["openrouter", "OPENROUTER_API_KEY"],67      ["cerebras", "CEREBRAS_API_KEY"],68    ] as const) {69      const key = readEnv(envName);70      test.skip(!key, `${envName} missing`);71      const item = page.locator("li", { has: page.getByRole("heading", { level: 3, name: names[provider] }) }).first();72      await item.getByRole("button", { name: /add key|replace key/i }).click();73      await page.locator("#provider-api-key").fill(key!);74      await page.getByRole("button", { name: /^connect$|^replace key$/i }).last().click();75      await expect(item).toContainText(/connected/i, { timeout: 60_000 });76      await expect(page.locator("#provider-api-key")).toHaveCount(0);77    }78    // never in the DOM79    const html = await page.content();80    expect(html).not.toContain(readEnv("ANTHROPIC_API_KEY")!.slice(10, 30));81    await expect.poll(() => sql(`select count(*) from provider_connections pc join users u on u.id=pc.user_id where u.email='${email}' and pc.status='valid'`)).toBe("9");82    // keys are encrypted at rest83    const stored = sql(`select encrypted_key from provider_connections pc join users u on u.id=pc.user_id where u.email='${email}' and provider='anthropic'`);84    expect(stored.startsWith("v1.")).toBe(true);85    expect(stored).not.toContain("sk-ant");86  });8788  test("select a model, send a message and receive a real stream", async ({ page }) => {89    await restoreSession(page);90    await page.goto("/app/chat");91    await page.getByRole("button", { name: /select model/i }).click();92    await page.getByLabel(/search models/i).fill("haiku 4.5");93    await page.getByRole("option").first().click();94    await expect(page.getByRole("button", { name: /select model/i })).toContainText(/haiku/i);95    await page.getByLabel(/^message$/i).fill("Reply with exactly the word PONG and nothing else.");96    await page.getByRole("button", { name: /^send$/i }).click();97    await expect(page.getByText(/PONG/).first()).toBeVisible({ timeout: 60_000 });98    await page.waitForURL(/\/app\/chat\/cnv_/, { timeout: 30_000 });99    conversationUrl = page.url();100    // footer meta appears (tokens + latency)101    await expect(page.getByText(/tok\/s|\bin ·/).first()).toBeVisible({ timeout: 30_000 });102    // sidebar shows the conversation with an auto title103    await expect(page.locator("aside").getByText(/reply with exactly/i).first()).toBeVisible();104  });105106  test("conversation persists after reload and a second model works", async ({ page }) => {107    await restoreSession(page);108    await page.goto(conversationUrl);109    await expect(page.getByText(/PONG/).first()).toBeVisible();110    // switch to an OpenAI model in the same conversation and continue111    await page.getByRole("button", { name: /select model/i }).click();112    await page.getByLabel(/search models/i).fill("gpt-5.4-nano");113    await page.getByRole("option").first().click();114    await page.getByLabel(/^message$/i).fill("Now reply with exactly the word PING.");115    await page.getByRole("button", { name: /^send$/i }).click();116    await expect(page.getByText(/PING/).first()).toBeVisible({ timeout: 60_000 });117    await page.reload();118    await expect(page.getByText(/PONG/).first()).toBeVisible();119    await expect(page.getByText(/PING/).first()).toBeVisible();120  });121122  test("model configuration is capability-driven", async ({ page }) => {123    await restoreSession(page);124    await page.goto("/app/chat");125    await page.getByRole("button", { name: /select model/i }).click();126    // Pin the native OpenAI catalog (OpenRouter also lists openai/* models with different parameter sheets).127    await page.getByRole("dialog").getByRole("button", { name: /^OpenAI$/ }).click();128    await page.getByLabel(/search models/i).fill("gpt-5.5");129    await page.getByRole("option").first().click();130    await page.getByRole("button", { name: /model configuration/i }).click();131    await expect(page.getByText(/reasoning/i).first()).toBeVisible();132    await expect(page.getByText(/^Effort$/)).toBeVisible();133    // seed/stop are not supported by the Responses API → never shown134    await expect(page.getByText(/^Seed$/)).toHaveCount(0);135    await expect(page.getByText(/^Stop sequences$/)).toHaveCount(0);136    await expect(page.getByText(/not available for this model/i)).toContainText(/seed/i);137    await page.keyboard.press("Escape");138  });139140  test("usage dashboard shows the requests", async ({ page }) => {141    await restoreSession(page);142    await page.goto("/app/usage");143    await expect(page.getByText(/requests/i).first()).toBeVisible();144    await expect.poll(() => Number(sql(`select count(*) from usage_records ur join users u on u.id=ur.user_id where u.email='${email}'`))).toBeGreaterThanOrEqual(2);145  });146147  test("change password, logout, login again", async ({ page }) => {148    await restoreSession(page);149    await page.goto("/app/settings/security");150    const newPassword = `${password}-2`;151    await page.getByLabel(/current password/i).fill(password);152    await page.getByLabel(/^new password$/i).fill(newPassword);153    const confirm = page.getByLabel(/confirm/i);154    if (await confirm.count()) await confirm.fill(newPassword);155    await page.getByRole("button", { name: /change password|update password/i }).click();156    await expect(page.getByText(/password (changed|updated)/i).first()).toBeVisible({ timeout: 20_000 });157    // sign out via the sidebar menu158    await page.goto("/app/chat");159    await page.getByText(email).first().click();160    await page.getByRole("menuitem", { name: /sign out/i }).click();161    await page.waitForURL(/\/login/);162    await login(page, email, newPassword);163    await expect(page).toHaveURL(/\/app/);164  });165});166