/** * Wave-2 QA sweep: every route added in wave 2 (compare, timeline, prices, benchmark leaderboards, hardware fit, history/as-of, * diff, graph, admin, typed listings, manifest/OG) at 390 and 1440 px, dark and light — HTTP status, console errors, horizontal * overflow, screenshot — plus interaction flows: compare tray (add 2 models → matrix), hardware-fit form submit, admin login → * connectors → Run now, diff with two dates, graph renders nodes. * * Run: node qa/screens-wave2.mjs [BASE_URL] [API_URL] [ADMIN_TOKEN] (defaults http://localhost:8330, http://127.0.0.1:8331, dev-admin-token) * Screenshots → qa/screens/wave2/. Slugs are discovered live from the API — nothing is hardcoded. */ import { chromium } from '/Users/simon-pierreboucher/Desktop/uqo-eval/node_modules/playwright/index.mjs'; import { mkdirSync } from 'node:fs'; const BASE = process.argv[2] ?? 'http://localhost:8330'; const API = process.argv[3] ?? 'http://127.0.0.1:8331'; const TOKEN = process.argv[4] ?? process.env.AIA_ADMIN_TOKEN ?? 'dev-admin-token'; const OUT = new URL('./screens/wave2/', import.meta.url).pathname; mkdirSync(OUT, { recursive: true }); const j = async (path, headers = {}) => (await fetch(`${API}/api/v1${path}`, { headers })).json(); const models = (await j('/models?limit=3&sort=quality')).items.map((m) => m.slug); const bench = (await j('/benchmarks')).items.sort((a, b) => Number(b.result_count) - Number(a.result_count))[0]?.slug; const benchModel = bench ? (await j(`/benchmarks/${bench}/results?limit=1`)).items[0]?.model?.slug : null; const hardware = (await j('/hardware?limit=1&sort=memory')).items[0]?.slug; const company = (await j('/companies?limit=1&sort=models')).items[0]?.slug; const today = new Date().toISOString().slice(0, 10); const weekAgo = new Date(Date.now() - 7 * 86400000).toISOString().slice(0, 10); const [m1, m2, m3] = models; console.log(`discovered models=${models.join(',')} bench=${bench} benchModel=${benchModel} hardware=${hardware} company=${company}`); const PAGES = [ '/compare', `/compare?ids=${m1},${m2}`, `/compare?ids=${m1},${m2},${m3}`, '/timeline', `/timeline?year=${today.slice(0, 4)}&category=price`, `/timeline?entity=${m1}`, '/prices', '/prices?sort=output&days=90&scale=log', '/benchmarks', `/benchmarks/${bench}`, `/benchmarks/${bench}?model=${benchModel}`, '/benchmarks/does-not-exist', '/hardware', '/hardware/fit', '/hardware/fit?memory_gb=32&quant=4bit&context=8192', `/hardware/${hardware}`, `/models/${m1}`, `/models/${m1}?tab=history`, `/models/${m1}?tab=history&asof=${today}`, `/models/${m1}?tab=history&asof=2020-01-01`, `/companies/${company}?tab=history`, '/diff', `/diff?a=${weekAgo}&b=${today}&scope=models`, `/graph/${m1}`, `/graph/${m1}?depth=2`, '/graph/does-not-exist', '/papers', '/frameworks', '/datasets', '/tools', '/developers', '/admin', '/admin/connectors', '/does-not-exist-wave2', ]; const LIGHT_SUBSET = new Set(['/compare', `/compare?ids=${m1},${m2}`, '/timeline', '/prices', `/benchmarks/${bench}`, '/hardware/fit?memory_gb=32&quant=4bit&context=8192', `/models/${m1}?tab=history&asof=${today}`, `/diff?a=${weekAgo}&b=${today}&scope=models`, `/graph/${m1}`, '/papers', '/frameworks', '/admin', '/developers']); const WIDTHS = [390, 1440]; const THEMES = ['dark', 'light']; const IGNORE = /favicon|Failed to load resource: the server responded with a status of 404|Failed to load resource: the server responded with a status of 401|Failed to load resource: the server responded with a status of 403/; const browser = await chromium.launch(); let failures = 0; const slug = (p) => p.replace(/[^a-z0-9]+/gi, '_').replace(/^_|_$/g, '') || 'home'; function watch(page) { const errors = []; const onErr = (e) => errors.push(String(e)); const onCon = (m) => { if (m.type() === 'error') errors.push(m.text()); }; page.on('pageerror', onErr); page.on('console', onCon); return { errors, off: () => (page.off('pageerror', onErr), page.off('console', onCon)) }; } const clean = (errors, expected) => errors.filter((e) => !IGNORE.test(e)).filter((e) => !(expected === 404 && /Encountered a script tag while rendering React component/.test(e))); for (const theme of THEMES) { for (const width of WIDTHS) { const mobile = width < 768; const ctx = await browser.newContext({ viewport: { width, height: mobile ? 844 : 900 }, deviceScaleFactor: 1, isMobile: mobile, hasTouch: mobile, colorScheme: theme }); await ctx.addInitScript((t) => localStorage.setItem('aia-theme', t), theme); const page = await ctx.newPage(); for (const path of PAGES) { if (theme === 'light' && !LIGHT_SUBSET.has(path)) continue; const w = watch(page); const t0 = Date.now(); const res = await page.goto(BASE + path, { waitUntil: 'networkidle', timeout: 90000 }).catch((e) => ({ status: () => `ERR ${e.message.slice(0, 40)}` })); await page.waitForTimeout(700); const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth).catch(() => -1); const applied = await page.evaluate(() => document.documentElement.getAttribute('data-theme')).catch(() => null); const expected = /does-not-exist/.test(path) ? 404 : 200; const status = res.status(); const filtered = clean(w.errors, expected); const ok = status === expected && overflow <= 0 && filtered.length === 0 && applied === theme; if (!ok) failures++; console.log(`${ok ? 'OK ' : 'FAIL'} ${theme.padEnd(5)} ${width} ${status} ${String(Date.now() - t0).padStart(5)}ms overflow=${overflow} errors=${filtered.length} theme=${applied} ${path}${filtered.length ? ' :: ' + filtered[0].slice(0, 160) : ''}`); await page.screenshot({ path: `${OUT}${theme}-${width}-${slug(path)}.png`, fullPage: false }).catch(() => undefined); w.off(); } await ctx.close(); } } /* ------------------------------------------------------------------------------------------------------------ flows */ async function flow(name, width, fn) { const mobile = width < 768; const ctx = await browser.newContext({ viewport: { width, height: mobile ? 844 : 900 }, deviceScaleFactor: 1, isMobile: mobile, hasTouch: mobile, colorScheme: 'dark' }); await ctx.addInitScript(() => localStorage.setItem('aia-theme', 'dark')); const page = await ctx.newPage(); const w = watch(page); try { await fn(page); const filtered = clean(w.errors, 200); const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); const ok = filtered.length === 0 && overflow <= 0; if (!ok) failures++; console.log(`${ok ? 'OK ' : 'FAIL'} flow ${name} @${width} errors=${filtered.length} overflow=${overflow}${filtered.length ? ' :: ' + filtered[0].slice(0, 160) : ''}`); } catch (e) { failures++; console.log(`FAIL flow ${name} @${width} :: ${String(e.message ?? e).slice(0, 200)}`); } await page.screenshot({ path: `${OUT}flow-${name}-${width}.png`, fullPage: false }).catch(() => undefined); w.off(); await ctx.close(); } for (const width of WIDTHS) { // Compare tray: add two models from the /models listing, then open /compare and expect the matrix. await flow('compare-tray', width, async (page) => { await page.goto(`${BASE}/models?sort=quality`, { waitUntil: 'networkidle' }); const buttons = page.locator('button[aria-pressed]'); if ((await buttons.count()) < 2) throw new Error('no compare buttons on /models'); await buttons.nth(0).click(); await buttons.nth(1).click(); await page.waitForTimeout(300); const stored = await page.evaluate(() => JSON.parse(localStorage.getItem('aia-compare') ?? '[]')); if (stored.length !== 2) throw new Error(`tray holds ${stored.length} items, expected 2`); await page.goto(`${BASE}/compare?ids=${stored.map((s) => s.slug).join(',')}`, { waitUntil: 'networkidle' }); const cells = await page.locator('table tbody tr').count(); if (cells < 3) throw new Error(`matrix has ${cells} rows`); if (!(await page.locator('table thead th').count()) >= 3) throw new Error('matrix header missing entity columns'); }); // Hardware fit: choose presets and submit the form. await flow('hardware-fit', width, async (page) => { await page.goto(`${BASE}/hardware/fit`, { waitUntil: 'networkidle' }); const form = page.locator('form[action="/hardware/fit"]').first(); await form.locator('select[name="memory_gb"]').selectOption('64'); await form.locator('select[name="quant"]').selectOption('8bit'); await form.locator('select[name="context"]').selectOption('32768'); await Promise.all([page.waitForURL(/memory_gb=64/), form.locator('button[type="submit"]').first().click()]); await page.waitForLoadState('networkidle'); const text = await page.evaluate(() => document.body.innerText); if (!/estimated/i.test(text)) throw new Error('missing ESTIMATED label'); if ((await page.locator('table tbody tr').count()) < 1) throw new Error('no fit rows'); }); // Admin: login → connectors table → Run now. await flow('admin', width, async (page) => { await page.goto(`${BASE}/admin`, { waitUntil: 'networkidle' }); const html = await page.content(); if (html.includes(TOKEN)) throw new Error('admin token leaked into HTML before login'); await page.locator('input[type="password"]').first().fill(TOKEN); await Promise.all([page.waitForURL(/\/admin\/(overview|connectors)/, { timeout: 30000 }), page.locator('form button[type="submit"]').first().click()]); await page.goto(`${BASE}/admin/connectors`, { waitUntil: 'networkidle' }); if ((await page.content()).includes(TOKEN)) throw new Error('admin token leaked into connectors HTML'); const rows = await page.locator('table tbody tr').count(); if (rows < 1) throw new Error('connectors table empty'); // Connectors with a run already pending render a disabled button — pick the first enabled one. const run = page.locator('button:not([disabled])', { hasText: /run now/i }).first(); if (!(await run.count())) throw new Error('no enabled Run now button'); await run.click(); await page.waitForLoadState('networkidle'); await page.waitForTimeout(500); const after = await page.evaluate(() => document.body.innerText); if (!/queued|run now|pending|scheduled|enqueued/i.test(after)) throw new Error('no run-now feedback'); }); // Diff with two dates. await flow('diff', width, async (page) => { await page.goto(`${BASE}/diff`, { waitUntil: 'networkidle' }); const form = page.locator('form[action="/diff"]').first(); await form.locator('input[name="a"]').fill(weekAgo); await form.locator('input[name="b"]').fill(today); await Promise.all([page.waitForURL(/a=\d{4}-\d{2}-\d{2}/), form.locator('button[type="submit"]').first().click()]); await page.waitForLoadState('networkidle'); const text = await page.evaluate(() => document.body.innerText); if (!/new entities/i.test(text)) throw new Error('diff sections missing'); }); // Graph renders nodes. await flow('graph', width, async (page) => { await page.goto(`${BASE}/graph/${m1}`, { waitUntil: 'networkidle' }); await page.waitForTimeout(1500); const n = await page.locator('svg [data-node], svg circle').count(); if (n < 2) throw new Error(`graph has ${n} nodes`); }); } await browser.close(); console.log(failures ? `\n${failures} failure(s)` : '\nall wave-2 checks OK'); process.exit(failures ? 1 : 0);