HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1/**2 * Wave-2 QA sweep: every route added in wave 2 (compare, timeline, prices, benchmark leaderboards, hardware fit, history/as-of,3 * diff, graph, admin, typed listings, manifest/OG) at 390 and 1440 px, dark and light — HTTP status, console errors, horizontal4 * overflow, screenshot — plus interaction flows: compare tray (add 2 models → matrix), hardware-fit form submit, admin login →5 * connectors → Run now, diff with two dates, graph renders nodes.6 *7 * Run: node qa/screens-wave2.mjs [BASE_URL] [API_URL] [ADMIN_TOKEN] (defaults http://localhost:8330, http://127.0.0.1:8331, dev-admin-token)8 * Screenshots → qa/screens/wave2/. Slugs are discovered live from the API — nothing is hardcoded.9 */10import { chromium } from '/Users/simon-pierreboucher/Desktop/uqo-eval/node_modules/playwright/index.mjs';11import { mkdirSync } from 'node:fs';1213const BASE = process.argv[2] ?? 'http://localhost:8330';14const API = process.argv[3] ?? 'http://127.0.0.1:8331';15const TOKEN = process.argv[4] ?? process.env.AIA_ADMIN_TOKEN ?? 'dev-admin-token';16const OUT = new URL('./screens/wave2/', import.meta.url).pathname;17mkdirSync(OUT, { recursive: true });1819const j = async (path, headers = {}) => (await fetch(`${API}/api/v1${path}`, { headers })).json();20const models = (await j('/models?limit=3&sort=quality')).items.map((m) => m.slug);21const bench = (await j('/benchmarks')).items.sort((a, b) => Number(b.result_count) - Number(a.result_count))[0]?.slug;22const benchModel = bench ? (await j(`/benchmarks/${bench}/results?limit=1`)).items[0]?.model?.slug : null;23const hardware = (await j('/hardware?limit=1&sort=memory')).items[0]?.slug;24const company = (await j('/companies?limit=1&sort=models')).items[0]?.slug;25const today = new Date().toISOString().slice(0, 10);26const weekAgo = new Date(Date.now() - 7 * 86400000).toISOString().slice(0, 10);27const [m1, m2, m3] = models;28console.log(`discovered models=${models.join(',')} bench=${bench} benchModel=${benchModel} hardware=${hardware} company=${company}`);2930const PAGES = [31 '/compare',32 `/compare?ids=${m1},${m2}`,33 `/compare?ids=${m1},${m2},${m3}`,34 '/timeline',35 `/timeline?year=${today.slice(0, 4)}&category=price`,36 `/timeline?entity=${m1}`,37 '/prices',38 '/prices?sort=output&days=90&scale=log',39 '/benchmarks',40 `/benchmarks/${bench}`,41 `/benchmarks/${bench}?model=${benchModel}`,42 '/benchmarks/does-not-exist',43 '/hardware',44 '/hardware/fit',45 '/hardware/fit?memory_gb=32&quant=4bit&context=8192',46 `/hardware/${hardware}`,47 `/models/${m1}`,48 `/models/${m1}?tab=history`,49 `/models/${m1}?tab=history&asof=${today}`,50 `/models/${m1}?tab=history&asof=2020-01-01`,51 `/companies/${company}?tab=history`,52 '/diff',53 `/diff?a=${weekAgo}&b=${today}&scope=models`,54 `/graph/${m1}`,55 `/graph/${m1}?depth=2`,56 '/graph/does-not-exist',57 '/papers',58 '/frameworks',59 '/datasets',60 '/tools',61 '/developers',62 '/admin',63 '/admin/connectors',64 '/does-not-exist-wave2',65];66const LIGHT_SUBSET = new Set(['/compare', `/compare?ids=${m1},${m2}`, '/timeline', '/prices', `/benchmarks/${bench}`, '/hardware/fit?memory_gb=32&quant=4bit&context=8192', `/models/${m1}?tab=history&asof=${today}`, `/diff?a=${weekAgo}&b=${today}&scope=models`, `/graph/${m1}`, '/papers', '/frameworks', '/admin', '/developers']);67const WIDTHS = [390, 1440];68const THEMES = ['dark', 'light'];69const IGNORE = /favicon|Failed to load resource: the server responded with a status of 404|Failed to load resource: the server responded with a status of 401|Failed to load resource: the server responded with a status of 403/;7071const browser = await chromium.launch();72let failures = 0;73const slug = (p) => p.replace(/[^a-z0-9]+/gi, '_').replace(/^_|_$/g, '') || 'home';7475function watch(page) {76 const errors = [];77 const onErr = (e) => errors.push(String(e));78 const onCon = (m) => {79 if (m.type() === 'error') errors.push(m.text());80 };81 page.on('pageerror', onErr);82 page.on('console', onCon);83 return { errors, off: () => (page.off('pageerror', onErr), page.off('console', onCon)) };84}85const clean = (errors, expected) => errors.filter((e) => !IGNORE.test(e)).filter((e) => !(expected === 404 && /Encountered a script tag while rendering React component/.test(e)));8687for (const theme of THEMES) {88 for (const width of WIDTHS) {89 const mobile = width < 768;90 const ctx = await browser.newContext({ viewport: { width, height: mobile ? 844 : 900 }, deviceScaleFactor: 1, isMobile: mobile, hasTouch: mobile, colorScheme: theme });91 await ctx.addInitScript((t) => localStorage.setItem('aia-theme', t), theme);92 const page = await ctx.newPage();93 for (const path of PAGES) {94 if (theme === 'light' && !LIGHT_SUBSET.has(path)) continue;95 const w = watch(page);96 const t0 = Date.now();97 const res = await page.goto(BASE + path, { waitUntil: 'networkidle', timeout: 90000 }).catch((e) => ({ status: () => `ERR ${e.message.slice(0, 40)}` }));98 await page.waitForTimeout(700);99 const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth).catch(() => -1);100 const applied = await page.evaluate(() => document.documentElement.getAttribute('data-theme')).catch(() => null);101 const expected = /does-not-exist/.test(path) ? 404 : 200;102 const status = res.status();103 const filtered = clean(w.errors, expected);104 const ok = status === expected && overflow <= 0 && filtered.length === 0 && applied === theme;105 if (!ok) failures++;106 console.log(`${ok ? 'OK ' : 'FAIL'} ${theme.padEnd(5)} ${width} ${status} ${String(Date.now() - t0).padStart(5)}ms overflow=${overflow} errors=${filtered.length} theme=${applied} ${path}${filtered.length ? ' :: ' + filtered[0].slice(0, 160) : ''}`);107 await page.screenshot({ path: `${OUT}${theme}-${width}-${slug(path)}.png`, fullPage: false }).catch(() => undefined);108 w.off();109 }110 await ctx.close();111 }112}113114/* ------------------------------------------------------------------------------------------------------------ flows */115async function flow(name, width, fn) {116 const mobile = width < 768;117 const ctx = await browser.newContext({ viewport: { width, height: mobile ? 844 : 900 }, deviceScaleFactor: 1, isMobile: mobile, hasTouch: mobile, colorScheme: 'dark' });118 await ctx.addInitScript(() => localStorage.setItem('aia-theme', 'dark'));119 const page = await ctx.newPage();120 const w = watch(page);121 try {122 await fn(page);123 const filtered = clean(w.errors, 200);124 const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth);125 const ok = filtered.length === 0 && overflow <= 0;126 if (!ok) failures++;127 console.log(`${ok ? 'OK ' : 'FAIL'} flow ${name} @${width} errors=${filtered.length} overflow=${overflow}${filtered.length ? ' :: ' + filtered[0].slice(0, 160) : ''}`);128 } catch (e) {129 failures++;130 console.log(`FAIL flow ${name} @${width} :: ${String(e.message ?? e).slice(0, 200)}`);131 }132 await page.screenshot({ path: `${OUT}flow-${name}-${width}.png`, fullPage: false }).catch(() => undefined);133 w.off();134 await ctx.close();135}136137for (const width of WIDTHS) {138 // Compare tray: add two models from the /models listing, then open /compare and expect the matrix.139 await flow('compare-tray', width, async (page) => {140 await page.goto(`${BASE}/models?sort=quality`, { waitUntil: 'networkidle' });141 const buttons = page.locator('button[aria-pressed]');142 if ((await buttons.count()) < 2) throw new Error('no compare buttons on /models');143 await buttons.nth(0).click();144 await buttons.nth(1).click();145 await page.waitForTimeout(300);146 const stored = await page.evaluate(() => JSON.parse(localStorage.getItem('aia-compare') ?? '[]'));147 if (stored.length !== 2) throw new Error(`tray holds ${stored.length} items, expected 2`);148 await page.goto(`${BASE}/compare?ids=${stored.map((s) => s.slug).join(',')}`, { waitUntil: 'networkidle' });149 const cells = await page.locator('table tbody tr').count();150 if (cells < 3) throw new Error(`matrix has ${cells} rows`);151 if (!(await page.locator('table thead th').count()) >= 3) throw new Error('matrix header missing entity columns');152 });153154 // Hardware fit: choose presets and submit the form.155 await flow('hardware-fit', width, async (page) => {156 await page.goto(`${BASE}/hardware/fit`, { waitUntil: 'networkidle' });157 const form = page.locator('form[action="/hardware/fit"]').first();158 await form.locator('select[name="memory_gb"]').selectOption('64');159 await form.locator('select[name="quant"]').selectOption('8bit');160 await form.locator('select[name="context"]').selectOption('32768');161 await Promise.all([page.waitForURL(/memory_gb=64/), form.locator('button[type="submit"]').first().click()]);162 await page.waitForLoadState('networkidle');163 const text = await page.evaluate(() => document.body.innerText);164 if (!/estimated/i.test(text)) throw new Error('missing ESTIMATED label');165 if ((await page.locator('table tbody tr').count()) < 1) throw new Error('no fit rows');166 });167168 // Admin: login → connectors table → Run now.169 await flow('admin', width, async (page) => {170 await page.goto(`${BASE}/admin`, { waitUntil: 'networkidle' });171 const html = await page.content();172 if (html.includes(TOKEN)) throw new Error('admin token leaked into HTML before login');173 await page.locator('input[type="password"]').first().fill(TOKEN);174 await Promise.all([page.waitForURL(/\/admin\/(overview|connectors)/, { timeout: 30000 }), page.locator('form button[type="submit"]').first().click()]);175 await page.goto(`${BASE}/admin/connectors`, { waitUntil: 'networkidle' });176 if ((await page.content()).includes(TOKEN)) throw new Error('admin token leaked into connectors HTML');177 const rows = await page.locator('table tbody tr').count();178 if (rows < 1) throw new Error('connectors table empty');179 // Connectors with a run already pending render a disabled button — pick the first enabled one.180 const run = page.locator('button:not([disabled])', { hasText: /run now/i }).first();181 if (!(await run.count())) throw new Error('no enabled Run now button');182 await run.click();183 await page.waitForLoadState('networkidle');184 await page.waitForTimeout(500);185 const after = await page.evaluate(() => document.body.innerText);186 if (!/queued|run now|pending|scheduled|enqueued/i.test(after)) throw new Error('no run-now feedback');187 });188189 // Diff with two dates.190 await flow('diff', width, async (page) => {191 await page.goto(`${BASE}/diff`, { waitUntil: 'networkidle' });192 const form = page.locator('form[action="/diff"]').first();193 await form.locator('input[name="a"]').fill(weekAgo);194 await form.locator('input[name="b"]').fill(today);195 await Promise.all([page.waitForURL(/a=\d{4}-\d{2}-\d{2}/), form.locator('button[type="submit"]').first().click()]);196 await page.waitForLoadState('networkidle');197 const text = await page.evaluate(() => document.body.innerText);198 if (!/new entities/i.test(text)) throw new Error('diff sections missing');199 });200201 // Graph renders nodes.202 await flow('graph', width, async (page) => {203 await page.goto(`${BASE}/graph/${m1}`, { waitUntil: 'networkidle' });204 await page.waitForTimeout(1500);205 const n = await page.locator('svg [data-node], svg circle').count();206 if (n < 2) throw new Error(`graph has ${n} nodes`);207 });208}209210await browser.close();211console.log(failures ? `\n${failures} failure(s)` : '\nall wave-2 checks OK');212process.exit(failures ? 1 : 0);213