HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1/**2 * Stream D1 QA sweep — models · artifacts · families · benchmarks · compare · licences.3 * Widths 320 · 360 · 375 · 390 · 430 · 768 · 1366 · 1440 · 1920, dark + light. Slugs are discovered live from the API.4 * Asserts: HTTP status (308 for merged / artifact slugs under /models), zero console errors, no horizontal overflow5 * (scrollWidth ≤ clientWidth), the evidence drawer opens from a model-page value, the matrix renders ≥ 1 heatmap cell,6 * the leaderboard has one row per model (no duplicate model slug in the first 50 rows).7 * Run: node qa/d1.mjs [BASE=http://localhost:8341] [API=http://127.0.0.1:8332] (FAST=1 → 390 + 1440 only)8 */9import { chromium } from '/Users/simon-pierreboucher/Desktop/uqo-eval/node_modules/playwright/index.mjs';10import { mkdirSync } from 'node:fs';1112const BASE = process.argv[2] ?? 'http://localhost:8341';13const API = (process.argv[3] ?? 'http://127.0.0.1:8332') + '/api/v1';14const OUT = new URL('./screens/d1/', import.meta.url).pathname;15mkdirSync(OUT, { recursive: true });16const WIDTHS = process.env.FAST === '1' ? [390, 1440] : [320, 360, 375, 390, 430, 768, 1366, 1440, 1920];17const THEMES = ['dark', 'light'];18const j = async (u) => (await fetch(API + u)).json();1920// ---------------------------------------------------------------------------------------------------------- discover slugs21const top = await j('/models?limit=3&sort=quality');22const [a, b, c] = top.items.map((m) => m.slug);23const artRes = await j('/models?include=artifacts&limit=80');24// prefer an artifact whose canonical model is resolved (the page shows a prominent link + note); fall back to any artifact25const artifactRow = artRes.items.find((m) => m.entity_type === 'artifact' && m.canonical) ?? artRes.items.find((m) => m.entity_type === 'artifact');26const artifact = artifactRow?.slug;27const artifactHasCanonical = !!artifactRow?.canonical;28const detail = await j(`/models/${encodeURIComponent(a)}`);29let merged = null;30for (const alias of detail.identity?.api_aliases ?? []) {31 if (alias.includes('/') || alias === a) continue;32 const d = await j(`/models/${encodeURIComponent(alias)}`);33 if (d.redirected_from && d.slug === a) {34 merged = alias;35 break;36 }37}38const fam = (await j('/families?limit=1&sort=models')).items[0]?.slug;39const benches = (await j('/benchmarks')).items.filter((x) => Number(x.result_count) > 0);40const bench = benches.find((x) => x.slug === 'gpqa-diamond')?.slug ?? benches[0]?.slug;41const lic = (await j('/licenses')).items[0]?.key ?? 'Apache-2.0';42console.log(`slugs: model=${a} artifact=${artifact} merged=${merged} family=${fam} benchmark=${bench} licence=${lic}`);4344const PAGES = [45 '/models',46 '/models?openness=open-weights&min_params=7B&sort=params',47 `/models/${a}`,48 `/models/${a}/diff/${b}`,49 `/artifacts/${artifact}`,50 `/compare?ids=${a},${b},${c}`,51 '/benchmarks',52 `/benchmarks/${bench}`,53 '/benchmarks/matrix',54 `/benchmarks/${bench}/cost-vs-performance`,55 '/families',56 `/families/${fam}`,57 '/licenses',58 `/licenses/${encodeURIComponent(lic)}`,59];60// 404 favicon noise and 429s from the API's per-IP rate limiter (the sweep loads a page every second) are not page bugs.61const filterErrors = (errors) => errors.filter((e) => !/favicon|Failed to load resource: the server responded with a status of (404|429)/.test(e));6263let failures = 0;64const ok = (cond, msg) => {65 if (!cond) failures++;66 console.log(`${cond ? 'OK ' : 'FAIL'} ${msg}`);67};6869// ---------------------------------------------------------------------------------------------------------- redirects70for (const [path, expect] of [71 [`/models/${artifact}`, `/artifacts/${artifact}`],72 [`/models/${merged}`, `/models/${a}`],73]) {74 if (!path.includes('null')) {75 const r = await fetch(BASE + path, { redirect: 'manual' });76 const loc = r.headers.get('location') ?? '';77 ok((r.status === 308 || r.status === 301) && loc.endsWith(expect), `redirect ${path} → ${r.status} ${loc}`);78 } else console.log(`SKIP redirect (no slug discovered) ${path}`);79}80for (const path of ['/models/does-not-exist-xyz', '/artifacts/does-not-exist-xyz', '/families/does-not-exist-xyz', '/licenses/does-not-exist-xyz', '/benchmarks/does-not-exist-xyz']) {81 const r = await fetch(BASE + path);82 ok(r.status === 404, `404 ${path} → ${r.status}`);83}84for (const path of [`/models/${a}/opengraph-image`, `/artifacts/${artifact}/opengraph-image`, `/benchmarks/${bench}/opengraph-image`, `/families/${fam}/opengraph-image`, `/licenses/${encodeURIComponent(lic)}/opengraph-image`]) {85 const r = await fetch(BASE + path);86 ok(r.status === 200 && (r.headers.get('content-type') ?? '').startsWith('image/png'), `og ${path} → ${r.status} ${r.headers.get('content-type')}`);87}8889// ---------------------------------------------------------------------------------------------------------- sweep90const browser = await chromium.launch();91for (const theme of THEMES) {92 for (const width of WIDTHS) {93 const mobile = width < 768;94 const ctx = await browser.newContext({ viewport: { width, height: mobile ? 844 : 900 }, deviceScaleFactor: 1, isMobile: mobile, hasTouch: mobile, colorScheme: theme });95 await ctx.addInitScript((t) => localStorage.setItem('aia-theme', t), theme);96 const page = await ctx.newPage();97 for (const path of PAGES) {98 const errors = [];99 const onErr = (e) => errors.push(String(e));100 const onCon = (m) => {101 if (m.type() === 'error') errors.push(m.text());102 };103 page.on('pageerror', onErr);104 page.on('console', onCon);105 const t0 = Date.now();106 const res = await page.goto(BASE + path, { waitUntil: 'networkidle', timeout: 90000 }).catch((e) => ({ status: () => `ERR ${e.message.slice(0, 40)}` }));107 await page.waitForTimeout(400);108 const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth).catch(() => -1);109 const filtered = filterErrors(errors);110 const status = res.status();111 const good = status === 200 && overflow <= 0 && filtered.length === 0;112 if (!good) failures++;113 console.log(`${good ? 'OK ' : 'FAIL'} ${theme.padEnd(5)} ${String(width).padStart(4)} ${status} ${String(Date.now() - t0).padStart(5)}ms overflow=${overflow} errors=${filtered.length} ${path}${filtered.length ? ' :: ' + filtered[0].slice(0, 160) : ''}`);114 if ([390, 1440].includes(width) || !good) await page.screenshot({ path: `${OUT}${theme}-${width}-${path.replace(/[^a-z0-9]+/gi, '_').replace(/^_|_$/g, '')}.png`, fullPage: false }).catch(() => undefined);115 page.off('pageerror', onErr);116 page.off('console', onCon);117 }118 await ctx.close();119 }120}121122// ---------------------------------------------------------------------------------------------------------- flows (1440 dark)123{124 const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, colorScheme: 'dark' });125 await ctx.addInitScript(() => localStorage.setItem('aia-theme', 'dark'));126 const page = await ctx.newPage();127128 // evidence drawer from a model page value129 await page.goto(`${BASE}/models/${a}`, { waitUntil: 'networkidle' });130 const trigger = page.locator('[data-identity-strip] [data-evidence]').first();131 ok((await trigger.count()) > 0, 'model page: identity strip has evidence triggers');132 await trigger.click();133 const drawer = await page.waitForSelector('[data-evidence-drawer]', { timeout: 8000 }).catch(() => null);134 ok(!!drawer, 'model page: evidence drawer opens from an identity-strip value');135 const drawerText = drawer ? await drawer.innerText() : '';136 ok(/Source|Tier|Observed/i.test(drawerText), 'evidence drawer shows source · tier · observed');137 await page.screenshot({ path: `${OUT}flow-evidence-drawer.png` });138 await page.keyboard.press('Escape');139 // section nav present with sections in order140 const navItems = await page.$$eval('[data-section-nav] a', (as) => as.map((x) => x.textContent.trim()));141 ok(navItems[0] === 'Overview' && navItems.at(-1) === 'Provenance', `section nav order: ${navItems.join(' · ')}`);142 // every rendered section id has a nav item143 const sectionIds = await page.$$eval('section[data-section]', (ss) => ss.map((s) => s.id));144 const navIds = await page.$$eval('[data-section-nav] a', (as) => as.map((x) => x.getAttribute('href').slice(1)));145 ok(sectionIds.every((id) => navIds.includes(id)), `all ${sectionIds.length} sections are in the nav`);146147 // models list: inspector reflects the inspected row; column chooser148 await page.goto(`${BASE}/models`, { waitUntil: 'networkidle' });149 const second = page.locator('[data-models-table] tr[data-row]').nth(1);150 const secondSlug = await second.getAttribute('data-row');151 await second.locator('[data-inspect]').click();152 await page.waitForTimeout(200);153 const inspected = await page.getAttribute('aside[aria-label="Inspector"] [data-models-inspector]', 'data-inspected');154 ok(inspected === secondSlug, `models inspector shows the inspected row (${inspected})`);155 await page.click('[data-column-chooser]');156 const chooserLabels = await page.$$eval('[role="group"][aria-label="Visible columns"] label', (ls) => ls.length);157 ok(chooserLabels >= 6, `column chooser lists ${chooserLabels} columns`);158 const understood = await page.goto(`${BASE}/models?openness=open-weights&min_params=7B`, { waitUntil: 'networkidle' }).then(() => page.$$eval('[data-understood] a', (as) => as.map((x) => x.textContent.trim())));159 ok(understood.some((t) => /parameters ≥ 7B/.test(t)) && understood.some((t) => /Open weights/.test(t)), `understood-as chips: ${understood.join(' | ')}`);160 const countText = await page.textContent('[data-models-count]');161 ok(/canonical models/.test(countText ?? ''), `count label: ${countText?.trim().slice(0, 60)}`);162163 // matrix renders cells164 await page.goto(`${BASE}/benchmarks/matrix`, { waitUntil: 'networkidle' });165 const cells = await page.$$eval('table.heatmap td:not(.empty)', (tds) => tds.length);166 ok(cells >= 1, `matrix renders ${cells} heatmap cells`);167168 // leaderboard: one row per model169 await page.goto(`${BASE}/benchmarks/${bench}`, { waitUntil: 'networkidle' });170 const slugs = await page.$$eval('[data-leaderboard] tr[data-model]', (trs) => trs.slice(0, 50).map((t) => t.getAttribute('data-model')));171 ok(slugs.length > 0 && new Set(slugs).size === slugs.length, `leaderboard: ${slugs.length} rows, ${new Set(slugs).size} distinct models in the first 50`);172 const lbApi = await j(`/benchmarks/${bench}/leaderboard?limit=50`);173 ok(slugs[0] === lbApi.items[0]?.model.slug, `leaderboard first row matches the API leader (${slugs[0]})`);174175 // compare: hide-identical toggle + differences-only link176 await page.goto(`${BASE}/compare?ids=${a},${b},${c}`, { waitUntil: 'networkidle' });177 const before = await page.$$eval('[data-compare-table] tr[data-dim]', (r) => r.length);178 await page.click('[data-hide-identical]');179 await page.waitForTimeout(150);180 const after = await page.$$eval('[data-compare-table] tr[data-dim]', (r) => r.length);181 ok(after >= before, `compare: ${before} rows with identical hidden → ${after} rows shown when unhidden`);182 const diffOnlyHref = await page.getAttribute('[data-diff-only]', 'href');183 ok((diffOnlyHref ?? '').includes('diff_only=1'), `compare: differences-only link → ${diffOnlyHref}`);184 const evidenceCells = await page.$$eval('[data-compare-table] [data-evidence]', (x) => x.length);185 ok(evidenceCells > 0, `compare: ${evidenceCells} cells open evidence`);186187 // artifact page note + canonical link188 await page.goto(`${BASE}/artifacts/${artifact}`, { waitUntil: 'networkidle' });189 if (artifactHasCanonical) ok((await page.locator('[data-artifact-note]').count()) === 1, 'artifact page shows the "not an independent model" note with a canonical link');190 else ok((await page.locator('[data-artifact-header]').count()) === 1, `artifact page renders (canonical model unresolved for ${artifact} — honest fallback note)`);191 await ctx.close();192}193194await browser.close();195console.log(failures ? `\n${failures} failure(s)` : '\nall D1 checks OK');196process.exit(failures ? 1 : 0);197