spb/zyquo-cloud-web Public MIT

1/*2 *  verify-matrix.mjs3 *  Zyquo Cloud Web4 *5 *  Author: Simon-Pierre Boucher6 *  Mail: contact@spboucher.ai7 *8 *  Phase 7 provider matrix: drives the app's own provider layer inside a real9 *  Chromium page (dev harness) with the user's real keys. Per model: streaming10 *  completion (token deltas, reasoning, usage, finish). Per provider:11 *  non-streaming completion, Stop/cancellation, vision probe (1×1 PNG on the12 *  cheapest vision model), Perplexity citations. Also audits every network13 *  origin the page contacted. Writes /tmp/zyquo-verify.json.14 *15 *  Usage: node scripts/verify-matrix.mjs [baseURL] [providers…]16 */1718import { chromium } from 'playwright'19import { readFileSync, writeFileSync } from 'node:fs'2021const baseURL = process.argv[2] ?? 'http://localhost:5173'22const onlyProviders = process.argv.slice(3)23const keys = JSON.parse(readFileSync(new URL('../.keys.local.json', import.meta.url), 'utf8'))2425// Deliberately skipped models (billing/time-outsized for a smoke matrix).26const SKIP = new Set(['sonar-deep-research'])27const PER_MODEL_TIMEOUT = 120_0002829const browser = await chromium.launch()30const page = await browser.newPage()31const origins = new Set()32page.on('request', (req) => origins.add(new URL(req.url()).origin))33page.setDefaultTimeout(PER_MODEL_TIMEOUT + 10_000)3435await page.goto(baseURL)36await page.evaluate((keyMap) => {37  localStorage.setItem('zyquo.cloud.web.keys', JSON.stringify(keyMap))38}, keys)39await page.reload()40await page.waitForFunction(() => window.__zyquo !== undefined)4142// A guaranteed-valid test image: 128×128 solid red, generated in-page.43const PNG_RED = await page.evaluate(() => {44  const canvas = document.createElement('canvas')45  canvas.width = 12846  canvas.height = 12847  const ctx = canvas.getContext('2d')48  ctx.fillStyle = '#d62828'49  ctx.fillRect(0, 0, 128, 128)50  return canvas.toDataURL('image/png').split(',')[1]51})5253/** Runs one streaming check inside the page. */54async function streamCheck(provider, modelID, opts = {}) {55  return page.evaluate(56    async ([provider, modelID, opts, png]) => {57      const z = window.__zyquo58      const model = z.findModel(provider, modelID)59      if (!model) return { ok: false, error: 'not in catalog' }60      const key = z.keys.getKey(provider)61      const client = z.clientFor(model)62      const controller = new AbortController()63      const start = performance.now()64      const result = {65        ok: false, text: 0, reasoning: 0, usage: null, finish: null,66        citations: 0, deltas: 0, error: null, ms: 0,67      }68      const attachments = opts.vision69        ? [{ id: 'v1', kind: 'image', fileName: 'dot.png', mimeType: 'image/png', data: png }]70        : []71      const timer = setTimeout(() => controller.abort(), opts.timeout ?? 110000)72      try {73        const events = client.streamChat(74          {75            model,76            messages: [{77              id: 'm1', role: 'user',78              text: opts.vision ? 'What color is this image? One word.' : (opts.prompt ?? 'Say OK.'),79              attachments, createdAt: Date.now(),80            }],81            parameters: { maxTokens: opts.maxTokens ?? 64 },82            stream: true,83          },84          key,85          controller.signal86        )87        for await (const event of events) {88          result.deltas++89          if (event.type === 'textDelta') result.text += event.text.length90          else if (event.type === 'reasoningDelta') result.reasoning += event.text.length91          else if (event.type === 'usage') result.usage = event.usage92          else if (event.type === 'citations') result.citations = event.citations.length93          else if (event.type === 'finished') result.finish = event.reason94        }95        result.ok = result.text > 0 || result.reasoning > 096        if (!result.ok) result.error = 'stream completed with no content'97      } catch (err) {98        result.error = err && err.message ? err.message : String(err)99      }100      clearTimeout(timer)101      result.ms = Math.round(performance.now() - start)102      return result103    },104    [provider, modelID, opts, PNG_RED]105  )106}107108async function completeCheck(provider, modelID) {109  return page.evaluate(110    async ([provider, modelID]) => {111      const z = window.__zyquo112      const model = z.findModel(provider, modelID)113      const key = z.keys.getKey(provider)114      try {115        const start = performance.now()116        const result = await z.clientFor(model).complete(117          {118            model,119            messages: [{ id: 'm1', role: 'user', text: 'Reply with exactly: OK', createdAt: Date.now() }],120            parameters: { maxTokens: 16 },121            stream: false,122          },123          key124        )125        return { ok: result.text.length > 0, ms: Math.round(performance.now() - start) }126      } catch (err) {127        return { ok: false, error: err && err.message ? err.message : String(err) }128      }129    },130    [provider, modelID]131  )132}133134async function cancelCheck(provider, modelID) {135  return page.evaluate(136    async ([provider, modelID]) => {137      const z = window.__zyquo138      const model = z.findModel(provider, modelID)139      const key = z.keys.getKey(provider)140      const controller = new AbortController()141      setTimeout(() => controller.abort(), 900)142      try {143        const events = z.clientFor(model).streamChat(144          {145            model,146            messages: [{ id: 'm1', role: 'user', text: 'Write a 500-word essay about the sea.', createdAt: Date.now() }],147            parameters: { maxTokens: 800 },148            stream: true,149          },150          key,151          controller.signal152        )153        for await (const _event of events) { /* drain until abort */ }154        return { ok: false, error: 'stream finished before abort' }155      } catch (err) {156        const message = err && err.message ? err.message : String(err)157        return { ok: /stopped|cancel/i.test(message), error: message }158      }159    },160    [provider, modelID]161  )162}163164const catalog = await page.evaluate(() =>165  window.__zyquo.CATALOG.map((m) => ({166    id: m.id, provider: m.provider,167    vision: m.capabilities.vision, reasoning: m.capabilities.reasoning,168    citations: m.capabilities.citations, legacy: m.isLegacy,169    outPrice: m.pricing ? m.pricing.outputPerMTok : null,170  }))171)172173const providers = [...new Set(catalog.map((m) => m.provider))].filter(174  (p) => onlyProviders.length === 0 || onlyProviders.includes(p)175)176const report = { startedAt: new Date().toISOString(), providers: {} }177178for (const provider of providers) {179  const models = catalog.filter((m) => m.provider === provider)180  const entry = { models: {}, nonStreaming: null, cancel: null, vision: null, citations: null }181  report.providers[provider] = entry182  console.log(`\n=== ${provider} (${models.length} models) ===`)183184  for (const model of models) {185    if (SKIP.has(model.id)) {186      entry.models[model.id] = { skipped: 'expensive deep-research model' }187      console.log(`  SKIP ${model.id}`)188      continue189    }190    const result = await streamCheck(provider, model.id, {191      maxTokens: model.reasoning ? 256 : 64,192    })193    entry.models[model.id] = result194    console.log(195      `  ${result.ok ? 'PASS' : 'FAIL'} ${model.id} (${result.ms}ms, text ${result.text}, reas ${result.reasoning}, deltas ${result.deltas}${result.error ? `, err: ${String(result.error).slice(0, 90)}` : ''})`196    )197  }198199  // Provider-level probes on the cheapest working model200  const working = models.filter((m) => entry.models[m.id] && entry.models[m.id].ok)201  const cheapest =202    working.filter((m) => !m.reasoning).sort((a, b) => (a.outPrice ?? 1e9) - (b.outPrice ?? 1e9))[0] ??203    working[0]204  if (cheapest) {205    entry.nonStreaming = { model: cheapest.id, ...(await completeCheck(provider, cheapest.id)) }206    console.log(`  non-streaming[${cheapest.id}]: ${entry.nonStreaming.ok ? 'PASS' : 'FAIL ' + (entry.nonStreaming.error ?? '')}`)207    entry.cancel = { model: cheapest.id, ...(await cancelCheck(provider, cheapest.id)) }208    console.log(`  cancel[${cheapest.id}]: ${entry.cancel.ok ? 'PASS' : 'FAIL ' + (entry.cancel.error ?? '')}`)209  }210  const visionModel = working211    .filter((m) => m.vision && !m.legacy)212    .sort((a, b) => (a.outPrice ?? 1e9) - (b.outPrice ?? 1e9))[0]213  if (visionModel) {214    const v = await streamCheck(provider, visionModel.id, { vision: true, maxTokens: 200 })215    entry.vision = { model: visionModel.id, ok: v.ok, error: v.error }216    console.log(`  vision[${visionModel.id}]: ${v.ok ? 'PASS' : 'FAIL ' + (v.error ?? '')}`)217  }218  if (provider === 'perplexity') {219    const c = await streamCheck(provider, 'sonar', {220      prompt: 'What is the tallest building in the world right now?',221      maxTokens: 128,222    })223    entry.citations = { model: 'sonar', ok: c.citations > 0, count: c.citations }224    console.log(`  citations[sonar]: ${c.citations > 0 ? `PASS (${c.citations})` : 'FAIL'}`)225  }226}227228report.contactedOrigins = [...origins].sort()229report.finishedAt = new Date().toISOString()230writeFileSync('/tmp/zyquo-verify.json', JSON.stringify(report, null, 2))231console.log('\nOrigins contacted:', report.contactedOrigins.join(', '))232console.log('Report → /tmp/zyquo-verify.json')233await browser.close()234