spb/llmindex Public

1/**2 * llmindex.io — DB seed: sync models from OpenRouter, optional demo score run3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * - Always: syncs the `models` table from OpenRouter /models (slug, pricing).8 * - If SEED_DEMO=1: creates a ScoreRun of kind "demo_seed" with deterministic,9 *   clearly-labelled ILLUSTRATIVE scores so the UI/API can be exercised before10 *   the first real eval + IRT fit. Demo runs are flagged in UI and API and are11 *   replaced in the leaderboard as soon as a real index_fit run completes.12 */13import { createHash } from 'node:crypto';14import { prisma } from './index';15import { OpenRouterClient } from '@llmindex/openrouter';16import { DOMAINS, INDEX_VERSION } from '@llmindex/scoring';1718// Curated providers whose flagship models are ranked by default. Slugs19// themselves are NEVER hardcoded — they come from the live /models sync.20const RANKED_PROVIDERS = ['anthropic', 'openai', 'google', 'meta-llama', 'mistralai', 'deepseek', 'qwen', 'x-ai'];21const RANKED_PER_PROVIDER = 2;2223async function syncModels(): Promise<void> {24  const client = new OpenRouterClient();25  const models = await client.listModels();26  console.log(`[seed] fetched ${models.length} models from OpenRouter`);2728  const now = new Date();29  for (const m of models) {30    const provider = m.id.split('/')[0] ?? 'unknown';31    const vision =32      m.architecture?.input_modalities?.includes('image') ??33      m.architecture?.modality?.includes('image') ??34      false;35    await prisma.model.upsert({36      where: { slug: m.id },37      create: {38        slug: m.id,39        name: m.name,40        provider,41        contextLength: m.context_length ?? null,42        promptPricePerM: m.pricing ? Number(m.pricing.prompt) * 1_000_000 : null,43        completionPricePerM: m.pricing ? Number(m.pricing.completion) * 1_000_000 : null,44        vision,45        syncedAt: now,46      },47      update: {48        name: m.name,49        contextLength: m.context_length ?? null,50        promptPricePerM: m.pricing ? Number(m.pricing.prompt) * 1_000_000 : null,51        completionPricePerM: m.pricing ? Number(m.pricing.completion) * 1_000_000 : null,52        vision,53        syncedAt: now,54        active: true,55      },56    });57  }58  // Models that vanished from OpenRouter become inactive (never deleted: audit trail).59  await prisma.model.updateMany({ where: { syncedAt: { lt: now } }, data: { active: false } });6061  // Curate the ranked subset. Preferred: explicit RANKED_MODELS env list (exact62  // OpenRouter slugs, comma-separated — configuration, not source). Fallback:63  // heuristic newest-priced models per provider.64  await prisma.model.updateMany({ data: { ranked: false } });65  const explicit = (process.env.RANKED_MODELS ?? '')66    .split(',')67    .map((s) => s.trim())68    .filter(Boolean);69  if (explicit.length > 0) {70    for (const slug of explicit) {71      const updated = await prisma.model.updateMany({ where: { slug, active: true }, data: { ranked: true } });72      if (updated.count === 0) console.warn(`[seed] RANKED_MODELS slug not in catalog: ${slug}`);73    }74    const ranked = await prisma.model.count({ where: { ranked: true } });75    console.log(`[seed] ranked subset (explicit): ${ranked}/${explicit.length} models`);76    return;77  }78  for (const provider of RANKED_PROVIDERS) {79    const candidates = await prisma.model.findMany({80      where: {81        provider,82        active: true,83        promptPricePerM: { gt: 0 },84        NOT: [85          { slug: { contains: ':free' } },86          { slug: { contains: '-base' } },87          { name: { contains: '(older' } },88        ],89      },90      orderBy: [{ contextLength: 'desc' }, { slug: 'desc' }],91      take: RANKED_PER_PROVIDER,92    });93    for (const c of candidates) {94      await prisma.model.update({ where: { id: c.id }, data: { ranked: true } });95    }96  }97  const ranked = await prisma.model.count({ where: { ranked: true } });98  console.log(`[seed] ranked subset: ${ranked} models`);99}100101/** Deterministic pseudo-metric in [0,1] from a label — demo data only. */102function demoMetric(label: string): number {103  const h = createHash('sha256').update(label).digest();104  return h.readUInt32BE(0) / 0xffffffff;105}106107async function seedDemoRun(): Promise<void> {108  const models = await prisma.model.findMany({ where: { ranked: true } });109  if (models.length === 0) {110    console.warn('[seed] no ranked models; skipping demo run');111    return;112  }113  const itemSetHash = createHash('sha256')114    .update('demo_seed:' + models.map((m) => m.slug).join(','))115    .digest('hex');116117  const existing = await prisma.scoreRun.findFirst({ where: { kind: 'demo_seed', itemSetHash } });118  if (existing) {119    console.log('[seed] demo run already present, skipping');120    return;121  }122123  const run = await prisma.scoreRun.create({124    data: {125      indexVersion: INDEX_VERSION,126      kind: 'demo_seed',127      status: 'complete',128      itemSetHash,129      modelSet: models.map((m) => m.slug),130      fitDiagnostics: { synthetic: true },131      notes:132        'DEMO DATA — deterministic illustrative scores, NOT a real evaluation. ' +133        'Replaced by the first index_fit run.',134      completedAt: new Date(),135    },136  });137138  for (const m of models) {139    const domainScores: number[] = [];140    for (const domain of DOMAINS) {141      const base = 0.35 + 0.55 * demoMetric(`${m.slug}:${domain}`);142      const score = Math.round(base * 1000);143      const half = Math.round(20 + 30 * demoMetric(`${m.slug}:${domain}:ci`));144      domainScores.push(score);145      await prisma.score.create({146        data: {147          runId: run.id,148          modelId: m.id,149          domain,150          score,151          scoreLow: Math.max(0, score - half),152          scoreHigh: Math.min(1000, score + half),153          subMetrics: {154            accuracy_irt: Number(base.toFixed(3)),155            consistency: Number((0.7 + 0.3 * demoMetric(`${m.slug}:${domain}:c`)).toFixed(3)),156            calibration: Number((0.6 + 0.4 * demoMetric(`${m.slug}:${domain}:k`)).toFixed(3)),157            contamination_delta: Number((0.1 * demoMetric(`${m.slug}:${domain}:d`)).toFixed(3)),158            latency_p50: Math.round(400 + 4000 * demoMetric(`${m.slug}:${domain}:l`)),159            cost_per_1k_items: Number((0.5 + 30 * demoMetric(`${m.slug}:cost`)).toFixed(2)),160          },161        },162      });163    }164    const global = Math.round(domainScores.reduce((a, b) => a + b, 0) / domainScores.length);165    await prisma.score.create({166      data: {167        runId: run.id,168        modelId: m.id,169        domain: 'global',170        score: global,171        scoreLow: Math.max(0, global - 35),172        scoreHigh: Math.min(1000, global + 35),173      },174    });175  }176  console.log(`[seed] demo run ${run.id} created for ${models.length} models`);177}178179async function main(): Promise<void> {180  await syncModels();181  if (process.env.SEED_DEMO === '1') await seedDemoRun();182}183184main()185  .catch((err) => {186    console.error('[seed] failed:', err);187    process.exitCode = 1;188  })189  .finally(() => prisma.$disconnect());190