/** * llmindex.io — DB seed: sync models from OpenRouter, optional demo score run * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * - Always: syncs the `models` table from OpenRouter /models (slug, pricing). * - If SEED_DEMO=1: creates a ScoreRun of kind "demo_seed" with deterministic, * clearly-labelled ILLUSTRATIVE scores so the UI/API can be exercised before * the first real eval + IRT fit. Demo runs are flagged in UI and API and are * replaced in the leaderboard as soon as a real index_fit run completes. */ import { createHash } from 'node:crypto'; import { prisma } from './index'; import { OpenRouterClient } from '@llmindex/openrouter'; import { DOMAINS, INDEX_VERSION } from '@llmindex/scoring'; // Curated providers whose flagship models are ranked by default. Slugs // themselves are NEVER hardcoded — they come from the live /models sync. const RANKED_PROVIDERS = ['anthropic', 'openai', 'google', 'meta-llama', 'mistralai', 'deepseek', 'qwen', 'x-ai']; const RANKED_PER_PROVIDER = 2; async function syncModels(): Promise { const client = new OpenRouterClient(); const models = await client.listModels(); console.log(`[seed] fetched ${models.length} models from OpenRouter`); const now = new Date(); for (const m of models) { const provider = m.id.split('/')[0] ?? 'unknown'; const vision = m.architecture?.input_modalities?.includes('image') ?? m.architecture?.modality?.includes('image') ?? false; await prisma.model.upsert({ where: { slug: m.id }, create: { slug: m.id, name: m.name, provider, contextLength: m.context_length ?? null, promptPricePerM: m.pricing ? Number(m.pricing.prompt) * 1_000_000 : null, completionPricePerM: m.pricing ? Number(m.pricing.completion) * 1_000_000 : null, vision, syncedAt: now, }, update: { name: m.name, contextLength: m.context_length ?? null, promptPricePerM: m.pricing ? Number(m.pricing.prompt) * 1_000_000 : null, completionPricePerM: m.pricing ? Number(m.pricing.completion) * 1_000_000 : null, vision, syncedAt: now, active: true, }, }); } // Models that vanished from OpenRouter become inactive (never deleted: audit trail). await prisma.model.updateMany({ where: { syncedAt: { lt: now } }, data: { active: false } }); // Curate the ranked subset. Preferred: explicit RANKED_MODELS env list (exact // OpenRouter slugs, comma-separated — configuration, not source). Fallback: // heuristic newest-priced models per provider. await prisma.model.updateMany({ data: { ranked: false } }); const explicit = (process.env.RANKED_MODELS ?? '') .split(',') .map((s) => s.trim()) .filter(Boolean); if (explicit.length > 0) { for (const slug of explicit) { const updated = await prisma.model.updateMany({ where: { slug, active: true }, data: { ranked: true } }); if (updated.count === 0) console.warn(`[seed] RANKED_MODELS slug not in catalog: ${slug}`); } const ranked = await prisma.model.count({ where: { ranked: true } }); console.log(`[seed] ranked subset (explicit): ${ranked}/${explicit.length} models`); return; } for (const provider of RANKED_PROVIDERS) { const candidates = await prisma.model.findMany({ where: { provider, active: true, promptPricePerM: { gt: 0 }, NOT: [ { slug: { contains: ':free' } }, { slug: { contains: '-base' } }, { name: { contains: '(older' } }, ], }, orderBy: [{ contextLength: 'desc' }, { slug: 'desc' }], take: RANKED_PER_PROVIDER, }); for (const c of candidates) { await prisma.model.update({ where: { id: c.id }, data: { ranked: true } }); } } const ranked = await prisma.model.count({ where: { ranked: true } }); console.log(`[seed] ranked subset: ${ranked} models`); } /** Deterministic pseudo-metric in [0,1] from a label — demo data only. */ function demoMetric(label: string): number { const h = createHash('sha256').update(label).digest(); return h.readUInt32BE(0) / 0xffffffff; } async function seedDemoRun(): Promise { const models = await prisma.model.findMany({ where: { ranked: true } }); if (models.length === 0) { console.warn('[seed] no ranked models; skipping demo run'); return; } const itemSetHash = createHash('sha256') .update('demo_seed:' + models.map((m) => m.slug).join(',')) .digest('hex'); const existing = await prisma.scoreRun.findFirst({ where: { kind: 'demo_seed', itemSetHash } }); if (existing) { console.log('[seed] demo run already present, skipping'); return; } const run = await prisma.scoreRun.create({ data: { indexVersion: INDEX_VERSION, kind: 'demo_seed', status: 'complete', itemSetHash, modelSet: models.map((m) => m.slug), fitDiagnostics: { synthetic: true }, notes: 'DEMO DATA — deterministic illustrative scores, NOT a real evaluation. ' + 'Replaced by the first index_fit run.', completedAt: new Date(), }, }); for (const m of models) { const domainScores: number[] = []; for (const domain of DOMAINS) { const base = 0.35 + 0.55 * demoMetric(`${m.slug}:${domain}`); const score = Math.round(base * 1000); const half = Math.round(20 + 30 * demoMetric(`${m.slug}:${domain}:ci`)); domainScores.push(score); await prisma.score.create({ data: { runId: run.id, modelId: m.id, domain, score, scoreLow: Math.max(0, score - half), scoreHigh: Math.min(1000, score + half), subMetrics: { accuracy_irt: Number(base.toFixed(3)), consistency: Number((0.7 + 0.3 * demoMetric(`${m.slug}:${domain}:c`)).toFixed(3)), calibration: Number((0.6 + 0.4 * demoMetric(`${m.slug}:${domain}:k`)).toFixed(3)), contamination_delta: Number((0.1 * demoMetric(`${m.slug}:${domain}:d`)).toFixed(3)), latency_p50: Math.round(400 + 4000 * demoMetric(`${m.slug}:${domain}:l`)), cost_per_1k_items: Number((0.5 + 30 * demoMetric(`${m.slug}:cost`)).toFixed(2)), }, }, }); } const global = Math.round(domainScores.reduce((a, b) => a + b, 0) / domainScores.length); await prisma.score.create({ data: { runId: run.id, modelId: m.id, domain: 'global', score: global, scoreLow: Math.max(0, global - 35), scoreHigh: Math.min(1000, global + 35), }, }); } console.log(`[seed] demo run ${run.id} created for ${models.length} models`); } async function main(): Promise { await syncModels(); if (process.env.SEED_DEMO === '1') await seedDemoRun(); } main() .catch((err) => { console.error('[seed] failed:', err); process.exitCode = 1; }) .finally(() => prisma.$disconnect());