spb/llmindex Public

1// llmindex.io — Prisma schema: models, eval items, responses, duels, score runs2// Author:  Simon-Pierre Boucher3// Contact: contact@spboucher.ai4// License: Proprietary — © Simon-Pierre Boucher, all rights reserved56generator client {7  provider = "prisma-client-js"8}910datasource db {11  provider = "postgresql"12  url      = env("DATABASE_URL")13}1415// LLMs synced daily from OpenRouter /models (slug, pricing → cost_per_1k_items).16model Model {17  id              String   @id @default(cuid())18  slug            String   @unique // OpenRouter slug, e.g. "anthropic/claude-..."19  name            String20  provider        String21  contextLength   Int?     @map("context_length")22  // USD per 1M tokens (converted from OpenRouter per-token pricing at sync time)23  promptPricePerM Float?   @map("prompt_price_per_m")24  completionPricePerM Float? @map("completion_price_per_m")25  active          Boolean  @default(true)26  ranked          Boolean  @default(false) // curated subset shown on the index27  vision          Boolean  @default(false) // accepts image input (vision_ocr domain)28  syncedAt        DateTime @map("synced_at")29  createdAt       DateTime @default(now()) @map("created_at")3031  responses ModelResponse[]32  scores    Score[]33  duelsAsA  PairwiseDuel[]  @relation("duel_model_a")34  duelsAsB  PairwiseDuel[]  @relation("duel_model_b")3536  @@map("models")37}3839// Instantiated evaluation items. answer_key/rubric are SERVER-SIDE ONLY:40// they must never be selected by web app queries or public API serializers.41model EvalItem {42  id          String   @id @default(cuid())43  templateId  String   @map("template_id") // versioned template id from packages/items44  domain      String45  prompt      String46  answerKey   String   @map("answer_key")47  rubric      String?48  perturbSeed String?  @map("perturb_seed") // seed used by the perturbation engine49  isAnchor    Boolean  @default(false) @map("is_anchor") // fixed longitudinal subset (≤20% of a run)50  irtA        Float?   @map("irt_a") // 2PL discrimination (from last fit)51  irtB        Float?   @map("irt_b") // 2PL difficulty (logits, from last fit)52  status      String   @default("active") // active | flagged_for_retirement | retired53  createdAt   DateTime @default(now()) @map("created_at")5455  responses ModelResponse[]56  duels     PairwiseDuel[]5758  @@index([domain, status])59  @@map("eval_items")60}6162// One row per batch (kind=eval_batch|duel_batch) or per index fit (kind=index_fit).63// Historical runs are immutable; every displayed score traces to one of these.64model ScoreRun {65  id             String    @id @default(cuid())66  indexVersion   String    @map("index_version") // INDEX_VERSION from packages/scoring67  kind           String // eval_batch | duel_batch | index_fit | demo_seed68  status         String    @default("pending") // pending | running | complete | degraded | failed69  itemSetHash    String    @map("item_set_hash") // sha256 over ordered item ids + perturb seeds70  modelSet       Json      @map("model_set") // array of model slugs in the run71  fitDiagnostics Json?     @map("fit_diagnostics") // convergence, judge agreement, bias rates…72  notes          String?73  createdAt      DateTime  @default(now()) @map("created_at")74  completedAt    DateTime? @map("completed_at")7576  responses ModelResponse[]77  duels     PairwiseDuel[]78  scores    Score[]7980  @@index([kind, status, createdAt])81  @@map("score_runs")82}8384// Full audit trail: a ranking without stored raw responses is invalid.85model ModelResponse {86  id              String   @id @default(cuid())87  runId           String   @map("run_id")88  modelId         String   @map("model_id")89  itemId          String   @map("item_id")90  sampleIndex     Int      @default(0) @map("sample_index") // >0 for consistency k-samples91  requestParams   Json     @map("request_params") // exact temperature, max_tokens, seed…92  rawResponse     Json     @map("raw_response")93  answerExtracted String?  @map("answer_extracted")94  correct         Boolean?95  confidence      Float? // model-reported confidence in [0,1] → calibration96  tokensIn        Int?     @map("tokens_in")97  tokensOut       Int?     @map("tokens_out")98  latencyMs       Int?     @map("latency_ms")99  costUsd         Float?   @map("cost_usd")100  error           String?101  createdAt       DateTime @default(now()) @map("created_at")102103  run   ScoreRun @relation(fields: [runId], references: [id])104  model Model    @relation(fields: [modelId], references: [id])105  item  EvalItem @relation(fields: [itemId], references: [id])106107  @@unique([runId, modelId, itemId, sampleIndex])108  @@index([modelId, createdAt])109  @@map("model_responses")110}111112// LLM-judged pairwise duels feeding the Bradley-Terry layer.113model PairwiseDuel {114  id              String   @id @default(cuid())115  runId           String   @map("run_id")116  domain          String117  itemId          String   @map("item_id")118  modelAId        String   @map("model_a_id")119  modelBId        String   @map("model_b_id")120  judgeSlug       String   @map("judge_slug") // never a model judging its own duel121  positionSwapped Boolean  @default(false) @map("position_swapped")122  winner          String // "a" | "b" | "tie"123  rawJudgment     Json     @map("raw_judgment")124  createdAt       DateTime @default(now()) @map("created_at")125126  run    ScoreRun @relation(fields: [runId], references: [id])127  item   EvalItem @relation(fields: [itemId], references: [id])128  modelA Model    @relation("duel_model_a", fields: [modelAId], references: [id])129  modelB Model    @relation("duel_model_b", fields: [modelBId], references: [id])130131  @@index([runId, domain])132  @@map("pairwise_duels")133}134135// Final scores. domain="global" for the Global Index; otherwise one of136// the domains in packages/scoring/src/domains.ts. Always with CI bounds.137model Score {138  id         String  @id @default(cuid())139  runId      String  @map("run_id")140  modelId    String  @map("model_id")141  domain     String // "global" | code | math | reasoning | …142  score      Float // 0–1000143  scoreLow   Float   @map("score_low")144  scoreHigh  Float   @map("score_high")145  // { accuracy_irt, consistency, calibration, contamination_delta, latency_p50, cost_per_1k_items }146  subMetrics Json?   @map("sub_metrics")147148  run   ScoreRun @relation(fields: [runId], references: [id])149  model Model    @relation(fields: [modelId], references: [id])150151  @@unique([runId, modelId, domain])152  @@index([domain])153  @@map("scores")154}155