spb/llmindex Public

1/**2 * llmindex.io — BullMQ workers: eval batches, duel scheduling, IRT refit triggers3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { Worker, type Job } from 'bullmq';8import IORedis from 'ioredis';9import { z } from 'zod';10import { redisUrl } from './env';11import { runEvalBatch } from './eval-runner';12import { runDuelBatch } from './duel-runner';13import { runRefit } from './refit';14import { DOMAINS, DUEL_DOMAINS } from '@llmindex/scoring';1516export const QUEUES = {17  eval: 'llmindex-eval',18  duel: 'llmindex-duel',19  refit: 'llmindex-refit',20} as const;2122const evalJob = z.object({23  modelSlug: z.string().min(1),24  domain: z.enum(DOMAINS),25  n: z.number().int().min(1).max(5000),26  kSamples: z.number().int().min(1).max(10).optional(),27  seed: z.string().optional(),28});2930const duelJob = z.object({31  domain: z.enum(DUEL_DOMAINS as [string, ...string[]]),32  pairs: z.number().int().min(1).max(5000),33  seed: z.string().optional(),34});3536function connection() {37  return new IORedis(redisUrl(), { maxRetriesPerRequest: null });38}3940function main(): void {41  const opts = { connection: connection(), concurrency: 1 };4243  new Worker(44    QUEUES.eval,45    async (job: Job) => {46      const params = evalJob.parse(job.data);47      return runEvalBatch({ ...params, domain: params.domain });48    },49    opts,50  );5152  new Worker(53    QUEUES.duel,54    async (job: Job) => {55      const params = duelJob.parse(job.data);56      return runDuelBatch({57        domain: params.domain as 'writing' | 'safety_refusal_quality',58        pairs: params.pairs,59        seed: params.seed,60      });61    },62    opts,63  );6465  new Worker(QUEUES.refit, async () => runRefit(), opts);6667  console.log(68    `[worker] listening on queues ${Object.values(QUEUES).join(', ')} via ${redisUrl().replace(/\/\/.*@/, '//***@')}`,69  );70}7172main();73