/**
 * llmindex.io — live benchmark orchestrator: one model at a time, refit after each
 * Author:  Simon-Pierre Boucher
 * Contact: contact@spboucher.ai
 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved
 *
 * Runs graded eval batches model-by-model across all IRT domains, publishing
 * live progress to Redis and triggering an IRT refit after every completed
 * model — so the public leaderboard grows gradually while the run is online.
 * (The first real fit appears once ≥2 models are complete, replacing the demo.)
 */
import { prisma } from '@llmindex/db';
import { DUEL_DOMAINS, IRT_DOMAINS, VISION_DOMAINS, type Domain } from '@llmindex/scoring';
import { runEvalBatch } from './eval-runner';
import { runDuelBatch } from './duel-runner';
import { runRefit } from './refit';
import { writeProgress, type BenchmarkProgress } from './progress';

export interface BenchmarkOptions {
  /** Items per domain per model. */
  n?: number;
  /** Samples per item (consistency). */
  k?: number;
  /** Explicit slugs; defaults to the ranked subset. */
  models?: string[];
  /**
   * Models evaluated concurrently (lanes). 4 lanes × 4 in-batch calls =
   * ≤16 concurrent OpenRouter requests — well inside paid-tier limits, and
   * every call retries 429s with exponential backoff regardless.
   */
  parallel?: number;
}

export async function runBenchmark(opts: BenchmarkOptions = {}): Promise<void> {
  const n = opts.n ?? 30;
  const k = opts.k ?? 2;
  const ranked = opts.models?.length
    ? await prisma.model.findMany({ where: { slug: { in: opts.models } }, orderBy: { slug: 'asc' } })
    : await prisma.model.findMany({ where: { ranked: true }, orderBy: { slug: 'asc' } });
  if (ranked.length === 0) throw new Error('No models to benchmark (run db:seed first)');

  const slugs = ranked.map((m) => m.slug);
  const seedBase = `bench:${Date.now()}`;
  const state: BenchmarkProgress = {
    active: true,
    started_at: new Date().toISOString(),
    models: slugs,
    completed: [],
    failed: [],
    current: null,
    last_refit_run_id: null,
  };
  await writeProgress(state);
  console.log(`[bench] starting: ${slugs.length} models × ${IRT_DOMAINS.length} domains (n=${n}, k=${k})`);

  const visionBySlug = new Map(ranked.map((m) => [m.slug, m.vision]));
  const lanes = Math.max(1, Math.min(12, opts.parallel ?? 6));

  // Global (model × domain) task pool, model-major so models finish (and hit
  // the leaderboard) roughly one at a time even with parallel lanes.
  interface Task {
    slug: string;
    domain: Domain;
    di: number;
    domainsTotal: number;
  }
  const tasks: Task[] = [];
  const remainingByModel = new Map<string, number>();
  const modelHadError = new Set<string>();
  for (const slug of slugs) {
    const domains: readonly Domain[] = visionBySlug.get(slug)
      ? IRT_DOMAINS
      : IRT_DOMAINS.filter((d) => !VISION_DOMAINS.includes(d));
    remainingByModel.set(slug, domains.length);
    domains.forEach((domain, di) =>
      tasks.push({ slug, domain, di: di + 1, domainsTotal: domains.length }),
    );
  }

  const laneCurrent: (typeof state.current)[] = Array.from({ length: lanes }, () => null);
  let refitChain: Promise<void> = Promise.resolve();
  const syncProgress = async (): Promise<void> => {
    const currents = laneCurrent.filter((c): c is NonNullable<typeof c> => c !== null);
    state.currents = currents;
    state.current = currents[0] ?? null;
    await writeProgress(state);
  };

  let cursor = 0;
  console.log(`[bench] parallel lanes=${lanes} → ≤${lanes * 4} concurrent OpenRouter calls`);
  await Promise.all(
    Array.from({ length: lanes }, (_, laneId) =>
      (async () => {
        for (;;) {
          const task = tasks[cursor++];
          if (!task) return;
          laneCurrent[laneId] = {
            model: task.slug,
            domain: task.domain,
            domain_index: task.di,
            domains_total: task.domainsTotal,
            calls_done: 0,
            calls_total: n * k,
          };
          await syncProgress();
          try {
            const result = await runEvalBatch({
              modelSlug: task.slug,
              domain: task.domain,
              n,
              kSamples: k,
              seed: `${seedBase}:${task.slug}:${task.domain}`,
              onProgress: async (done, total) => {
                const cur = laneCurrent[laneId];
                if (cur && cur.model === task.slug && cur.domain === task.domain) {
                  cur.calls_done = done;
                  cur.calls_total = total;
                  await syncProgress();
                }
              },
            });
            if (result.status === 'degraded') {
              console.warn(`[bench] ${task.slug}/${task.domain} degraded (excluded from fits)`);
            }
          } catch (err) {
            console.warn(`[bench] ${task.slug}/${task.domain} failed: ${String(err).slice(0, 300)}`);
            modelHadError.add(task.slug);
          }
          laneCurrent[laneId] = null;
          const left = (remainingByModel.get(task.slug) ?? 1) - 1;
          remainingByModel.set(task.slug, left);
          if (left === 0) {
            (modelHadError.has(task.slug) ? state.failed : state.completed).push(task.slug);
            // Refits are serialized (never concurrent) via a promise chain.
            refitChain = refitChain.then(async () => {
              try {
                const refit = await runRefit();
                if (refit) state.last_refit_run_id = refit.runId;
              } catch (err) {
                console.warn(`[bench] refit after ${task.slug} failed: ${String(err).slice(0, 300)}`);
              }
            });
            console.log(
              `[bench] ${task.slug} done (${state.completed.length + state.failed.length}/${slugs.length})`,
            );
          }
          await syncProgress();
        }
      })(),
    ),
  );
  await refitChain;

  // Judged duel domains (writing, safety, svg_design): pairwise + 3-judge
  // panel + Bradley-Terry. Requires JUDGE_MODELS in the environment.
  if (process.env.JUDGE_MODELS) {
    const pairsPerDomain = Math.min(400, Math.max(60, slugs.length * 6));
    for (const domain of DUEL_DOMAINS) {
      state.current = {
        model: `duels:${domain}`,
        domain,
        domain_index: 1,
        domains_total: 1,
        calls_done: 0,
        calls_total: pairsPerDomain,
      };
      await writeProgress(state);
      try {
        await runDuelBatch({
          domain: domain as 'writing' | 'safety_refusal_quality' | 'svg_design',
          pairs: pairsPerDomain,
          seed: `${seedBase}:duel:${domain}`,
          onProgress: async (done, total) => {
            if (state.current) {
              state.current.calls_done = done;
              state.current.calls_total = total;
              await writeProgress(state);
            }
          },
        });
      } catch (err) {
        console.warn(`[bench] duel domain ${domain} failed: ${String(err).slice(0, 300)}`);
      }
    }
    state.current = null;
    try {
      const refit = await runRefit();
      if (refit) state.last_refit_run_id = refit.runId;
    } catch (err) {
      console.warn(`[bench] final refit failed: ${String(err).slice(0, 300)}`);
    }
  } else {
    console.warn('[bench] JUDGE_MODELS not set — skipping duel domains (writing/safety/svg)');
  }

  state.active = false;
  state.note = `benchmark complete: ${state.completed.length} ok, ${state.failed.length} failed`;
  await writeProgress(state);
  console.log(`[bench] ${state.note}`);
}