spb/polyllm

1import { describe, it, expect } from "vitest";2import type { PolyModel } from "@/lib/ai/core/types";3import { buildBadgeContext, deriveBadges, lifecycleStatus, isCodingModel, isOpenWeightsModel, speedTier, quantile, isNewModel } from "@/lib/models/badges";4import { presetCompatibility, unsupportedSettings, pruneSettingsForModel, groupsFor } from "@/lib/models/params";56const NOW = Date.parse("2026-09-11T12:00:00Z");7const daysAgo = (d: number) => new Date(NOW - d * 86_400_000).toISOString();8const caps = (over: Partial<PolyModel["capabilities"]> = {}): PolyModel["capabilities"] => ({ text: true, vision: false, audioInput: false, audioOutput: false, imageGeneration: false, video: false, reasoning: false, tools: false, structuredOutput: false, streaming: true, files: false, webSearch: false, ...over });910function m(id: string, over: Partial<PolyModel> = {}): PolyModel {11  return { key: `openai/${id}`, id, provider: "openai", displayName: id, capabilities: caps(), parameters: {}, status: "active", pricing: null, metadata: { firstSeenAt: daysAgo(120) }, ...over };12}1314const ALL: PolyModel[] = [15  m("gpt-5.5", { capabilities: caps({ reasoning: true, vision: true, tools: true }), limits: { contextTokens: 1_050_000 }, pricing: { inputPerMillion: 5, outputPerMillion: 30 } }),16  m("gpt-5.4-mini", { capabilities: caps({ reasoning: true, vision: true }), limits: { contextTokens: 400_000 }, pricing: { inputPerMillion: 0.4, outputPerMillion: 1.6 } }),17  m("gpt-5.4-nano", { pricing: { inputPerMillion: 0.1, outputPerMillion: 0.4 }, limits: { contextTokens: 128_000 } }),18  m("gpt-5.5-pro", { pricing: { inputPerMillion: 30, outputPerMillion: 120 }, limits: { contextTokens: 400_000 } }),19  m("gpt-5.3-codex", { pricing: { inputPerMillion: 2, outputPerMillion: 8 }, limits: { contextTokens: 400_000 }, metadata: { firstSeenAt: daysAgo(3) } }),20  m("brand-new", { metadata: { firstSeenAt: daysAgo(1) } }),21];2223describe("badges", () => {24  const ctx = buildBadgeContext(ALL, NOW);25  it("computes quantiles", () => {26    expect(quantile([1, 2, 3, 4], 0.5)).toBe(2.5);27    expect(quantile([1, 2, 3, 4, 5], 0.25)).toBe(2);28  });29  it("derives CHEAP from real pricing quantiles, not fixed numbers", () => {30    expect(ctx.cheapThreshold).toBeGreaterThan(0);31    expect(deriveBadges(ALL[2], ctx).map((b) => b.kind)).toContain("cheap");32    expect(deriveBadges(ALL[3], ctx).map((b) => b.kind)).not.toContain("cheap");33  });34  it("derives capability and size badges", () => {35    const k = deriveBadges(ALL[0], ctx).map((b) => b.kind);36    expect(k).toEqual(expect.arrayContaining(["reasoning", "vision", "long-context"]));37    expect(k).not.toContain("fast");38    expect(deriveBadges(ALL[1], ctx).map((b) => b.kind)).toContain("fast");39    expect(isCodingModel(ALL[4])).toBe(true);40    expect(deriveBadges(ALL[4], ctx).map((b) => b.kind)).toEqual(expect.arrayContaining(["new", "coding"]));41  });42  it("treats a fresh registry as not-new (everything would be new otherwise)", () => {43    const fresh = ALL.map((x) => ({ ...x, metadata: { firstSeenAt: daysAgo(1) } }));44    const c = buildBadgeContext(fresh, NOW);45    expect(c.newViaFirstSeen).toBe(false);46    expect(isNewModel(fresh[0], c)).toBe(false);47    // provider-reported release dates still count48    expect(isNewModel({ ...fresh[0], metadata: { createdAt: daysAgo(2) } }, c)).toBe(true);49  });50  it("speed tiers and open weights heuristics", () => {51    expect(speedTier(ALL[2], ctx)).toBe("fast");52    expect(speedTier(ALL[3], ctx)).toBe("frontier");53    expect(isOpenWeightsModel(m("meta-llama/llama-3.1-8b-instruct"))).toBe(true);54    expect(isOpenWeightsModel(ALL[0])).toBe(false);55    expect(isOpenWeightsModel(m("x", { metadata: { openWeights: true } }))).toBe(true);56  });57  it("lifecycle: retiring > deprecated > unavailable > preview > new > active", () => {58    expect(lifecycleStatus(m("a", { metadata: { shutdownDate: daysAgo(-30) } }), { now: NOW }).status).toBe("retiring");59    expect(lifecycleStatus(m("a", { metadata: { shutdownDate: daysAgo(-300) } }), { now: NOW }).status).toBe("active");60    expect(lifecycleStatus(m("a", { metadata: { shutdownDate: daysAgo(5) } }), { now: NOW }).status).toBe("deprecated");61    expect(lifecycleStatus(m("a", { status: "deprecated" }), { now: NOW, connected: false }).status).toBe("deprecated");62    expect(lifecycleStatus(m("a"), { now: NOW, connected: false }).status).toBe("unavailable");63    expect(lifecycleStatus(m("a", { status: "preview" }), { now: NOW, connected: true }).status).toBe("preview");64    expect(lifecycleStatus(ALL[5], { ctx, connected: true }).status).toBe("new");65    expect(lifecycleStatus(ALL[0], { ctx, connected: true }).status).toBe("active");66    expect(lifecycleStatus(m("a", { status: "unknown" }), { now: NOW }).label).toBe("Unverified");67  });68});6970describe("parameter sheet", () => {71  const model: PolyModel = m("gpt-5.5", {72    capabilities: caps({ reasoning: true, tools: true, structuredOutput: true, webSearch: true }),73    parameters: { temperature: true, topP: true, maxTokens: true, reasoningEffort: true, reasoningEffortLevels: ["low", "medium", "high"], verbosity: true, stop: false, seed: false },74    limits: { maxOutputTokens: 128_000 },75  });76  it("lists only supported groups", () => {77    const groups = groupsFor(model).map((g) => g.group.key);78    expect(groups).toEqual(["generation", "reasoning", "output", "tools"]);79    expect(groupsFor(model).find((g) => g.group.key === "generation")!.defs.map((d) => d.key)).toEqual(["temperature", "topP", "maxTokens"]);80  });81  it("flags unsupported settings and bad effort levels", () => {82    const u = unsupportedSettings({ temperature: 0.2, seed: 42, stop: ["x"], reasoningEffort: "xhigh", includeReasoning: true, tools: [] }, model);83    expect(u.map((x) => x.key).sort()).toEqual(["reasoningEffort", "seed", "stop"]);84    expect(u.find((x) => x.key === "reasoningEffort")!.reason).toMatch(/low \/ medium \/ high/);85  });86  it("computes preset compatibility and prunes", () => {87    const ok = presetCompatibility({ parameters: { temperature: 0.3, reasoningEffort: "high" }, tools: { builtin: ["calculator"] } }, model);88    expect(ok.compatible).toBe(true);89    expect(ok.total).toBe(3);90    const bad = presetCompatibility({ parameters: { temperature: 0.3, seed: 1 }, tools: {} }, model);91    expect(bad.compatible).toBe(false);92    expect(bad.unsupported[0].key).toBe("seed");93    expect(pruneSettingsForModel({ temperature: 0.3, seed: 1, topK: 4 }, model)).toEqual({ temperature: 0.3 });94  });95});96