spb/polyllm

1import { describe, it, expect } from "vitest";2import { computeSavings, coversCapabilities, type ModelUsageAgg } from "@/lib/usage/savings";3import type { PolyModel, ModelCapabilities } from "@/lib/ai/core/types";45const caps = (over: Partial<ModelCapabilities> = {}): ModelCapabilities => ({ text: true, vision: true, audioInput: false, audioOutput: false, imageGeneration: false, video: false, reasoning: true, tools: true, structuredOutput: true, streaming: true, files: true, webSearch: false, ...over });67function model(key: string, input: number, output: number, extra: Partial<PolyModel> = {}): PolyModel {8  const [provider, id] = key.split("/") as [PolyModel["provider"], string];9  return { key, id, provider, displayName: id, family: "Claude", capabilities: caps(), parameters: {}, status: "active", pricing: { inputPerMillion: input, outputPerMillion: output, cachedInputPerMillion: input / 10 }, ...extra };10}1112const registry = new Map<string, PolyModel>(13  [14    model("anthropic/claude-opus-4", 15, 75, { displayName: "Claude Opus 4" }),15    model("anthropic/claude-sonnet-4", 3, 15, { displayName: "Claude Sonnet 4" }),16    model("anthropic/claude-haiku-4", 0.8, 4, { displayName: "Claude Haiku 4", capabilities: caps({ files: false }) }),17    model("anthropic/claude-old", 1, 5, { status: "deprecated" }),18    model("openai/gpt-x", 2, 8, { family: "GPT", capabilities: caps() }),19  ].map((m) => [m.key, m]),20);2122const opusRow: ModelUsageAgg = { modelKey: "anthropic/claude-opus-4", provider: "anthropic", requests: 40, inputTokens: 1_000_000, outputTokens: 200_000, cachedTokens: 0, reasoningTokens: 0, costUsd: 30 };2324describe("savings opportunities", () => {25  it("picks the closest cheaper sibling on the same provider, not blindly the cheapest", () => {26    const [opp] = computeSavings([opusRow], registry);27    expect(opp).toBeDefined();28    expect(opp.toModelKey).toBe("anthropic/claude-sonnet-4"); // Haiku is cheaper but Sonnet is the next tier and still saves > 40 %29    // Opus: 1M × $15 + 0.2M × $75 = $30 ; Sonnet: $3 + $3 = $630    expect(opp.currentCostUsd).toBeCloseTo(30, 4);31    expect(opp.alternativeCostUsd).toBeCloseTo(6, 4);32    expect(opp.savingsUsd).toBeCloseTo(24, 4);33    expect(opp.savingsPct).toBe(80);34    expect(opp.headline).toContain("Claude Opus 4");35    expect(opp.headline).toContain("Claude Sonnet 4");36    expect(opp.headline).toMatch(/\$24\.00/);37  });3839  it("re-prices cached input at the cached rate", () => {40    const row = { ...opusRow, cachedTokens: 500_000 };41    const [opp] = computeSavings([row], registry);42    // Opus: 0.5M × 15 + 0.5M × 1.5 + 0.2M × 75 = 7.5 + 0.75 + 15 = 23.2543    expect(opp.currentCostUsd).toBeCloseTo(23.25, 4);44  });4546  it("never proposes a sibling that lacks a capability the source has", () => {47    const only = new Map(registry);48    only.delete("anthropic/claude-sonnet-4"); // leaves Haiku (no files) as the only cheaper sibling49    expect(computeSavings([opusRow], only)).toEqual([]);50    expect(coversCapabilities(registry.get("anthropic/claude-haiku-4")!, registry.get("anthropic/claude-opus-4")!)).toBe(false);51    expect(coversCapabilities(registry.get("anthropic/claude-sonnet-4")!, registry.get("anthropic/claude-opus-4")!)).toBe(true);52  });5354  it("ignores deprecated siblings, other providers, unpriced sources and tiny amounts", () => {55    const sonnetRow: ModelUsageAgg = { ...opusRow, modelKey: "anthropic/claude-sonnet-4", costUsd: 6 };56    const noHaiku = new Map(registry);57    noHaiku.delete("anthropic/claude-haiku-4");58    expect(computeSavings([sonnetRow], noHaiku)).toEqual([]); // only deprecated claude-old and OpenAI remain59    const unpriced = new Map(registry);60    unpriced.set("anthropic/claude-opus-4", { ...registry.get("anthropic/claude-opus-4")!, pricing: null });61    expect(computeSavings([opusRow], unpriced)).toEqual([]);62    const tiny: ModelUsageAgg = { ...opusRow, inputTokens: 100, outputTokens: 10, costUsd: 0.002 };63    expect(computeSavings([tiny], registry)).toEqual([]);64  });6566  it("requires the sibling's context window to fit the average prompt", () => {67    const small = new Map(registry);68    small.set("anthropic/claude-sonnet-4", { ...registry.get("anthropic/claude-sonnet-4")!, limits: { contextTokens: 8_000 } });69    small.delete("anthropic/claude-haiku-4");70    const row = { ...opusRow, requests: 10 }; // 100K tokens per request71    expect(computeSavings([row], small)).toEqual([]);72  });7374  it("returns the top 3 sorted by savings", () => {75    const rows: ModelUsageAgg[] = ["a", "b", "c", "d"].map((s, i) => ({ ...opusRow, modelKey: `anthropic/claude-opus-4`, requests: 10, inputTokens: (i + 1) * 100_000, outputTokens: 0, costUsd: 1 }));76    const out = computeSavings(rows, registry, { top: 3 });77    expect(out).toHaveLength(3);78    expect(out[0].savingsUsd).toBeGreaterThanOrEqual(out[1].savingsUsd);79    expect(out[1].savingsUsd).toBeGreaterThanOrEqual(out[2].savingsUsd);80  });81});82