"""/benchmarks listing, /benchmarks/{slug}, /benchmarks/{slug}/results (leaderboard), /benchmarks/{slug}/history.""" from __future__ import annotations from typing import Any from fastapi import APIRouter, Query, Request from aiatlas.api.common import ( ENTITY_COLS, ENTITY_FROM, PAGINATION, RESULT_COLS, RESULT_FROM, Pagination, cached, entity_cols, entity_summary, page, resolve_entity, resolve_id, result_row, ) from aiatlas.api.detail import leaderboard from aiatlas.api.routers.entities import detail_for_type from aiatlas.db import connection, fetch_all, fetch_val router = APIRouter(prefix="/api/v1/benchmarks", tags=["benchmarks"]) @router.get("") @cached(300) async def list_benchmarks(request: Request, category: str | None = None) -> dict[str, Any]: where = "e.entity_type = 'benchmark' and e.merged_into is null" + (" and e.attributes->>'category' ilike :cat" if category else "") async with connection() as conn: rows = await fetch_all(conn, f""" select {ENTITY_COLS}, (select count(*) from benchmark_results r where r.benchmark_id = e.id and r.valid_to is null) as result_count, (select count(distinct r.model_id) from benchmark_results r where r.benchmark_id = e.id and r.valid_to is null) as model_count, top.score as top_score, {entity_cols("tm", "t_")} from {ENTITY_FROM} left join lateral (select r.model_id, r.score from benchmark_results r where r.benchmark_id = e.id and r.valid_to is null order by case when r.higher_is_better then -r.score else r.score end limit 1) top on true left join entities tm on tm.id = top.model_id left join entities tmo on tmo.id = tm.organization_id where {where} order by result_count desc, e.canonical_name""", cat=category) items = [] for r in rows: top_model = entity_summary(r, "t_") items.append({**(entity_summary(r) or {}), "result_count": int(r["result_count"] or 0), "model_count": int(r["model_count"] or 0), "top": {"model": top_model, "score": r["top_score"]} if top_model else None}) return {"items": items} @router.get("/{slug}") @cached(300) async def get_benchmark(request: Request, slug: str) -> dict[str, Any]: return await detail_for_type(slug, ("benchmark",)) @router.get("/{slug}/results") @cached(300) async def benchmark_results(request: Request, slug: str, p: Pagination = PAGINATION, config: str | None = Query(None, max_length=200), history: int = Query(0, ge=0, le=1)) -> dict[str, Any]: async with connection() as conn: bench = await resolve_entity(conn, slug, ("benchmark",)) items = await leaderboard(conn, bench["id"], limit=p.limit, offset=p.offset, config=config, history=bool(history)) where = "r.benchmark_id = :id" + ("" if history else " and r.valid_to is null") + (" and r.config::text ilike :cfg" if config else "") total = await fetch_val(conn, f"select count(*) from benchmark_results r where {where}", id=bench["id"], cfg=f"%{config}%" if config else None) out = page(items, int(total or 0), p) out["benchmark"] = entity_summary(bench) return out @router.get("/{slug}/history") @cached(300) async def benchmark_history(request: Request, slug: str, model: str | None = None, limit: int = Query(1000, ge=1, le=5000)) -> dict[str, Any]: async with connection() as conn: bench = await resolve_entity(conn, slug, ("benchmark",)) where = ["r.benchmark_id = :id"] params: dict[str, Any] = {"id": bench["id"], "lim": limit} if model: where.append("r.model_id = :model") params["model"] = await resolve_id(conn, model) rows = await fetch_all(conn, f"select {RESULT_COLS} from {RESULT_FROM} where {' and '.join(where)} order by r.observed_at asc, r.id limit :lim", **params) return {"benchmark": entity_summary(bench), "items": [result_row(r) for r in rows]}