"""Live counters and aggregate statistics — always computed from the database, never hardcoded. API 1.1 semantics: `entities.model` counts CANONICAL model releases (entity_type 'model', not merged) — artifacts (checkpoints, quantisations, conversions) are `entities.artifact`, folded evaluation variants are excluded through `merged_into`. `organizations_total` is the number shown by /companies (company + organization + lab + university). `definitions` says how every counter is counted.""" from __future__ import annotations from typing import Any from sqlalchemy.ext.asyncio import AsyncConnection from aiatlas.db import execute, fetch_all, fetch_one, jsonb, transaction from aiatlas.sdk.archive import archive_size ORG_TYPES = ("company", "organization", "lab", "university") DEFINITIONS: dict[str, str] = { "models": "Canonical model releases (artifacts, quantisations, conversions and folded evaluation variants excluded; merged duplicates excluded).", "artifacts": "Checkpoints, quantisations, conversions and packagings of a canonical model (entity_type 'artifact').", "model_families": "Model families (Llama 4, Qwen3.6, Claude…) grouping canonical releases.", "organizations_total": "Companies + organizations + labs + universities, merged duplicates excluded — the same universe as /companies.", "entities_total": "All live entities of every type (merged duplicates excluded).", "change_events": "All change events ever recorded, including the initial back-filled corpus.", "change_events_24h": "Events OBSERVED in the last 24 hours (legacy counter — includes back-filled history when a connector first runs).", "change_events_live_24h": "Events that OCCURRED in the last 24 hours, excluding back-fill and source-document changes — what actually happened today.", "change_events_7d": "Events observed in the last 7 days (legacy counter).", "benchmark_results": "Current benchmark result rows (one per model × benchmark × metric × configuration run).", "prices_current": "Live provider price offers (model × provider × provider model id).", "claims_current": "Current temporal claims (one per entity × property).", "relations": "Live relations in the knowledge graph.", "sources": "Enabled sources (websites, registries, leaderboards) crawled by AI Atlas connectors.", } async def live_counts(conn: AsyncConnection) -> dict[str, Any]: by_type = await fetch_all(conn, "select entity_type, count(*) as n from entities where merged_into is null group by 1") counts = {r["entity_type"]: int(r["n"]) for r in by_type} infra = await fetch_one(conn, """select (select count(*) from sources where enabled) as sources, (select count(*) from connectors) as connectors, (select count(*) from connectors where enabled) as connectors_enabled, (select count(*) from documents) as documents, (select count(*) from snapshots) as snapshots, (select count(*) from claims) as claims, (select count(*) from claims where status = 'current') as claims_current, (select count(*) from relations where valid_to is null) as relations, (select count(*) from change_events) as change_events, (select count(*) from change_events where observed_at > now() - interval '24 hours') as change_events_24h, (select count(*) from change_events where is_backfill = false and event_type <> 'DOCUMENT_CHANGED' and occurred_at > now() - interval '24 hours') as change_events_live_24h, (select count(*) from change_events where is_backfill = false and event_type <> 'DOCUMENT_CHANGED' and occurred_at > now() - interval '7 days') as change_events_live_7d, (select count(*) from change_events where observed_at > now() - interval '7 days') as change_events_7d, (select count(*) from benchmark_results where valid_to is null and is_current) as benchmark_results, (select count(*) from prices where valid_to is null) as prices_current, (select count(*) from prices) as prices_total, (select count(*) from jobs where status = 'queued') as jobs_queued, (select count(*) from jobs where status = 'dead') as jobs_dead, (select count(*) from review_queue where status = 'pending') as review_pending, (select count(*) from llm_jobs) as llm_jobs, (select coalesce(sum(input_tokens),0) + coalesce(sum(output_tokens),0) from llm_jobs) as llm_tokens, (select max(observed_at) from snapshots) as last_snapshot_at, (select max(observed_at) from change_events) as last_event_at, (select min(first_seen_at) from entities) as first_entity_at""") out = {"entities": counts, "entities_total": sum(counts.values()), **{k: (int(v) if isinstance(v, int) else v) for k, v in (infra or {}).items()}} out["organizations_total"] = sum(counts.get(t, 0) for t in ORG_TYPES) out["artifacts"] = counts.get("artifact", 0) out["model_families"] = counts.get("model_family", 0) out["definitions"] = DEFINITIONS return out async def compute_stats() -> dict[str, Any]: async with transaction() as conn: counts = await live_counts(conn) counts["archive"] = archive_size() await execute(conn, "insert into stats_snapshots (counts) values (cast(:c as jsonb))", c=jsonb(counts)) return counts async def history(conn: AsyncConnection, days: int = 90) -> list[dict[str, Any]]: return await fetch_all(conn, """select distinct on (date_trunc('day', computed_at)) date_trunc('day', computed_at) as day, counts from stats_snapshots where computed_at > now() - make_interval(days => :d) order by 1 desc, computed_at desc""", d=days) __all__ = ["DEFINITIONS", "ORG_TYPES", "compute_stats", "history", "live_counts"]