"""Deterministic sanity checks. Each check returns `Anomaly` records — flags with evidence, never deletions or silent fixes. Severity: `critical` (value is physically impossible or contradicts itself), `warning` (implausible, needs a look), `info` (worth knowing, e.g. a duplicate candidate). `dedupe_key` keeps one open row per (entity, check).""" from __future__ import annotations from dataclasses import dataclass, field from datetime import UTC, date, datetime from typing import Any from aiatlas.ontology.benchmarks import metric_bounds MAX_PARAMS = 10e12 # 10T MAX_CONTEXT = 100_000_000 # 100M tokens MAX_OUTPUT = 100_000_000 MAX_PRICE_PER_MTOK = 5_000.0 # USD per 1M tokens MAX_MEMORY_GB = 100_000 MIN_RELEASE = date(2010, 1, 1) @dataclass class Anomaly: check: str severity: str message: str entity_id: str | None = None value: Any = None detail: dict[str, Any] = field(default_factory=dict) @property def dedupe_key(self) -> str: return f"{self.check}:{self.entity_id or self.detail.get('key', '')}" def _num(v: Any) -> float | None: if isinstance(v, bool): return None if isinstance(v, (int, float)): return float(v) if isinstance(v, str): try: return float(v.replace(",", "")) except ValueError: return None return None def _date(v: Any) -> date | None: if isinstance(v, datetime): return v.date() if isinstance(v, date): return v if isinstance(v, str) and len(v) >= 4 and v[:4].isdigit(): try: parts = v[:10].split("-") y = int(parts[0]); m = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 1; d = int(parts[2]) if len(parts) > 2 and parts[2][:2].isdigit() else 1 return date(y, max(1, min(12, m)), max(1, min(28, d))) except ValueError: return None return None def check_model(entity_id: str, name: str, attrs: dict[str, Any], *, today: date | None = None) -> list[Anomaly]: today = today or datetime.now(UTC).date() out: list[Anomaly] = [] params = _num(attrs.get("parameter_count")) active = _num(attrs.get("active_parameter_count")) ctx = _num(attrs.get("context_length")) max_out = _num(attrs.get("max_output_tokens")) rel = _date(attrs.get("release_date")) dep = _date(attrs.get("deprecation_date")) ret = _date(attrs.get("retirement_date")) cutoff = _date(attrs.get("knowledge_cutoff")) if params is not None and params > MAX_PARAMS: out.append(Anomaly("params_too_large", "critical", f"{name}: parameter_count {params:.3g} exceeds {MAX_PARAMS:.0e}", entity_id, params)) if params is not None and params < 1e5: out.append(Anomaly("params_too_small", "warning", f"{name}: parameter_count {params:.3g} below 100K", entity_id, params)) if params is not None and active is not None and active > params: out.append(Anomaly("active_gt_total", "critical", f"{name}: active parameters {active:.3g} > total {params:.3g}", entity_id, active, {"parameter_count": params, "active_parameter_count": active})) if ctx is not None and ctx > MAX_CONTEXT: out.append(Anomaly("context_too_large", "critical", f"{name}: context_length {ctx:.0f} exceeds 100M tokens", entity_id, ctx)) if ctx is not None and ctx < 256: out.append(Anomaly("context_too_small", "warning", f"{name}: context_length {ctx:.0f} below 256 tokens", entity_id, ctx)) if max_out is not None and ctx is not None and max_out > ctx: out.append(Anomaly("max_output_gt_context", "warning", f"{name}: max_output_tokens {max_out:.0f} > context_length {ctx:.0f}", entity_id, max_out, {"context_length": ctx})) if rel and rel > today: out.append(Anomaly("release_in_future", "critical", f"{name}: release_date {rel.isoformat()} is in the future", entity_id, rel.isoformat())) if rel and rel < MIN_RELEASE: out.append(Anomaly("release_too_old", "warning", f"{name}: release_date {rel.isoformat()} before 2010", entity_id, rel.isoformat())) if rel and dep and dep < rel: out.append(Anomaly("deprecated_before_release", "critical", f"{name}: deprecation_date {dep} before release_date {rel}", entity_id, dep.isoformat(), {"release_date": rel.isoformat()})) if rel and ret and ret < rel: out.append(Anomaly("retired_before_release", "critical", f"{name}: retirement_date {ret} before release_date {rel}", entity_id, ret.isoformat(), {"release_date": rel.isoformat()})) if dep and ret and ret < dep: out.append(Anomaly("retired_before_deprecated", "warning", f"{name}: retirement_date {ret} before deprecation_date {dep}", entity_id, ret.isoformat())) if cutoff and rel and cutoff > rel: out.append(Anomaly("cutoff_after_release", "warning", f"{name}: knowledge_cutoff {cutoff} after release_date {rel}", entity_id, cutoff.isoformat())) status = str(attrs.get("status") or "").lower() if status in ("deprecated", "retired") and rel and rel > today: out.append(Anomaly("deprecated_future_release", "critical", f"{name}: status {status} but release in the future", entity_id, status)) if attrs.get("openness") in ("open-weights", "open-source") and not any(attrs.get(k) for k in ("hf_repo", "repository_url", "model_card_url", "weights_url")): out.append(Anomaly("open_without_weights_url", "info", f"{name}: labelled {attrs.get('openness')} but no weights location recorded", entity_id, attrs.get("openness"))) return out def check_price(row: dict[str, Any]) -> list[Anomaly]: out: list[Anomaly] = [] key = f"{row.get('model_id')}:{row.get('provider_id')}:{row.get('provider_model_id') or ''}" label = f"{row.get('model_name') or row.get('model_id')} @ {row.get('provider_name') or row.get('provider_id')}" inp, outp = _num(row.get("input_per_mtok")), _num(row.get("output_per_mtok")) for k, v in (("input_per_mtok", inp), ("output_per_mtok", outp), ("cached_input_per_mtok", _num(row.get("cached_input_per_mtok"))), ("batch_input_per_mtok", _num(row.get("batch_input_per_mtok"))), ("batch_output_per_mtok", _num(row.get("batch_output_per_mtok")))): if v is not None and v < 0: out.append(Anomaly("negative_price", "critical", f"{label}: {k} is negative ({v})", row.get("model_id"), v, {"key": key, "field": k, "price_id": row.get("id")})) if v is not None and v > MAX_PRICE_PER_MTOK: out.append(Anomaly("price_too_high", "warning", f"{label}: {k} = ${v:g} per 1M tokens", row.get("model_id"), v, {"key": key, "field": k, "price_id": row.get("id")})) if inp is not None and outp is not None and outp == 0 and inp > 0 and not (row.get("features") or {}).get("free"): out.append(Anomaly("zero_output_price", "warning", f"{label}: output price is 0 while input is ${inp:g}", row.get("model_id"), 0, {"key": key, "price_id": row.get("id")})) if inp is not None and outp is not None and inp > 0 and outp > 0 and inp > outp * 4: out.append(Anomaly("input_gt_output_price", "info", f"{label}: input ${inp:g} is more than 4× output ${outp:g}", row.get("model_id"), inp, {"key": key, "price_id": row.get("id")})) cached = _num(row.get("cached_input_per_mtok")) if cached is not None and inp is not None and cached > inp: out.append(Anomaly("cached_gt_input_price", "warning", f"{label}: cached input ${cached:g} > input ${inp:g}", row.get("model_id"), cached, {"key": key, "price_id": row.get("id")})) return out def check_price_movement(old: dict[str, Any], new: dict[str, Any]) -> list[Anomaly]: out: list[Anomaly] = [] for k in ("input_per_mtok", "output_per_mtok"): a, b = _num(old.get(k)), _num(new.get(k)) if a and b and a > 0 and b > 0 and (b / a > 100 or a / b > 100): out.append(Anomaly("price_jump_100x", "critical", f"{k} moved {a:g} → {b:g} (>100×)", new.get("model_id"), b, {"key": f"{new.get('model_id')}:{new.get('provider_id')}:{k}", "old": a, "new": b})) return out def check_result(row: dict[str, Any]) -> list[Anomaly]: out: list[Anomaly] = [] score = _num(row.get("score")) lo, hi = metric_bounds(row.get("metric"), row.get("unit")) label = f"{row.get('model_name') or row.get('model_id')} on {row.get('benchmark_name') or row.get('benchmark_id')}" if score is not None and hi is not None and score > hi + 1e-9: out.append(Anomaly("score_above_max", "critical", f"{label}: {score:g} > metric maximum {hi:g}", row.get("model_id"), score, {"key": row.get("id"), "result_id": row.get("id"), "metric": row.get("metric")})) if score is not None and lo is not None and score < lo - 1e-9: out.append(Anomaly("score_below_min", "critical", f"{label}: {score:g} < metric minimum {lo:g}", row.get("model_id"), score, {"key": row.get("id"), "result_id": row.get("id"), "metric": row.get("metric")})) ev = _date(row.get("evaluated_at")) rel = _date(row.get("model_release_date")) if ev and rel and ev < rel and (rel - ev).days > 45: out.append(Anomaly("evaluated_before_release", "warning", f"{label}: evaluated {ev} before model release {rel}", row.get("model_id"), ev.isoformat(), {"key": row.get("id"), "result_id": row.get("id")})) return out def check_hardware(entity_id: str, name: str, attrs: dict[str, Any]) -> list[Anomaly]: out: list[Anomaly] = [] mem = attrs.get("memory_gb") mems = mem if isinstance(mem, list) else [mem] for m in mems: v = _num(m) if v is not None and (v <= 0 or v > MAX_MEMORY_GB): out.append(Anomaly("memory_implausible", "critical", f"{name}: memory_gb {v:g} implausible", entity_id, v)) bw = _num(attrs.get("memory_bandwidth_gbs")) if bw is not None and (bw <= 0 or bw > 100_000): out.append(Anomaly("bandwidth_implausible", "warning", f"{name}: memory_bandwidth_gbs {bw:g} implausible", entity_id, bw)) tdp = _num(attrs.get("tdp_watts")) if tdp is not None and (tdp <= 0 or tdp > 200_000): out.append(Anomaly("tdp_implausible", "warning", f"{name}: tdp_watts {tdp:g} implausible", entity_id, tdp)) return out __all__ = ["Anomaly", "check_hardware", "check_model", "check_price", "check_price_movement", "check_result"]