HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Deterministic sanity checks. Each check returns `Anomaly` records — flags with evidence, never deletions or silent fixes.23Severity: `critical` (value is physically impossible or contradicts itself), `warning` (implausible, needs a look), `info` (worth4knowing, e.g. a duplicate candidate). `dedupe_key` keeps one open row per (entity, check)."""5from __future__ import annotations67from dataclasses import dataclass, field8from datetime import UTC, date, datetime9from typing import Any1011from aiatlas.ontology.benchmarks import metric_bounds1213MAX_PARAMS = 10e12 # 10T14MAX_CONTEXT = 100_000_000 # 100M tokens15MAX_OUTPUT = 100_000_00016MAX_PRICE_PER_MTOK = 5_000.0 # USD per 1M tokens17MAX_MEMORY_GB = 100_00018MIN_RELEASE = date(2010, 1, 1)192021@dataclass22class Anomaly:23 check: str24 severity: str25 message: str26 entity_id: str | None = None27 value: Any = None28 detail: dict[str, Any] = field(default_factory=dict)2930 @property31 def dedupe_key(self) -> str:32 return f"{self.check}:{self.entity_id or self.detail.get('key', '')}"333435def _num(v: Any) -> float | None:36 if isinstance(v, bool):37 return None38 if isinstance(v, (int, float)):39 return float(v)40 if isinstance(v, str):41 try:42 return float(v.replace(",", ""))43 except ValueError:44 return None45 return None464748def _date(v: Any) -> date | None:49 if isinstance(v, datetime):50 return v.date()51 if isinstance(v, date):52 return v53 if isinstance(v, str) and len(v) >= 4 and v[:4].isdigit():54 try:55 parts = v[:10].split("-")56 y = int(parts[0]); m = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 1; d = int(parts[2]) if len(parts) > 2 and parts[2][:2].isdigit() else 157 return date(y, max(1, min(12, m)), max(1, min(28, d)))58 except ValueError:59 return None60 return None616263def check_model(entity_id: str, name: str, attrs: dict[str, Any], *, today: date | None = None) -> list[Anomaly]:64 today = today or datetime.now(UTC).date()65 out: list[Anomaly] = []66 params = _num(attrs.get("parameter_count"))67 active = _num(attrs.get("active_parameter_count"))68 ctx = _num(attrs.get("context_length"))69 max_out = _num(attrs.get("max_output_tokens"))70 rel = _date(attrs.get("release_date"))71 dep = _date(attrs.get("deprecation_date"))72 ret = _date(attrs.get("retirement_date"))73 cutoff = _date(attrs.get("knowledge_cutoff"))7475 if params is not None and params > MAX_PARAMS:76 out.append(Anomaly("params_too_large", "critical", f"{name}: parameter_count {params:.3g} exceeds {MAX_PARAMS:.0e}", entity_id, params))77 if params is not None and params < 1e5:78 out.append(Anomaly("params_too_small", "warning", f"{name}: parameter_count {params:.3g} below 100K", entity_id, params))79 if params is not None and active is not None and active > params:80 out.append(Anomaly("active_gt_total", "critical", f"{name}: active parameters {active:.3g} > total {params:.3g}", entity_id, active,81 {"parameter_count": params, "active_parameter_count": active}))82 if ctx is not None and ctx > MAX_CONTEXT:83 out.append(Anomaly("context_too_large", "critical", f"{name}: context_length {ctx:.0f} exceeds 100M tokens", entity_id, ctx))84 if ctx is not None and ctx < 256:85 out.append(Anomaly("context_too_small", "warning", f"{name}: context_length {ctx:.0f} below 256 tokens", entity_id, ctx))86 if max_out is not None and ctx is not None and max_out > ctx:87 out.append(Anomaly("max_output_gt_context", "warning", f"{name}: max_output_tokens {max_out:.0f} > context_length {ctx:.0f}", entity_id, max_out,88 {"context_length": ctx}))89 if rel and rel > today:90 out.append(Anomaly("release_in_future", "critical", f"{name}: release_date {rel.isoformat()} is in the future", entity_id, rel.isoformat()))91 if rel and rel < MIN_RELEASE:92 out.append(Anomaly("release_too_old", "warning", f"{name}: release_date {rel.isoformat()} before 2010", entity_id, rel.isoformat()))93 if rel and dep and dep < rel:94 out.append(Anomaly("deprecated_before_release", "critical", f"{name}: deprecation_date {dep} before release_date {rel}", entity_id, dep.isoformat(),95 {"release_date": rel.isoformat()}))96 if rel and ret and ret < rel:97 out.append(Anomaly("retired_before_release", "critical", f"{name}: retirement_date {ret} before release_date {rel}", entity_id, ret.isoformat(),98 {"release_date": rel.isoformat()}))99 if dep and ret and ret < dep:100 out.append(Anomaly("retired_before_deprecated", "warning", f"{name}: retirement_date {ret} before deprecation_date {dep}", entity_id, ret.isoformat()))101 if cutoff and rel and cutoff > rel:102 out.append(Anomaly("cutoff_after_release", "warning", f"{name}: knowledge_cutoff {cutoff} after release_date {rel}", entity_id, cutoff.isoformat()))103 status = str(attrs.get("status") or "").lower()104 if status in ("deprecated", "retired") and rel and rel > today:105 out.append(Anomaly("deprecated_future_release", "critical", f"{name}: status {status} but release in the future", entity_id, status))106 if attrs.get("openness") in ("open-weights", "open-source") and not any(attrs.get(k) for k in ("hf_repo", "repository_url", "model_card_url", "weights_url")):107 out.append(Anomaly("open_without_weights_url", "info", f"{name}: labelled {attrs.get('openness')} but no weights location recorded", entity_id, attrs.get("openness")))108 return out109110111def check_price(row: dict[str, Any]) -> list[Anomaly]:112 out: list[Anomaly] = []113 key = f"{row.get('model_id')}:{row.get('provider_id')}:{row.get('provider_model_id') or ''}"114 label = f"{row.get('model_name') or row.get('model_id')} @ {row.get('provider_name') or row.get('provider_id')}"115 inp, outp = _num(row.get("input_per_mtok")), _num(row.get("output_per_mtok"))116 for k, v in (("input_per_mtok", inp), ("output_per_mtok", outp), ("cached_input_per_mtok", _num(row.get("cached_input_per_mtok"))),117 ("batch_input_per_mtok", _num(row.get("batch_input_per_mtok"))), ("batch_output_per_mtok", _num(row.get("batch_output_per_mtok")))):118 if v is not None and v < 0:119 out.append(Anomaly("negative_price", "critical", f"{label}: {k} is negative ({v})", row.get("model_id"), v, {"key": key, "field": k, "price_id": row.get("id")}))120 if v is not None and v > MAX_PRICE_PER_MTOK:121 out.append(Anomaly("price_too_high", "warning", f"{label}: {k} = ${v:g} per 1M tokens", row.get("model_id"), v, {"key": key, "field": k, "price_id": row.get("id")}))122 if inp is not None and outp is not None and outp == 0 and inp > 0 and not (row.get("features") or {}).get("free"):123 out.append(Anomaly("zero_output_price", "warning", f"{label}: output price is 0 while input is ${inp:g}", row.get("model_id"), 0, {"key": key, "price_id": row.get("id")}))124 if inp is not None and outp is not None and inp > 0 and outp > 0 and inp > outp * 4:125 out.append(Anomaly("input_gt_output_price", "info", f"{label}: input ${inp:g} is more than 4× output ${outp:g}", row.get("model_id"), inp, {"key": key, "price_id": row.get("id")}))126 cached = _num(row.get("cached_input_per_mtok"))127 if cached is not None and inp is not None and cached > inp:128 out.append(Anomaly("cached_gt_input_price", "warning", f"{label}: cached input ${cached:g} > input ${inp:g}", row.get("model_id"), cached, {"key": key, "price_id": row.get("id")}))129 return out130131132def check_price_movement(old: dict[str, Any], new: dict[str, Any]) -> list[Anomaly]:133 out: list[Anomaly] = []134 for k in ("input_per_mtok", "output_per_mtok"):135 a, b = _num(old.get(k)), _num(new.get(k))136 if a and b and a > 0 and b > 0 and (b / a > 100 or a / b > 100):137 out.append(Anomaly("price_jump_100x", "critical", f"{k} moved {a:g} → {b:g} (>100×)", new.get("model_id"), b,138 {"key": f"{new.get('model_id')}:{new.get('provider_id')}:{k}", "old": a, "new": b}))139 return out140141142def check_result(row: dict[str, Any]) -> list[Anomaly]:143 out: list[Anomaly] = []144 score = _num(row.get("score"))145 lo, hi = metric_bounds(row.get("metric"), row.get("unit"))146 label = f"{row.get('model_name') or row.get('model_id')} on {row.get('benchmark_name') or row.get('benchmark_id')}"147 if score is not None and hi is not None and score > hi + 1e-9:148 out.append(Anomaly("score_above_max", "critical", f"{label}: {score:g} > metric maximum {hi:g}", row.get("model_id"), score,149 {"key": row.get("id"), "result_id": row.get("id"), "metric": row.get("metric")}))150 if score is not None and lo is not None and score < lo - 1e-9:151 out.append(Anomaly("score_below_min", "critical", f"{label}: {score:g} < metric minimum {lo:g}", row.get("model_id"), score,152 {"key": row.get("id"), "result_id": row.get("id"), "metric": row.get("metric")}))153 ev = _date(row.get("evaluated_at"))154 rel = _date(row.get("model_release_date"))155 if ev and rel and ev < rel and (rel - ev).days > 45:156 out.append(Anomaly("evaluated_before_release", "warning", f"{label}: evaluated {ev} before model release {rel}", row.get("model_id"), ev.isoformat(),157 {"key": row.get("id"), "result_id": row.get("id")}))158 return out159160161def check_hardware(entity_id: str, name: str, attrs: dict[str, Any]) -> list[Anomaly]:162 out: list[Anomaly] = []163 mem = attrs.get("memory_gb")164 mems = mem if isinstance(mem, list) else [mem]165 for m in mems:166 v = _num(m)167 if v is not None and (v <= 0 or v > MAX_MEMORY_GB):168 out.append(Anomaly("memory_implausible", "critical", f"{name}: memory_gb {v:g} implausible", entity_id, v))169 bw = _num(attrs.get("memory_bandwidth_gbs"))170 if bw is not None and (bw <= 0 or bw > 100_000):171 out.append(Anomaly("bandwidth_implausible", "warning", f"{name}: memory_bandwidth_gbs {bw:g} implausible", entity_id, bw))172 tdp = _num(attrs.get("tdp_watts"))173 if tdp is not None and (tdp <= 0 or tdp > 200_000):174 out.append(Anomaly("tdp_implausible", "warning", f"{name}: tdp_watts {tdp:g} implausible", entity_id, tdp))175 return out176177178__all__ = ["Anomaly", "check_hardware", "check_model", "check_price", "check_price_movement", "check_result"]179