SPB Git
spboucher.ai

spb/ai-atlas

Public

ontology: official family orgs, effort-config helpers, chained/budget effort suffixes, hyphenated family versions, property-level normalisation, run keys split from task keys

- models: FAMILY_ORGS / official_orgs / is_official_org; effort_config() and base_name() shared by the writer and the canonicalizer;
  effort suffixes chain ("-thinking-64k-high-effort"), LiveBench budgets ("32k thinking", "(no thinking)"); "max"/"fast"/"instant" are model
  tiers (Qwen3-Max, GPT-5.1-Codex-Max, Grok 4.1 Fast), no longer effort markers; family_release_hint reads "gpt-5-4" as 5.4 and never a size (38B)
- taxonomy: normalize_property(entity_type, prop, value) → (canonical, value_raw, mappings); unknown values kept, mapped to NULL
- benchmarks: RUN_KEYS (release, index_version, version, dataset_revision) leave config_key so a newer run replaces the older one; they are
  condition keys for comparability (partially comparable)
- tests/test_ontology.py: licence table, openness derivation, taxonomy, 22 model names, comparability trio, metric bounds/trust

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 12, 2026) parent 631e059 Browse files

4 changed files +350 −21

4 changed files
Unified Split
modified src/aiatlas/ontology/benchmarks.py +10 −6
@@ -144,11 +144,13 @@ def trust_level(source_key: str | None, config: dict[str, Any] | None = None, *,
144 144
145 # ---------------------------------------------------------------------------------------------- comparability 145 # ---------------------------------------------------------------------------------------------- comparability
146 # Config keys that change the *task* (must match for full comparability). 146 # Config keys that change the *task* (must match for full comparability).
147−TASK_KEYS = ("variant", "board", "dataset_revision", "harness", "evaluator", "index_version", "release", "version", "subset", "split", "shots", 147+TASK_KEYS = ("variant", "board", "harness", "evaluator", "subset", "split", "shots", "pass_count", "attempts", "language", "scaffold", "agent", "system")
148− "pass_count", "attempts", "language", "scaffold", "agent", "system") 148+# Config keys that identify the *run* (a LiveBench release, an AA index version, a dataset revision): a newer run supersedes the older one
149+# for the same task (`run_group`, one current row per task), and two runs of the same task are only partially comparable.
150+RUN_KEYS = ("release", "index_version", "version", "dataset_revision")
149 # Config keys that change the *conditions* (mismatch → partially comparable). 151 # Config keys that change the *conditions* (mismatch → partially comparable).
150 CONDITION_KEYS = ("reasoning_effort", "reasoning", "thinking_budget", "temperature", "judge", "tools", "tool_use", "max_tokens", "context_length", 152 CONDITION_KEYS = ("reasoning_effort", "reasoning", "thinking_budget", "temperature", "judge", "tools", "tool_use", "max_tokens", "context_length",
151− "sampling", "aggregation", "edit_format", "model_tag") 153+ "sampling", "aggregation", "edit_format", "model_tag", *RUN_KEYS)
152 # Keys that are pure bookkeeping (never affect comparability). 154 # Keys that are pure bookkeeping (never affect comparability).
153 IGNORED_KEYS = {"aa_slug", "livebench_model_id", "api_model_id", "date", "submission", "checked_by_swebench", "open_source_system", "system_org", 155 IGNORED_KEYS = {"aa_slug", "livebench_model_id", "api_model_id", "date", "submission", "checked_by_swebench", "open_source_system", "system_org",
154 "total_cost_usd", "cost_per_instance_usd", "dirname", "command", "versions", "test_cases", "seconds_per_case", "estimated", 156 "total_cost_usd", "cost_per_instance_usd", "dirname", "command", "versions", "test_cases", "seconds_per_case", "estimated",
@@ -162,7 +164,8 @@ def _clean(v: Any) -> Any:
162 164
163 165
164 def config_key(config: dict[str, Any] | None, metric: str | None = None) -> str: 166 def config_key(config: dict[str, Any] | None, metric: str | None = None) -> str:
165− """Stable hash of the comparability-relevant part of a result configuration (task keys + metric).""" 167+ """Stable hash of the comparability-relevant part of a result configuration (task keys + metric). Run keys (release, index
168+ version) are deliberately excluded: they define `run_group`, so the newest run of a task replaces the older ones on leaderboards."""
166 cfg = config or {} 169 cfg = config or {}
167 core = {k: _clean(cfg[k]) for k in TASK_KEYS if k in cfg and cfg[k] not in (None, "", [], {})} 170 core = {k: _clean(cfg[k]) for k in TASK_KEYS if k in cfg and cfg[k] not in (None, "", [], {})}
168 core["metric"] = normalize_metric(metric) or "" 171 core["metric"] = normalize_metric(metric) or ""
@@ -215,5 +218,6 @@ def run_group_from_config(config: dict[str, Any] | None) -> str | None:
215 return None 218 return None
216 219
217 220
218−__all__ = ["COMPARABLE", "CONDITION_KEYS", "FAMILIES", "IGNORED_KEYS", "METRICS", "NOT_COMPARABLE", "PARTIAL", "TASK_KEYS", "TRUST_LABELS", "TRUST_LEVELS", 221+__all__ = ["COMPARABLE", "CONDITION_KEYS", "FAMILIES", "IGNORED_KEYS", "METRICS", "NOT_COMPARABLE", "PARTIAL", "RUN_KEYS", "TASK_KEYS", "TRUST_LABELS",
219− "comparability", "config_key", "family_of", "metric_bounds", "normalize_metric", "run_group_from_config", "trust_level", "variant_from_config"] 222+ "TRUST_LEVELS", "comparability", "config_key", "family_of", "metric_bounds", "normalize_metric", "run_group_from_config", "trust_level",
223+ "variant_from_config"]
modified src/aiatlas/ontology/models.py +102 −13
@@ -43,12 +43,15 @@ CONVERTER_ORGS = {
43 EFFORT_SUFFIXES: dict[str, dict[str, str]] = { 43 EFFORT_SUFFIXES: dict[str, dict[str, str]] = {
44 "xhigh": {"reasoning_effort": "xhigh"}, "x-high": {"reasoning_effort": "xhigh"}, "extra-high": {"reasoning_effort": "xhigh"}, 44 "xhigh": {"reasoning_effort": "xhigh"}, "x-high": {"reasoning_effort": "xhigh"}, "extra-high": {"reasoning_effort": "xhigh"},
45 "high": {"reasoning_effort": "high"}, "medium": {"reasoning_effort": "medium"}, "low": {"reasoning_effort": "low"}, "minimal": {"reasoning_effort": "minimal"}, 45 "high": {"reasoning_effort": "high"}, "medium": {"reasoning_effort": "medium"}, "low": {"reasoning_effort": "low"}, "minimal": {"reasoning_effort": "minimal"},
46− "max": {"reasoning_effort": "max"}, "max-effort": {"reasoning_effort": "max"}, "high-effort": {"reasoning_effort": "high"}, "low-effort": {"reasoning_effort": "low"}, 46+ "max-effort": {"reasoning_effort": "max"}, "high-effort": {"reasoning_effort": "high"}, "low-effort": {"reasoning_effort": "low"},
47 "medium-effort": {"reasoning_effort": "medium"}, "xhigh-effort": {"reasoning_effort": "xhigh"}, 47 "medium-effort": {"reasoning_effort": "medium"}, "xhigh-effort": {"reasoning_effort": "xhigh"},
48 "thinking": {"reasoning": "on"}, "reasoning": {"reasoning": "on"}, "think": {"reasoning": "on"}, "thinking-on": {"reasoning": "on"}, 48 "thinking": {"reasoning": "on"}, "reasoning": {"reasoning": "on"}, "think": {"reasoning": "on"}, "thinking-on": {"reasoning": "on"},
49 "non-reasoning": {"reasoning": "off"}, "no-reasoning": {"reasoning": "off"}, "non-thinking": {"reasoning": "off"}, "nothink": {"reasoning": "off"}, 49 "non-reasoning": {"reasoning": "off"}, "no-reasoning": {"reasoning": "off"}, "non-thinking": {"reasoning": "off"}, "nothink": {"reasoning": "off"},
50− "no-think": {"reasoning": "off"}, "instant": {"reasoning": "off"}, "thinking-off": {"reasoning": "off"}, "fast": {"reasoning": "off"}, 50+ "no-think": {"reasoning": "off"}, "no-thinking": {"reasoning": "off"}, "thinking-off": {"reasoning": "off"}, "without-thinking": {"reasoning": "off"},
51+ "with-thinking": {"reasoning": "on"},
51 "adaptive": {"reasoning": "adaptive"}, "adaptive-reasoning": {"reasoning": "adaptive"}, 52 "adaptive": {"reasoning": "adaptive"}, "adaptive-reasoning": {"reasoning": "adaptive"},
53+ # NOT effort suffixes although evaluators sometimes use them: "max" (Qwen3-Max, GPT-5.1-Codex-Max are models), "fast" (Grok 4.1 Fast),
54+ # "instant" (Claude Instant) — they name a model tier, so they are only recognised in the explicit "-max-effort" form above.
52 "thinking-16k": {"reasoning": "on", "thinking_budget": "16k"}, "thinking-32k": {"reasoning": "on", "thinking_budget": "32k"}, 55 "thinking-16k": {"reasoning": "on", "thinking_budget": "16k"}, "thinking-32k": {"reasoning": "on", "thinking_budget": "32k"},
53 "thinking-64k": {"reasoning": "on", "thinking_budget": "64k"}, "thinking-128k": {"reasoning": "on", "thinking_budget": "128k"}, 56 "thinking-64k": {"reasoning": "on", "thinking_budget": "64k"}, "thinking-128k": {"reasoning": "on", "thinking_budget": "128k"},
54 "thinking-8k": {"reasoning": "on", "thinking_budget": "8k"}, "thinking-4k": {"reasoning": "on", "thinking_budget": "4k"}, "thinking-1k": {"reasoning": "on", "thinking_budget": "1k"}, 57 "thinking-8k": {"reasoning": "on", "thinking_budget": "8k"}, "thinking-4k": {"reasoning": "on", "thinking_budget": "4k"}, "thinking-1k": {"reasoning": "on", "thinking_budget": "1k"},
@@ -56,6 +59,10 @@ EFFORT_SUFFIXES: dict[str, dict[str, str]] = {
56 # order matters: try the longest compound suffixes first 59 # order matters: try the longest compound suffixes first
57 _EFFORT_ORDERED = sorted(EFFORT_SUFFIXES, key=len, reverse=True) 60 _EFFORT_ORDERED = sorted(EFFORT_SUFFIXES, key=len, reverse=True)
58 _EFFORT_RE = re.compile(r"[-_ ](" + "|".join(re.escape(s) for s in _EFFORT_ORDERED) + r")$", re.I) 61 _EFFORT_RE = re.compile(r"[-_ ](" + "|".join(re.escape(s) for s in _EFFORT_ORDERED) + r")$", re.I)
62+# LiveBench-style budgets: "32k-thinking", "24k-think", "default-think(ing)"
63+_BUDGET_RE = re.compile(r"[-_ ]((\d+)k)[-_ ]?(thinking|think)$", re.I)
64+_DEFAULT_THINK_RE = re.compile(r"[-_ ](default[-_ ]?(thinking|think))$", re.I)
65+_MAX_EFFORT_SUFFIXES = 3 # "claude-opus-4-5-20251101-thinking-64k-high-effort" → thinking-64k + high-effort
59 # Words that are *part of a model name*, never an effort suffix, when they precede the suffix (e.g. "Kimi K2 Thinking" is a distinct release). 66 # Words that are *part of a model name*, never an effort suffix, when they precede the suffix (e.g. "Kimi K2 Thinking" is a distinct release).
60 OFFICIAL_THINKING_RELEASES = {"kimi-k2-thinking", "qwen3-235b-a22b-thinking-2507", "qwen3-30b-a3b-thinking-2507", "qwen3-4b-thinking-2507", "glm-4.5-air-thinking", 67 OFFICIAL_THINKING_RELEASES = {"kimi-k2-thinking", "qwen3-235b-a22b-thinking-2507", "qwen3-30b-a3b-thinking-2507", "qwen3-4b-thinking-2507", "glm-4.5-air-thinking",
61 "gemini-2.5-flash-thinking", "grok-3-mini-thinking", "gemini-2-0-flash-thinking-exp-1219", "gemini-2-0-flash-thinking-exp-01-21"} 68 "gemini-2.5-flash-thinking", "grok-3-mini-thinking", "gemini-2-0-flash-thinking-exp-1219", "gemini-2-0-flash-thinking-exp-01-21"}
@@ -116,13 +123,33 @@ def analyze_model_name(raw: str) -> NameAnalysis:
116 low = re.sub(r"[()\[\]]+", "-", low).strip("-") 123 low = re.sub(r"[()\[\]]+", "-", low).strip("-")
117 low = re.sub(r"-{2,}", "-", low) 124 low = re.sub(r"-{2,}", "-", low)
118 125
119− # evaluator effort suffix (only when the remaining stem is not itself an official "Thinking" release) 126+ # evaluator effort suffixes (up to three, e.g. "-thinking-64k-high-effort"), only when the stem is not itself an official "Thinking" release
120− m = _EFFORT_RE.search(low) 127+ stripped: list[str] = []
121− if m and low not in OFFICIAL_THINKING_RELEASES: 128+ for _ in range(_MAX_EFFORT_SUFFIXES):
122− suffix = m.group(1).lower() 129+ if low in OFFICIAL_THINKING_RELEASES:
123− a.effort = dict(EFFORT_SUFFIXES[suffix]) 130+ break
124− a.effort_suffix = suffix 131+ m = _BUDGET_RE.search(low) # "-32k-thinking" before the bare "-thinking"
125− low = low[: m.start()] 132+ if m:
133+ a.effort = {"reasoning": "on", "thinking_budget": m.group(1).lower(), **a.effort}
134+ stripped.insert(0, m.group(0)[1:].lower())
135+ low = low[: m.start()]
136+ continue
137+ m = _DEFAULT_THINK_RE.search(low)
138+ if m:
139+ a.effort = {"reasoning": "on", **a.effort}
140+ stripped.insert(0, m.group(1).lower())
141+ low = low[: m.start()]
142+ continue
143+ m = _EFFORT_RE.search(low)
144+ if m:
145+ suffix = m.group(1).lower()
146+ a.effort = {**EFFORT_SUFFIXES[suffix], **a.effort}
147+ stripped.insert(0, suffix)
148+ low = low[: m.start()]
149+ continue
150+ break
151+ if stripped:
152+ a.effort_suffix = "-".join(stripped)
126 153
127 # sizes 154 # sizes
128 am = _ACTIVE_RE.search(low) 155 am = _ACTIVE_RE.search(low)
@@ -252,10 +279,13 @@ def family_release_hint(name: str) -> str | None:
252 if not fam: 279 if not fam:
253 return None 280 return None
254 n = name.split("/")[-1] 281 n = name.split("/")[-1]
255− m = re.search(re.escape(fam.split(" ")[0]) + r"[\s\-]?(\d+(?:\.\d+)?)", n, re.I) 282+ # version = digits right after the family word, optionally ".minor" or "-minor" (AA/OpenRouter slugs write 5.4 as 5-4);
283+ # a number followed by a size unit (38B, 235B, 1.5T) is a parameter count, never a version.
284+ m = re.search(re.escape(fam.split(" ")[0]) + r"[\s\-]?(\d+)(?![0-9]*\.?\d*[bmt](?![a-z]))(?:[.\-](\d+)(?![0-9]*[bmt](?![a-z])))?(?![0-9])", n, re.I)
256 if m: 285 if m:
286+ version = m.group(1) + (f".{m.group(2)}" if m.group(2) else "")
257 joined = fam in ("Qwen", "GLM", "Phi", "Yi", "Step") or re.search(re.escape(fam) + r"\d", n, re.I) 287 joined = fam in ("Qwen", "GLM", "Phi", "Yi", "Step") or re.search(re.escape(fam) + r"\d", n, re.I)
258− return f"{fam}{m.group(1)}" if joined else f"{fam} {m.group(1)}" 288+ return f"{fam}{version}" if joined else f"{fam} {version}"
259 return fam 289 return fam
260 290
261 291
@@ -267,5 +297,64 @@ def variant_key(name: str) -> str:
267 return "-".join(toks) 297 return "-".join(toks)
268 298
269 299
270−__all__ = ["ARTIFACT_PACKAGING", "CONVERTER_ORGS", "EFFORT_SUFFIXES", "NameAnalysis", "PRECISION_FORMATS", "QUANT_FORMATS", "analyze_model_name", 300+# ---------------------------------------------------------------------------------------------- official organisations per family
271− "family_hint", "family_release_hint", "variant_key"] 301+# family root (lowercase `family_hint`) → organisation slugs / hub org names (lowercase) that publish the *official* checkpoints.
302+# A repository under one of these orgs is the model (or its official checkpoint), never a third-party artifact.
303+FAMILY_ORGS: dict[str, tuple[str, ...]] = {
304+ "llama": ("meta-llama", "meta", "meta-ai", "facebook"), "qwen": ("qwen", "alibaba", "alibaba-cloud", "alibaba-qwen"), "gemma": ("google", "google-deepmind"),
305+ "gemini": ("google", "google-deepmind"), "palm": ("google",), "claude": ("anthropic",), "gpt": ("openai",), "gpt-oss": ("openai",), "openai o-series": ("openai",),
306+ "whisper": ("openai",), "dall·e": ("openai",), "sora": ("openai",), "deepseek": ("deepseek-ai", "deepseek"), "kimi": ("moonshotai", "moonshot-ai", "moonshot"),
307+ "glm": ("zai-org", "thudm", "zhipu-ai", "z-ai", "zhipu"), "mistral": ("mistralai", "mistral-ai", "mistral"), "mixtral": ("mistralai", "mistral-ai", "mistral"),
308+ "ministral": ("mistralai", "mistral-ai", "mistral"), "codestral": ("mistralai", "mistral-ai", "mistral"), "devstral": ("mistralai", "mistral-ai", "mistral"),
309+ "magistral": ("mistralai", "mistral-ai", "mistral"), "pixtral": ("mistralai", "mistral-ai", "mistral"), "voxtral": ("mistralai", "mistral-ai", "mistral"),
310+ "phi": ("microsoft",), "grok": ("xai", "x-ai", "xai-org"), "command": ("cohereforai", "cohere", "coherelabs", "cohere-labs"), "aya": ("cohereforai", "cohere", "coherelabs"),
311+ "nemotron": ("nvidia",), "granite": ("ibm-granite", "ibm"), "olmo": ("allenai", "ai2"), "molmo": ("allenai", "ai2"), "falcon": ("tiiuae", "tii"), "yi": ("01-ai",),
312+ "minimax": ("minimaxai", "minimax"), "hunyuan": ("tencent", "tencent-hunyuan"), "ernie": ("baidu",), "seed": ("bytedance-seed", "bytedance"), "doubao": ("bytedance",),
313+ "step": ("stepfun-ai", "stepfun"), "internlm": ("internlm", "shanghai-ai-laboratory", "opengvlab"), "jamba": ("ai21labs", "ai21"), "lfm": ("liquidai", "liquid-ai"),
314+ "exaone": ("lgai-exaone", "lg-ai-research"), "solar": ("upstage",), "nova": ("amazon", "aws"), "titan": ("amazon", "aws"), "sonar": ("perplexity-ai", "perplexity"),
315+ "stable diffusion": ("stabilityai", "stability-ai"), "flux": ("black-forest-labs",), "veo": ("google",), "imagen": ("google",), "cogito": ("deepcogito",),
316+ "hermes": ("nousresearch",), "smollm": ("huggingfacetb", "hugging-face"), "bert": ("google", "google-bert"), "t5": ("google", "google-t5"), "clip": ("openai",),
317+}
318+
319+
320+def official_orgs(name: str) -> tuple[str, ...]:
321+ """Official organisation slugs for the family the name belongs to (empty when unknown)."""
322+ fam = family_hint(name)
323+ return FAMILY_ORGS.get(fam.lower(), ()) if fam else ()
324+
325+
326+def is_official_org(repo_org: str | None, name: str) -> bool:
327+ return bool(repo_org) and repo_org.lower() in official_orgs(name)
328+
329+
330+def effort_config(name: str, config: dict | None) -> dict:
331+ """Result configuration for an evaluation-effort variant: the effort dict (`reasoning_effort`, `reasoning`, `thinking_budget`) is merged
332+ into the config and the evaluator's variant slug is kept as `aa_variant_slug` so folded results stay distinguishable. Names that are
333+ not effort variants return the config unchanged."""
334+ cfg = dict(config or {})
335+ a = analyze_model_name(name)
336+ if not a.is_effort_variant:
337+ return cfg
338+ for k, v in a.effort.items():
339+ cfg.setdefault(k, v)
340+ slug = cfg.get("aa_slug") or a.raw.split("/")[-1].strip().lower().replace(" ", "-")
341+ cfg.setdefault("aa_variant_slug", slug)
342+ return cfg
343+
344+
345+def base_name(name: str) -> str:
346+ """The name with the evaluator effort suffix removed ("gpt-5-4-mini-medium" → "gpt-5-4-mini", "Claude Sonnet 4 (no thinking)" →
347+ "Claude Sonnet 4"); unchanged when not a variant."""
348+ a = analyze_model_name(name)
349+ if not a.is_effort_variant or not a.effort_suffix:
350+ return name.strip()
351+ n = name.strip()
352+ # the suffix was detected on a normalised form (spaces/underscores/parentheses → "-"): strip it from the raw name with the same tolerance
353+ tokens = [re.escape(t) for t in a.effort_suffix.split("-") if t]
354+ pattern = r"[\s\-_(]+" + r"[\s\-_]*".join(tokens) + r"[)\s]*$"
355+ out = re.sub(pattern, "", n, flags=re.I)
356+ return out.strip().rstrip("-_ (").strip() or n
357+
358+
359+__all__ = ["ARTIFACT_PACKAGING", "CONVERTER_ORGS", "EFFORT_SUFFIXES", "FAMILY_ORGS", "NameAnalysis", "PRECISION_FORMATS", "QUANT_FORMATS", "analyze_model_name",
360+ "base_name", "effort_config", "family_hint", "family_release_hint", "is_official_org", "official_orgs", "variant_key"]
modified src/aiatlas/ontology/taxonomy.py +76 −2
@@ -149,6 +149,79 @@ def normalize_framework_kind(raw: Any) -> str | None:
149 return _FW_ALIASES.get(s) or _FW_ALIASES.get(s.replace("_", "-")) or _FW_ALIASES.get(s.replace("-", " ")) 149 return _FW_ALIASES.get(s) or _FW_ALIASES.get(s.replace("_", "-")) or _FW_ALIASES.get(s.replace("-", " "))
150 150
151 151
152+# ---------------------------------------------------------------------------------------------- property-level normalisation (writer + canonicalizer)
153+# property → taxonomy domain (the entity type picks the normaliser for `kind`)
154+TAXONOMY_PROPERTIES = {"license", "openness", "modalities", "modalities_input", "modalities_output", "status", "kind", "org_kind"}
155+_ORG_TYPES = {"company", "organization", "lab", "university", "provider"}
156+_FRAMEWORK_TYPES = {"framework", "library", "tool", "runtime", "agent", "mcp_server", "repository"}
157+
158+
159+def normalize_property(entity_type: str | None, prop: str, value: Any) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]:
160+ """Canonicalise one attribute value at write time.
161+
162+ Returns `(canonical_value, value_raw, mappings)` where `value_raw` is the source string when it differed from the canonical value
163+ (None otherwise) and `mappings` lists `(domain, raw, canonical|None)` pairs observed — unknown values are KEPT AS-IS (never dropped)
164+ and mapped to canonical=None so the taxonomy backlog stays visible."""
165+ if value is None or prop not in TAXONOMY_PROPERTIES:
166+ return value, None, []
167+ if prop == "license":
168+ from aiatlas.ontology.licenses import normalize_license
169+
170+ return _scalar("license", value, normalize_license)
171+ if prop == "openness":
172+ from aiatlas.ontology.openness import normalize_openness
173+
174+ return _scalar("openness", value, normalize_openness)
175+ if prop == "status":
176+ return _scalar("status", value, normalize_status)
177+ if prop == "org_kind":
178+ return _scalar("org_kind", value, normalize_org_kind)
179+ if prop == "kind":
180+ if entity_type == "hardware":
181+ return _scalar("hardware_kind", value, normalize_hardware_kind)
182+ if entity_type in _FRAMEWORK_TYPES:
183+ return _scalar("framework_kind", value, normalize_framework_kind)
184+ return value, None, []
185+ if prop in ("modalities", "modalities_input", "modalities_output"):
186+ items = value if isinstance(value, (list, tuple, set)) else re.split(r"[,/;+&]|\band\b|→|->", str(value))
187+ out: list[str] = []
188+ mappings: list[tuple[str, str, str | None]] = []
189+ changed = False
190+ for it in items:
191+ raw = str(it).strip()
192+ if not raw:
193+ continue
194+ canon = normalize_modality(raw)
195+ if canon is None and raw.lower() in _MODALITY_ALIASES: # "multimodal" & co: a property of the set, not a modality
196+ mappings.append(("modality", raw, None))
197+ changed = True
198+ continue
199+ mappings.append(("modality", raw, canon))
200+ keep = canon or raw
201+ if keep != raw:
202+ changed = True
203+ if keep not in out:
204+ out.append(keep)
205+ canonical = sorted(out)
206+ if not isinstance(value, (list, tuple, set)):
207+ changed = True
208+ elif sorted(str(x).strip() for x in value if str(x).strip()) != canonical:
209+ changed = True
210+ raw_repr = (", ".join(str(x) for x in value) if isinstance(value, (list, tuple, set)) else str(value)) if changed else None
211+ return canonical, raw_repr, mappings
212+ return value, None, []
213+
214+
215+def _scalar(domain: str, value: Any, normaliser) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]:
216+ if not isinstance(value, str):
217+ return value, None, []
218+ raw = value.strip()
219+ canon = normaliser(raw)
220+ if canon is None:
221+ return raw, None, [(domain, raw, None)]
222+ return canon, (raw if raw != canon else None), [(domain, raw, canon)]
223+
224+
152 # ---------------------------------------------------------------------------------------------- generic helper 225 # ---------------------------------------------------------------------------------------------- generic helper
153 def canonical_enum(value: Any, normaliser) -> tuple[Any, str | None]: 226 def canonical_enum(value: Any, normaliser) -> tuple[Any, str | None]:
154 """Return (canonical_or_original, raw_if_changed). Lets the writer store `x` canonical and `x_raw` when the source label differed.""" 227 """Return (canonical_or_original, raw_if_changed). Lets the writer store `x` canonical and `x_raw` when the source label differed."""
@@ -161,6 +234,7 @@ def canonical_enum(value: Any, normaliser) -> tuple[Any, str | None]:
161 234
162 235
163 __all__ = [ 236 __all__ = [
164− "FRAMEWORK_KINDS", "HARDWARE_KINDS", "MODALITIES", "MODEL_STATUSES", "ORG_KINDS", "ORG_TYPE_DEFAULT_KIND", "canonical_enum", 237+ "FRAMEWORK_KINDS", "HARDWARE_KINDS", "MODALITIES", "MODEL_STATUSES", "ORG_KINDS", "ORG_TYPE_DEFAULT_KIND", "TAXONOMY_PROPERTIES", "canonical_enum",
165− "normalize_framework_kind", "normalize_hardware_kind", "normalize_modalities", "normalize_modality", "normalize_org_kind", "normalize_status", 238+ "normalize_framework_kind", "normalize_hardware_kind", "normalize_modalities", "normalize_modality", "normalize_org_kind", "normalize_property",
239+ "normalize_status",
166 ] 240 ]
added tests/test_ontology.py +162 −0
@@ -0,0 +1,162 @@
1+"""Ontology — pure functions, no database."""
2+from __future__ import annotations
3+
4+import pytest
5+
6+from aiatlas.ontology import benchmarks as b
7+from aiatlas.ontology.licenses import LICENSES, license_info, normalize_license
8+from aiatlas.ontology.models import analyze_model_name, base_name, effort_config, family_hint, family_release_hint, is_official_org, variant_key
9+from aiatlas.ontology.openness import derive_openness, normalize_openness, openness_dimensions
10+from aiatlas.ontology.taxonomy import normalize_hardware_kind, normalize_modalities, normalize_org_kind, normalize_property, normalize_status
11+
12+
13+# ---------------------------------------------------------------------------------------------- licences
14+@pytest.mark.parametrize("raw", ["Apache 2.0", "apache-2.0", "Apache-2.0", "Apache License 2.0", "apache license, version 2.0", "APACHE2", "License: apache-2.0"])
15+def test_apache_variants_one_key(raw: str) -> None:
16+ assert normalize_license(raw) == "Apache-2.0"
17+
18+
19+@pytest.mark.parametrize("raw,key", [
20+ ("mit", "MIT"), ("Modified MIT", "MIT-Modified"), ("CC BY-NC 4.0", "CC-BY-NC-4.0"), ("cc-by-nc-4.0", "CC-BY-NC-4.0"), ("cc-by-4.0", "CC-BY-4.0"),
21+ ("llama3", "Llama-3-Community"), ("llama3.1", "Llama-3.1-Community"), ("llama3.2", "Llama-3.2-Community"), ("Llama 4 Community License", "Llama-4-Community"),
22+ ("gemma", "Gemma-Terms"), ("openrail++", "OpenRAIL++-M"), ("bigscience-bloom-rail-1.0", "BigScience-BLOOM-RAIL-1.0"), ("other", "Other"),
23+ ("apple-amlr", "Apple-AMLR"), ("bsd-3-clause", "BSD-3-Clause"), ("proprietary", "Proprietary"), ("GPLv3", "GPL-3.0"),
24+])
25+def test_license_table(raw: str, key: str) -> None:
26+ assert normalize_license(raw) == key
27+ assert key in LICENSES
28+
29+
30+def test_unknown_license_is_none_not_guessed() -> None:
31+ assert normalize_license("weird-license-xyz") is None
32+ assert normalize_license("") is None and normalize_license(None) is None
33+ assert license_info("weird") is None
34+
35+
36+# ---------------------------------------------------------------------------------------------- openness
37+def test_openness_derivation() -> None:
38+ llama = openness_dimensions(weights_available=True, license_key="Llama-3.1-Community")
39+ assert derive_openness(llama, license_key="Llama-3.1-Community") == "restricted-weights"
40+ apache = openness_dimensions(weights_available=True, license_key="Apache-2.0")
41+ assert derive_openness(apache, license_key="Apache-2.0") == "open-weights"
42+ assert derive_openness({**apache, "source_code_available": True}, license_key="Apache-2.0") == "open-source"
43+ prop = openness_dimensions(weights_available=None, license_key="Proprietary")
44+ assert prop["weights_available"] is False and derive_openness(prop, license_key="Proprietary") == "proprietary"
45+ assert derive_openness(openness_dimensions(weights_available=None), license_key=None) == "unknown"
46+ nc = openness_dimensions(weights_available=True, license_key="CC-BY-NC-4.0")
47+ assert nc["commercial_use_allowed"] is False and derive_openness(nc, license_key="CC-BY-NC-4.0") == "restricted-weights"
48+
49+
50+@pytest.mark.parametrize("raw,canon", [("open-weights", "open-weights"), ("restricted", "restricted-weights"), ("gated", "restricted-weights"), ("closed", "proprietary"),
51+ ("Open Source", "open-source"), ("", "unknown"), ("banana", None)])
52+def test_normalize_openness(raw: str, canon: str | None) -> None:
53+ assert normalize_openness(raw) == canon
54+
55+
56+# ---------------------------------------------------------------------------------------------- taxonomy
57+def test_modalities_and_status_and_kinds() -> None:
58+ assert normalize_modalities(["Text", "text", "pdf", "vision"]) == ["document", "image", "text"]
59+ assert normalize_modalities("text, image and audio") == ["audio", "image", "text"]
60+ assert normalize_status("limited-availability") == "limited-availability" and normalize_status("Available") == "active"
61+ assert normalize_status("archived") == "archived" and normalize_status("sunset") == "deprecated" and normalize_status("shutdown") == "retired"
62+ assert normalize_hardware_kind("computer") == "system" and normalize_hardware_kind("soc") == "soc" and normalize_hardware_kind("GPU") == "gpu"
63+ assert normalize_org_kind("startup") == "company" and normalize_org_kind("research institute") == "lab" and normalize_org_kind("organization") is None
64+
65+
66+def test_normalize_property_keeps_unknown_values() -> None:
67+ value, raw, maps = normalize_property("model", "license", "Apache 2.0")
68+ assert value == "Apache-2.0" and raw == "Apache 2.0" and ("license", "Apache 2.0", "Apache-2.0") in maps
69+ value, raw, maps = normalize_property("model", "license", "totally-custom")
70+ assert value == "totally-custom" and raw is None and maps == [("license", "totally-custom", None)]
71+ value, raw, maps = normalize_property("model", "modalities", ["Text", "smell", "pdf"])
72+ assert value == ["document", "smell", "text"] and raw == "Text, smell, pdf" # unknown modality kept, never dropped
73+ assert ("modality", "smell", None) in maps
74+ value, raw, _ = normalize_property("model", "modalities", ["text"])
75+ assert value == ["text"] and raw is None
76+ assert normalize_property("hardware", "kind", "computer")[0] == "system"
77+ assert normalize_property("framework", "kind", "framework")[0] == "training-framework"
78+ assert normalize_property("model", "kind", "whatever")[0] == "whatever" # `kind` is only a taxonomy for hardware/frameworks
79+ assert normalize_property("model", "parameter_count", 7_000_000_000) == (7_000_000_000, None, [])
80+
81+
82+# ---------------------------------------------------------------------------------------------- model names
83+NAMES = {
84+ # name → (is_artifact, is_effort_variant, base_key, family_release_hint)
85+ "Qwen3.6-35B-A3B": (False, False, "qwen3-6-35b-a3b", "Qwen3.6"),
86+ "unsloth/Qwen3.6-35B-A3B-GGUF": (True, False, "qwen3-6-35b-a3b", "Qwen3.6"),
87+ "Qwen3.6-35B-A3B-FP8": (True, False, "qwen3-6-35b-a3b", "Qwen3.6"),
88+ "mlx-community/Kimi-K2.5": (True, False, "kimi-k2-5", "Kimi"),
89+ "zai-org/GLM-5-FP8": (True, False, "glm-5", "GLM5"),
90+ "amd/Llama-3.3-70B-Instruct-MXFP4": (True, False, "llama-3-3-70b-instruct", "Llama 3.3"),
91+ "bartowski/Qwen3.8-27B-GGUF": (True, False, "qwen3-8-27b", "Qwen3.8"),
92+ "meta-llama/Llama-4-Maverick-17B-128E-Instruct": (False, False, "llama-4-maverick-17b-128e-instruct", "Llama 4"),
93+ "claude-fable-5-1-xhigh": (False, True, "claude-fable-5-1", "Claude"),
94+ "gpt-6-astra-high": (False, True, "gpt-6-astra", "GPT 6"),
95+ "deepseek-v4-pro-0424-non-reasoning": (False, True, "deepseek-v4-pro-0424", "DeepSeek"),
96+ "gpt-5-4-mini-medium": (False, True, "gpt-5-4-mini", "GPT 5.4"),
97+ "Qwen3 Max Thinking": (False, True, "qwen3-max", "Qwen3"),
98+ "Kimi K2 Thinking": (False, False, "kimi-k2-thinking", "Kimi"),
99+ "Qwen3-Max": (False, False, "qwen3-max", "Qwen3"), # "max" is a tier, not an effort
100+ "GPT-5.1-Codex-Max": (False, False, "gpt-5-1-codex-max", "GPT 5.1"),
101+ "Grok 4.1 Fast": (False, False, "grok-4-1-fast", "Grok 4.1"),
102+ "gemini-2-0-flash-thinking-exp-1219": (False, False, "gemini-2-0-flash-thinking-exp-1219", "Gemini 2.0"),
103+ "Llama 3.1 70B Instruct": (False, False, "llama-3-1-70b-instruct", "Llama 3.1"),
104+ "Qwen 38B": (False, False, "qwen-38b", "Qwen"),
105+ "Qwen3-235B-A22B": (False, False, "qwen3-235b-a22b", "Qwen3"),
106+ "deepseek-ai/DeepSeek-V4-Flash-0731": (False, False, "deepseek-v4-flash-0731", "DeepSeek"),
107+}
108+
109+
110+@pytest.mark.parametrize("name", sorted(NAMES))
111+def test_analyze_model_name(name: str) -> None:
112+ is_artifact, is_variant, base, fam = NAMES[name]
113+ a = analyze_model_name(name)
114+ assert a.is_artifact is is_artifact, (name, a)
115+ assert a.is_effort_variant is is_variant, (name, a)
116+ assert a.base_key == base, (name, a.base_key)
117+ assert family_release_hint(name) == fam, (name, family_release_hint(name))
118+
119+
120+def test_variant_key_groups_artifacts_with_their_model() -> None:
121+ assert variant_key("Qwen3.6-35B-A3B") == variant_key("unsloth/Qwen3.6-35B-A3B-GGUF") == variant_key("Qwen3.6 35B A3B FP8") == "qwen3-6-35b-a3b"
122+ assert variant_key("Qwen3-8B") != variant_key("Qwen 38B")
123+ assert variant_key("Llama 3.1 70B Instruct") == variant_key("meta-llama/Llama-3.1-70B-Instruct") == "llama-3-1-70b"
124+
125+
126+def test_effort_helpers() -> None:
127+ assert base_name("gpt-5-4-mini-medium") == "gpt-5-4-mini" and base_name("Qwen3 Max Thinking") == "Qwen3 Max" and base_name("Qwen3-Max") == "Qwen3-Max"
128+ cfg = effort_config("gpt-5-4-mini-medium", {"aa_slug": "gpt-5-4-mini-medium", "evaluator": "AA"})
129+ assert cfg["reasoning_effort"] == "medium" and cfg["aa_variant_slug"] == "gpt-5-4-mini-medium" and cfg["evaluator"] == "AA"
130+ assert effort_config("Qwen3-Max", {"x": 1}) == {"x": 1}
131+ assert analyze_model_name("claude-fable-5-1-xhigh").parameter_count is None
132+ assert analyze_model_name("Qwen3.6-35B-A3B").active_parameter_count == 3_000_000_000
133+ assert family_hint("o3-mini") == "OpenAI o-series" and family_hint("random-thing") is None
134+ assert is_official_org("meta-llama", "Llama 4 Maverick") and not is_official_org("unsloth", "Llama 4 Maverick")
135+
136+
137+# ---------------------------------------------------------------------------------------------- benchmarks
138+def test_comparability_trio() -> None:
139+ a = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "high"}
140+ same = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "high"}
141+ cond = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "low"}
142+ task = {"variant": "Lite", "system": "mini-SWE-agent", "reasoning_effort": "high"}
143+ assert b.comparability(a, same, "resolved", "resolved")[0] == b.COMPARABLE
144+ assert b.comparability(a, cond, "resolved", "resolved")[0] == b.PARTIAL
145+ assert b.comparability(a, task, "resolved", "resolved")[0] == b.NOT_COMPARABLE
146+ assert b.comparability(a, same, "resolved", "pass@1")[0] == b.NOT_COMPARABLE
147+ assert b.comparability(a, same, same_benchmark=False)[0] == b.NOT_COMPARABLE
148+ # config_key ignores condition and bookkeeping keys, keeps task keys + metric
149+ assert b.config_key(a, "resolved") == b.config_key(cond, "resolved") == b.config_key({**a, "aa_slug": "x", "date": "2026"}, "resolved")
150+ assert b.config_key(a, "resolved") != b.config_key(task, "resolved") != b.config_key(a, "pass@1")
151+
152+
153+def test_metric_bounds_and_trust() -> None:
154+ assert b.metric_bounds("accuracy") == (0, 100) and b.metric_bounds("pass rate (2 attempts)") == (0, 100) and b.metric_bounds("category:Reasoning") == (0, 100)
155+ assert b.metric_bounds("elo") == (0, None) and b.metric_bounds("score") == (None, None) and b.metric_bounds("weird", "%") == (0, 100)
156+ assert b.normalize_metric("Percent Resolved") == "resolved" and b.normalize_metric("Elo / Bradley–Terry score") == "elo"
157+ assert b.trust_level("swebench.com", {"checked_by_swebench": True}) == "official-benchmark"
158+ assert b.trust_level("swebench.com", {"checked_by_swebench": False}) == "community"
159+ assert b.trust_level("artificialanalysis.ai") == "independent-evaluator" and b.trust_level("docs.claude.com") == "official-model-card"
160+ assert b.trust_level(None, extractor="llm") == "unverified"
161+ assert b.run_group_from_config({"release": "2026-06-25"}) == "2026-06-25" and b.run_group_from_config({"index_version": "4.3"}) == "4.3"
162+ assert b.variant_from_config({"board": "Multilingual"}) == "Multilingual" and b.family_of("swe-bench-verified") == ("swe-bench", "Verified")
163
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere