HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Small canonical enums and their normalisers. Raw source strings are never mutated in snapshots — only the materialised attribute2is canonical; connectors and the writer call these before writing a claim."""3from __future__ import annotations45import re6from typing import Any78# ---------------------------------------------------------------------------------------------- modalities9MODALITIES = ("text", "image", "audio", "video", "document", "code", "embedding", "3d", "structured", "action")10_MODALITY_ALIASES = {11 "text": "text", "texts": "text", "language": "text", "nlp": "text", "chat": "text", "textual": "text",12 "image": "image", "images": "image", "vision": "image", "visual": "image", "picture": "image", "photo": "image", "img": "image",13 "audio": "audio", "speech": "audio", "voice": "audio", "sound": "audio", "music": "audio",14 "video": "video", "videos": "video",15 "pdf": "document", "document": "document", "documents": "document", "file": "document", "files": "document", "doc": "document",16 "code": "code", "coding": "code", "programming": "code",17 "embedding": "embedding", "embeddings": "embedding", "vector": "embedding",18 "3d": "3d", "mesh": "3d", "point cloud": "3d",19 "structured": "structured", "json": "structured", "table": "structured", "tabular": "structured", "time-series": "structured", "time series": "structured",20 "action": "action", "actions": "action", "computer-use": "action", "computer use": "action", "tool": "action", "robotics": "action",21 "multimodal": None, "multi-modal": None, "omni": None, "any": None, # not a modality — a property of the set22}232425def normalize_modality(raw: Any) -> str | None:26 if raw is None:27 return None28 s = str(raw).strip().lower().replace("_", "-")29 if s in _MODALITY_ALIASES:30 return _MODALITY_ALIASES[s]31 s2 = s.replace("-", " ")32 return _MODALITY_ALIASES.get(s2)333435def normalize_modalities(raw: Any) -> list[str]:36 """Any iterable / comma string → sorted canonical unique list (unknown values dropped)."""37 if raw is None:38 return []39 items = raw if isinstance(raw, (list, tuple, set)) else re.split(r"[,/;+&]|\band\b|→|->", str(raw))40 out: set[str] = set()41 for it in items:42 m = normalize_modality(it)43 if m:44 out.add(m)45 return sorted(out)464748# ---------------------------------------------------------------------------------------------- model / entity status49MODEL_STATUSES = ("announced", "preview", "active", "limited-availability", "deprecated", "retired", "archived", "unknown")50_STATUS_ALIASES = {51 "active": "active", "available": "active", "ga": "active", "general availability": "active", "generally available": "active", "live": "active",52 "released": "active", "stable": "active", "current": "active", "supported": "active", "production": "active", "online": "active",53 "preview": "preview", "beta": "preview", "alpha": "preview", "experimental": "preview", "public preview": "preview", "early access": "preview",54 "research preview": "preview", "exp": "preview",55 "limited-availability": "limited-availability", "limited availability": "limited-availability", "limited": "limited-availability",56 "waitlist": "limited-availability", "private preview": "limited-availability", "invite-only": "limited-availability", "gated": "limited-availability",57 "deprecated": "deprecated", "legacy": "deprecated", "sunset": "deprecated", "sunsetting": "deprecated", "end-of-life announced": "deprecated",58 "deprecation": "deprecated",59 "retired": "retired", "discontinued": "retired", "shut down": "retired", "shutdown": "retired", "removed": "retired", "end of life": "retired",60 "eol": "retired", "decommissioned": "retired", "offline": "retired", "delisted": "retired",61 "announced": "announced", "upcoming": "announced", "coming soon": "announced", "planned": "announced", "unreleased": "announced",62 "archived": "archived",63 "unknown": "unknown", "": "unknown",64}656667def normalize_status(raw: Any) -> str | None:68 if raw is None:69 return None70 s = str(raw).strip().lower().replace("_", "-")71 if s in _STATUS_ALIASES:72 return _STATUS_ALIASES[s]73 return _STATUS_ALIASES.get(s.replace("-", " "))747576# ---------------------------------------------------------------------------------------------- organization kinds77ORG_KINDS = ("company", "lab", "university", "nonprofit", "government", "community", "consortium", "individual")78_ORG_KIND_ALIASES = {79 "company": "company", "corporation": "company", "startup": "company", "enterprise": "company", "vendor": "company", "business": "company",80 "lab": "lab", "laboratory": "lab", "research lab": "lab", "research-lab": "lab", "research institute": "lab", "institute": "lab", "research": "lab",81 "university": "university", "academic": "university", "school": "university", "college": "university",82 "nonprofit": "nonprofit", "non-profit": "nonprofit", "non profit": "nonprofit", "foundation": "nonprofit", "ngo": "nonprofit",83 "government": "government", "public": "government", "agency": "government", "state": "government",84 "community": "community", "open-source community": "community", "collective": "community", "hf-community": "community", "group": "community",85 "consortium": "consortium", "alliance": "consortium", "coalition": "consortium",86 "individual": "individual", "person": "individual", "independent": "individual", "user": "individual",87}88# entity_type → default org kind (the entity_type stays; org_kind is the canonical sub-kind)89ORG_TYPE_DEFAULT_KIND = {"company": "company", "lab": "lab", "university": "university", "organization": None}909192def normalize_org_kind(raw: Any) -> str | None:93 if raw is None:94 return None95 s = str(raw).strip().lower().replace("_", "-")96 return _ORG_KIND_ALIASES.get(s) or _ORG_KIND_ALIASES.get(s.replace("-", " "))979899# ---------------------------------------------------------------------------------------------- hardware kinds100HARDWARE_KINDS = ("gpu", "accelerator", "npu", "cpu", "soc", "system", "server", "workstation", "cloud-instance", "rack")101_HW_ALIASES = {102 "gpu": "gpu", "graphics card": "gpu", "graphics": "gpu", "dgpu": "gpu",103 "accelerator": "accelerator", "tpu": "accelerator", "asic": "accelerator", "ai accelerator": "accelerator", "lpu": "accelerator", "wse": "accelerator",104 "npu": "npu", "neural engine": "npu",105 "cpu": "cpu", "processor": "cpu",106 "soc": "soc", "system on chip": "soc", "system-on-chip": "soc", "apple silicon": "soc", "chip": "soc",107 "system": "system", "computer": "system", "desktop": "system", "laptop": "system", "mini": "system", "mac": "system",108 "server": "server", "node": "server", "dgx": "server", "appliance": "server",109 "workstation": "workstation",110 "cloud-instance": "cloud-instance", "cloud instance": "cloud-instance", "instance": "cloud-instance", "vm": "cloud-instance",111 "rack": "rack", "superpod": "rack", "pod": "rack", "cluster": "rack", "nvl72": "rack",112}113114115def normalize_hardware_kind(raw: Any) -> str | None:116 if raw is None:117 return None118 s = str(raw).strip().lower().replace("_", "-")119 return _HW_ALIASES.get(s) or _HW_ALIASES.get(s.replace("-", " "))120121122# ---------------------------------------------------------------------------------------------- framework / library / tool kinds123FRAMEWORK_KINDS = ("training-framework", "inference-engine", "serving-engine", "library", "runtime", "agent-framework", "orchestration",124 "evaluation-harness", "sdk", "tool", "application", "agent", "mcp-server", "vector-database", "observability", "data-tooling")125_FW_ALIASES = {126 "framework": "training-framework", "training framework": "training-framework", "training-framework": "training-framework", "deep learning framework": "training-framework",127 "inference engine": "inference-engine", "inference-engine": "inference-engine", "inference": "inference-engine", "engine": "inference-engine",128 "serving": "serving-engine", "serving engine": "serving-engine", "serving-engine": "serving-engine", "server": "serving-engine",129 "library": "library", "lib": "library", "package": "library",130 "runtime": "runtime",131 "agent framework": "agent-framework", "agent-framework": "agent-framework", "agents": "agent-framework",132 "orchestration": "orchestration", "workflow": "orchestration", "pipeline": "orchestration",133 "evaluation": "evaluation-harness", "eval": "evaluation-harness", "evaluation harness": "evaluation-harness", "evaluation-harness": "evaluation-harness", "benchmarking": "evaluation-harness",134 "sdk": "sdk", "client": "sdk", "api client": "sdk",135 "tool": "tool", "cli": "tool", "utility": "tool",136 "application": "application", "app": "application", "product": "application", "ui": "application", "desktop app": "application",137 "agent": "agent", "coding agent": "agent", "assistant": "agent",138 "mcp": "mcp-server", "mcp server": "mcp-server", "mcp-server": "mcp-server", "mcp_server": "mcp-server",139 "vector database": "vector-database", "vector-database": "vector-database", "vector db": "vector-database", "vector store": "vector-database",140 "observability": "observability", "tracing": "observability", "monitoring": "observability", "experiment tracking": "observability",141 "data": "data-tooling", "data tooling": "data-tooling", "data-tooling": "data-tooling", "datasets": "data-tooling", "tokenizer": "data-tooling",142}143144145def normalize_framework_kind(raw: Any) -> str | None:146 if raw is None:147 return None148 s = str(raw).strip().lower()149 return _FW_ALIASES.get(s) or _FW_ALIASES.get(s.replace("_", "-")) or _FW_ALIASES.get(s.replace("-", " "))150151152# ---------------------------------------------------------------------------------------------- property-level normalisation (writer + canonicalizer)153# property → taxonomy domain (the entity type picks the normaliser for `kind`)154TAXONOMY_PROPERTIES = {"license", "openness", "modalities", "modalities_input", "modalities_output", "status", "kind", "org_kind"}155_ORG_TYPES = {"company", "organization", "lab", "university", "provider"}156_FRAMEWORK_TYPES = {"framework", "library", "tool", "runtime", "agent", "mcp_server", "repository"}157158159def normalize_property(entity_type: str | None, prop: str, value: Any) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]:160 """Canonicalise one attribute value at write time.161162 Returns `(canonical_value, value_raw, mappings)` where `value_raw` is the source string when it differed from the canonical value163 (None otherwise) and `mappings` lists `(domain, raw, canonical|None)` pairs observed — unknown values are KEPT AS-IS (never dropped)164 and mapped to canonical=None so the taxonomy backlog stays visible."""165 if value is None or prop not in TAXONOMY_PROPERTIES:166 return value, None, []167 if prop == "license":168 from aiatlas.ontology.licenses import normalize_license169170 return _scalar("license", value, normalize_license)171 if prop == "openness":172 from aiatlas.ontology.openness import normalize_openness173174 return _scalar("openness", value, normalize_openness)175 if prop == "status":176 return _scalar("status", value, normalize_status)177 if prop == "org_kind":178 return _scalar("org_kind", value, normalize_org_kind)179 if prop == "kind":180 if entity_type == "hardware":181 return _scalar("hardware_kind", value, normalize_hardware_kind)182 if entity_type in _FRAMEWORK_TYPES:183 return _scalar("framework_kind", value, normalize_framework_kind)184 return value, None, []185 if prop in ("modalities", "modalities_input", "modalities_output"):186 items = value if isinstance(value, (list, tuple, set)) else re.split(r"[,/;+&]|\band\b|→|->", str(value))187 out: list[str] = []188 mappings: list[tuple[str, str, str | None]] = []189 changed = False190 for it in items:191 raw = str(it).strip()192 if not raw:193 continue194 canon = normalize_modality(raw)195 if canon is None and raw.lower() in _MODALITY_ALIASES: # "multimodal" & co: a property of the set, not a modality196 mappings.append(("modality", raw, None))197 changed = True198 continue199 mappings.append(("modality", raw, canon))200 keep = canon or raw201 if keep != raw:202 changed = True203 if keep not in out:204 out.append(keep)205 canonical = sorted(out)206 if not isinstance(value, (list, tuple, set)):207 changed = True208 elif sorted(str(x).strip() for x in value if str(x).strip()) != canonical:209 changed = True210 raw_repr = (", ".join(str(x) for x in value) if isinstance(value, (list, tuple, set)) else str(value)) if changed else None211 return canonical, raw_repr, mappings212 return value, None, []213214215# raw values that legitimately have no canonical counterpart (kept as-is, never reported as an unmapped violation)216NOT_A_VIOLATION = {("framework_kind", "model"), ("org_kind", "organization")}217218219def _scalar(domain: str, value: Any, normaliser) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]:220 if not isinstance(value, str):221 return value, None, []222 raw = value.strip()223 canon = normaliser(raw)224 if canon is None:225 if (domain, raw.lower()) in NOT_A_VIOLATION:226 return raw, None, []227 return raw, None, [(domain, raw, None)]228 return canon, (raw if raw != canon else None), [(domain, raw, canon)]229230231# ---------------------------------------------------------------------------------------------- generic helper232def canonical_enum(value: Any, normaliser) -> tuple[Any, str | None]:233 """Return (canonical_or_original, raw_if_changed). Lets the writer store `x` canonical and `x_raw` when the source label differed."""234 if value is None:235 return None, None236 canon = normaliser(value)237 if canon is None:238 return value, None239 return canon, (str(value) if str(value) != canon else None)240241242__all__ = [243 "FRAMEWORK_KINDS", "HARDWARE_KINDS", "MODALITIES", "MODEL_STATUSES", "ORG_KINDS", "ORG_TYPE_DEFAULT_KIND", "TAXONOMY_PROPERTIES", "canonical_enum",244 "normalize_framework_kind", "normalize_hardware_kind", "normalize_modalities", "normalize_modality", "normalize_org_kind", "normalize_property",245 "normalize_status",246]247