spb/ai-atlas

1"""Numbers as they appear in AI documentation: `70B`, `1.5T`, `128K context`, `$3.00 / 1M tokens`, `72.4%`."""2from __future__ import annotations34import re56_MULT = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12, "g": 1e9}78_PARAMS = re.compile(r"(?<![\w.])(\d+(?:[.,]\d+)?)\s*([kKmMbBtT])\b(?:\s*(?:params?|parameters?))?", re.IGNORECASE)9_PARAMS_WORD = re.compile(r"(\d+(?:[.,]\d+)?)\s*(billion|million|trillion)\s*(?:params?|parameters?)", re.IGNORECASE)10_CONTEXT = re.compile(r"(\d+(?:[.,]\d+)?)\s*([kKmM])?\s*(?:tokens?|token|ctx|context)?", re.IGNORECASE)11_MONEY = re.compile(r"(?:US)?\$\s*(\d+(?:[.,]\d+)?)\s*(?:/|per)\s*(?:1\s*)?([mMkK])\s*(?:tok(?:ens?)?)?", re.IGNORECASE)12_MONEY_SIMPLE = re.compile(r"(?:US)?\$\s*(\d+(?:\.\d+)?)")13_PCT = re.compile(r"(-?\d+(?:\.\d+)?)\s*%")141516def _num(s: str) -> float:17    return float(s.replace(",", "."))181920def parse_param_count(text: str) -> int | None:21    """'Qwen3-235B-A22B' → 235e9 ; '7.6 billion parameters' → 7.6e9. Returns total parameters (not active)."""22    m = _PARAMS_WORD.search(text)23    if m:24        return int(_num(m.group(1)) * {"million": 1e6, "billion": 1e9, "trillion": 1e12}[m.group(2).lower()])25    best: int | None = None26    for m in _PARAMS.finditer(text):27        val = _num(m.group(1)) * _MULT[m.group(2).lower()]28        if 1e6 <= val <= 5e13 and (best is None or val > best):29            best = int(val)30    return best313233def parse_active_params(text: str) -> int | None:34    """'235B-A22B' / '30B-A3B' → active parameters of an MoE model."""35    m = re.search(r"-A(\d+(?:\.\d+)?)([bBmM])\b", text)36    if not m:37        m = re.search(r"(\d+(?:\.\d+)?)\s*([bB])\s*active", text, re.IGNORECASE)38    if not m:39        return None40    return int(_num(m.group(1)) * _MULT[m.group(2).lower()])414243def parse_context_length(text: str) -> int | None:44    """'128K' → 128000 ; '1M' → 1000000 ; '200,000 tokens' → 200000 ; '32768' → 32768."""45    t = text.strip().replace("tokens", "").replace("token", "")46    m = re.search(r"(\d{1,3}(?:,\d{3})+|\d+(?:\.\d+)?)\s*([kKmM])?\b", t)47    if not m:48        return None49    raw = m.group(1)50    if "," in raw and not m.group(2):51        val = float(raw.replace(",", ""))52    else:53        val = float(raw.replace(",", "."))54        if m.group(2):55            unit = m.group(2).lower()56            val *= 1024 if (unit == "k" and val in (8, 16, 32, 64, 128, 256, 512)) and "1024" in text else _MULT[unit]57    if val < 256 or val > 1e8:58        return None59    return int(val)606162def parse_tokens(text: str) -> int | None:63    return parse_context_length(text)646566def parse_money_per_mtok(text: str) -> float | None:67    """'$3.00 / 1M tokens' → 3.0 ; '$0.15/M' → 0.15 ; '$2 per 1K tokens' → 2000.0 (normalised to per million)."""68    m = _MONEY.search(text)69    if m:70        val = _num(m.group(1))71        unit = m.group(2).lower()72        return val * 1000 if unit == "k" else val73    m = _MONEY_SIMPLE.search(text)74    if m and "token" in text.lower():75        return float(m.group(1))76    return None777879def parse_money(text: str) -> float | None:80    m = _MONEY_SIMPLE.search(text.replace(",", ""))81    return float(m.group(1)) if m else None828384def parse_percent(text: str) -> float | None:85    m = _PCT.search(text)86    if m:87        return float(m.group(1))88    m = re.search(r"(?<![\d.])(\d{1,3}(?:\.\d+)?)(?![\d.%])", text)89    if m:90        v = float(m.group(1))91        return v if 0 <= v <= 100 else None92    return None939495def parse_int(text: str) -> int | None:96    m = re.search(r"-?\d[\d,]*", text)97    if not m:98        return None99    try:100        return int(m.group(0).replace(",", ""))101    except ValueError:102        return None103104105__all__ = [106    "parse_active_params",107    "parse_context_length",108    "parse_int",109    "parse_money",110    "parse_money_per_mtok",111    "parse_param_count",112    "parse_percent",113    "parse_tokens",114]115