HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Shared helpers for lab connectors: announcement events from feeds/listings, key/value tables, model-name normalisation."""2from __future__ import annotations34import re5from datetime import datetime6from typing import Any78from aiatlas.connectors._identity import family_ref9from aiatlas.ontology.licenses import normalize_license10from aiatlas.ontology.openness import derive_openness, openness_dimensions11from aiatlas.ontology.taxonomy import normalize_modalities, normalize_status12from aiatlas.sdk.extract.dates import parse_datetime13from aiatlas.sdk.extract.feeds import FeedItem14from aiatlas.sdk.facts import EntityRef, Facts, Target1516RELEASE_WORDS = re.compile(r"\b(introducing|announcing|launch(?:ing|es|ed)?|releas(?:e|es|ing|ed)|now available|new model|preview|deprecat|pricing|price)\b", re.IGNORECASE)17MODEL_WORDS = re.compile(r"\b(model|gpt|claude|gemini|llama|mistral|qwen|deepseek|grok|command|phi|nemotron|gemma|sonnet|opus|haiku|o\d|codex|sora|veo|imagen|whisper|"18 r"embedding|reasoning|agent)\w*", re.IGNORECASE)192021def announcement_events(facts: Facts, org: EntityRef, items: list[FeedItem], *, source_name: str, follow: bool = True, max_follow: int = 30,22 needs_llm: bool = True, importance_default: int = 1) -> int:23 """One ANNOUNCEMENT event per feed item (deduped by URL) and, optionally, a follow-up fetch of the article for LLM extraction."""24 n = 025 for i, it in enumerate(items):26 if not it.url or not it.title:27 continue28 text = f"{it.title} {it.summary or ''}"29 is_release = bool(RELEASE_WORDS.search(text) and MODEL_WORDS.search(text))30 importance = 2 if is_release else importance_default31 facts.event("ANNOUNCEMENT", "release" if is_release else "company", f"{org.name}: {it.title}", entity=org, importance=importance,32 effective_at=it.published_at, dedupe_key=f"ANNOUNCEMENT:{it.url}", source_url=it.url,33 meta={"source": source_name, "categories": it.categories[:5], "summary": (it.summary or "")[:300], "is_release": is_release})34 if follow and i < max_follow:35 facts.follow(it.url, doc_type="news", needs_llm=needs_llm and is_release, priority=1 if is_release else 3,36 meta={"llm_task": "release_announcement", "published_at": it.published_at.isoformat() if it.published_at else None, "title": it.title})37 n += 138 return n394041def kv_tables(tables: list[dict[str, Any]]) -> dict[str, str]:42 """Merge 2-column key/value tables into one dict (lower-cased keys)."""43 out: dict[str, str] = {}44 for t in tables:45 rows = t.get("rows") or []46 headers = t.get("headers") or []47 if headers and len(headers) == 2 and rows:48 for r in rows:49 if len(r) >= 2 and r[0]:50 out.setdefault(r[0].strip().lower(), r[1].strip())51 elif rows and all(len(r) == 2 for r in rows):52 for r in rows:53 if r[0]:54 out.setdefault(r[0].strip().lower(), r[1].strip())55 return out565758def transpose_feature_table(table: dict[str, Any]) -> dict[str, dict[str, str]]:59 """Feature-comparison table (first column = feature, other columns = models) → {model: {feature: value}}."""60 headers = table.get("headers") or []61 if len(headers) < 2:62 return {}63 models = [clean_cell(h) for h in headers[1:]]64 out: dict[str, dict[str, str]] = {m: {} for m in models if m}65 for row in table.get("rows") or []:66 if not row:67 continue68 feature = clean_cell(row[0]).lower()69 for m, val in zip(models, row[1:], strict=False):70 if m:71 out[m][feature] = clean_cell(val)72 return out737475_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")76_CODE = re.compile(r"`([^`]*)`")777879def clean_cell(s: str) -> str:80 import html as _html8182 s = _html.unescape(s or "")83 s = _LINK.sub(r"\1", s)84 s = _CODE.sub(r"\1", s)85 s = re.sub(r"\*\*|__", "", s)86 s = re.sub(r"<[^>]+>", " ", s)87 return re.sub(r"\s+", " ", s).strip()888990def link_in_cell(s: str) -> str | None:91 m = re.search(r"\]\((https?://[^)\s]+)\)", s or "")92 return m.group(1) if m else None939495def parse_retirement(text: str) -> tuple[str | None, bool]:96 """'Not sooner than September 1, 2027' → ('2027-09-01', True=tentative)."""97 if not text or text.strip().lower() in ("n/a", "—", "-", ""):98 return None, False99 tentative = "not sooner" in text.lower() or "tentative" in text.lower()100 m = re.search(r"([A-Z][a-z]+ \d{1,2}, \d{4}|\d{4}-\d{2}-\d{2})", text)101 if not m:102 return None, tentative103 dt = parse_datetime(m.group(1))104 return (dt.date().isoformat() if dt else None), tentative105106107def money(cell: str) -> float | None:108 """'$10 / MTok', '$0.25 / MTok1' (footnote), '$12.50 per 1M tokens' → 10.0 …"""109 if not cell:110 return None111 m = re.search(r"\$\s*(\d+(?:\.\d+)?)", cell.replace(",", ""))112 if not m:113 return None114 val = float(m.group(1))115 low = cell.lower()116 if re.search(r"/\s*1?k\b|per\s*1?k\b|1,000 tokens", low):117 return val * 1000118 return val119120121def tokens(cell: str) -> int | None:122 from aiatlas.sdk.extract.numbers import parse_context_length123124 return parse_context_length(cell or "")125126127def month_year(cell: str) -> str | None:128 """'Jun 2026' → '2026-06'; 'Reliable knowledge cutoff' cells."""129 if not cell:130 return None131 dt = parse_datetime(cell.strip())132 if dt and re.search(r"[A-Za-z]{3,9}\.?,? (\d{1,2},? )?\d{4}", cell):133 return f"{dt.year:04d}-{dt.month:02d}"134 if re.fullmatch(r"\d{4}-\d{2}(-\d{2})?", cell.strip()):135 return cell.strip()136 return None137138139def model_ref(facts: Facts, name: str, org: EntityRef, *, api_id: str | None = None, provider_key: str | None = None, family: str | None = None,140 aliases: list[str] | None = None, identifiers: dict[str, str] | None = None) -> EntityRef:141 """Official model EntityRef (identity = the lab's API id when given). `family` = the lab's own family label ("Claude", "Gemini") — when142 absent the ontology infers the versioned family ("Llama 3.1", "Qwen3") from the name; both become a `model_family` hint."""143 ids: dict[str, str] = dict(identifiers or {})144 if api_id and provider_key:145 ids[f"{provider_key}_model_id"] = api_id146 fam = EntityRef(entity_type="model_family", name=family, organization=org, identifiers={"family_key": f"{re.sub(r'[^a-z0-9.]+', '-', family.lower()).strip('-')}@{org.slug_hint}"}) if family else family_ref(name, org)147 ref = facts.entity("model", name, identifiers=ids, organization=org, aliases=aliases or [], family=fam, identity_confidence="high" if ids else "medium")148 facts.relate(org, "develops", ref)149 if family:150 facts.claim(ref, "family", family)151 elif fam is not None:152 facts.claim(ref, "family", fam.name)153 return ref154155156# ---------------------------------------------------------------------------------------------- canonical vocabularies for lab docs157# source label (lower-cased, punctuation-insensitive) → canonical capability slug158CAPABILITY_MAP: dict[str, str] = {159 "function calling": "function_calling", "function_calling": "function_calling", "tool calling": "function_calling", "tool_calling": "function_calling",160 "tool use": "function_calling", "tool_use": "function_calling", "tools": "function_calling", "native tool use": "function_calling",161 "structured outputs": "structured_output", "structured_outputs": "structured_output", "structured output": "structured_output", "json mode": "structured_output",162 "json_mode": "structured_output", "json output": "structured_output", "response_format": "structured_output",163 "reasoning": "reasoning", "thinking": "reasoning", "extended thinking": "reasoning", "extended_thinking": "reasoning", "adaptive thinking": "reasoning",164 "vision": "vision", "image understanding": "vision", "image input": "vision", "image_input": "vision", "image inputs": "vision",165 "audio understanding": "audio_input", "audio input": "audio_input", "audio_input": "audio_input", "speech input": "audio_input",166 "audio generation": "audio_output", "audio output": "audio_output", "audio_output": "audio_output", "speech generation": "audio_output", "text to speech": "audio_output",167 "image generation": "image_generation", "image_generation": "image_generation", "video generation": "video_generation", "video_generation": "video_generation",168 "code execution": "code_execution", "code_execution": "code_execution", "code interpreter": "code_execution", "code_interpreter": "code_execution",169 "grounding with google search": "search_grounding", "search grounding": "search_grounding", "search_grounding": "search_grounding", "web search": "search_grounding",170 "web_search": "search_grounding", "grounding with google maps": "maps_grounding",171 "caching": "caching", "prompt caching": "caching", "prompt_caching": "caching", "context caching": "caching", "context_caching": "caching",172 "batch api": "batch", "batch": "batch", "batch_api": "batch", "batch mode": "batch",173 "fine tuning": "fine_tuning", "fine-tuning": "fine_tuning", "fine_tuning": "fine_tuning", "tuning": "fine_tuning", "supervised fine-tuning": "fine_tuning",174 "streaming": "streaming", "live api": "live_api", "live_api": "live_api", "realtime": "realtime", "computer use": "computer_use", "computer_use": "computer_use",175 "file search": "file_search", "file_search": "file_search", "url context": "url_context", "url_context": "url_context", "mcp": "mcp", "distillation": "distillation",176 "predicted outputs": "predicted_outputs", "predicted_outputs": "predicted_outputs", "embeddings": "embeddings", "flex inference": "flex_inference",177 "priority inference": "priority_inference", "citations": "citations", "pdf support": "document_input", "document understanding": "document_input", "files api": "document_input",178 "web fetch": "web_fetch", "memory": "memory", "agent skills": "agent_skills", "evals": "evals",179}180181182def normalize_capability(raw: str) -> str:183 s = re.sub(r"\s+", " ", str(raw).strip().lower())184 return CAPABILITY_MAP.get(s) or CAPABILITY_MAP.get(s.replace("_", " ")) or CAPABILITY_MAP.get(s.replace("-", " ")) or re.sub(r"[^a-z0-9]+", "_", s).strip("_")185186187def normalize_capabilities(values: list[str] | None) -> list[str]:188 """Source labels / slugs → sorted canonical capability slugs (unknown labels are kept as slugs, never dropped)."""189 if not values:190 return []191 return sorted({normalize_capability(v) for v in values if str(v).strip()})192193194def claim_license(facts: Facts, ref: EntityRef, raw: str | None, *, weights_available: bool | None = None) -> str | None:195 """`license` = ontology key (raw label kept in `license_raw`); with `weights_available` the openness category is derived, never asserted."""196 if not raw:197 return None198 key = normalize_license(raw)199 facts.claim(ref, "license_raw", raw)200 facts.claim(ref, "license", key or raw)201 if weights_available is not None:202 if weights_available:203 facts.claim(ref, "weights_available", True)204 dims = openness_dimensions(weights_available=weights_available, license_key=key)205 facts.claim(ref, "openness", derive_openness(dims, license_key=key))206 return key207208209def claim_status(facts: Facts, ref: EntityRef, raw: str | None) -> str | None:210 if not raw:211 return None212 canon = normalize_status(raw)213 facts.claim(ref, "status", canon or raw)214 if canon and canon != str(raw).strip().lower():215 facts.claim(ref, "status_raw", raw)216 return canon217218219def claim_api_aliases(facts: Facts, ref: EntityRef, aliases: list[str] | str | None) -> None:220 """`api_aliases` is always a list; `api_alias` (the first one) is kept for compatibility."""221 items = [aliases] if isinstance(aliases, str) else list(aliases or [])222 items = [a.strip() for a in items if isinstance(a, str) and a.strip()]223 if not items:224 return225 facts.claim(ref, "api_aliases", sorted(dict.fromkeys(items)))226 facts.claim(ref, "api_alias", items[0])227228229def first_target_with(targets: list[Target], key: str) -> Target | None:230 return next((t for t in targets if t.key == key), None)231232233def iso_date(dt: datetime | None) -> str | None:234 return dt.date().isoformat() if dt else None235236237_FLIGHT_CHUNK = re.compile(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)', re.DOTALL)238239240def next_flight_payload(content: bytes | str) -> str:241 """Concatenate the React Server Components payload a Next.js (App Router) page streams through `self.__next_f.push([1, "…"])`.242 The chunks are JS string literals, so they decode as JSON strings. Returns '' when the page is not a Next.js RSC page."""243 import json244245 html = content.decode("utf-8", errors="replace") if isinstance(content, bytes) else content246 parts: list[str] = []247 for m in _FLIGHT_CHUNK.finditer(html):248 try:249 parts.append(json.loads(f'"{m.group(1)}"'))250 except ValueError:251 continue252 return "".join(parts)253254255def json_after(text: str, marker: str) -> Any | None:256 """Parse the JSON object/array that starts right after `marker` (brace-matched, string-aware)."""257 import json258259 i = text.find(marker)260 if i < 0:261 return None262 j = i + len(marker)263 while j < len(text) and text[j] not in "{[":264 j += 1265 if j >= len(text):266 return None267 opener = text[j]268 closer = "}" if opener == "{" else "]"269 depth = 0270 in_str = False271 k = j272 while k < len(text):273 ch = text[k]274 if in_str:275 if ch == "\\":276 k += 1277 elif ch == '"':278 in_str = False279 elif ch == '"':280 in_str = True281 elif ch == opener:282 depth += 1283 elif ch == closer:284 depth -= 1285 if depth == 0:286 break287 k += 1288 try:289 return json.loads(text[j:k + 1])290 except ValueError:291 return None292293294def slug_of(url: str) -> str:295 """Last non-empty path segment of a URL, without extension: '/api/docs/models/gpt-4.1.md' → 'gpt-4.1'."""296 from urllib.parse import urlparse297298 path = urlparse(url).path.rstrip("/")299 seg = path.rsplit("/", 1)[-1]300 return re.sub(r"\.(md|mdx|html?)$", "", seg)301302303def claim_modalities(facts: Facts, ref: EntityRef, inputs: Any = None, outputs: Any = None) -> tuple[list[str], list[str]]:304 """Canonical modality lists (`pdf` → document, `Images` → image); `modalities` = union. Only what the page states."""305 mi, mo = normalize_modalities(inputs), normalize_modalities(outputs)306 if mi:307 facts.claim(ref, "modalities_input", mi)308 if mo:309 facts.claim(ref, "modalities_output", mo)310 if mi or mo:311 facts.claim(ref, "modalities", sorted(set(mi) | set(mo)))312 if "image" in mi:313 facts.claim(ref, "vision", True)314 if "audio" in mi:315 facts.claim(ref, "audio", True)316 return mi, mo317318319__all__ = [320 "CAPABILITY_MAP",321 "MODEL_WORDS",322 "RELEASE_WORDS",323 "announcement_events",324 "claim_api_aliases",325 "claim_license",326 "claim_modalities",327 "claim_status",328 "clean_cell",329 "first_target_with",330 "iso_date",331 "json_after",332 "kv_tables",333 "link_in_cell",334 "model_ref",335 "money",336 "month_year",337 "next_flight_payload",338 "normalize_capabilities",339 "normalize_capability",340 "parse_retirement",341 "slug_of",342 "tokens",343 "transpose_feature_table",344]345