1"""Shared helpers for lab connectors: announcement events from feeds/listings, key/value tables, model-name normalisation."""2from __future__ import annotations34import re5from datetime import datetime6from typing import Any78from aiatlas.sdk.extract.dates import parse_datetime9from aiatlas.sdk.extract.feeds import FeedItem10from aiatlas.sdk.facts import EntityRef, Facts, Target1112RELEASE_WORDS = re.compile(r"\b(introducing|announcing|launch(?:ing|es|ed)?|releas(?:e|es|ing|ed)|now available|new model|preview|deprecat|pricing|price)\b", re.I)13MODEL_WORDS = re.compile(r"\b(model|gpt|claude|gemini|llama|mistral|qwen|deepseek|grok|command|phi|nemotron|gemma|sonnet|opus|haiku|o\d|codex|sora|veo|imagen|whisper|"14 r"embedding|reasoning|agent)\w*", re.I)151617def announcement_events(facts: Facts, org: EntityRef, items: list[FeedItem], *, source_name: str, follow: bool = True, max_follow: int = 30,18 needs_llm: bool = True, importance_default: int = 1) -> int:19 """One ANNOUNCEMENT event per feed item (deduped by URL) and, optionally, a follow-up fetch of the article for LLM extraction."""20 n = 021 for i, it in enumerate(items):22 if not it.url or not it.title:23 continue24 text = f"{it.title} {it.summary or ''}"25 is_release = bool(RELEASE_WORDS.search(text) and MODEL_WORDS.search(text))26 importance = 2 if is_release else importance_default27 facts.event("ANNOUNCEMENT", "release" if is_release else "company", f"{org.name}: {it.title}", entity=org, importance=importance,28 effective_at=it.published_at, dedupe_key=f"ANNOUNCEMENT:{it.url}", source_url=it.url,29 meta={"source": source_name, "categories": it.categories[:5], "summary": (it.summary or "")[:300], "is_release": is_release})30 if follow and i < max_follow:31 facts.follow(it.url, doc_type="news", needs_llm=needs_llm and is_release, priority=1 if is_release else 3,32 meta={"llm_task": "release_announcement", "published_at": it.published_at.isoformat() if it.published_at else None, "title": it.title})33 n += 134 return n353637def kv_tables(tables: list[dict[str, Any]]) -> dict[str, str]:38 """Merge 2-column key/value tables into one dict (lower-cased keys)."""39 out: dict[str, str] = {}40 for t in tables:41 rows = t.get("rows") or []42 headers = t.get("headers") or []43 if headers and len(headers) == 2 and rows:44 for r in rows:45 if len(r) >= 2 and r[0]:46 out.setdefault(r[0].strip().lower(), r[1].strip())47 elif rows and all(len(r) == 2 for r in rows):48 for r in rows:49 if r[0]:50 out.setdefault(r[0].strip().lower(), r[1].strip())51 return out525354def transpose_feature_table(table: dict[str, Any]) -> dict[str, dict[str, str]]:55 """Feature-comparison table (first column = feature, other columns = models) → {model: {feature: value}}."""56 headers = table.get("headers") or []57 if len(headers) < 2:58 return {}59 models = [clean_cell(h) for h in headers[1:]]60 out: dict[str, dict[str, str]] = {m: {} for m in models if m}61 for row in table.get("rows") or []:62 if not row:63 continue64 feature = clean_cell(row[0]).lower()65 for m, val in zip(models, row[1:], strict=False):66 if m:67 out[m][feature] = clean_cell(val)68 return out697071_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")72_CODE = re.compile(r"`([^`]*)`")737475def clean_cell(s: str) -> str:76 s = _LINK.sub(r"\1", s or "")77 s = _CODE.sub(r"\1", s)78 s = re.sub(r"\*\*|__", "", s)79 s = re.sub(r"<[^>]+>", " ", s)80 return re.sub(r"\s+", " ", s).strip()818283def link_in_cell(s: str) -> str | None:84 m = re.search(r"\]\((https?://[^)\s]+)\)", s or "")85 return m.group(1) if m else None868788def parse_retirement(text: str) -> tuple[str | None, bool]:89 """'Not sooner than September 1, 2027' → ('2027-09-01', True=tentative)."""90 if not text or text.strip().lower() in ("n/a", "—", "-", ""):91 return None, False92 tentative = "not sooner" in text.lower() or "tentative" in text.lower()93 m = re.search(r"([A-Z][a-z]+ \d{1,2}, \d{4}|\d{4}-\d{2}-\d{2})", text)94 if not m:95 return None, tentative96 dt = parse_datetime(m.group(1))97 return (dt.date().isoformat() if dt else None), tentative9899100def money(cell: str) -> float | None:101 """'$10 / MTok', '$0.25 / MTok1' (footnote), '$12.50 per 1M tokens' → 10.0 …"""102 if not cell:103 return None104 m = re.search(r"\$\s*(\d+(?:\.\d+)?)", cell.replace(",", ""))105 if not m:106 return None107 val = float(m.group(1))108 low = cell.lower()109 if re.search(r"/\s*1?k\b|per\s*1?k\b|1,000 tokens", low):110 return val * 1000111 return val112113114def tokens(cell: str) -> int | None:115 from aiatlas.sdk.extract.numbers import parse_context_length116117 return parse_context_length(cell or "")118119120def month_year(cell: str) -> str | None:121 """'Jun 2026' → '2026-06'; 'Reliable knowledge cutoff' cells."""122 if not cell:123 return None124 dt = parse_datetime(cell.strip())125 if dt and re.search(r"[A-Za-z]{3,9}\.? \d{4}", cell):126 return f"{dt.year:04d}-{dt.month:02d}"127 if re.fullmatch(r"\d{4}-\d{2}(-\d{2})?", cell.strip()):128 return cell.strip()129 return None130131132def model_ref(facts: Facts, name: str, org: EntityRef, *, api_id: str | None = None, provider_key: str | None = None, family: str | None = None,133 aliases: list[str] | None = None) -> EntityRef:134 ids: dict[str, str] = {}135 if api_id and provider_key:136 ids[f"{provider_key}_model_id"] = api_id137 ref = facts.entity("model", name, identifiers=ids, organization=org, aliases=aliases or [])138 facts.relate(org, "develops", ref)139 if family:140 facts.claim(ref, "family", family)141 return ref142143144def first_target_with(targets: list[Target], key: str) -> Target | None:145 return next((t for t in targets if t.key == key), None)146147148def iso_date(dt: datetime | None) -> str | None:149 return dt.date().isoformat() if dt else None150151152__all__ = ["announcement_events", "kv_tables", "transpose_feature_table", "clean_cell", "link_in_cell", "parse_retirement", "money", "tokens", "month_year",153 "model_ref", "first_target_with", "iso_date", "RELEASE_WORDS", "MODEL_WORDS"]154