"""Shared helpers for lab connectors: announcement events from feeds/listings, key/value tables, model-name normalisation.""" from __future__ import annotations import re from datetime import datetime from typing import Any from aiatlas.sdk.extract.dates import parse_datetime from aiatlas.sdk.extract.feeds import FeedItem from aiatlas.sdk.facts import EntityRef, Facts, Target RELEASE_WORDS = re.compile(r"\b(introducing|announcing|launch(?:ing|es|ed)?|releas(?:e|es|ing|ed)|now available|new model|preview|deprecat|pricing|price)\b", re.I) MODEL_WORDS = re.compile(r"\b(model|gpt|claude|gemini|llama|mistral|qwen|deepseek|grok|command|phi|nemotron|gemma|sonnet|opus|haiku|o\d|codex|sora|veo|imagen|whisper|" r"embedding|reasoning|agent)\w*", re.I) def announcement_events(facts: Facts, org: EntityRef, items: list[FeedItem], *, source_name: str, follow: bool = True, max_follow: int = 30, needs_llm: bool = True, importance_default: int = 1) -> int: """One ANNOUNCEMENT event per feed item (deduped by URL) and, optionally, a follow-up fetch of the article for LLM extraction.""" n = 0 for i, it in enumerate(items): if not it.url or not it.title: continue text = f"{it.title} {it.summary or ''}" is_release = bool(RELEASE_WORDS.search(text) and MODEL_WORDS.search(text)) importance = 2 if is_release else importance_default facts.event("ANNOUNCEMENT", "release" if is_release else "company", f"{org.name}: {it.title}", entity=org, importance=importance, effective_at=it.published_at, dedupe_key=f"ANNOUNCEMENT:{it.url}", source_url=it.url, meta={"source": source_name, "categories": it.categories[:5], "summary": (it.summary or "")[:300], "is_release": is_release}) if follow and i < max_follow: facts.follow(it.url, doc_type="news", needs_llm=needs_llm and is_release, priority=1 if is_release else 3, meta={"llm_task": "release_announcement", "published_at": it.published_at.isoformat() if it.published_at else None, "title": it.title}) n += 1 return n def kv_tables(tables: list[dict[str, Any]]) -> dict[str, str]: """Merge 2-column key/value tables into one dict (lower-cased keys).""" out: dict[str, str] = {} for t in tables: rows = t.get("rows") or [] headers = t.get("headers") or [] if headers and len(headers) == 2 and rows: for r in rows: if len(r) >= 2 and r[0]: out.setdefault(r[0].strip().lower(), r[1].strip()) elif rows and all(len(r) == 2 for r in rows): for r in rows: if r[0]: out.setdefault(r[0].strip().lower(), r[1].strip()) return out def transpose_feature_table(table: dict[str, Any]) -> dict[str, dict[str, str]]: """Feature-comparison table (first column = feature, other columns = models) → {model: {feature: value}}.""" headers = table.get("headers") or [] if len(headers) < 2: return {} models = [clean_cell(h) for h in headers[1:]] out: dict[str, dict[str, str]] = {m: {} for m in models if m} for row in table.get("rows") or []: if not row: continue feature = clean_cell(row[0]).lower() for m, val in zip(models, row[1:], strict=False): if m: out[m][feature] = clean_cell(val) return out _LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") _CODE = re.compile(r"`([^`]*)`") def clean_cell(s: str) -> str: s = _LINK.sub(r"\1", s or "") s = _CODE.sub(r"\1", s) s = re.sub(r"\*\*|__", "", s) s = re.sub(r"<[^>]+>", " ", s) return re.sub(r"\s+", " ", s).strip() def link_in_cell(s: str) -> str | None: m = re.search(r"\]\((https?://[^)\s]+)\)", s or "") return m.group(1) if m else None def parse_retirement(text: str) -> tuple[str | None, bool]: """'Not sooner than September 1, 2027' → ('2027-09-01', True=tentative).""" if not text or text.strip().lower() in ("n/a", "—", "-", ""): return None, False tentative = "not sooner" in text.lower() or "tentative" in text.lower() m = re.search(r"([A-Z][a-z]+ \d{1,2}, \d{4}|\d{4}-\d{2}-\d{2})", text) if not m: return None, tentative dt = parse_datetime(m.group(1)) return (dt.date().isoformat() if dt else None), tentative def money(cell: str) -> float | None: """'$10 / MTok', '$0.25 / MTok1' (footnote), '$12.50 per 1M tokens' → 10.0 …""" if not cell: return None m = re.search(r"\$\s*(\d+(?:\.\d+)?)", cell.replace(",", "")) if not m: return None val = float(m.group(1)) low = cell.lower() if re.search(r"/\s*1?k\b|per\s*1?k\b|1,000 tokens", low): return val * 1000 return val def tokens(cell: str) -> int | None: from aiatlas.sdk.extract.numbers import parse_context_length return parse_context_length(cell or "") def month_year(cell: str) -> str | None: """'Jun 2026' → '2026-06'; 'Reliable knowledge cutoff' cells.""" if not cell: return None dt = parse_datetime(cell.strip()) if dt and re.search(r"[A-Za-z]{3,9}\.? \d{4}", cell): return f"{dt.year:04d}-{dt.month:02d}" if re.fullmatch(r"\d{4}-\d{2}(-\d{2})?", cell.strip()): return cell.strip() return None def model_ref(facts: Facts, name: str, org: EntityRef, *, api_id: str | None = None, provider_key: str | None = None, family: str | None = None, aliases: list[str] | None = None) -> EntityRef: ids: dict[str, str] = {} if api_id and provider_key: ids[f"{provider_key}_model_id"] = api_id ref = facts.entity("model", name, identifiers=ids, organization=org, aliases=aliases or []) facts.relate(org, "develops", ref) if family: facts.claim(ref, "family", family) return ref def first_target_with(targets: list[Target], key: str) -> Target | None: return next((t for t in targets if t.key == key), None) def iso_date(dt: datetime | None) -> str | None: return dt.date().isoformat() if dt else None __all__ = ["announcement_events", "kv_tables", "transpose_feature_table", "clean_cell", "link_in_cell", "parse_retirement", "money", "tokens", "month_year", "model_ref", "first_target_with", "iso_date", "RELEASE_WORDS", "MODEL_WORDS"]