"""RSS / Atom / JSON Feed connector → news items (title, url, published_at, summary, language). One block per entry keyed by the entry's canonical URL / id so the block diff equals the entry delta. Uses feedparser (bounded input).""" from __future__ import annotations import hashlib from collections.abc import Mapping from typing import Any import feedparser from companyatlas.connectors._util import parse_date, strip_html from companyatlas.fetch import FetchResult from companyatlas.sdk.connector import Connector, ConnectorMeta, register from companyatlas.sdk.models import Block, ExtractedNewsItem, Extraction from companyatlas.sdk.normalize import language_guess, normalized_text, simhash from companyatlas.taxonomy import FetchMode, Surface from companyatlas.urls import canonicalize_url MAX_ENTRIES = 200 CATEGORY_BY_SURFACE = {Surface.NEWSROOM: "press", Surface.BLOG: "blog", Surface.CHANGELOG: "changelog", Surface.RESEARCH: "research", Surface.INVESTOR_RELATIONS: "ir", Surface.FEED: "blog"} def _json_feed_items(data: dict[str, Any]) -> list[ExtractedNewsItem]: out: list[ExtractedNewsItem] = [] for it in (data.get("items") or [])[:MAX_ENTRIES]: if not isinstance(it, dict) or not it.get("title") or not it.get("url"): continue summary = strip_html(it.get("summary") or it.get("content_text") or it.get("content_html") or "")[:600] or None out.append(ExtractedNewsItem(title=str(it["title"])[:300], url=str(it["url"]), published_at=parse_date(it.get("date_published")), summary=summary, language=(it.get("language") or data.get("language") or None))) return out @register class FeedConnector(Connector): meta = ConnectorMeta(connector_id="feed-v1", name="RSS / Atom / JSON feed", version="1", category=Surface.FEED, fetch_mode=FetchMode.FEED, default_interval_s=2 * 3600, surfaces=(), # newsroom/blog/changelog HTML pages stay with generic-html; feeds win by URL or surface url_pattern=r"(/(feed|rss|atom|feeds)(\.xml|\.json|/|$)|\.(rss|atom)$|/rss\.xml|/feed\.xml|/atom\.xml|/index\.xml$|feed\.json$)", priority=45, accept="application/rss+xml,application/atom+xml,application/feed+json,application/xml,text/xml;q=0.9,*/*;q=0.5", description="Syndication feeds → news items") def extract(self, sensor: Mapping[str, Any], result: FetchResult) -> Extraction: surface = str(sensor.get("surface") or Surface.FEED) category = (sensor.get("config") or {}).get("category") or CATEGORY_BY_SURFACE.get(surface, "other") # type: ignore[arg-type] items: list[ExtractedNewsItem] = [] title: str | None = None lang: str | None = None if result.is_json: data = result.json() if isinstance(data, dict): items = _json_feed_items(data) title = data.get("title") lang = data.get("language") else: parsed = feedparser.parse(result.content[: 8 * 1024 * 1024]) if parsed.bozo and not parsed.entries: raise ValueError(f"unparseable feed: {getattr(parsed, 'bozo_exception', 'unknown error')}") title = parsed.feed.get("title") lang = parsed.feed.get("language") for e in parsed.entries[:MAX_ENTRIES]: link = e.get("link") or next((ln.get("href") for ln in e.get("links", []) if ln.get("href")), None) etitle = (e.get("title") or "").strip() if not link or not etitle: continue published = e.get("published") or e.get("updated") or e.get("created") summary = strip_html(e.get("summary") or (e.get("content") or [{}])[0].get("value", ""))[:600] or None items.append(ExtractedNewsItem(title=etitle[:300], url=link, published_at=parse_date(published), summary=summary, language=(e.get("language") or lang or None))) for it in items: it.category = category if not it.language: it.language = language_guess(f"{it.title} {it.summary or ''}") blocks: list[Block] = [] seen: set[str] = set() lines: list[str] = [] for i, it in enumerate(items): canon = canonicalize_url(it.url) if canon in seen: continue seen.add(canon) text = f"{it.title}\n{it.summary or ''}".strip() lines.append(it.title) blocks.append(Block(key=f"entry:{hashlib.blake2b(canon.encode('utf-8'), digest_size=8).hexdigest()}", kind="news_item", text=text, path=title or "Feed", hash=hashlib.sha256(normalized_text(text).encode()).hexdigest()[:16], simhash=simhash(text), weight=1.2, order=i, attrs={"url": canon, "published_at": it.published_at.isoformat() if it.published_at else None})) meta = {"feed_title": title, "entry_count": len(items), "category": category, "structured": True} return Extraction(text="\n".join(lines), blocks=blocks, title=title, language=(lang or "").split("-")[0] or None, meta=meta, news=items) __all__ = ["FeedConnector"]