HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""arXiv — Atom API listing (public document, not a commercial API) + category RSS feeds. Tier 1 (the papers' own metadata).23 * https://rss.arxiv.org/rss/cs.LG (+ cs.CL, cs.AI, cs.CV) — daily announcements (new / cross / replace), ~250–600 papers per feed.4 * https://export.arxiv.org/api/query?search_query=cat:cs.LG+OR+cat:cs.CL+OR+cat:cs.AI+OR+cat:cs.CV&sortBy=submittedDate&sortOrder=descending5 paginated with `start`/`max_results=200` (≤ 5 pages) — arXiv asks for ≤ 1 request every 3 s → 4/min. **Opt-in** (`config["atom_pages"] > 0`):6 on 2026-09-11 `https://export.arxiv.org/robots.txt` is `User-agent: * / Disallow: /`, so the SDK's robots policy blocks the Atom API7 (3 `blocked_source` review items per run). The RSS feeds are not disallowed and already exceed `expected_min_records`.89Paper entities are keyed by the version-less arXiv id (`{"arxiv": "2509.01234"}`). Authors are a claim on the paper; a `researcher`10entity is created only when the feed carries an identifier (ORCID) — never from a bare name. PDFs are not fetched (`needs_llm=False`);11a later job builds paper passports.12"""13from __future__ import annotations1415import re16from datetime import UTC, datetime17from typing import Any1819import feedparser2021from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext22from aiatlas.sdk.extract.dates import parse_datetime23from aiatlas.sdk.extract.html import clean_text24from aiatlas.sdk.facts import EntityRef, Facts, Target25from aiatlas.sdk.fetch import FetchResult2627ATOM = "https://export.arxiv.org/api/query"28RSS = "https://rss.arxiv.org/rss/{cat}"29DEFAULT_QUERY = "cat:cs.LG+OR+cat:cs.CL+OR+cat:cs.AI+OR+cat:cs.CV"30ARXIV_ID = re.compile(r"(\d{4}\.\d{4,5})(v\d+)?")31OLD_ID = re.compile(r"([a-z-]+(?:\.[A-Z]{2})?/\d{7})(v\d+)?")32ANNOUNCE = re.compile(r"^arXiv:\S+\s+Announce Type:\s*(\w+)\s*", re.IGNORECASE)333435class ArxivConnector(BaseConnector):36 name = "arxiv"37 label = "arXiv — recent cs.LG / cs.CL / cs.AI / cs.CV papers (Atom API + RSS)"38 description = "Atom listing of the newest submissions in the core ML categories and the daily category RSS feeds."39 source_key = "arxiv.org"40 version = "2"41 parser_version = "2"42 interval_seconds = 6 * 360043 min_interval_seconds = 2 * 360044 max_interval_seconds = 8640045 rate_per_min = 446 # Documented exception: export.arxiv.org/robots.txt disallows generic crawlers, but arXiv's API Terms of Use designate this host47 # for programmatic access (≤ 1 request / 3 s). We stay at 4 requests / min, identify ourselves and honour the rate.48 respect_robots = False49 tier = 150 priority = 151 expected_min_records = 10052 concurrency = 153 needs_llm = False5455 async def discover(self, ctx: RunContext) -> list[Target]:56 pages = min(5, max(0, int(self.config.get("atom_pages", self.config.get("pages", 0)))))57 per_page = int(self.config.get("max_results", 200))58 query = self.config.get("query", DEFAULT_QUERY)59 targets = [Target(url=f"{ATOM}?search_query={query}&sortBy=submittedDate&sortOrder=descending&start={i * per_page}&max_results={per_page}",60 doc_type="feed", key=f"atom:{i}", min_bytes=2000, priority=1, meta={"page": i}) for i in range(pages)]61 for cat in self.config.get("rss_categories", ["cs.LG", "cs.CL", "cs.AI", "cs.CV"]):62 targets.append(Target(url=RSS.format(cat=cat), doc_type="feed", key=f"rss:{cat}", min_bytes=2000, priority=2, meta={"category": cat}))63 return targets6465 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:66 facts = Facts()67 feed = feedparser.parse(res.content)68 is_rss = (target.key or "").startswith("rss:")69 for e in feed.entries:70 self._paper(facts, e, is_rss=is_rss)71 facts.document_title = clean_text(feed.feed.get("title") or "") or None72 return facts7374 def _paper(self, facts: Facts, e: Any, *, is_rss: bool) -> None:75 raw_id = e.get("id") or e.get("link") or ""76 arxiv_id = _arxiv_id(raw_id) or _arxiv_id(e.get("link") or "")77 title = clean_text(re.sub(r"\s+", " ", e.get("title") or "")).strip()78 if not arxiv_id or not title:79 return80 summary = e.get("summary") or ""81 announce = None82 if is_rss:83 m = ANNOUNCE.match(summary)84 if m:85 announce = m.group(1).lower()86 summary = summary[m.end():]87 summary = re.sub(r"^Abstract:\s*", "", summary.strip(), flags=re.IGNORECASE)88 abstract = clean_text(re.sub(r"\s+", " ", summary)).strip()89 published = _dt(e.get("published_parsed")) or parse_datetime(e.get("published"))90 updated = (_dt(e["updated_parsed"]) if "updated_parsed" in e else None) or (parse_datetime(e["updated"]) if "updated" in e else None)91 authors = _authors(e)92 categories = [t.get("term") for t in e.get("tags", []) if isinstance(t, dict) and t.get("term")]93 primary = (e.get("arxiv_primary_category") or {}).get("term") if isinstance(e.get("arxiv_primary_category"), dict) else None94 ref = facts.entity("paper", title[:300], identifiers={"arxiv": arxiv_id}, first_seen_hint=published)95 facts.claim(ref, "arxiv_id", arxiv_id)96 facts.claim(ref, "authors", authors[:100])97 facts.claim(ref, "published_at", published.isoformat(timespec="seconds") if published else None)98 if updated and (not published or updated != published):99 facts.claim(ref, "updated_at", updated.isoformat(timespec="seconds"))100 facts.claim(ref, "abstract", abstract[:6000] or None)101 facts.claim(ref, "categories", categories)102 facts.claim(ref, "primary_category", primary or (categories[0] if categories else None))103 facts.claim(ref, "pdf_url", f"https://arxiv.org/pdf/{arxiv_id}")104 facts.claim(ref, "official_url", f"https://arxiv.org/abs/{arxiv_id}")105 doi = e.get("arxiv_doi")106 facts.claim(ref, "doi", doi.strip() if isinstance(doi, str) and doi.strip() else None)107 comment = e.get("arxiv_comment")108 facts.claim(ref, "comment", clean_text(comment)[:1000] if isinstance(comment, str) and comment.strip() else None)109 jref = e.get("arxiv_journal_ref")110 facts.claim(ref, "journal_ref", clean_text(jref)[:500] if isinstance(jref, str) and jref.strip() else None)111 if announce:112 facts.claim(ref, "arxiv_announce_type", announce)113 # researchers only with an identifier: arXiv feeds carry no ORCID / author id, so authors stay a claim on the paper (homonyms114 # would merge and spelling variants split name-only entities). An ORCID in the feed (rare `arxiv:author` extension) creates one.115 for author in (e.get("authors") or [])[:20]:116 orcid = _orcid(author)117 if orcid:118 person = EntityRef(entity_type="researcher", name=(author.get("name") or orcid)[:200], identifiers={"orcid": orcid}, identity_confidence="high")119 facts.entities.append(person)120 facts.relate(person, "authored", ref)121 if doi and isinstance(doi, str) and doi.strip():122 ref.identifiers["doi"] = doi.strip()123124125ORCID = re.compile(r"(\d{4}-\d{4}-\d{4}-\d{3}[\dX])")126127128def _orcid(author: Any) -> str | None:129 if not isinstance(author, dict):130 return None131 for key in ("orcid", "arxiv_orcid", "uri", "href", "id"):132 v = author.get(key)133 if isinstance(v, str) and "orcid.org" in v.lower() or (isinstance(v, str) and key == "orcid"):134 m = ORCID.search(v)135 if m:136 return m.group(1)137 return None138139140def _arxiv_id(s: str) -> str | None:141 m = ARXIV_ID.search(s)142 if m:143 return m.group(1)144 m = OLD_ID.search(s.replace("http://arxiv.org/abs/", "").replace("https://arxiv.org/abs/", ""))145 return m.group(1) if m else None146147148def _authors(e: Any) -> list[str]:149 names: list[str] = []150 for a in e.get("authors", []) or []:151 n = a.get("name") if isinstance(a, dict) else None152 if n:153 names.extend(_split_names(n))154 if not names and e.get("author"):155 names = _split_names(e["author"])156 seen: set[str] = set()157 out: list[str] = []158 for n in names:159 n = clean_text(n).strip()160 if n and n.lower() not in seen:161 seen.add(n.lower())162 out.append(n)163 return out164165166def _split_names(s: str) -> list[str]:167 # RSS dc:creator packs every author into one string: "Wenzhe Jin, Haina Tang"168 parts = re.split(r",\s+|\s+and\s+", s)169 return [p.strip() for p in parts if p.strip()]170171172def _dt(struct: Any) -> datetime | None:173 if not struct:174 return None175 try:176 return datetime(*struct[:6], tzinfo=UTC)177 except Exception: # noqa: BLE001178 return None179180181CONNECTORS = [ArxivConnector]182