"""OpenReview — conference submissions (ICLR / NeurIPS / ICML). Tier 1 (the venue's own records). Status (probed 2026-09-11): both public JSON endpoints answer **HTTP 403 `ChallengeRequiredError`** ("Challenge verification required", redirecting to https://openreview.net/challenge) for our identified UA *and* for a browser-like UA with browser Accept headers: https://api.openreview.net/notes?invitation=ICLR.cc/2026/Conference/-/Blind_Submission&limit=5 https://api2.openreview.net/notes?content.venueid=ICLR.cc/2026/Conference&limit=5 The challenge is an access control, so we do not bypass it: the connector ships **disabled by default** (`enabled_by_default = False`) and is exercised against a fixture built from the documented API v2 response shape (`{"notes": [{id, forum, cdate, pdate, mdate, content: {title: {value}, authors: {value: [...]}, abstract: {value}, keywords: {value}, venue: {value}, venueid: {value}, pdf: {value}}}], "count": N}` — see https://docs.openreview.net/reference/api-v2). Enable it (`aia connectors` / `update connectors set enabled = true`) once OpenReview serves JSON to identified crawlers again; `discover()` already emits the right targets. """ from __future__ import annotations from datetime import UTC, datetime from typing import Any from urllib.parse import quote from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext from aiatlas.sdk.facts import EntityRef, Facts, Target from aiatlas.sdk.fetch import FetchResult API2 = "https://api2.openreview.net/notes" DEFAULT_VENUES = ["ICLR.cc/2026/Conference", "NeurIPS.cc/2025/Conference", "ICML.cc/2026/Conference"] BROWSER_ACCEPT = "application/json, text/plain, */*" class OpenReviewConnector(BaseConnector): name = "openreview" label = "OpenReview — accepted/submitted papers of recent ML conferences" description = "Public notes of the configured venue ids (API v2 JSON). Disabled by default: the endpoint currently requires a browser challenge." source_key = "openreview.net" version = "2" parser_version = "2" interval_seconds = 86400 min_interval_seconds = 12 * 3600 max_interval_seconds = 7 * 86400 rate_per_min = 10 tier = 1 priority = 2 expected_min_records = 20 concurrency = 1 enabled_by_default = False async def discover(self, ctx: RunContext) -> list[Target]: venues = self.config.get("venues") or DEFAULT_VENUES limit = int(self.config.get("limit", 100)) pages = int(self.config.get("pages", 3)) out: list[Target] = [] for venue in venues: for p in range(pages): url = f"{API2}?content.venueid={quote(venue, safe='')}&limit={limit}&offset={p * limit}&sort=cdate:desc" out.append(Target(url=url, doc_type="listing", key=f"notes:{venue}:{p}", accept=BROWSER_ACCEPT, min_bytes=20, meta={"venue": venue, "page": p, "content_type": "application/json"})) return out async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts: facts = Facts() data = parsed.json if parsed.kind == "json" else None if not isinstance(data, dict): return facts notes = data.get("notes") or [] for note in notes: self._note(facts, note, target.meta.get("venue")) facts.document_title = f"OpenReview notes — {target.meta.get('venue') or ''}".strip() return facts def _note(self, facts: Facts, note: dict[str, Any], venue_hint: str | None) -> None: content = note.get("content") or {} title = _val(content.get("title")) note_id = note.get("id") if not note_id or not isinstance(title, str) or not title.strip(): return authors = _val(content.get("authors")) or [] authors = [a for a in authors if isinstance(a, str)] if isinstance(authors, list) else [] pdate = _epoch_ms(note.get("pdate")) or _epoch_ms(note.get("cdate")) ref = facts.entity("paper", title.strip()[:300], identifiers={"openreview": note_id}, first_seen_hint=pdate) facts.claim(ref, "openreview_id", note_id) facts.claim(ref, "authors", authors[:100]) facts.claim(ref, "abstract", (_val(content.get("abstract")) or "")[:6000] or None) facts.claim(ref, "keywords", _val(content.get("keywords")) if isinstance(_val(content.get("keywords")), list) else None) facts.claim(ref, "venue", _val(content.get("venue")) or venue_hint) facts.claim(ref, "venue_id", _val(content.get("venueid")) or venue_hint) facts.claim(ref, "published_at", pdate.isoformat(timespec="seconds") if pdate else None) mdate = _epoch_ms(note.get("mdate")) facts.claim(ref, "updated_at", mdate.isoformat(timespec="seconds") if mdate else None) pdf = _val(content.get("pdf")) facts.claim(ref, "pdf_url", f"https://openreview.net{pdf}" if isinstance(pdf, str) and pdf.startswith("/") else pdf if isinstance(pdf, str) else None) facts.claim(ref, "official_url", f"https://openreview.net/forum?id={note.get('forum') or note_id}") tldr = _val(content.get("TLDR")) or _val(content.get("tldr")) facts.claim(ref, "tldr", tldr if isinstance(tldr, str) else None) primary = _val(content.get("primary_area")) facts.claim(ref, "primary_area", primary if isinstance(primary, str) else None) # researchers only with an identifier: OpenReview profile ids (`~Ada_Placeholder1`) — name-only authors stay a claim on the paper author_ids = _val(content.get("authorids")) or [] author_ids = [a for a in author_ids if isinstance(a, str)] if isinstance(author_ids, list) else [] for i, pid in enumerate(author_ids[:20]): if not pid.startswith("~"): continue name = authors[i] if i < len(authors) else pid.strip("~").replace("_", " ").rstrip("0123456789") person = EntityRef(entity_type="researcher", name=name[:200], identifiers={"openreview_profile": pid}, aliases=[pid], identity_confidence="high") facts.entities.append(person) facts.claim(person, "openreview_profile_url", f"https://openreview.net/profile?id={pid}") facts.relate(person, "authored", ref, attributes={"position": i + 1}) def _val(field: Any) -> Any: """API v2 wraps every content field as {"value": …}; API v1 returns bare values.""" if isinstance(field, dict) and "value" in field: return field["value"] return field def _epoch_ms(v: Any) -> datetime | None: if isinstance(v, (int, float)) and v > 0: return datetime.fromtimestamp(v / 1000 if v > 1e11 else v, tz=UTC) return None CONNECTORS = [OpenReviewConnector]