spb/ai-atlas

1"""OpenReview — conference submissions (ICLR / NeurIPS / ICML). Tier 1 (the venue's own records).23Status (probed 2026-09-11): both public JSON endpoints answer **HTTP 403 `ChallengeRequiredError`** ("Challenge verification required",4redirecting to https://openreview.net/challenge) for our identified UA *and* for a browser-like UA with browser Accept headers:5    https://api.openreview.net/notes?invitation=ICLR.cc/2026/Conference/-/Blind_Submission&limit=56    https://api2.openreview.net/notes?content.venueid=ICLR.cc/2026/Conference&limit=57The challenge is an access control, so we do not bypass it: the connector ships **disabled by default** (`enabled_by_default = False`)8and is exercised against a fixture built from the documented API v2 response shape (`{"notes": [{id, forum, cdate, pdate, mdate,9content: {title: {value}, authors: {value: [...]}, abstract: {value}, keywords: {value}, venue: {value}, venueid: {value}, pdf: {value}}}],10"count": N}` — see https://docs.openreview.net/reference/api-v2). Enable it (`aia connectors` / `update connectors set enabled = true`)11once OpenReview serves JSON to identified crawlers again; `discover()` already emits the right targets.12"""13from __future__ import annotations1415from datetime import UTC, datetime16from typing import Any17from urllib.parse import quote1819from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext20from aiatlas.sdk.facts import EntityRef, Facts, Target21from aiatlas.sdk.fetch import FetchResult2223API2 = "https://api2.openreview.net/notes"24DEFAULT_VENUES = ["ICLR.cc/2026/Conference", "NeurIPS.cc/2025/Conference", "ICML.cc/2026/Conference"]25BROWSER_ACCEPT = "application/json, text/plain, */*"262728class OpenReviewConnector(BaseConnector):29    name = "openreview"30    label = "OpenReview — accepted/submitted papers of recent ML conferences"31    description = "Public notes of the configured venue ids (API v2 JSON). Disabled by default: the endpoint currently requires a browser challenge."32    source_key = "openreview.net"33    version = "2"34    parser_version = "2"35    interval_seconds = 8640036    min_interval_seconds = 12 * 360037    max_interval_seconds = 7 * 8640038    rate_per_min = 1039    tier = 140    priority = 241    expected_min_records = 2042    concurrency = 143    enabled_by_default = False4445    async def discover(self, ctx: RunContext) -> list[Target]:46        venues = self.config.get("venues") or DEFAULT_VENUES47        limit = int(self.config.get("limit", 100))48        pages = int(self.config.get("pages", 3))49        out: list[Target] = []50        for venue in venues:51            for p in range(pages):52                url = f"{API2}?content.venueid={quote(venue, safe='')}&limit={limit}&offset={p * limit}&sort=cdate:desc"53                out.append(Target(url=url, doc_type="listing", key=f"notes:{venue}:{p}", accept=BROWSER_ACCEPT, min_bytes=20,54                                  meta={"venue": venue, "page": p, "content_type": "application/json"}))55        return out5657    async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:58        facts = Facts()59        data = parsed.json if parsed.kind == "json" else None60        if not isinstance(data, dict):61            return facts62        notes = data.get("notes") or []63        for note in notes:64            self._note(facts, note, target.meta.get("venue"))65        facts.document_title = f"OpenReview notes — {target.meta.get('venue') or ''}".strip()66        return facts6768    def _note(self, facts: Facts, note: dict[str, Any], venue_hint: str | None) -> None:69        content = note.get("content") or {}70        title = _val(content.get("title"))71        note_id = note.get("id")72        if not note_id or not isinstance(title, str) or not title.strip():73            return74        authors = _val(content.get("authors")) or []75        authors = [a for a in authors if isinstance(a, str)] if isinstance(authors, list) else []76        pdate = _epoch_ms(note.get("pdate")) or _epoch_ms(note.get("cdate"))77        ref = facts.entity("paper", title.strip()[:300], identifiers={"openreview": note_id}, first_seen_hint=pdate)78        facts.claim(ref, "openreview_id", note_id)79        facts.claim(ref, "authors", authors[:100])80        facts.claim(ref, "abstract", (_val(content.get("abstract")) or "")[:6000] or None)81        facts.claim(ref, "keywords", _val(content.get("keywords")) if isinstance(_val(content.get("keywords")), list) else None)82        facts.claim(ref, "venue", _val(content.get("venue")) or venue_hint)83        facts.claim(ref, "venue_id", _val(content.get("venueid")) or venue_hint)84        facts.claim(ref, "published_at", pdate.isoformat(timespec="seconds") if pdate else None)85        mdate = _epoch_ms(note.get("mdate"))86        facts.claim(ref, "updated_at", mdate.isoformat(timespec="seconds") if mdate else None)87        pdf = _val(content.get("pdf"))88        facts.claim(ref, "pdf_url", f"https://openreview.net{pdf}" if isinstance(pdf, str) and pdf.startswith("/") else pdf if isinstance(pdf, str) else None)89        facts.claim(ref, "official_url", f"https://openreview.net/forum?id={note.get('forum') or note_id}")90        tldr = _val(content.get("TLDR")) or _val(content.get("tldr"))91        facts.claim(ref, "tldr", tldr if isinstance(tldr, str) else None)92        primary = _val(content.get("primary_area"))93        facts.claim(ref, "primary_area", primary if isinstance(primary, str) else None)94        # researchers only with an identifier: OpenReview profile ids (`~Ada_Placeholder1`) — name-only authors stay a claim on the paper95        author_ids = _val(content.get("authorids")) or []96        author_ids = [a for a in author_ids if isinstance(a, str)] if isinstance(author_ids, list) else []97        for i, pid in enumerate(author_ids[:20]):98            if not pid.startswith("~"):99                continue100            name = authors[i] if i < len(authors) else pid.strip("~").replace("_", " ").rstrip("0123456789")101            person = EntityRef(entity_type="researcher", name=name[:200], identifiers={"openreview_profile": pid}, aliases=[pid], identity_confidence="high")102            facts.entities.append(person)103            facts.claim(person, "openreview_profile_url", f"https://openreview.net/profile?id={pid}")104            facts.relate(person, "authored", ref, attributes={"position": i + 1})105106107def _val(field: Any) -> Any:108    """API v2 wraps every content field as {"value": …}; API v1 returns bare values."""109    if isinstance(field, dict) and "value" in field:110        return field["value"]111    return field112113114def _epoch_ms(v: Any) -> datetime | None:115    if isinstance(v, (int, float)) and v > 0:116        return datetime.fromtimestamp(v / 1000 if v > 1e11 else v, tz=UTC)117    return None118119120CONNECTORS = [OpenReviewConnector]121