spb/ai-atlas

1"""Apple Machine Learning Research — RSS feed → `paper` entities (title, date, abstract, URL) + ANNOUNCEMENT events.23Each research page is followed deterministically (no LLM) to pick up the arXiv / PDF links and the author list when present.4"""5from __future__ import annotations67import re89from aiatlas.registry import org_ref10from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext11from aiatlas.sdk.extract.html import node_text12from aiatlas.sdk.facts import EntityRef, Facts, Target13from aiatlas.sdk.fetch import FetchResult1415from ._common import announcement_events1617FEED = "https://machinelearning.apple.com/rss.xml"18ARXIV = re.compile(r"arxiv\.org/(?:abs|pdf)/(\d{4}\.\d{4,5})(?:v\d+)?")192021class AppleMLConnector(BaseConnector):22    name = "apple_ml"23    label = "Apple Machine Learning Research — feed"24    description = "Apple Machine Learning Research RSS: research items as paper entities (+ arXiv links from the research pages) and events."25    source_key = "machinelearning.apple.com"26    version = "1"27    parser_version = "1"28    interval_seconds = 2160029    min_interval_seconds = 720030    rate_per_min = 1031    tier = 132    priority = 133    expected_min_records = 534    concurrency = 23536    async def discover(self, ctx: RunContext) -> list[Target]:37        return [Target(url=FEED, doc_type="feed", key="feed", min_bytes=500)]3839    async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:40        facts = Facts()41        org = org_ref("apple")42        facts.entities.append(org)43        if parsed.kind == "feed":44            for it in parsed.feed_items:45                if not it.url or not it.title:46                    continue47                paper = facts.entity("paper", it.title, identifiers={"url": it.url.rstrip("/")}, organization=org)48                facts.claim(paper, "published_at", it.published_at.date().isoformat() if it.published_at else None)49                facts.claim(paper, "abstract", (it.summary or "")[:2000] or None)50                facts.claim(paper, "authors", it.authors or None)51                facts.claim(paper, "paper_url", it.url)52                facts.relate(paper, "published_by", org)53                facts.follow(it.url, doc_type="paper_page", entity=paper, key=f"paper:{it.url.rstrip('/').rsplit('/', 1)[-1]}", priority=2, min_bytes=1000)54            announcement_events(facts, org, parsed.feed_items, source_name="machinelearning.apple.com", follow=False, importance_default=1)55            facts.document_title, facts.document_entity = "Apple Machine Learning Research", org56        elif target.doc_type == "paper_page" and parsed.html:57            url = (res.final_url or res.url).rstrip("/")58            title = next((t for lvl, t in parsed.html.headings if lvl == 1), None) or (parsed.html.title or "").split(" - Apple")[0].strip()59            paper = target.entity or (EntityRef(entity_type="paper", name=title, identifiers={"url": url}, organization=org) if title else None)60            if paper is not None:61                self._paper_page(facts, org, paper, parsed)62        return facts6364    def _paper_page(self, facts: Facts, org: EntityRef, paper: EntityRef, parsed: Parsed) -> None:65        html = parsed.html66        assert html67        if paper not in facts.entities:68            facts.entities.append(paper)69            facts.relate(paper, "published_by", org)70        facts.document_entity = paper71        facts.document_title = html.title72        for href, _text in html.links:73            m = ARXIV.search(href)74            if m:75                paper.identifiers.setdefault("arxiv", m.group(1))76                facts.claim(paper, "arxiv_id", m.group(1))77                facts.claim(paper, "pdf_url", f"https://arxiv.org/pdf/{m.group(1)}")78                break79        else:80            pdf = next((h for h, _t in html.links if h.lower().endswith(".pdf")), None)81            facts.claim(paper, "pdf_url", pdf)82        # <p><span class="a11y">Authors</span>Name‡, Name†**, …</p> — footnote marks (affiliations) stripped83        authors: list[str] = []84        for p in html.css("p"):85            span = p.css_first("span.a11y")86            if span is None or node_text(span).lower() != "authors":87                continue88            text = node_text(p)[len(node_text(span)):]89            for raw in text.split(","):90                name_ = re.sub(r"[‡†§*¶#0-9]+", "", raw).strip()91                if 3 <= len(name_) <= 60 and name_ not in authors:92                    authors.append(name_)93            break94        if authors:95            facts.claim(paper, "authors", authors[:40])96        # publication date comes from the feed (day precision); the page only states the month979899100CONNECTORS = [AppleMLConnector]101