Repository Public Research preview
1"""Polite HTTP client: per-host rate limiting, retries with exponential backoff, robots.txt checks.23Uses httpx (already a dependency of the anthropic SDK). Never follows cross-host redirects silently4for robots-protected hosts. API endpoints (S3, OAI-PMH, Crossref, arXiv API) are consulted under their5published usage policies; see docs/data_governance/LICENSE_MATRIX.md.6"""7from __future__ import annotations89import threading10import time11import urllib.robotparser as robotparser12from dataclasses import dataclass, field13from urllib.parse import urlparse1415import httpx1617USER_AGENT = "HTR-research/0.1 (UQO WP-AI-0002; academic text-mining; contact via project repository)"181920@dataclass21class HostPolicy:22 min_interval_s: float = 0.34 # ~3 req/s default23 check_robots: bool = False242526DEFAULT_POLICIES: dict[str, HostPolicy] = {27 "pmc-oa-opendata.s3.amazonaws.com": HostPolicy(0.05, False), # AWS Open Data, no published quota28 "export.arxiv.org": HostPolicy(3.0, False), # arXiv API ToU: 1 request / 3 s29 "arxiv.org": HostPolicy(3.0, False),30 "oaipmh.arxiv.org": HostPolicy(3.0, False),31 "api.crossref.org": HostPolicy(1.0, False), # polite pool, mailto in UA32 "eutils.ncbi.nlm.nih.gov": HostPolicy(0.34, False), # 3 req/s without key33}343536@dataclass37class RateLimiter:38 policies: dict[str, HostPolicy] = field(default_factory=lambda: dict(DEFAULT_POLICIES))39 _last: dict[str, float] = field(default_factory=dict)40 _lock: threading.Lock = field(default_factory=threading.Lock)4142 def wait(self, host: str) -> None:43 pol = self.policies.get(host, HostPolicy())44 with self._lock:45 now = time.monotonic()46 last = self._last.get(host, 0.0)47 delay = pol.min_interval_s - (now - last)48 if delay > 0:49 time.sleep(delay)50 self._last[host] = time.monotonic()515253class PoliteClient:54 def __init__(self, timeout: float = 60.0, max_retries: int = 5, limiter: RateLimiter | None = None, mailto: str | None = None):55 ua = USER_AGENT + (f" mailto:{mailto}" if mailto else "")56 self._client = httpx.Client(timeout=timeout, headers={"User-Agent": ua}, follow_redirects=True)57 self.max_retries = max_retries58 self.limiter = limiter or RateLimiter()59 self._robots: dict[str, robotparser.RobotFileParser | None] = {}60 self.stats = {"requests": 0, "retries": 0, "bytes": 0}6162 def _allowed_by_robots(self, url: str) -> bool:63 host = urlparse(url).netloc64 if host not in self._robots:65 rp = robotparser.RobotFileParser()66 try:67 rp.set_url(f"{urlparse(url).scheme}://{host}/robots.txt")68 rp.read()69 self._robots[host] = rp70 except Exception: # noqa: BLE00171 self._robots[host] = None72 rp = self._robots[host]73 return True if rp is None else rp.can_fetch(USER_AGENT, url)7475 def get(self, url: str, *, params: dict | None = None, check_robots: bool | None = None, accept: str | None = None) -> httpx.Response:76 host = urlparse(url).netloc77 pol = self.limiter.policies.get(host, HostPolicy())78 if (check_robots if check_robots is not None else pol.check_robots) and not self._allowed_by_robots(url):79 raise PermissionError(f"robots.txt disallows {url}")80 headers = {"Accept": accept} if accept else None81 last_exc: Exception | None = None82 for attempt in range(self.max_retries + 1):83 self.limiter.wait(host)84 try:85 self.stats["requests"] += 186 r = self._client.get(url, params=params, headers=headers)87 if r.status_code in (429, 500, 502, 503, 504):88 retry_after = r.headers.get("Retry-After")89 wait = float(retry_after) if retry_after and retry_after.isdigit() else min(60.0, 2.0**attempt)90 time.sleep(wait)91 self.stats["retries"] += 192 continue93 self.stats["bytes"] += len(r.content)94 return r95 except (httpx.TransportError, httpx.TimeoutException) as e: # pragma: no cover - network96 last_exc = e97 time.sleep(min(60.0, 2.0**attempt))98 self.stats["retries"] += 199 raise RuntimeError(f"GET {url} failed after {self.max_retries} retries: {last_exc}")100101 def close(self) -> None:102 self._client.close()103