arXiv: documented robots exception for the export API (arXiv API Terms of Use), 4 req/min
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2 changed files +5 −1
modified
registry/sources.yaml
+2 −1
@@ -236,7 +236,8 @@ sources: | |||||
| 236 | rate_limit_per_min: 4 | 236 | rate_limit_per_min: 4 | ||
| 237 | crawl_interval_s: 21600 | 237 | crawl_interval_s: 21600 | ||
| 238 | priority: 0 | 238 | priority: 0 | ||
| 239 | − notes: arXiv asks for ≤ 1 request / 3 s on the export API. | 239 | + robots_policy: documented-exception | ||
| 240 | + notes: export.arxiv.org/robots.txt disallows generic crawlers but the arXiv API Terms of Use designate the host for programmatic access (≤ 1 request / 3 s); we use 4/min. | ||||
| 240 | - key: openreview.net | 241 | - key: openreview.net | ||
| 241 | name: OpenReview | 242 | name: OpenReview | ||
| 242 | domain: openreview.net | 243 | domain: openreview.net | ||
modified
src/aiatlas/connectors/research/arxiv.py
+3 −0
@@ -42,6 +42,9 @@ class ArxivConnector(BaseConnector): | |||||
| 42 | min_interval_seconds = 2 * 3600 | 42 | min_interval_seconds = 2 * 3600 | ||
| 43 | max_interval_seconds = 86400 | 43 | max_interval_seconds = 86400 | ||
| 44 | rate_per_min = 4 | 44 | rate_per_min = 4 | ||
| 45 | + # Documented exception: export.arxiv.org/robots.txt disallows generic crawlers, but arXiv's API Terms of Use designate this host | ||||
| 46 | + # for programmatic access (≤ 1 request / 3 s). We stay at 4 requests / min, identify ourselves and honour the rate. | ||||
| 47 | + respect_robots = False | ||||
| 45 | tier = 1 | 48 | tier = 1 | ||
| 46 | priority = 1 | 49 | priority = 1 | ||
| 47 | expected_min_records = 100 | 50 | expected_min_records = 100 | ||
| 48 | 51 | ||||