spb/ai-atlas

1import type { Metadata } from 'next';2import Link from 'next/link';3import { Container, PageHeader, Section } from '@/components/ui/section';4import { BOT_UA, CONTACT_EMAIL, routes } from '@/lib/site';56export const metadata: Metadata = { title: 'AIAtlasBot — crawler policy', description: 'About AIAtlasBot, the crawler that reads public AI documentation for AI Atlas: purpose, robots.txt compliance, rate limits and how to contact us or opt out.', alternates: { canonical: '/bot' } };78export default function BotPage() {9  return (10    <Container>11      <PageHeader eyebrow="Crawler" title={<span className="mono">{BOT_UA}</span>} lede="The crawler behind AI Atlas. It reads public documentation, pricing pages, model cards, papers, feeds and repositories to keep the atlas current and sourced." />12      <div className="prose-atlas max-w-3xl text-[15px] leading-relaxed text-ink-2">13        <p>14          User-Agent: <span className="mono text-ink">{BOT_UA}/0.1 (+https://www.ai-atlas.co/bot; {CONTACT_EMAIL})</span>15        </p>16      </div>17      <Section eyebrow="Purpose" title="What it collects, and why" hairline={false}>18        <p className="max-w-3xl text-sm leading-relaxed text-ink-2">AIAtlasBot fetches publicly accessible pages that describe AI models, organizations, providers, prices, benchmarks, hardware, frameworks and datasets. Pages are archived and turned into attributed facts; each fact on AI Atlas links back to the page it came from, giving publishers credit and traffic. It does not collect personal data, does not log in, and does not fetch content behind paywalls or access controls.</p>19      </Section>20      <Section eyebrow="Behaviour" title="How it behaves">21        <ul className="max-w-3xl list-disc space-y-1.5 pl-5 text-sm leading-relaxed text-ink-2">22          <li>Honours <span className="mono">robots.txt</span> (including <span className="mono">Crawl-delay</span>) for <span className="mono">{BOT_UA}</span> and <span className="mono">*</span>.</li>23          <li>Per-domain rate limits (typically a few requests per minute; arXiv ≤ 4/min, Hugging Face ≤ 30/min, GitHub ≤ 20/min).</li>24          <li>Conditional requests (<span className="mono">ETag</span> / <span className="mono">If-Modified-Since</span>) and content hashing so unchanged pages cost nothing.</li>25          <li>Adaptive intervals: pages that rarely change are visited less often; failures back off with a circuit breaker.</li>26          <li>Never bypasses CAPTCHAs or bot walls. If a site blocks us, the document is marked blocked and left alone.</li>27        </ul>28      </Section>29      <Section eyebrow="Opt out" title="Blocking or contacting us">30        <p className="max-w-3xl text-sm leading-relaxed text-ink-2">To exclude your site, add to your robots.txt:</p>31        <pre className="mt-3 max-w-3xl border border-rule bg-surface p-4 text-[13px] text-ink"><code>{`User-agent: ${BOT_UA}\nDisallow: /`}</code></pre>32        <p className="mt-3 max-w-3xl text-sm leading-relaxed text-ink-2">33          Changes are picked up on the next visit. For questions, corrections or removal requests write to <a href={`mailto:${CONTACT_EMAIL}`} className="link">{CONTACT_EMAIL}</a>. The full list of sources is public on <Link href={routes.sources()} className="link">/sources</Link>.34        </p>35      </Section>36    </Container>37  );38}39