spb/company-atlas

1import type { Metadata } from 'next';2import Link from 'next/link';3import { Container, PageHeader } from '@/components/ui/section';4import { BOT_UA, CONTACT_EMAIL, routes, SITE_URL } from '@/lib/site';56export const metadata: Metadata = { title: 'CompanyAtlasBot — our crawler', description: 'Identity of the Company Atlas crawler, what it collects, how it behaves, and how to opt out with robots.txt.' };78export default function BotPage() {9  return (10    <Container>11      <PageHeader eyebrow="Crawler" title={BOT_UA} lede="Company Atlas observes publicly accessible corporate web pages with a clearly identified, polite crawler. This page is linked from its User-Agent string." />12      <div className="prose-atlas max-w-3xl text-[15px] leading-relaxed text-ink-2">13        <h2>Identity</h2>14        <pre>15          <code>{`User-Agent: ${BOT_UA}/0.1 (+${SITE_URL}/bot; ${CONTACT_EMAIL})`}</code>16        </pre>17        <p>18          Requests originate from the MacLustr infrastructure operated by Simon-Pierre Boucher (Québec, Canada). Contact for any question, rate concern or removal request: <a href={`mailto:${CONTACT_EMAIL}`}>{CONTACT_EMAIL}</a>.19        </p>20        <h2>What we collect</h2>21        <ul>22          <li>Publicly accessible pages of company websites: homepage, about, careers and job boards, newsroom, blog and feeds, products, pricing, leadership, locations, documentation, changelogs, legal/terms/privacy, security, investor relations, partners, customers, sitemaps.</li>23          <li>Public structured data on those pages (JSON-LD, microdata, RSS/Atom) and public JSON endpoints that the page itself loads.</li>24          <li>Text and structure only. Images, videos, fonts, binaries and tracking scripts are not downloaded by default.</li>25        </ul>26        <h2>What we never do</h2>27        <ul>28          <li>Never log in, never bypass authentication, paywalls, CAPTCHAs or bot challenges; a challenge page is recorded as a failure and the sensor backs off.</li>29          <li>Never collect non-public or personal consumer data. People appear only in their public professional context on the company’s own pages.</li>30          <li>Never access private networks, localhost or infrastructure endpoints.</li>31        </ul>32        <h2>How it behaves</h2>33        <ul>34          <li>35            Honours <code>robots.txt</code> (including <code>Crawl-delay</code>) for <code>{BOT_UA}</code> and <code>*</code>.36          </li>37          <li>Per-domain concurrency of 1–2 and per-domain rate limits; adaptive schedules from every 5–15 minutes for very active pages to every 3–7 days for stable ones; conditional requests (ETag / Last-Modified) whenever the server supports them.</li>38          <li>Retry budgets with exponential backoff; sustained 429/403 responses pause the domain.</li>39          <li>Redirect and size caps; no endless pagination, calendars or faceted crawling.</li>40        </ul>41        <h2>Opting out</h2>42        <p>Add to your robots.txt:</p>43        <pre>44          <code>{`User-agent: ${BOT_UA}45Disallow: /`}</code>46        </pre>47        <p>48          Or disallow only specific paths. Changes are picked up within 24 hours. To also remove already collected observations of your pages, write to <a href={`mailto:${CONTACT_EMAIL}`}>{CONTACT_EMAIL}</a> from a company address; we suppress the surfaces and mark the history as unavailable rather than pretending it never existed.49        </p>50        <h2>Attribution</h2>51        <p>52          Every event on <Link href={routes.home()}>Company Atlas</Link> links to the public page it was detected on. See also the <Link href={routes.about()}>about page</Link> and the <Link href={routes.api()}>API terms</Link>.53        </p>54      </div>55    </Container>56  );57}58