TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import type { Metadata } from "next";2import { PageHeader, Panel } from "@/components/ui";34export const metadata: Metadata = { title: "WebSensorBot", description: "About the WebSensor crawler: what it fetches, how often, and how to contact us." };56export default function BotPage() {7 return (8 <>9 <PageHeader kicker="Crawler policy" title="WebSensorBot" description="WebSensor monitors official public endpoints of well-known organizations and detects meaningful changes. This page describes how our fetcher behaves." />10 <div className="grid gap-4 lg:grid-cols-2">11 <Panel title="Identification">12 <p className="text-[13px]">User agent:</p>13 <pre className="mt-1 rounded-md border border-line bg-panel-2 p-2 font-mono text-[12px]">WebSensorBot/0.1 (+https://www.websensor.io/bot; contact@websensor.io)</pre>14 <p className="mt-3 text-[13px] text-fg-muted">Requests originate from a small, fixed set of addresses. Contact: <a href="mailto:contact@spboucher.ai" className="text-info hover:underline">contact@spboucher.ai</a> (Simon-Pierre Boucher). WebSensor is hosted on <a href="https://www.maclustr.io" className="text-info hover:underline" target="_blank" rel="noopener noreferrer">MacLustr</a>.</p>15 </Panel>16 <Panel title="What we fetch">17 <ul className="list-disc space-y-1 pl-5 text-[13px] text-fg-muted">18 <li>Official RSS/Atom feeds, sitemaps, status-page APIs, GitHub release feeds and public JSON APIs first.</li>19 <li>A small number of high-value HTML pages (pricing, changelog, security, documentation) when no structured alternative exists.</li>20 <li>No login, no CAPTCHAs, no paywalls, no private infrastructure. Robots directives and rate limits are honoured.</li>21 </ul>22 </Panel>23 <Panel title="How often">24 <ul className="list-disc space-y-1 pl-5 text-[13px] text-fg-muted">25 <li>Conditional requests (<code>If-None-Match</code> / <code>If-Modified-Since</code>) whenever the server supports them — most checks are 304s.</li>26 <li>Adaptive polling: from about once a minute for critical status feeds to a few times a day for slow-moving pages; at most 2 concurrent connections per host.</li>27 <li>Back-off on errors and on HTTP 429 (Retry-After honoured).</li>28 </ul>29 </Panel>30 <Panel title="Opting out">31 <p className="text-[13px] text-fg-muted">Add a <code>Disallow</code> rule for <code>WebSensorBot</code> in your <code>robots.txt</code>, or email us and we will disable the sensors for your domain.</p>32 </Panel>33 </div>34 </>35 );36}37