spb/spbgit

1/**2 * ─────────────────────────────────────────────3 *  SPB Git — Personal Git Platform4 * ─────────────────────────────────────────────5 *  Author  : Simon-Pierre Boucher6 *  Contact : contact@spboucher.ai7 *  File    : src/lib/crawl-guard.mjs8 *  Purpose : Throttle requests for SHA-pinned (historical) pages — crawlers walking every file at every commit9 *  License : MIT © Simon-Pierre Boucher10 * ─────────────────────────────────────────────11 */1213/**14 * In November 2026, 87 % of production requests were two crawlers fetching blob/tree/commit/raw URLs pinned to15 * historical commit SHAs. Each such page is rendered and cached once per (repository, sha): memory restarts16 * (10–30 per hour) and a 76 GB cache followed. Historical pages stay available to people, but one address may17 * only request a few of them per minute; robots.txt asks crawlers not to request them at all.18 */19const HISTORICAL = /^\/(?:[^/]+\/(?:blob|tree|blame|commits|raw)\/([0-9a-f]{7,40})(?:\/|$|\?)|raw\/[^/]+\/([0-9a-f]{7,40})(?:\/|$|\?)|[^/]+\/commit\/[0-9a-f]{7,40}(?:$|\?|\/))/;2021/** @param {string} url request URL (path + query) @returns {boolean} true when the page is pinned to a commit SHA */22export function isHistoricalUrl(url) {23  return HISTORICAL.test(url);24}2526/**27 * Fixed-window counter per client address.28 * @param {{max: number, windowMs?: number}} options29 * @returns {{hit: (ip: string, now?: number) => {allowed: boolean, retryAfter: number}, size: () => number}}30 */31export function createCrawlGuard({ max, windowMs = 60_000 }) {32  /** @type {Map<string, {start: number, count: number}>} */33  const windows = new Map();34  let lastSweep = 0;35  return {36    hit(ip, now = Date.now()) {37      if (now - lastSweep > windowMs) {38        for (const [key, w] of windows) if (now - w.start >= windowMs) windows.delete(key);39        lastSweep = now;40      }41      let w = windows.get(ip);42      if (!w || now - w.start >= windowMs) {43        w = { start: now, count: 0 };44        windows.set(ip, w);45      }46      w.count += 1;47      return { allowed: w.count <= max, retryAfter: Math.max(1, Math.ceil((w.start + windowMs - now) / 1000)) };48    },49    size: () => windows.size,50  };51}52