spb/datacenterindex

1import { afterEach, describe, expect, it } from "vitest";2import { botToken, getRobots, groupAllows, isAllowedByRobots, matchLen, parseRobots, resetRobotsCache, robotsAllows, robotsCacheSize, ROBOTS_CACHE_MAX } from "./robots.js";34const ORIGIN = "https://example.com";56afterEach(() => resetRobotsCache());78describe("parseRobots", () => {9  it("picks the group addressed to our token and the * group separately", () => {10    const r = parseRobots(`User-agent: *\nDisallow: /private\nCrawl-delay: 2\n\nUser-agent: DataCenterIndexBot\nAllow: /private/public-report\nDisallow: /tmp\nCrawl-delay: 5\nSitemap: https://example.com/sitemap.xml`, "datacenterindexbot");11    expect(r.agent).toEqual({ allow: ["/private/public-report"], disallow: ["/tmp"], crawlDelay: 5 });12    expect(r.star).toEqual({ allow: [], disallow: ["/private"], crawlDelay: 2 });13    expect(r.crawlDelay).toBe(5); // our group's delay wins14    expect(r.sitemaps).toEqual(["https://example.com/sitemap.xml"]);15  });16  it("falls back to * when no group names us; consecutive User-agent lines share one group; comments stripped", () => {17    const r = parseRobots(`# comment\nUser-agent: Googlebot\nUser-agent: Bingbot\nDisallow: /g\n\nUser-agent: *\nDisallow: /x # trailing\nAllow: /x/y`);18    expect(r.agent).toBeNull();19    expect(r.star).toEqual({ allow: ["/x/y"], disallow: ["/x"], crawlDelay: null });20    expect(r.disallow).toEqual(["/x"]);21  });22  it("matches a partial token (product name without version)", () => {23    const r = parseRobots(`User-agent: datacenterindex\nDisallow: /a\n\nUser-agent: *\nDisallow: /b`, "datacenterindexbot");24    expect(r.agent?.disallow).toEqual(["/a"]);25  });26  it("derives the bot token from the User-Agent string", () => {27    expect(botToken("DataCenterIndexBot/0.1 (+https://www.datacenterindex.io/bot)")).toBe("datacenterindexbot");28    expect(botToken("mybot")).toBe("mybot");29  });30  it("ignores invalid crawl-delays", () => {31    expect(parseRobots("User-agent: *\nCrawl-delay: fast").crawlDelay).toBeNull();32    expect(parseRobots("User-agent: *\nCrawl-delay: -3").crawlDelay).toBeNull();33    expect(parseRobots("User-agent: *\nCrawl-delay: 0.5").crawlDelay).toBe(0.5);34  });35});3637describe("matchLen / groupAllows", () => {38  it("prefix match, * wildcard and $ end anchor", () => {39    expect(matchLen("/a", "/abc")).toBe(2);40    expect(matchLen("/a*c", "/abc")).toBe(4);41    expect(matchLen("/a*c", "/abd")).toBe(-1);42    expect(matchLen("/*.pdf$", "/docs/x.pdf")).toBe(7);43    expect(matchLen("/*.pdf$", "/docs/x.pdf?dl=1")).toBe(-1); // anchored: must end there44    expect(matchLen("/docs$", "/docs")).toBe(6);45    expect(matchLen("/docs$", "/docs/")).toBe(-1);46    expect(matchLen("", "/x")).toBe(-1);47  });48  it("percent-decodes both the pattern and the path", () => {49    expect(matchLen("/caf%C3%A9", "/café")).toBeGreaterThan(0);50    expect(matchLen("/café", "/caf%C3%A9")).toBeGreaterThan(0);51    expect(matchLen("/a%2Fb", "/a/b")).toBeGreaterThan(0);52    expect(matchLen("/%ZZ", "/%ZZ")).toBeGreaterThan(0); // malformed escapes compare raw53  });54  it("longest match wins, allow ties beat disallow", () => {55    const g = { allow: ["/private/report"], disallow: ["/private"], crawlDelay: null };56    expect(groupAllows(g, "/private/report.pdf")).toBe(true);57    expect(groupAllows(g, "/private/other")).toBe(false);58    expect(groupAllows(g, "/public")).toBe(true);59    expect(groupAllows({ allow: ["/p"], disallow: ["/p"], crawlDelay: null }, "/p")).toBe(true);60    expect(groupAllows(null, "/anything")).toBe(true);61  });62});6364describe("robotsAllows", () => {65  it("requires both our group and the * group to allow", () => {66    const r = parseRobots(`User-agent: *\nDisallow: /star-only\n\nUser-agent: datacenterindexbot\nDisallow: /bot-only`);67    expect(robotsAllows(r, `${ORIGIN}/star-only/x`)).toBe(false);68    expect(robotsAllows(r, `${ORIGIN}/bot-only/x`)).toBe(false);69    expect(robotsAllows(r, `${ORIGIN}/ok`)).toBe(true);70  });71  it("evaluates path + query and honours $ anchors on the query", () => {72    const r = parseRobots(`User-agent: *\nDisallow: /*?*sort=\nDisallow: /exact$`);73    expect(robotsAllows(r, `${ORIGIN}/list?sort=asc`)).toBe(false);74    expect(robotsAllows(r, `${ORIGIN}/list?page=2`)).toBe(true);75    expect(robotsAllows(r, `${ORIGIN}/exact`)).toBe(false);76    expect(robotsAllows(r, `${ORIGIN}/exact/child`)).toBe(true);77  });78  it("Disallow: (empty) allows everything", () => {79    expect(robotsAllows(parseRobots("User-agent: *\nDisallow:"), `${ORIGIN}/x`)).toBe(true);80  });81});8283describe("getRobots cache + fetch outcomes", () => {84  const fetchWith = (status: number, text = "", error?: { code: string }) => async () => ({ status, text, error: error ?? null });85  it("200 → rules; 404/410 → allow all", async () => {86    const r = await getRobots(`${ORIGIN}/x`, fetchWith(200, "User-agent: *\nDisallow: /x"));87    expect(robotsAllows(r, `${ORIGIN}/x`)).toBe(false);88    resetRobotsCache();89    const r404 = await getRobots(`${ORIGIN}/x`, fetchWith(404));90    expect(r404.unknown).toBeFalsy();91    expect(robotsAllows(r404, `${ORIGIN}/x`)).toBe(true);92  });93  it("503 / timeout with nothing cached → unknown (short TTL); with a cache → previous rules kept", async () => {94    const u = await getRobots(`${ORIGIN}/x`, fetchWith(503));95    expect(u.unknown).toBe(true);96    expect(u.ttlMs).toBe(3_600_000);97    expect((await isAllowedByRobots(`${ORIGIN}/x`)).unknown).toBe(true);98    resetRobotsCache();99    await getRobots(`${ORIGIN}/x`, fetchWith(200, "User-agent: *\nDisallow: /x"));100    resetRobotsCache([[ORIGIN, { ...(await getRobots(`${ORIGIN}/x`, fetchWith(200, "User-agent: *\nDisallow: /x"))), fetchedAt: 0 }]]); // expired entry101    const kept = await getRobots(`${ORIGIN}/x`, fetchWith(0, "", { code: "timeout" }));102    expect(kept.unknown).toBeFalsy();103    expect(robotsAllows(kept, `${ORIGIN}/x`)).toBe(false);104  });105  it("caches per origin and evicts the least recently used entry beyond the cap", async () => {106    let calls = 0;107    const f = async () => { calls++; return { status: 200, text: "User-agent: *\nDisallow: /x", error: null }; };108    await getRobots(`${ORIGIN}/a`, f); await getRobots(`${ORIGIN}/b`, f);109    expect(calls).toBe(1);110    resetRobotsCache();111    for (let i = 0; i < ROBOTS_CACHE_MAX + 10; i++) await getRobots(`https://h${i}.example/`, f);112    expect(robotsCacheSize()).toBe(ROBOTS_CACHE_MAX);113    // the very first origin was evicted, a recent one is still cached114    calls = 0;115    await getRobots("https://h0.example/", f);116    expect(calls).toBe(1);117    calls = 0;118    await getRobots(`https://h${ROBOTS_CACHE_MAX + 9}.example/`, f);119    expect(calls).toBe(0);120  });121});122