spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { crawlDelay, isAllowed, parseRobots } from "./robots.js";45describe("parseRobots / isAllowed", () => {6  it("allows everything when robots is empty", () => {7    const r = parseRobots("");8    expect(isAllowed(r, "/anything")).toBe(true);9  });1011  it("honors a wildcard Disallow", () => {12    const r = parseRobots("User-agent: *\nDisallow: /private/");13    expect(isAllowed(r, "/private/x")).toBe(false);14    expect(isAllowed(r, "/public/x")).toBe(true);15  });1617  it("applies longest-match-wins between Allow and Disallow", () => {18    const r = parseRobots("User-agent: *\nDisallow: /a/\nAllow: /a/b");19    expect(isAllowed(r, "/a/b/c")).toBe(true);20    expect(isAllowed(r, "/a/x")).toBe(false);21  });2223  it("supports * and $ wildcards", () => {24    const r = parseRobots("User-agent: *\nDisallow: /*.pdf$");25    expect(isAllowed(r, "/docs/file.pdf")).toBe(false);26    expect(isAllowed(r, "/docs/file.pdf?x=1")).toBe(true);27    expect(isAllowed(r, "/docs/file.html")).toBe(true);28  });2930  it("selects the most specific matching user-agent group", () => {31    const r = parseRobots(32      "User-agent: *\nDisallow: /\n\nUser-agent: Tendril\nAllow: /\nDisallow: /secret",33    );34    expect(isAllowed(r, "/page", "Tendril/1.0")).toBe(true);35    expect(isAllowed(r, "/secret", "Tendril/1.0")).toBe(false);36    expect(isAllowed(r, "/page", "SomeOtherBot")).toBe(false);37  });3839  it("treats an empty Disallow as allow-all", () => {40    const r = parseRobots("User-agent: *\nDisallow:");41    expect(isAllowed(r, "/anything")).toBe(true);42  });4344  it("groups consecutive user-agent lines together", () => {45    const r = parseRobots("User-agent: A\nUser-agent: Tendril\nDisallow: /x");46    expect(isAllowed(r, "/x", "Tendril")).toBe(false);47  });4849  it("collects Sitemap directives and Crawl-delay", () => {50    const r = parseRobots("Sitemap: https://e.com/sitemap.xml\nUser-agent: *\nCrawl-delay: 5\nDisallow: /y");51    expect(r.sitemaps).toEqual(["https://e.com/sitemap.xml"]);52    expect(crawlDelay(r)).toBe(5);53  });54});55