spb/fetcha

1import { describe, expect, it } from "vitest";2import { assertUrlAllowed, isBlockedIP, isBlockedHostname, FetchaError, fetchRequestSchema, generateApiKey, hashApiKey, looksBlocked, htmlToText, normalizeGeo } from "../src/index";34describe("SSRF protection", () => {5  it("blocks private and reserved IPv4 ranges", () => {6    for (const ip of ["127.0.0.1", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "0.0.0.0", "100.64.0.1", "224.0.0.1", "255.255.255.255"]) {7      expect(isBlockedIP(ip), ip).toBe(true);8    }9    for (const ip of ["8.8.8.8", "1.1.1.1", "93.184.216.34", "172.32.0.1", "11.0.0.1"]) {10      expect(isBlockedIP(ip), ip).toBe(false);11    }12  });1314  it("blocks IPv6 loopback, link-local, ULA and v4-mapped private", () => {15    for (const ip of ["::1", "::", "fe80::1", "fd00::1", "fc00::1", "::ffff:127.0.0.1", "::ffff:10.0.0.1", "ff02::1"]) {16      expect(isBlockedIP(ip), ip).toBe(true);17    }18    expect(isBlockedIP("2606:4700:4700::1111")).toBe(false);19  });2021  it("blocks internal hostnames", () => {22    for (const h of ["localhost", "LOCALHOST.", "foo.local", "metadata.google.internal", "api.internal", "intranet", "m3u96a.maclustr.io"]) {23      expect(isBlockedHostname(h), h).toBe(true);24    }25    expect(isBlockedHostname("example.com")).toBe(false);26  });2728  it("rejects non-http schemes and credentials", async () => {29    for (const u of ["file:///etc/passwd", "ftp://example.com/x", "gopher://x", "data:text/html,hi", "javascript:alert(1)"]) {30      await expect(assertUrlAllowed(u, { resolve: false })).rejects.toMatchObject({ code: "URL_NOT_ALLOWED" });31    }32    await expect(assertUrlAllowed("https://user:pw@example.com/", { resolve: false })).rejects.toMatchObject({ code: "URL_NOT_ALLOWED" });33    await expect(assertUrlAllowed("not a url", { resolve: false })).rejects.toMatchObject({ code: "INVALID_REQUEST" });34  });3536  it("rejects literal private IPs and numeric encodings", async () => {37    await expect(assertUrlAllowed("http://169.254.169.254/latest/meta-data")).rejects.toBeInstanceOf(FetchaError);38    await expect(assertUrlAllowed("http://[::1]:8080/")).rejects.toMatchObject({ code: "URL_NOT_ALLOWED" });39    await expect(assertUrlAllowed("http://2130706433/")).rejects.toMatchObject({ code: "URL_NOT_ALLOWED" });40    await expect(assertUrlAllowed("http://0x7f000001/")).rejects.toMatchObject({ code: "URL_NOT_ALLOWED" });41  });4243  it("allows public URLs without resolving when asked", async () => {44    const r = await assertUrlAllowed("https://example.com/path?q=1", { resolve: false });45    expect(r.hostname).toBe("example.com");46  });47});4849describe("request schema", () => {50  it("applies defaults and uppercases country", () => {51    const r = fetchRequestSchema.parse({ url: "https://example.com", country: "ca" });52    expect(r.method).toBe("GET");53    expect(r.network).toBe("auto");54    expect(r.timeout).toBe(30_000);55    expect(r.country).toBe("CA");56    expect(r.follow_redirects).toBe(true);57  });58  it("rejects unknown fields and bad values", () => {59    expect(fetchRequestSchema.safeParse({ url: "https://e.com", foo: 1 }).success).toBe(false);60    expect(fetchRequestSchema.safeParse({ url: "https://e.com", timeout: 10 }).success).toBe(false);61    expect(fetchRequestSchema.safeParse({ url: "https://e.com", network: "vpn" }).success).toBe(false);62  });63});6465describe("api keys", () => {66  it("generates prefixed keys and stable hashes", () => {67    const k = generateApiKey("live");68    expect(k.plaintext.startsWith("fch_live_")).toBe(true);69    expect(k.prefix).toBe(k.plaintext.slice(0, 13));70    expect(hashApiKey(k.plaintext)).toBe(k.hash);71    expect(generateApiKey("test").plaintext.startsWith("fch_test_")).toBe(true);72  });73});7475describe("block detection & text", () => {76  it("detects challenge pages and captchas", () => {77    expect(looksBlocked(403, "", {}).blocked).toBe(true);78    expect(looksBlocked(503, "<title>Just a moment...</title>", {}).blocked).toBe(true);79    expect(looksBlocked(200, "<html><head><title>Security check</title></head><body><div class='g-recaptcha'></div>Please verify you are human</body></html>", {}).blocked).toBe(true);80    expect(looksBlocked(200, "<html><body><h1>Contact</h1><form><div class='g-recaptcha'></div><button>Send</button></form></body></html>", {}).blocked).toBe(false);81    expect(looksBlocked(200, "<html><body>Hello</body></html>", {}).blocked).toBe(false);82    expect(looksBlocked(200, "<html><body></body></html>", { "cf-mitigated": "challenge" })).toMatchObject({ blocked: true, reason: "cloudflare_challenge", challenge: true });83    expect(looksBlocked(403, "<html><body>Access Denied</body></html>", { server: "AkamaiGHost", "set-cookie": "_abck=1; Path=/" })).toMatchObject({ blocked: true, vendor: "akamai" });84    expect(looksBlocked(429, "", { "retry-after": "3" })).toMatchObject({ blocked: true, reason: "rate_limited", retryAfterMs: 3000 });85    expect(looksBlocked(200, "<html><head><script src='/x.js'></script></head><body></body></html>", {})).toMatchObject({ blocked: true, reason: "empty_html" });86    expect(looksBlocked(200, "<html><head><title>DataDome</title></head><body><script src='https://geo.captcha-delivery.com/captcha/'></script></body></html>", {})).toMatchObject({ blocked: true, vendor: "datadome" });87    // Vendor beacons on a normal page must not count as blocks.88    const normal = `<html><head><title>Cloudflare Challenge - Example</title><script src="https://js.datadome.co/tags.js"></script><script src="/challenge.js?x"></script><script src="/ips.js"></script></head><body><main><h1>You bypassed the Cloudflare challenge!</h1><p>${"content ".repeat(200)}</p></main><script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script><script>window._pxvid='x'; _Incapsula_Resource='y';</script></body></html>`;89    expect(looksBlocked(200, normal, { server: "cloudflare" }).blocked).toBe(false);90    expect(looksBlocked(403, normal, { server: "cloudflare" })).toMatchObject({ blocked: true, vendor: "cloudflare" });91    expect(looksBlocked(200, "<html><head><title>Just a moment...</title><script src='/cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1'></script></head><body><div id='challenge-running'></div></body></html>", {})).toMatchObject({ blocked: true, reason: "cloudflare_challenge" });92  });93  it("extracts readable text", () => {94    expect(htmlToText("<h1>Hi</h1><script>x()</script><p>There &amp; back</p>")).toBe("Hi\nThere & back");95  });96  it("normalizes geography", () => {97    expect(normalizeGeo({ country: "ca", region: "QC", city: "Québec" })).toEqual({ country: "CA", region: "quebec", city: "quebec" });98    expect(normalizeGeo({ country: "US", region: "ny" }).region).toBe("new_york");99  });100});101