spb/datacenterindex

1import { afterEach, describe, expect, it } from "vitest";2import { gzipSync } from "node:zlib";3import type { RawDocument } from "./types.js";4import { Budget, applyRobotsDirectives, budgetDay, expectedCredits, gunzipLimit, looksBlockedOrEmpty, parseRetryAfter, robotsDirective, safeGunzip, setBudgetStore } from "./fetchers.js";56function doc(p: Partial<RawDocument>): RawDocument {7  return { url: "https://x.com/a", finalUrl: "https://x.com/a", fetchedAt: "2026-09-12T00:00:00Z", status: 200, contentType: "text/html", body: Buffer.alloc(0), text: "", headers: {}, etag: null, lastModified: null, notModified: false, fetcher: "direct", level: 1, durationMs: 1, credits: 0, ...p };8}910describe("looksBlockedOrEmpty", () => {11  const filler = "<p>" + "Real content about data centers. ".repeat(80) + "</p>";12  it("status-based blocks", () => {13    for (const s of [401, 403, 429, 503]) expect(looksBlockedOrEmpty(doc({ status: s }))).toBe(true);14    expect(looksBlockedOrEmpty(doc({ status: 500 }))).toBe(false);15    expect(looksBlockedOrEmpty(doc({ status: 200, contentType: "application/json", text: "{}" }))).toBe(false);16  });17  it("interstitial only when the page carries little text (Cloudflare beacon on real pages is fine)", () => {18    expect(looksBlockedOrEmpty(doc({ text: "<html><title>Just a moment...</title><body><script src=/cdn-cgi/challenge-platform/x></script></body></html>" }))).toBe(true);19    expect(looksBlockedOrEmpty(doc({ text: `<html><head><script src="/cdn-cgi/challenge-platform/h/b"></script></head><body>${filler}</body></html>` }))).toBe(false);20  });21  it("empty JS shells are blocked-looking, small real pages are not", () => {22    expect(looksBlockedOrEmpty(doc({ text: '<html><body><div id="__next"></div><script src="app.js"></script></body></html>' }))).toBe(true);23    expect(looksBlockedOrEmpty(doc({ text: "<html><body><h1>DC1</h1><p>12 MW, Ashburn</p></body></html>" }))).toBe(false);24  });25});2627describe("gzip bomb guard", () => {28  it("caps decompressed output at max(4×maxBytes, 64 MB) and returns an error above it", () => {29    expect(gunzipLimit(1024)).toBe(64 * 1024 * 1024);30    expect(gunzipLimit(25 * 1024 * 1024)).toBe(100 * 1024 * 1024);31    const bomb = gzipSync(Buffer.alloc(200 * 1024, 0)); // 200 KB of zeros → ~200 bytes gzipped32    const ok = safeGunzip(bomb, 1024 * 1024);33    expect(ok.error).toBeUndefined();34    expect(ok.body.length).toBe(200 * 1024);35    // simulate a tiny cap by exercising the underlying option path: a 200 KB payload with a 64 MB floor is fine, so test36    // the error branch through a body far above a synthetic limit37    const huge = gzipSync(Buffer.alloc(70 * 1024 * 1024, 0));38    const r = safeGunzip(huge, 1024);39    expect(r.error).toMatch(/exceeds/);40  });41  it("leaves non-gzip bodies untouched", () => {42    const b = Buffer.from("plain");43    expect(safeGunzip(b, 1024).body).toBe(b);44  });45});4647describe("robots directives (noarchive / noindex)", () => {48  it("reads X-Robots-Tag and <meta name=robots>", () => {49    expect(robotsDirective(doc({ headers: { "x-robots-tag": "noarchive, nosnippet" } }))).toBe("header:noarchive");50    expect(robotsDirective(doc({ headers: { "x-robots-tag": "noindex" } }))).toBe("header:noindex");51    expect(robotsDirective(doc({ text: '<html><head><meta name="ROBOTS" content="index, noarchive"></head></html>' }))).toBe("meta:noarchive");52    expect(robotsDirective(doc({ text: '<meta content="noindex,follow" name="robots">' }))).toBe("meta:noindex");53    expect(robotsDirective(doc({ text: '<meta name="robots" content="index, follow, max-snippet:-1">' }))).toBeNull();54    expect(robotsDirective(doc({ headers: { "x-robots-tag": "nosnippet" } }))).toBeNull();55    expect(robotsDirective(doc({ contentType: "application/json", text: '<meta name="robots" content="noarchive">' }))).toBeNull(); // only HTML bodies56  });57  it("applyRobotsDirectives sets meta.noarchive and is idempotent", () => {58    const d = applyRobotsDirectives(doc({ headers: { "x-robots-tag": "noarchive" }, meta: { attempts: ["L1:200"] } }));59    expect(d.meta).toEqual({ attempts: ["L1:200"], noarchive: true, robotsDirective: "header:noarchive" });60    expect(applyRobotsDirectives(d).meta).toEqual(d.meta);61    expect(applyRobotsDirectives(doc({})).meta).toBeUndefined();62  });63});6465describe("parseRetryAfter / expectedCredits", () => {66  it("accepts delta-seconds and HTTP dates, bounded to 7 days", () => {67    const now = Date.parse("2026-09-12T12:00:00Z");68    expect(parseRetryAfter("120", now)).toBe(120_000);69    expect(parseRetryAfter("Sat, 12 Sep 2026 12:10:00 GMT", now)).toBe(600_000);70    expect(parseRetryAfter("Fri, 11 Sep 2026 12:00:00 GMT", now)).toBe(0); // in the past71    expect(parseRetryAfter("99999999", now)).toBe(7 * 86_400_000);72    expect(parseRetryAfter("soon", now)).toBeNull();73    expect(parseRetryAfter(null, now)).toBeNull();74  });75  it("expected premium cost per level", () => {76    expect(expectedCredits(1)).toBe(0);77    expect(expectedCredits(2)).toBe(0);78    expect(expectedCredits(3)).toBe(1);79    expect(expectedCredits(4)).toBe(6);80    expect(expectedCredits(4, { renderJs: false })).toBe(1);81  });82});8384describe("Budget", () => {85  afterEach(() => { setBudgetStore(null); delete process.env.DCI_TEST_BUDGET; });86  it("tracks local spend against the env limit and rolls per UTC day", () => {87    process.env.DCI_TEST_BUDGET = "10";88    const b = new Budget("scrapfly", "DCI_TEST_BUDGET", 400);89    expect(b.limit()).toBe(10);90    expect(b.remaining()).toBe(10);91    b.spend(4);92    expect(b.used()).toBe(4);93    expect(b.remaining()).toBe(6);94    b.spend(10);95    expect(b.remaining()).toBe(0);96    expect(b.snapshot()).toEqual({ day: budgetDay(), used: 14, local: 14, limit: 10 });97  });98  it("uses the fallback limit without env and never goes below its own spend when a shared store lags", () => {99    const b = new Budget("firecrawl", "DCI_TEST_BUDGET", 200);100    expect(b.limit()).toBe(200);101    const spent: number[] = [];102    setBudgetStore({ used: () => 50, spend: (_p, _d, n) => { spent.push(n); } });103    expect(b.used()).toBe(50); // shared view104    b.spend(60);105    expect(spent).toEqual([60]);106    expect(b.used()).toBe(60); // local 60 > shared 50107    setBudgetStore({ used: () => 100, spend: () => { throw new Error("redis down"); } });108    expect(() => b.spend(1)).not.toThrow();109    expect(b.used()).toBe(100);110  });111});112