spb/datacenterindex

1import { afterEach, beforeEach, describe, expect, it } from "vitest";2import type { FetchLevel } from "@dci/core";3import type { Fetcher, FetchOptions, RawDocument } from "./types.js";4import { fetchWithEscalation, fetchers } from "./fetchers.js";56/**7 * fetchWithEscalation with the four level fetchers stubbed (no network). Each stub records the levels it was asked8 * for and returns a scripted document; the real registry entries are restored after every test.9 */10function stub(level: FetchLevel, script: (opts: FetchOptions) => Partial<RawDocument> | { unavailable: true }, calls: FetchLevel[]): Fetcher {11  return {12    name: level >= 4 ? "scrapfly" : level === 3 ? "firecrawl" : "direct",13    level,14    available() { const r = script({}); return !("unavailable" in r); },15    async fetch(url, opts = {}) {16      calls.push(level);17      const r = script(opts) as Partial<RawDocument>;18      const body = Buffer.from(r.text ?? "", "utf8");19      return { url, finalUrl: url, fetchedAt: "2026-09-12T00:00:00Z", status: 200, contentType: "text/html", body, text: "", headers: {}, etag: null, lastModified: null, notModified: false, fetcher: this.name, level, durationMs: 1, credits: level === 3 ? 1 : level === 4 ? 6 : 0, ...r };20    },21  };22}2324const RICH = "<html><body>" + "<p>Real facility content, 12 MW, Ashburn, colocation.</p>".repeat(40) + "</body></html>";25const SHELL = '<html><body><div id="__next"></div></body></html>';26const saved: Record<FetchLevel, Fetcher> = { ...fetchers };27let calls: FetchLevel[] = [];2829beforeEach(() => { calls.length = 0; });30afterEach(() => { for (const l of [1, 2, 3, 4] as FetchLevel[]) fetchers[l] = saved[l]; });3132describe("fetchWithEscalation", () => {33  it("returns the first good direct document without touching premium levels", async () => {34    fetchers[1] = stub(1, () => ({ text: RICH }), calls);35    fetchers[2] = stub(2, () => ({ text: RICH }), calls);36    fetchers[3] = stub(3, () => ({ text: RICH }), calls);37    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 4 });38    expect(d.level).toBe(1);39    expect(calls).toEqual([1]);40    expect(d.meta?.attempts).toEqual(["L1:200"]);41  });4243  it("escalates past a 403 and a JS shell, then stops at the first real page", async () => {44    fetchers[1] = stub(1, () => ({ status: 403, text: "" }), calls);45    fetchers[2] = stub(2, () => ({ text: SHELL }), calls);46    fetchers[3] = stub(3, () => ({ text: RICH }), calls);47    fetchers[4] = stub(4, () => ({ text: RICH }), calls);48    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 4 });49    expect(calls).toEqual([1, 2, 3]);50    expect(d.level).toBe(3);51    expect(d.meta?.attempts).toEqual(["L1:403", "L2:200", "L3:200"]);52  });5354  it("HTTP 429 is never escalated and exposes retryAfterMs from Retry-After", async () => {55    fetchers[1] = stub(1, () => ({ status: 429, headers: { "retry-after": "90" }, text: "" }), calls);56    fetchers[2] = stub(2, () => ({ text: RICH }), calls);57    fetchers[3] = stub(3, () => ({ text: RICH }), calls);58    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 4 });59    expect(calls).toEqual([1]);60    expect(d.status).toBe(429);61    expect(d.meta).toMatchObject({ rateLimited: true, retryAfterMs: 90_000, attempts: ["L1:429"] });62    expect(d.credits).toBe(0);63  });6465  it("a 429 without Retry-After still stops escalation (no retryAfterMs key)", async () => {66    fetchers[1] = stub(1, () => ({ status: 429, text: "" }), calls);67    fetchers[2] = stub(2, () => ({ text: RICH }), calls);68    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 2 });69    expect(calls).toEqual([1]);70    expect(d.meta?.rateLimited).toBe(true);71    expect(d.meta).not.toHaveProperty("retryAfterMs");72  });7374  it("creditsLeft: skips premium levels whose expected cost exceeds the remaining run budget", async () => {75    fetchers[1] = stub(1, () => ({ status: 403, text: "" }), calls);76    fetchers[2] = stub(2, () => ({ status: 403, text: "" }), calls);77    fetchers[3] = stub(3, () => ({ text: SHELL, credits: 1 }), calls); // Firecrawl returns a shell → would escalate to L478    fetchers[4] = stub(4, () => ({ text: RICH, credits: 6 }), calls);79    // 5 credits left: L3 (1) is affordable, L4 (6) is not → never spends on both80    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 4, creditsLeft: 5 });81    expect(calls).toEqual([1, 2, 3]);82    expect(d.level).toBe(3);83    // L3 spent 1 of the 5 credits; the skipped L4 (cost 6 > 4 left) is visible in the attempt trace84    expect(d.meta?.attempts).toEqual(["L1:403", "L2:403", "L3:200", "L4:over_budget(6>4)"]);85    // 1 credit left and Scrapfly ASP-only (cost 1): after L3 spends it, L4 is over budget86    calls.length = 0;87    const d2 = await fetchWithEscalation("https://x.com/a", { maxLevel: 4, creditsLeft: 1, renderJs: false });88    expect(calls).toEqual([1, 2, 3]);89    expect(d2.level).toBe(3);90    // 0 credits left: no premium at all, best direct attempt is returned91    calls.length = 0;92    const d3 = await fetchWithEscalation("https://x.com/a", { maxLevel: 4, creditsLeft: 0 });93    expect(calls).toEqual([1, 2]);94    expect(d3.status).toBe(403);95    expect(d3.meta?.attempts).toEqual(["L1:403", "L2:403", "L3:over_budget(1>0)", "L4:over_budget(6>0)"]);96    // no creditsLeft given → legacy behaviour, both premium levels reachable97    calls.length = 0;98    const d4 = await fetchWithEscalation("https://x.com/a", { maxLevel: 4 });99    expect(calls).toEqual([1, 2, 3, 4]);100    expect(d4.level).toBe(4);101  });102103  it("unavailable premium fetchers are skipped and recorded", async () => {104    fetchers[1] = stub(1, () => ({ status: 403, text: "" }), calls);105    fetchers[2] = stub(2, () => ({ status: 403, text: "" }), calls);106    fetchers[3] = stub(3, () => ({ unavailable: true }), calls);107    fetchers[4] = stub(4, () => ({ text: RICH }), calls);108    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 4 });109    expect(calls).toEqual([1, 2, 4]);110    expect(d.meta?.attempts).toEqual(["L1:403", "L2:403", "L3:unavailable", "L4:200"]);111  });112113  it("hard failures (ssrf, dns, too_large, 404, 410) are returned immediately", async () => {114    for (const [code, status] of [["ssrf_blocked", 0], ["dns", 0], ["too_large", 200]] as const) {115      calls.length = 0;116      fetchers[1] = stub(1, () => ({ status, text: "", error: { code, message: code } }), calls);117      fetchers[2] = stub(2, () => ({ text: RICH }), calls);118      const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 2 });119      expect(calls, code).toEqual([1]);120      expect(d.error?.code).toBe(code);121    }122    for (const status of [404, 410]) {123      calls.length = 0;124      fetchers[1] = stub(1, () => ({ status, text: "" }), calls);125      const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 2 });126      expect(calls).toEqual([1]);127      expect(d.status).toBe(status);128    }129  });130131  it("propagates noarchive from headers or meta robots on whichever level served the page", async () => {132    fetchers[1] = stub(1, () => ({ status: 403, text: "" }), calls);133    fetchers[2] = stub(2, () => ({ text: RICH, headers: { "x-robots-tag": "noarchive" } }), calls);134    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 2 });135    expect(d.level).toBe(2);136    expect(d.meta).toMatchObject({ noarchive: true, robotsDirective: "header:noarchive" });137    calls.length = 0;138    fetchers[1] = stub(1, () => ({ text: '<html><head><meta name="robots" content="noindex"></head><body>' + RICH + "</body></html>" }), calls);139    const d2 = await fetchWithEscalation("https://x.com/a", { maxLevel: 2 });140    expect(d2.meta).toMatchObject({ noarchive: true, robotsDirective: "meta:noindex" });141    fetchers[1] = stub(1, () => ({ text: RICH }), calls);142    const clean = await fetchWithEscalation("https://x.com/a", { maxLevel: 2 });143    expect(clean.meta?.noarchive).toBeUndefined();144  });145146  it("304 is returned as-is", async () => {147    fetchers[1] = stub(1, () => ({ status: 304, notModified: true, text: "" }), calls);148    fetchers[2] = stub(2, () => ({ text: RICH }), calls);149    const d = await fetchWithEscalation("https://x.com/a", { maxLevel: 2, etag: '"e"' });150    expect(d.notModified).toBe(true);151    expect(calls).toEqual([1]);152  });153});154