spb/datacenterindex

1import { afterEach, describe, expect, it } from "vitest";2import { capDiscovered } from "@dci/connectors";3import type { DiscoveredUrl } from "@dci/connectors";4import { fetchOutcome, metrics, renderMetrics, resetMetrics } from "./prom.js";5import { parseQueues } from "./env.js";6import { isDiscoveryGroup, premiumAllowedAfterErrors } from "./scheduling.js";7import { discoveryDue } from "./scheduler.js";8import { budgetKey, connectorBudgetKey } from "./budget.js";910afterEach(() => resetMetrics());1112describe("prometheus registry", () => {13  it("renders counters, gauges and histograms in exposition format", () => {14    metrics.fetches.inc({ connector: "equinix", level: "L1", outcome: "ok" });15    metrics.fetches.inc({ connector: "equinix", level: "L1", outcome: "ok" });16    metrics.credits.inc({ provider: "scrapfly" }, 5.5);17    metrics.dailyBudget.set(400, { provider: "scrapfly" });18    metrics.fetchDuration.observe(0.3);19    metrics.fetchDuration.observe(7);20    metrics.events.inc(undefined, 3);21    const out = renderMetrics();22    expect(out).toContain('dci_crawl_fetches_total{connector="equinix",level="L1",outcome="ok"} 2');23    expect(out).toContain('dci_crawl_credits_total{provider="scrapfly"} 5.5');24    expect(out).toContain('dci_crawl_daily_budget{provider="scrapfly"} 400');25    expect(out).toContain('dci_crawl_fetch_duration_seconds_bucket{le="0.5"} 1');26    expect(out).toContain('dci_crawl_fetch_duration_seconds_bucket{le="+Inf"} 2');27    expect(out).toContain("dci_crawl_fetch_duration_seconds_count 2");28    expect(out).toContain("dci_events_total 3");29    expect(out).toContain("# TYPE dci_queue_jobs gauge");30    expect(out.endsWith("\n")).toBe(true);31  });32  it("escapes label values and ignores zero increments", () => {33    metrics.ingest.inc({ connector: 'a"b\\c', result: "created" }, 0);34    metrics.ingest.inc({ connector: 'a"b\\c', result: "created" }, 2);35    expect(renderMetrics()).toContain('dci_ingest_entities_total{connector="a\\"b\\\\c",result="created"} 2');36  });37  it("classifies fetch outcomes", () => {38    expect(fetchOutcome({ notModified: true, status: 304 })).toBe("not_modified");39    expect(fetchOutcome({ notModified: false, status: 200 })).toBe("ok");40    expect(fetchOutcome({ notModified: false, status: 403 })).toBe("blocked");41    expect(fetchOutcome({ notModified: false, status: 500 })).toBe("error");42    expect(fetchOutcome({ notModified: false, status: 0, error: { code: "robots_disallow" } })).toBe("blocked");43    expect(fetchOutcome({ notModified: false, status: 0, error: { code: "timeout" } })).toBe("error");44  });45});4647describe("env / budgets", () => {48  it("parses DCI_QUEUES", () => {49    expect(parseQueues(undefined)).toEqual(["crawl", "maintenance"]);50    expect(parseQueues("maintenance")).toEqual(["maintenance"]);51    expect(parseQueues(" Crawl , bogus ")).toEqual(["crawl"]);52    expect(parseQueues("bogus")).toEqual(["crawl", "maintenance"]);53  });54  it("builds the Redis budget keys the API reads", () => {55    expect(budgetKey("scrapfly", "2026-09-11")).toBe("dci:budget:scrapfly:2026-09-11");56    expect(connectorBudgetKey("equinix", "2026-09-11")).toBe("dci:budget:connector:equinix:2026-09-11");57  });58});5960describe("cost control rules", () => {61  it("discovery groups are direct-only", () => {62    expect(isDiscoveryGroup("sitemap")).toBe(true);63    expect(isDiscoveryGroup("rss")).toBe(true);64    expect(isDiscoveryGroup("facility_pages")).toBe(false);65    expect(isDiscoveryGroup(undefined)).toBe(false);66  });67  it("premium retries are gated after two consecutive failures", () => {68    expect(premiumAllowedAfterErrors(0)).toBe(true);69    expect(premiumAllowedAfterErrors(1)).toBe(true);70    expect(premiumAllowedAfterErrors(2)).toBe(false);71    expect(premiumAllowedAfterErrors(3)).toBe(false);72    expect(premiumAllowedAfterErrors(4)).toBe(true);73    expect(premiumAllowedAfterErrors(5)).toBe(false);74    expect(premiumAllowedAfterErrors(8)).toBe(true);75  });76});7778describe("scheduler", () => {79  const now = new Date("2026-09-11T12:00:00Z");80  const H = 3_600_000;81  it("a never-discovered connector is due (full), one that was just discovered with zero docs is not", () => {82    expect(discoveryDue({ docCount: 0, lastDiscoverAt: null }, 6 * H, now)).toEqual({ due: true, task: "full" });83    expect(discoveryDue({ docCount: 0, lastDiscoverAt: new Date(now.getTime() - 5 * H).toISOString() }, 6 * H, now)).toEqual({ due: false, task: "full" });84    expect(discoveryDue({ docCount: 0, lastDiscoverAt: new Date(now.getTime() - 7 * H).toISOString() }, 6 * H, now)).toEqual({ due: true, task: "full" });85  });86  it("a populated connector re-discovers on cadence", () => {87    expect(discoveryDue({ docCount: 10, lastDiscoverAt: new Date(now.getTime() - 5 * H).toISOString() }, 6 * H, now)).toEqual({ due: false, task: "discover" });88    expect(discoveryDue({ docCount: 10, lastDiscoverAt: new Date(now.getTime() - 6 * H).toISOString() }, 6 * H, now)).toEqual({ due: true, task: "discover" });89  });90});9192describe("discovery cap", () => {93  const u = (url: string, group: string, priority: number): DiscoveredUrl => ({ url, group, priority });94  it("keeps the highest-priority groups instead of the first N sitemap entries", () => {95    const urls = [u("/blog/1", "default", 30), u("/blog/2", "default", 30), u("/dc/a", "facility_pages", 60), u("/news/x", "newsroom", 50), u("/dc/b", "facility_pages", 60)];96    expect(capDiscovered(urls, 3).map((x) => x.url)).toEqual(["/dc/a", "/dc/b", "/news/x"]);97    expect(capDiscovered(urls, 10)).toHaveLength(5);98    expect(capDiscovered(urls, 0)).toHaveLength(5);99  });100  it("is stable for equal priorities", () => {101    const urls = [u("/a", "default", 30), u("/b", "default", 30), u("/c", "default", 30)];102    expect(capDiscovered(urls, 2).map((x) => x.url)).toEqual(["/a", "/b"]);103  });104});105