import { afterEach, describe, expect, it } from "vitest"; import { capDiscovered } from "@dci/connectors"; import type { DiscoveredUrl } from "@dci/connectors"; import { fetchOutcome, metrics, renderMetrics, resetMetrics } from "./prom.js"; import { parseQueues } from "./env.js"; import { isDiscoveryGroup, premiumAllowedAfterErrors } from "./scheduling.js"; import { discoveryDue } from "./scheduler.js"; import { budgetKey, connectorBudgetKey } from "./budget.js"; afterEach(() => resetMetrics()); describe("prometheus registry", () => { it("renders counters, gauges and histograms in exposition format", () => { metrics.fetches.inc({ connector: "equinix", level: "L1", outcome: "ok" }); metrics.fetches.inc({ connector: "equinix", level: "L1", outcome: "ok" }); metrics.credits.inc({ provider: "scrapfly" }, 5.5); metrics.dailyBudget.set(400, { provider: "scrapfly" }); metrics.fetchDuration.observe(0.3); metrics.fetchDuration.observe(7); metrics.events.inc(undefined, 3); const out = renderMetrics(); expect(out).toContain('dci_crawl_fetches_total{connector="equinix",level="L1",outcome="ok"} 2'); expect(out).toContain('dci_crawl_credits_total{provider="scrapfly"} 5.5'); expect(out).toContain('dci_crawl_daily_budget{provider="scrapfly"} 400'); expect(out).toContain('dci_crawl_fetch_duration_seconds_bucket{le="0.5"} 1'); expect(out).toContain('dci_crawl_fetch_duration_seconds_bucket{le="+Inf"} 2'); expect(out).toContain("dci_crawl_fetch_duration_seconds_count 2"); expect(out).toContain("dci_events_total 3"); expect(out).toContain("# TYPE dci_queue_jobs gauge"); expect(out.endsWith("\n")).toBe(true); }); it("escapes label values and ignores zero increments", () => { metrics.ingest.inc({ connector: 'a"b\\c', result: "created" }, 0); metrics.ingest.inc({ connector: 'a"b\\c', result: "created" }, 2); expect(renderMetrics()).toContain('dci_ingest_entities_total{connector="a\\"b\\\\c",result="created"} 2'); }); it("classifies fetch outcomes", () => { expect(fetchOutcome({ notModified: true, status: 304 })).toBe("not_modified"); expect(fetchOutcome({ notModified: false, status: 200 })).toBe("ok"); expect(fetchOutcome({ notModified: false, status: 403 })).toBe("blocked"); expect(fetchOutcome({ notModified: false, status: 500 })).toBe("error"); expect(fetchOutcome({ notModified: false, status: 0, error: { code: "robots_disallow" } })).toBe("blocked"); expect(fetchOutcome({ notModified: false, status: 0, error: { code: "timeout" } })).toBe("error"); }); }); describe("env / budgets", () => { it("parses DCI_QUEUES", () => { expect(parseQueues(undefined)).toEqual(["crawl", "maintenance"]); expect(parseQueues("maintenance")).toEqual(["maintenance"]); expect(parseQueues(" Crawl , bogus ")).toEqual(["crawl"]); expect(parseQueues("bogus")).toEqual(["crawl", "maintenance"]); }); it("builds the Redis budget keys the API reads", () => { expect(budgetKey("scrapfly", "2026-09-11")).toBe("dci:budget:scrapfly:2026-09-11"); expect(connectorBudgetKey("equinix", "2026-09-11")).toBe("dci:budget:connector:equinix:2026-09-11"); }); }); describe("cost control rules", () => { it("discovery groups are direct-only", () => { expect(isDiscoveryGroup("sitemap")).toBe(true); expect(isDiscoveryGroup("rss")).toBe(true); expect(isDiscoveryGroup("facility_pages")).toBe(false); expect(isDiscoveryGroup(undefined)).toBe(false); }); it("premium retries are gated after two consecutive failures", () => { expect(premiumAllowedAfterErrors(0)).toBe(true); expect(premiumAllowedAfterErrors(1)).toBe(true); expect(premiumAllowedAfterErrors(2)).toBe(false); expect(premiumAllowedAfterErrors(3)).toBe(false); expect(premiumAllowedAfterErrors(4)).toBe(true); expect(premiumAllowedAfterErrors(5)).toBe(false); expect(premiumAllowedAfterErrors(8)).toBe(true); }); }); describe("scheduler", () => { const now = new Date("2026-09-11T12:00:00Z"); const H = 3_600_000; it("a never-discovered connector is due (full), one that was just discovered with zero docs is not", () => { expect(discoveryDue({ docCount: 0, lastDiscoverAt: null }, 6 * H, now)).toEqual({ due: true, task: "full" }); expect(discoveryDue({ docCount: 0, lastDiscoverAt: new Date(now.getTime() - 5 * H).toISOString() }, 6 * H, now)).toEqual({ due: false, task: "full" }); expect(discoveryDue({ docCount: 0, lastDiscoverAt: new Date(now.getTime() - 7 * H).toISOString() }, 6 * H, now)).toEqual({ due: true, task: "full" }); }); it("a populated connector re-discovers on cadence", () => { expect(discoveryDue({ docCount: 10, lastDiscoverAt: new Date(now.getTime() - 5 * H).toISOString() }, 6 * H, now)).toEqual({ due: false, task: "discover" }); expect(discoveryDue({ docCount: 10, lastDiscoverAt: new Date(now.getTime() - 6 * H).toISOString() }, 6 * H, now)).toEqual({ due: true, task: "discover" }); }); }); describe("discovery cap", () => { const u = (url: string, group: string, priority: number): DiscoveredUrl => ({ url, group, priority }); it("keeps the highest-priority groups instead of the first N sitemap entries", () => { const urls = [u("/blog/1", "default", 30), u("/blog/2", "default", 30), u("/dc/a", "facility_pages", 60), u("/news/x", "newsroom", 50), u("/dc/b", "facility_pages", 60)]; expect(capDiscovered(urls, 3).map((x) => x.url)).toEqual(["/dc/a", "/dc/b", "/news/x"]); expect(capDiscovered(urls, 10)).toHaveLength(5); expect(capDiscovered(urls, 0)).toHaveLength(5); }); it("is stable for equal priorities", () => { const urls = [u("/a", "default", 30), u("/b", "default", 30), u("/c", "default", 30)]; expect(capDiscovered(urls, 2).map((x) => x.url)).toEqual(["/a", "/b"]); }); });