spb/datacenterindex

1import { describe, expect, it } from "vitest";2import type { RawDocument } from "@dci/connectors";3import { baseIntervalMs, encodeDiscoveredFrom, healthFrom, intervalScale, maxLevelForBudget, minIntervalMs, nextCheckAfter, parseDiscoveredFrom, ratioSignificance, shouldSkipExtraction, updateChangeScore, versionSignificance } from "./scheduling.js";4import { planFetchBookkeeping } from "./documents.js";5import { fingerprintOf, pLimit, stripRuntimeNoise } from "./pipeline.js";67const DAY = 86_400_000;8const now = new Date("2026-09-11T12:00:00Z");910function raw(partial: Partial<RawDocument>): RawDocument {11  return { url: "https://example.com/a", finalUrl: "https://example.com/a", fetchedAt: now.toISOString(), status: 200, contentType: "text/html", body: Buffer.from("<html><body>x</body></html>"), text: "<html><body>x</body></html>", headers: {}, etag: null, lastModified: null, notModified: false, fetcher: "direct", level: 1, durationMs: 10, credits: 0, ...partial };12}1314describe("intervals", () => {15  it("resolves group → default → weekly", () => {16    expect(baseIntervalMs({ newsroom: "daily", default: "monthly" }, "newsroom")).toBe(DAY);17    expect(baseIntervalMs({ newsroom: "daily", default: "monthly" }, "other")).toBe(30 * DAY);18    expect(baseIntervalMs({ newsroom: "daily" }, "other")).toBe(7 * DAY);19    expect(baseIntervalMs({ x: "not-an-interval" }, "x")).toBe(7 * DAY);20  });21  it("never → Infinity, min interval picks the shortest finite one", () => {22    expect(baseIntervalMs({ archive: "never" }, "archive")).toBe(Number.POSITIVE_INFINITY);23    expect(minIntervalMs({ a: "weekly", b: "6h", c: "never" })).toBe(6 * 3_600_000);24  });25});2627describe("change score EMA + scale", () => {28  it("moves toward 1 on change and toward 0 when static, clamped", () => {29    expect(updateChangeScore(0.5, true)).toBeCloseTo(0.65, 3);30    expect(updateChangeScore(0.5, false)).toBeCloseTo(0.35, 3);31    let s = 0.5;32    for (let i = 0; i < 20; i++) s = updateChangeScore(s, false);33    expect(s).toBeLessThan(0.01);34    expect(updateChangeScore(Number.NaN, true)).toBeCloseTo(0.65, 3);35  });36  it("frequent changers ×0.5, static ×2 up to ×4", () => {37    expect(intervalScale(0.9)).toBe(0.5);38    expect(intervalScale(0.5)).toBe(0.5);39    expect(intervalScale(0.3)).toBe(1);40    expect(intervalScale(0.15)).toBe(2);41    expect(intervalScale(0.02)).toBe(4);42  });43});4445describe("nextCheckAfter", () => {46  it("scales the base interval adaptively", () => {47    const fast = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.8, statusCode: 200, consecutiveErrors: 0, hadError: false });48    expect(fast.nextCheck!.getTime() - now.getTime()).toBe(DAY / 2);49    const slow = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.01, statusCode: 200, consecutiveErrors: 0, hadError: false });50    expect(slow.nextCheck!.getTime() - now.getTime()).toBe(4 * DAY);51    expect(slow.quarantined).toBe(false);52  });53  it("404 backs off ×4 then quarantines after 3 consecutive", () => {54    const first = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: 404, consecutiveErrors: 1, hadError: false });55    expect(first.nextCheck!.getTime() - now.getTime()).toBe(4 * DAY);56    expect(first.quarantined).toBe(false);57    const third = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: 410, consecutiveErrors: 3, hadError: false });58    expect(third.nextCheck).toBeNull();59    expect(third.quarantined).toBe(true);60  });61  it("transport errors back off exponentially, capped at 8×, and never → null", () => {62    const e1 = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: null, consecutiveErrors: 1, hadError: true });63    expect(e1.nextCheck!.getTime() - now.getTime()).toBe(DAY);64    const e5 = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: null, consecutiveErrors: 5, hadError: true });65    expect(e5.nextCheck!.getTime() - now.getTime()).toBe(8 * DAY);66    expect(nextCheckAfter({ now, baseMs: Number.POSITIVE_INFINITY, changeScore: 0.5, statusCode: 200, consecutiveErrors: 0, hadError: false }).nextCheck).toBeNull();67  });68  it("429 honours Retry-After (floor 15 min, cap 8× base), otherwise falls back to the error backoff", () => {69    const ra = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: 429, consecutiveErrors: 1, hadError: false, retryAfterMs: 3_600_000 });70    expect(ra.nextCheck!.getTime() - now.getTime()).toBe(3_600_000);71    expect(ra.reason).toBe("retry-after 3600s");72    expect(nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: 429, consecutiveErrors: 1, hadError: false, retryAfterMs: 1_000 }).nextCheck!.getTime() - now.getTime()).toBe(15 * 60_000);73    expect(nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: 429, consecutiveErrors: 1, hadError: false, retryAfterMs: 30 * DAY }).nextCheck!.getTime() - now.getTime()).toBe(8 * DAY);74    const noHeader = nextCheckAfter({ now, baseMs: DAY, changeScore: 0.5, statusCode: 429, consecutiveErrors: 2, hadError: false, retryAfterMs: null });75    expect(noHeader.reason).toBe("error backoff ×2");76  });77  it("planFetchBookkeeping forwards doc.meta.retryAfterMs from a 429", () => {78    const p = planFetchBookkeeping({ changeFrequencyScore: 0.5, errorCount: 0, discoveredFrom: encodeDiscoveredFrom("newsroom", null) }, raw({ status: 429, body: Buffer.alloc(0), text: "", meta: { retryAfterMs: 2 * 3_600_000 } }), { contentHash: null, changed: false, storageKey: null, title: null }, { newsroom: "daily" }, now);79    expect(p.failed).toBe(true);80    expect(p.error).toBe("HTTP 429");81    expect(Date.parse(p.nextCheck!) - now.getTime()).toBe(2 * 3_600_000);82  });83  it("enforces a 15 min floor", () => {84    const r = nextCheckAfter({ now, baseMs: 60_000, changeScore: 0.9, statusCode: 200, consecutiveErrors: 0, hadError: false });85    expect(r.nextCheck!.getTime() - now.getTime()).toBe(15 * 60_000);86  });87});8889describe("planFetchBookkeeping (documents)", () => {90  const doc = { changeFrequencyScore: 0.5, errorCount: 0, discoveredFrom: encodeDiscoveredFrom("newsroom", "https://example.com/sitemap.xml") };91  const schedule = { newsroom: "daily", facility_pages: "weekly" };92  it("uses the group's interval and updates the score on success", () => {93    const p = planFetchBookkeeping(doc, raw({}), { contentHash: "h1", changed: true, storageKey: "k", title: "t" }, schedule, now);94    expect(p.failed).toBe(false);95    expect(p.changed).toBe(true);96    expect(p.changeScore).toBeCloseTo(0.65, 3);97    expect(p.errorCount).toBe(0);98    expect(Date.parse(p.nextCheck!) - now.getTime()).toBe(DAY / 2); // 0.65 ≥ 0.5 → ×0.599  });100  it("counts consecutive 404s and quarantines on the third", () => {101    const p1 = planFetchBookkeeping({ ...doc, errorCount: 0 }, raw({ status: 404, body: Buffer.alloc(0), text: "" }), { contentHash: null, changed: false, storageKey: null, title: null }, schedule, now);102    expect(p1.failed).toBe(true);103    expect(p1.errorCount).toBe(1);104    expect(p1.error).toBe("HTTP 404");105    expect(p1.changeScore).toBe(0.5); // untouched on failure106    const p3 = planFetchBookkeeping({ ...doc, errorCount: 2 }, raw({ status: 410, body: Buffer.alloc(0), text: "" }), { contentHash: null, changed: false, storageKey: null, title: null }, schedule, now);107    expect(p3.quarantined).toBe(true);108    expect(p3.nextCheck).toBeNull();109  });110  it("treats transport errors as failures with their code", () => {111    const p = planFetchBookkeeping({ ...doc, errorCount: 1 }, raw({ status: 0, error: { code: "timeout", message: "AbortError" } }), { contentHash: null, changed: false, storageKey: null, title: null }, schedule, now);112    expect(p.failed).toBe(true);113    expect(p.error).toMatch(/^timeout/);114    expect(p.errorCount).toBe(2);115    expect(Date.parse(p.nextCheck!) - now.getTime()).toBe(2 * DAY);116  });117  it("304 is a success that lowers the change score", () => {118    const p = planFetchBookkeeping(doc, raw({ status: 304, notModified: true, body: Buffer.alloc(0), text: "" }), { contentHash: "h0", changed: false, storageKey: "k", title: null }, schedule, now);119    expect(p.failed).toBe(false);120    expect(p.notModified).toBe(true);121    expect(p.changeScore).toBeCloseTo(0.35, 3);122  });123});124125describe("fingerprint / skip logic", () => {126  it("ignores volatile noise between two fetches of the same page", () => {127    const a = raw({ text: '<html><head><script nonce="a1">x("2026-09-11T01:00")</script></head><body data-build="a1">\n  <p>Hello</p> <!-- build 1 --> <input type="hidden" name="csrf" value="aaa"></body></html>' });128    const b = raw({ text: '<html><head><script nonce="b2">y("2026-09-11T02:00")</script></head><body data-build="b2">\n\n  <p>Hello</p>   <!-- build 2 --> <input type="hidden" name="csrf" value="bbb"></body></html>' });129    const c = raw({ text: "<html><body><p>Hello world</p></body></html>" });130    expect(fingerprintOf(a)).toBe(fingerprintOf(b));131    expect(fingerprintOf(a)).not.toBe(fingerprintOf(c));132  });133  it("ignores epoch cache-busters stamped into URLs on every render", () => {134    const a = raw({ text: '<html><head><meta property="og:url" content="https://x.com/p?t=1789110041117?latest"/></head><body><a href="https://t.co/share?url=https://x.com/p?t=1789110041118&amp;title=A">x</a><p>Body</p></body></html>' });135    const b = raw({ text: '<html><head><meta property="og:url" content="https://x.com/p?t=1789110320767?latest"/></head><body><a href="https://t.co/share?url=https://x.com/p?t=1789110320767&amp;title=A">x</a><p>Body</p></body></html>' });136    expect(fingerprintOf(a)).toBe(fingerprintOf(b));137    expect(stripRuntimeNoise("/a?t=1789110041117&x=1")).toBe("/a?t=&x=1");138    expect(stripRuntimeNoise("/a?id=12345&page=2")).toBe("/a?id=12345&page=2"); // short ids untouched139    expect(stripRuntimeNoise("capacity 120 MW, opened 2024, 1,000 racks")).toBe("capacity 120 MW, opened 2024, 1,000 racks");140  });141  it("hashes binary bodies (PDF) by bytes", () => {142    const pdf1 = raw({ contentType: "application/pdf", text: "", body: Buffer.from("%PDF-1.4 one") });143    const pdf2 = raw({ contentType: "application/pdf", text: "", body: Buffer.from("%PDF-1.4 two") });144    expect(fingerprintOf(pdf1)).not.toBe(fingerprintOf(pdf2));145    expect(fingerprintOf(pdf1)).toBe(fingerprintOf(raw({ contentType: "application/pdf", text: "", body: Buffer.from("%PDF-1.4 one") })));146  });147  it("skips extraction only when hash and extractor version are unchanged and not forced", () => {148    const base = { newHash: "h", storedHash: "h", force: false, extractorVersion: "v1", storedExtractorVersion: "v1", notModified: false };149    expect(shouldSkipExtraction(base)).toBe(true);150    expect(shouldSkipExtraction({ ...base, force: true })).toBe(false);151    expect(shouldSkipExtraction({ ...base, storedHash: "other" })).toBe(false);152    expect(shouldSkipExtraction({ ...base, storedHash: null })).toBe(false);153    expect(shouldSkipExtraction({ ...base, storedExtractorVersion: "v0" })).toBe(false);154    expect(shouldSkipExtraction({ ...base, storedExtractorVersion: null })).toBe(true);155    expect(shouldSkipExtraction({ ...base, storedHash: "x", notModified: true })).toBe(true);156  });157});158159describe("significance, health, budget, discovered_from", () => {160  it("significance from field changes else ratio", () => {161    expect(versionSignificance([{ significance: 90 }, { significance: 40 }], 0.5)).toBe(90);162    expect(versionSignificance([], 0.01)).toBe(10);163    expect(versionSignificance([], 0.05)).toBe(20);164    expect(versionSignificance([], 0.3)).toBe(30);165    expect(ratioSignificance(0)).toBe(0);166  });167  it("health thresholds 20 % / 50 %", () => {168    expect(healthFrom(100, 5)).toBe("ok");169    expect(healthFrom(100, 19)).toBe("ok");170    expect(healthFrom(100, 20)).toBe("degraded");171    expect(healthFrom(100, 50)).toBe("failing");172    expect(healthFrom(0, 0)).toBe("ok");173  });174  it("premium levels are capped at L2 once the run budget is spent", () => {175    expect(maxLevelForBudget(4, 0, 100)).toBe(4);176    expect(maxLevelForBudget(4, 100, 100)).toBe(2);177    expect(maxLevelForBudget(3, 250, 200)).toBe(2);178    expect(maxLevelForBudget(2, 999, 0)).toBe(2);179    expect(maxLevelForBudget(1, 0, 0)).toBe(1);180  });181  it("round-trips group and origin in discovered_from", () => {182    const enc = encodeDiscoveredFrom("facility_pages", "https://example.com/sitemap.xml");183    expect(parseDiscoveredFrom(enc)).toEqual({ group: "facility_pages", from: "https://example.com/sitemap.xml" });184    expect(parseDiscoveredFrom(null)).toEqual({ group: "default", from: null });185    expect(parseDiscoveredFrom("newsroom|")).toEqual({ group: "newsroom", from: null });186    expect(parseDiscoveredFrom("https://legacy")).toEqual({ group: "default", from: "https://legacy" });187  });188});189190describe("pLimit", () => {191  it("never exceeds the concurrency and resolves everything", async () => {192    const limit = pLimit(2);193    let active = 0, peak = 0;194    const results = await Promise.all(195      Array.from({ length: 7 }, (_, i) =>196        limit(async () => {197          active++; peak = Math.max(peak, active);198          await new Promise((r) => setTimeout(r, 5));199          active--;200          return i;201        }),202      ),203    );204    expect(results).toEqual([0, 1, 2, 3, 4, 5, 6]);205    expect(peak).toBe(2);206  });207  it("propagates rejections without blocking the queue", async () => {208    const limit = pLimit(1);209    await expect(limit(async () => { throw new Error("boom"); })).rejects.toThrow("boom");210    expect(await limit(async () => 42)).toBe(42);211  });212});213214describe("connectorHealthFrom", () => {215  it("labels blocked, schema change and no-new-content connectors", async () => {216    const { connectorHealthFrom } = await import("./scheduling.js");217    expect(connectorHealthFrom({ runHealth: "failing", status: "partial", fetched: 10, blocked: 8, discovered: 0, previouslyDiscovered: null, extracted: 0, entities: 0, docsTotal: 50 })).toBe("blocked");218    expect(connectorHealthFrom({ runHealth: "ok", status: "ok", fetched: 12, blocked: 0, discovered: 0, previouslyDiscovered: null, extracted: 12, entities: 0, docsTotal: 200 })).toBe("schema_change");219    expect(connectorHealthFrom({ runHealth: "ok", status: "ok", fetched: 0, blocked: 0, discovered: 0, previouslyDiscovered: 40, extracted: 0, entities: 0, docsTotal: 40 })).toBe("no_new_content");220    expect(connectorHealthFrom({ runHealth: "ok", status: "ok", fetched: 8, blocked: 0, discovered: 3, previouslyDiscovered: 40, extracted: 8, entities: 8, docsTotal: 40 })).toBe("ok");221    expect(connectorHealthFrom({ runHealth: "degraded", status: "failed", fetched: 8, blocked: 0, discovered: 3, previouslyDiscovered: 40, extracted: 8, entities: 8, docsTotal: 40 })).toBe("failing");222  });223});224