spb/datacenterindex

1import type { Provenance } from "@dci/core";2import { contentFingerprint } from "@dci/core";3import type { ConnectorContext, FetchOptions, RawDocument } from "./types.js";4import type { ConnectorConfig } from "./config.js";5import { fetchWithEscalation, BROWSER_UA } from "./fetchers.js";6import { isAllowedByRobots } from "./robots.js";7import { acquire, configureHost } from "./ratelimit.js";8import type { FetchLevel } from "@dci/core";910/**11 * Minimal in-memory ConnectorContext for dry runs / tests / connector authoring.12 * Applies robots, rate limits and escalation exactly like the production runtime, but persists nothing.13 */14export function createTestContext(cfg: ConnectorConfig, opts: { verbose?: boolean; sourceId?: string } = {}): ConnectorContext & { fetched: RawDocument[]; credits: number } {15  const state = new Map<string, unknown>();16  const hashes = new Map<string, string>();17  const host = cfg.domain.replace(/^https?:\/\//, "").split("/")[0]!;18  configureHost(host, cfg.fetch.rpm, cfg.fetch.concurrency);19  const fetched: RawDocument[] = [];20  const ctx = {21    connectorId: cfg.id,22    sourceId: opts.sourceId ?? `src_${cfg.id}`,23    runId: `run_test_${Date.now().toString(36)}`,24    fetched,25    credits: 0,26    dryRun: true,27    env: process.env,28    now: () => new Date().toISOString(),29    log(level: "debug" | "info" | "warn" | "error", msg: string, extra?: Record<string, unknown>) {30      if (level === "debug" && !opts.verbose) return;31      console.error(`[${cfg.id}] ${level.toUpperCase()} ${msg}${extra ? " " + JSON.stringify(extra) : ""}`);32    },33    async getState<T>(key: string): Promise<T | null> { return (state.get(key) as T) ?? null; },34    async setState(key: string, value: unknown) { state.set(key, value); },35    async isKnownUnchanged(url: string, hash: string) { return hashes.get(url) === hash; },36    provenance(url: string, extra: Partial<Provenance> = {}): Provenance {37      const now = new Date().toISOString();38      return { sourceId: ctx.sourceId, connectorId: cfg.id, url, firstObserved: now, lastObserved: now, retrievedAt: now, confidence: cfg.kind === "operator" || cfg.kind === "government" || cfg.kind === "filing" || cfg.kind === "utility" || cfg.kind === "cloud_provider" || cfg.kind === "registry" ? "high" : "moderate", extractorVersion: cfg.parserVersion, ...extra };39    },40    async fetch(url: string, o: FetchOptions = {}): Promise<RawDocument> {41      let robotsUnknown = false;42      let crawlDelay: number | null = null;43      if (cfg.fetch.respectRobots) {44        const r = await isAllowedByRobots(url).catch(() => ({ allowed: true, crawlDelay: null as number | null, sitemaps: [] as string[], unknown: true }));45        robotsUnknown = r.unknown;46        crawlDelay = r.crawlDelay;47        if (!r.allowed) { ctx.log("warn", `robots.txt disallows ${url}`); return { url, finalUrl: url, fetchedAt: new Date().toISOString(), status: 0, contentType: null, body: Buffer.alloc(0), text: "", headers: {}, etag: null, lastModified: null, notModified: false, fetcher: "direct", level: 1, durationMs: 0, credits: 0, error: { code: "robots_disallow", message: "disallowed by robots.txt" } }; }48      }49      const release = await acquire(new URL(url).hostname, crawlDelay != null ? Math.min(30_000, Math.max(0, crawlDelay * 1000)) : 0);50      try {51        const level = Math.max(o.level ?? 1, cfg.fetch.level) as FetchLevel;52        // robots.txt unavailable → direct fetches only (same rule as the production context); run budget bounds premium spend53        const maxLevel = (robotsUnknown ? Math.min(2, cfg.fetch.maxLevel) : cfg.fetch.maxLevel) as FetchLevel;54        const creditsLeft = Math.max(0, cfg.fetch.maxCreditsPerRun - ctx.credits);55        const doc = await fetchWithEscalation(url, { ...o, level: Math.min(level, maxLevel) as FetchLevel, maxLevel, creditsLeft, renderJs: o.renderJs ?? cfg.fetch.renderJs, country: o.country ?? cfg.fetch.country, waitForSelector: o.waitForSelector ?? cfg.fetch.waitForSelector, timeoutMs: o.timeoutMs ?? cfg.fetch.timeoutMs, maxBytes: o.maxBytes ?? cfg.fetch.maxBytes, accept: o.accept ?? cfg.fetch.accept, headers: { ...(cfg.fetch.userAgent === "browser" ? { "user-agent": BROWSER_UA } : {}), ...(cfg.fetch.headers ?? {}), ...(o.headers ?? {}) } });56        if (robotsUnknown) doc.meta = { ...(doc.meta ?? {}), robotsUnknown: true };57        ctx.credits += doc.credits;58        if (!doc.error && doc.body.length) hashes.set(url, contentFingerprint(doc.text, doc.contentType));59        fetched.push(doc);60        ctx.log("debug", `fetch ${url} → ${doc.error?.code ?? doc.status} via ${doc.fetcher} L${doc.level} ${doc.durationMs}ms ${doc.body.length}b`);61        return doc;62      } finally { release(); }63    },64  };65  return ctx;66}67