spb/rareindex

1import { existsSync, readFileSync, readdirSync } from 'node:fs';2import path from 'node:path';3import { fileURLToPath } from 'node:url';4import { z } from 'zod';5import { EngineSchema } from '@rareindex/shared';6import { domainOf, RefreshClassSchema } from './types.js';78/**9 * Central per-domain acquisition policy (SPEC §16). One file — connectors/domains.json — holds10 * every rate limit, concurrency cap, timeout, retry count, engine policy, locale, currency and11 * crawl-depth setting. Connector code must not hard-code these; it asks `policyFor(url)`.12 *13 * Resolution order: exact host → parent domain (sub.example.com → example.com) → `*` defaults.14 */15export const DomainPolicySchema = z.object({16  /** minimum spacing between two requests to this host */17  minIntervalMs: z.number().int().nonnegative().default(1000),18  /** maximum simultaneous in-flight requests to this host (all connectors together) */19  concurrency: z.number().int().positive().default(2),20  timeoutMs: z.number().int().positive().default(45_000),21  maxRetries: z.number().int().nonnegative().default(2),22  /** allowed engines in order; connectors may narrow but not widen */23  engines: z.array(EngineSchema).optional(),24  /** Firecrawl policy */25  firecrawl: z.object({ waitForMs: z.number().int().nonnegative().optional(), onlyMainContent: z.boolean().optional(), maxCreditsPerRun: z.number().int().positive().optional() }).default({}),26  /** Scrapfly policy */27  scrapfly: z.object({ renderJs: z.boolean().optional(), asp: z.boolean().optional(), country: z.string().optional(), proxyPool: z.enum(['datacenter', 'residential']).optional(), maxCreditsPerRun: z.number().int().positive().optional() }).default({}),28  /** ISO country used for geo routing (Firecrawl location / Scrapfly country) */29  country: z.string().optional(),30  locale: z.string().optional(),31  currency: z.string().optional(),32  /** default page depth for incremental crawls */33  crawlDepth: z.number().int().positive().default(3),34  /** hard cap of pages a single backfill run may fetch */35  backfillMaxPages: z.number().int().positive().default(200),36  /** honest bot UA (default) or a browser-like UA (only where the site serves bots a degraded page and terms allow) */37  userAgent: z.enum(['bot', 'browser']).default('bot'),38  refreshClass: RefreshClassSchema.optional(),39  /** consecutive failures before the circuit opens */40  circuitFailures: z.number().int().positive().default(6),41  /** how long the circuit stays open */42  circuitCooldownMs: z.number().int().positive().default(10 * 60_000),43  /** robots.txt / terms observations kept next to the policy for auditability */44  notes: z.string().optional(),45});46export type DomainPolicy = z.infer<typeof DomainPolicySchema>;4748/** Patch form (no defaults applied — Zod's .partial() would still fill defaults, breaking layered merges). */49export const DomainPolicyPatchSchema = z.object({50  minIntervalMs: z.number().int().nonnegative().optional(),51  concurrency: z.number().int().positive().optional(),52  timeoutMs: z.number().int().positive().optional(),53  maxRetries: z.number().int().nonnegative().optional(),54  engines: z.array(EngineSchema).optional(),55  firecrawl: z.object({ waitForMs: z.number().int().nonnegative().optional(), onlyMainContent: z.boolean().optional(), maxCreditsPerRun: z.number().int().positive().optional() }).optional(),56  scrapfly: z.object({ renderJs: z.boolean().optional(), asp: z.boolean().optional(), country: z.string().optional(), proxyPool: z.enum(['datacenter', 'residential']).optional(), maxCreditsPerRun: z.number().int().positive().optional() }).optional(),57  country: z.string().optional(),58  locale: z.string().optional(),59  currency: z.string().optional(),60  crawlDepth: z.number().int().positive().optional(),61  backfillMaxPages: z.number().int().positive().optional(),62  userAgent: z.enum(['bot', 'browser']).optional(),63  refreshClass: RefreshClassSchema.optional(),64  circuitFailures: z.number().int().positive().optional(),65  circuitCooldownMs: z.number().int().positive().optional(),66  notes: z.string().optional(),67});68export type DomainPolicyPatch = z.infer<typeof DomainPolicyPatchSchema>;6970export const DomainsFileSchema = z.object({71  version: z.string().default('1.0'),72  defaults: DomainPolicyPatchSchema.default({}),73  domains: z.record(z.string(), DomainPolicyPatchSchema).default({}),74});75export type DomainsFile = z.infer<typeof DomainsFileSchema>;7677const here = path.dirname(fileURLToPath(import.meta.url));78export const DOMAINS_PATH = path.resolve(here, '../../../connectors/domains.json');79/** Fragment directory: connectors/domains.d/<group>.json = { domains: {...} } merged over domains.json (lets parallel contributors add hosts without conflicts). */80export const DOMAINS_DIR = path.resolve(here, '../../../connectors/domains.d');8182let cached: DomainsFile | null = null;83const policyCache = new Map<string, DomainPolicy>();8485export function loadDomains(force = false): DomainsFile {86  if (cached && !force) return cached;87  if (!existsSync(DOMAINS_PATH)) {88    cached = DomainsFileSchema.parse({});89    return cached;90  }91  const base = DomainsFileSchema.parse(JSON.parse(readFileSync(DOMAINS_PATH, 'utf8')));92  if (existsSync(DOMAINS_DIR)) {93    for (const f of readdirSync(DOMAINS_DIR).filter((x) => x.endsWith('.json')).sort()) {94      // A malformed fragment must never take the whole pipeline down: skip it loudly (pnpm registry validates strictly).95      try {96        const frag = DomainsFileSchema.parse({ ...JSON.parse(readFileSync(path.join(DOMAINS_DIR, f), 'utf8')), version: base.version });97        for (const [host, patch] of Object.entries(frag.domains)) base.domains[host] = { ...(base.domains[host] ?? {}), ...patch };98      } catch (err) {99        console.warn(`[domains] ignoring invalid fragment ${f}: ${err instanceof Error ? err.message.slice(0, 300) : String(err)}`);100      }101    }102  }103  cached = base;104  return cached;105}106107/** Testing/embedding hook: replace the loaded file. */108export function setDomains(file: DomainsFile): void {109  cached = DomainsFileSchema.parse(file);110  policyCache.clear();111}112113/** Resolve the effective policy for a URL or host. */114export function policyFor(urlOrHost: string): DomainPolicy {115  const host = urlOrHost.includes('://') ? domainOf(urlOrHost) : urlOrHost.replace(/^www\./, '').toLowerCase();116  const hit = policyCache.get(host);117  if (hit) return hit;118  const file = loadDomains();119  const layers: DomainPolicyPatch[] = [file.defaults];120  const parts = host.split('.');121  // parent domains first, exact host last (exact wins)122  for (let i = parts.length - 2; i >= 0; i--) {123    const candidate = parts.slice(i).join('.');124    const layer = file.domains[candidate];125    if (layer) layers.push(layer);126  }127  const merged = layers.reduce<Record<string, unknown>>((acc, l) => ({ ...acc, ...stripUndefined(l), firecrawl: { ...((acc.firecrawl as object) ?? {}), ...(l.firecrawl ?? {}) }, scrapfly: { ...((acc.scrapfly as object) ?? {}), ...(l.scrapfly ?? {}) } }), {});128  const policy = DomainPolicySchema.parse(merged);129  policyCache.set(host, policy);130  return policy;131}132133export function clearPolicyCache(): void {134  policyCache.clear();135}136137function stripUndefined<T extends object>(o: T): Partial<T> {138  return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined)) as Partial<T>;139}140