import { eq, sql } from 'drizzle-orm'; import { crawlState } from '@rareindex/database'; import type { BudgetStore } from '@rareindex/connectors'; import { db } from './db.ts'; /** * Crawl budget engine (§170): per-URL content hash + change-interval estimate. * nextFetchAt = lastFetched + clamp(estimatedChangeInterval / 2, minH, maxH). * Unchanged fetches lengthen the interval (×1.5), changes shorten it (×0.5). */ export function createBudgetStore(opts: { connectorId: string; minHours?: number; maxHours?: number }): BudgetStore { const minH = opts.minHours ?? 1; const maxH = opts.maxHours ?? 24 * 14; return { async get(urlHash) { const [row] = await db().select({ contentHash: crawlState.contentHash, nextFetchAt: crawlState.nextFetchAt }).from(crawlState).where(eq(crawlState.urlHash, urlHash)).limit(1); return row ? { contentHash: row.contentHash, nextFetchAt: row.nextFetchAt } : null; }, async record(input) { const now = new Date(); const [prev] = await db().select().from(crawlState).where(eq(crawlState.urlHash, input.urlHash)).limit(1); let interval = prev?.changeIntervalHours ?? 24; if (prev) interval = input.changed ? Math.max(minH, interval * 0.5) : Math.min(maxH, interval * 1.5); const failures = input.status && input.status >= 400 ? (prev?.failures ?? 0) + 1 : 0; const backoff = failures > 0 ? Math.min(maxH, 2 ** failures) : interval / 2; const nextFetchAt = new Date(now.getTime() + Math.max(minH, backoff) * 3600_000); await db() .insert(crawlState) .values({ urlHash: input.urlHash, connectorId: input.connectorId, url: input.url, contentHash: input.contentHash, lastFetchedAt: now, lastChangedAt: input.changed ? now : null, fetchCount: 1, changeCount: input.changed ? 1 : 0, changeIntervalHours: interval, nextFetchAt, lastStatus: input.status, failures, }) .onConflictDoUpdate({ target: crawlState.urlHash, set: { contentHash: input.contentHash, lastFetchedAt: now, lastChangedAt: input.changed ? now : sql`${crawlState.lastChangedAt}`, fetchCount: sql`${crawlState.fetchCount} + 1`, changeCount: sql`${crawlState.changeCount} + ${input.changed ? 1 : 0}`, changeIntervalHours: interval, nextFetchAt, lastStatus: input.status, failures, }, }); }, }; }