spb/rareindex

1import { eq, sql } from 'drizzle-orm';2import { crawlState } from '@rareindex/database';3import type { BudgetStore } from '@rareindex/connectors';4import { db } from './db.ts';56/**7 * Crawl budget engine (§170): per-URL content hash + change-interval estimate.8 * nextFetchAt = lastFetched + clamp(estimatedChangeInterval / 2, minH, maxH).9 * Unchanged fetches lengthen the interval (×1.5), changes shorten it (×0.5).10 */11export function createBudgetStore(opts: { connectorId: string; minHours?: number; maxHours?: number }): BudgetStore {12  const minH = opts.minHours ?? 1;13  const maxH = opts.maxHours ?? 24 * 14;14  return {15    async get(urlHash) {16      const [row] = await db().select({ contentHash: crawlState.contentHash, nextFetchAt: crawlState.nextFetchAt }).from(crawlState).where(eq(crawlState.urlHash, urlHash)).limit(1);17      return row ? { contentHash: row.contentHash, nextFetchAt: row.nextFetchAt } : null;18    },19    async record(input) {20      const now = new Date();21      const [prev] = await db().select().from(crawlState).where(eq(crawlState.urlHash, input.urlHash)).limit(1);22      let interval = prev?.changeIntervalHours ?? 24;23      if (prev) interval = input.changed ? Math.max(minH, interval * 0.5) : Math.min(maxH, interval * 1.5);24      const failures = input.status && input.status >= 400 ? (prev?.failures ?? 0) + 1 : 0;25      const backoff = failures > 0 ? Math.min(maxH, 2 ** failures) : interval / 2;26      const nextFetchAt = new Date(now.getTime() + Math.max(minH, backoff) * 3600_000);27      await db()28        .insert(crawlState)29        .values({30          urlHash: input.urlHash,31          connectorId: input.connectorId,32          url: input.url,33          contentHash: input.contentHash,34          lastFetchedAt: now,35          lastChangedAt: input.changed ? now : null,36          fetchCount: 1,37          changeCount: input.changed ? 1 : 0,38          changeIntervalHours: interval,39          nextFetchAt,40          lastStatus: input.status,41          failures,42        })43        .onConflictDoUpdate({44          target: crawlState.urlHash,45          set: {46            contentHash: input.contentHash,47            lastFetchedAt: now,48            lastChangedAt: input.changed ? now : sql`${crawlState.lastChangedAt}`,49            fetchCount: sql`${crawlState.fetchCount} + 1`,50            changeCount: sql`${crawlState.changeCount} + ${input.changed ? 1 : 0}`,51            changeIntervalHours: interval,52            nextFetchAt,53            lastStatus: input.status,54            failures,55          },56        });57    },58  };59}60