1import { sha256, type ConnectorHealth, type NormalizedRecord } from '@rareindex/shared';2import { healthFromRuns } from './health.js';3import type { ConnectorMeta, CrawlContext, HealthContext, RareIndexConnector, RawRecordInput, RawRecordLike } from './types.js';45/**6 * Convenience base class. Connectors may implement the interface directly; this class adds7 * rate limiting, checkpointing helpers and a default health check.8 */9export abstract class BaseConnector implements RareIndexConnector {10 abstract readonly version: string;11 abstract readonly parserVersion: string;12 readonly urlPatterns?: RegExp[];13 /** minimum ms between requests to the source (politeness) */14 protected minIntervalMs = 500;15 private lastRequestAt = 0;1617 constructor(public readonly meta: ConnectorMeta) {}1819 abstract crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput>;20 abstract normalize(raw: RawRecordLike): Promise<NormalizedRecord[]>;2122 async healthCheck(ctx: HealthContext): Promise<ConnectorHealth> {23 const h = healthFromRuns(ctx, this.meta.schemaVersion);24 if (ctx.recentRuns.length === 0) {25 // Fresh connector: light probe of the source root.26 const res = await ctx.fetch(this.meta.sourceUrl, { engines: ['api'], failOnHttpError: false, responseType: 'text', minQuality: 0 });27 h.status = res.httpStatus && res.httpStatus < 500 ? 'unknown' : 'failing';28 h.last_error = res.error;29 }30 return h;31 }3233 /** Politeness delay between requests. */34 protected async throttle(): Promise<void> {35 const wait = this.lastRequestAt + this.minIntervalMs - Date.now();36 if (wait > 0) await new Promise((r) => setTimeout(r, wait));37 this.lastRequestAt = Date.now();38 }3940 /** Stable content hash for dedupe of raw payloads. */41 protected hash(payload: unknown): string {42 return sha256(typeof payload === 'string' ? payload : JSON.stringify(payload));43 }4445 /** Helper: yield-limit aware counter. */46 protected reached(ctx: CrawlContext, count: number): boolean {47 return ctx.options.limit !== undefined && count >= ctx.options.limit;48 }49}50