spb/rareindex

1import { sha256, type ConnectorHealth, type NormalizedRecord } from '@rareindex/shared';2import { healthFromRuns } from './health.js';3import { policyFor, type DomainPolicy } from './domains.js';4import type { ConnectorMeta, CrawlContext, HealthContext, RareIndexConnector, RawRecordInput, RawRecordLike } from './types.js';56/**7 * Convenience base class. Connectors may implement the interface directly; this class adds8 * rate limiting, checkpointing helpers and a default health check.9 */10export abstract class BaseConnector implements RareIndexConnector {11  abstract readonly version: string;12  abstract readonly parserVersion: string;13  readonly urlPatterns?: RegExp[];14  /** minimum ms between requests to the source (politeness) */15  protected minIntervalMs = 500;16  private lastRequestAt = 0;1718  constructor(public readonly meta: ConnectorMeta) {}1920  abstract crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput>;21  abstract normalize(raw: RawRecordLike): Promise<NormalizedRecord[]>;2223  async healthCheck(ctx: HealthContext): Promise<ConnectorHealth> {24    const h = healthFromRuns(ctx, this.meta.schemaVersion);25    if (ctx.recentRuns.length === 0) {26      // Fresh connector: light probe of the source root.27      const res = await ctx.fetch(this.meta.sourceUrl, { engines: ['api'], failOnHttpError: false, responseType: 'text', minQuality: 0 });28      h.status = res.httpStatus && res.httpStatus < 500 ? 'unknown' : 'failing';29      h.last_error = res.error;30    }31    return h;32  }3334  /** Politeness delay between requests: the larger of the connector's own interval and the domain policy (SPEC §16). */35  protected async throttle(url?: string): Promise<void> {36    const interval = Math.max(this.minIntervalMs, policyFor(url ?? this.meta.sourceUrl).minIntervalMs);37    const wait = this.lastRequestAt + interval - Date.now();38    if (wait > 0) await new Promise((r) => setTimeout(r, wait));39    this.lastRequestAt = Date.now();40  }4142  /** Effective per-domain policy for this connector's source. */43  protected get policy(): DomainPolicy {44    return policyFor(this.meta.sourceUrl);45  }4647  /** Stable content hash for dedupe of raw payloads. */48  protected hash(payload: unknown): string {49    return sha256(typeof payload === 'string' ? payload : JSON.stringify(payload));50  }5152  /** Helper: yield-limit aware counter. */53  protected reached(ctx: CrawlContext, count: number): boolean {54    return ctx.options.limit !== undefined && count >= ctx.options.limit;55  }56}57