1import { z } from 'zod';2import type { Logger } from '@rareindex/shared';3import { EngineSchema, SourceTypeSchema, type Engine, type ExtractionResult, type NormalizedRecord, type RawRecord, type ConnectorHealth, type RecordKind } from '@rareindex/shared';45/** Registry entry — mirrors connectors/registry.json (§106). */6export const ConnectorMetaSchema = z.object({7 id: z.string().regex(/^[a-z0-9-]+$/),8 displayName: z.string(),9 sourceId: z.string(),10 sourceName: z.string(),11 sourceType: SourceTypeSchema,12 sourceUrl: z.string().url(),13 /** module path relative to /connectors, e.g. "api/scryfall" */14 module: z.string(),15 enginePriority: z.array(EngineSchema).min(1),16 categories: z.array(z.string()).min(1),17 regions: z.array(z.string()).default([]),18 languages: z.array(z.string()).default(['en']),19 currency: z.array(z.string()).default(['USD']),20 supportsListings: z.boolean().default(false),21 supportsSold: z.boolean().default(false),22 supportsAuctions: z.boolean().default(false),23 supportsImages: z.boolean().default(true),24 supportsCatalog: z.boolean().default(false),25 supportsPopulation: z.boolean().default(false),26 supportsLookup: z.boolean().default(false),27 refreshFrequencyMinutes: z.number().int().positive().default(1440),28 priority: z.enum(['high', 'medium', 'low']).default('medium'),29 /** 0–1 baseline trust for the source (§143) */30 trustScore: z.number().min(0).max(1).default(0.6),31 attributionRequired: z.boolean().default(true),32 termsUrl: z.string().optional(),33 /** legal/ethical notes: what we do and do not fetch from this source */34 accessNotes: z.string().optional(),35 enabled: z.boolean().default(true),36 schemaVersion: z.string().default('1.0'),37 config: z.record(z.string(), z.unknown()).default({}),38});39export type ConnectorMeta = z.infer<typeof ConnectorMetaSchema>;4041export const RegistrySchema = z.object({42 version: z.string(),43 connectors: z.array(ConnectorMetaSchema),44});45export type Registry = z.infer<typeof RegistrySchema>;4647/** What a connector yields while crawling; the worker adds ids, run id and persistence. */48export interface RawRecordInput {49 url: string;50 externalId?: string | null;51 kind: RecordKind;52 engine: Engine;53 httpStatus?: number | null;54 payload: unknown;55 /** optional large text (HTML/markdown) persisted to disk, not the DB */56 snapshot?: string | null;57 fetchedAt?: Date;58}5960export type QualityField = 'title' | 'price' | 'status' | 'date' | 'images' | 'description' | 'identifiers' | 'category' | 'currency';6162export interface FetchOptions {63 /** Restrict/override the engines to try, in order. Defaults to meta.enginePriority. */64 engines?: Engine[];65 /** Fields the parser is expected to find; drives the quality score (§199). */66 expect?: QualityField[];67 /** Parse the fetched document; return the fields found so the router can score quality. */68 parse?: (res: ExtractionResult) => Partial<Record<QualityField, unknown>> | null;69 /** Minimum quality to accept an engine result before falling back (default 0.6). */70 minQuality?: number;71 /** Engine hints */72 renderJs?: boolean;73 waitForMs?: number;74 country?: string;75 headers?: Record<string, string>;76 /** Firecrawl JSON extraction schema (when using the json format) */77 jsonSchema?: Record<string, unknown>;78 jsonPrompt?: string;79 /** Scrapfly extraction model (e.g. "product") */80 extractionModel?: string;81 timeoutMs?: number;82 /** skip cache/budget checks */83 force?: boolean;84 /** treat non-2xx as failure (default true) */85 failOnHttpError?: boolean;86 /** Body for POST requests (api engine) */87 method?: 'GET' | 'POST';88 body?: unknown;89 /** Return type for api engine: json (default) or text */90 responseType?: 'json' | 'text';91}9293export interface CrawlOptions {94 mode: 'incremental' | 'backfill' | 'probe';95 /** soft cap on records to yield (probe/tests) */96 limit?: number;97 /** cursor persisted between runs (connector-defined shape) */98 cursor?: Record<string, unknown>;99 /** optional category filter (slugs) */100 categories?: string[];101 /** optional seed URLs/queries (manual runs) */102 seeds?: string[];103}104105export interface EngineStats {106 attempts: number;107 success: number;108 credits: number;109 ms: number;110}111112export interface CrawlContext {113 meta: ConnectorMeta;114 log: Logger;115 options: CrawlOptions;116 /** Routed fetch: api/http → firecrawl → scrapfly (§198). Records stats and cost. */117 fetch(url: string, opts?: FetchOptions): Promise<ExtractionResult>;118 /** Firecrawl search (when available) — returns urls + snippets/markdown. */119 search?(query: string, opts?: { limit?: number; scrape?: boolean; country?: string }): Promise<Array<{ url: string; title?: string; description?: string; markdown?: string }>>;120 /** Firecrawl map — discover URLs of a site. */121 map?(url: string, opts?: { search?: string; limit?: number }): Promise<string[]>;122 /** Crawl budget: true when the URL content changed since last time or is due (§170). */123 shouldFetch(url: string): Promise<boolean>;124 /** Persist an updated cursor (called by connector at checkpoints). */125 setCursor(cursor: Record<string, unknown>): Promise<void>;126 /** Report a non-fatal anomaly for health tracking (§105). */127 anomaly(kind: string, detail?: string): void;128 engineStats: Record<string, EngineStats>;129 /** Cooperative cancellation */130 signal?: AbortSignal;131}132133export interface HealthContext {134 meta: ConnectorMeta;135 log: Logger;136 fetch(url: string, opts?: FetchOptions): Promise<ExtractionResult>;137 /** Recent run metrics supplied by the worker (may be empty for a fresh connector). */138 recentRuns: Array<{ startedAt: Date; status: string; pagesAttempted: number; pagesSuccess: number; recordsRaw: number; recordsDuplicate: number; engineStats: Record<string, EngineStats>; anomalies: string[]; error: string | null }>;139}140141/**142 * The connector contract (§102). Every source implements exactly this. Everything else —143 * persistence, dedupe, entity resolution, valuation — is generic pipeline code.144 */145export interface RareIndexConnector {146 readonly meta: ConnectorMeta;147 /** semantic version of connector code; bump when crawl logic changes */148 readonly version: string;149 /** version of the parsing/normalisation logic; bump when field mapping changes */150 readonly parserVersion: string;151 /** Yield raw records. Must be resumable through options.cursor. */152 crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput>;153 /** Pure function: raw payload → canonical normalised records. Must be deterministic and fixture-testable. */154 normalize(raw: RawRecordLike): Promise<NormalizedRecord[]>;155 /** Liveness/probe check. Base implementation derives status from recent runs and a light probe. */156 healthCheck(ctx: HealthContext): Promise<ConnectorHealth>;157 /** Optional: resolve a single source URL (scanner/paste URL, §114). */158 lookup?(url: string, ctx: CrawlContext): Promise<RawRecordInput[]>;159 /** Optional: URL patterns this connector understands (for lookup routing). */160 readonly urlPatterns?: RegExp[];161}162163export type RawRecordLike = Pick<RawRecord, 'url' | 'kind' | 'payload' | 'fetchedAt' | 'engine'> & { externalId?: string | null; id?: string; connectorId?: string; sourceId?: string };164165export type ConnectorFactory = () => RareIndexConnector | Promise<RareIndexConnector>;166