TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import type { Logger } from '@rareindex/shared';3import { EngineSchema, SourceTypeSchema, type Engine, type ExtractionResult, type NormalizedRecord, type RawRecord, type ConnectorHealth, type RecordKind } from '@rareindex/shared';45/** Explicit connector capabilities (SPEC §5). Derived from the supports* flags when omitted. */6export const CapabilitySchema = z.enum([7 'search', 'live_listings', 'sold_listings', 'auction_results', 'grading_population', 'cert_lookup', 'catalog', 'images', 'seller_data', 'price_guide', 'historical_backfill', 'news', 'url_lookup',8]);9export type Capability = z.infer<typeof CapabilitySchema>;1011/** Refresh scheduling classes (SPEC §17). */12export const RefreshClassSchema = z.enum(['hot', 'active', 'normal', 'archive']);13export type RefreshClass = z.infer<typeof RefreshClassSchema>;14export const REFRESH_CLASS_MINUTES: Record<RefreshClass, { min: number; max: number; default: number }> = {15 hot: { min: 1, max: 5, default: 5 },16 active: { min: 15, max: 60, default: 30 },17 normal: { min: 360, max: 1440, default: 1440 },18 archive: { min: 7 * 1440, max: 31 * 1440, default: 7 * 1440 },19};2021export function refreshClassFor(minutes: number): RefreshClass {22 if (minutes <= 5) return 'hot';23 if (minutes <= 60) return 'active';24 if (minutes <= 1440) return 'normal';25 return 'archive';26}2728/** Registry entry — mirrors connectors/registry.json (§106). */29export const ConnectorMetaSchema = z.object({30 id: z.string().regex(/^[a-z0-9-]+$/),31 displayName: z.string(),32 sourceId: z.string(),33 sourceName: z.string(),34 sourceType: SourceTypeSchema,35 sourceUrl: z.string().url(),36 /** module path relative to /connectors, e.g. "api/scryfall" */37 module: z.string(),38 enginePriority: z.array(EngineSchema).min(1),39 categories: z.array(z.string()).min(1),40 regions: z.array(z.string()).default([]),41 languages: z.array(z.string()).default(['en']),42 currency: z.array(z.string()).default(['USD']),43 supportsListings: z.boolean().default(false),44 supportsSold: z.boolean().default(false),45 supportsAuctions: z.boolean().default(false),46 supportsImages: z.boolean().default(true),47 supportsCatalog: z.boolean().default(false),48 supportsPopulation: z.boolean().default(false),49 supportsLookup: z.boolean().default(false),50 refreshFrequencyMinutes: z.number().int().positive().default(1440),51 priority: z.enum(['high', 'medium', 'low']).default('medium'),52 /** 0–1 baseline trust for the source (§143) */53 trustScore: z.number().min(0).max(1).default(0.6),54 attributionRequired: z.boolean().default(true),55 termsUrl: z.string().optional(),56 /** legal/ethical notes: what we do and do not fetch from this source */57 accessNotes: z.string().optional(),58 enabled: z.boolean().default(true),59 schemaVersion: z.string().default('1.0'),60 config: z.record(z.string(), z.unknown()).default({}),61 /** Explicit capabilities (SPEC §5); when empty they are derived from the supports* flags. */62 capabilities: z.array(CapabilitySchema).default([]),63 /** Scheduling class (SPEC §17); derived from refreshFrequencyMinutes when omitted. */64 refreshClass: RefreshClassSchema.optional(),65 /** Primary host (derived from sourceUrl by the registry builder). Key into connectors/domains.json. */66 domain: z.string().optional(),67 /** Primary ISO country of the source (defaults to regions[0]). */68 country: z.string().optional(),69 /** Environment variables the connector needs (API keys). Missing → connector reported as `disabled`, never crashes. */70 requires: z.array(z.string()).default([]),71 /** Short human label of how data is acquired: "official API", "Shopify products.json", "Firecrawl markdown", "Scrapfly render"… */72 acquisitionMethod: z.string().optional(),73 logoUrl: z.string().optional(),74 /** Does the source expose deep historical results worth a resumable backfill (SPEC §9)? */75 historicalDepth: z.enum(['none', 'months', 'years', 'decades']).default('none'),76});77export type ConnectorMeta = z.infer<typeof ConnectorMetaSchema>;7879/** Effective capability list: declared ∪ derived from the supports* flags (never fewer than the flags say). */80export function capabilitiesOf(meta: ConnectorMeta): Capability[] {81 const set = new Set<Capability>(meta.capabilities);82 if (meta.supportsListings) set.add('live_listings');83 if (meta.supportsSold) set.add('sold_listings');84 if (meta.supportsAuctions) set.add('auction_results');85 if (meta.supportsImages) set.add('images');86 if (meta.supportsCatalog) set.add('catalog');87 if (meta.supportsPopulation) set.add('grading_population');88 if (meta.supportsLookup) set.add('url_lookup');89 if (meta.historicalDepth !== 'none') set.add('historical_backfill');90 return [...set];91}9293/** Effective refresh class (declared or derived from the refresh frequency). */94export function effectiveRefreshClass(meta: ConnectorMeta): RefreshClass {95 return meta.refreshClass ?? refreshClassFor(meta.refreshFrequencyMinutes);96}9798/** Env variables declared in `requires` that are absent from process.env. Empty = runnable. */99export function missingRequirements(meta: ConnectorMeta, envSource: Record<string, string | undefined> = process.env): string[] {100 return (meta.requires ?? []).filter((k) => !envSource[k] || !String(envSource[k]).trim());101}102103export function domainOf(url: string): string {104 try {105 return new URL(url).hostname.replace(/^www\./, '').toLowerCase();106 } catch {107 return url;108 }109}110111export const RegistrySchema = z.object({112 version: z.string(),113 connectors: z.array(ConnectorMetaSchema),114});115export type Registry = z.infer<typeof RegistrySchema>;116117/** What a connector yields while crawling; the worker adds ids, run id and persistence. */118export interface RawRecordInput {119 url: string;120 externalId?: string | null;121 kind: RecordKind;122 engine: Engine;123 httpStatus?: number | null;124 payload: unknown;125 /** optional large text (HTML/markdown) persisted to disk, not the DB */126 snapshot?: string | null;127 fetchedAt?: Date;128}129130export type QualityField = 'title' | 'price' | 'status' | 'date' | 'images' | 'description' | 'identifiers' | 'category' | 'currency';131132export interface FetchOptions {133 /** Restrict/override the engines to try, in order. Defaults to meta.enginePriority. */134 engines?: Engine[];135 /** Fields the parser is expected to find; drives the quality score (§199). */136 expect?: QualityField[];137 /** Parse the fetched document; return the fields found so the router can score quality. */138 parse?: (res: ExtractionResult) => Partial<Record<QualityField, unknown>> | null;139 /** Minimum quality to accept an engine result before falling back (default 0.6). */140 minQuality?: number;141 /** Engine hints */142 renderJs?: boolean;143 waitForMs?: number;144 country?: string;145 headers?: Record<string, string>;146 /** Firecrawl JSON extraction schema (when using the json format) */147 jsonSchema?: Record<string, unknown>;148 jsonPrompt?: string;149 /** Scrapfly extraction model (e.g. "product") */150 extractionModel?: string;151 timeoutMs?: number;152 /** skip cache/budget checks */153 force?: boolean;154 /** treat non-2xx as failure (default true) */155 failOnHttpError?: boolean;156 /** Body for POST requests (api engine) */157 method?: 'GET' | 'POST';158 body?: unknown;159 /** Return type for api engine: json (default), text, or binary (PDF, gzip) */160 responseType?: 'json' | 'text' | 'binary';161 /** Bypass the per-domain circuit breaker / concurrency gate (health probes). */162 ignoreCircuit?: boolean;163}164165export interface CrawlOptions {166 mode: 'incremental' | 'backfill' | 'probe';167 /** soft cap on records to yield (probe/tests) */168 limit?: number;169 /** cursor persisted between runs (connector-defined shape) */170 cursor?: Record<string, unknown>;171 /** optional category filter (slugs) */172 categories?: string[];173 /** optional seed URLs/queries (manual runs) */174 seeds?: string[];175}176177export interface EngineStats {178 attempts: number;179 success: number;180 credits: number;181 ms: number;182 /** HTTP status histogram, e.g. {"200": 40, "403": 2} (SPEC §12 HTTP error distribution) */183 statuses?: Record<string, number>;184 /** Anti-bot challenge / captcha / block pages detected (SPEC §12) */185 blocked?: number;186 /** Requests refused by the per-domain circuit breaker */187 circuitOpen?: number;188}189190/** Backfill progress reported by connectors during `mode: 'backfill'` (SPEC §9). */191export interface BackfillProgress {192 /** last fully processed page/cursor position (connector-defined) */193 page?: number;194 totalPages?: number | null;195 itemsProcessed?: number;196 /** oldest date reached so far (crawling backwards) */197 reachedDate?: Date | null;198 cursor?: Record<string, unknown>;199}200201export interface CrawlContext {202 meta: ConnectorMeta;203 log: Logger;204 options: CrawlOptions;205 /** Routed fetch: api/http → firecrawl → scrapfly (§198). Records stats and cost. */206 fetch(url: string, opts?: FetchOptions): Promise<ExtractionResult>;207 /** Firecrawl search (when available) — returns urls + snippets/markdown. */208 search?(query: string, opts?: { limit?: number; scrape?: boolean; country?: string }): Promise<Array<{ url: string; title?: string; description?: string; markdown?: string }>>;209 /** Firecrawl map — discover URLs of a site. */210 map?(url: string, opts?: { search?: string; limit?: number }): Promise<string[]>;211 /** Crawl budget: true when the URL content changed since last time or is due (§170). */212 shouldFetch(url: string): Promise<boolean>;213 /** Persist an updated cursor (called by connector at checkpoints). */214 setCursor(cursor: Record<string, unknown>): Promise<void>;215 /** Report a non-fatal anomaly for health tracking (§105). */216 anomaly(kind: string, detail?: string): void;217 /** Report backfill progress (SPEC §9). No-op outside backfill mode. */218 progress(p: BackfillProgress): Promise<void>;219 engineStats: Record<string, EngineStats>;220 /** Cooperative cancellation */221 signal?: AbortSignal;222}223224export interface HealthContext {225 meta: ConnectorMeta;226 log: Logger;227 fetch(url: string, opts?: FetchOptions): Promise<ExtractionResult>;228 /** Recent run metrics supplied by the worker (may be empty for a fresh connector). */229 recentRuns: Array<{ startedAt: Date; status: string; pagesAttempted: number; pagesSuccess: number; recordsRaw: number; recordsDuplicate: number; engineStats: Record<string, EngineStats>; anomalies: string[]; error: string | null }>;230}231232/**233 * The connector contract (§102). Every source implements exactly this. Everything else —234 * persistence, dedupe, entity resolution, valuation — is generic pipeline code.235 */236export interface RareIndexConnector {237 readonly meta: ConnectorMeta;238 /** semantic version of connector code; bump when crawl logic changes */239 readonly version: string;240 /** version of the parsing/normalisation logic; bump when field mapping changes */241 readonly parserVersion: string;242 /** Yield raw records. Must be resumable through options.cursor. */243 crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput>;244 /** Pure function: raw payload → canonical normalised records. Must be deterministic and fixture-testable. */245 normalize(raw: RawRecordLike): Promise<NormalizedRecord[]>;246 /** Liveness/probe check. Base implementation derives status from recent runs and a light probe. */247 healthCheck(ctx: HealthContext): Promise<ConnectorHealth>;248 /** Optional: resolve a single source URL (scanner/paste URL, §114). */249 lookup?(url: string, ctx: CrawlContext): Promise<RawRecordInput[]>;250 /** Optional: URL patterns this connector understands (for lookup routing). */251 readonly urlPatterns?: RegExp[];252}253254export type RawRecordLike = Pick<RawRecord, 'url' | 'kind' | 'payload' | 'fetchedAt' | 'engine'> & { externalId?: string | null; id?: string; connectorId?: string; sourceId?: string };255256export type ConnectorFactory = (meta: ConnectorMeta) => RareIndexConnector | Promise<RareIndexConnector>;257