spb/rareindex

1import { existsSync, readFileSync } from 'node:fs';2import path from 'node:path';3import { fileURLToPath } from 'node:url';4import { z } from 'zod';5import { SourceTypeSchema } from '@rareindex/shared';67/**8 * Machine-readable source registry (SPEC §4): every candidate source we researched, implemented or9 * rejected, with the access/legal/technical facts needed to decide and to build a connector.10 * File: data/sources/sources.json. `docs/connectors/SOURCES.md` is generated from it.11 */12export const SourceStatusSchema = z.enum(['implemented', 'partial', 'planned', 'researching', 'rejected', 'blocked', 'gated']);13export const PaginationStyleSchema = z.enum(['page_number', 'cursor', 'offset', 'infinite_scroll', 'sitemap', 'feed', 'bulk_file', 'none', 'unknown']);14export const DifficultySchema = z.enum(['trivial', 'easy', 'medium', 'hard', 'very_hard']);15export const AntiBotSchema = z.enum(['none', 'low', 'medium', 'high', 'blocking']);16export const FreshnessSchema = z.enum(['realtime', 'hourly', 'daily', 'weekly', 'static', 'unknown']);1718export const SourceEntrySchema = z.object({19  id: z.string().regex(/^[a-z0-9-]+$/),20  name: z.string(),21  domain: z.string(),22  country: z.string(),23  supportedCountries: z.array(z.string()).default([]),24  languages: z.array(z.string()).default(['en']),25  sourceType: SourceTypeSchema,26  categories: z.array(z.string()).min(1),27  subcategories: z.array(z.string()).default([]),28  api: z.object({ available: z.boolean(), docsUrl: z.string().nullable().default(null), auth: z.enum(['none', 'api_key', 'oauth', 'partner', 'login']).default('none'), notes: z.string().nullable().default(null) }).prefault({ available: false }),29  urls: z.object({ search: z.string().nullable().default(null), product: z.string().nullable().default(null), sold: z.string().nullable().default(null), sitemap: z.string().nullable().default(null), robots: z.string().nullable().default(null) }).prefault({}),30  data: z.object({31    liveListings: z.boolean().default(false),32    soldResults: z.boolean().default(false),33    auctionResults: z.boolean().default(false),34    historical: z.enum(['none', 'months', 'years', 'decades']).default('none'),35    images: z.boolean().default(true),36    seller: z.boolean().default(false),37    saleDate: z.boolean().default(false),38    realizedPrice: z.boolean().default(false),39    fees: z.boolean().default(false),40    grading: z.boolean().default(false),41    certNumber: z.boolean().default(false),42    itemId: z.boolean().default(true),43    gtin: z.boolean().default(false),44    population: z.boolean().default(false),45    catalog: z.boolean().default(false),46  }).prefault({}),47  access: z.object({48    pagination: PaginationStyleSchema.default('unknown'),49    jsRendering: z.boolean().default(false),50    antiBot: AntiBotSchema.default('none'),51    cloudflare: z.boolean().default(false),52    robots: z.string().nullable().default(null),53    rateLimit: z.string().nullable().default(null),54    preferredEngine: z.enum(['api', 'feed', 'firecrawl', 'scrapfly', 'shopify', 'woocommerce', 'browser', 'none']).default('api'),55  }).prefault({}),56  currency: z.array(z.string()).default(['USD']),57  freshness: FreshnessSchema.default('unknown'),58  priority: z.enum(['wave1', 'wave2', 'wave3', 'wave4', 'later']).default('later'),59  difficulty: DifficultySchema.default('medium'),60  /** 0–1 expected reliability of parsed data */61  reliability: z.number().min(0).max(1).default(0.6),62  legal: z.string().nullable().default(null),63  status: SourceStatusSchema.default('researching'),64  /** connector ids implementing this source */65  connectors: z.array(z.string()).default([]),66  /** why rejected/blocked, or what remains for partial */67  statusReason: z.string().nullable().default(null),68  /** facts verified live (probe) vs desk research */69  verified: z.boolean().default(false),70  verifiedAt: z.string().nullable().default(null),71  notes: z.string().nullable().default(null),72});73export type SourceEntry = z.infer<typeof SourceEntrySchema>;7475export const SourcesFileSchema = z.object({ version: z.string().default('1.0'), updatedAt: z.string().optional(), sources: z.array(SourceEntrySchema) });76export type SourcesFile = z.infer<typeof SourcesFileSchema>;7778const here = path.dirname(fileURLToPath(import.meta.url));79export const SOURCES_PATH = path.resolve(here, '../../../data/sources/sources.json');8081let cached: SourcesFile | null = null;82export function loadSources(force = false): SourcesFile {83  if (cached && !force) return cached;84  if (!existsSync(SOURCES_PATH)) {85    cached = { version: '1.0', sources: [] };86    return cached;87  }88  cached = SourcesFileSchema.parse(JSON.parse(readFileSync(SOURCES_PATH, 'utf8')));89  const ids = new Set<string>();90  for (const s of cached.sources) {91    if (ids.has(s.id)) throw new Error(`sources.json: duplicate source id ${s.id}`);92    ids.add(s.id);93  }94  return cached;95}9697export function getSource(id: string): SourceEntry | undefined {98  return loadSources().sources.find((s) => s.id === id);99}100