import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared'; /** * Numista catalogue API v3 (https://en.numista.com/api/doc/) โ€” the reference catalogue of world coins, * banknotes and exonumia (N# numbers). Requires a personal API key (header `Numista-API-Key`; free plan * 2 000 requests/month). Gated: the connector is reported DISABLED until NUMISTA_API_KEY exists. * Records: catalog_item per type (identifiers numista_id) and, when `fetchPrices` is on, guide_value * observations per grade for the first issue of each type (/types/{id}/issues/{issue}/prices). * Licence: attribution "Source: Numista" + N# is mandatory; bulk download and re-serving the data as a * feed are forbidden โ€” we store only the identifying fields we need and always attribute. */ const API = 'https://api.numista.com/v3'; const PARSER_VERSION = '1.0.0'; export const TypeSchema = z.object({ id: z.number(), title: z.string(), category: z.string().nullable().optional(), object_type: z.object({ id: z.number().nullable().optional(), name: z.string().nullable().optional() }).nullable().optional(), issuer: z.object({ code: z.string().nullable().optional(), name: z.string().nullable().optional() }).nullable().optional(), min_year: z.number().nullable().optional(), max_year: z.number().nullable().optional(), obverse_thumbnail: z.string().nullable().optional(), reverse_thumbnail: z.string().nullable().optional(), /** filled from GET /types/{id} when `fetchDetails` is on */ value: z.object({ text: z.string().nullable().optional(), numeric_value: z.number().nullable().optional(), currency: z.object({ id: z.number().nullable().optional(), name: z.string().nullable().optional(), full_name: z.string().nullable().optional() }).nullable().optional() }).nullable().optional(), composition: z.object({ text: z.string().nullable().optional() }).nullable().optional(), ruler: z.array(z.object({ id: z.number().nullable().optional(), name: z.string().nullable().optional() })).nullable().optional(), url: z.string().nullable().optional(), references: z.array(z.object({ catalogue: z.object({ code: z.string().nullable().optional() }).nullable().optional(), number: z.string().nullable().optional() })).nullable().optional(), }); export type NumistaType = z.infer; export const PricesSchema = z.object({ issue_id: z.number().nullable(), currency: z.string(), prices: z.array(z.object({ grade: z.string().optional(), gradde: z.string().optional(), price: z.number() })) }); export const PayloadSchema = z.object({ kind: z.literal('types_page'), url: z.string(), query: z.record(z.string(), z.string()), page: z.number().int(), count: z.number().int().nullable(), types: z.array(TypeSchema), /** N# โ†’ issue prices (guide values) when fetched */ prices: z.record(z.string(), PricesSchema).default({}), /** observation date = fetch date of the price call (Numista prices are live estimates) */ pricesFetchedAt: z.string().nullable().default(null), }); export type Payload = z.infer; const GRADE_LABEL: Record = { g: 'G', vg: 'VG', f: 'F', vf: 'VF', xf: 'XF', au: 'AU', unc: 'UNC' }; const GRADE_CONDITION: Record = { g: 'good', vg: 'very_good', f: 'fine', vf: 'very_fine', xf: 'extremely_fine', au: 'about_uncirculated', unc: 'mint_state' }; export function numistaCategory(t: NumistaType): string { const c = (t.category ?? '').toLowerCase(); if (c === 'banknote') return 'banknotes'; if (c === 'exonumia') return 'medals'; return 'coins'; } interface Cursor { seedIndex?: number; page?: number; done?: boolean; updatedAt?: string; } export class NumistaConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/numista\.com\/(?:catalogue\/pieces)?(\d+)/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const key = process.env.NUMISTA_API_KEY; if (!key) { ctx.anomaly('config_missing', 'NUMISTA_API_KEY is not set (connector is gated)'); return; } const headers = { 'Numista-API-Key': key }; const seeds = (this.meta.config.seeds as Array> | undefined) ?? []; const count = Number(this.meta.config.count ?? 50); const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 4); const fetchPrices = Boolean(this.meta.config.fetchPrices ?? false); const priceCurrency = String(this.meta.config.priceCurrency ?? 'EUR'); const maxPricesPerRun = Number(this.meta.config.maxPricesPerRun ?? 20); const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) }; let seedIndex = Math.min(cursor.seedIndex ?? 0, Math.max(seeds.length - 1, 0)); let page = cursor.page ?? 1; let pages = 0; let priceCalls = 0; let yielded = 0; while (seedIndex < seeds.length && pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) { const seed = seeds[seedIndex]!; const q = new URLSearchParams({ ...seed, page: String(page), count: String(count), lang: 'en' }); const url = `${API}/types?${q}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], headers, expect: ['title', 'identifiers'], parse: (r) => { const t = (r.json as { types?: Array<{ id?: number; title?: string }> } | null)?.types?.[0]; return t ? { title: t.title, identifiers: { numista: String(t.id) } } : null; } }); pages++; const parsed = res.success ? z.object({ count: z.union([z.number(), z.string()]).nullable().optional(), types: z.array(z.unknown()) }).safeParse(res.json) : null; if (!parsed?.success) { ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const types = parsed.data.types.map((t) => TypeSchema.safeParse(t)).filter((r) => r.success).map((r) => r.data); const prices: Record> = {}; let pricesFetchedAt: string | null = null; if (fetchPrices) { for (const t of types) { if (priceCalls >= maxPricesPerRun || ctx.signal?.aborted) break; await this.throttle(); const issues = await ctx.fetch(`${API}/types/${t.id}/issues?lang=en`, { engines: ['api'], headers, minQuality: 0 }); priceCalls++; const first = (issues.json as Array<{ id?: number }> | null)?.[0]?.id; if (!issues.success || !first) continue; await this.throttle(); const pr = await ctx.fetch(`${API}/types/${t.id}/issues/${first}/prices?currency=${priceCurrency}&lang=en`, { engines: ['api'], headers, minQuality: 0 }); priceCalls++; const pp = pr.success ? PricesSchema.omit({ issue_id: true }).safeParse(pr.json) : null; if (pp?.success) { prices[String(t.id)] = { issue_id: first, ...pp.data }; pricesFetchedAt = pr.fetchedAt.toISOString(); } } } if (types.length) { yielded++; const payload: Payload = { kind: 'types_page', url, query: seed, page, count: parsed.data.count !== null && parsed.data.count !== undefined ? Number(parsed.data.count) : null, types, prices, pricesFetchedAt }; yield { url, externalId: `types:${JSON.stringify(seed)}:p${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } const total = parsed.data.count !== null && parsed.data.count !== undefined ? Number(parsed.data.count) : null; const more = types.length === count && (total === null || page * count < total); if (more) page++; else { seedIndex++; page = 1; } await ctx.setCursor({ seedIndex: seedIndex >= seeds.length ? 0 : seedIndex, page: seedIndex >= seeds.length ? 1 : page, done: ctx.options.mode === 'backfill' && seedIndex >= seeds.length, updatedAt: new Date().toISOString() }); if (ctx.options.mode === 'backfill') await ctx.progress({ page: seedIndex, totalPages: seeds.length, itemsProcessed: yielded }); } } async normalize(raw: RawRecordLike): Promise { const p = PayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const t of p.types) { const categorySlug = numistaCategory(t); const refs: Record = {}; for (const r of t.references ?? []) if (r.catalogue?.code && r.number) refs[`${r.catalogue.code.toLowerCase()}_number`] = r.number; const attributes = AssetAttributesSchema.parse({ categorySlug, name: t.title, series: t.object_type?.name ?? null, year: t.min_year ?? null, country: null, material: t.composition?.text?.match(/^(gold|silver|copper|bronze|nickel|brass|aluminium|aluminum|steel|zinc|tin|iron|platinum|billon|electrum)/i)?.[1]?.toLowerCase() ?? null, identifiers: { numista_id: String(t.id), ...refs }, metadata: { issuer: t.issuer?.name ?? null, issuer_code: t.issuer?.code ?? null, min_year: t.min_year ?? null, max_year: t.max_year ?? null, denomination: t.value?.text ?? null, currency_name: t.value?.currency?.name ?? null, composition: t.composition?.text ?? null, ruler: t.ruler?.map((r) => r.name).filter(Boolean).join(', ') || null, numista_category: t.category ?? null, attribution: `Source: Numista (N#${t.id})`, }, }); const sourceUrl = t.url ?? `https://en.numista.com/${t.id}`; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle: `${t.title} ยท N#${t.id}`, imageUrls: [t.obverse_thumbnail, t.reverse_thumbnail].filter((u): u is string => Boolean(u)), attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `type:${t.id}`, confidence: 0.95 })); const pr = p.prices[String(t.id)]; if (pr && p.pricesFetchedAt) { const currency = pr.currency.toUpperCase() as CurrencyCode; for (const g of pr.prices) { const code = (g.grade ?? g.gradde ?? '').toLowerCase(); if (!code || !(code in GRADE_LABEL) || !(g.price > 0)) continue; out.push( NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `type:${t.id}:issue:${pr.issue_id ?? 'first'}:${code}`, confidence: 0.75, grade: { grader: null, grade: GRADE_LABEL[code], qualifier: null, certificationNumber: null }, condition: { condition: GRADE_CONDITION[code], conditionRaw: code, completeness: null }, priceKind: 'guide_value', price: g.price, currency, observationDate: new Date(p.pricesFetchedAt), sampleSize: null, }), ); } } } return out; } } export default (meta: ConnectorMeta) => new NumistaConnector(meta);