TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared';45/**6 * Numista catalogue API v3 (https://en.numista.com/api/doc/) — the reference catalogue of world coins,7 * banknotes and exonumia (N# numbers). Requires a personal API key (header `Numista-API-Key`; free plan8 * 2 000 requests/month). Gated: the connector is reported DISABLED until NUMISTA_API_KEY exists.9 * Records: catalog_item per type (identifiers numista_id) and, when `fetchPrices` is on, guide_value10 * observations per grade for the first issue of each type (/types/{id}/issues/{issue}/prices).11 * Licence: attribution "Source: Numista" + N# is mandatory; bulk download and re-serving the data as a12 * feed are forbidden — we store only the identifying fields we need and always attribute.13 */1415const API = 'https://api.numista.com/v3';16const PARSER_VERSION = '1.0.0';1718export const TypeSchema = z.object({19 id: z.number(),20 title: z.string(),21 category: z.string().nullable().optional(),22 object_type: z.object({ id: z.number().nullable().optional(), name: z.string().nullable().optional() }).nullable().optional(),23 issuer: z.object({ code: z.string().nullable().optional(), name: z.string().nullable().optional() }).nullable().optional(),24 min_year: z.number().nullable().optional(),25 max_year: z.number().nullable().optional(),26 obverse_thumbnail: z.string().nullable().optional(),27 reverse_thumbnail: z.string().nullable().optional(),28 /** filled from GET /types/{id} when `fetchDetails` is on */29 value: z.object({ text: z.string().nullable().optional(), numeric_value: z.number().nullable().optional(), currency: z.object({ id: z.number().nullable().optional(), name: z.string().nullable().optional(), full_name: z.string().nullable().optional() }).nullable().optional() }).nullable().optional(),30 composition: z.object({ text: z.string().nullable().optional() }).nullable().optional(),31 ruler: z.array(z.object({ id: z.number().nullable().optional(), name: z.string().nullable().optional() })).nullable().optional(),32 url: z.string().nullable().optional(),33 references: z.array(z.object({ catalogue: z.object({ code: z.string().nullable().optional() }).nullable().optional(), number: z.string().nullable().optional() })).nullable().optional(),34});35export type NumistaType = z.infer<typeof TypeSchema>;36export const PricesSchema = z.object({ issue_id: z.number().nullable(), currency: z.string(), prices: z.array(z.object({ grade: z.string().optional(), gradde: z.string().optional(), price: z.number() })) });37export const PayloadSchema = z.object({38 kind: z.literal('types_page'),39 url: z.string(),40 query: z.record(z.string(), z.string()),41 page: z.number().int(),42 count: z.number().int().nullable(),43 types: z.array(TypeSchema),44 /** N# → issue prices (guide values) when fetched */45 prices: z.record(z.string(), PricesSchema).default({}),46 /** observation date = fetch date of the price call (Numista prices are live estimates) */47 pricesFetchedAt: z.string().nullable().default(null),48});49export type Payload = z.infer<typeof PayloadSchema>;5051const GRADE_LABEL: Record<string, string> = { g: 'G', vg: 'VG', f: 'F', vf: 'VF', xf: 'XF', au: 'AU', unc: 'UNC' };52const GRADE_CONDITION: Record<string, string> = { g: 'good', vg: 'very_good', f: 'fine', vf: 'very_fine', xf: 'extremely_fine', au: 'about_uncirculated', unc: 'mint_state' };5354export function numistaCategory(t: NumistaType): string {55 const c = (t.category ?? '').toLowerCase();56 if (c === 'banknote') return 'banknotes';57 if (c === 'exonumia') return 'medals';58 return 'coins';59}6061interface Cursor {62 seedIndex?: number;63 page?: number;64 done?: boolean;65 updatedAt?: string;66}6768export class NumistaConnector extends BaseConnector {69 readonly version = '1.0.0';70 readonly parserVersion = PARSER_VERSION;71 protected override minIntervalMs = 1500;72 override readonly urlPatterns = [/numista\.com\/(?:catalogue\/pieces)?(\d+)/i];7374 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {75 const key = process.env.NUMISTA_API_KEY;76 if (!key) {77 ctx.anomaly('config_missing', 'NUMISTA_API_KEY is not set (connector is gated)');78 return;79 }80 const headers = { 'Numista-API-Key': key };81 const seeds = (this.meta.config.seeds as Array<Record<string, string>> | undefined) ?? [];82 const count = Number(this.meta.config.count ?? 50);83 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 4);84 const fetchPrices = Boolean(this.meta.config.fetchPrices ?? false);85 const priceCurrency = String(this.meta.config.priceCurrency ?? 'EUR');86 const maxPricesPerRun = Number(this.meta.config.maxPricesPerRun ?? 20);87 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };88 let seedIndex = Math.min(cursor.seedIndex ?? 0, Math.max(seeds.length - 1, 0));89 let page = cursor.page ?? 1;90 let pages = 0;91 let priceCalls = 0;92 let yielded = 0;93 while (seedIndex < seeds.length && pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) {94 const seed = seeds[seedIndex]!;95 const q = new URLSearchParams({ ...seed, page: String(page), count: String(count), lang: 'en' });96 const url = `${API}/types?${q}`;97 await this.throttle();98 const res = await ctx.fetch(url, { engines: ['api'], headers, expect: ['title', 'identifiers'], parse: (r) => {99 const t = (r.json as { types?: Array<{ id?: number; title?: string }> } | null)?.types?.[0];100 return t ? { title: t.title, identifiers: { numista: String(t.id) } } : null;101 } });102 pages++;103 const parsed = res.success ? z.object({ count: z.union([z.number(), z.string()]).nullable().optional(), types: z.array(z.unknown()) }).safeParse(res.json) : null;104 if (!parsed?.success) {105 ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);106 break;107 }108 const types = parsed.data.types.map((t) => TypeSchema.safeParse(t)).filter((r) => r.success).map((r) => r.data);109 const prices: Record<string, z.infer<typeof PricesSchema>> = {};110 let pricesFetchedAt: string | null = null;111 if (fetchPrices) {112 for (const t of types) {113 if (priceCalls >= maxPricesPerRun || ctx.signal?.aborted) break;114 await this.throttle();115 const issues = await ctx.fetch(`${API}/types/${t.id}/issues?lang=en`, { engines: ['api'], headers, minQuality: 0 });116 priceCalls++;117 const first = (issues.json as Array<{ id?: number }> | null)?.[0]?.id;118 if (!issues.success || !first) continue;119 await this.throttle();120 const pr = await ctx.fetch(`${API}/types/${t.id}/issues/${first}/prices?currency=${priceCurrency}&lang=en`, { engines: ['api'], headers, minQuality: 0 });121 priceCalls++;122 const pp = pr.success ? PricesSchema.omit({ issue_id: true }).safeParse(pr.json) : null;123 if (pp?.success) {124 prices[String(t.id)] = { issue_id: first, ...pp.data };125 pricesFetchedAt = pr.fetchedAt.toISOString();126 }127 }128 }129 if (types.length) {130 yielded++;131 const payload: Payload = { kind: 'types_page', url, query: seed, page, count: parsed.data.count !== null && parsed.data.count !== undefined ? Number(parsed.data.count) : null, types, prices, pricesFetchedAt };132 yield { url, externalId: `types:${JSON.stringify(seed)}:p${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };133 }134 const total = parsed.data.count !== null && parsed.data.count !== undefined ? Number(parsed.data.count) : null;135 const more = types.length === count && (total === null || page * count < total);136 if (more) page++;137 else {138 seedIndex++;139 page = 1;140 }141 await ctx.setCursor({ seedIndex: seedIndex >= seeds.length ? 0 : seedIndex, page: seedIndex >= seeds.length ? 1 : page, done: ctx.options.mode === 'backfill' && seedIndex >= seeds.length, updatedAt: new Date().toISOString() });142 if (ctx.options.mode === 'backfill') await ctx.progress({ page: seedIndex, totalPages: seeds.length, itemsProcessed: yielded });143 }144 }145146 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {147 const p = PayloadSchema.parse(raw.payload);148 const out: NormalizedRecord[] = [];149 for (const t of p.types) {150 const categorySlug = numistaCategory(t);151 const refs: Record<string, string> = {};152 for (const r of t.references ?? []) if (r.catalogue?.code && r.number) refs[`${r.catalogue.code.toLowerCase()}_number`] = r.number;153 const attributes = AssetAttributesSchema.parse({154 categorySlug,155 name: t.title,156 series: t.object_type?.name ?? null,157 year: t.min_year ?? null,158 country: null,159 material: t.composition?.text?.match(/^(gold|silver|copper|bronze|nickel|brass|aluminium|aluminum|steel|zinc|tin|iron|platinum|billon|electrum)/i)?.[1]?.toLowerCase() ?? null,160 identifiers: { numista_id: String(t.id), ...refs },161 metadata: {162 issuer: t.issuer?.name ?? null,163 issuer_code: t.issuer?.code ?? null,164 min_year: t.min_year ?? null,165 max_year: t.max_year ?? null,166 denomination: t.value?.text ?? null,167 currency_name: t.value?.currency?.name ?? null,168 composition: t.composition?.text ?? null,169 ruler: t.ruler?.map((r) => r.name).filter(Boolean).join(', ') || null,170 numista_category: t.category ?? null,171 attribution: `Source: Numista (N#${t.id})`,172 },173 });174 const sourceUrl = t.url ?? `https://en.numista.com/${t.id}`;175 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle: `${t.title} · N#${t.id}`, imageUrls: [t.obverse_thumbnail, t.reverse_thumbnail].filter((u): u is string => Boolean(u)), attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };176 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `type:${t.id}`, confidence: 0.95 }));177 const pr = p.prices[String(t.id)];178 if (pr && p.pricesFetchedAt) {179 const currency = pr.currency.toUpperCase() as CurrencyCode;180 for (const g of pr.prices) {181 const code = (g.grade ?? g.gradde ?? '').toLowerCase();182 if (!code || !(code in GRADE_LABEL) || !(g.price > 0)) continue;183 out.push(184 NormalizedPriceObservationSchema.parse({185 kind: 'price_observation',186 ...base,187 externalId: `type:${t.id}:issue:${pr.issue_id ?? 'first'}:${code}`,188 confidence: 0.75,189 grade: { grader: null, grade: GRADE_LABEL[code], qualifier: null, certificationNumber: null },190 condition: { condition: GRADE_CONDITION[code], conditionRaw: code, completeness: null },191 priceKind: 'guide_value',192 price: g.price,193 currency,194 observationDate: new Date(p.pricesFetchedAt),195 sampleSize: null,196 }),197 );198 }199 }200 }201 return out;202 }203}204205export default (meta: ConnectorMeta) => new NumistaConnector(meta);206