import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, num, priceObservation } from '../_lib/shared.js'; import { BOT_HEADERS, dayOf } from '../_lib/wave4.js'; /** Card Kingdom public price lists → Magic price observations keyed by scryfall_id. */ const PARSER_VERSION = '1.0.0'; const SINGLES = 'https://api.cardkingdom.com/api/pricelist'; const SEALED = 'https://api.cardkingdom.com/api/sealed_pricelist'; const SingleSchema = z.object({ id: z.number(), sku: z.string().nullable().optional(), scryfall_id: z.string().nullable().optional(), url: z.string().nullable().optional(), name: z.string(), variation: z.string().nullable().optional(), edition: z.string(), is_foil: z.union([z.string(), z.boolean()]).optional(), price_retail: z.union([z.string(), z.number()]).nullable().optional(), qty_retail: z.number().nullable().optional(), price_buy: z.union([z.string(), z.number()]).nullable().optional(), qty_buying: z.number().nullable().optional(), }); const SealedSchema = z.object({ id: z.number(), url: z.string().nullable().optional(), name: z.string(), edition: z.string().nullable().optional(), price_retail: z.union([z.string(), z.number()]).nullable().optional(), qty_retail: z.number().nullable().optional(), price_buy: z.union([z.string(), z.number()]).nullable().optional(), qty_buying: z.number().nullable().optional() }); const RawPayloadSchema = z.discriminatedUnion('kind', [ z.object({ kind: z.literal('single'), row: SingleSchema, createdAt: z.string().nullable(), baseUrl: z.string() }), z.object({ kind: z.literal('sealed'), row: SealedSchema, createdAt: z.string().nullable(), baseUrl: z.string() }), ]); export type CardKingdomPayload = z.infer; export function isFoil(v: string | boolean | undefined): boolean { return v === true || String(v).toLowerCase() === 'true'; } export class CardKingdomConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; async *crawl(ctx: CrawlContext): AsyncIterable { const minRetail = Number(this.meta.config.minRetailUsd ?? 0); let count = 0; const singles = await ctx.fetch(SINGLES, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 180_000 }); let doc = singles.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } | null; if ((!doc || !Array.isArray(doc.data)) && typeof singles.html === 'string' && singles.html.startsWith('{')) doc = JSON.parse(singles.html) as typeof doc; // served as text/html if (!singles.success || !doc || !Array.isArray(doc.data)) throw new Error(`cardkingdom pricelist failed: ${singles.error ?? singles.httpStatus}`); const createdAt = doc.meta?.created_at ?? null; const baseUrl = doc.meta?.base_url ?? 'https://www.cardkingdom.com/'; for (const r of doc.data) { const parsed = SingleSchema.safeParse(r); if (!parsed.success) { ctx.anomaly('parse_failure_row', parsed.error.issues[0]?.message); continue; } const retail = num(parsed.data.price_retail); if (retail === null || retail < minRetail) continue; if (this.reached(ctx, count)) return; count++; yield { url: `${baseUrl}${parsed.data.url ?? `mtg/${parsed.data.id}`}`, externalId: `s${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: singles.httpStatus, payload: { kind: 'single', row: parsed.data, createdAt, baseUrl } satisfies CardKingdomPayload, fetchedAt: singles.fetchedAt }; } if (this.meta.config.includeSealed !== false) { const sealed = await ctx.fetch(SEALED, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 120_000 }); let sdoc = sealed.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } | null; if ((!sdoc || !Array.isArray(sdoc.data)) && typeof sealed.html === 'string' && sealed.html.startsWith('{')) sdoc = JSON.parse(sealed.html) as typeof sdoc; if (sealed.success && sdoc && Array.isArray(sdoc.data)) { for (const r of sdoc.data) { const parsed = SealedSchema.safeParse(r); if (!parsed.success || num(parsed.data.price_retail) === null) continue; if (this.reached(ctx, count)) return; count++; yield { url: `${baseUrl}${parsed.data.url ?? `mtg-sealed/${parsed.data.id}`}`, externalId: `x${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: sealed.httpStatus, payload: { kind: 'sealed', row: parsed.data, createdAt: sdoc.meta?.created_at ?? null, baseUrl } satisfies CardKingdomPayload, fetchedAt: sealed.fetchedAt }; } } else ctx.anomaly('page_fetch_failed', `sealed_pricelist: ${sealed.error ?? sealed.httpStatus}`); } await ctx.setCursor({ completedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = RawPayloadSchema.parse(raw.payload); const observedAt = raw.fetchedAt; const created = p.createdAt ? new Date(p.createdAt.replace(' ', 'T') + 'Z') : null; const observationDate = created && !Number.isNaN(created.getTime()) ? dayOf(created) : dayOf(observedAt); const out: NormalizedRecord[] = []; if (p.kind === 'single') { const r = p.row; const foil = isFoil(r.is_foil); const variation = (r.variation ?? '').trim(); const variant = [variation || null, foil ? 'Foil' : null].filter(Boolean).join(' ') || null; const number = r.sku?.match(/-([A-Za-z0-9]+)$/)?.[1] ?? null; const identifiers: Record = { cardkingdom_id: String(r.id) }; if (r.scryfall_id) identifiers.scryfall_id = r.scryfall_id; if (r.sku) identifiers.cardkingdom_sku = r.sku; const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition, name: r.name, number, variant, language: 'English', identifiers, metadata: { finish: foil ? 'foil' : 'nonfoil', variation: variation || null } }); const rawTitle = makeTitle({ name: r.name, set: r.edition, number, variant }); out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION })); const retail = num(r.price_retail); if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy), qty_retail: r.qty_retail ?? null, qty_buying: r.qty_buying ?? null } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null })); return out; } const r = p.row; const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition ?? null, name: r.name, identifiers: { cardkingdom_sealed_id: String(r.id) }, metadata: { sealed: true } }); const rawTitle = makeTitle({ name: r.name, set: r.edition ?? null }); out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.75, parserVersion: PARSER_VERSION })); const retail = num(r.price_retail); if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy) } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null })); return out; } } export default (meta: ConnectorMeta) => new CardKingdomConnector(meta);