TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation } from '../_lib/shared.js';5import { BOT_HEADERS, dayOf } from '../_lib/wave4.js';67/** Card Kingdom public price lists → Magic price observations keyed by scryfall_id. */8const PARSER_VERSION = '1.0.0';9const SINGLES = 'https://api.cardkingdom.com/api/pricelist';10const SEALED = 'https://api.cardkingdom.com/api/sealed_pricelist';1112const SingleSchema = z.object({13 id: z.number(),14 sku: z.string().nullable().optional(),15 scryfall_id: z.string().nullable().optional(),16 url: z.string().nullable().optional(),17 name: z.string(),18 variation: z.string().nullable().optional(),19 edition: z.string(),20 is_foil: z.union([z.string(), z.boolean()]).optional(),21 price_retail: z.union([z.string(), z.number()]).nullable().optional(),22 qty_retail: z.number().nullable().optional(),23 price_buy: z.union([z.string(), z.number()]).nullable().optional(),24 qty_buying: z.number().nullable().optional(),25});26const SealedSchema = z.object({ id: z.number(), url: z.string().nullable().optional(), name: z.string(), edition: z.string().nullable().optional(), price_retail: z.union([z.string(), z.number()]).nullable().optional(), qty_retail: z.number().nullable().optional(), price_buy: z.union([z.string(), z.number()]).nullable().optional(), qty_buying: z.number().nullable().optional() });27const RawPayloadSchema = z.discriminatedUnion('kind', [28 z.object({ kind: z.literal('single'), row: SingleSchema, createdAt: z.string().nullable(), baseUrl: z.string() }),29 z.object({ kind: z.literal('sealed'), row: SealedSchema, createdAt: z.string().nullable(), baseUrl: z.string() }),30]);31export type CardKingdomPayload = z.infer<typeof RawPayloadSchema>;3233export function isFoil(v: string | boolean | undefined): boolean {34 return v === true || String(v).toLowerCase() === 'true';35}3637export class CardKingdomConnector extends BaseConnector {38 readonly version = '1.0.0';39 readonly parserVersion = PARSER_VERSION;4041 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {42 const minRetail = Number(this.meta.config.minRetailUsd ?? 0);43 let count = 0;44 const singles = await ctx.fetch(SINGLES, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 180_000 });45 let doc = singles.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } | null;46 if ((!doc || !Array.isArray(doc.data)) && typeof singles.html === 'string' && singles.html.startsWith('{')) doc = JSON.parse(singles.html) as typeof doc; // served as text/html47 if (!singles.success || !doc || !Array.isArray(doc.data)) throw new Error(`cardkingdom pricelist failed: ${singles.error ?? singles.httpStatus}`);48 const createdAt = doc.meta?.created_at ?? null;49 const baseUrl = doc.meta?.base_url ?? 'https://www.cardkingdom.com/';50 for (const r of doc.data) {51 const parsed = SingleSchema.safeParse(r);52 if (!parsed.success) {53 ctx.anomaly('parse_failure_row', parsed.error.issues[0]?.message);54 continue;55 }56 const retail = num(parsed.data.price_retail);57 if (retail === null || retail < minRetail) continue;58 if (this.reached(ctx, count)) return;59 count++;60 yield { url: `${baseUrl}${parsed.data.url ?? `mtg/${parsed.data.id}`}`, externalId: `s${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: singles.httpStatus, payload: { kind: 'single', row: parsed.data, createdAt, baseUrl } satisfies CardKingdomPayload, fetchedAt: singles.fetchedAt };61 }62 if (this.meta.config.includeSealed !== false) {63 const sealed = await ctx.fetch(SEALED, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 120_000 });64 let sdoc = sealed.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } | null;65 if ((!sdoc || !Array.isArray(sdoc.data)) && typeof sealed.html === 'string' && sealed.html.startsWith('{')) sdoc = JSON.parse(sealed.html) as typeof sdoc;66 if (sealed.success && sdoc && Array.isArray(sdoc.data)) {67 for (const r of sdoc.data) {68 const parsed = SealedSchema.safeParse(r);69 if (!parsed.success || num(parsed.data.price_retail) === null) continue;70 if (this.reached(ctx, count)) return;71 count++;72 yield { url: `${baseUrl}${parsed.data.url ?? `mtg-sealed/${parsed.data.id}`}`, externalId: `x${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: sealed.httpStatus, payload: { kind: 'sealed', row: parsed.data, createdAt: sdoc.meta?.created_at ?? null, baseUrl } satisfies CardKingdomPayload, fetchedAt: sealed.fetchedAt };73 }74 } else ctx.anomaly('page_fetch_failed', `sealed_pricelist: ${sealed.error ?? sealed.httpStatus}`);75 }76 await ctx.setCursor({ completedAt: new Date().toISOString() });77 }7879 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {80 const p = RawPayloadSchema.parse(raw.payload);81 const observedAt = raw.fetchedAt;82 const created = p.createdAt ? new Date(p.createdAt.replace(' ', 'T') + 'Z') : null;83 const observationDate = created && !Number.isNaN(created.getTime()) ? dayOf(created) : dayOf(observedAt);84 const out: NormalizedRecord[] = [];85 if (p.kind === 'single') {86 const r = p.row;87 const foil = isFoil(r.is_foil);88 const variation = (r.variation ?? '').trim();89 const variant = [variation || null, foil ? 'Foil' : null].filter(Boolean).join(' ') || null;90 const number = r.sku?.match(/-([A-Za-z0-9]+)$/)?.[1] ?? null;91 const identifiers: Record<string, string> = { cardkingdom_id: String(r.id) };92 if (r.scryfall_id) identifiers.scryfall_id = r.scryfall_id;93 if (r.sku) identifiers.cardkingdom_sku = r.sku;94 const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition, name: r.name, number, variant, language: 'English', identifiers, metadata: { finish: foil ? 'foil' : 'nonfoil', variation: variation || null } });95 const rawTitle = makeTitle({ name: r.name, set: r.edition, number, variant });96 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION }));97 const retail = num(r.price_retail);98 if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy), qty_retail: r.qty_retail ?? null, qty_buying: r.qty_buying ?? null } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null }));99 return out;100 }101 const r = p.row;102 const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition ?? null, name: r.name, identifiers: { cardkingdom_sealed_id: String(r.id) }, metadata: { sealed: true } });103 const rawTitle = makeTitle({ name: r.name, set: r.edition ?? null });104 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.75, parserVersion: PARSER_VERSION }));105 const retail = num(r.price_retail);106 if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy) } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null }));107 return out;108 }109}110111export default (meta: ConnectorMeta) => new CardKingdomConnector(meta);112