import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, num, priceObservation } from '../_lib/shared.js'; const API = 'https://db.ygoprodeck.com/api/v7/cardinfo.php'; const CardSchema = z.object({ id: z.number(), name: z.string(), type: z.string().optional(), humanReadableCardType: z.string().optional(), frameType: z.string().optional(), race: z.string().optional(), attribute: z.string().optional(), archetype: z.string().optional(), atk: z.number().nullable().optional(), def: z.number().nullable().optional(), level: z.number().nullable().optional(), ygoprodeck_url: z.string().optional(), card_sets: z.array(z.object({ set_name: z.string(), set_code: z.string(), set_rarity: z.string().optional(), set_rarity_code: z.string().optional(), set_price: z.string().optional() })).optional(), card_images: z.array(z.object({ id: z.number(), image_url: z.string().optional(), image_url_small: z.string().optional(), image_url_cropped: z.string().optional() })).optional(), card_prices: z.array(z.record(z.string(), z.string().nullable())).optional(), misc_info: z.array(z.object({ konami_id: z.number().optional(), tcg_date: z.string().optional(), ocg_date: z.string().optional(), md_rarity: z.string().optional(), treated_as: z.string().optional() }).loose()).optional(), }); type YgoCard = z.infer; const KEEP = ['id', 'name', 'type', 'humanReadableCardType', 'frameType', 'race', 'attribute', 'archetype', 'atk', 'def', 'level', 'ygoprodeck_url', 'card_sets', 'card_images', 'card_prices', 'misc_info'] as const; export function trimCard(card: Record): YgoCard { const out: Record = {}; for (const k of KEEP) if (card[k] !== undefined) out[k] = card[k]; if (Array.isArray(out.misc_info)) out.misc_info = (out.misc_info as Array>).map((m) => ({ konami_id: m.konami_id, tcg_date: m.tcg_date, ocg_date: m.ocg_date, md_rarity: m.md_rarity, treated_as: m.treated_as })); return CardSchema.parse(out); } /** Set code prefix → language marker (e.g. LOB-EN001 → EN, LOB-FR001 → FR, LOB-001 → unmarked). */ export function languageFromSetCode(code: string): string | null { const m = code.match(/^[A-Z0-9]+-([A-Z]{2})\d/); if (!m) return null; const map: Record = { EN: 'English', FR: 'French', DE: 'German', IT: 'Italian', SP: 'Spanish', PT: 'Portuguese', JP: 'Japanese', KR: 'Korean', AE: 'English (Asian-English)', NA: 'English', E: 'English' }; return map[m[1]!] ?? null; } const CARD_PRICE_SOURCES: Array<[key: string, source: string, currency: 'USD' | 'EUR']> = [ ['tcgplayer_price', 'tcgplayer', 'USD'], ['cardmarket_price', 'cardmarket', 'EUR'], ['ebay_price', 'ebay', 'USD'], ['amazon_price', 'amazon', 'USD'], ['coolstuffinc_price', 'coolstuffinc', 'USD'], ]; export class YgoProDeckConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = '1.0.0'; protected override minIntervalMs = 100; async *crawl(ctx: CrawlContext): AsyncIterable { const misc = ctx.meta.config.misc !== false; const url = `${API}${misc ? '?misc=yes' : ''}`; const res = await ctx.fetch(url, { engines: ['api'], timeoutMs: 180_000 }); if (!res.success || !res.json) throw new Error(`ygoprodeck cardinfo failed: ${res.error}`); const data = (res.json as { data?: Record[] }).data ?? []; if (data.length < 1000) ctx.anomaly('empty_page', `only ${data.length} cards returned`); const fetchedAt = res.fetchedAt; let count = 0; for (const c of data) { let card: YgoCard; try { card = trimCard(c); } catch (err) { ctx.anomaly('parse_failure', `card ${String(c.id)}: ${err instanceof Error ? err.message : String(err)}`); continue; } yield { url: card.ygoprodeck_url ?? `${API}?id=${card.id}`, externalId: String(card.id), kind: 'catalog_item', engine: 'api', httpStatus: 200, payload: { card }, fetchedAt }; count++; if (this.reached(ctx, count)) break; } await ctx.setCursor({ lastRunAt: fetchedAt.toISOString(), count }); } async normalize(raw: RawRecordLike): Promise { const { card } = z.object({ card: CardSchema }).parse(raw.payload); const misc = card.misc_info?.[0]; const images = [card.card_images?.[0]?.image_url].filter((x): x is string => Boolean(x)); const sourceUrl = card.ygoprodeck_url ?? raw.url; const observedAt = raw.fetchedAt; const out: NormalizedRecord[] = []; const printings = card.card_sets?.length ? card.card_sets : []; const baseIdentifiers: Record = { ygo_id: String(card.id) }; if (misc?.konami_id) baseIdentifiers.konami_id = String(misc.konami_id); const tcgYear = misc?.tcg_date ? Number(misc.tcg_date.slice(0, 4)) : null; const buildAttrs = (p: (typeof printings)[number] | null) => attrs({ categorySlug: 'yugioh', franchise: 'Yu-Gi-Oh!', brand: 'Konami', set: p?.set_name ?? null, setCode: p ? p.set_code.split('-')[0]! : null, name: card.name, number: p?.set_code ?? null, year: p ? null : tcgYear, // per-printing release year is not provided by the API; do not guess variant: null, language: p ? languageFromSetCode(p.set_code) : null, rarity: p?.set_rarity ?? null, identifiers: { ...baseIdentifiers, ...(p ? { set_code: p.set_code } : {}) }, metadata: { cardType: card.humanReadableCardType ?? card.type ?? null, frameType: card.frameType ?? null, race: card.race ?? null, attribute: card.attribute ?? null, archetype: card.archetype ?? null, atk: card.atk ?? null, def: card.def ?? null, level: card.level ?? null, tcgDate: misc?.tcg_date ?? null, ocgDate: misc?.ocg_date ?? null, rarityCode: p?.set_rarity_code ?? null, }, }); // The same set code can appear with several rarities (e.g. Common + Secret Rare): keep each // (set_code, rarity) pair once — they are distinct market objects. const seen = new Set(); const unique = printings.filter((p) => { const k = `${p.set_code}|${p.set_rarity ?? ''}`; if (seen.has(k)) return false; seen.add(k); return true; }); const targets = unique.length ? unique : [null]; const codeCounts = new Map(); for (const p of unique) codeCounts.set(p.set_code, (codeCounts.get(p.set_code) ?? 0) + 1); const raritySlug = (r: string | undefined) => (r ?? 'unknown').toLowerCase().replace(/[^a-z0-9]+/g, '-'); for (const [i, p] of targets.entries()) { const a = buildAttrs(p); const printingId = p ? `${card.id}:${p.set_code}${(codeCounts.get(p.set_code) ?? 1) > 1 ? `:${raritySlug(p.set_rarity)}` : ''}` : String(card.id); const title = p ? `${card.name} · ${p.set_name} ${p.set_code}${p.set_rarity ? ` (${p.set_rarity})` : ''}` : makeTitle({ name: card.name }); out.push( catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: printingId, rawTitle: title, description: card.humanReadableCardType ?? null, imageUrls: images, attributes: a, observedAt, confidence: 0.95, parserVersion: this.parserVersion, releaseDate: null, }), ); const setPrice = num(p?.set_price); if (p && setPrice !== null) { out.push( priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${printingId}:set_price`, rawTitle: title, imageUrls: images, attributes: { ...a, metadata: { ...a.metadata, price_scope: 'printing', priceSource: 'ygoprodeck' } }, observedAt, confidence: 0.7, parserVersion: this.parserVersion, priceKind: 'guide_value', price: setPrice, currency: 'USD', observationDate: observedAt, sampleSize: null, }), ); } // Card-level marketplace prices attach to the first listed printing only. if (i === 0) { const cp = card.card_prices?.[0] ?? {}; for (const [key, source, currency] of CARD_PRICE_SOURCES) { const price = num(cp[key]); if (price === null) continue; out.push( priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.id}:${key}`, rawTitle: title, imageUrls: images, attributes: { ...a, metadata: { ...a.metadata, price_scope: 'card', priceSource: source } }, observedAt, confidence: 0.6, parserVersion: this.parserVersion, priceKind: 'market', price, currency, observationDate: observedAt, sampleSize: null, }), ); } } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new YgoProDeckConnector(meta); }