TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation } from '../_lib/shared.js';56const API = 'https://db.ygoprodeck.com/api/v7/cardinfo.php';78const CardSchema = z.object({9 id: z.number(),10 name: z.string(),11 type: z.string().optional(),12 humanReadableCardType: z.string().optional(),13 frameType: z.string().optional(),14 race: z.string().optional(),15 attribute: z.string().optional(),16 archetype: z.string().optional(),17 atk: z.number().nullable().optional(),18 def: z.number().nullable().optional(),19 level: z.number().nullable().optional(),20 ygoprodeck_url: z.string().optional(),21 card_sets: z.array(z.object({ set_name: z.string(), set_code: z.string(), set_rarity: z.string().optional(), set_rarity_code: z.string().optional(), set_price: z.string().optional() })).optional(),22 card_images: z.array(z.object({ id: z.number(), image_url: z.string().optional(), image_url_small: z.string().optional(), image_url_cropped: z.string().optional() })).optional(),23 card_prices: z.array(z.record(z.string(), z.string().nullable())).optional(),24 misc_info: z.array(z.object({ konami_id: z.number().optional(), tcg_date: z.string().optional(), ocg_date: z.string().optional(), md_rarity: z.string().optional(), treated_as: z.string().optional() }).loose()).optional(),25});26type YgoCard = z.infer<typeof CardSchema>;2728const KEEP = ['id', 'name', 'type', 'humanReadableCardType', 'frameType', 'race', 'attribute', 'archetype', 'atk', 'def', 'level', 'ygoprodeck_url', 'card_sets', 'card_images', 'card_prices', 'misc_info'] as const;2930export function trimCard(card: Record<string, unknown>): YgoCard {31 const out: Record<string, unknown> = {};32 for (const k of KEEP) if (card[k] !== undefined) out[k] = card[k];33 if (Array.isArray(out.misc_info)) out.misc_info = (out.misc_info as Array<Record<string, unknown>>).map((m) => ({ konami_id: m.konami_id, tcg_date: m.tcg_date, ocg_date: m.ocg_date, md_rarity: m.md_rarity, treated_as: m.treated_as }));34 return CardSchema.parse(out);35}3637/** Set code prefix → language marker (e.g. LOB-EN001 → EN, LOB-FR001 → FR, LOB-001 → unmarked). */38export function languageFromSetCode(code: string): string | null {39 const m = code.match(/^[A-Z0-9]+-([A-Z]{2})\d/);40 if (!m) return null;41 const map: Record<string, string> = { EN: 'English', FR: 'French', DE: 'German', IT: 'Italian', SP: 'Spanish', PT: 'Portuguese', JP: 'Japanese', KR: 'Korean', AE: 'English (Asian-English)', NA: 'English', E: 'English' };42 return map[m[1]!] ?? null;43}4445const CARD_PRICE_SOURCES: Array<[key: string, source: string, currency: 'USD' | 'EUR']> = [46 ['tcgplayer_price', 'tcgplayer', 'USD'],47 ['cardmarket_price', 'cardmarket', 'EUR'],48 ['ebay_price', 'ebay', 'USD'],49 ['amazon_price', 'amazon', 'USD'],50 ['coolstuffinc_price', 'coolstuffinc', 'USD'],51];5253export class YgoProDeckConnector extends BaseConnector {54 readonly version = '1.0.0';55 readonly parserVersion = '1.0.0';56 protected override minIntervalMs = 100;5758 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {59 const misc = ctx.meta.config.misc !== false;60 const url = `${API}${misc ? '?misc=yes' : ''}`;61 const res = await ctx.fetch(url, { engines: ['api'], timeoutMs: 180_000 });62 if (!res.success || !res.json) throw new Error(`ygoprodeck cardinfo failed: ${res.error}`);63 const data = (res.json as { data?: Record<string, unknown>[] }).data ?? [];64 if (data.length < 1000) ctx.anomaly('empty_page', `only ${data.length} cards returned`);65 const fetchedAt = res.fetchedAt;66 let count = 0;67 for (const c of data) {68 let card: YgoCard;69 try {70 card = trimCard(c);71 } catch (err) {72 ctx.anomaly('parse_failure', `card ${String(c.id)}: ${err instanceof Error ? err.message : String(err)}`);73 continue;74 }75 yield { url: card.ygoprodeck_url ?? `${API}?id=${card.id}`, externalId: String(card.id), kind: 'catalog_item', engine: 'api', httpStatus: 200, payload: { card }, fetchedAt };76 count++;77 if (this.reached(ctx, count)) break;78 }79 await ctx.setCursor({ lastRunAt: fetchedAt.toISOString(), count });80 }8182 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {83 const { card } = z.object({ card: CardSchema }).parse(raw.payload);84 const misc = card.misc_info?.[0];85 const images = [card.card_images?.[0]?.image_url].filter((x): x is string => Boolean(x));86 const sourceUrl = card.ygoprodeck_url ?? raw.url;87 const observedAt = raw.fetchedAt;88 const out: NormalizedRecord[] = [];89 const printings = card.card_sets?.length ? card.card_sets : [];9091 const baseIdentifiers: Record<string, string> = { ygo_id: String(card.id) };92 if (misc?.konami_id) baseIdentifiers.konami_id = String(misc.konami_id);93 const tcgYear = misc?.tcg_date ? Number(misc.tcg_date.slice(0, 4)) : null;9495 const buildAttrs = (p: (typeof printings)[number] | null) =>96 attrs({97 categorySlug: 'yugioh',98 franchise: 'Yu-Gi-Oh!',99 brand: 'Konami',100 set: p?.set_name ?? null,101 setCode: p ? p.set_code.split('-')[0]! : null,102 name: card.name,103 number: p?.set_code ?? null,104 year: p ? null : tcgYear, // per-printing release year is not provided by the API; do not guess105 variant: null,106 language: p ? languageFromSetCode(p.set_code) : null,107 rarity: p?.set_rarity ?? null,108 identifiers: { ...baseIdentifiers, ...(p ? { set_code: p.set_code } : {}) },109 metadata: {110 cardType: card.humanReadableCardType ?? card.type ?? null,111 frameType: card.frameType ?? null,112 race: card.race ?? null,113 attribute: card.attribute ?? null,114 archetype: card.archetype ?? null,115 atk: card.atk ?? null,116 def: card.def ?? null,117 level: card.level ?? null,118 tcgDate: misc?.tcg_date ?? null,119 ocgDate: misc?.ocg_date ?? null,120 rarityCode: p?.set_rarity_code ?? null,121 },122 });123124 // The same set code can appear with several rarities (e.g. Common + Secret Rare): keep each125 // (set_code, rarity) pair once — they are distinct market objects.126 const seen = new Set<string>();127 const unique = printings.filter((p) => {128 const k = `${p.set_code}|${p.set_rarity ?? ''}`;129 if (seen.has(k)) return false;130 seen.add(k);131 return true;132 });133 const targets = unique.length ? unique : [null];134 const codeCounts = new Map<string, number>();135 for (const p of unique) codeCounts.set(p.set_code, (codeCounts.get(p.set_code) ?? 0) + 1);136 const raritySlug = (r: string | undefined) => (r ?? 'unknown').toLowerCase().replace(/[^a-z0-9]+/g, '-');137 for (const [i, p] of targets.entries()) {138 const a = buildAttrs(p);139 const printingId = p ? `${card.id}:${p.set_code}${(codeCounts.get(p.set_code) ?? 1) > 1 ? `:${raritySlug(p.set_rarity)}` : ''}` : String(card.id);140 const title = p ? `${card.name} · ${p.set_name} ${p.set_code}${p.set_rarity ? ` (${p.set_rarity})` : ''}` : makeTitle({ name: card.name });141 out.push(142 catalogItem({143 kind: 'catalog_item',144 connectorId: this.meta.id,145 sourceId: this.meta.sourceId,146 sourceUrl,147 externalId: printingId,148 rawTitle: title,149 description: card.humanReadableCardType ?? null,150 imageUrls: images,151 attributes: a,152 observedAt,153 confidence: 0.95,154 parserVersion: this.parserVersion,155 releaseDate: null,156 }),157 );158 const setPrice = num(p?.set_price);159 if (p && setPrice !== null) {160 out.push(161 priceObservation({162 kind: 'price_observation',163 connectorId: this.meta.id,164 sourceId: this.meta.sourceId,165 sourceUrl,166 externalId: `${printingId}:set_price`,167 rawTitle: title,168 imageUrls: images,169 attributes: { ...a, metadata: { ...a.metadata, price_scope: 'printing', priceSource: 'ygoprodeck' } },170 observedAt,171 confidence: 0.7,172 parserVersion: this.parserVersion,173 priceKind: 'guide_value',174 price: setPrice,175 currency: 'USD',176 observationDate: observedAt,177 sampleSize: null,178 }),179 );180 }181 // Card-level marketplace prices attach to the first listed printing only.182 if (i === 0) {183 const cp = card.card_prices?.[0] ?? {};184 for (const [key, source, currency] of CARD_PRICE_SOURCES) {185 const price = num(cp[key]);186 if (price === null) continue;187 out.push(188 priceObservation({189 kind: 'price_observation',190 connectorId: this.meta.id,191 sourceId: this.meta.sourceId,192 sourceUrl,193 externalId: `${card.id}:${key}`,194 rawTitle: title,195 imageUrls: images,196 attributes: { ...a, metadata: { ...a.metadata, price_scope: 'card', priceSource: source } },197 observedAt,198 confidence: 0.6,199 parserVersion: this.parserVersion,200 priceKind: 'market',201 price,202 currency,203 observationDate: observedAt,204 sampleSize: null,205 }),206 );207 }208 }209 }210 return out;211 }212}213214export default function createConnector(meta: ConnectorMeta) {215 return new YgoProDeckConnector(meta);216}217