import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js'; /** * TCGdex connector — Pokémon TCG catalog for every language (en, ja, fr, de, es, it, pt, ko, zh…) * with TCGplayer / Cardmarket pricing per variant on English cards. Card ids share pokemontcg.io's * scheme for English sets (base1-4, swsh1-1), so `pokemontcg_id` lines up with the pokemontcg * connector and set codes use the same PTCGO abbreviation (tcgOnline) as pokemontcg's ptcgoCode. */ const API = 'https://api.tcgdex.net/v2'; const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' }; const PARSER_VERSION = '1.0.0'; const SetBriefSchema = z.object({ id: z.string(), name: z.string(), cardCount: z.object({ total: z.number(), official: z.number() }).partial().optional() }); const SetSchema = z.object({ id: z.string(), name: z.string(), releaseDate: z.string().optional(), tcgOnline: z.string().optional(), abbreviation: z.object({ official: z.string().optional(), localized: z.string().optional() }).partial().optional(), serie: z.object({ id: z.string(), name: z.string() }).optional(), cardCount: z.object({ total: z.number().optional(), official: z.number().optional() }).partial().optional(), cards: z.array(z.object({ id: z.string(), localId: z.string(), name: z.string(), image: z.string().optional() })), }); const TcgplayerPricesSchema = z.record(z.string(), z.unknown()); const CardSchema = z.object({ id: z.string(), localId: z.string(), name: z.string(), rarity: z.string().optional(), category: z.string().optional(), illustrator: z.string().optional(), image: z.string().optional(), hp: z.number().optional(), types: z.array(z.string()).optional(), dexId: z.array(z.number()).optional(), regulationMark: z.string().optional(), updated: z.string().optional(), set: z.object({ id: z.string(), name: z.string() }), variants: z.object({ firstEdition: z.boolean().optional(), holo: z.boolean().optional(), normal: z.boolean().optional(), reverse: z.boolean().optional(), wPromo: z.boolean().optional() }).partial().optional(), variants_detailed: z .array( z .object({ type: z.string(), subtype: z.string().optional(), stamp: z.array(z.string()).optional(), thirdParty: z.object({ cardmarket: z.number().optional(), tcgplayer: z.number().optional() }).partial().optional(), pricing: z.object({ cardmarket: z.record(z.string(), z.unknown()).nullable().optional(), tcgplayer: z.record(z.string(), z.unknown()).nullable().optional() }).partial().nullable().optional(), }) .loose(), ) .optional(), pricing: z.object({ cardmarket: z.record(z.string(), z.unknown()).nullable().optional(), tcgplayer: z.record(z.string(), z.unknown()).nullable().optional() }).partial().nullable().optional(), }); type TcgCard = z.infer; const RawPayloadSchema = z.object({ card: CardSchema, set: z.object({ id: z.string(), name: z.string(), code: z.string().nullable(), releaseDate: z.string().nullable(), serie: z.string().nullable(), total: z.number().nullable() }), language: z.string(), }); /** Keep the payload compact: drop attacks/abilities/legal text. */ export function trimCard(raw: Record): TcgCard { const keep = ['id', 'localId', 'name', 'rarity', 'category', 'illustrator', 'image', 'hp', 'types', 'dexId', 'regulationMark', 'updated', 'set', 'variants', 'variants_detailed', 'pricing']; const out: Record = {}; for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k]; if (out.set && typeof out.set === 'object') out.set = { id: (out.set as { id: string }).id, name: (out.set as { name: string }).name }; return CardSchema.parse(out); } const LANG_NAME: Record = { en: 'English', ja: 'Japanese', fr: 'French', de: 'German', es: 'Spanish', it: 'Italian', pt: 'Portuguese', ko: 'Korean', zh: 'Chinese', 'zh-tw': 'Chinese (Traditional)', 'zh-cn': 'Chinese (Simplified)', nl: 'Dutch', pl: 'Polish', ru: 'Russian', th: 'Thai', id: 'Indonesian' }; /** Variant vocabulary aligned with the pokemontcg connector. */ function variantLabel(type: string, stamp?: string[]): string | null { const first = stamp?.some((s) => /1st/i.test(s)); if (type === 'reverse') return first ? '1st Edition Reverse Holo' : 'Reverse Holo'; if (type === 'holo') return first ? '1st Edition Holo' : 'Holo'; if (type === 'firstEdition') return '1st Edition'; return first ? '1st Edition' : null; } export class TcgdexConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 120; override readonly urlPatterns = [/^https?:\/\/(www\.)?tcgdex\.net\/(?:[a-z]{2}\/)?(?:database|cards?)\//i]; private async get(ctx: CrawlContext, url: string) { await this.throttle(); return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS, timeoutMs: 60_000 }), (r) => r.success && r.json !== null, 4, 1500); } async *crawl(ctx: CrawlContext): AsyncIterable { const languages = ctx.options.seeds?.length ? ctx.options.seeds.filter((s) => /^[a-z]{2}(-[a-z]{2})?$/.test(s)) : ((this.meta.config.languages as string[] | undefined) ?? ['en']); const seedSets = ctx.options.seeds?.filter((s) => !/^[a-z]{2}(-[a-z]{2})?$/.test(s)) ?? []; const cursor = ctx.options.cursor ?? {}; let langIndex = Number(cursor.langIndex ?? 0); let setIndex = Number(cursor.setIndex ?? 0); let count = 0; for (; langIndex < languages.length; langIndex++, setIndex = 0) { const lang = languages[langIndex]!; const setsRes = await this.get(ctx, `${API}/${lang}/sets`); if (!setsRes.success || !Array.isArray(setsRes.json)) { ctx.anomaly('sets_unavailable', `${lang}: ${setsRes.error ?? setsRes.httpStatus}`); continue; } const sets = z.array(SetBriefSchema.loose()).parse(setsRes.json).filter((s) => !seedSets.length || seedSets.includes(s.id)); for (; setIndex < sets.length; setIndex++) { const brief = sets[setIndex]!; if (ctx.signal?.aborted) return; const setRes = await this.get(ctx, `${API}/${lang}/sets/${encodeURIComponent(brief.id)}`); if (!setRes.success || !setRes.json) { ctx.anomaly('set_unavailable', `${lang}/${brief.id}`); continue; } const parsedSet = SetSchema.loose().safeParse(setRes.json); if (!parsedSet.success) { ctx.anomaly('parse_failure_set', `${lang}/${brief.id}: ${parsedSet.error.issues[0]?.message}`); continue; } const set = parsedSet.data; const setMeta = { id: set.id, name: set.name, code: set.tcgOnline ?? set.abbreviation?.official ?? null, releaseDate: set.releaseDate ?? null, serie: set.serie?.name ?? null, total: set.cardCount?.official ?? set.cardCount?.total ?? null }; for (const c of set.cards) { if (this.reached(ctx, count)) return; const url = `${API}/${lang}/cards/${encodeURIComponent(c.id)}`; const cardRes = await this.get(ctx, url); if (!cardRes.success || !cardRes.json) { ctx.anomaly('card_unavailable', `${lang}/${c.id}`); continue; } let card: TcgCard; try { card = trimCard(cardRes.json as Record); } catch (err) { ctx.anomaly('parse_failure_card', `${lang}/${c.id}: ${err instanceof Error ? err.message : String(err)}`); continue; } count++; yield { url: `https://tcgdex.net/${lang}/database/${set.id}/${card.localId}`, externalId: `${lang}:${card.id}`, kind: 'catalog_item', engine: 'api', httpStatus: cardRes.httpStatus, payload: { card, set: setMeta, language: lang }, fetchedAt: cardRes.fetchedAt }; } await ctx.setCursor({ langIndex, setIndex: setIndex + 1, updatedAt: new Date().toISOString() }); } } await ctx.setCursor({ langIndex: 0, setIndex: 0, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const { card, set, language } = RawPayloadSchema.parse(raw.payload); const year = set.releaseDate ? Number(set.releaseDate.slice(0, 4)) || null : null; const setCode = set.code ?? set.id.toUpperCase(); const langName = LANG_NAME[language] ?? language; const sourceUrl = raw.url; const images = card.image ? [`${card.image}/high.webp`] : []; const variants = card.variants_detailed?.length ? card.variants_detailed : Object.entries(card.variants ?? {}) .filter(([, v]) => v) .map(([k]) => ({ type: k === 'firstEdition' ? 'firstEdition' : k, subtype: undefined, stamp: undefined, thirdParty: undefined, pricing: undefined })) .filter((v) => v.type !== 'wPromo'); if (!variants.length) variants.push({ type: 'normal', subtype: undefined, stamp: undefined, thirdParty: undefined, pricing: undefined }); const out: NormalizedRecord[] = []; const seenVariant = new Set(); for (const v of variants) { const variant = variantLabel(v.type, v.stamp) ?? (v.subtype && v.subtype !== 'unlimited' && v.subtype !== 'standard' ? cap(v.subtype) : null); const vkey = variant ?? ''; if (seenVariant.has(vkey)) continue; seenVariant.add(vkey); const identifiers: Record = { tcgdex_id: `${language}:${card.id}` }; if (language === 'en' && !variant?.includes('1st Edition')) identifiers.pokemontcg_id = card.id; if (v.thirdParty?.tcgplayer) identifiers.tcgplayer_id = String(v.thirdParty.tcgplayer); if (v.thirdParty?.cardmarket) identifiers.cardmarket_id = String(v.thirdParty.cardmarket); const a = attrs({ categorySlug: 'pokemon', franchise: 'Pokémon', brand: 'The Pokémon Company', series: set.serie, set: set.name, setCode, name: card.name, number: card.localId.replace(/^0+(?=\d)/, ''), year, variant, language: langName, rarity: card.rarity ?? null, identifiers, metadata: { hp: card.hp, types: card.types, illustrator: card.illustrator, dexIds: card.dexId, regulationMark: card.regulationMark, category: card.category, tcgdex_set: set.id }, }); const rawTitle = makeTitle({ name: card.name, set: set.name, number: a.number, total: set.total, year, variant }); const observedAt = card.updated ? new Date(card.updated) : raw.fetchedAt; out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: set.releaseDate ? new Date(set.releaseDate) : null })); const pricing = (v.pricing ?? (variants.length === 1 ? card.pricing : null)) ?? null; const tcg = pricing?.tcgplayer as Record | null | undefined; if (tcg) { const updated = typeof tcg.updated === 'string' ? new Date(tcg.updated) : null; for (const [k, val] of Object.entries(tcg)) { if (!val || typeof val !== 'object') continue; const bucket = val as Record; const kinds: Array<[string, 'market' | 'low' | 'mid' | 'high']> = [['marketPrice', 'market'], ['lowPrice', 'low'], ['midPrice', 'mid'], ['highPrice', 'high']]; for (const [field, priceKind] of kinds) { const price = num(bucket[field]); if (!price || !updated) continue; out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}:${k}:${priceKind}`, rawTitle, imageUrls: images, attributes: a, observedAt: updated, confidence: 0.8, parserVersion: PARSER_VERSION, priceKind, price, currency: 'USD', observationDate: updated, sampleSize: null })); } } } const cm = pricing?.cardmarket as Record | null | undefined; if (cm) { const updated = typeof cm.updated === 'string' ? new Date(cm.updated) : null; const kinds: Array<[string, 'market' | 'low' | 'trend' | 'average_7d' | 'average_30d']> = [['avg', 'market'], ['low', 'low'], ['trend', 'trend'], ['avg7', 'average_7d'], ['avg30', 'average_30d']]; for (const [field, priceKind] of kinds) { const price = num(cm[field]); if (!price || !updated) continue; out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}:cm:${priceKind}`, rawTitle, imageUrls: images, attributes: a, observedAt: updated, confidence: 0.75, parserVersion: PARSER_VERSION, priceKind, price, currency: 'EUR', observationDate: updated, sampleSize: null })); } } } return out; } } function cap(s: string): string { return s.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()); } export default (meta: ConnectorMeta) => new TcgdexConnector(meta);