TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';56/**7 * TCGdex connector — Pokémon TCG catalog for every language (en, ja, fr, de, es, it, pt, ko, zh…)8 * with TCGplayer / Cardmarket pricing per variant on English cards. Card ids share pokemontcg.io's9 * scheme for English sets (base1-4, swsh1-1), so `pokemontcg_id` lines up with the pokemontcg10 * connector and set codes use the same PTCGO abbreviation (tcgOnline) as pokemontcg's ptcgoCode.11 */12const API = 'https://api.tcgdex.net/v2';13const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };14const PARSER_VERSION = '1.0.0';1516const SetBriefSchema = z.object({ id: z.string(), name: z.string(), cardCount: z.object({ total: z.number(), official: z.number() }).partial().optional() });17const SetSchema = z.object({18 id: z.string(),19 name: z.string(),20 releaseDate: z.string().optional(),21 tcgOnline: z.string().optional(),22 abbreviation: z.object({ official: z.string().optional(), localized: z.string().optional() }).partial().optional(),23 serie: z.object({ id: z.string(), name: z.string() }).optional(),24 cardCount: z.object({ total: z.number().optional(), official: z.number().optional() }).partial().optional(),25 cards: z.array(z.object({ id: z.string(), localId: z.string(), name: z.string(), image: z.string().optional() })),26});27const TcgplayerPricesSchema = z.record(z.string(), z.unknown());28const CardSchema = z.object({29 id: z.string(),30 localId: z.string(),31 name: z.string(),32 rarity: z.string().optional(),33 category: z.string().optional(),34 illustrator: z.string().optional(),35 image: z.string().optional(),36 hp: z.number().optional(),37 types: z.array(z.string()).optional(),38 dexId: z.array(z.number()).optional(),39 regulationMark: z.string().optional(),40 updated: z.string().optional(),41 set: z.object({ id: z.string(), name: z.string() }),42 variants: z.object({ firstEdition: z.boolean().optional(), holo: z.boolean().optional(), normal: z.boolean().optional(), reverse: z.boolean().optional(), wPromo: z.boolean().optional() }).partial().optional(),43 variants_detailed: z44 .array(45 z46 .object({47 type: z.string(),48 subtype: z.string().optional(),49 stamp: z.array(z.string()).optional(),50 thirdParty: z.object({ cardmarket: z.number().optional(), tcgplayer: z.number().optional() }).partial().optional(),51 pricing: z.object({ cardmarket: z.record(z.string(), z.unknown()).nullable().optional(), tcgplayer: z.record(z.string(), z.unknown()).nullable().optional() }).partial().nullable().optional(),52 })53 .loose(),54 )55 .optional(),56 pricing: z.object({ cardmarket: z.record(z.string(), z.unknown()).nullable().optional(), tcgplayer: z.record(z.string(), z.unknown()).nullable().optional() }).partial().nullable().optional(),57});58type TcgCard = z.infer<typeof CardSchema>;5960const RawPayloadSchema = z.object({61 card: CardSchema,62 set: z.object({ id: z.string(), name: z.string(), code: z.string().nullable(), releaseDate: z.string().nullable(), serie: z.string().nullable(), total: z.number().nullable() }),63 language: z.string(),64});6566/** Keep the payload compact: drop attacks/abilities/legal text. */67export function trimCard(raw: Record<string, unknown>): TcgCard {68 const keep = ['id', 'localId', 'name', 'rarity', 'category', 'illustrator', 'image', 'hp', 'types', 'dexId', 'regulationMark', 'updated', 'set', 'variants', 'variants_detailed', 'pricing'];69 const out: Record<string, unknown> = {};70 for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];71 if (out.set && typeof out.set === 'object') out.set = { id: (out.set as { id: string }).id, name: (out.set as { name: string }).name };72 return CardSchema.parse(out);73}7475const LANG_NAME: Record<string, string> = { en: 'English', ja: 'Japanese', fr: 'French', de: 'German', es: 'Spanish', it: 'Italian', pt: 'Portuguese', ko: 'Korean', zh: 'Chinese', 'zh-tw': 'Chinese (Traditional)', 'zh-cn': 'Chinese (Simplified)', nl: 'Dutch', pl: 'Polish', ru: 'Russian', th: 'Thai', id: 'Indonesian' };7677/** Variant vocabulary aligned with the pokemontcg connector. */78function variantLabel(type: string, stamp?: string[]): string | null {79 const first = stamp?.some((s) => /1st/i.test(s));80 if (type === 'reverse') return first ? '1st Edition Reverse Holo' : 'Reverse Holo';81 if (type === 'holo') return first ? '1st Edition Holo' : 'Holo';82 if (type === 'firstEdition') return '1st Edition';83 return first ? '1st Edition' : null;84}8586export class TcgdexConnector extends BaseConnector {87 readonly version = '1.0.0';88 readonly parserVersion = PARSER_VERSION;89 protected override minIntervalMs = 120;90 override readonly urlPatterns = [/^https?:\/\/(www\.)?tcgdex\.net\/(?:[a-z]{2}\/)?(?:database|cards?)\//i];9192 private async get(ctx: CrawlContext, url: string) {93 await this.throttle();94 return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS, timeoutMs: 60_000 }), (r) => r.success && r.json !== null, 4, 1500);95 }9697 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {98 const languages = ctx.options.seeds?.length ? ctx.options.seeds.filter((s) => /^[a-z]{2}(-[a-z]{2})?$/.test(s)) : ((this.meta.config.languages as string[] | undefined) ?? ['en']);99 const seedSets = ctx.options.seeds?.filter((s) => !/^[a-z]{2}(-[a-z]{2})?$/.test(s)) ?? [];100 const cursor = ctx.options.cursor ?? {};101 let langIndex = Number(cursor.langIndex ?? 0);102 let setIndex = Number(cursor.setIndex ?? 0);103 let count = 0;104 for (; langIndex < languages.length; langIndex++, setIndex = 0) {105 const lang = languages[langIndex]!;106 const setsRes = await this.get(ctx, `${API}/${lang}/sets`);107 if (!setsRes.success || !Array.isArray(setsRes.json)) {108 ctx.anomaly('sets_unavailable', `${lang}: ${setsRes.error ?? setsRes.httpStatus}`);109 continue;110 }111 const sets = z.array(SetBriefSchema.loose()).parse(setsRes.json).filter((s) => !seedSets.length || seedSets.includes(s.id));112 for (; setIndex < sets.length; setIndex++) {113 const brief = sets[setIndex]!;114 if (ctx.signal?.aborted) return;115 const setRes = await this.get(ctx, `${API}/${lang}/sets/${encodeURIComponent(brief.id)}`);116 if (!setRes.success || !setRes.json) {117 ctx.anomaly('set_unavailable', `${lang}/${brief.id}`);118 continue;119 }120 const parsedSet = SetSchema.loose().safeParse(setRes.json);121 if (!parsedSet.success) {122 ctx.anomaly('parse_failure_set', `${lang}/${brief.id}: ${parsedSet.error.issues[0]?.message}`);123 continue;124 }125 const set = parsedSet.data;126 const setMeta = { id: set.id, name: set.name, code: set.tcgOnline ?? set.abbreviation?.official ?? null, releaseDate: set.releaseDate ?? null, serie: set.serie?.name ?? null, total: set.cardCount?.official ?? set.cardCount?.total ?? null };127 for (const c of set.cards) {128 if (this.reached(ctx, count)) return;129 const url = `${API}/${lang}/cards/${encodeURIComponent(c.id)}`;130 const cardRes = await this.get(ctx, url);131 if (!cardRes.success || !cardRes.json) {132 ctx.anomaly('card_unavailable', `${lang}/${c.id}`);133 continue;134 }135 let card: TcgCard;136 try {137 card = trimCard(cardRes.json as Record<string, unknown>);138 } catch (err) {139 ctx.anomaly('parse_failure_card', `${lang}/${c.id}: ${err instanceof Error ? err.message : String(err)}`);140 continue;141 }142 count++;143 yield { url: `https://tcgdex.net/${lang}/database/${set.id}/${card.localId}`, externalId: `${lang}:${card.id}`, kind: 'catalog_item', engine: 'api', httpStatus: cardRes.httpStatus, payload: { card, set: setMeta, language: lang }, fetchedAt: cardRes.fetchedAt };144 }145 await ctx.setCursor({ langIndex, setIndex: setIndex + 1, updatedAt: new Date().toISOString() });146 }147 }148 await ctx.setCursor({ langIndex: 0, setIndex: 0, updatedAt: new Date().toISOString() });149 }150151 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {152 const { card, set, language } = RawPayloadSchema.parse(raw.payload);153 const year = set.releaseDate ? Number(set.releaseDate.slice(0, 4)) || null : null;154 const setCode = set.code ?? set.id.toUpperCase();155 const langName = LANG_NAME[language] ?? language;156 const sourceUrl = raw.url;157 const images = card.image ? [`${card.image}/high.webp`] : [];158 const variants = card.variants_detailed?.length159 ? card.variants_detailed160 : Object.entries(card.variants ?? {})161 .filter(([, v]) => v)162 .map(([k]) => ({ type: k === 'firstEdition' ? 'firstEdition' : k, subtype: undefined, stamp: undefined, thirdParty: undefined, pricing: undefined }))163 .filter((v) => v.type !== 'wPromo');164 if (!variants.length) variants.push({ type: 'normal', subtype: undefined, stamp: undefined, thirdParty: undefined, pricing: undefined });165 const out: NormalizedRecord[] = [];166 const seenVariant = new Set<string>();167 for (const v of variants) {168 const variant = variantLabel(v.type, v.stamp) ?? (v.subtype && v.subtype !== 'unlimited' && v.subtype !== 'standard' ? cap(v.subtype) : null);169 const vkey = variant ?? '';170 if (seenVariant.has(vkey)) continue;171 seenVariant.add(vkey);172 const identifiers: Record<string, string> = { tcgdex_id: `${language}:${card.id}` };173 if (language === 'en' && !variant?.includes('1st Edition')) identifiers.pokemontcg_id = card.id;174 if (v.thirdParty?.tcgplayer) identifiers.tcgplayer_id = String(v.thirdParty.tcgplayer);175 if (v.thirdParty?.cardmarket) identifiers.cardmarket_id = String(v.thirdParty.cardmarket);176 const a = attrs({177 categorySlug: 'pokemon',178 franchise: 'Pokémon',179 brand: 'The Pokémon Company',180 series: set.serie,181 set: set.name,182 setCode,183 name: card.name,184 number: card.localId.replace(/^0+(?=\d)/, ''),185 year,186 variant,187 language: langName,188 rarity: card.rarity ?? null,189 identifiers,190 metadata: { hp: card.hp, types: card.types, illustrator: card.illustrator, dexIds: card.dexId, regulationMark: card.regulationMark, category: card.category, tcgdex_set: set.id },191 });192 const rawTitle = makeTitle({ name: card.name, set: set.name, number: a.number, total: set.total, year, variant });193 const observedAt = card.updated ? new Date(card.updated) : raw.fetchedAt;194 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: set.releaseDate ? new Date(set.releaseDate) : null }));195196 const pricing = (v.pricing ?? (variants.length === 1 ? card.pricing : null)) ?? null;197 const tcg = pricing?.tcgplayer as Record<string, unknown> | null | undefined;198 if (tcg) {199 const updated = typeof tcg.updated === 'string' ? new Date(tcg.updated) : null;200 for (const [k, val] of Object.entries(tcg)) {201 if (!val || typeof val !== 'object') continue;202 const bucket = val as Record<string, unknown>;203 const kinds: Array<[string, 'market' | 'low' | 'mid' | 'high']> = [['marketPrice', 'market'], ['lowPrice', 'low'], ['midPrice', 'mid'], ['highPrice', 'high']];204 for (const [field, priceKind] of kinds) {205 const price = num(bucket[field]);206 if (!price || !updated) continue;207 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}:${k}:${priceKind}`, rawTitle, imageUrls: images, attributes: a, observedAt: updated, confidence: 0.8, parserVersion: PARSER_VERSION, priceKind, price, currency: 'USD', observationDate: updated, sampleSize: null }));208 }209 }210 }211 const cm = pricing?.cardmarket as Record<string, unknown> | null | undefined;212 if (cm) {213 const updated = typeof cm.updated === 'string' ? new Date(cm.updated) : null;214 const kinds: Array<[string, 'market' | 'low' | 'trend' | 'average_7d' | 'average_30d']> = [['avg', 'market'], ['low', 'low'], ['trend', 'trend'], ['avg7', 'average_7d'], ['avg30', 'average_30d']];215 for (const [field, priceKind] of kinds) {216 const price = num(cm[field]);217 if (!price || !updated) continue;218 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}:cm:${priceKind}`, rawTitle, imageUrls: images, attributes: a, observedAt: updated, confidence: 0.75, parserVersion: PARSER_VERSION, priceKind, price, currency: 'EUR', observationDate: updated, sampleSize: null }));219 }220 }221 }222 return out;223 }224}225226function cap(s: string): string {227 return s.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());228}229230export default (meta: ConnectorMeta) => new TcgdexConnector(meta);231