TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, parseSlashDate, priceObservation, withRetries, yearOf } from '../_lib/shared.js';56const API = 'https://api.pokemontcg.io/v2';78const SetSchema = z.object({9 id: z.string(),10 name: z.string(),11 series: z.string().optional(),12 printedTotal: z.number().optional(),13 total: z.number().optional(),14 ptcgoCode: z.string().optional(),15 releaseDate: z.string().optional(),16 updatedAt: z.string().optional(),17 images: z.object({ symbol: z.string().optional(), logo: z.string().optional() }).optional(),18});19type PkSet = z.infer<typeof SetSchema>;2021const PriceBlock = z.object({ low: z.number().nullable().optional(), mid: z.number().nullable().optional(), high: z.number().nullable().optional(), market: z.number().nullable().optional(), directLow: z.number().nullable().optional() });2223const CardSchema = z.object({24 id: z.string(),25 name: z.string(),26 supertype: z.string().optional(),27 subtypes: z.array(z.string()).optional(),28 hp: z.string().optional(),29 types: z.array(z.string()).optional(),30 number: z.string(),31 artist: z.string().optional(),32 rarity: z.string().optional(),33 flavorText: z.string().optional(),34 nationalPokedexNumbers: z.array(z.number()).optional(),35 regulationMark: z.string().nullable().optional(),36 images: z.object({ small: z.string().optional(), large: z.string().optional() }).optional(),37 set: SetSchema,38 tcgplayer: z.object({ url: z.string().optional(), updatedAt: z.string().optional(), prices: z.record(z.string(), PriceBlock).optional() }).optional(),39 cardmarket: z.object({ url: z.string().optional(), updatedAt: z.string().optional(), prices: z.record(z.string(), z.number().nullable()).optional() }).optional(),40});41type PkCard = z.infer<typeof CardSchema>;4243const RawPayloadSchema = z.object({ card: CardSchema, source: z.enum(['api', 'github-mirror']).default('api') });4445const KEEP = ['id', 'name', 'supertype', 'subtypes', 'hp', 'types', 'number', 'artist', 'rarity', 'flavorText', 'nationalPokedexNumbers', 'regulationMark', 'images', 'set', 'tcgplayer', 'cardmarket'] as const;4647export function trimCard(card: Record<string, unknown>, set?: PkSet): PkCard {48 const out: Record<string, unknown> = {};49 for (const k of KEEP) if (card[k] !== undefined) out[k] = card[k];50 if (!out.set && set) out.set = set;51 if (out.set && typeof out.set === 'object') {52 const s = out.set as Record<string, unknown>;53 delete s.legalities;54 }55 return CardSchema.parse(out);56}5758/** TCGplayer price-variant key → RareIndex variant label (null = base printing). */59export const TCGPLAYER_VARIANTS: Record<string, string | null> = {60 normal: null,61 holofoil: 'Holo',62 reverseHolofoil: 'Reverse Holo',63 unlimited: null,64 unlimitedHolofoil: 'Holo',65 '1stEdition': '1st Edition',66 '1stEditionNormal': '1st Edition',67 '1stEditionHolofoil': '1st Edition Holo',68};6970export class PokemonTcgConnector extends BaseConnector {71 readonly version = '1.0.0';72 readonly parserVersion = '1.0.0';73 protected override minIntervalMs = 2100; // ≈ 28 req/min, under the keyless 30/min ceiling7475 private headers(): Record<string, string> {76 const key = process.env.POKEMONTCG_API_KEY;77 return key ? { 'x-api-key': key } : {};78 }7980 private async apiGet(ctx: CrawlContext, url: string) {81 await this.throttle();82 return withRetries(83 () => ctx.fetch(url, { engines: ['api'], headers: this.headers(), timeoutMs: 120_000 }),84 (r) => r.success && r.json !== null,85 5,86 2000,87 );88 }8990 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {91 const pageSize = Number(ctx.meta.config.pageSize ?? 250);92 const mirror = String(ctx.meta.config.mirrorBase ?? 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master');93 let sets: PkSet[] = [];94 const setsRes = await this.apiGet(ctx, `${API}/sets?pageSize=250&orderBy=releaseDate`);95 if (setsRes.success && setsRes.json) {96 sets = z.array(SetSchema).parse((setsRes.json as { data: unknown[] }).data);97 } else {98 ctx.anomaly('blocked_or_down', `sets endpoint failed (${setsRes.error}); using GitHub mirror for the set list`);99 const m = await ctx.fetch(`${mirror}/sets/en.json`, { engines: ['api'] });100 if (!m.success || !m.json) throw new Error(`pokemontcg: sets unavailable from API and mirror: ${m.error}`);101 sets = z.array(SetSchema).parse(m.json);102 }103 sets.sort((a, b) => (a.releaseDate ?? '').localeCompare(b.releaseDate ?? ''));104 const filterIds = ctx.options.seeds?.length ? new Set(ctx.options.seeds) : null;105 let setIndex = Number(ctx.options.cursor?.setIndex ?? 0);106 let page = Number(ctx.options.cursor?.page ?? 1);107 let count = 0;108 for (; setIndex < sets.length; setIndex++, page = 1) {109 const set = sets[setIndex]!;110 if (filterIds && !filterIds.has(set.id)) continue;111 let usedMirror = false;112 for (;;) {113 const url = `${API}/cards?q=set.id:${encodeURIComponent(set.id)}&pageSize=${pageSize}&page=${page}`;114 const res = await this.apiGet(ctx, url);115 let cards: Record<string, unknown>[] = [];116 let totalCount = 0;117 if (res.success && res.json) {118 const body = res.json as { data: Record<string, unknown>[]; totalCount?: number; count?: number };119 cards = body.data ?? [];120 totalCount = body.totalCount ?? cards.length;121 } else if (page === 1) {122 // API down for this set → public GitHub mirror (same schema, no prices).123 const m = await ctx.fetch(`${mirror}/cards/en/${encodeURIComponent(set.id)}.json`, { engines: ['api'] });124 if (!m.success || !Array.isArray(m.json)) {125 ctx.anomaly('empty_page', `set ${set.id}: API ${res.error}; mirror ${m.error ?? 'not an array'}`);126 break;127 }128 cards = m.json as Record<string, unknown>[];129 totalCount = cards.length;130 usedMirror = true;131 ctx.anomaly('fallback_feed', `set ${set.id} served from GitHub mirror (no prices)`);132 } else {133 ctx.anomaly('empty_page', `set ${set.id} page ${page}: ${res.error}`);134 break;135 }136 const fetchedAt = new Date();137 for (const c of cards) {138 let card: PkCard;139 try {140 card = trimCard(c, set);141 } catch (err) {142 ctx.anomaly('parse_failure', `card ${String(c.id)}: ${err instanceof Error ? err.message : String(err)}`);143 continue;144 }145 yield {146 url: `${API}/cards/${card.id}`,147 externalId: card.id,148 kind: 'catalog_item',149 engine: usedMirror ? 'feed' : 'api',150 httpStatus: 200,151 payload: { card, source: usedMirror ? 'github-mirror' : 'api' },152 fetchedAt,153 };154 count++;155 if (this.reached(ctx, count)) return;156 }157 if (usedMirror || page * pageSize >= totalCount || cards.length === 0) break;158 page++;159 await ctx.setCursor({ setIndex, page });160 }161 await ctx.setCursor({ setIndex: setIndex + 1, page: 1 });162 }163 await ctx.setCursor({ setIndex: 0, page: 1, completedAt: new Date().toISOString() });164 ctx.log.info({ count }, 'pokemontcg crawl done');165 }166167 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {168 const { card, source } = RawPayloadSchema.parse(raw.payload);169 const set = card.set;170 const year = yearOf(set.releaseDate);171 const images = [card.images?.large ?? card.images?.small].filter((x): x is string => Boolean(x));172 const sourceUrl = card.tcgplayer?.url ?? `https://pokemontcg.io/card/${card.id}`;173 const setCode = set.ptcgoCode ?? set.id.toUpperCase();174 const out: NormalizedRecord[] = [];175176 const buildAttrs = (variant: string | null) =>177 attrs({178 categorySlug: 'pokemon',179 franchise: 'Pokémon',180 brand: 'The Pokémon Company',181 series: set.series ?? null,182 set: set.name,183 setCode,184 name: card.name,185 number: card.number,186 year,187 variant,188 language: 'English',189 rarity: card.rarity ?? null,190 identifiers: { pokemontcg_id: card.id, pokemontcg_set_id: set.id, ...(card.tcgplayer?.url ? { tcgplayer_url: card.tcgplayer.url } : {}), ...(card.cardmarket?.url ? { cardmarket_url: card.cardmarket.url } : {}) },191 metadata: {192 supertype: card.supertype ?? null,193 subtypes: card.subtypes ?? [],194 hp: card.hp ?? null,195 types: card.types ?? [],196 artist: card.artist ?? null,197 nationalPokedexNumbers: card.nationalPokedexNumbers ?? [],198 regulationMark: card.regulationMark ?? null,199 printedTotal: set.printedTotal ?? null,200 setTotal: set.total ?? null,201 setSymbol: set.images?.symbol ?? null,202 dataSource: source,203 },204 });205206 const tcg = card.tcgplayer;207 const tcgDate = parseSlashDate(tcg?.updatedAt) ?? raw.fetchedAt;208 const variantKeys = Object.keys(tcg?.prices ?? {}).filter((k) => k in TCGPLAYER_VARIANTS);209 const variants: Array<string | null> = variantKeys.length ? [...new Set(variantKeys.map((k) => TCGPLAYER_VARIANTS[k] ?? null))] : [null];210 // Base printing: if the card only exists as holo, the canonical variant is 'Holo'.211 for (const variant of variants) {212 const a = buildAttrs(variant);213 out.push(214 catalogItem({215 kind: 'catalog_item',216 connectorId: this.meta.id,217 sourceId: this.meta.sourceId,218 sourceUrl,219 externalId: `${card.id}:${variant ?? 'base'}`,220 rawTitle: makeTitle({ name: card.name, set: set.name, number: card.number, total: set.printedTotal ?? null, year, variant }),221 description: card.flavorText ?? null,222 imageUrls: images,223 attributes: a,224 observedAt: raw.fetchedAt,225 confidence: 0.97,226 parserVersion: this.parserVersion,227 releaseDate: parseSlashDate(set.releaseDate),228 }),229 );230 }231232 const kinds: Array<['low' | 'mid' | 'high' | 'market', 'low' | 'mid' | 'high' | 'market']> = [233 ['market', 'market'],234 ['low', 'low'],235 ['mid', 'mid'],236 ['high', 'high'],237 ];238 for (const key of variantKeys) {239 const block = tcg!.prices![key]!;240 const variant = TCGPLAYER_VARIANTS[key] ?? null;241 const a = buildAttrs(variant);242 for (const [field, priceKind] of kinds) {243 const price = num(block[field]);244 if (price === null) continue;245 out.push(246 priceObservation({247 kind: 'price_observation',248 connectorId: this.meta.id,249 sourceId: this.meta.sourceId,250 sourceUrl,251 externalId: `${card.id}:tcgplayer:${key}:${field}`,252 rawTitle: makeTitle({ name: card.name, set: set.name, number: card.number, total: set.printedTotal ?? null, year, variant }),253 imageUrls: images,254 attributes: { ...a, metadata: { ...a.metadata, priceSource: 'tcgplayer', priceVariantKey: key } },255 condition: { condition: 'near_mint', conditionRaw: 'TCGplayer near mint market', completeness: null },256 observedAt: tcgDate,257 confidence: 0.9,258 parserVersion: this.parserVersion,259 priceKind,260 price,261 currency: 'USD',262 observationDate: tcgDate,263 sampleSize: null,264 }),265 );266 }267 }268269 const cm = card.cardmarket;270 if (cm?.prices) {271 const cmDate = parseSlashDate(cm.updatedAt) ?? raw.fetchedAt;272 // Cardmarket main prices refer to the card's primary printing; reverseHolo* fields to the reverse holo.273 const primary = variants.includes('Holo') && !variants.includes(null) ? 'Holo' : variants.includes(null) ? null : (variants[0] ?? null);274 const main: Array<[string, 'market' | 'low' | 'trend' | 'average_7d' | 'average_30d']> = [275 ['averageSellPrice', 'market'],276 ['lowPrice', 'low'],277 ['trendPrice', 'trend'],278 ['avg7', 'average_7d'],279 ['avg30', 'average_30d'],280 ];281 const reverse: Array<[string, 'market' | 'low' | 'trend' | 'average_7d' | 'average_30d']> = [282 ['reverseHoloSell', 'market'],283 ['reverseHoloLow', 'low'],284 ['reverseHoloTrend', 'trend'],285 ['reverseHoloAvg7', 'average_7d'],286 ['reverseHoloAvg30', 'average_30d'],287 ];288 const emit = (pairs: typeof main, variant: string | null, tag: string) => {289 const a = buildAttrs(variant);290 for (const [field, priceKind] of pairs) {291 const price = num(cm.prices![field]);292 if (price === null) continue;293 out.push(294 priceObservation({295 kind: 'price_observation',296 connectorId: this.meta.id,297 sourceId: this.meta.sourceId,298 sourceUrl: cm.url ?? sourceUrl,299 externalId: `${card.id}:cardmarket:${tag}:${field}`,300 rawTitle: makeTitle({ name: card.name, set: set.name, number: card.number, total: set.printedTotal ?? null, year, variant }),301 imageUrls: images,302 attributes: { ...a, metadata: { ...a.metadata, priceSource: 'cardmarket', priceVariantKey: tag } },303 condition: { condition: null, conditionRaw: 'Cardmarket aggregate', completeness: null },304 observedAt: cmDate,305 confidence: 0.75,306 parserVersion: this.parserVersion,307 priceKind,308 price,309 currency: 'EUR',310 observationDate: cmDate,311 sampleSize: null,312 }),313 );314 }315 };316 emit(main, primary, 'main');317 if (variants.includes('Reverse Holo')) emit(reverse, 'Reverse Holo', 'reverse');318 }319 return out;320 }321}322323export default function createConnector(meta: ConnectorMeta) {324 return new PokemonTcgConnector(meta);325}326