import { z } from 'zod'; import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; import { PriceChartingLikeConnector, parseProductPage as parsePage, splitTitle, setNameFromConsole, type ProductPayload, type SiteOptions } from '../_lib/pc-core.js'; /** * PriceCharting connector — video games, LEGO, Funko, comics AND trading cards (Pokémon, Magic, * Yu-Gi-Oh!) with per-grade eBay completed sales (Ungraded, Grade 1–9.5, PSA/BGS/CGC/SGC/TAG/ACE 10). * * Card products are enriched at crawl time against reference catalogs so their sales attach to the * canonical assets created by the API connectors: * - Pokémon: set name → pokemontcg set (id + ptcgoCode) via the maintainers' GitHub mirror; * `pokemontcg_id` = `${setId}-${number}` for unlimited printings. * - Magic: set name → Scryfall set code; exact-name lookup → scryfall_id + collector number. * - Yu-Gi-Oh!: set code is the prefix of the printed card code (LOB-001 → LOB). */ const BASE = 'https://www.pricecharting.com'; const POKEMON_SETS_MIRROR = 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/sets/en.json'; const SCRYFALL_SETS = 'https://api.scryfall.com/sets'; const SCRYFALL_NAMED = 'https://api.scryfall.com/cards/named'; const SCRYFALL_HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' }; const PkSetSchema = z.object({ id: z.string(), name: z.string(), ptcgoCode: z.string().optional(), releaseDate: z.string().optional(), series: z.string().optional() }); const ScrySetSchema = z.object({ code: z.string(), name: z.string(), set_type: z.string(), released_at: z.string().optional(), digital: z.boolean().optional() }); const norm = (s: string) => s.toLowerCase().replace(/&/g, 'and').replace(/[^a-z0-9]+/g, ' ').trim(); const normNoSet = (s: string) => norm(s).replace(/\bset\b/g, '').replace(/\s+/g, ' ').trim(); /** Classic Magic sets whose PriceCharting console names differ from Scryfall's. */ const MAGIC_OVERRIDES: Record = { alpha: 'lea', beta: 'leb', unlimited: '2ed', revised: '3ed', 'fourth edition': '4ed', 'fifth edition': '5ed', 'sixth edition': '6ed', 'seventh edition': '7ed', 'eighth edition': '8ed', 'ninth edition': '9ed', 'tenth edition': '10e', 'collectors edition': 'ced', 'international collectors edition': 'cei', 'the dark': 'drk', 'urzas saga': 'usg', 'urzas legacy': 'ulg', 'urzas destiny': 'uds', 'ravnica city of guilds': 'rav', 'time spiral timeshifted': 'tsb', 'commander legends': 'cmr', 'secret lair drop': 'sld', 'lord of the rings': 'ltr', 'marvel spider man': 'spm', 'avatar the last airbender': 'tla', 'teenage mutant ninja turtles': 'tmn', 'the hobbit': 'hbt', }; const POKEMON_OVERRIDES: Record = { base: 'base1', 'base 2': 'base4', promo: 'basep', 'wizards black star promos': 'basep', 'black star promo': 'basep', 'legendary collection': 'base6', expedition: 'ecard1', aquapolis: 'ecard2', skyridge: 'ecard3', 'xy evolutions': 'xy12', evolutions: 'xy12', 'pokemon go': 'pgo', 'mcdonalds 25th anniversary': 'mcd21', 'mcdonalds collection 2019': 'mcd19', 'celebrations classic collection': 'cel25c', 'shining legends': 'sm35', 'dragon majesty': 'sm75', 'detective pikachu': 'det1', 'generations': 'g1', 'double crisis': 'dc1', }; export class PriceChartingConnector extends PriceChartingLikeConnector { protected readonly siteOptions: SiteOptions = { base: BASE, site: 'pricecharting', categoryPrefix: (cat) => ({ 'pokemon-cards': 'pokemon-', 'magic-cards': 'magic-', 'yugioh-cards': 'yugioh-' })[cat] ?? cat.replace(/-cards$/, '-'), }; override readonly urlPatterns = [/^https?:\/\/(www\.)?pricecharting\.com\/game\/[^/]+\/[^/?#]+/i]; private pokemonSets: z.infer[] | null = null; private scrySets: z.infer[] | null = null; private namedCache = new Map(); protected override async prepare(ctx: CrawlContext): Promise { if (!this.pokemonSets) { const r = await ctx.fetch(POKEMON_SETS_MIRROR, { engines: ['api'], force: true, responseType: 'text' }); let data: unknown = r.json; if (!data && r.html) { try { data = JSON.parse(r.html); // GitHub raw serves text/plain } catch { data = null; } } if (r.success && Array.isArray(data)) this.pokemonSets = z.array(PkSetSchema.loose()).parse(data); else ctx.anomaly('reference_unavailable', `pokemon sets mirror: ${r.error ?? r.httpStatus}`); } if (!this.scrySets) { const r = await ctx.fetch(SCRYFALL_SETS, { engines: ['api'], headers: SCRYFALL_HEADERS, force: true }); const data = (r.json as { data?: unknown[] } | null)?.data; if (r.success && Array.isArray(data)) this.scrySets = z.array(ScrySetSchema.loose()).parse(data).filter((s) => !s.digital); else ctx.anomaly('reference_unavailable', `scryfall sets: ${r.error ?? r.httpStatus}`); } } /** Pokémon console name → pokemontcg set. */ private pokemonSet(consoleName: string): { id: string; code: string; name: string } | null { if (!this.pokemonSets) return null; const raw = setNameFromConsole(consoleName, 'cards'); if (/japanese|korean|chinese/i.test(raw)) return null; // non-English printings are separate catalogs const key = normNoSet(raw); const byId = (id: string) => this.pokemonSets!.find((s) => s.id === id); const override = POKEMON_OVERRIDES[key]; const stripped = key.replace(/^(xy|sm|swsh|sv|bw|hgss|dp|ex|me)\s+/, ''); const hit = (override ? byId(override) : undefined) ?? this.pokemonSets.find((s) => norm(s.name) === norm(raw)) ?? this.pokemonSets.find((s) => normNoSet(s.name) === key) ?? this.pokemonSets.find((s) => normNoSet(s.name) === stripped) ?? this.pokemonSets.find((s) => normNoSet(s.name).replace(/^(xy|sm|swsh|sv|bw|hgss|dp|ex|me)\s+/, '') === stripped); if (!hit) { const contains = this.pokemonSets.filter((s) => normNoSet(s.name) === key || (key.length > 6 && normNoSet(s.name).startsWith(key))); if (contains.length === 1) return toPk(contains[0]!); return null; } return toPk(hit); function toPk(s: z.infer) { return { id: s.id, code: (s.ptcgoCode ?? s.id).toUpperCase(), name: s.name }; } } /** Magic console name → Scryfall set. */ private magicSet(consoleName: string): { code: string; name: string } | null { if (!this.scrySets) return null; const raw = setNameFromConsole(consoleName, 'cards'); const key = norm(raw); const override = MAGIC_OVERRIDES[key]; const byCode = (c: string) => this.scrySets!.find((s) => s.code === c); const hit = (override ? byCode(override) : undefined) ?? this.scrySets.find((s) => norm(s.name) === key) ?? this.scrySets.find((s) => norm(s.name) === `limited edition ${key}`) ?? this.scrySets.find((s) => norm(s.name).replace(/\bedition\b/g, '').trim() === key.replace(/\bedition\b/g, '').trim()); if (hit) return { code: hit.code, name: hit.name }; const TYPES = ['expansion', 'core', 'masters', 'draft_innovation', 'commander', 'masterpiece', 'promo', 'box', 'funny', 'starter']; const candidates = this.scrySets.filter((s) => (norm(s.name).startsWith(key) || norm(s.name).includes(key)) && TYPES.includes(s.set_type)); if (candidates.length === 1) return { code: candidates[0]!.code, name: candidates[0]!.name }; const main = candidates.filter((s) => s.set_type === 'expansion' || s.set_type === 'core'); if (main.length === 1) return { code: main[0]!.code, name: main[0]!.name }; return null; } protected override async enrich(ctx: CrawlContext, payload: ProductPayload): Promise { const c = payload.consoleUri; if (c.startsWith('pokemon-')) { const set = this.pokemonSet(payload.consoleName); if (!set) return payload; const { number, variant } = splitTitle(payload.title, 'cards'); const isBasePrinting = !variant || !/1st edition|shadowless/i.test(variant); const pokemontcgId = number && isBasePrinting ? `${set.id}-${number.split('/')[0]}` : undefined; return { ...payload, setRef: { id: set.id, code: set.code, name: set.name, source: 'pokemontcg-mirror' }, cardRef: pokemontcgId ? { pokemontcg_id: pokemontcgId } : null }; } if (c.startsWith('magic-')) { const set = this.magicSet(payload.consoleName); if (!set) return payload; const { name, number } = splitTitle(payload.title, 'cards'); let cardRef: ProductPayload['cardRef'] = null; const cacheKey = `${set.code}|${name.toLowerCase()}|${number ?? ''}`; if (!this.namedCache.has(cacheKey)) { await new Promise((r) => setTimeout(r, 120)); // Scryfall politeness (≤ 10 req/s) const url = number ? `https://api.scryfall.com/cards/${encodeURIComponent(set.code)}/${encodeURIComponent(number)}` : `${SCRYFALL_NAMED}?exact=${encodeURIComponent(name)}&set=${encodeURIComponent(set.code)}`; const r = await ctx.fetch(url, { engines: ['api'], headers: SCRYFALL_HEADERS, force: true, failOnHttpError: false }); const j = r.json as { id?: string; collector_number?: string; tcgplayer_id?: number; object?: string } | null; this.namedCache.set(cacheKey, r.success && j?.object === 'card' && j.id && j.collector_number ? { id: j.id, number: j.collector_number, ...(j.tcgplayer_id ? { tcgplayer_id: String(j.tcgplayer_id) } : {}) } : null); if (this.namedCache.size > 5000) this.namedCache.clear(); } const hit = this.namedCache.get(cacheKey) ?? null; if (hit) cardRef = { scryfall_id: hit.id, number: hit.number, ...(hit.tcgplayer_id ? { tcgplayer_id: hit.tcgplayer_id } : {}) }; return { ...payload, setRef: { id: set.code, code: set.code.toUpperCase(), name: set.name, source: 'scryfall' }, cardRef }; } return payload; } } export default (meta: ConnectorMeta) => new PriceChartingConnector(meta); export const parseProductPage = (html: string, url: string) => parsePage(html, url, 'pricecharting'); export { ProductPayloadSchema, type ProductPayload } from '../_lib/pc-core.js';