TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors';3import { PriceChartingLikeConnector, parseProductPage as parsePage, splitTitle, setNameFromConsole, type ProductPayload, type SiteOptions } from '../_lib/pc-core.js';45/**6 * PriceCharting connector — video games, LEGO, Funko, comics AND trading cards (Pokémon, Magic,7 * Yu-Gi-Oh!) with per-grade eBay completed sales (Ungraded, Grade 1–9.5, PSA/BGS/CGC/SGC/TAG/ACE 10).8 *9 * Card products are enriched at crawl time against reference catalogs so their sales attach to the10 * canonical assets created by the API connectors:11 * - Pokémon: set name → pokemontcg set (id + ptcgoCode) via the maintainers' GitHub mirror;12 * `pokemontcg_id` = `${setId}-${number}` for unlimited printings.13 * - Magic: set name → Scryfall set code; exact-name lookup → scryfall_id + collector number.14 * - Yu-Gi-Oh!: set code is the prefix of the printed card code (LOB-001 → LOB).15 */1617const BASE = 'https://www.pricecharting.com';18const POKEMON_SETS_MIRROR = 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/sets/en.json';19const SCRYFALL_SETS = 'https://api.scryfall.com/sets';20const SCRYFALL_NAMED = 'https://api.scryfall.com/cards/named';21const SCRYFALL_HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };2223const PkSetSchema = z.object({ id: z.string(), name: z.string(), ptcgoCode: z.string().optional(), releaseDate: z.string().optional(), series: z.string().optional() });24const ScrySetSchema = z.object({ code: z.string(), name: z.string(), set_type: z.string(), released_at: z.string().optional(), digital: z.boolean().optional() });2526const norm = (s: string) => s.toLowerCase().replace(/&/g, 'and').replace(/[^a-z0-9]+/g, ' ').trim();27const normNoSet = (s: string) => norm(s).replace(/\bset\b/g, '').replace(/\s+/g, ' ').trim();2829/** Classic Magic sets whose PriceCharting console names differ from Scryfall's. */30const MAGIC_OVERRIDES: Record<string, string> = {31 alpha: 'lea',32 beta: 'leb',33 unlimited: '2ed',34 revised: '3ed',35 'fourth edition': '4ed',36 'fifth edition': '5ed',37 'sixth edition': '6ed',38 'seventh edition': '7ed',39 'eighth edition': '8ed',40 'ninth edition': '9ed',41 'tenth edition': '10e',42 'collectors edition': 'ced',43 'international collectors edition': 'cei',44 'the dark': 'drk',45 'urzas saga': 'usg',46 'urzas legacy': 'ulg',47 'urzas destiny': 'uds',48 'ravnica city of guilds': 'rav',49 'time spiral timeshifted': 'tsb',50 'commander legends': 'cmr',51 'secret lair drop': 'sld',52 'lord of the rings': 'ltr',53 'marvel spider man': 'spm',54 'avatar the last airbender': 'tla',55 'teenage mutant ninja turtles': 'tmn',56 'the hobbit': 'hbt',57};58const POKEMON_OVERRIDES: Record<string, string> = {59 base: 'base1',60 'base 2': 'base4',61 promo: 'basep',62 'wizards black star promos': 'basep',63 'black star promo': 'basep',64 'legendary collection': 'base6',65 expedition: 'ecard1',66 aquapolis: 'ecard2',67 skyridge: 'ecard3',68 'xy evolutions': 'xy12',69 evolutions: 'xy12',70 'pokemon go': 'pgo',71 'mcdonalds 25th anniversary': 'mcd21',72 'mcdonalds collection 2019': 'mcd19',73 'celebrations classic collection': 'cel25c',74 'shining legends': 'sm35',75 'dragon majesty': 'sm75',76 'detective pikachu': 'det1',77 'generations': 'g1',78 'double crisis': 'dc1',79};8081export class PriceChartingConnector extends PriceChartingLikeConnector {82 protected readonly siteOptions: SiteOptions = {83 base: BASE,84 site: 'pricecharting',85 categoryPrefix: (cat) => ({ 'pokemon-cards': 'pokemon-', 'magic-cards': 'magic-', 'yugioh-cards': 'yugioh-' })[cat] ?? cat.replace(/-cards$/, '-'),86 };87 override readonly urlPatterns = [/^https?:\/\/(www\.)?pricecharting\.com\/game\/[^/]+\/[^/?#]+/i];8889 private pokemonSets: z.infer<typeof PkSetSchema>[] | null = null;90 private scrySets: z.infer<typeof ScrySetSchema>[] | null = null;91 private namedCache = new Map<string, { id: string; number: string; tcgplayer_id?: string } | null>();9293 protected override async prepare(ctx: CrawlContext): Promise<void> {94 if (!this.pokemonSets) {95 const r = await ctx.fetch(POKEMON_SETS_MIRROR, { engines: ['api'], force: true, responseType: 'text' });96 let data: unknown = r.json;97 if (!data && r.html) {98 try {99 data = JSON.parse(r.html); // GitHub raw serves text/plain100 } catch {101 data = null;102 }103 }104 if (r.success && Array.isArray(data)) this.pokemonSets = z.array(PkSetSchema.loose()).parse(data);105 else ctx.anomaly('reference_unavailable', `pokemon sets mirror: ${r.error ?? r.httpStatus}`);106 }107 if (!this.scrySets) {108 const r = await ctx.fetch(SCRYFALL_SETS, { engines: ['api'], headers: SCRYFALL_HEADERS, force: true });109 const data = (r.json as { data?: unknown[] } | null)?.data;110 if (r.success && Array.isArray(data)) this.scrySets = z.array(ScrySetSchema.loose()).parse(data).filter((s) => !s.digital);111 else ctx.anomaly('reference_unavailable', `scryfall sets: ${r.error ?? r.httpStatus}`);112 }113 }114115 /** Pokémon console name → pokemontcg set. */116 private pokemonSet(consoleName: string): { id: string; code: string; name: string } | null {117 if (!this.pokemonSets) return null;118 const raw = setNameFromConsole(consoleName, 'cards');119 if (/japanese|korean|chinese/i.test(raw)) return null; // non-English printings are separate catalogs120 const key = normNoSet(raw);121 const byId = (id: string) => this.pokemonSets!.find((s) => s.id === id);122 const override = POKEMON_OVERRIDES[key];123 const stripped = key.replace(/^(xy|sm|swsh|sv|bw|hgss|dp|ex|me)\s+/, '');124 const hit =125 (override ? byId(override) : undefined) ??126 this.pokemonSets.find((s) => norm(s.name) === norm(raw)) ??127 this.pokemonSets.find((s) => normNoSet(s.name) === key) ??128 this.pokemonSets.find((s) => normNoSet(s.name) === stripped) ??129 this.pokemonSets.find((s) => normNoSet(s.name).replace(/^(xy|sm|swsh|sv|bw|hgss|dp|ex|me)\s+/, '') === stripped);130 if (!hit) {131 const contains = this.pokemonSets.filter((s) => normNoSet(s.name) === key || (key.length > 6 && normNoSet(s.name).startsWith(key)));132 if (contains.length === 1) return toPk(contains[0]!);133 return null;134 }135 return toPk(hit);136 function toPk(s: z.infer<typeof PkSetSchema>) {137 return { id: s.id, code: (s.ptcgoCode ?? s.id).toUpperCase(), name: s.name };138 }139 }140141 /** Magic console name → Scryfall set. */142 private magicSet(consoleName: string): { code: string; name: string } | null {143 if (!this.scrySets) return null;144 const raw = setNameFromConsole(consoleName, 'cards');145 const key = norm(raw);146 const override = MAGIC_OVERRIDES[key];147 const byCode = (c: string) => this.scrySets!.find((s) => s.code === c);148 const hit = (override ? byCode(override) : undefined) ?? this.scrySets.find((s) => norm(s.name) === key) ?? this.scrySets.find((s) => norm(s.name) === `limited edition ${key}`) ?? this.scrySets.find((s) => norm(s.name).replace(/\bedition\b/g, '').trim() === key.replace(/\bedition\b/g, '').trim());149 if (hit) return { code: hit.code, name: hit.name };150 const TYPES = ['expansion', 'core', 'masters', 'draft_innovation', 'commander', 'masterpiece', 'promo', 'box', 'funny', 'starter'];151 const candidates = this.scrySets.filter((s) => (norm(s.name).startsWith(key) || norm(s.name).includes(key)) && TYPES.includes(s.set_type));152 if (candidates.length === 1) return { code: candidates[0]!.code, name: candidates[0]!.name };153 const main = candidates.filter((s) => s.set_type === 'expansion' || s.set_type === 'core');154 if (main.length === 1) return { code: main[0]!.code, name: main[0]!.name };155 return null;156 }157158 protected override async enrich(ctx: CrawlContext, payload: ProductPayload): Promise<ProductPayload> {159 const c = payload.consoleUri;160 if (c.startsWith('pokemon-')) {161 const set = this.pokemonSet(payload.consoleName);162 if (!set) return payload;163 const { number, variant } = splitTitle(payload.title, 'cards');164 const isBasePrinting = !variant || !/1st edition|shadowless/i.test(variant);165 const pokemontcgId = number && isBasePrinting ? `${set.id}-${number.split('/')[0]}` : undefined;166 return { ...payload, setRef: { id: set.id, code: set.code, name: set.name, source: 'pokemontcg-mirror' }, cardRef: pokemontcgId ? { pokemontcg_id: pokemontcgId } : null };167 }168 if (c.startsWith('magic-')) {169 const set = this.magicSet(payload.consoleName);170 if (!set) return payload;171 const { name, number } = splitTitle(payload.title, 'cards');172 let cardRef: ProductPayload['cardRef'] = null;173 const cacheKey = `${set.code}|${name.toLowerCase()}|${number ?? ''}`;174 if (!this.namedCache.has(cacheKey)) {175 await new Promise((r) => setTimeout(r, 120)); // Scryfall politeness (≤ 10 req/s)176 const url = number177 ? `https://api.scryfall.com/cards/${encodeURIComponent(set.code)}/${encodeURIComponent(number)}`178 : `${SCRYFALL_NAMED}?exact=${encodeURIComponent(name)}&set=${encodeURIComponent(set.code)}`;179 const r = await ctx.fetch(url, { engines: ['api'], headers: SCRYFALL_HEADERS, force: true, failOnHttpError: false });180 const j = r.json as { id?: string; collector_number?: string; tcgplayer_id?: number; object?: string } | null;181 this.namedCache.set(cacheKey, r.success && j?.object === 'card' && j.id && j.collector_number ? { id: j.id, number: j.collector_number, ...(j.tcgplayer_id ? { tcgplayer_id: String(j.tcgplayer_id) } : {}) } : null);182 if (this.namedCache.size > 5000) this.namedCache.clear();183 }184 const hit = this.namedCache.get(cacheKey) ?? null;185 if (hit) cardRef = { scryfall_id: hit.id, number: hit.number, ...(hit.tcgplayer_id ? { tcgplayer_id: hit.tcgplayer_id } : {}) };186 return { ...payload, setRef: { id: set.code, code: set.code.toUpperCase(), name: set.name, source: 'scryfall' }, cardRef };187 }188 return payload;189 }190}191192export default (meta: ConnectorMeta) => new PriceChartingConnector(meta);193export const parseProductPage = (html: string, url: string) => parsePage(html, url, 'pricecharting');194export { ProductPayloadSchema, type ProductPayload } from '../_lib/pc-core.js';195