import { z } from 'zod'; import { BaseConnector, missingRequirements, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { normalizeCondition } from '@rareindex/taxonomy'; import { attrs, catalogItem, makeTitle } from '../_lib/shared.js'; import { JSON_HEADERS } from '../_g1-cards-eu-jp-lib/index.js'; /** * CardTrader API v2 (gated: CARDTRADER_API_TOKEN). games → expansions → blueprints/export (catalog with * Scryfall/Cardmarket/TCGplayer ids) → marketplace/products (cheapest public offers, EUR/USD). * One raw record per blueprint (with its offers); normalize emits a catalog item and one listing per offer. */ const API = 'https://api.cardtrader.com/api/v2'; const PARSER_VERSION = '1.0.0'; export const GameSchema = z.object({ id: z.number().int(), name: z.string(), display_name: z.string().nullable().optional() }); export const ExpansionSchema = z.object({ id: z.number().int(), game_id: z.number().int(), code: z.string().nullable().optional(), name: z.string() }); export const BlueprintSchema = z.object({ id: z.number().int(), name: z.string(), version: z.string().nullable().optional(), game_id: z.number().int(), category_id: z.number().int().nullable().optional(), expansion_id: z.number().int().nullable().optional(), image_url: z.string().nullable().optional(), scryfall_id: z.string().nullable().optional(), card_market_ids: z.array(z.number()).nullable().optional(), tcg_player_id: z.union([z.string(), z.number()]).nullable().optional(), fixed_properties: z.record(z.string(), z.unknown()).nullable().optional(), }); export type Blueprint = z.infer; export const ProductSchema = z.object({ id: z.number().int(), blueprint_id: z.number().int(), name_en: z.string().nullable().optional(), quantity: z.number().int().nullable().optional(), price: z.object({ cents: z.number(), currency: z.string() }), description: z.string().nullable().optional(), properties_hash: z.record(z.string(), z.unknown()).default({}), expansion: z.object({ id: z.number().int().optional(), code: z.string().nullable().optional(), name_en: z.string().nullable().optional() }).nullable().optional(), user: z.object({ id: z.number().int().optional(), username: z.string().nullable().optional(), country_code: z.string().nullable().optional(), user_type: z.string().nullable().optional(), can_sell_via_hub: z.boolean().optional() }).nullable().optional(), graded: z.boolean().nullable().optional(), on_vacation: z.boolean().nullable().optional(), bundle_size: z.number().int().nullable().optional(), }); export type Product = z.infer; const RawPayloadSchema = z.object({ game: GameSchema, categorySlug: z.string(), expansion: ExpansionSchema, blueprint: BlueprintSchema, products: z.array(ProductSchema) }); export type CardtraderPayload = z.infer; const CURRENCIES = new Set(['EUR', 'USD', 'GBP', 'CHF', 'CAD', 'AUD', 'JPY', 'SEK', 'NOK', 'DKK', 'PLN', 'CZK']); /** Game name → taxonomy slug via the configurable substring map (null = not tracked). */ export function slugForGame(name: string, map: Record): string | null { const s = name.toLowerCase(); for (const [needle, slug] of Object.entries(map)) if (s.includes(needle)) return slug; return null; } /** properties_hash → normalised bits (condition slug, language, foil, first edition, signed/altered). */ export function offerProperties(h: Record): { conditionRaw: string | null; condition: string | null; language: string | null; foil: boolean; firstEdition: boolean; signed: boolean; altered: boolean; reverse: boolean } { const conditionRaw = typeof h.condition === 'string' ? h.condition : null; const langKey = Object.keys(h).find((k) => /_language$|^language$/.test(k)); const lang = langKey && typeof h[langKey] === 'string' ? (h[langKey] as string) : null; const LANG: Record = { en: 'English', it: 'Italian', fr: 'French', de: 'German', es: 'Spanish', pt: 'Portuguese', jp: 'Japanese', ja: 'Japanese', ko: 'Korean', ru: 'Russian', zh: 'Chinese', 'zh-cn': 'Chinese', 'zh-tw': 'Chinese' }; const foil = Object.entries(h).some(([k, v]) => /_foil$|^foil$/.test(k) && (v === true || v === 'true')); const reverse = Object.entries(h).some(([k, v]) => /reverse/.test(k) && (v === true || v === 'true')); const firstEdition = Object.entries(h).some(([k, v]) => /first_edition/.test(k) && (v === true || v === 'true')); return { conditionRaw, condition: normalizeCondition('trading_cards', conditionRaw), language: lang ? (LANG[lang.toLowerCase()] ?? lang) : null, foil, firstEdition, signed: h.signed === true, altered: h.altered === true, reverse }; } export class CardtraderConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1000; private headers(): Record { return { ...JSON_HEADERS, authorization: `Bearer ${process.env.CARDTRADER_API_TOKEN ?? ''}` }; } private async get(ctx: CrawlContext, path: string, schema: z.ZodType): Promise { const url = `${API}${path}`; await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], headers: this.headers(), minQuality: 0 }); if (!res.success || res.json === null) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return null; } const parsed = schema.safeParse(res.json); if (!parsed.success) { ctx.anomaly('schema_drift', `${url}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`); return null; } return parsed.data; } async *crawl(ctx: CrawlContext): AsyncIterable { const missing = missingRequirements(this.meta); if (missing.length) { ctx.log.warn({ missing }, 'cardtrader disabled: missing API token'); return; } const slugMap = (this.meta.config.gameSlugs ?? {}) as Record; const includeMarketplace = this.meta.config.includeMarketplace !== false; const backfill = ctx.options.mode === 'backfill'; const perRun = backfill ? Infinity : Number(this.meta.config.expansionsPerRun ?? 30); const games = await this.get(ctx, '/games', z.array(GameSchema.loose())); if (!games) throw new Error('cardtrader: /games failed'); const tracked = new Map; slug: string }>(); for (const g of games) { const slug = slugForGame(`${g.name} ${g.display_name ?? ''}`, slugMap); if (slug) tracked.set(g.id, { game: GameSchema.parse(g), slug }); } const allExp = await this.get(ctx, '/expansions', z.array(ExpansionSchema.loose())); if (!allExp) throw new Error('cardtrader: /expansions failed'); let expansions = allExp.filter((e) => tracked.has(e.game_id)).map((e) => ExpansionSchema.parse(e)).sort((a, b) => b.id - a.id); if (ctx.options.seeds?.length) expansions = expansions.filter((e) => ctx.options.seeds!.includes(String(e.id)) || (e.code && ctx.options.seeds!.includes(e.code))); expansions = expansions.slice(0, Number.isFinite(perRun) ? perRun : expansions.length); let expIdx = Number(ctx.options.cursor?.expIdx ?? 0); let count = 0; for (; expIdx < expansions.length; expIdx++) { if (ctx.signal?.aborted) return; const expansion = expansions[expIdx]!; const { game, slug } = tracked.get(expansion.game_id)!; const blueprints = await this.get(ctx, `/blueprints/export?expansion_id=${expansion.id}`, z.array(z.unknown())); if (!blueprints) continue; const offers = new Map(); if (includeMarketplace) { const mp = await this.get(ctx, `/marketplace/products?expansion_id=${expansion.id}`, z.record(z.string(), z.array(z.unknown()))); for (const [bp, list] of Object.entries(mp ?? {})) { const parsed = list.map((p) => ProductSchema.safeParse(p)).filter((p) => p.success).map((p) => (p as { data: Product }).data); offers.set(Number(bp), parsed); } } for (const raw of blueprints) { const bp = BlueprintSchema.safeParse(raw); if (!bp.success) { ctx.anomaly('parse_failure_blueprint', `${expansion.id}: ${bp.error.issues[0]?.message}`); continue; } if (this.reached(ctx, count)) { await ctx.setCursor({ expIdx }); return; } count++; const payload: CardtraderPayload = { game, categorySlug: slug, expansion, blueprint: bp.data, products: offers.get(bp.data.id) ?? [] }; yield { url: `https://www.cardtrader.com/cards/${bp.data.id}`, externalId: String(bp.data.id), kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() }; } await ctx.setCursor({ expIdx: expIdx + 1 }); await ctx.progress({ page: expIdx + 1, totalPages: expansions.length, itemsProcessed: count }); } await ctx.setCursor({ expIdx: 0, completedAt: new Date().toISOString(), done: true }); } async normalize(raw: RawRecordLike): Promise { const { game, categorySlug, expansion, blueprint, products } = RawPayloadSchema.parse(raw.payload); const fp = blueprint.fixed_properties ?? {}; const str = (v: unknown) => (typeof v === 'string' && v.trim() ? v.trim() : typeof v === 'number' ? String(v) : null); const number = str(fp.collector_number) ?? str(fp.number); const rarity = str(Object.entries(fp).find(([k]) => /rarity/.test(k))?.[1]); const ids: Record = { cardtrader_blueprint_id: String(blueprint.id) }; if (blueprint.scryfall_id) ids.scryfall_id = blueprint.scryfall_id; if (blueprint.card_market_ids?.length) ids.cardmarket_id = String(blueprint.card_market_ids[0]); if (blueprint.tcg_player_id) ids.tcgplayer_id = String(blueprint.tcg_player_id); const base = attrs({ categorySlug, franchise: game.display_name ?? game.name, set: expansion.name, setCode: expansion.code?.toUpperCase() ?? null, name: blueprint.name, number, variant: blueprint.version ?? null, language: null, rarity, identifiers: ids, metadata: { cardtrader_game_id: game.id, cardtrader_expansion_id: expansion.id, category_id: blueprint.category_id ?? null, fixed_properties: fp }, }); const images = blueprint.image_url ? [blueprint.image_url.startsWith('http') ? blueprint.image_url : `https://www.cardtrader.com${blueprint.image_url}`] : []; const rawTitle = makeTitle({ name: blueprint.name, set: expansion.name, number, variant: blueprint.version ?? null }); const out: NormalizedRecord[] = [catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: String(blueprint.id), rawTitle, imageUrls: images, attributes: base, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null })]; for (const p of products) { const currency = p.price.currency.toUpperCase(); if (!CURRENCIES.has(currency) || !(p.price.cents > 0)) continue; const props = offerProperties(p.properties_hash); const variant = [blueprint.version, props.firstEdition ? '1st Edition' : null, props.reverse ? 'Reverse Holo' : props.foil ? 'Foil' : null].filter(Boolean).join(' ') || null; const bundle = (p.bundle_size ?? 1) > 1; const professional = p.user?.user_type === 'professional'; out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${blueprint.id}:${p.id}`, rawTitle: `${p.name_en ?? blueprint.name} · ${expansion.name}${variant ? ` ${variant}` : ''}${props.conditionRaw ? ` · ${props.conditionRaw}` : ''}${props.language ? ` (${props.language})` : ''}`, description: p.description?.slice(0, 500) ?? null, imageUrls: images, attributes: { ...base, variant, language: props.language, metadata: { ...base.metadata, properties: p.properties_hash, signed: props.signed, altered: props.altered, graded: p.graded ?? false, bundle_size: p.bundle_size ?? 1, seller_type: p.user?.user_type ?? null, seller_country: p.user?.country_code ?? null, cardtrader_zero: p.user?.can_sell_via_hub ?? null } }, grade: { grader: null, grade: null, qualifier: p.graded ? 'graded (grader not exposed by API)' : null, certificationNumber: null }, condition: { condition: props.condition, conditionRaw: props.conditionRaw, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: p.price.cents / 100, currency, seller: professional ? (p.user?.username ?? null) : null, location: p.user?.country_code ?? null, quantity: bundle ? p.bundle_size : (p.quantity ?? null), availability: p.on_vacation ? 'unknown' : 'available', }), ); } return out; } } export default (meta: ConnectorMeta) => new CardtraderConnector(meta);