TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, missingRequirements, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { normalizeCondition } from '@rareindex/taxonomy';5import { attrs, catalogItem, makeTitle } from '../_lib/shared.js';6import { JSON_HEADERS } from '../_g1-cards-eu-jp-lib/index.js';78/**9 * CardTrader API v2 (gated: CARDTRADER_API_TOKEN). games → expansions → blueprints/export (catalog with10 * Scryfall/Cardmarket/TCGplayer ids) → marketplace/products (cheapest public offers, EUR/USD).11 * One raw record per blueprint (with its offers); normalize emits a catalog item and one listing per offer.12 */13const API = 'https://api.cardtrader.com/api/v2';14const PARSER_VERSION = '1.0.0';1516export const GameSchema = z.object({ id: z.number().int(), name: z.string(), display_name: z.string().nullable().optional() });17export const ExpansionSchema = z.object({ id: z.number().int(), game_id: z.number().int(), code: z.string().nullable().optional(), name: z.string() });18export const BlueprintSchema = z.object({19 id: z.number().int(),20 name: z.string(),21 version: z.string().nullable().optional(),22 game_id: z.number().int(),23 category_id: z.number().int().nullable().optional(),24 expansion_id: z.number().int().nullable().optional(),25 image_url: z.string().nullable().optional(),26 scryfall_id: z.string().nullable().optional(),27 card_market_ids: z.array(z.number()).nullable().optional(),28 tcg_player_id: z.union([z.string(), z.number()]).nullable().optional(),29 fixed_properties: z.record(z.string(), z.unknown()).nullable().optional(),30});31export type Blueprint = z.infer<typeof BlueprintSchema>;32export const ProductSchema = z.object({33 id: z.number().int(),34 blueprint_id: z.number().int(),35 name_en: z.string().nullable().optional(),36 quantity: z.number().int().nullable().optional(),37 price: z.object({ cents: z.number(), currency: z.string() }),38 description: z.string().nullable().optional(),39 properties_hash: z.record(z.string(), z.unknown()).default({}),40 expansion: z.object({ id: z.number().int().optional(), code: z.string().nullable().optional(), name_en: z.string().nullable().optional() }).nullable().optional(),41 user: z.object({ id: z.number().int().optional(), username: z.string().nullable().optional(), country_code: z.string().nullable().optional(), user_type: z.string().nullable().optional(), can_sell_via_hub: z.boolean().optional() }).nullable().optional(),42 graded: z.boolean().nullable().optional(),43 on_vacation: z.boolean().nullable().optional(),44 bundle_size: z.number().int().nullable().optional(),45});46export type Product = z.infer<typeof ProductSchema>;47const RawPayloadSchema = z.object({ game: GameSchema, categorySlug: z.string(), expansion: ExpansionSchema, blueprint: BlueprintSchema, products: z.array(ProductSchema) });48export type CardtraderPayload = z.infer<typeof RawPayloadSchema>;4950const CURRENCIES = new Set(['EUR', 'USD', 'GBP', 'CHF', 'CAD', 'AUD', 'JPY', 'SEK', 'NOK', 'DKK', 'PLN', 'CZK']);5152/** Game name → taxonomy slug via the configurable substring map (null = not tracked). */53export function slugForGame(name: string, map: Record<string, string>): string | null {54 const s = name.toLowerCase();55 for (const [needle, slug] of Object.entries(map)) if (s.includes(needle)) return slug;56 return null;57}5859/** properties_hash → normalised bits (condition slug, language, foil, first edition, signed/altered). */60export function offerProperties(h: Record<string, unknown>): { conditionRaw: string | null; condition: string | null; language: string | null; foil: boolean; firstEdition: boolean; signed: boolean; altered: boolean; reverse: boolean } {61 const conditionRaw = typeof h.condition === 'string' ? h.condition : null;62 const langKey = Object.keys(h).find((k) => /_language$|^language$/.test(k));63 const lang = langKey && typeof h[langKey] === 'string' ? (h[langKey] as string) : null;64 const LANG: Record<string, string> = { en: 'English', it: 'Italian', fr: 'French', de: 'German', es: 'Spanish', pt: 'Portuguese', jp: 'Japanese', ja: 'Japanese', ko: 'Korean', ru: 'Russian', zh: 'Chinese', 'zh-cn': 'Chinese', 'zh-tw': 'Chinese' };65 const foil = Object.entries(h).some(([k, v]) => /_foil$|^foil$/.test(k) && (v === true || v === 'true'));66 const reverse = Object.entries(h).some(([k, v]) => /reverse/.test(k) && (v === true || v === 'true'));67 const firstEdition = Object.entries(h).some(([k, v]) => /first_edition/.test(k) && (v === true || v === 'true'));68 return { conditionRaw, condition: normalizeCondition('trading_cards', conditionRaw), language: lang ? (LANG[lang.toLowerCase()] ?? lang) : null, foil, firstEdition, signed: h.signed === true, altered: h.altered === true, reverse };69}7071export class CardtraderConnector extends BaseConnector {72 readonly version = '1.0.0';73 readonly parserVersion = PARSER_VERSION;74 protected override minIntervalMs = 1000;7576 private headers(): Record<string, string> {77 return { ...JSON_HEADERS, authorization: `Bearer ${process.env.CARDTRADER_API_TOKEN ?? ''}` };78 }7980 private async get<T>(ctx: CrawlContext, path: string, schema: z.ZodType<T>): Promise<T | null> {81 const url = `${API}${path}`;82 await this.throttle(url);83 const res = await ctx.fetch(url, { engines: ['api'], headers: this.headers(), minQuality: 0 });84 if (!res.success || res.json === null) {85 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);86 return null;87 }88 const parsed = schema.safeParse(res.json);89 if (!parsed.success) {90 ctx.anomaly('schema_drift', `${url}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`);91 return null;92 }93 return parsed.data;94 }9596 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {97 const missing = missingRequirements(this.meta);98 if (missing.length) {99 ctx.log.warn({ missing }, 'cardtrader disabled: missing API token');100 return;101 }102 const slugMap = (this.meta.config.gameSlugs ?? {}) as Record<string, string>;103 const includeMarketplace = this.meta.config.includeMarketplace !== false;104 const backfill = ctx.options.mode === 'backfill';105 const perRun = backfill ? Infinity : Number(this.meta.config.expansionsPerRun ?? 30);106 const games = await this.get(ctx, '/games', z.array(GameSchema.loose()));107 if (!games) throw new Error('cardtrader: /games failed');108 const tracked = new Map<number, { game: z.infer<typeof GameSchema>; slug: string }>();109 for (const g of games) {110 const slug = slugForGame(`${g.name} ${g.display_name ?? ''}`, slugMap);111 if (slug) tracked.set(g.id, { game: GameSchema.parse(g), slug });112 }113 const allExp = await this.get(ctx, '/expansions', z.array(ExpansionSchema.loose()));114 if (!allExp) throw new Error('cardtrader: /expansions failed');115 let expansions = allExp.filter((e) => tracked.has(e.game_id)).map((e) => ExpansionSchema.parse(e)).sort((a, b) => b.id - a.id);116 if (ctx.options.seeds?.length) expansions = expansions.filter((e) => ctx.options.seeds!.includes(String(e.id)) || (e.code && ctx.options.seeds!.includes(e.code)));117 expansions = expansions.slice(0, Number.isFinite(perRun) ? perRun : expansions.length);118 let expIdx = Number(ctx.options.cursor?.expIdx ?? 0);119 let count = 0;120 for (; expIdx < expansions.length; expIdx++) {121 if (ctx.signal?.aborted) return;122 const expansion = expansions[expIdx]!;123 const { game, slug } = tracked.get(expansion.game_id)!;124 const blueprints = await this.get(ctx, `/blueprints/export?expansion_id=${expansion.id}`, z.array(z.unknown()));125 if (!blueprints) continue;126 const offers = new Map<number, Product[]>();127 if (includeMarketplace) {128 const mp = await this.get(ctx, `/marketplace/products?expansion_id=${expansion.id}`, z.record(z.string(), z.array(z.unknown())));129 for (const [bp, list] of Object.entries(mp ?? {})) {130 const parsed = list.map((p) => ProductSchema.safeParse(p)).filter((p) => p.success).map((p) => (p as { data: Product }).data);131 offers.set(Number(bp), parsed);132 }133 }134 for (const raw of blueprints) {135 const bp = BlueprintSchema.safeParse(raw);136 if (!bp.success) {137 ctx.anomaly('parse_failure_blueprint', `${expansion.id}: ${bp.error.issues[0]?.message}`);138 continue;139 }140 if (this.reached(ctx, count)) {141 await ctx.setCursor({ expIdx });142 return;143 }144 count++;145 const payload: CardtraderPayload = { game, categorySlug: slug, expansion, blueprint: bp.data, products: offers.get(bp.data.id) ?? [] };146 yield { url: `https://www.cardtrader.com/cards/${bp.data.id}`, externalId: String(bp.data.id), kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };147 }148 await ctx.setCursor({ expIdx: expIdx + 1 });149 await ctx.progress({ page: expIdx + 1, totalPages: expansions.length, itemsProcessed: count });150 }151 await ctx.setCursor({ expIdx: 0, completedAt: new Date().toISOString(), done: true });152 }153154 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {155 const { game, categorySlug, expansion, blueprint, products } = RawPayloadSchema.parse(raw.payload);156 const fp = blueprint.fixed_properties ?? {};157 const str = (v: unknown) => (typeof v === 'string' && v.trim() ? v.trim() : typeof v === 'number' ? String(v) : null);158 const number = str(fp.collector_number) ?? str(fp.number);159 const rarity = str(Object.entries(fp).find(([k]) => /rarity/.test(k))?.[1]);160 const ids: Record<string, string> = { cardtrader_blueprint_id: String(blueprint.id) };161 if (blueprint.scryfall_id) ids.scryfall_id = blueprint.scryfall_id;162 if (blueprint.card_market_ids?.length) ids.cardmarket_id = String(blueprint.card_market_ids[0]);163 if (blueprint.tcg_player_id) ids.tcgplayer_id = String(blueprint.tcg_player_id);164 const base = attrs({165 categorySlug,166 franchise: game.display_name ?? game.name,167 set: expansion.name,168 setCode: expansion.code?.toUpperCase() ?? null,169 name: blueprint.name,170 number,171 variant: blueprint.version ?? null,172 language: null,173 rarity,174 identifiers: ids,175 metadata: { cardtrader_game_id: game.id, cardtrader_expansion_id: expansion.id, category_id: blueprint.category_id ?? null, fixed_properties: fp },176 });177 const images = blueprint.image_url ? [blueprint.image_url.startsWith('http') ? blueprint.image_url : `https://www.cardtrader.com${blueprint.image_url}`] : [];178 const rawTitle = makeTitle({ name: blueprint.name, set: expansion.name, number, variant: blueprint.version ?? null });179 const out: NormalizedRecord[] = [catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: String(blueprint.id), rawTitle, imageUrls: images, attributes: base, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null })];180 for (const p of products) {181 const currency = p.price.currency.toUpperCase();182 if (!CURRENCIES.has(currency) || !(p.price.cents > 0)) continue;183 const props = offerProperties(p.properties_hash);184 const variant = [blueprint.version, props.firstEdition ? '1st Edition' : null, props.reverse ? 'Reverse Holo' : props.foil ? 'Foil' : null].filter(Boolean).join(' ') || null;185 const bundle = (p.bundle_size ?? 1) > 1;186 const professional = p.user?.user_type === 'professional';187 out.push(188 NormalizedListingSchema.parse({189 kind: 'listing',190 connectorId: this.meta.id,191 sourceId: this.meta.sourceId,192 sourceUrl: raw.url,193 externalId: `${blueprint.id}:${p.id}`,194 rawTitle: `${p.name_en ?? blueprint.name} · ${expansion.name}${variant ? ` ${variant}` : ''}${props.conditionRaw ? ` · ${props.conditionRaw}` : ''}${props.language ? ` (${props.language})` : ''}`,195 description: p.description?.slice(0, 500) ?? null,196 imageUrls: images,197 attributes: { ...base, variant, language: props.language, metadata: { ...base.metadata, properties: p.properties_hash, signed: props.signed, altered: props.altered, graded: p.graded ?? false, bundle_size: p.bundle_size ?? 1, seller_type: p.user?.user_type ?? null, seller_country: p.user?.country_code ?? null, cardtrader_zero: p.user?.can_sell_via_hub ?? null } },198 grade: { grader: null, grade: null, qualifier: p.graded ? 'graded (grader not exposed by API)' : null, certificationNumber: null },199 condition: { condition: props.condition, conditionRaw: props.conditionRaw, completeness: null },200 observedAt: raw.fetchedAt,201 confidence: 0.8,202 parserVersion: PARSER_VERSION,203 listingType: 'fixed_price',204 price: p.price.cents / 100,205 currency,206 seller: professional ? (p.user?.username ?? null) : null,207 location: p.user?.country_code ?? null,208 quantity: bundle ? p.bundle_size : (p.quantity ?? null),209 availability: p.on_vacation ? 'unknown' : 'available',210 }),211 );212 }213 return out;214 }215}216217export default (meta: ConnectorMeta) => new CardtraderConnector(meta);218