TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, priceObservation } from '../_lib/shared.js';5import { BOT_HEADERS, fetchGzipJson, isoDay, magicFinishVariant, magicTreatments, streamGzipDataEntries } from '../_lib/wave4.js';67/**8 * MTGJSON connector — per-set card files (identifiers across Scryfall/TCGplayer/Cardmarket/Card Kingdom)9 * joined with the daily price file. One raw record per printing; normalize emits catalog items per10 * finish and dated price observations per provider × finish.11 */12const API = 'https://mtgjson.com/api/v5';13const PARSER_VERSION = '1.0.0';1415const SetSchema = z.object({ code: z.string(), name: z.string(), releaseDate: z.string().nullable().optional(), type: z.string().nullable().optional(), tcgplayerGroupId: z.number().nullable().optional(), mcmId: z.number().nullable().optional(), isOnlineOnly: z.boolean().optional(), totalSetSize: z.number().optional() });16const CardSchema = z.object({17 uuid: z.string(),18 name: z.string(),19 number: z.string(),20 rarity: z.string().nullable().optional(),21 finishes: z.array(z.string()).default([]),22 language: z.string().nullable().optional(),23 identifiers: z.record(z.string(), z.string()).default({}),24 frameEffects: z.array(z.string()).optional(),25 promoTypes: z.array(z.string()).optional(),26 isFullArt: z.boolean().optional(),27 borderColor: z.string().optional(),28 frameVersion: z.string().optional(),29 isPromo: z.boolean().optional(),30 isReserved: z.boolean().optional(),31 artist: z.string().nullable().optional(),32 type: z.string().nullable().optional(),33 manaCost: z.string().nullable().optional(),34});35/** provider → finish → { date: price } */36const ProviderPricesSchema = z.object({ retail: z.record(z.string(), z.record(z.string(), z.number())).optional(), buylist: z.record(z.string(), z.record(z.string(), z.number())).optional(), currency: z.string().optional() });37const PaperPricesSchema = z.record(z.string(), ProviderPricesSchema);38const RawPayloadSchema = z.object({ set: SetSchema, card: CardSchema, prices: PaperPricesSchema.nullable(), priceDate: z.string().nullable() });39export type MtgjsonPayload = z.infer<typeof RawPayloadSchema>;4041type PriceMap = Map<string, z.infer<typeof PaperPricesSchema>>;4243const CARD_KEEP = ['uuid', 'name', 'number', 'rarity', 'finishes', 'language', 'identifiers', 'frameEffects', 'promoTypes', 'isFullArt', 'borderColor', 'frameVersion', 'isPromo', 'isReserved', 'artist', 'type', 'manaCost'] as const;44export function trimCard(c: Record<string, unknown>): z.infer<typeof CardSchema> {45 const out: Record<string, unknown> = {};46 for (const k of CARD_KEEP) if (c[k] !== undefined) out[k] = c[k];47 return CardSchema.parse(out);48}49export function trimSet(s: Record<string, unknown>): z.infer<typeof SetSchema> {50 const out: Record<string, unknown> = {};51 for (const k of ['code', 'name', 'releaseDate', 'type', 'tcgplayerGroupId', 'mcmId', 'isOnlineOnly', 'totalSetSize']) if (s[k] !== undefined) out[k] = s[k];52 return SetSchema.parse(out);53}5455/** Keep only the latest `days` dates per provider/finish (payload compactness). */56function trimPrices(paper: unknown, days: number): z.infer<typeof PaperPricesSchema> | null {57 const parsed = PaperPricesSchema.safeParse(paper);58 if (!parsed.success) return null;59 const out: z.infer<typeof PaperPricesSchema> = {};60 for (const [provider, p] of Object.entries(parsed.data)) {61 const trimmed: z.infer<typeof ProviderPricesSchema> = { currency: p.currency };62 for (const kind of ['retail', 'buylist'] as const) {63 const byFinish = p[kind];64 if (!byFinish) continue;65 const tf: Record<string, Record<string, number>> = {};66 for (const [finish, series] of Object.entries(byFinish)) {67 const dates = Object.keys(series).sort().slice(-days);68 tf[finish] = Object.fromEntries(dates.map((d) => [d, series[d]!]));69 }70 trimmed[kind] = tf;71 }72 out[provider] = trimmed;73 }74 return out;75}7677export class MtgjsonConnector extends BaseConnector {78 readonly version = '1.0.0';79 readonly parserVersion = PARSER_VERSION;80 protected override minIntervalMs = 250;8182 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {83 const backfill = ctx.options.mode === 'backfill';84 const maxSets = backfill ? Infinity : Number(this.meta.config.maxSetsPerRun ?? 60);85 const listRes = await ctx.fetch(`${API}/SetList.json`, { engines: ['api'], headers: BOT_HEADERS });86 const list = (listRes.json as { data?: unknown[] } | null)?.data;87 if (!listRes.success || !Array.isArray(list)) throw new Error(`mtgjson SetList failed: ${listRes.error}`);88 let sets = list.map((s) => trimSet(s as Record<string, unknown>)).filter((s) => !s.isOnlineOnly);89 if (ctx.options.seeds?.length) sets = sets.filter((s) => ctx.options.seeds!.includes(s.code));90 sets.sort((a, b) => (b.releaseDate ?? '').localeCompare(a.releaseDate ?? ''));91 sets = sets.slice(0, Number.isFinite(maxSets) ? maxSets : sets.length);9293 // Daily prices (small) — joined by uuid. Backfill streams the 90-day history instead.94 const prices: PriceMap = new Map();95 let priceDate: string | null = null;96 if (!backfill) {97 const today = await fetchGzipJson<{ meta?: { date?: string }; data?: Record<string, { paper?: unknown }> }>(`${API}/AllPricesToday.json.gz`, ctx.signal);98 priceDate = today.meta?.date ?? null;99 for (const [uuid, v] of Object.entries(today.data ?? {})) {100 const t = trimPrices(v.paper, 1);101 if (t) prices.set(uuid, t);102 }103 const s = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });104 s.attempts++;105 s.success++;106 }107108 let count = 0;109 let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);110 const cards = new Map<string, { set: z.infer<typeof SetSchema>; card: z.infer<typeof CardSchema>; url: string }>();111 for (; setIdx < sets.length; setIdx++) {112 if (ctx.signal?.aborted) return;113 const set = sets[setIdx]!;114 await this.throttle();115 const res = await ctx.fetch(`${API}/${set.code}.json`, { engines: ['api'], headers: BOT_HEADERS });116 const data = (res.json as { data?: { cards?: unknown[] } } | null)?.data;117 if (!res.success || !Array.isArray(data?.cards)) {118 ctx.anomaly('page_fetch_failed', `${set.code}: ${res.error ?? res.httpStatus}`);119 continue;120 }121 for (const raw of data.cards) {122 let card: z.infer<typeof CardSchema>;123 try {124 card = trimCard(raw as Record<string, unknown>);125 } catch (err) {126 ctx.anomaly('parse_failure_card', `${set.code}: ${err instanceof Error ? err.message : String(err)}`);127 continue;128 }129 const url = `https://mtgjson.com/api/v5/${set.code}.json#${card.uuid}`;130 if (backfill) {131 cards.set(card.uuid, { set, card, url });132 continue;133 }134 if (this.reached(ctx, count)) return;135 count++;136 const payload: MtgjsonPayload = { set, card, prices: prices.get(card.uuid) ?? null, priceDate };137 yield { url, externalId: card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };138 }139 await ctx.setCursor({ setIdx: setIdx + 1, priceDate });140 }141 if (backfill) {142 // Stream the 90-day history and yield one record per known printing.143 const days = Number(this.meta.config.historyDays ?? 90);144 const queue: RawRecordInput[] = [];145 const seen = await streamGzipDataEntries(146 `${API}/AllPrices.json.gz`,147 (uuid, value) => {148 const c = cards.get(uuid);149 if (!c) return;150 const paper = (value as { paper?: unknown })?.paper;151 const trimmed = trimPrices(paper, days);152 const payload: MtgjsonPayload = { set: c.set, card: c.card, prices: trimmed, priceDate: null };153 queue.push({ url: c.url, externalId: c.card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() });154 },155 { signal: ctx.signal, limit: ctx.options.limit ? ctx.options.limit * 4 : undefined },156 );157 ctx.log.info({ seen, matched: queue.length }, 'mtgjson AllPrices streamed');158 for (const r of queue) {159 if (this.reached(ctx, count)) return;160 count++;161 yield r;162 }163 }164 await ctx.setCursor({ setIdx: 0, priceDate, completedAt: new Date().toISOString() });165 }166167 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {168 const { set, card, prices } = RawPayloadSchema.parse(raw.payload);169 const year = set.releaseDate ? Number(set.releaseDate.slice(0, 4)) || null : null;170 const ids: Record<string, string> = { mtgjson_uuid: card.uuid };171 const map: Array<[string, string]> = [['scryfallId', 'scryfall_id'], ['scryfallOracleId', 'oracle_id'], ['tcgplayerProductId', 'tcgplayer_id'], ['tcgplayerEtchedProductId', 'tcgplayer_etched_id'], ['mcmId', 'cardmarket_id'], ['cardKingdomId', 'cardkingdom_id'], ['cardKingdomFoilId', 'cardkingdom_foil_id'], ['cardKingdomEtchedId', 'cardkingdom_etched_id'], ['cardsphereId', 'cardsphere_id'], ['multiverseId', 'multiverse_id'], ['mtgoId', 'mtgo_id']];172 for (const [from, to] of map) if (card.identifiers[from]) ids[to] = card.identifiers[from]!;173 const treat = magicTreatments({ ...card, setType: set.type ?? null, releaseDate: set.releaseDate ?? null });174 const baseVariant = treat.length ? treat.join(' ') : null;175 const finishes = card.finishes.length ? card.finishes : ['nonfoil'];176 const observedAt = raw.fetchedAt;177 const providers = new Set((this.meta.config.providers as string[] | undefined) ?? ['tcgplayer', 'cardmarket', 'cardkingdom', 'cardsphere', 'manapool']);178 const out: NormalizedRecord[] = [];179 for (const finish of finishes) {180 const variant = [baseVariant, magicFinishVariant(finish)].filter(Boolean).join(' ') || null;181 const a = attrs({182 categorySlug: 'magic_the_gathering',183 franchise: 'Magic: The Gathering',184 brand: 'Wizards of the Coast',185 set: set.name,186 setCode: set.code.toUpperCase(),187 name: card.name,188 number: card.number,189 year,190 variant,191 language: card.language ?? 'English',192 rarity: card.rarity ?? null,193 identifiers: { ...ids, finish },194 metadata: { artist: card.artist ?? null, type_line: card.type ?? null, mana_cost: card.manaCost ?? null, promo: card.isPromo ?? false, reserved: card.isReserved ?? false, set_type: set.type ?? null },195 });196 const rawTitle = makeTitle({ name: card.name, set: set.name, number: card.number, year, variant });197 const sourceUrl = `https://scryfall.com/card/${set.code.toLowerCase()}/${encodeURIComponent(card.number)}`;198 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.uuid}:${finish}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: set.releaseDate ? new Date(set.releaseDate) : null }));199 if (!prices) continue;200 for (const [provider, p] of Object.entries(prices)) {201 if (!providers.has(provider)) continue;202 // MTGJSON price files key finishes as normal|foil|etched while card.finishes uses nonfoil|foil|etched.203 const priceKey = finish === 'nonfoil' ? 'normal' : finish;204 const retail = p.retail?.[priceKey];205 if (!retail) continue;206 const currency = (p.currency ?? (provider === 'cardmarket' ? 'EUR' : 'USD')) as 'USD' | 'EUR';207 const buy = p.buylist?.[priceKey];208 for (const [date, price] of Object.entries(retail)) {209 const observationDate = isoDay(date);210 if (!observationDate || !(price > 0)) continue;211 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.uuid}:${finish}:${provider}:${date}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider, buylist: buy?.[date] ?? null } }, observedAt, confidence: provider === 'tcgplayer' || provider === 'cardmarket' ? 0.75 : 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency, observationDate, sampleSize: null }));212 }213 }214 }215 return out;216 }217}218219export default (meta: ConnectorMeta) => new MtgjsonConnector(meta);220