TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { createGunzip } from 'node:zlib';2import { Readable } from 'node:stream';3import { createInterface } from 'node:readline';4import { z } from 'zod';5import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';6import { type NormalizedRecord } from '@rareindex/shared';7import { attrs, catalogItem, makeTitle, num, priceObservation, yearOf } from '../_lib/shared.js';89const API = 'https://api.scryfall.com';10const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json;q=0.9,*/*;q=0.8' };1112/** Trimmed printing payload we persist as the raw record (everything normalize() needs, nothing more). */13const CardPayloadSchema = z.object({14 id: z.string(),15 oracle_id: z.string().optional(),16 name: z.string(),17 lang: z.string().optional(),18 released_at: z.string().optional(),19 set: z.string(),20 set_name: z.string(),21 set_type: z.string().optional(),22 collector_number: z.string(),23 rarity: z.string().optional(),24 layout: z.string().optional(),25 finishes: z.array(z.string()).optional(),26 frame_effects: z.array(z.string()).optional().nullable(),27 promo_types: z.array(z.string()).optional().nullable(),28 promo: z.boolean().optional(),29 reserved: z.boolean().optional(),30 reprint: z.boolean().optional(),31 variation: z.boolean().optional(),32 full_art: z.boolean().optional(),33 border_color: z.string().optional(),34 frame: z.string().optional(),35 digital: z.boolean().optional(),36 artist: z.string().optional(),37 mana_cost: z.string().optional(),38 type_line: z.string().optional(),39 cmc: z.number().optional(),40 colors: z.array(z.string()).optional(),41 tcgplayer_id: z.number().optional(),42 tcgplayer_etched_id: z.number().optional(),43 cardmarket_id: z.number().optional(),44 mtgo_id: z.number().optional(),45 arena_id: z.number().optional(),46 multiverse_ids: z.array(z.number()).optional(),47 scryfall_uri: z.string().optional(),48 image_uris: z.record(z.string(), z.string()).optional(),49 card_faces: z.array(z.object({ name: z.string().optional(), image_uris: z.record(z.string(), z.string()).optional() })).optional(),50 prices: z.record(z.string(), z.string().nullable()).optional(),51});52type CardPayload = z.infer<typeof CardPayloadSchema>;5354const RawPayloadSchema = z.object({55 card: CardPayloadSchema,56 /** ISO timestamp of the bulk file (prices' observation time); null for single-card lookups */57 bulkUpdatedAt: z.string().nullable().default(null),58});5960const KEEP: Array<keyof CardPayload> = ['id', 'oracle_id', 'name', 'lang', 'released_at', 'set', 'set_name', 'set_type', 'collector_number', 'rarity', 'layout', 'finishes', 'frame_effects', 'promo_types', 'promo', 'reserved', 'reprint', 'variation', 'full_art', 'border_color', 'frame', 'digital', 'artist', 'mana_cost', 'type_line', 'cmc', 'colors', 'tcgplayer_id', 'tcgplayer_etched_id', 'cardmarket_id', 'mtgo_id', 'arena_id', 'multiverse_ids', 'scryfall_uri', 'image_uris', 'card_faces', 'prices'];6162export function trimCard(card: Record<string, unknown>): CardPayload {63 const out: Record<string, unknown> = {};64 for (const k of KEEP) if (card[k] !== undefined) out[k] = card[k];65 if (Array.isArray(out.card_faces)) out.card_faces = (out.card_faces as Array<Record<string, unknown>>).map((f) => ({ name: f.name, image_uris: f.image_uris }));66 return CardPayloadSchema.parse(out);67}6869function finishVariant(finish: string): string | null {70 if (finish === 'foil') return 'Foil';71 if (finish === 'etched') return 'Etched Foil';72 return null;73}7475/** Frame effects / promo types that materially identify a distinct printing treatment on the market. */76const VARIANT_FRAME_EFFECTS = new Set(['showcase', 'extendedart', 'inverted', 'shatteredglass', 'etched', 'textured']);77const VARIANT_PROMO_TYPES = new Set(['serialized', 'prerelease', 'promopack', 'judgegift', 'buyabox', 'gameday', 'textured', 'galaxyfoil', 'surgefoil', 'stepandcompleat', 'confettifoil', 'oilslick', 'halofoil', 'neonink', 'ripplefoil', 'fracturefoil', 'rainbowfoil', 'raisedfoil', 'invisibleink', 'doublerainbow', 'manafoil', 'firstplacefoil', 'dragonscalefoil', 'silverfoil', 'gilded', 'embossed', 'startercollection', 'schinesealtart', 'datestamped', 'playerrewards', 'arenaleague', 'fnm', 'release', 'launch', 'convention', 'mediainsert', 'wizardsplaynetwork', 'thick', 'poster']);78const LABELS: Record<string, string> = { extendedart: 'Extended Art', promopack: 'Promo Pack', judgegift: 'Judge Promo', buyabox: 'Buy-a-Box', gameday: 'Game Day', galaxyfoil: 'Galaxy Foil', surgefoil: 'Surge Foil', stepandcompleat: 'Step-and-Compleat', confettifoil: 'Confetti Foil', oilslick: 'Oil Slick', halofoil: 'Halo Foil', neonink: 'Neon Ink', ripplefoil: 'Ripple Foil', fracturefoil: 'Fracture Foil', rainbowfoil: 'Rainbow Foil', raisedfoil: 'Raised Foil', invisibleink: 'Invisible Ink', doublerainbow: 'Double Rainbow', manafoil: 'Mana Foil', firstplacefoil: 'First Place Foil', dragonscalefoil: 'Dragon Scale Foil', silverfoil: 'Silver Foil', startercollection: 'Starter Collection', schinesealtart: 'Chinese Alt Art', datestamped: 'Date Stamped', playerrewards: 'Player Rewards', arenaleague: 'Arena League', fnm: 'FNM', mediainsert: 'Media Insert', wizardsplaynetwork: 'WPN', shatteredglass: 'Shattered Glass' };7980function treatments(card: CardPayload): string[] {81 const t: string[] = [];82 for (const fe of card.frame_effects ?? []) if (VARIANT_FRAME_EFFECTS.has(fe)) t.push(fe);83 for (const pt of card.promo_types ?? []) if (VARIANT_PROMO_TYPES.has(pt)) t.push(pt);84 if (card.full_art) t.push('fullart');85 if (card.border_color === 'borderless') t.push('borderless');86 if (card.frame === '1997' && card.set_type !== 'core' && card.set_type !== 'expansion' && (card.released_at ?? '') > '2010') t.push('retro');87 return [...new Set(t)].map((k) => LABELS[k] ?? (k === 'fullart' ? 'Full Art' : k.charAt(0).toUpperCase() + k.slice(1)));88}8990export class ScryfallConnector extends BaseConnector {91 readonly version = '1.0.0';92 readonly parserVersion = '1.0.0';93 override readonly urlPatterns = [/^https?:\/\/(www\.)?scryfall\.com\/card\/[^/]+\/[^/?#]+/i];94 protected override minIntervalMs = 100;9596 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {97 const bulkType = String(ctx.meta.config.bulkType ?? 'default_cards');98 const list = await ctx.fetch(`${API}/bulk-data`, { engines: ['api'], headers: HEADERS });99 if (!list.success || !list.json) throw new Error(`scryfall bulk-data listing failed: ${list.error}`);100 const entries = (list.json as { data: Array<{ type: string; updated_at: string; download_uri?: string; jsonl_download_uri?: string }> }).data;101 const entry = entries.find((e) => e.type === bulkType);102 if (!entry) throw new Error(`scryfall bulk type ${bulkType} not found`);103 const prev = ctx.options.cursor?.updatedAt as string | undefined;104 if (ctx.options.mode === 'incremental' && prev && prev === entry.updated_at) {105 ctx.log.info({ updatedAt: prev }, 'scryfall bulk unchanged, skipping');106 return;107 }108 const uri = entry.jsonl_download_uri ?? entry.download_uri;109 if (!uri) throw new Error('scryfall bulk entry has no download uri');110 const jsonl = Boolean(entry.jsonl_download_uri);111 const started = Date.now();112 const res = await fetch(uri, { headers: HEADERS, signal: ctx.signal });113 const stats = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });114 stats.attempts++;115 if (!res.ok || !res.body) throw new Error(`scryfall bulk download HTTP ${res.status}`);116 stats.success++;117 let count = 0;118 const fetchedAt = new Date();119 let body: NodeJS.ReadableStream = Readable.fromWeb(res.body as import('node:stream/web').ReadableStream);120 if (uri.endsWith('.gz')) body = body.pipe(createGunzip());121 const rl = createInterface({ input: body, crlfDelay: Infinity });122 for await (const rawLine of rl) {123 let line = rawLine.trim();124 if (!jsonl) {125 // legacy JSON array format: one object per line, wrapped by [ ] and separated by commas126 if (line === '[' || line === ']' || !line) continue;127 if (line.endsWith(',')) line = line.slice(0, -1);128 }129 if (!line) continue;130 let card: Record<string, unknown>;131 try {132 card = JSON.parse(line) as Record<string, unknown>;133 } catch {134 ctx.anomaly('parse_failure', 'bulk line is not JSON');135 continue;136 }137 if (card.object !== 'card' || card.digital === true) continue;138 let payload: CardPayload;139 try {140 payload = trimCard(card);141 } catch (err) {142 ctx.anomaly('parse_failure', `card ${String(card.id)}: ${err instanceof Error ? err.message : String(err)}`);143 continue;144 }145 yield {146 url: payload.scryfall_uri ?? `${API}/cards/${payload.id}`,147 externalId: payload.id,148 kind: 'catalog_item',149 engine: 'api',150 httpStatus: 200,151 payload: { card: payload, bulkUpdatedAt: entry.updated_at },152 fetchedAt,153 };154 count++;155 if (this.reached(ctx, count)) break;156 }157 stats.ms += Date.now() - started;158 rl.close();159 if (!this.reached(ctx, count)) await ctx.setCursor({ updatedAt: entry.updated_at, count });160 ctx.log.info({ count, ms: Date.now() - started }, 'scryfall bulk crawl done');161 }162163 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {164 const m = url.match(/scryfall\.com\/card\/([^/]+)\/([^/?#]+)/i);165 if (!m) return [];166 await this.throttle();167 const res = await ctx.fetch(`${API}/cards/${encodeURIComponent(m[1]!)}/${encodeURIComponent(m[2]!)}`, { engines: ['api'], headers: HEADERS });168 if (!res.success || !res.json) return [];169 const payload = trimCard(res.json as Record<string, unknown>);170 return [{ url: payload.scryfall_uri ?? url, externalId: payload.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card: payload, bulkUpdatedAt: null }, fetchedAt: res.fetchedAt }];171 }172173 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {174 const { card, bulkUpdatedAt } = RawPayloadSchema.parse(raw.payload);175 const year = yearOf(card.released_at);176 const sourceUrl = card.scryfall_uri ?? raw.url;177 const images = card.image_uris ? [card.image_uris.large ?? card.image_uris.normal ?? card.image_uris.png].filter((x): x is string => Boolean(x)) : (card.card_faces ?? []).map((f) => f.image_uris?.large ?? f.image_uris?.normal).filter((x): x is string => Boolean(x));178 const identifiers: Record<string, string> = { scryfall_id: card.id };179 if (card.oracle_id) identifiers.oracle_id = card.oracle_id;180 if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id);181 if (card.tcgplayer_etched_id) identifiers.tcgplayer_etched_id = String(card.tcgplayer_etched_id);182 if (card.cardmarket_id) identifiers.cardmarket_id = String(card.cardmarket_id);183 if (card.mtgo_id) identifiers.mtgo_id = String(card.mtgo_id);184 if (card.arena_id) identifiers.arena_id = String(card.arena_id);185 if (card.multiverse_ids?.length) identifiers.multiverse_id = String(card.multiverse_ids[0]);186 const treat = treatments(card);187 const baseVariant = treat.length ? treat.join(' ') : null;188 const finishes = card.finishes?.length ? card.finishes : ['nonfoil'];189 const observedAt = bulkUpdatedAt ? new Date(bulkUpdatedAt) : raw.fetchedAt;190 const out: NormalizedRecord[] = [];191192 const buildAttrs = (finish: string) => {193 const fv = finishVariant(finish);194 const variant = [baseVariant, fv].filter(Boolean).join(' ') || null;195 return attrs({196 categorySlug: 'magic_the_gathering',197 franchise: 'Magic: The Gathering',198 brand: 'Wizards of the Coast',199 set: card.set_name,200 setCode: card.set.toUpperCase(),201 name: card.name,202 number: card.collector_number,203 year,204 variant,205 language: card.lang ?? null,206 rarity: card.rarity ?? null,207 identifiers: { ...identifiers, finish },208 metadata: {209 mana_cost: card.mana_cost ?? null,210 type_line: card.type_line ?? null,211 cmc: card.cmc ?? null,212 colors: card.colors ?? [],213 artist: card.artist ?? null,214 reserved: card.reserved ?? false,215 promo: card.promo ?? false,216 reprint: card.reprint ?? false,217 set_type: card.set_type ?? null,218 layout: card.layout ?? null,219 frame: card.frame ?? null,220 border_color: card.border_color ?? null,221 finish,222 treatments: treat,223 },224 });225 };226227 for (const finish of finishes) {228 const a = buildAttrs(finish);229 out.push(230 catalogItem({231 kind: 'catalog_item',232 connectorId: this.meta.id,233 sourceId: this.meta.sourceId,234 sourceUrl,235 externalId: `${card.id}:${finish}`,236 rawTitle: makeTitle({ name: card.name, set: card.set_name, number: card.collector_number, year, variant: a.variant }),237 description: card.type_line ?? null,238 imageUrls: images,239 attributes: a,240 observedAt,241 confidence: 0.98,242 parserVersion: this.parserVersion,243 releaseDate: card.released_at ?? null,244 }),245 );246 }247248 const prices = card.prices ?? {};249 const obsDate = observedAt;250 const priceMap: Array<[key: string, finish: string, currency: 'USD' | 'EUR']> = [251 ['usd', 'nonfoil', 'USD'],252 ['usd_foil', 'foil', 'USD'],253 ['usd_etched', 'etched', 'USD'],254 ['eur', 'nonfoil', 'EUR'],255 ['eur_foil', 'foil', 'EUR'],256 ];257 for (const [key, finish, currency] of priceMap) {258 const price = num(prices[key]);259 if (price === null) continue;260 if (!finishes.includes(finish)) continue;261 const a = buildAttrs(finish);262 out.push(263 priceObservation({264 kind: 'price_observation',265 connectorId: this.meta.id,266 sourceId: this.meta.sourceId,267 sourceUrl,268 externalId: `${card.id}:${key}`,269 rawTitle: makeTitle({ name: card.name, set: card.set_name, number: card.collector_number, year, variant: a.variant }),270 imageUrls: images,271 attributes: a,272 condition: { condition: 'near_mint', conditionRaw: 'NM (market price)', completeness: null },273 observedAt,274 confidence: 0.9,275 parserVersion: this.parserVersion,276 priceKind: 'market',277 price,278 currency,279 observationDate: obsDate,280 sampleSize: null,281 }),282 );283 }284 return out;285 }286}287288export default function createConnector(meta: ConnectorMeta) {289 return new ScryfallConnector(meta);290}291