TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as htmlUtil, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';5import { dayOf, isoDay } from '../_lib/tcg-shared.js';67/**8 * MTGGoldfish — set pages embed a JSON card list with current paper/online prices; the public9 * price-history component returns a daily paper price series (2010 →) per printing.10 */11const BASE = 'https://www.mtggoldfish.com';12const PARSER_VERSION = '1.0.0';13const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'text/html,application/xhtml+xml' };1415const CardSchema = z.object({16 name: z.string(), // "Black Lotus [LEA]" — MTGGoldfish card id17 set: z.string(),18 display_name: z.string(),19 rarity: z.string().nullable().optional(),20 foil: z.boolean().optional(),21 card_num: z.union([z.number(), z.string()]).nullable().optional(),22 finish: z.string().nullable().optional(),23 card_uuid: z.string().nullable().optional(),24 paper: z.number().nullable(),25 online: z.number().nullable(),26 link: z.string().nullable(),27 image: z.string().nullable(),28});29type Card = z.infer<typeof CardSchema>;3031const SetPayloadSchema = z.object({ kind: z.literal('card'), setName: z.string().nullable(), setSlug: z.string(), card: CardSchema });32const HistoryPayloadSchema = z.object({ kind: z.literal('history'), setName: z.string().nullable(), card: CardSchema, series: z.array(z.tuple([z.string(), z.number()])) });3334/** Parse the escaped JSON card list embedded in a set page. */35export function parseSetPage(html: string): { setName: string | null; cards: Card[] } {36 const idx = html.indexOf('"cards":[');37 if (idx < 0) return { setName: titleOf(html), cards: [] };38 const start = html.lastIndexOf('="', idx) + 2;39 const end = html.indexOf('"', start);40 const raw = html.slice(start, end);41 const decoded = raw.replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');42 let json: { cards?: Array<Record<string, unknown>> };43 try {44 json = JSON.parse(decoded) as { cards?: Array<Record<string, unknown>> };45 } catch {46 return { setName: titleOf(html), cards: [] };47 }48 const cards: Card[] = [];49 for (const c of json.cards ?? []) {50 const prices = (c.prices ?? {}) as Record<string, { current_price?: string | null } | null>;51 const images = (c.source_image_variants ?? {}) as Record<string, string>;52 const links = (c.links ?? {}) as Record<string, string>;53 const parsed = CardSchema.safeParse({54 name: c.name,55 set: c.set,56 display_name: c.display_name,57 rarity: c.rarity,58 foil: c.foil,59 card_num: c.card_num,60 finish: c.finish,61 card_uuid: c.card_uuid,62 paper: num(prices.paper?.current_price),63 online: num(prices.online?.current_price),64 link: links.default ?? null,65 image: images['672x938'] ?? images['265x370'] ?? null,66 });67 if (parsed.success) cards.push(parsed.data);68 }69 return { setName: titleOf(html), cards };70}7172function titleOf(html: string): string | null {73 const m = html.match(/<title>([^<]+)<\/title>/);74 if (!m) return null;75 return m[1]!.replace(/\s*[|·-]\s*MTGGoldfish.*$/i, '').replace(/\s*Price(s| Guide).*$/i, '').replace(/\s*Set Information.*$/i, '').replace(/\s*\([A-Z0-9]{2,6}\)\s*$/, '').trim() || null;76}7778/** "d += '2010-11-02, 3102.99\n'" lines → [[date, price], …] */79export function parseHistory(js: string): Array<[string, number]> {80 const out: Array<[string, number]> = [];81 for (const m of js.matchAll(/(\d{4}-\d{2}-\d{2}),\s*([0-9.]+)/g)) {82 const price = Number(m[2]);83 if (Number.isFinite(price) && price > 0) out.push([m[1]!, price]);84 }85 return out;86}8788export class MtgGoldfishConnector extends BaseConnector {89 readonly version = '1.0.0';90 readonly parserVersion = PARSER_VERSION;91 protected override minIntervalMs = 1500;92 override readonly urlPatterns = [/mtggoldfish\.com\/price\/[^/]+\/\d+\//i, /mtggoldfish\.com\/sets\/[^/?#]+/i];9394 private async page(ctx: CrawlContext, url: string) {95 await this.throttle();96 return withRetries(() => ctx.fetch(url, { engines: ['api', 'firecrawl'], headers: HEADERS, responseType: 'text', minQuality: 0.3 }), (r) => r.success && Boolean(r.html || r.markdown), 3, 2500);97 }9899 private async discoverSets(ctx: CrawlContext): Promise<string[]> {100 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];101 if (seeds.length) return seeds;102 const res = await this.page(ctx, `${BASE}/sets`);103 const text = res.html ?? '';104 const slugs = [...new Set([...text.matchAll(/href="\/sets\/([^"#?]+)"/g)].map((m) => m[1]!))];105 return slugs;106 }107108 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {109 const sets = await this.discoverSets(ctx);110 const maxSets = ctx.options.mode === 'backfill' ? Infinity : Number(this.meta.config.maxSetsPerRun ?? 60);111 const hist = (this.meta.config.history ?? {}) as { enabled?: boolean; minPrice?: number; maxPerRun?: number; days?: number };112 let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);113 if (setIdx >= sets.length) setIdx = 0;114 let done = 0;115 let count = 0;116 let historyFetched = 0;117 for (; setIdx < sets.length && done < maxSets; setIdx++, done++) {118 const slug = sets[setIdx]!;119 const res = await this.page(ctx, `${BASE}/sets/${slug}`);120 if (!res.success || !res.html) {121 ctx.anomaly('page_fetch_failed', `${slug}: ${res.error ?? res.httpStatus}`);122 continue;123 }124 const { setName, cards } = parseSetPage(res.html);125 if (!cards.length) ctx.anomaly('empty_page', `set ${slug}`);126 for (const card of cards) {127 if (this.reached(ctx, count)) return;128 count++;129 const url = card.link ? `${BASE}${card.link}` : `${BASE}/sets/${slug}`;130 yield { url, externalId: card.card_uuid ?? card.name, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'card', setName, setSlug: slug, card }, fetchedAt: res.fetchedAt };131 if (hist.enabled !== false && card.paper !== null && card.paper >= Number(hist.minPrice ?? 100) && historyFetched < Number(hist.maxPerRun ?? 150) && !card.foil) {132 historyFetched++;133 const q = new URLSearchParams({ card_id: card.name, selector: '#tab-paper', type: 'paper', price_type: '' });134 // The component is an XHR endpoint: needs the XHR headers and must never fall back to Firecrawl (406 otherwise).135 await this.throttle();136 const h = await ctx.fetch(`${BASE}/price_history_component?${q.toString()}`, { engines: ['api'], headers: { ...HEADERS, accept: 'text/javascript, application/javascript, */*;q=0.1', 'x-requested-with': 'XMLHttpRequest' }, responseType: 'text', minQuality: 0.3 });137 if (h.success && h.html) {138 const cutoff = Date.now() - Number(hist.days ?? 1095) * 86_400_000;139 const series = parseHistory(h.html).filter(([d]) => new Date(`${d}T00:00:00Z`).getTime() >= cutoff);140 if (series.length) yield { url, externalId: `${card.card_uuid ?? card.name}:history`, kind: 'price_observation', engine: h.engine, httpStatus: h.httpStatus, payload: { kind: 'history', setName, card, series }, fetchedAt: h.fetchedAt };141 } else ctx.anomaly('page_fetch_failed', `history ${card.name}: ${h.error ?? h.httpStatus}`);142 }143 }144 await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() });145 }146 if (setIdx >= sets.length) await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() });147 }148149 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {150 const m = url.match(/mtggoldfish\.com\/price\/([^/]+)\/(\d+)\//i);151 if (!m) return [];152 // Card pages do not embed the JSON list; resolve through the set page.153 const res = await this.page(ctx, `${BASE}/sets/${m[1]}`);154 if (!res.success || !res.html) return [];155 const { setName, cards } = parseSetPage(res.html);156 const card = cards.find((c) => String(c.card_num) === m[2] && !c.foil) ?? cards.find((c) => String(c.card_num) === m[2]);157 if (!card) return [];158 return [{ url, externalId: card.card_uuid ?? card.name, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'card', setName, setSlug: m[1]!, card }, fetchedAt: res.fetchedAt }];159 }160161 private attrsFor(card: Card, setName: string | null) {162 const variant = card.foil || /foil/i.test(card.finish ?? '') ? (/etched/i.test(card.finish ?? '') ? 'Etched Foil' : 'Foil') : null;163 return {164 variant,165 a: attrs({166 categorySlug: 'magic_the_gathering',167 franchise: 'Magic: The Gathering',168 brand: 'Wizards of the Coast',169 set: setName ?? card.set,170 setCode: card.set.toUpperCase(),171 name: card.display_name,172 number: card.card_num !== null && card.card_num !== undefined ? String(card.card_num) : null,173 year: null,174 variant,175 language: 'English',176 rarity: card.rarity ?? null,177 identifiers: { mtggoldfish_card_id: card.name, ...(card.card_uuid ? { mtggoldfish_uuid: card.card_uuid } : {}) },178 metadata: { finish: card.finish ?? null, online_price: card.online },179 }),180 };181 }182183 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {184 const payload = raw.payload as { kind?: string };185 if (payload.kind === 'history') {186 const { card, setName, series } = HistoryPayloadSchema.parse(raw.payload);187 const { a, variant } = this.attrsFor(card, setName);188 const rawTitle = makeTitle({ name: card.display_name, set: setName ?? card.set, number: card.card_num !== null && card.card_num !== undefined ? String(card.card_num) : null, variant });189 const out: NormalizedRecord[] = [];190 const seen = new Set<string>();191 for (const [date, price] of series) {192 if (seen.has(date)) continue;193 seen.add(date);194 const day = isoDay(date);195 if (!day) continue;196 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.card_uuid ?? card.name}:paper:${date}`, rawTitle, imageUrls: card.image ? [card.image] : [], attributes: a, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency: 'USD', observationDate: day, sampleSize: null }));197 }198 return out;199 }200 const { card, setName } = SetPayloadSchema.parse(raw.payload);201 const { a, variant } = this.attrsFor(card, setName);202 const number = card.card_num !== null && card.card_num !== undefined ? String(card.card_num) : null;203 const rawTitle = makeTitle({ name: card.display_name, set: setName ?? card.set, number, variant });204 const images = card.image ? [card.image] : [];205 const out: NormalizedRecord[] = [206 catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: card.card_uuid ?? card.name, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null }),207 ];208 if (card.paper) {209 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.card_uuid ?? card.name}:paper:current`, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.75, parserVersion: PARSER_VERSION, priceKind: 'market', price: card.paper, currency: 'USD', observationDate: dayOf(raw.fetchedAt), sampleSize: null }));210 }211 return out;212 }213}214215export default (meta: ConnectorMeta) => new MtgGoldfishConnector(meta);216export { htmlUtil as _html };217