import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, withRetries } from '../_lib/shared.js'; import { UA_HEADERS } from '../_lib/tcg-shared.js'; /** DigimonCard.io — full Digimon Card Game catalog in one public JSON call (no prices). */ const API = 'https://digimoncard.io/api-public'; const PARSER_VERSION = '1.0.0'; const CardSchema = z .object({ name: z.string(), type: z.string().nullable().optional(), id: z.string(), color: z.string().nullable().optional(), color2: z.string().nullable().optional(), rarity: z.string().nullable().optional(), stage: z.string().nullable().optional(), attribute: z.string().nullable().optional(), level: z.number().nullable().optional(), dp: z.number().nullable().optional(), artist: z.string().nullable().optional(), series: z.string().nullable().optional(), pretty_url: z.string().nullable().optional(), date_added: z.string().nullable().optional(), tcgplayer_name: z.string().nullable().optional(), tcgplayer_id: z.number().nullable().optional(), set_name: z.array(z.string()).nullable().optional(), }) .loose(); export type DigimonCard = z.infer; export function trimCard(raw: Record): DigimonCard { const keep = ['name', 'type', 'id', 'color', 'color2', 'rarity', 'stage', 'attribute', 'level', 'dp', 'artist', 'series', 'pretty_url', 'date_added', 'tcgplayer_name', 'tcgplayer_id', 'set_name']; const out: Record = {}; for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k]; return CardSchema.parse(out); } /** "BT5-103" → { prefix: "BT5", code: "BT-05" }; "ST1-03" → { prefix "ST1", code "ST-1" }; "P-021" → { prefix "P", code "P" } */ export function setPrefix(id: string): { prefix: string; code: string } { const m = id.match(/^([A-Za-z]+)(\d*)-/); if (!m) return { prefix: id, code: id }; const letters = m[1]!.toUpperCase(); const digits = m[2] ?? ''; const code = digits ? `${letters}-${letters === 'BT' || letters === 'EX' ? digits.padStart(2, '0') : digits}` : letters; return { prefix: `${letters}${digits}`, code }; } /** Pick the primary set name for a card from its set list using the number prefix. */ export function primarySet(id: string, sets: string[] | null | undefined): string | null { if (!sets?.length) return null; const { code, prefix } = setPrefix(id); const letters = prefix.replace(/\d+$/, ''); const digits = prefix.slice(letters.length); const candidates = [code, `${letters}-${digits}`, `${letters}-${digits.padStart(2, '0')}`, `${letters}${digits}`]; for (const s of sets) { const head = s.split(':')[0]!.trim().toUpperCase(); if (candidates.includes(head)) return s; } return sets[0] ?? null; } export class DigimonCardConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1000; override readonly urlPatterns = [/digimoncard\.io\/card\/[a-z0-9-]+/i]; async *crawl(ctx: CrawlContext): AsyncIterable { await this.throttle(); const res = await withRetries(() => ctx.fetch(`${API}/search?series=${encodeURIComponent('Digimon Card Game')}`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 120_000 }), (r) => r.success && Array.isArray(r.json), 3, 3000); if (!res.success || !Array.isArray(res.json)) throw new Error(`digimoncard: catalog unavailable (${res.error ?? res.httpStatus})`); const seen = new Map(); let count = 0; for (const raw of res.json as Record[]) { let card: DigimonCard; try { card = trimCard(raw); } catch (err) { ctx.anomaly('parse_failure_card', err instanceof Error ? err.message : String(err)); continue; } if (ctx.options.seeds?.length && !ctx.options.seeds.includes(setPrefix(card.id).prefix)) continue; const n = (seen.get(card.id) ?? 0) + 1; seen.set(card.id, n); if (this.reached(ctx, count)) return; count++; yield { url: `https://digimoncard.io/card/${card.pretty_url ?? card.id.toLowerCase()}`, externalId: n > 1 ? `${card.id}#${n}` : card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, alt: n }, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ updatedAt: new Date().toISOString(), cards: count }); } async lookup(url: string, ctx: CrawlContext): Promise { const m = url.match(/digimoncard\.io\/card\/([a-z0-9-]+)/i); if (!m) return []; const id = m[1]!.split('-').slice(-2).join('-').toUpperCase(); // pretty_url ends with the card number const res = await ctx.fetch(`${API}/search?card=${encodeURIComponent(id)}`, { engines: ['api'], headers: UA_HEADERS }); if (!res.success || !Array.isArray(res.json) || !res.json.length) return []; const card = trimCard(res.json[0] as Record); return [{ url, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, alt: 1 }, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const { card, alt } = z.object({ card: CardSchema, alt: z.number().default(1) }).parse(raw.payload); const set = primarySet(card.id, card.set_name); const { code } = setPrefix(card.id); const identifiers: Record = { digimoncard_id: card.id }; if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id); const variant = alt > 1 ? `Alternate Art ${alt - 1}` : null; const a = attrs({ categorySlug: 'digimon_tcg', franchise: 'Digimon', brand: 'Bandai', set, setCode: code, name: card.name, number: card.id, year: null, variant, language: 'English', rarity: card.rarity ?? null, identifiers, metadata: { type: card.type, color: card.color, color2: card.color2, stage: card.stage, attribute: card.attribute, level: card.level, dp: card.dp, artist: card.artist, sets: card.set_name ?? [] }, }); const rawTitle = makeTitle({ name: card.name, set, number: card.id, variant }); return [ catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: raw.externalId ?? card.id, rawTitle, imageUrls: [`https://images.digimoncard.io/images/cards/${card.id}.jpg`], attributes: a, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null, }), ]; } } export default (meta: ConnectorMeta) => new DigimonCardConnector(meta);