import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js'; /** * Lorcast connector — Disney Lorcana catalog (all sets, promos) with TCGplayer market prices * (usd / usd_foil). Open API, no key: https://lorcast.com/docs/api */ const API = 'https://api.lorcast.com/v0'; const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' }; const PARSER_VERSION = '1.0.0'; const SetSchema = z.object({ id: z.string(), name: z.string(), code: z.string(), released_at: z.string().nullable().optional(), prereleased_at: z.string().nullable().optional() }); const CardSchema = z .object({ id: z.string(), name: z.string(), version: z.string().nullable().optional(), layout: z.string().optional(), released_at: z.string().nullable().optional(), image_uris: z.record(z.string(), z.record(z.string(), z.string())).optional(), ink: z.string().nullable().optional(), type: z.array(z.string()).optional(), rarity: z.string().optional(), illustrators: z.array(z.string()).optional(), collector_number: z.string(), lang: z.string().optional(), set: z.object({ id: z.string(), code: z.string(), name: z.string() }), tcgplayer_id: z.number().nullable().optional(), prices: z.object({ usd: z.number().nullable().optional(), usd_foil: z.number().nullable().optional() }).partial().nullable().optional(), }) .loose(); type LorCard = z.infer; export function trimCard(raw: Record): LorCard { const keep = ['id', 'name', 'version', 'layout', 'released_at', 'image_uris', 'ink', 'type', 'rarity', 'illustrators', 'collector_number', 'lang', 'set', 'tcgplayer_id', 'prices']; const out: Record = {}; for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k]; return CardSchema.parse(out); } const RawPayloadSchema = z.object({ card: CardSchema, set: SetSchema }); export class LorcastConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 250; private async get(ctx: CrawlContext, url: string) { await this.throttle(); return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS }), (r) => r.success && r.json !== null, 4, 1500); } async *crawl(ctx: CrawlContext): AsyncIterable { const setsRes = await this.get(ctx, `${API}/sets`); const setsJson = (setsRes.json as { results?: unknown[] } | null)?.results; if (!setsRes.success || !Array.isArray(setsJson)) throw new Error(`lorcast: sets unavailable (${setsRes.error ?? setsRes.httpStatus})`); const sets = z.array(SetSchema.loose()).parse(setsJson).filter((s) => !ctx.options.seeds?.length || ctx.options.seeds.includes(s.code)); let setIndex = Number(ctx.options.cursor?.setIndex ?? 0); let count = 0; for (; setIndex < sets.length; setIndex++) { const set = sets[setIndex]!; if (ctx.signal?.aborted) return; const seenIds = new Set(); for (let page = 1; page < 50; page++) { const res = await this.get(ctx, `${API}/cards/search?q=${encodeURIComponent(`set:${set.code}`)}&page=${page}`); const results = (res.json as { results?: unknown[] } | null)?.results; if (!res.success || !Array.isArray(results) || results.length === 0) break; let fresh = 0; for (const raw of results) { let card: LorCard; try { card = trimCard(raw as Record); } catch (err) { ctx.anomaly('parse_failure_card', `${set.code}: ${err instanceof Error ? err.message : String(err)}`); continue; } if (seenIds.has(card.id)) continue; seenIds.add(card.id); fresh++; if (this.reached(ctx, count)) return; count++; yield { url: `https://lorcast.com/cards/${card.id}`, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, set }, fetchedAt: res.fetchedAt }; } if (fresh === 0) break; // API returned the same page again → no pagination } await ctx.setCursor({ setIndex: setIndex + 1, updatedAt: new Date().toISOString() }); } await ctx.setCursor({ setIndex: 0, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const { card, set } = RawPayloadSchema.parse(raw.payload); const year = set.released_at ? Number(set.released_at.slice(0, 4)) || null : card.released_at ? Number(card.released_at.slice(0, 4)) || null : null; const identifiers: Record = { lorcast_id: card.id }; if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id); const digital = card.image_uris?.digital; const images = digital ? [digital.large ?? digital.normal ?? digital.small].filter((x): x is string => Boolean(x)) : []; const rarity = card.rarity ? card.rarity.replace(/_/g, ' ') : null; const name = card.version ? `${card.name} - ${card.version}` : card.name; const build = (variant: string | null) => attrs({ categorySlug: 'disney_lorcana', franchise: 'Disney Lorcana', brand: 'Ravensburger', set: set.name, setCode: set.code, name, number: card.collector_number, year, variant, language: card.lang === 'en' || !card.lang ? 'English' : card.lang, rarity, identifiers, metadata: { ink: card.ink, type: card.type, illustrators: card.illustrators, layout: card.layout, lorcast_set: set.id }, }); const out: NormalizedRecord[] = []; const observedAt = raw.fetchedAt; const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate())); const isEnchanted = /enchanted/i.test(card.rarity ?? ''); const variantsToEmit: Array<[string | null, number | null]> = [ [isEnchanted ? 'Enchanted' : null, num(card.prices?.usd)], [isEnchanted ? 'Enchanted' : 'Foil', num(card.prices?.usd_foil)], ]; const seen = new Set(); for (const [variant, price] of variantsToEmit) { const key = variant ?? ''; const a = build(variant); const rawTitle = makeTitle({ name, set: set.name, number: card.collector_number, year, variant }); if (!seen.has(key)) { seen.add(key); out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.id}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: set.released_at ? new Date(set.released_at) : null })); } if (price) { // Lorcast relays TCGplayer market prices without a timestamp → dated by fetch day, moderate confidence. out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.id}:${variant ?? 'normal'}:market`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency: 'USD', observationDate: obsDate, sampleSize: null })); } } return out; } } export default (meta: ConnectorMeta) => new LorcastConnector(meta);