spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';56/**7 * Lorcast connector — Disney Lorcana catalog (all sets, promos) with TCGplayer market prices8 * (usd / usd_foil). Open API, no key: https://lorcast.com/docs/api9 */10const API = 'https://api.lorcast.com/v0';11const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };12const PARSER_VERSION = '1.0.0';1314const SetSchema = z.object({ id: z.string(), name: z.string(), code: z.string(), released_at: z.string().nullable().optional(), prereleased_at: z.string().nullable().optional() });15const CardSchema = z16  .object({17    id: z.string(),18    name: z.string(),19    version: z.string().nullable().optional(),20    layout: z.string().optional(),21    released_at: z.string().nullable().optional(),22    image_uris: z.record(z.string(), z.record(z.string(), z.string())).optional(),23    ink: z.string().nullable().optional(),24    type: z.array(z.string()).optional(),25    rarity: z.string().optional(),26    illustrators: z.array(z.string()).optional(),27    collector_number: z.string(),28    lang: z.string().optional(),29    set: z.object({ id: z.string(), code: z.string(), name: z.string() }),30    tcgplayer_id: z.number().nullable().optional(),31    prices: z.object({ usd: z.number().nullable().optional(), usd_foil: z.number().nullable().optional() }).partial().nullable().optional(),32  })33  .loose();34type LorCard = z.infer<typeof CardSchema>;3536export function trimCard(raw: Record<string, unknown>): LorCard {37  const keep = ['id', 'name', 'version', 'layout', 'released_at', 'image_uris', 'ink', 'type', 'rarity', 'illustrators', 'collector_number', 'lang', 'set', 'tcgplayer_id', 'prices'];38  const out: Record<string, unknown> = {};39  for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];40  return CardSchema.parse(out);41}4243const RawPayloadSchema = z.object({ card: CardSchema, set: SetSchema });4445export class LorcastConnector extends BaseConnector {46  readonly version = '1.0.0';47  readonly parserVersion = PARSER_VERSION;48  protected override minIntervalMs = 250;4950  private async get(ctx: CrawlContext, url: string) {51    await this.throttle();52    return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS }), (r) => r.success && r.json !== null, 4, 1500);53  }5455  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {56    const setsRes = await this.get(ctx, `${API}/sets`);57    const setsJson = (setsRes.json as { results?: unknown[] } | null)?.results;58    if (!setsRes.success || !Array.isArray(setsJson)) throw new Error(`lorcast: sets unavailable (${setsRes.error ?? setsRes.httpStatus})`);59    const sets = z.array(SetSchema.loose()).parse(setsJson).filter((s) => !ctx.options.seeds?.length || ctx.options.seeds.includes(s.code));60    let setIndex = Number(ctx.options.cursor?.setIndex ?? 0);61    let count = 0;62    for (; setIndex < sets.length; setIndex++) {63      const set = sets[setIndex]!;64      if (ctx.signal?.aborted) return;65      const seenIds = new Set<string>();66      for (let page = 1; page < 50; page++) {67        const res = await this.get(ctx, `${API}/cards/search?q=${encodeURIComponent(`set:${set.code}`)}&page=${page}`);68        const results = (res.json as { results?: unknown[] } | null)?.results;69        if (!res.success || !Array.isArray(results) || results.length === 0) break;70        let fresh = 0;71        for (const raw of results) {72          let card: LorCard;73          try {74            card = trimCard(raw as Record<string, unknown>);75          } catch (err) {76            ctx.anomaly('parse_failure_card', `${set.code}: ${err instanceof Error ? err.message : String(err)}`);77            continue;78          }79          if (seenIds.has(card.id)) continue;80          seenIds.add(card.id);81          fresh++;82          if (this.reached(ctx, count)) return;83          count++;84          yield { url: `https://lorcast.com/cards/${card.id}`, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, set }, fetchedAt: res.fetchedAt };85        }86        if (fresh === 0) break; // API returned the same page again → no pagination87      }88      await ctx.setCursor({ setIndex: setIndex + 1, updatedAt: new Date().toISOString() });89    }90    await ctx.setCursor({ setIndex: 0, updatedAt: new Date().toISOString() });91  }9293  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {94    const { card, set } = RawPayloadSchema.parse(raw.payload);95    const year = set.released_at ? Number(set.released_at.slice(0, 4)) || null : card.released_at ? Number(card.released_at.slice(0, 4)) || null : null;96    const identifiers: Record<string, string> = { lorcast_id: card.id };97    if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id);98    const digital = card.image_uris?.digital;99    const images = digital ? [digital.large ?? digital.normal ?? digital.small].filter((x): x is string => Boolean(x)) : [];100    const rarity = card.rarity ? card.rarity.replace(/_/g, ' ') : null;101    const name = card.version ? `${card.name} - ${card.version}` : card.name;102    const build = (variant: string | null) =>103      attrs({104        categorySlug: 'disney_lorcana',105        franchise: 'Disney Lorcana',106        brand: 'Ravensburger',107        set: set.name,108        setCode: set.code,109        name,110        number: card.collector_number,111        year,112        variant,113        language: card.lang === 'en' || !card.lang ? 'English' : card.lang,114        rarity,115        identifiers,116        metadata: { ink: card.ink, type: card.type, illustrators: card.illustrators, layout: card.layout, lorcast_set: set.id },117      });118    const out: NormalizedRecord[] = [];119    const observedAt = raw.fetchedAt;120    const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));121    const isEnchanted = /enchanted/i.test(card.rarity ?? '');122    const variantsToEmit: Array<[string | null, number | null]> = [123      [isEnchanted ? 'Enchanted' : null, num(card.prices?.usd)],124      [isEnchanted ? 'Enchanted' : 'Foil', num(card.prices?.usd_foil)],125    ];126    const seen = new Set<string>();127    for (const [variant, price] of variantsToEmit) {128      const key = variant ?? '';129      const a = build(variant);130      const rawTitle = makeTitle({ name, set: set.name, number: card.collector_number, year, variant });131      if (!seen.has(key)) {132        seen.add(key);133        out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.id}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: set.released_at ? new Date(set.released_at) : null }));134      }135      if (price) {136        // Lorcast relays TCGplayer market prices without a timestamp → dated by fetch day, moderate confidence.137        out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.id}:${variant ?? 'normal'}:market`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency: 'USD', observationDate: obsDate, sampleSize: null }));138      }139    }140    return out;141  }142}143144export default (meta: ConnectorMeta) => new LorcastConnector(meta);145