spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, withRetries } from '../_lib/shared.js';5import { UA_HEADERS } from '../_lib/tcg-shared.js';67/** DigimonCard.io — full Digimon Card Game catalog in one public JSON call (no prices). */8const API = 'https://digimoncard.io/api-public';9const PARSER_VERSION = '1.0.0';1011const CardSchema = z12  .object({13    name: z.string(),14    type: z.string().nullable().optional(),15    id: z.string(),16    color: z.string().nullable().optional(),17    color2: z.string().nullable().optional(),18    rarity: z.string().nullable().optional(),19    stage: z.string().nullable().optional(),20    attribute: z.string().nullable().optional(),21    level: z.number().nullable().optional(),22    dp: z.number().nullable().optional(),23    artist: z.string().nullable().optional(),24    series: z.string().nullable().optional(),25    pretty_url: z.string().nullable().optional(),26    date_added: z.string().nullable().optional(),27    tcgplayer_name: z.string().nullable().optional(),28    tcgplayer_id: z.number().nullable().optional(),29    set_name: z.array(z.string()).nullable().optional(),30  })31  .loose();32export type DigimonCard = z.infer<typeof CardSchema>;3334export function trimCard(raw: Record<string, unknown>): DigimonCard {35  const keep = ['name', 'type', 'id', 'color', 'color2', 'rarity', 'stage', 'attribute', 'level', 'dp', 'artist', 'series', 'pretty_url', 'date_added', 'tcgplayer_name', 'tcgplayer_id', 'set_name'];36  const out: Record<string, unknown> = {};37  for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];38  return CardSchema.parse(out);39}4041/** "BT5-103" → { prefix: "BT5", code: "BT-05" }; "ST1-03" → { prefix "ST1", code "ST-1" }; "P-021" → { prefix "P", code "P" } */42export function setPrefix(id: string): { prefix: string; code: string } {43  const m = id.match(/^([A-Za-z]+)(\d*)-/);44  if (!m) return { prefix: id, code: id };45  const letters = m[1]!.toUpperCase();46  const digits = m[2] ?? '';47  const code = digits ? `${letters}-${letters === 'BT' || letters === 'EX' ? digits.padStart(2, '0') : digits}` : letters;48  return { prefix: `${letters}${digits}`, code };49}5051/** Pick the primary set name for a card from its set list using the number prefix. */52export function primarySet(id: string, sets: string[] | null | undefined): string | null {53  if (!sets?.length) return null;54  const { code, prefix } = setPrefix(id);55  const letters = prefix.replace(/\d+$/, '');56  const digits = prefix.slice(letters.length);57  const candidates = [code, `${letters}-${digits}`, `${letters}-${digits.padStart(2, '0')}`, `${letters}${digits}`];58  for (const s of sets) {59    const head = s.split(':')[0]!.trim().toUpperCase();60    if (candidates.includes(head)) return s;61  }62  return sets[0] ?? null;63}6465export class DigimonCardConnector extends BaseConnector {66  readonly version = '1.0.0';67  readonly parserVersion = PARSER_VERSION;68  protected override minIntervalMs = 1000;69  override readonly urlPatterns = [/digimoncard\.io\/card\/[a-z0-9-]+/i];7071  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {72    await this.throttle();73    const res = await withRetries(() => ctx.fetch(`${API}/search?series=${encodeURIComponent('Digimon Card Game')}`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 120_000 }), (r) => r.success && Array.isArray(r.json), 3, 3000);74    if (!res.success || !Array.isArray(res.json)) throw new Error(`digimoncard: catalog unavailable (${res.error ?? res.httpStatus})`);75    const seen = new Map<string, number>();76    let count = 0;77    for (const raw of res.json as Record<string, unknown>[]) {78      let card: DigimonCard;79      try {80        card = trimCard(raw);81      } catch (err) {82        ctx.anomaly('parse_failure_card', err instanceof Error ? err.message : String(err));83        continue;84      }85      if (ctx.options.seeds?.length && !ctx.options.seeds.includes(setPrefix(card.id).prefix)) continue;86      const n = (seen.get(card.id) ?? 0) + 1;87      seen.set(card.id, n);88      if (this.reached(ctx, count)) return;89      count++;90      yield { url: `https://digimoncard.io/card/${card.pretty_url ?? card.id.toLowerCase()}`, externalId: n > 1 ? `${card.id}#${n}` : card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, alt: n }, fetchedAt: res.fetchedAt };91    }92    await ctx.setCursor({ updatedAt: new Date().toISOString(), cards: count });93  }9495  async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {96    const m = url.match(/digimoncard\.io\/card\/([a-z0-9-]+)/i);97    if (!m) return [];98    const id = m[1]!.split('-').slice(-2).join('-').toUpperCase(); // pretty_url ends with the card number99    const res = await ctx.fetch(`${API}/search?card=${encodeURIComponent(id)}`, { engines: ['api'], headers: UA_HEADERS });100    if (!res.success || !Array.isArray(res.json) || !res.json.length) return [];101    const card = trimCard(res.json[0] as Record<string, unknown>);102    return [{ url, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, alt: 1 }, fetchedAt: res.fetchedAt }];103  }104105  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {106    const { card, alt } = z.object({ card: CardSchema, alt: z.number().default(1) }).parse(raw.payload);107    const set = primarySet(card.id, card.set_name);108    const { code } = setPrefix(card.id);109    const identifiers: Record<string, string> = { digimoncard_id: card.id };110    if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id);111    const variant = alt > 1 ? `Alternate Art ${alt - 1}` : null;112    const a = attrs({113      categorySlug: 'digimon_tcg',114      franchise: 'Digimon',115      brand: 'Bandai',116      set,117      setCode: code,118      name: card.name,119      number: card.id,120      year: null,121      variant,122      language: 'English',123      rarity: card.rarity ?? null,124      identifiers,125      metadata: { type: card.type, color: card.color, color2: card.color2, stage: card.stage, attribute: card.attribute, level: card.level, dp: card.dp, artist: card.artist, sets: card.set_name ?? [] },126    });127    const rawTitle = makeTitle({ name: card.name, set, number: card.id, variant });128    return [129      catalogItem({130        kind: 'catalog_item',131        connectorId: this.meta.id,132        sourceId: this.meta.sourceId,133        sourceUrl: raw.url,134        externalId: raw.externalId ?? card.id,135        rawTitle,136        imageUrls: [`https://images.digimoncard.io/images/cards/${card.id}.jpg`],137        attributes: a,138        observedAt: raw.fetchedAt,139        confidence: 0.85,140        parserVersion: PARSER_VERSION,141        releaseDate: null,142      }),143    ];144  }145}146147export default (meta: ConnectorMeta) => new DigimonCardConnector(meta);148