spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, withRetries } from '../_lib/shared.js';5import { UA_HEADERS } from '../_lib/tcg-shared.js';67/** Grand Archive official index — editions with publisher-declared circulations (print runs). */8const API = 'https://api.gatcg.com';9const PARSER_VERSION = '1.0.0';1011const CirculationSchema = z.object({ uuid: z.string().optional(), kind: z.string().nullable().optional(), foil: z.boolean().optional(), population: z.number().nullable().optional(), population_operator: z.string().nullable().optional(), printing: z.boolean().optional() });12const EditionSchema = z13  .object({14    uuid: z.string(),15    slug: z.string().nullable().optional(),16    collector_number: z.string().nullable().optional(),17    rarity: z.number().nullable().optional(),18    illustrator: z.string().nullable().optional(),19    image: z.string().nullable().optional(),20    configuration: z.string().nullable().optional(),21    set: z.object({ id: z.string().optional(), name: z.string(), prefix: z.string().nullable().optional(), release_date: z.string().nullable().optional(), language: z.string().nullable().optional() }),22    circulations: z.array(CirculationSchema).default([]),23  })24  .loose();25const CardSchema = z26  .object({27    uuid: z.string(),28    name: z.string(),29    slug: z.string().nullable().optional(),30    element: z.string().nullable().optional(),31    types: z.array(z.string()).optional(),32    classes: z.array(z.string()).optional(),33    editions: z.array(EditionSchema).default([]),34  })35  .loose();36export type GaCard = z.infer<typeof CardSchema>;3738const RARITY: Record<number, string> = { 1: 'Common', 2: 'Uncommon', 3: 'Rare', 4: 'Super Rare', 5: 'Ultra Rare', 6: 'Promotional', 7: 'Collector Super Rare', 8: 'Collector Ultra Rare', 9: 'Collector Promo' };3940export function trimCard(raw: Record<string, unknown>): GaCard {41  const editions = Array.isArray(raw.editions) ? (raw.editions as Record<string, unknown>[]).map((e) => ({42    uuid: e.uuid, slug: e.slug, collector_number: e.collector_number, rarity: e.rarity, illustrator: e.illustrator, image: e.image, configuration: e.configuration,43    set: e.set && typeof e.set === 'object' ? { id: (e.set as Record<string, unknown>).id, name: (e.set as Record<string, unknown>).name, prefix: (e.set as Record<string, unknown>).prefix, release_date: (e.set as Record<string, unknown>).release_date, language: (e.set as Record<string, unknown>).language } : undefined,44    circulations: Array.isArray(e.circulations) ? (e.circulations as Record<string, unknown>[]).map((c) => ({ uuid: c.uuid, kind: c.kind, foil: c.foil, population: c.population, population_operator: c.population_operator, printing: c.printing })) : [],45  })) : [];46  return CardSchema.parse({ uuid: raw.uuid, name: raw.name, slug: raw.slug, element: raw.element, types: raw.types, classes: raw.classes, editions });47}4849export class GrandArchiveConnector extends BaseConnector {50  readonly version = '1.0.0';51  readonly parserVersion = PARSER_VERSION;52  protected override minIntervalMs = 600;5354  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {55    const pageSize = Number(this.meta.config.pageSize ?? 50);56    let page = Number(ctx.options.cursor?.page ?? 1);57    let count = 0;58    for (; page < 400; page++) {59      if (ctx.signal?.aborted) return;60      await this.throttle();61      const res = await withRetries(() => ctx.fetch(`${API}/cards/search?page=${page}&page_size=${pageSize}`, { engines: ['api'], headers: UA_HEADERS }), (r) => r.success && r.json !== null, 3, 2000);62      const body = res.json as { data?: unknown[]; has_more?: boolean } | null;63      if (!res.success || !Array.isArray(body?.data)) {64        ctx.anomaly('page_fetch_failed', `page ${page}: ${res.error ?? res.httpStatus}`);65        break;66      }67      for (const raw of body.data) {68        let card: GaCard;69        try {70          card = trimCard(raw as Record<string, unknown>);71        } catch (err) {72          ctx.anomaly('parse_failure_card', `page ${page}: ${err instanceof Error ? err.message : String(err)}`);73          continue;74        }75        if (this.reached(ctx, count)) return;76        count++;77        yield { url: `https://index.gatcg.com/card/${card.slug ?? card.uuid}`, externalId: card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card }, fetchedAt: res.fetchedAt };78      }79      await ctx.setCursor({ page: page + 1, updatedAt: new Date().toISOString() });80      if (!body.has_more) break;81    }82    await ctx.setCursor({ page: 1, updatedAt: new Date().toISOString() });83  }8485  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {86    const { card } = z.object({ card: CardSchema }).parse(raw.payload);87    const out: NormalizedRecord[] = [];88    for (const ed of card.editions) {89      const y = ed.set.release_date ? Number(ed.set.release_date.slice(0, 4)) : NaN;90      const year = Number.isFinite(y) && y >= 1990 ? y : null; // epoch placeholders (1970) are not release years91      const images = ed.image ? [`${API}${ed.image.startsWith('/') ? '' : '/'}${ed.image}`] : [];92      const circulations = ed.circulations.length ? ed.circulations : [{ kind: 'NONFOIL', foil: false, population: null, population_operator: null }];93      const seen = new Set<string>();94      for (const c of circulations) {95        const variant = c.foil || c.kind === 'FOIL' ? 'Foil' : null;96        const key = variant ?? '';97        if (seen.has(key)) continue;98        seen.add(key);99        const exact = c.population_operator === '=' || c.population_operator === 'EXACT';100        const a = attrs({101          categorySlug: 'other_tcg',102          franchise: 'Grand Archive',103          brand: 'Weebs of the Shore',104          set: ed.set.name,105          setCode: ed.set.prefix ?? null,106          name: card.name,107          number: ed.collector_number ?? null,108          year,109          variant,110          language: ed.set.language === 'EN' || !ed.set.language ? 'English' : ed.set.language,111          rarity: ed.rarity ? (RARITY[ed.rarity] ?? String(ed.rarity)) : null,112          productionQuantity: exact && c.population ? c.population : null,113          identifiers: { gatcg_edition_id: ed.uuid, gatcg_card_id: card.uuid, ...(ed.slug ? { gatcg_slug: ed.slug } : {}) },114          metadata: { element: card.element, types: card.types ?? [], classes: card.classes ?? [], illustrator: ed.illustrator, configuration: ed.configuration, population: c.population ?? null, population_operator: c.population_operator ?? null, population_exact: exact },115        });116        out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${ed.uuid}${variant ? `:${variant}` : ''}`, rawTitle: makeTitle({ name: card.name, set: ed.set.name, number: ed.collector_number ?? null, year, variant }), imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: year && ed.set.release_date ? new Date(ed.set.release_date) : null }));117      }118    }119    return out;120  }121}122123export default (meta: ConnectorMeta) => new GrandArchiveConnector(meta);124