import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, num, parseSlashDate, priceObservation, withRetries } from '../_lib/shared.js'; /** * OPTCG API connector — One Piece Card Game (English) catalog per set with TCGplayer-derived * market / inventory prices and the date they were scraped. Open API, no key (https://optcgapi.com). */ const API = 'https://optcgapi.com/api'; const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' }; const PARSER_VERSION = '1.0.0'; const SetSchema = z.object({ set_name: z.string(), set_id: z.string() }); const CardSchema = z .object({ card_set_id: z.string(), card_name: z.string(), set_name: z.string(), set_id: z.string(), rarity: z.string().nullable().optional(), card_color: z.string().nullable().optional(), card_type: z.string().nullable().optional(), card_cost: z.union([z.string(), z.number()]).nullable().optional(), card_power: z.union([z.string(), z.number()]).nullable().optional(), sub_types: z.string().nullable().optional(), attribute: z.string().nullable().optional(), card_image: z.string().nullable().optional(), market_price: z.number().nullable().optional(), inventory_price: z.number().nullable().optional(), date_scraped: z.string().nullable().optional(), }) .loose(); type OpCard = z.infer; export function trimCard(raw: Record): OpCard { const keep = ['card_set_id', 'card_name', 'set_name', 'set_id', 'rarity', 'card_color', 'card_type', 'card_cost', 'card_power', 'sub_types', 'attribute', 'card_image', 'market_price', 'inventory_price', 'date_scraped']; const out: Record = {}; for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k]; return CardSchema.parse(out); } const RawPayloadSchema = z.object({ card: CardSchema, set: SetSchema }); const RARITY: Record = { C: 'Common', UC: 'Uncommon', R: 'Rare', SR: 'Super Rare', SEC: 'Secret Rare', L: 'Leader', SP: 'Special', P: 'Promo', TR: 'Treasure Rare' }; export class OptcgConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 500; private async get(ctx: CrawlContext, url: string) { await this.throttle(); return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS }), (r) => r.success && r.json !== null, 4, 1500); } async *crawl(ctx: CrawlContext): AsyncIterable { const setsRes = await this.get(ctx, `${API}/allSets/`); if (!setsRes.success || !Array.isArray(setsRes.json)) throw new Error(`optcg: sets unavailable (${setsRes.error ?? setsRes.httpStatus})`); const sets = z.array(SetSchema.loose()).parse(setsRes.json).filter((s) => !ctx.options.seeds?.length || ctx.options.seeds.includes(s.set_id)); let setIndex = Number(ctx.options.cursor?.setIndex ?? 0); let count = 0; for (; setIndex < sets.length; setIndex++) { const set = sets[setIndex]!; if (ctx.signal?.aborted) return; const res = await this.get(ctx, `${API}/sets/${encodeURIComponent(set.set_id)}/`); if (!res.success || !Array.isArray(res.json)) { ctx.anomaly('set_unavailable', `${set.set_id}: ${res.error ?? res.httpStatus}`); continue; } const seen = new Set(); for (const raw of res.json as unknown[]) { let card: OpCard; try { card = trimCard(raw as Record); } catch (err) { ctx.anomaly('parse_failure_card', `${set.set_id}: ${err instanceof Error ? err.message : String(err)}`); continue; } // The API repeats alternate arts with the same card_set_id; keep them distinct by image id. const imageId = card.card_image?.match(/\/([^/]+)\.(?:jpg|png|webp)$/i)?.[1] ?? null; const key = imageId && imageId !== card.card_set_id ? `${card.card_set_id}:${imageId}` : card.card_set_id; if (seen.has(key)) continue; seen.add(key); if (this.reached(ctx, count)) return; count++; yield { url: `https://optcgapi.com/cards/${encodeURIComponent(card.card_set_id)}/`, externalId: key, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, set }, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ setIndex: setIndex + 1, updatedAt: new Date().toISOString() }); } await ctx.setCursor({ setIndex: 0, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const { card, set } = RawPayloadSchema.parse(raw.payload); const imageId = card.card_image?.match(/\/([^/]+)\.(?:jpg|png|webp)$/i)?.[1] ?? null; const isAlt = imageId ? imageId !== card.card_set_id : false; const altSuffix = imageId && isAlt ? imageId.replace(card.card_set_id, '').replace(/^[_-]/, '') : null; const variant = isAlt ? (/p\d*$/i.test(altSuffix ?? '') || /_p/i.test(altSuffix ?? '') ? `Alternate Art${altSuffix ? ` ${altSuffix.toUpperCase()}` : ''}` : `Alternate Art ${altSuffix}`) : null; const identifiers: Record = { optcg_id: card.card_set_id }; if (imageId) identifiers.optcg_image_id = imageId; const a = attrs({ categorySlug: 'one_piece_card_game', franchise: 'One Piece', brand: 'Bandai', set: card.set_name || set.set_name, setCode: card.set_id || set.set_id, name: card.card_name, number: card.card_set_id, year: null, variant, language: 'English', rarity: card.rarity ? (RARITY[card.rarity] ?? card.rarity) : null, identifiers, metadata: { color: card.card_color, type: card.card_type, cost: card.card_cost, power: card.card_power, subTypes: card.sub_types, attribute: card.attribute, rarityCode: card.rarity }, }); const rawTitle = makeTitle({ name: card.card_name, set: a.set, number: card.card_set_id, variant }); const images = card.card_image ? [card.card_image] : []; const out: NormalizedRecord[] = []; out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: raw.externalId ?? card.card_set_id, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null })); const scraped = parseSlashDate(card.date_scraped) ?? new Date(Date.UTC(raw.fetchedAt.getUTCFullYear(), raw.fetchedAt.getUTCMonth(), raw.fetchedAt.getUTCDate())); const market = num(card.market_price); if (market) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${raw.externalId ?? card.card_set_id}:market`, rawTitle, imageUrls: images, attributes: a, observedAt: scraped, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: market, currency: 'USD', observationDate: scraped, sampleSize: null })); const low = num(card.inventory_price); if (low) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${raw.externalId ?? card.card_set_id}:low`, rawTitle, imageUrls: images, attributes: a, observedAt: scraped, confidence: 0.65, parserVersion: PARSER_VERSION, priceKind: 'low', price: low, currency: 'USD', observationDate: scraped, sampleSize: null })); return out; } } export default (meta: ConnectorMeta) => new OptcgConnector(meta);