TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, parseSlashDate, priceObservation, withRetries } from '../_lib/shared.js';56/**7 * OPTCG API connector — One Piece Card Game (English) catalog per set with TCGplayer-derived8 * market / inventory prices and the date they were scraped. Open API, no key (https://optcgapi.com).9 */10const API = 'https://optcgapi.com/api';11const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };12const PARSER_VERSION = '1.0.0';1314const SetSchema = z.object({ set_name: z.string(), set_id: z.string() });15const CardSchema = z16 .object({17 card_set_id: z.string(),18 card_name: z.string(),19 set_name: z.string(),20 set_id: z.string(),21 rarity: z.string().nullable().optional(),22 card_color: z.string().nullable().optional(),23 card_type: z.string().nullable().optional(),24 card_cost: z.union([z.string(), z.number()]).nullable().optional(),25 card_power: z.union([z.string(), z.number()]).nullable().optional(),26 sub_types: z.string().nullable().optional(),27 attribute: z.string().nullable().optional(),28 card_image: z.string().nullable().optional(),29 market_price: z.number().nullable().optional(),30 inventory_price: z.number().nullable().optional(),31 date_scraped: z.string().nullable().optional(),32 })33 .loose();34type OpCard = z.infer<typeof CardSchema>;3536export function trimCard(raw: Record<string, unknown>): OpCard {37 const keep = ['card_set_id', 'card_name', 'set_name', 'set_id', 'rarity', 'card_color', 'card_type', 'card_cost', 'card_power', 'sub_types', 'attribute', 'card_image', 'market_price', 'inventory_price', 'date_scraped'];38 const out: Record<string, unknown> = {};39 for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];40 return CardSchema.parse(out);41}4243const RawPayloadSchema = z.object({ card: CardSchema, set: SetSchema });4445const RARITY: Record<string, string> = { C: 'Common', UC: 'Uncommon', R: 'Rare', SR: 'Super Rare', SEC: 'Secret Rare', L: 'Leader', SP: 'Special', P: 'Promo', TR: 'Treasure Rare' };4647export class OptcgConnector extends BaseConnector {48 readonly version = '1.0.0';49 readonly parserVersion = PARSER_VERSION;50 protected override minIntervalMs = 500;5152 private async get(ctx: CrawlContext, url: string) {53 await this.throttle();54 return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS }), (r) => r.success && r.json !== null, 4, 1500);55 }5657 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {58 const setsRes = await this.get(ctx, `${API}/allSets/`);59 if (!setsRes.success || !Array.isArray(setsRes.json)) throw new Error(`optcg: sets unavailable (${setsRes.error ?? setsRes.httpStatus})`);60 const sets = z.array(SetSchema.loose()).parse(setsRes.json).filter((s) => !ctx.options.seeds?.length || ctx.options.seeds.includes(s.set_id));61 let setIndex = Number(ctx.options.cursor?.setIndex ?? 0);62 let count = 0;63 for (; setIndex < sets.length; setIndex++) {64 const set = sets[setIndex]!;65 if (ctx.signal?.aborted) return;66 const res = await this.get(ctx, `${API}/sets/${encodeURIComponent(set.set_id)}/`);67 if (!res.success || !Array.isArray(res.json)) {68 ctx.anomaly('set_unavailable', `${set.set_id}: ${res.error ?? res.httpStatus}`);69 continue;70 }71 const seen = new Set<string>();72 for (const raw of res.json as unknown[]) {73 let card: OpCard;74 try {75 card = trimCard(raw as Record<string, unknown>);76 } catch (err) {77 ctx.anomaly('parse_failure_card', `${set.set_id}: ${err instanceof Error ? err.message : String(err)}`);78 continue;79 }80 // The API repeats alternate arts with the same card_set_id; keep them distinct by image id.81 const imageId = card.card_image?.match(/\/([^/]+)\.(?:jpg|png|webp)$/i)?.[1] ?? null;82 const key = imageId && imageId !== card.card_set_id ? `${card.card_set_id}:${imageId}` : card.card_set_id;83 if (seen.has(key)) continue;84 seen.add(key);85 if (this.reached(ctx, count)) return;86 count++;87 yield { url: `https://optcgapi.com/cards/${encodeURIComponent(card.card_set_id)}/`, externalId: key, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, set }, fetchedAt: res.fetchedAt };88 }89 await ctx.setCursor({ setIndex: setIndex + 1, updatedAt: new Date().toISOString() });90 }91 await ctx.setCursor({ setIndex: 0, updatedAt: new Date().toISOString() });92 }9394 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {95 const { card, set } = RawPayloadSchema.parse(raw.payload);96 const imageId = card.card_image?.match(/\/([^/]+)\.(?:jpg|png|webp)$/i)?.[1] ?? null;97 const isAlt = imageId ? imageId !== card.card_set_id : false;98 const altSuffix = imageId && isAlt ? imageId.replace(card.card_set_id, '').replace(/^[_-]/, '') : null;99 const variant = isAlt ? (/p\d*$/i.test(altSuffix ?? '') || /_p/i.test(altSuffix ?? '') ? `Alternate Art${altSuffix ? ` ${altSuffix.toUpperCase()}` : ''}` : `Alternate Art ${altSuffix}`) : null;100 const identifiers: Record<string, string> = { optcg_id: card.card_set_id };101 if (imageId) identifiers.optcg_image_id = imageId;102 const a = attrs({103 categorySlug: 'one_piece_card_game',104 franchise: 'One Piece',105 brand: 'Bandai',106 set: card.set_name || set.set_name,107 setCode: card.set_id || set.set_id,108 name: card.card_name,109 number: card.card_set_id,110 year: null,111 variant,112 language: 'English',113 rarity: card.rarity ? (RARITY[card.rarity] ?? card.rarity) : null,114 identifiers,115 metadata: { color: card.card_color, type: card.card_type, cost: card.card_cost, power: card.card_power, subTypes: card.sub_types, attribute: card.attribute, rarityCode: card.rarity },116 });117 const rawTitle = makeTitle({ name: card.card_name, set: a.set, number: card.card_set_id, variant });118 const images = card.card_image ? [card.card_image] : [];119 const out: NormalizedRecord[] = [];120 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: raw.externalId ?? card.card_set_id, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null }));121 const scraped = parseSlashDate(card.date_scraped) ?? new Date(Date.UTC(raw.fetchedAt.getUTCFullYear(), raw.fetchedAt.getUTCMonth(), raw.fetchedAt.getUTCDate()));122 const market = num(card.market_price);123 if (market) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${raw.externalId ?? card.card_set_id}:market`, rawTitle, imageUrls: images, attributes: a, observedAt: scraped, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: market, currency: 'USD', observationDate: scraped, sampleSize: null }));124 const low = num(card.inventory_price);125 if (low) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${raw.externalId ?? card.card_set_id}:low`, rawTitle, imageUrls: images, attributes: a, observedAt: scraped, confidence: 0.65, parserVersion: PARSER_VERSION, priceKind: 'low', price: low, currency: 'USD', observationDate: scraped, sampleSize: null }));126 return out;127 }128}129130export default (meta: ConnectorMeta) => new OptcgConnector(meta);131