TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';5import { UA_HEADERS, dayOf, isoDay, splitNumber, tcgplayerImage, variantFromSubType } from '../_lib/tcg-shared.js';67/**8 * TCGCSV connector — public daily mirror of TCGplayer's catalog + prices for ~90 card games.9 * One raw record per TCGplayer product (with its price rows); normalize emits one catalog_item per10 * printing subtype and price_observations (market/low/…) dated with the mirror's last-updated stamp.11 */12const API = 'https://tcgcsv.com/tcgplayer';13const PARSER_VERSION = '1.0.0';1415const GroupSchema = z.object({ groupId: z.number(), name: z.string(), abbreviation: z.string().nullable().optional(), isSupplemental: z.boolean().optional(), publishedOn: z.string().nullable().optional(), categoryId: z.number().optional() });16const ExtSchema = z.object({ name: z.string(), displayName: z.string().optional(), value: z.string() });17const ProductSchema = z.object({18 productId: z.number(),19 name: z.string(),20 cleanName: z.string().optional(),21 imageUrl: z.string().nullable().optional(),22 url: z.string().optional(),23 modifiedOn: z.string().optional(),24 extendedData: z.array(ExtSchema).default([]),25});26const PriceSchema = z.object({27 productId: z.number(),28 lowPrice: z.number().nullable().optional(),29 midPrice: z.number().nullable().optional(),30 highPrice: z.number().nullable().optional(),31 marketPrice: z.number().nullable().optional(),32 directLowPrice: z.number().nullable().optional(),33 subTypeName: z.string().nullable().optional(),34});35const CategoryCfgSchema = z.object({ slug: z.string(), franchise: z.string().nullable(), brand: z.string().nullable(), language: z.string().optional() });36const RawPayloadSchema = z.object({37 categoryId: z.number(),38 category: CategoryCfgSchema,39 group: GroupSchema,40 product: ProductSchema,41 prices: z.array(PriceSchema),42 updatedAt: z.string().nullable(),43});44export type TcgcsvPayload = z.infer<typeof RawPayloadSchema>;4546export function trimProduct(p: Record<string, unknown>) {47 const keep = ['productId', 'name', 'cleanName', 'imageUrl', 'url', 'modifiedOn', 'extendedData'];48 const out: Record<string, unknown> = {};49 for (const k of keep) if (p[k] !== undefined) out[k] = p[k];50 return ProductSchema.parse(out);51}5253export class TcgcsvConnector extends BaseConnector {54 readonly version = '1.0.0';55 readonly parserVersion = PARSER_VERSION;56 protected override minIntervalMs = 260;5758 private categories(): Array<[number, z.infer<typeof CategoryCfgSchema>]> {59 const cfg = (this.meta.config.categories ?? {}) as Record<string, unknown>;60 return Object.entries(cfg).map(([id, c]) => [Number(id), CategoryCfgSchema.parse(c)] as [number, z.infer<typeof CategoryCfgSchema>]);61 }6263 private async get(ctx: CrawlContext, url: string) {64 await this.throttle();65 return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: UA_HEADERS }), (r) => r.success && (r.json !== null || Boolean(r.html)), 4, 1500);66 }6768 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {69 const stamp = await this.get(ctx, 'https://tcgcsv.com/last-updated.txt');70 const updatedAt = typeof stamp.html === 'string' ? stamp.html.trim() : typeof stamp.json === 'string' ? String(stamp.json).trim() : null;71 const cats = this.categories().filter(([id]) => !ctx.options.seeds?.length || ctx.options.seeds.includes(String(id)) || ctx.options.seeds.includes(String(id)));72 const maxGroups = ctx.options.mode === 'backfill' ? Infinity : Number(this.meta.config.maxGroupsPerCategory ?? 40);73 let catIdx = Number(ctx.options.cursor?.catIdx ?? 0);74 let groupIdx = Number(ctx.options.cursor?.groupIdx ?? 0);75 let count = 0;76 for (; catIdx < cats.length; catIdx++, groupIdx = 0) {77 const [categoryId, category] = cats[catIdx]!;78 const gRes = await this.get(ctx, `${API}/${categoryId}/groups`);79 const gJson = gRes.json as { results?: unknown[] } | null;80 if (!gRes.success || !Array.isArray(gJson?.results)) {81 ctx.anomaly('page_fetch_failed', `groups ${categoryId}: ${gRes.error ?? gRes.httpStatus}`);82 continue;83 }84 const groups = z85 .array(GroupSchema.loose())86 .parse(gJson.results)87 .sort((a, b) => (b.publishedOn ?? '').localeCompare(a.publishedOn ?? ''))88 .slice(0, maxGroups);89 for (; groupIdx < groups.length; groupIdx++) {90 if (ctx.signal?.aborted) return;91 const group = GroupSchema.parse(groups[groupIdx]);92 const pRes = await this.get(ctx, `${API}/${categoryId}/${group.groupId}/products`);93 const products = (pRes.json as { results?: unknown[] } | null)?.results;94 if (!pRes.success || !Array.isArray(products)) {95 ctx.anomaly('page_fetch_failed', `products ${categoryId}/${group.groupId}: ${pRes.error ?? pRes.httpStatus}`);96 continue;97 }98 const prRes = await this.get(ctx, `${API}/${categoryId}/${group.groupId}/prices`);99 const priceRows = (prRes.json as { results?: unknown[] } | null)?.results;100 const byProduct = new Map<number, z.infer<typeof PriceSchema>[]>();101 if (Array.isArray(priceRows)) {102 for (const row of priceRows) {103 const parsed = PriceSchema.safeParse(row);104 if (!parsed.success) continue;105 const list = byProduct.get(parsed.data.productId) ?? [];106 list.push(parsed.data);107 byProduct.set(parsed.data.productId, list);108 }109 } else {110 ctx.anomaly('page_fetch_failed', `prices ${categoryId}/${group.groupId}: ${prRes.error ?? prRes.httpStatus}`);111 }112 for (const raw of products) {113 let product: z.infer<typeof ProductSchema>;114 try {115 product = trimProduct(raw as Record<string, unknown>);116 } catch (err) {117 ctx.anomaly('parse_failure_product', `${group.groupId}: ${err instanceof Error ? err.message : String(err)}`);118 continue;119 }120 if (this.reached(ctx, count)) return;121 count++;122 const payload: TcgcsvPayload = { categoryId, category, group, product, prices: byProduct.get(product.productId) ?? [], updatedAt };123 yield { url: product.url ?? `https://www.tcgplayer.com/product/${product.productId}`, externalId: String(product.productId), kind: 'catalog_item', engine: 'api', httpStatus: pRes.httpStatus, payload, fetchedAt: pRes.fetchedAt };124 }125 await ctx.setCursor({ catIdx, groupIdx: groupIdx + 1, updatedAt });126 }127 }128 await ctx.setCursor({ catIdx: 0, groupIdx: 0, updatedAt, completedAt: new Date().toISOString() });129 }130131 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {132 const { category, group, product, prices, updatedAt } = RawPayloadSchema.parse(raw.payload);133 const ext = new Map(product.extendedData.map((e) => [e.name.toLowerCase(), e.value]));134 const { number, total } = splitNumber(ext.get('number') ?? null);135 const rarity = ext.get('rarity') ?? null;136 const isSealed = !number && !rarity;137 const year = group.publishedOn ? Number(group.publishedOn.slice(0, 4)) || null : null;138 const images = tcgplayerImage(product.imageUrl);139 // "Charizard - 4/102" style names carry the number; keep the clean name.140 const name = (product.cleanName ?? product.name).replace(/\s+-\s+[A-Za-z0-9]+\/[A-Za-z0-9]+$/, '').trim() || product.name;141 const observedAt = raw.fetchedAt;142 const obsDate = isoDay(updatedAt) ?? dayOf(observedAt);143 const priceKinds = new Set(((this.meta.config.priceKinds as string[] | undefined) ?? ['market', 'low']));144 const build = (variant: string | null) =>145 attrs({146 categorySlug: category.slug,147 franchise: category.franchise,148 brand: category.brand,149 set: group.name,150 setCode: group.abbreviation ?? null,151 name,152 number,153 year,154 variant,155 language: category.language ?? 'English',156 rarity,157 identifiers: { tcgplayer_id: String(product.productId), tcgplayer_group_id: String(group.groupId) },158 metadata: { sealed: isSealed, total, card_type: ext.get('card type') ?? ext.get('cardtype') ?? null, supplemental: group.isSupplemental ?? false, tcgplayer_url: product.url ?? null },159 });160 const out: NormalizedRecord[] = [];161 const subTypes = prices.length ? [...new Set(prices.map((p) => p.subTypeName ?? 'Normal'))] : ['Normal'];162 const seenVariants = new Set<string>();163 for (const subType of subTypes) {164 const variant = variantFromSubType(subType);165 const key = variant ?? '';166 const a = build(variant);167 const rawTitle = makeTitle({ name, set: group.name, number, total, year, variant });168 if (!seenVariants.has(key)) {169 seenVariants.add(key);170 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${product.productId}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: group.publishedOn ? new Date(group.publishedOn) : null }));171 }172 const row = prices.find((p) => (p.subTypeName ?? 'Normal') === subType);173 if (!row) continue;174 const pairs: Array<['market' | 'low' | 'mid' | 'high', number | null]> = [175 ['market', num(row.marketPrice)],176 ['low', num(row.lowPrice)],177 ['mid', num(row.midPrice)],178 ['high', num(row.highPrice)],179 ];180 for (const [kind, price] of pairs) {181 if (!price || !priceKinds.has(kind)) continue;182 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${product.productId}:${subType}:${kind}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.75, parserVersion: PARSER_VERSION, priceKind: kind, price, currency: 'USD', observationDate: obsDate, sampleSize: null }));183 }184 }185 return out;186 }187}188189export default (meta: ConnectorMeta) => new TcgcsvConnector(meta);190