TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, priceObservation } from '../_lib/shared.js';5import { JSON_HEADERS, dayOf, isoDay } from '../_g1-cards-eu-jp-lib/index.js';67/**8 * Cardmarket public bulk files: daily price guide (EUR) joined with the product catalog per game.9 * One raw record per product; normalize emits a catalog item (plus a foil variant when foil prices10 * exist) and one price observation per Cardmarket price field, dated by the file's own createdAt.11 */12const BASE = 'https://downloads.s3.cardmarket.com/productCatalog';13const PARSER_VERSION = '1.0.0';1415const GameCfgSchema = z.object({ slug: z.string(), name: z.string(), franchise: z.string().nullable().optional(), brand: z.string().nullable().optional(), foilVariant: z.string().default('Foil') });16export type GameCfg = z.infer<typeof GameCfgSchema>;1718export const ProductSchema = z.object({19 idProduct: z.number().int(),20 name: z.string(),21 idCategory: z.number().int(),22 categoryName: z.string(),23 idExpansion: z.number().int().nullable().optional(),24 idMetacard: z.number().int().nullable().optional(),25 dateAdded: z.string().nullable().optional(),26});27export type CardmarketProduct = z.infer<typeof ProductSchema>;2829const price = z.number().nullable().optional();30export const PriceRowSchema = z.object({31 idProduct: z.number().int(),32 idCategory: z.number().int().optional(),33 avg: price,34 low: price,35 trend: price,36 avg1: price,37 avg7: price,38 avg30: price,39 'avg-foil': price,40 'low-foil': price,41 'trend-foil': price,42 'avg1-foil': price,43 'avg7-foil': price,44 'avg30-foil': price,45});46export type PriceRow = z.infer<typeof PriceRowSchema>;4748const RawPayloadSchema = z.object({49 gameId: z.number().int(),50 game: GameCfgSchema,51 product: ProductSchema,52 prices: PriceRowSchema.nullable(),53 single: z.boolean(),54 priceGuideCreatedAt: z.string().nullable(),55 productListCreatedAt: z.string().nullable(),56});57export type CardmarketPayload = z.infer<typeof RawPayloadSchema>;5859const PriceGuideFileSchema = z.object({ version: z.number().optional(), createdAt: z.string().nullable().optional(), priceGuides: z.array(z.unknown()) });60const ProductFileSchema = z.object({ version: z.number().optional(), createdAt: z.string().nullable().optional(), products: z.array(z.unknown()) });6162/** Cardmarket price field → RareIndex priceKind. */63export const PRICE_FIELDS: Array<[keyof PriceRow, 'market' | 'low' | 'mid' | 'trend' | 'average_7d' | 'average_30d']> = [64 ['avg', 'mid'],65 ['low', 'low'],66 ['trend', 'trend'],67 ['avg1', 'market'],68 ['avg7', 'average_7d'],69 ['avg30', 'average_30d'],70];7172export function hasAnyPrice(row: PriceRow | null | undefined, foil = false): boolean {73 if (!row) return false;74 return PRICE_FIELDS.some(([f]) => {75 const v = row[foil ? (`${f}-foil` as keyof PriceRow) : f];76 return typeof v === 'number' && v > 0;77 });78}7980/**81 * Cardmarket product names embed disambiguators: "Kakuna [Bug Bite | Primal Clash]", "Roronoa Zoro (OP01-001)",82 * "Auron (1-001)", "Forest (V.1)". Returns the clean name plus the parts we can use.83 */84export function parseProductName(raw: string): { name: string; number: string | null; version: string | null; disambiguation: string | null } {85 let name = raw.replace(/'/g, "'").replace(/&/g, '&').replace(/\s+/g, ' ').trim();86 let disambiguation: string | null = null;87 let number: string | null = null;88 let version: string | null = null;89 const br = name.match(/\s*\[([^\]]+)\]\s*$/);90 if (br) {91 disambiguation = br[1]!.trim();92 name = name.slice(0, br.index).trim();93 }94 for (;;) {95 const m = name.match(/\s*\(([^()]+)\)\s*$/);96 if (!m) break;97 const inner = m[1]!.trim();98 if (/^V\.\s?\d+$/i.test(inner)) version = version ?? inner.replace(/\s/g, '');99 else if (/^(?:[A-Z]{1,6}\d{0,3}[A-Z]?-\d{1,4}[A-Za-z]{0,3}|\d{1,2}-\d{3}[A-Z]?|[A-Z]{1,3}\d{1,3}-\d{1,3})$/.test(inner)) number = number ?? inner;100 else break;101 name = name.slice(0, m.index).trim();102 }103 return { name: name || raw.trim(), number, version, disambiguation };104}105106const SEALED_RE = /booster|box|display|bundle|deck|case|collection|tin|pack|kit|set$|starter|bundle|toolkit|fat pack|gift/i;107108export class CardmarketPriceGuideConnector extends BaseConnector {109 readonly version = '1.0.0';110 readonly parserVersion = PARSER_VERSION;111 protected override minIntervalMs = 2000;112113 private games(): Array<[number, GameCfg]> {114 const cfg = (this.meta.config.games ?? {}) as Record<string, unknown>;115 return Object.entries(cfg)116 .map(([id, c]) => [Number(id), GameCfgSchema.parse(c)] as [number, GameCfg])117 .sort((a, b) => a[0] - b[0]);118 }119120 private async file<T>(ctx: CrawlContext, url: string, schema: z.ZodType<T>): Promise<{ data: T | null; status: number | null; fetchedAt: Date; error: string | null }> {121 await this.throttle(url);122 const res = await ctx.fetch(url, { engines: ['api'], headers: JSON_HEADERS, timeoutMs: 240_000, minQuality: 0 });123 if (!res.success || res.json === null || res.json === undefined) return { data: null, status: res.httpStatus ?? null, fetchedAt: res.fetchedAt, error: res.error ?? `HTTP ${res.httpStatus}` };124 const parsed = schema.safeParse(res.json);125 if (!parsed.success) return { data: null, status: res.httpStatus ?? null, fetchedAt: res.fetchedAt, error: `schema: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}` };126 return { data: parsed.data, status: res.httpStatus ?? null, fetchedAt: res.fetchedAt, error: null };127 }128129 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {130 let games = this.games();131 if (ctx.options.seeds?.length) games = games.filter(([id]) => ctx.options.seeds!.includes(String(id)));132 const includeNonSingles = this.meta.config.includeNonSingles !== false;133 const onlyPriced = this.meta.config.onlyPriced !== false;134 let gameIdx = Number(ctx.options.cursor?.gameIdx ?? 0);135 let offset = Number(ctx.options.cursor?.offset ?? 0);136 let count = 0;137 let processed = 0;138 for (; gameIdx < games.length; gameIdx++, offset = 0) {139 if (ctx.signal?.aborted) return;140 const [gameId, game] = games[gameIdx]!;141 const guideUrl = `${BASE}/priceGuide/price_guide_${gameId}.json`;142 const guide = await this.file(ctx, guideUrl, PriceGuideFileSchema);143 if (!guide.data) {144 ctx.anomaly('page_fetch_failed', `${guideUrl}: ${guide.error}`);145 continue;146 }147 const prices = new Map<number, PriceRow>();148 let badRows = 0;149 for (const row of guide.data.priceGuides) {150 const p = PriceRowSchema.safeParse(row);151 if (p.success) prices.set(p.data.idProduct, p.data);152 else badRows++;153 }154 if (badRows) ctx.anomaly('schema_drift', `price_guide_${gameId}: ${badRows} unparseable rows`);155 const priceGuideCreatedAt = guide.data.createdAt ?? null;156157 const lists: Array<{ product: CardmarketProduct; single: boolean }> = [];158 let productListCreatedAt: string | null = null;159 const kinds: Array<['singles' | 'nonsingles', boolean]> = includeNonSingles ? [['singles', true], ['nonsingles', false]] : [['singles', true]];160 for (const [kind, single] of kinds) {161 const url = `${BASE}/productList/products_${kind}_${gameId}.json`;162 const f = await this.file(ctx, url, ProductFileSchema);163 if (!f.data) {164 ctx.anomaly('page_fetch_failed', `${url}: ${f.error}`);165 continue;166 }167 productListCreatedAt = productListCreatedAt ?? f.data.createdAt ?? null;168 let bad = 0;169 for (const raw of f.data.products) {170 const p = ProductSchema.safeParse(raw);171 if (!p.success) {172 bad++;173 continue;174 }175 lists.push({ product: p.data, single });176 }177 if (bad) ctx.anomaly('schema_drift', `products_${kind}_${gameId}: ${bad} unparseable products`);178 }179 const items = onlyPriced ? lists.filter((x) => hasAnyPrice(prices.get(x.product.idProduct)) || hasAnyPrice(prices.get(x.product.idProduct), true)) : lists;180 ctx.log.info({ gameId, products: lists.length, priced: items.length, createdAt: priceGuideCreatedAt }, 'cardmarket bulk files loaded');181 for (; offset < items.length; offset++) {182 if (ctx.signal?.aborted) return;183 if (this.reached(ctx, count)) {184 await ctx.setCursor({ gameIdx, offset });185 return;186 }187 const { product, single } = items[offset]!;188 count++;189 processed++;190 const payload: CardmarketPayload = { gameId, game, product, prices: prices.get(product.idProduct) ?? null, single, priceGuideCreatedAt, productListCreatedAt };191 yield { url: `${guideUrl}#idProduct=${product.idProduct}`, externalId: String(product.idProduct), kind: 'catalog_item', engine: 'api', httpStatus: guide.status, payload, fetchedAt: guide.fetchedAt };192 if (offset > 0 && offset % 2000 === 0) await ctx.setCursor({ gameIdx, offset: offset + 1 });193 }194 await ctx.setCursor({ gameIdx: gameIdx + 1, offset: 0, priceGuideCreatedAt });195 await ctx.progress({ page: gameIdx + 1, totalPages: games.length, itemsProcessed: processed });196 }197 await ctx.setCursor({ gameIdx: 0, offset: 0, completedAt: new Date().toISOString(), done: true });198 }199200 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {201 const { gameId, game, product, prices, single, priceGuideCreatedAt } = RawPayloadSchema.parse(raw.payload);202 const parsed = parseProductName(product.name);203 const observedAt = raw.fetchedAt;204 const obsDate = isoDay(priceGuideCreatedAt) ?? dayOf(observedAt);205 const sealed = !single && SEALED_RE.test(product.categoryName);206 const priceKinds = new Set((this.meta.config.priceKinds as string[] | undefined) ?? PRICE_FIELDS.map(([f]) => f));207 const ids: Record<string, string> = { cardmarket_id: String(product.idProduct) };208 if (single && product.idMetacard) ids.cardmarket_metacard_id = String(product.idMetacard);209 if (product.idExpansion) ids.cardmarket_expansion_id = String(product.idExpansion);210 const build = (variant: string | null) =>211 attrs({212 categorySlug: game.slug,213 franchise: game.franchise ?? null,214 brand: game.brand ?? null,215 name: parsed.name,216 number: parsed.number,217 variant,218 language: null,219 identifiers: ids,220 metadata: {221 cardmarket_game_id: gameId,222 cardmarket_game: game.name,223 cardmarket_category: product.categoryName,224 cardmarket_category_id: product.idCategory,225 cardmarket_expansion_id: product.idExpansion ?? null,226 single,227 sealed,228 version: parsed.version,229 disambiguation: parsed.disambiguation,230 date_added: product.dateAdded && !product.dateAdded.startsWith('0000') ? product.dateAdded : null,231 },232 });233 const out: NormalizedRecord[] = [];234 const variants: Array<{ variant: string | null; foil: boolean }> = [{ variant: null, foil: false }];235 if (single && hasAnyPrice(prices, true)) variants.push({ variant: game.foilVariant, foil: true });236 for (const { variant, foil } of variants) {237 const a = build(variant);238 const rawTitle = makeTitle({ name: parsed.name, number: parsed.number, variant }) + (parsed.disambiguation ? ` [${parsed.disambiguation}]` : '') + (single ? '' : ` — ${product.categoryName}`);239 out.push(240 catalogItem({241 kind: 'catalog_item',242 connectorId: this.meta.id,243 sourceId: this.meta.sourceId,244 sourceUrl: raw.url,245 externalId: `${product.idProduct}${foil ? ':foil' : ''}`,246 rawTitle,247 imageUrls: [],248 attributes: a,249 condition: { completeness: sealed ? 'sealed' : null },250 observedAt,251 confidence: 0.8,252 parserVersion: PARSER_VERSION,253 releaseDate: null,254 }),255 );256 if (!prices) continue;257 for (const [field, priceKind] of PRICE_FIELDS) {258 if (!priceKinds.has(field)) continue;259 const key = foil ? (`${field}-foil` as keyof PriceRow) : field;260 const v = prices[key];261 if (typeof v !== 'number' || !(v > 0)) continue;262 out.push(263 priceObservation({264 kind: 'price_observation',265 connectorId: this.meta.id,266 sourceId: this.meta.sourceId,267 sourceUrl: raw.url,268 externalId: `${product.idProduct}:${foil ? 'foil' : 'base'}:${field}`,269 rawTitle,270 imageUrls: [],271 attributes: { ...a, metadata: { ...a.metadata, cardmarket_field: key, foil } },272 condition: { completeness: sealed ? 'sealed' : null },273 observedAt,274 confidence: 0.75,275 parserVersion: PARSER_VERSION,276 priceKind,277 price: v,278 currency: 'EUR',279 observationDate: obsDate,280 sampleSize: null,281 }),282 );283 }284 }285 return out;286 }287}288289export default (meta: ConnectorMeta) => new CardmarketPriceGuideConnector(meta);290