import { z } from 'zod'; import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedListingSchema, type Engine, type NormalizedRecord } from '@rareindex/shared'; import { attrs } from '../_lib/shared.js'; import { HTML_HEADERS, JP_EXCLUDE_RE, cleanText, isJpBundle, jpCondition, jpVariant, parseJpCardTitle, yen } from '../_g1-cards-eu-jp-lib/index.js'; /** * Cardrush (カードラッシュ) — one storefront per game on the same shop platform. Category listing pages * (product-list/?page=n) → one raw record per page; normalize emits one JPY listing per product. */ const PARSER_VERSION = '1.0.0'; const StoreSchema = z.object({ host: z.string(), categorySlug: z.string(), franchise: z.string().nullable().optional(), brand: z.string().nullable().optional(), bracket: z.enum(['rarity', 'set']).default('rarity'), engines: z.array(z.enum(['api', 'feed', 'firecrawl', 'scrapfly', 'browser', 'manual'])).optional(), pagesPerList: z.number().int().positive().optional(), lists: z.record(z.string(), z.string()), }); export type CardrushStore = z.infer; export const ItemSchema = z.object({ productId: z.string(), url: z.string(), title: z.string(), modelNumber: z.string().nullable(), priceJpy: z.number().nullable(), stock: z.number().int().nullable(), soldOut: z.boolean(), image: z.string().nullable() }); export type CardrushItem = z.infer; const RawPayloadSchema = z.object({ store: StoreSchema.omit({ lists: true, engines: true, pagesPerList: true }), listId: z.string(), listName: z.string(), page: z.number().int(), items: z.array(ItemSchema) }); export type CardrushPayload = z.infer; /** Parse a product-list page: .item_data blocks + pager. */ export function parseListPage(doc: string, host: string): { title: string | null; items: CardrushItem[]; hasNext: boolean } { const $ = html.load(doc); const items: CardrushItem[] = []; $('.item_data').each((_, el) => { const $el = $(el); const productId = $el.attr('data-product-id')?.trim() || $el.find('a.item_data_link').attr('href')?.match(/\/product\/(\d+)/)?.[1] || ''; const href = $el.find('a.item_data_link').attr('href') ?? (productId ? `https://${host}/product/${productId}` : null); const title = cleanText($el.find('.goods_name').first().text()); if (!productId || !href || !title) return; const modelNumber = cleanText($el.find('.model_number_value').first().text()); const priceJpy = yen(cleanText($el.find('.selling_price .figure').first().text())); const stockText = cleanText($el.find('.stock').first().text()) ?? ''; const soldOut = $el.find('.stock.soldout').length > 0 || /在庫なし|売り切れ|SOLD/i.test(stockText); const stockM = stockText.replace(/[,,]/g, '').match(/(\d+)\s*(?:枚|点|個)/); const stock = soldOut ? 0 : stockM ? Number(stockM[1]) : null; const img = $el.find('.global_photo img').first(); const image = img.attr('data-x2') ?? img.attr('src') ?? null; items.push({ productId, url: href.startsWith('http') ? href : `https://${host}${href}`, title, modelNumber, priceJpy, stock, soldOut, image }); }); const hasNext = $('.pager a.to_next_page').length > 0; const title = cleanText($('title').first().text())?.split(' - ')[0]?.trim() ?? null; return { title, items, hasNext }; } export class CardrushConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 4000; private stores(): CardrushStore[] { return z.array(StoreSchema).parse(this.meta.config.stores ?? []); } async *crawl(ctx: CrawlContext): AsyncIterable { let stores = this.stores(); if (ctx.options.seeds?.length) stores = stores.filter((s) => ctx.options.seeds!.some((seed) => seed === s.host || seed === s.categorySlug || seed.startsWith(`${s.host}/`))); const backfill = ctx.options.mode === 'backfill'; let storeIdx = Number(ctx.options.cursor?.storeIdx ?? 0); let listIdx = Number(ctx.options.cursor?.listIdx ?? 0); let page = Number(ctx.options.cursor?.page ?? 1); let count = 0; for (; storeIdx < stores.length; storeIdx++, listIdx = 0) { const store = stores[storeIdx]!; const lists = Object.entries(store.lists); const maxPages = backfill ? this.policy.backfillMaxPages : Number(store.pagesPerList ?? this.meta.config.pagesPerList ?? 3); const engines = (store.engines ?? ['api']) as Engine[]; for (; listIdx < lists.length; listIdx++, page = 1) { const [listId, listName] = lists[listIdx]!; for (; page <= maxPages; page++) { if (ctx.signal?.aborted) return; if (this.reached(ctx, count)) { await ctx.setCursor({ storeIdx, listIdx, page }); return; } const url = `https://${store.host}/product-list/${listId}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(url); const res = await ctx.fetch(url, { engines, headers: HTML_HEADERS, responseType: 'text', renderJs: false, country: 'jp', expect: ['title', 'price'], parse: (r) => { const parsed = parseListPage(r.html ?? '', store.host); return { title: parsed.items.length ? 'ok' : null, price: parsed.items.some((i) => i.priceJpy) ? 1 : null }; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const { title, items, hasNext } = parseListPage(res.html, store.host); if (!items.length) { if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no .item_data blocks`); break; } count++; const payload: CardrushPayload = { store: { host: store.host, categorySlug: store.categorySlug, franchise: store.franchise ?? null, brand: store.brand ?? null, bracket: store.bracket }, listId, listName: title ?? listName, page, items }; yield { url, externalId: `${store.host}:${listId}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ storeIdx, listIdx, page: page + 1 }); await ctx.progress({ page, totalPages: null, itemsProcessed: count }); if (!hasNext) break; } } } await ctx.setCursor({ storeIdx: 0, listIdx: 0, page: 1, completedAt: new Date().toISOString(), done: true }); } async normalize(raw: RawRecordLike): Promise { const p = RawPayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const observedAt = raw.fetchedAt; const seen = new Set(); for (const it of p.items) { if (it.priceJpy === null || seen.has(it.productId)) continue; seen.add(it.productId); if (JP_EXCLUDE_RE.test(it.title)) continue; const t = parseJpCardTitle(it.title, { bracket: p.store.bracket }); const language = t.language ?? (t.notes.some((n) => /英語/.test(n)) ? 'English' : 'Japanese'); const isBundle = isJpBundle(it.title, t.quantity); const variant = jpVariant(t.notes, t.name); const a = attrs({ categorySlug: p.store.categorySlug, franchise: p.store.franchise ?? null, brand: p.store.brand ?? null, setCode: t.setCode, name: t.name, number: t.number, variant, language, rarity: t.rarity, // model_number is the shop's katakana/romaji reading of the name (a search key), NOT a SKU/GTIN → never emitted as `sku` identifiers: { cardrush_product_id: `${p.store.host}/${it.productId}`, ...(it.modelNumber ? { cardrush_model_number: it.modelNumber } : {}) }, metadata: { store: p.store.host, list_id: p.listId, list_name: p.listName, total: t.total, notes: t.notes, quantity: t.quantity, sealed: t.sealed }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: `${p.store.host}:${it.productId}`, rawTitle: it.title, imageUrls: it.image ? [it.image] : [], attributes: a, grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier, certificationNumber: null }, condition: { condition: jpCondition(t.conditionRaw), conditionRaw: t.conditionRaw, completeness: t.sealed ? 'sealed' : null }, observedAt, confidence: t.grade ? 0.8 : 0.75, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.priceJpy, currency: 'JPY', seller: 'Cardrush', location: 'JP', quantity: isBundle ? t.quantity : it.stock, availability: it.soldOut ? 'ended' : 'available', }), ); } return out; } } export default (meta: ConnectorMeta) => new CardrushConnector(meta);