TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type Engine, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../_lib/shared.js';5import { HTML_HEADERS, JP_EXCLUDE_RE, cleanText, isJpBundle, jpCondition, jpVariant, parseJpCardTitle, yen } from '../_g1-cards-eu-jp-lib/index.js';67/**8 * Cardrush (カードラッシュ) — one storefront per game on the same shop platform. Category listing pages9 * (product-list/<id>?page=n) → one raw record per page; normalize emits one JPY listing per product.10 */11const PARSER_VERSION = '1.0.0';1213const StoreSchema = z.object({14 host: z.string(),15 categorySlug: z.string(),16 franchise: z.string().nullable().optional(),17 brand: z.string().nullable().optional(),18 bracket: z.enum(['rarity', 'set']).default('rarity'),19 engines: z.array(z.enum(['api', 'feed', 'firecrawl', 'scrapfly', 'browser', 'manual'])).optional(),20 pagesPerList: z.number().int().positive().optional(),21 lists: z.record(z.string(), z.string()),22});23export type CardrushStore = z.infer<typeof StoreSchema>;2425export const ItemSchema = z.object({ productId: z.string(), url: z.string(), title: z.string(), modelNumber: z.string().nullable(), priceJpy: z.number().nullable(), stock: z.number().int().nullable(), soldOut: z.boolean(), image: z.string().nullable() });26export type CardrushItem = z.infer<typeof ItemSchema>;27const RawPayloadSchema = z.object({ store: StoreSchema.omit({ lists: true, engines: true, pagesPerList: true }), listId: z.string(), listName: z.string(), page: z.number().int(), items: z.array(ItemSchema) });28export type CardrushPayload = z.infer<typeof RawPayloadSchema>;2930/** Parse a product-list page: .item_data blocks + pager. */31export function parseListPage(doc: string, host: string): { title: string | null; items: CardrushItem[]; hasNext: boolean } {32 const $ = html.load(doc);33 const items: CardrushItem[] = [];34 $('.item_data').each((_, el) => {35 const $el = $(el);36 const productId = $el.attr('data-product-id')?.trim() || $el.find('a.item_data_link').attr('href')?.match(/\/product\/(\d+)/)?.[1] || '';37 const href = $el.find('a.item_data_link').attr('href') ?? (productId ? `https://${host}/product/${productId}` : null);38 const title = cleanText($el.find('.goods_name').first().text());39 if (!productId || !href || !title) return;40 const modelNumber = cleanText($el.find('.model_number_value').first().text());41 const priceJpy = yen(cleanText($el.find('.selling_price .figure').first().text()));42 const stockText = cleanText($el.find('.stock').first().text()) ?? '';43 const soldOut = $el.find('.stock.soldout').length > 0 || /在庫なし|売り切れ|SOLD/i.test(stockText);44 const stockM = stockText.replace(/[，,]/g, '').match(/(\d+)\s*(?:枚|点|個)/);45 const stock = soldOut ? 0 : stockM ? Number(stockM[1]) : null;46 const img = $el.find('.global_photo img').first();47 const image = img.attr('data-x2') ?? img.attr('src') ?? null;48 items.push({ productId, url: href.startsWith('http') ? href : `https://${host}${href}`, title, modelNumber, priceJpy, stock, soldOut, image });49 });50 const hasNext = $('.pager a.to_next_page').length > 0;51 const title = cleanText($('title').first().text())?.split(' - ')[0]?.trim() ?? null;52 return { title, items, hasNext };53}5455export class CardrushConnector extends BaseConnector {56 readonly version = '1.0.0';57 readonly parserVersion = PARSER_VERSION;58 protected override minIntervalMs = 4000;5960 private stores(): CardrushStore[] {61 return z.array(StoreSchema).parse(this.meta.config.stores ?? []);62 }6364 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {65 let stores = this.stores();66 if (ctx.options.seeds?.length) stores = stores.filter((s) => ctx.options.seeds!.some((seed) => seed === s.host || seed === s.categorySlug || seed.startsWith(`${s.host}/`)));67 const backfill = ctx.options.mode === 'backfill';68 let storeIdx = Number(ctx.options.cursor?.storeIdx ?? 0);69 let listIdx = Number(ctx.options.cursor?.listIdx ?? 0);70 let page = Number(ctx.options.cursor?.page ?? 1);71 let count = 0;72 for (; storeIdx < stores.length; storeIdx++, listIdx = 0) {73 const store = stores[storeIdx]!;74 const lists = Object.entries(store.lists);75 const maxPages = backfill ? this.policy.backfillMaxPages : Number(store.pagesPerList ?? this.meta.config.pagesPerList ?? 3);76 const engines = (store.engines ?? ['api']) as Engine[];77 for (; listIdx < lists.length; listIdx++, page = 1) {78 const [listId, listName] = lists[listIdx]!;79 for (; page <= maxPages; page++) {80 if (ctx.signal?.aborted) return;81 if (this.reached(ctx, count)) {82 await ctx.setCursor({ storeIdx, listIdx, page });83 return;84 }85 const url = `https://${store.host}/product-list/${listId}${page > 1 ? `?page=${page}` : ''}`;86 await this.throttle(url);87 const res = await ctx.fetch(url, {88 engines,89 headers: HTML_HEADERS,90 responseType: 'text',91 renderJs: false,92 country: 'jp',93 expect: ['title', 'price'],94 parse: (r) => {95 const parsed = parseListPage(r.html ?? '', store.host);96 return { title: parsed.items.length ? 'ok' : null, price: parsed.items.some((i) => i.priceJpy) ? 1 : null };97 },98 });99 if (!res.success || !res.html) {100 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);101 break;102 }103 const { title, items, hasNext } = parseListPage(res.html, store.host);104 if (!items.length) {105 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no .item_data blocks`);106 break;107 }108 count++;109 const payload: CardrushPayload = { store: { host: store.host, categorySlug: store.categorySlug, franchise: store.franchise ?? null, brand: store.brand ?? null, bracket: store.bracket }, listId, listName: title ?? listName, page, items };110 yield { url, externalId: `${store.host}:${listId}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };111 await ctx.setCursor({ storeIdx, listIdx, page: page + 1 });112 await ctx.progress({ page, totalPages: null, itemsProcessed: count });113 if (!hasNext) break;114 }115 }116 }117 await ctx.setCursor({ storeIdx: 0, listIdx: 0, page: 1, completedAt: new Date().toISOString(), done: true });118 }119120 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {121 const p = RawPayloadSchema.parse(raw.payload);122 const out: NormalizedRecord[] = [];123 const observedAt = raw.fetchedAt;124 const seen = new Set<string>();125 for (const it of p.items) {126 if (it.priceJpy === null || seen.has(it.productId)) continue;127 seen.add(it.productId);128 if (JP_EXCLUDE_RE.test(it.title)) continue;129 const t = parseJpCardTitle(it.title, { bracket: p.store.bracket });130 const language = t.language ?? (t.notes.some((n) => /英語/.test(n)) ? 'English' : 'Japanese');131 const isBundle = isJpBundle(it.title, t.quantity);132 const variant = jpVariant(t.notes, t.name);133 const a = attrs({134 categorySlug: p.store.categorySlug,135 franchise: p.store.franchise ?? null,136 brand: p.store.brand ?? null,137 setCode: t.setCode,138 name: t.name,139 number: t.number,140 variant,141 language,142 rarity: t.rarity,143 // model_number is the shop's katakana/romaji reading of the name (a search key), NOT a SKU/GTIN → never emitted as `sku`144 identifiers: { cardrush_product_id: `${p.store.host}/${it.productId}`, ...(it.modelNumber ? { cardrush_model_number: it.modelNumber } : {}) },145 metadata: { store: p.store.host, list_id: p.listId, list_name: p.listName, total: t.total, notes: t.notes, quantity: t.quantity, sealed: t.sealed },146 });147 out.push(148 NormalizedListingSchema.parse({149 kind: 'listing',150 connectorId: this.meta.id,151 sourceId: this.meta.sourceId,152 sourceUrl: it.url,153 externalId: `${p.store.host}:${it.productId}`,154 rawTitle: it.title,155 imageUrls: it.image ? [it.image] : [],156 attributes: a,157 grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier, certificationNumber: null },158 condition: { condition: jpCondition(t.conditionRaw), conditionRaw: t.conditionRaw, completeness: t.sealed ? 'sealed' : null },159 observedAt,160 confidence: t.grade ? 0.8 : 0.75,161 parserVersion: PARSER_VERSION,162 listingType: 'fixed_price',163 price: it.priceJpy,164 currency: 'JPY',165 seller: 'Cardrush',166 location: 'JP',167 quantity: isBundle ? t.quantity : it.stock,168 availability: it.soldOut ? 'ended' : 'available',169 }),170 );171 }172 return out;173 }174}175176export default (meta: ConnectorMeta) => new CardrushConnector(meta);177