import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parseSourceDate, type NormalizedRecord } from '@rareindex/shared'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; const BASE = 'https://www.suruga-ya.jp'; const PARSER_VERSION = '1.0.0'; export const CardSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), image: z.string().nullable(), typeLabel: z.string().nullable(), releaseDate: z.string().nullable(), brand: z.string().nullable(), price: z.number().nullable(), soldOut: z.boolean(), listPrice: z.number().nullable(), marketplacePrice: z.number().nullable(), used: z.boolean(), }); export type Card = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) }); export type PagePayload = z.infer; export function yen(s: string | null | undefined): number | null { const m = s?.replace(/,/g, '').match(/[¥¥]\s*(\d+)/); return m ? Number(m[1]) : null; } /** Parse a search results page into compact cards. */ export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload { const $ = H.load(htmlText); const items: Card[] = []; $('.item').each((_, el) => { const e = $(el); const a = e.find('a[href*="/product/detail/"], a[href*="/product/other/"]').first(); const href = a.attr('href'); if (!href) return; const url = href.split('?')[0]!; const id = url.match(/\/product\/(?:detail|other)\/([A-Za-z0-9]+)/)?.[1]; const title = H.text(e.find('h3.product-name').first()) ?? H.text(a); if (!id || !title) return; const image = e.find('img').first().attr('src') ?? null; const typeLabel = H.text(e.find('.condition').filter((__, c) => Boolean($(c).text().trim())).first()); const releaseRaw = H.text(e.find('.release_date').first()); const releaseDate = releaseRaw?.match(/(\d{4}\/\d{2}\/\d{2})/)?.[1] ?? null; const brand = H.text(e.find('.brand').first())?.replace(/^\[|\]$/g, '').trim() || null; const priceText = H.text(e.find('.item_price .price').first()) ?? ''; const soldOut = /品切れ/.test(priceText); const price = soldOut ? null : yen(priceText); const listPrice = yen(H.text(e.find('.price_teika').first())); const marketplacePrice = yen(e.find('.highlight-box strong').map((__, x) => $(x).text()).get().find((t) => /[¥¥]/.test(t)) ?? null); const used = /中古/.test(e.text()) || /中古/.test(priceText); items.push({ id, url: url.startsWith('http') ? url : BASE + url, title, image: image && image.startsWith('http') ? image : image ? BASE + image : null, typeLabel: typeLabel ?? null, releaseDate, brand, price, soldOut, listPrice, marketplacePrice, used }); }); return { kind: 'search_page', url: pageUrl, categorySlug, items }; } /** * Japanese card titles look like "114/083[SAR]:【PSA/GEM MT 10】(キラ)メガゲッコウガex". * Extract card number, rarity code, grade and the bare name; everything unknown stays null. */ export function parseJpCardTitle(title: string): { number: string | null; rarity: string | null; name: string; grader: string | null; grade: string | null } { const number = title.match(/^(\d{1,3}\/\d{1,3})/)?.[1] ?? null; const rarity = title.match(/\[([A-Z]{1,4})\]/)?.[1] ?? null; const g = parseGradeFromTitle(title.replace(/【([A-Z]{2,4})\/?/g, '【$1 ').replace(/[【】]/g, ' ')); let name = title.replace(/^\d{1,3}\/\d{1,3}\s*/, '').replace(/\[[A-Z]{1,4}\]\s*[::]?\s*/, '').replace(/【[^】]*】\s*/g, '').replace(/\((キラ|ミラー|ノーマル)\)\s*/g, '').trim(); if (!name) name = title; return { number, rarity, name, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade }; } export class SurugayaConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.seeds as Array<{ query: string; categorySlug: string }> | undefined) ?? []; const pages = Number(this.meta.config.pagesPerSeed ?? 1); const cap = ctx.options.limit; let count = 0; const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = startSeed; i < seeds.length; i++) { const seed = seeds[i]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return; const url = `${BASE}/search?${seed.query}${page > 1 ? `&page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { // Firecrawl only: Scrapfly's rendered fallback costs ~40 credits/page here and adds nothing. engines: ['firecrawl'], minQuality: 0.3, // pages where every card is sold out (no price) are still valid catalog data expect: ['title', 'price'], parse: (r) => { const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null; return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price || x.marketplacePrice) ? 1 : null } : null; }, }); const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null; if (!payload?.items.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`); break; } count++; yield { url, externalId: `${seed.query}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const c of p.items) { const release = c.releaseDate ? parseSourceDate(c.releaseDate) : null; const year = release ? release.getUTCFullYear() : null; const isCard = ['pokemon', 'yugioh', 'one_piece_card_game'].includes(p.categorySlug); const card = isCard ? parseJpCardTitle(c.title) : null; const attributes = AssetAttributesSchema.parse({ categorySlug: p.categorySlug, brand: c.brand, franchise: c.brand, name: card?.name ?? c.title, number: card?.number ?? null, rarity: card?.rarity ?? null, year, language: 'Japanese', region: 'JP', originalMsrp: c.listPrice, originalMsrpCurrency: c.listPrice ? 'JPY' : null, identifiers: { surugaya_id: c.id }, metadata: { type_label: c.typeLabel, release: c.releaseDate }, }); const rawTitle = `${c.title}${c.brand ? ` [${c.brand}]` : ''}${year ? ` (${year})` : ''}`; const grade = { grader: card?.grader ?? null, grade: card?.grade ?? null, qualifier: null, certificationNumber: null }; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: release })); const price = c.price ?? c.marketplacePrice; if (price) { const viaMarketplace = c.price === null && c.marketplacePrice !== null; out.push( NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: viaMarketplace ? `${c.id}:mp` : c.id, confidence: 0.8, listingType: 'fixed_price', price, currency: 'JPY', seller: viaMarketplace ? 'Suruga-ya marketplace seller' : 'Suruga-ya', location: 'Japan', condition: { condition: c.used || viaMarketplace ? null : 'mint_in_box', conditionRaw: viaMarketplace ? '中古 (marketplace)' : c.used ? '中古' : '新品', completeness: null }, availability: 'available', }), ); } else if (c.soldOut) { out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.7, listingType: 'fixed_price', price: null, currency: 'JPY', seller: 'Suruga-ya', location: 'Japan', availability: 'sold' })); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new SurugayaConnector(meta); }