TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';56const BASE = 'https://www.suruga-ya.jp';7const PARSER_VERSION = '1.0.0';89export const CardSchema = z.object({10 id: z.string(),11 url: z.string(),12 title: z.string(),13 image: z.string().nullable(),14 typeLabel: z.string().nullable(),15 releaseDate: z.string().nullable(),16 brand: z.string().nullable(),17 price: z.number().nullable(),18 soldOut: z.boolean(),19 listPrice: z.number().nullable(),20 marketplacePrice: z.number().nullable(),21 used: z.boolean(),22});23export type Card = z.infer<typeof CardSchema>;24export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) });25export type PagePayload = z.infer<typeof PagePayloadSchema>;2627export function yen(s: string | null | undefined): number | null {28 const m = s?.replace(/,/g, '').match(/[￥¥]\s*(\d+)/);29 return m ? Number(m[1]) : null;30}3132/** Parse a search results page into compact cards. */33export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {34 const $ = H.load(htmlText);35 const items: Card[] = [];36 $('.item').each((_, el) => {37 const e = $(el);38 const a = e.find('a[href*="/product/detail/"], a[href*="/product/other/"]').first();39 const href = a.attr('href');40 if (!href) return;41 const url = href.split('?')[0]!;42 const id = url.match(/\/product\/(?:detail|other)\/([A-Za-z0-9]+)/)?.[1];43 const title = H.text(e.find('h3.product-name').first()) ?? H.text(a);44 if (!id || !title) return;45 const image = e.find('img').first().attr('src') ?? null;46 const typeLabel = H.text(e.find('.condition').filter((__, c) => Boolean($(c).text().trim())).first());47 const releaseRaw = H.text(e.find('.release_date').first());48 const releaseDate = releaseRaw?.match(/(\d{4}\/\d{2}\/\d{2})/)?.[1] ?? null;49 const brand = H.text(e.find('.brand').first())?.replace(/^\[|\]$/g, '').trim() || null;50 const priceText = H.text(e.find('.item_price .price').first()) ?? '';51 const soldOut = /品切れ/.test(priceText);52 const price = soldOut ? null : yen(priceText);53 const listPrice = yen(H.text(e.find('.price_teika').first()));54 const marketplacePrice = yen(e.find('.highlight-box strong').map((__, x) => $(x).text()).get().find((t) => /[￥¥]/.test(t)) ?? null);55 const used = /中古/.test(e.text()) || /中古/.test(priceText);56 items.push({ id, url: url.startsWith('http') ? url : BASE + url, title, image: image && image.startsWith('http') ? image : image ? BASE + image : null, typeLabel: typeLabel ?? null, releaseDate, brand, price, soldOut, listPrice, marketplacePrice, used });57 });58 return { kind: 'search_page', url: pageUrl, categorySlug, items };59}6061/**62 * Japanese card titles look like "114/083[SAR]：【PSA/GEM MT 10】(キラ)メガゲッコウガex".63 * Extract card number, rarity code, grade and the bare name; everything unknown stays null.64 */65export function parseJpCardTitle(title: string): { number: string | null; rarity: string | null; name: string; grader: string | null; grade: string | null } {66 const number = title.match(/^(\d{1,3}\/\d{1,3})/)?.[1] ?? null;67 const rarity = title.match(/\[([A-Z]{1,4})\]/)?.[1] ?? null;68 const g = parseGradeFromTitle(title.replace(/【([A-Z]{2,4})\/?/g, '【$1 ').replace(/[【】]/g, ' '));69 let name = title.replace(/^\d{1,3}\/\d{1,3}\s*/, '').replace(/\[[A-Z]{1,4}\]\s*[：:]?\s*/, '').replace(/【[^】]*】\s*/g, '').replace(/\((キラ|ミラー|ノーマル)\)\s*/g, '').trim();70 if (!name) name = title;71 return { number, rarity, name, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade };72}7374export class SurugayaConnector extends BaseConnector {75 readonly version = '1.0.0';76 readonly parserVersion = PARSER_VERSION;77 protected override minIntervalMs = 2000;7879 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {80 const seeds = (this.meta.config.seeds as Array<{ query: string; categorySlug: string }> | undefined) ?? [];81 const pages = Number(this.meta.config.pagesPerSeed ?? 1);82 const cap = ctx.options.limit;83 let count = 0;84 const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;85 for (let i = startSeed; i < seeds.length; i++) {86 const seed = seeds[i]!;87 for (let page = 1; page <= pages; page++) {88 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;89 const url = `${BASE}/search?${seed.query}${page > 1 ? `&page=${page}` : ''}`;90 await this.throttle();91 const res = await ctx.fetch(url, {92 // Firecrawl only: Scrapfly's rendered fallback costs ~40 credits/page here and adds nothing.93 engines: ['firecrawl'],94 minQuality: 0.3, // pages where every card is sold out (no price) are still valid catalog data95 expect: ['title', 'price'],96 parse: (r) => {97 const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null;98 return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price || x.marketplacePrice) ? 1 : null } : null;99 },100 });101 const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null;102 if (!payload?.items.length) {103 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`);104 break;105 }106 count++;107 yield { url, externalId: `${seed.query}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };108 }109 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });110 }111 }112113 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {114 const p = PagePayloadSchema.parse(raw.payload);115 const out: NormalizedRecord[] = [];116 for (const c of p.items) {117 const release = c.releaseDate ? parseSourceDate(c.releaseDate) : null;118 const year = release ? release.getUTCFullYear() : null;119 const isCard = ['pokemon', 'yugioh', 'one_piece_card_game'].includes(p.categorySlug);120 const card = isCard ? parseJpCardTitle(c.title) : null;121 const attributes = AssetAttributesSchema.parse({122 categorySlug: p.categorySlug,123 brand: c.brand,124 franchise: c.brand,125 name: card?.name ?? c.title,126 number: card?.number ?? null,127 rarity: card?.rarity ?? null,128 year,129 language: 'Japanese',130 region: 'JP',131 originalMsrp: c.listPrice,132 originalMsrpCurrency: c.listPrice ? 'JPY' : null,133 identifiers: { surugaya_id: c.id },134 metadata: { type_label: c.typeLabel, release: c.releaseDate },135 });136 const rawTitle = `${c.title}${c.brand ? ` [${c.brand}]` : ''}${year ? ` (${year})` : ''}`;137 const grade = { grader: card?.grader ?? null, grade: card?.grade ?? null, qualifier: null, certificationNumber: null };138 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };139 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: release }));140 const price = c.price ?? c.marketplacePrice;141 if (price) {142 const viaMarketplace = c.price === null && c.marketplacePrice !== null;143 out.push(144 NormalizedListingSchema.parse({145 kind: 'listing',146 ...base,147 externalId: viaMarketplace ? `${c.id}:mp` : c.id,148 confidence: 0.8,149 listingType: 'fixed_price',150 price,151 currency: 'JPY',152 seller: viaMarketplace ? 'Suruga-ya marketplace seller' : 'Suruga-ya',153 location: 'Japan',154 condition: { condition: c.used || viaMarketplace ? null : 'mint_in_box', conditionRaw: viaMarketplace ? '中古 (marketplace)' : c.used ? '中古' : '新品', completeness: null },155 availability: 'available',156 }),157 );158 } else if (c.soldOut) {159 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.7, listingType: 'fixed_price', price: null, currency: 'JPY', seller: 'Suruga-ya', location: 'Japan', availability: 'sold' }));160 }161 }162 return out;163 }164}165166export default function createConnector(meta: ConnectorMeta) {167 return new SurugayaConnector(meta);168}169