import { z } from 'zod'; import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle } from '../_lib/shared.js'; import { HTML_HEADERS, cleanText, jpVariant, splitCardNumber, yen } from '../_g1-cards-eu-jp-lib/index.js'; /** * Yuyu-tei (遊々亭): per-expansion sell pages listing every single with rarity, number, JPY price * and stock. One raw record per expansion page; normalize emits a catalog item and a listing per card. */ const SITE = 'https://yuyu-tei.jp'; const PARSER_VERSION = '1.0.0'; const GameCfgSchema = z.object({ slug: z.string(), name: z.string(), franchise: z.string().nullable().optional(), brand: z.string().nullable().optional() }); export type GameCfg = z.infer; export const ItemSchema = z.object({ cardId: z.string(), url: z.string(), name: z.string(), number: z.string().nullable(), rarity: z.string().nullable(), priceJpy: z.number().nullable(), stock: z.number().int().nullable(), image: z.string().nullable(), kizu: z.boolean(), }); export type YuyuItem = z.infer; const RawPayloadSchema = z.object({ game: z.string(), gameCfg: GameCfgSchema, set: z.object({ code: z.string(), name: z.string() }), items: z.array(ItemSchema) }); export type YuyuPayload = z.infer; /** Expansion list from a game top page: + . */ export function parseVersions(doc: string): Array<{ code: string; name: string }> { const $ = html.load(doc); const out: Array<{ code: string; name: string }> = []; const seen = new Set(); $('input[name="vers[]"]').each((_, el) => { const code = $(el).attr('value')?.trim(); if (!code || seen.has(code)) return; const id = $(el).attr('id'); const label = cleanText((id ? $(`label[for="${id}"]`).first().text() : '') || $(el).next('label').text()) ?? code; seen.add(code); out.push({ code, name: label }); }); return out; } /** Parse a sell page (/sell//s/): rarity sections → .card-product blocks. */ export function parseSellPage(doc: string): { setName: string | null; items: YuyuItem[] } { const $ = html.load(doc); const title = cleanText($('title').first().text()); const setName = title ? title.split('|')[0]!.trim() || null : null; const items: YuyuItem[] = []; $('.cards-list').each((_, section) => { const rarity = cleanText($(section).find('h3 span').first().text()); $(section) .find('.card-product') .each((__, el) => { const $el = $(el); const a = $el.find('a[href*="/card/"]').first(); const href = a.attr('href'); if (!href) return; const cardId = $el.find('.cart_cid').attr('value')?.trim() || href.split('/').pop() || ''; const name = cleanText($el.find('h4').first().text()); if (!name || !cardId) return; const number = cleanText($el.find('span.border').first().text()); const priceText = cleanText($el.find('strong').first().text()); const stockText = cleanText($el.find('.cart_sell_zaiko').first().text()) ?? ''; const stockM = stockText.replace(/[,,]/g, '').match(/(\d+)\s*点/); const stock = stockM ? Number(stockM[1]) : /在庫\s*:?\s*なし|売り切れ|SOLD/i.test(stockText) ? 0 : null; const img = $el.find('img.card').attr('src') ?? null; const kizu = ($el.find('.cart_kizu').attr('value') ?? '0') !== '0'; items.push({ cardId, url: href.startsWith('http') ? href : `${SITE}${href}`, name, number: number && number !== '-' ? number : null, rarity: rarity && rarity !== '-' ? rarity : null, priceJpy: yen(priceText), stock, image: img, kizu }); }); }); return { setName, items }; } export class YuyuTeiConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 4000; private games(): Array<[string, GameCfg]> { const cfg = (this.meta.config.games ?? {}) as Record; return Object.entries(cfg).map(([code, c]) => [code, GameCfgSchema.parse(c)] as [string, GameCfg]); } private async page(ctx: CrawlContext, url: string) { await this.throttle(url); return ctx.fetch(url, { engines: ['api'], headers: HTML_HEADERS, responseType: 'text', minQuality: 0 }); } async *crawl(ctx: CrawlContext): AsyncIterable { let games = this.games(); if (ctx.options.seeds?.length) games = games.filter(([code]) => ctx.options.seeds!.some((s) => s === code || s.startsWith(`${code}:`))); const backfill = ctx.options.mode === 'backfill'; const perGame = backfill ? Infinity : Number(this.meta.config.setsPerGame ?? 6); let gameIdx = Number(ctx.options.cursor?.gameIdx ?? 0); let setIdx = Number(ctx.options.cursor?.setIdx ?? 0); let count = 0; for (; gameIdx < games.length; gameIdx++, setIdx = 0) { const [game, gameCfg] = games[gameIdx]!; const topUrl = `${SITE}/top/${game}`; const top = await this.page(ctx, topUrl); if (!top.success || !top.html) { ctx.anomaly('page_fetch_failed', `${topUrl}: ${top.error ?? top.httpStatus}`); continue; } let versions = parseVersions(top.html); if (!versions.length) { ctx.anomaly('selector_missing', `${topUrl}: no vers[] checkboxes`); continue; } const wanted = ctx.options.seeds?.filter((s) => s.startsWith(`${game}:`)).map((s) => s.slice(game.length + 1)) ?? []; if (wanted.length) versions = versions.filter((v) => wanted.includes(v.code)); versions = versions.slice(0, Number.isFinite(perGame) ? perGame : versions.length); for (; setIdx < versions.length; setIdx++) { if (ctx.signal?.aborted) return; if (this.reached(ctx, count)) { await ctx.setCursor({ gameIdx, setIdx }); return; } const v = versions[setIdx]!; const url = `${SITE}/sell/${game}/s/${v.code}`; const res = await this.page(ctx, url); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); continue; } const { setName, items } = parseSellPage(res.html); if (!items.length) { ctx.anomaly('parse_failure_page', `${url}: no .card-product blocks`); continue; } const noPrice = items.filter((i) => i.priceJpy === null).length; if (noPrice > items.length / 2) ctx.anomaly('price_parse_failure', `${url}: ${noPrice}/${items.length} items without price`); count++; const payload: YuyuPayload = { game, gameCfg, set: { code: v.code, name: setName ?? v.name }, items }; yield { url, externalId: `${game}:${v.code}`, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ gameIdx, setIdx: setIdx + 1 }); await ctx.progress({ page: setIdx + 1, totalPages: versions.length, itemsProcessed: count }); } } await ctx.setCursor({ gameIdx: 0, setIdx: 0, completedAt: new Date().toISOString(), done: true }); } async normalize(raw: RawRecordLike): Promise { const p = RawPayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const observedAt = raw.fetchedAt; // "[M6] 拡張パック ストームエメラルダ | シングルカード販売 | …" → code "M6", name "拡張パック ストームエメラルダ" const sm = p.set.name.match(/^\[([^\]]+)\]\s*(.+?)(?:\s*シングルカード販売)?$/); const setCode = sm?.[1] ?? p.set.code.toUpperCase(); const setName = sm?.[2]?.trim() || p.set.name; const seen = new Set(); for (const it of p.items) { if (seen.has(it.cardId)) continue; seen.add(it.cardId); const { number, total } = splitCardNumber(it.number); const variant = jpVariant([], it.name); const name = it.name.replace(/[((][^()()]*(パラレル|ミラー)[^()()]*[))]/g, '').replace(/\s+/g, ' ').trim() || it.name; const a = attrs({ categorySlug: p.gameCfg.slug, franchise: p.gameCfg.franchise ?? null, brand: p.gameCfg.brand ?? null, set: setName, setCode, name, number, variant, language: 'Japanese', rarity: it.rarity, identifiers: { yuyutei_card: `${p.game}/${p.set.code}/${it.cardId}` }, metadata: { game: p.gameCfg.name, total, yuyutei_version: p.set.code, damaged: it.kizu }, }); const rawTitle = makeTitle({ name: it.name, set: setName, number, total, variant }); const images = it.image ? [it.image.replace('/100_140/', '/front/')] : []; out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: `card:${p.game}:${p.set.code}:${it.cardId}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION, releaseDate: null })); if (it.priceJpy === null) continue; out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: `${p.game}:${p.set.code}:${it.cardId}${it.kizu ? ':kizu' : ''}`, rawTitle, imageUrls: images, attributes: a, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: it.kizu ? 'キズあり' : null, completeness: null }, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.priceJpy, currency: 'JPY', seller: 'Yuyu-tei', location: 'JP', quantity: it.stock, availability: it.stock === 0 ? 'ended' : 'available', }), ); } return out; } } export default (meta: ConnectorMeta) => new YuyuTeiConnector(meta);