TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle } from '../_lib/shared.js';5import { HTML_HEADERS, cleanText, jpVariant, splitCardNumber, yen } from '../_g1-cards-eu-jp-lib/index.js';67/**8 * Yuyu-tei (遊々亭): per-expansion sell pages listing every single with rarity, number, JPY price9 * and stock. One raw record per expansion page; normalize emits a catalog item and a listing per card.10 */11const SITE = 'https://yuyu-tei.jp';12const PARSER_VERSION = '1.0.0';1314const GameCfgSchema = z.object({ slug: z.string(), name: z.string(), franchise: z.string().nullable().optional(), brand: z.string().nullable().optional() });15export type GameCfg = z.infer<typeof GameCfgSchema>;1617export const ItemSchema = z.object({18 cardId: z.string(),19 url: z.string(),20 name: z.string(),21 number: z.string().nullable(),22 rarity: z.string().nullable(),23 priceJpy: z.number().nullable(),24 stock: z.number().int().nullable(),25 image: z.string().nullable(),26 kizu: z.boolean(),27});28export type YuyuItem = z.infer<typeof ItemSchema>;29const RawPayloadSchema = z.object({ game: z.string(), gameCfg: GameCfgSchema, set: z.object({ code: z.string(), name: z.string() }), items: z.array(ItemSchema) });30export type YuyuPayload = z.infer<typeof RawPayloadSchema>;3132/** Expansion list from a game top page: <input name="vers[]" value="m06"> + <label>[M6] 拡張パック …</label>. */33export function parseVersions(doc: string): Array<{ code: string; name: string }> {34 const $ = html.load(doc);35 const out: Array<{ code: string; name: string }> = [];36 const seen = new Set<string>();37 $('input[name="vers[]"]').each((_, el) => {38 const code = $(el).attr('value')?.trim();39 if (!code || seen.has(code)) return;40 const id = $(el).attr('id');41 const label = cleanText((id ? $(`label[for="${id}"]`).first().text() : '') || $(el).next('label').text()) ?? code;42 seen.add(code);43 out.push({ code, name: label });44 });45 return out;46}4748/** Parse a sell page (/sell/<game>/s/<ver>): rarity sections → .card-product blocks. */49export function parseSellPage(doc: string): { setName: string | null; items: YuyuItem[] } {50 const $ = html.load(doc);51 const title = cleanText($('title').first().text());52 const setName = title ? title.split('|')[0]!.trim() || null : null;53 const items: YuyuItem[] = [];54 $('.cards-list').each((_, section) => {55 const rarity = cleanText($(section).find('h3 span').first().text());56 $(section)57 .find('.card-product')58 .each((__, el) => {59 const $el = $(el);60 const a = $el.find('a[href*="/card/"]').first();61 const href = a.attr('href');62 if (!href) return;63 const cardId = $el.find('.cart_cid').attr('value')?.trim() || href.split('/').pop() || '';64 const name = cleanText($el.find('h4').first().text());65 if (!name || !cardId) return;66 const number = cleanText($el.find('span.border').first().text());67 const priceText = cleanText($el.find('strong').first().text());68 const stockText = cleanText($el.find('.cart_sell_zaiko').first().text()) ?? '';69 const stockM = stockText.replace(/[，,]/g, '').match(/(\d+)\s*点/);70 const stock = stockM ? Number(stockM[1]) : /在庫\s*:?\s*なし|売り切れ|SOLD/i.test(stockText) ? 0 : null;71 const img = $el.find('img.card').attr('src') ?? null;72 const kizu = ($el.find('.cart_kizu').attr('value') ?? '0') !== '0';73 items.push({ cardId, url: href.startsWith('http') ? href : `${SITE}${href}`, name, number: number && number !== '-' ? number : null, rarity: rarity && rarity !== '-' ? rarity : null, priceJpy: yen(priceText), stock, image: img, kizu });74 });75 });76 return { setName, items };77}7879export class YuyuTeiConnector extends BaseConnector {80 readonly version = '1.0.0';81 readonly parserVersion = PARSER_VERSION;82 protected override minIntervalMs = 4000;8384 private games(): Array<[string, GameCfg]> {85 const cfg = (this.meta.config.games ?? {}) as Record<string, unknown>;86 return Object.entries(cfg).map(([code, c]) => [code, GameCfgSchema.parse(c)] as [string, GameCfg]);87 }8889 private async page(ctx: CrawlContext, url: string) {90 await this.throttle(url);91 return ctx.fetch(url, { engines: ['api'], headers: HTML_HEADERS, responseType: 'text', minQuality: 0 });92 }9394 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {95 let games = this.games();96 if (ctx.options.seeds?.length) games = games.filter(([code]) => ctx.options.seeds!.some((s) => s === code || s.startsWith(`${code}:`)));97 const backfill = ctx.options.mode === 'backfill';98 const perGame = backfill ? Infinity : Number(this.meta.config.setsPerGame ?? 6);99 let gameIdx = Number(ctx.options.cursor?.gameIdx ?? 0);100 let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);101 let count = 0;102 for (; gameIdx < games.length; gameIdx++, setIdx = 0) {103 const [game, gameCfg] = games[gameIdx]!;104 const topUrl = `${SITE}/top/${game}`;105 const top = await this.page(ctx, topUrl);106 if (!top.success || !top.html) {107 ctx.anomaly('page_fetch_failed', `${topUrl}: ${top.error ?? top.httpStatus}`);108 continue;109 }110 let versions = parseVersions(top.html);111 if (!versions.length) {112 ctx.anomaly('selector_missing', `${topUrl}: no vers[] checkboxes`);113 continue;114 }115 const wanted = ctx.options.seeds?.filter((s) => s.startsWith(`${game}:`)).map((s) => s.slice(game.length + 1)) ?? [];116 if (wanted.length) versions = versions.filter((v) => wanted.includes(v.code));117 versions = versions.slice(0, Number.isFinite(perGame) ? perGame : versions.length);118 for (; setIdx < versions.length; setIdx++) {119 if (ctx.signal?.aborted) return;120 if (this.reached(ctx, count)) {121 await ctx.setCursor({ gameIdx, setIdx });122 return;123 }124 const v = versions[setIdx]!;125 const url = `${SITE}/sell/${game}/s/${v.code}`;126 const res = await this.page(ctx, url);127 if (!res.success || !res.html) {128 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);129 continue;130 }131 const { setName, items } = parseSellPage(res.html);132 if (!items.length) {133 ctx.anomaly('parse_failure_page', `${url}: no .card-product blocks`);134 continue;135 }136 const noPrice = items.filter((i) => i.priceJpy === null).length;137 if (noPrice > items.length / 2) ctx.anomaly('price_parse_failure', `${url}: ${noPrice}/${items.length} items without price`);138 count++;139 const payload: YuyuPayload = { game, gameCfg, set: { code: v.code, name: setName ?? v.name }, items };140 yield { url, externalId: `${game}:${v.code}`, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };141 await ctx.setCursor({ gameIdx, setIdx: setIdx + 1 });142 await ctx.progress({ page: setIdx + 1, totalPages: versions.length, itemsProcessed: count });143 }144 }145 await ctx.setCursor({ gameIdx: 0, setIdx: 0, completedAt: new Date().toISOString(), done: true });146 }147148 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {149 const p = RawPayloadSchema.parse(raw.payload);150 const out: NormalizedRecord[] = [];151 const observedAt = raw.fetchedAt;152 // "[M6] 拡張パック ストームエメラルダ | シングルカード販売 | …" → code "M6", name "拡張パック ストームエメラルダ"153 const sm = p.set.name.match(/^\[([^\]]+)\]\s*(.+?)(?:\s*シングルカード販売)?$/);154 const setCode = sm?.[1] ?? p.set.code.toUpperCase();155 const setName = sm?.[2]?.trim() || p.set.name;156 const seen = new Set<string>();157 for (const it of p.items) {158 if (seen.has(it.cardId)) continue;159 seen.add(it.cardId);160 const { number, total } = splitCardNumber(it.number);161 const variant = jpVariant([], it.name);162 const name = it.name.replace(/[（(][^()（）]*(パラレル|ミラー)[^()（）]*[)）]/g, '').replace(/\s+/g, ' ').trim() || it.name;163 const a = attrs({164 categorySlug: p.gameCfg.slug,165 franchise: p.gameCfg.franchise ?? null,166 brand: p.gameCfg.brand ?? null,167 set: setName,168 setCode,169 name,170 number,171 variant,172 language: 'Japanese',173 rarity: it.rarity,174 identifiers: { yuyutei_card: `${p.game}/${p.set.code}/${it.cardId}` },175 metadata: { game: p.gameCfg.name, total, yuyutei_version: p.set.code, damaged: it.kizu },176 });177 const rawTitle = makeTitle({ name: it.name, set: setName, number, total, variant });178 const images = it.image ? [it.image.replace('/100_140/', '/front/')] : [];179 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: `card:${p.game}:${p.set.code}:${it.cardId}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION, releaseDate: null }));180 if (it.priceJpy === null) continue;181 out.push(182 NormalizedListingSchema.parse({183 kind: 'listing',184 connectorId: this.meta.id,185 sourceId: this.meta.sourceId,186 sourceUrl: it.url,187 externalId: `${p.game}:${p.set.code}:${it.cardId}${it.kizu ? ':kizu' : ''}`,188 rawTitle,189 imageUrls: images,190 attributes: a,191 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },192 condition: { condition: null, conditionRaw: it.kizu ? 'キズあり' : null, completeness: null },193 observedAt,194 confidence: 0.8,195 parserVersion: PARSER_VERSION,196 listingType: 'fixed_price',197 price: it.priceJpy,198 currency: 'JPY',199 seller: 'Yuyu-tei',200 location: 'JP',201 quantity: it.stock,202 availability: it.stock === 0 ? 'ended' : 'available',203 }),204 );205 }206 return out;207 }208}209210export default (meta: ConnectorMeta) => new YuyuTeiConnector(meta);211