import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, isoDateOnly, posNumber, refineCategory, unixToDate, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js'; /** * Ruten (露天拍賣, Taiwan) — public search JSON (ids) + public item JSON (title, TWD price, stock, sold count, dates). * Asking prices → `listing`; `sold_num` is kept as metadata only (aggregate, not dated transactions). */ const SEARCH = 'https://rtapi.ruten.com.tw/api/search/v3/index.php/core/prod'; const ITEMS = 'https://rapi.ruten.com.tw/api/items/v2/list'; const SITE = 'https://www.ruten.com.tw'; const PARSER_VERSION = '1.0.0'; const PAGE_SIZE = 30; export const ItemSchema = z.object({ id: z.string(), name: z.string(), class: z.string().nullable(), currency: z.string().nullable(), price: z.number(), priceMin: z.number().nullable(), priceMax: z.number().nullable(), soldNum: z.number().nullable(), watchNum: z.number().nullable(), stockStatus: z.number().nullable(), available: z.boolean().nullable(), /** unix seconds */ postTime: z.number().nullable(), /** "2026-07-16" */ updateTime: z.string().nullable(), storeName: z.string().nullable(), image: z.string().nullable(), mode: z.string().nullable(), saleEndTime: z.number().nullable(), translatedName: z.string().nullable(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, offset: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) }); export type PagePayload = z.infer; const SearchResponse = z.object({ TotalRows: z.number().optional(), Rows: z.array(z.object({ Id: z.string() })).default([]) }); const ItemsResponse = z.object({ status: z.string().optional(), data: z.array(z.record(z.string(), z.unknown())).default([]) }); /** Map one raw rapi item object onto the compact payload shape. */ export function toItem(raw: Record): Item | null { const id = raw.id !== undefined ? String(raw.id) : null; const name = typeof raw.name === 'string' ? raw.name : null; const price = posNumber(raw.goods_price ?? raw.selling_g_now_price); if (!id || !name || price === null) return null; const range = (raw.goods_price_range ?? {}) as { min?: unknown; max?: unknown }; const images = (raw.images ?? {}) as { url?: unknown }; const img = Array.isArray(images.url) ? (images.url[0] as string | undefined) ?? null : null; return { id, name, class: typeof raw.class === 'string' ? raw.class : null, currency: typeof raw.currency === 'string' ? raw.currency : null, price, priceMin: posNumber(range.min), priceMax: posNumber(range.max), soldNum: intOrNull(raw.sold_num), watchNum: intOrNull(raw.watch_num), stockStatus: intOrNull(raw.stock_status), available: typeof raw.available === 'boolean' ? raw.available : null, postTime: intOrNull(raw.post_time), updateTime: typeof raw.update_time === 'string' ? raw.update_time : null, storeName: typeof raw.store_name === 'string' && raw.store_name.trim() ? raw.store_name.trim() : null, image: img, mode: typeof raw.mode === 'string' ? raw.mode : null, saleEndTime: intOrNull(raw.sale_end_time), translatedName: typeof raw.translated_name === 'string' ? raw.translated_name : null, }; } export function searchUrl(q: string, offset: number, limit = PAGE_SIZE, sort = 'rnk/dc'): string { return `${SEARCH}?q=${encodeURIComponent(q)}&type=direct&sort=${encodeURIComponent(sort)}&offset=${offset}&limit=${limit}`; } export function itemsUrl(ids: string[]): string { return `${ITEMS}?gno=${ids.join(',')}&level=simple`; } export class RutenConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; override readonly urlPatterns = [/^https?:\/\/(?:www\.)?ruten\.com\.tw\/item\/show\?(\d{8,})/i, /^https?:\/\/(?:www\.)?ruten\.com\.tw\/item\/(\d{8,})/i]; private seeds(ctx: CrawlContext): KeywordSeed[] { if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => KeywordSeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') })); const seeds = z.array(KeywordSeedSchema).parse(this.meta.config.seeds ?? []); const filter = ctx.options.categories; return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds; } private async fetchItems(ctx: CrawlContext, ids: string[]): Promise<{ items: Item[]; res: Awaited> }> { const url = itemsUrl(ids); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price', 'currency'], parse: (r) => { const parsed = ItemsResponse.safeParse(r.json); const first = parsed.success ? parsed.data.data.map(toItem).find(Boolean) : null; return first ? { title: first.name, price: first.price, currency: first.currency } : null; } }); const parsed = ItemsResponse.safeParse(res.json); const items = parsed.success ? parsed.data.data.map(toItem).filter((x): x is Item => Boolean(x)) : []; return { items, res }; } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = this.seeds(ctx); const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 2); const sort = String(this.meta.config.sort ?? 'rnk/dc'); const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number; done?: boolean }; let count = 0; for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) { const seed = seeds[si]!; let page = si === (cur.seedIndex ?? 0) && cur.page ? cur.page : 1; for (; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const offset = (page - 1) * PAGE_SIZE + 1; const sUrl = searchUrl(seed.q, offset, PAGE_SIZE, sort); await this.throttle(sUrl); const sRes = await ctx.fetch(sUrl, { engines: ['api'], responseType: 'json', minQuality: 0 }); const search = SearchResponse.safeParse(sRes.json); if (!sRes.success || !search.success) { ctx.anomaly(sRes.success ? 'schema_drift' : 'page_fetch_failed', `${sUrl}: ${sRes.error ?? sRes.httpStatus}`); break; } const ids = search.data.Rows.map((r) => r.Id); if (!ids.length) break; const { items, res } = await this.fetchItems(ctx, ids); if (!items.length) { ctx.anomaly(res.success ? 'parse_failure_page' : 'page_fetch_failed', `${itemsUrl(ids.slice(0, 3))}…: ${res.error ?? res.httpStatus}`); break; } const payload: PagePayload = { kind: 'search_page', seed, offset, total: search.data.TotalRows ?? null, items }; count++; yield { url: sUrl, externalId: `search:${seed.q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() }); await ctx.progress({ page, totalPages: payload.total ? Math.min(pages, Math.ceil(payload.total / PAGE_SIZE)) : null, itemsProcessed: count }); if (ids.length < PAGE_SIZE || (payload.total !== null && offset - 1 + ids.length >= payload.total)) break; } await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() }); } await ctx.setCursor({ done: true, at: new Date().toISOString() }); } async lookup(url: string, ctx: CrawlContext): Promise { const id = this.urlPatterns.map((re) => url.match(re)?.[1]).find(Boolean); if (!id) return []; const { items, res } = await this.fetchItems(ctx, [id]); if (!items.length) return []; const seed: KeywordSeed = { q: id, category: String(this.meta.config.defaultCategory ?? 'trading_cards'), language: null }; return [{ url: `${SITE}/item/show?${id}`, externalId: `item:${id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'search_page', seed, offset: 1, total: 1, items } satisfies PagePayload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const seen = new Set(); for (const it of p.items) { if (seen.has(it.id)) continue; seen.add(it.id); const currency = (it.currency ?? 'TWD').toUpperCase(); if (currency !== 'TWD') continue; // Ruten lists in TWD; anything else is schema drift we do not guess at const title = cleanTitle(it.name); const categorySlug = p.seed.category === 'auto' ? 'trading_cards' : refineCategory(p.seed.category, title); const conditionRaw = cjkConditionRaw(title); const inStock = it.available !== false && (it.stockStatus === null || it.stockStatus > 0); const availability = inStock ? 'available' : (it.soldNum ?? 0) > 0 ? 'sold' : 'ended'; const listedAt = unixToDate(it.postTime); const saleEnd = unixToDate(it.saleEndTime); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${SITE}/item/show?${it.id}`, externalId: it.id, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes: AssetAttributesSchema.parse({ categorySlug, name: title, language: p.seed.language ?? cjkLanguage(title), country: 'TW', identifiers: { ruten_item_id: it.id }, metadata: { ruten_class: it.class, sold_count: it.soldNum, watch_count: it.watchNum, price_min: it.priceMin, price_max: it.priceMax, updated_on: it.updateTime, seed_query: p.seed.q, listing_mode: it.mode }, }), grade: { ...cjkGrade(title), certificationNumber: null }, condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.price, currency: 'TWD', seller: it.storeName, location: 'Taiwan', quantity: null, listedAt: listedAt ?? isoDateOnly(it.updateTime), // Ruten's default sale_end_time is a far-future placeholder (2037); only keep real deadlines. endsAt: saleEnd && saleEnd.getUTCFullYear() < 2036 ? saleEnd : null, availability, bidCount: null, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new RutenConnector(meta); }