TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, isoDateOnly, posNumber, refineCategory, unixToDate, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * Ruten (露天拍賣, Taiwan) — public search JSON (ids) + public item JSON (title, TWD price, stock, sold count, dates).9 * Asking prices → `listing`; `sold_num` is kept as metadata only (aggregate, not dated transactions).10 */1112const SEARCH = 'https://rtapi.ruten.com.tw/api/search/v3/index.php/core/prod';13const ITEMS = 'https://rapi.ruten.com.tw/api/items/v2/list';14const SITE = 'https://www.ruten.com.tw';15const PARSER_VERSION = '1.0.0';16const PAGE_SIZE = 30;1718export const ItemSchema = z.object({19 id: z.string(),20 name: z.string(),21 class: z.string().nullable(),22 currency: z.string().nullable(),23 price: z.number(),24 priceMin: z.number().nullable(),25 priceMax: z.number().nullable(),26 soldNum: z.number().nullable(),27 watchNum: z.number().nullable(),28 stockStatus: z.number().nullable(),29 available: z.boolean().nullable(),30 /** unix seconds */31 postTime: z.number().nullable(),32 /** "2026-07-16" */33 updateTime: z.string().nullable(),34 storeName: z.string().nullable(),35 image: z.string().nullable(),36 mode: z.string().nullable(),37 saleEndTime: z.number().nullable(),38 translatedName: z.string().nullable(),39});40export type Item = z.infer<typeof ItemSchema>;4142export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, offset: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) });43export type PagePayload = z.infer<typeof PagePayloadSchema>;4445const SearchResponse = z.object({ TotalRows: z.number().optional(), Rows: z.array(z.object({ Id: z.string() })).default([]) });46const ItemsResponse = z.object({ status: z.string().optional(), data: z.array(z.record(z.string(), z.unknown())).default([]) });4748/** Map one raw rapi item object onto the compact payload shape. */49export function toItem(raw: Record<string, unknown>): Item | null {50 const id = raw.id !== undefined ? String(raw.id) : null;51 const name = typeof raw.name === 'string' ? raw.name : null;52 const price = posNumber(raw.goods_price ?? raw.selling_g_now_price);53 if (!id || !name || price === null) return null;54 const range = (raw.goods_price_range ?? {}) as { min?: unknown; max?: unknown };55 const images = (raw.images ?? {}) as { url?: unknown };56 const img = Array.isArray(images.url) ? (images.url[0] as string | undefined) ?? null : null;57 return {58 id,59 name,60 class: typeof raw.class === 'string' ? raw.class : null,61 currency: typeof raw.currency === 'string' ? raw.currency : null,62 price,63 priceMin: posNumber(range.min),64 priceMax: posNumber(range.max),65 soldNum: intOrNull(raw.sold_num),66 watchNum: intOrNull(raw.watch_num),67 stockStatus: intOrNull(raw.stock_status),68 available: typeof raw.available === 'boolean' ? raw.available : null,69 postTime: intOrNull(raw.post_time),70 updateTime: typeof raw.update_time === 'string' ? raw.update_time : null,71 storeName: typeof raw.store_name === 'string' && raw.store_name.trim() ? raw.store_name.trim() : null,72 image: img,73 mode: typeof raw.mode === 'string' ? raw.mode : null,74 saleEndTime: intOrNull(raw.sale_end_time),75 translatedName: typeof raw.translated_name === 'string' ? raw.translated_name : null,76 };77}7879export function searchUrl(q: string, offset: number, limit = PAGE_SIZE, sort = 'rnk/dc'): string {80 return `${SEARCH}?q=${encodeURIComponent(q)}&type=direct&sort=${encodeURIComponent(sort)}&offset=${offset}&limit=${limit}`;81}82export function itemsUrl(ids: string[]): string {83 return `${ITEMS}?gno=${ids.join(',')}&level=simple`;84}8586export class RutenConnector extends BaseConnector {87 readonly version = '1.0.0';88 readonly parserVersion = PARSER_VERSION;89 protected override minIntervalMs = 2000;90 override readonly urlPatterns = [/^https?:\/\/(?:www\.)?ruten\.com\.tw\/item\/show\?(\d{8,})/i, /^https?:\/\/(?:www\.)?ruten\.com\.tw\/item\/(\d{8,})/i];9192 private seeds(ctx: CrawlContext): KeywordSeed[] {93 if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => KeywordSeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') }));94 const seeds = z.array(KeywordSeedSchema).parse(this.meta.config.seeds ?? []);95 const filter = ctx.options.categories;96 return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds;97 }9899 private async fetchItems(ctx: CrawlContext, ids: string[]): Promise<{ items: Item[]; res: Awaited<ReturnType<CrawlContext['fetch']>> }> {100 const url = itemsUrl(ids);101 await this.throttle(url);102 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price', 'currency'], parse: (r) => {103 const parsed = ItemsResponse.safeParse(r.json);104 const first = parsed.success ? parsed.data.data.map(toItem).find(Boolean) : null;105 return first ? { title: first.name, price: first.price, currency: first.currency } : null;106 } });107 const parsed = ItemsResponse.safeParse(res.json);108 const items = parsed.success ? parsed.data.data.map(toItem).filter((x): x is Item => Boolean(x)) : [];109 return { items, res };110 }111112 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {113 const seeds = this.seeds(ctx);114 const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 2);115 const sort = String(this.meta.config.sort ?? 'rnk/dc');116 const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number; done?: boolean };117 let count = 0;118 for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) {119 const seed = seeds[si]!;120 let page = si === (cur.seedIndex ?? 0) && cur.page ? cur.page : 1;121 for (; page <= pages; page++) {122 if (ctx.signal?.aborted || this.reached(ctx, count)) return;123 const offset = (page - 1) * PAGE_SIZE + 1;124 const sUrl = searchUrl(seed.q, offset, PAGE_SIZE, sort);125 await this.throttle(sUrl);126 const sRes = await ctx.fetch(sUrl, { engines: ['api'], responseType: 'json', minQuality: 0 });127 const search = SearchResponse.safeParse(sRes.json);128 if (!sRes.success || !search.success) {129 ctx.anomaly(sRes.success ? 'schema_drift' : 'page_fetch_failed', `${sUrl}: ${sRes.error ?? sRes.httpStatus}`);130 break;131 }132 const ids = search.data.Rows.map((r) => r.Id);133 if (!ids.length) break;134 const { items, res } = await this.fetchItems(ctx, ids);135 if (!items.length) {136 ctx.anomaly(res.success ? 'parse_failure_page' : 'page_fetch_failed', `${itemsUrl(ids.slice(0, 3))}…: ${res.error ?? res.httpStatus}`);137 break;138 }139 const payload: PagePayload = { kind: 'search_page', seed, offset, total: search.data.TotalRows ?? null, items };140 count++;141 yield { url: sUrl, externalId: `search:${seed.q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };142 await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() });143 await ctx.progress({ page, totalPages: payload.total ? Math.min(pages, Math.ceil(payload.total / PAGE_SIZE)) : null, itemsProcessed: count });144 if (ids.length < PAGE_SIZE || (payload.total !== null && offset - 1 + ids.length >= payload.total)) break;145 }146 await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() });147 }148 await ctx.setCursor({ done: true, at: new Date().toISOString() });149 }150151 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {152 const id = this.urlPatterns.map((re) => url.match(re)?.[1]).find(Boolean);153 if (!id) return [];154 const { items, res } = await this.fetchItems(ctx, [id]);155 if (!items.length) return [];156 const seed: KeywordSeed = { q: id, category: String(this.meta.config.defaultCategory ?? 'trading_cards'), language: null };157 return [{ url: `${SITE}/item/show?${id}`, externalId: `item:${id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'search_page', seed, offset: 1, total: 1, items } satisfies PagePayload, fetchedAt: res.fetchedAt }];158 }159160 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {161 const p = PagePayloadSchema.parse(raw.payload);162 const out: NormalizedRecord[] = [];163 const seen = new Set<string>();164 for (const it of p.items) {165 if (seen.has(it.id)) continue;166 seen.add(it.id);167 const currency = (it.currency ?? 'TWD').toUpperCase();168 if (currency !== 'TWD') continue; // Ruten lists in TWD; anything else is schema drift we do not guess at169 const title = cleanTitle(it.name);170 const categorySlug = p.seed.category === 'auto' ? 'trading_cards' : refineCategory(p.seed.category, title);171 const conditionRaw = cjkConditionRaw(title);172 const inStock = it.available !== false && (it.stockStatus === null || it.stockStatus > 0);173 const availability = inStock ? 'available' : (it.soldNum ?? 0) > 0 ? 'sold' : 'ended';174 const listedAt = unixToDate(it.postTime);175 const saleEnd = unixToDate(it.saleEndTime);176 out.push(177 NormalizedListingSchema.parse({178 kind: 'listing',179 connectorId: this.meta.id,180 sourceId: this.meta.sourceId,181 sourceUrl: `${SITE}/item/show?${it.id}`,182 externalId: it.id,183 rawTitle: it.name,184 imageUrls: it.image ? [it.image] : [],185 attributes: AssetAttributesSchema.parse({186 categorySlug,187 name: title,188 language: p.seed.language ?? cjkLanguage(title),189 country: 'TW',190 identifiers: { ruten_item_id: it.id },191 metadata: { ruten_class: it.class, sold_count: it.soldNum, watch_count: it.watchNum, price_min: it.priceMin, price_max: it.priceMax, updated_on: it.updateTime, seed_query: p.seed.q, listing_mode: it.mode },192 }),193 grade: { ...cjkGrade(title), certificationNumber: null },194 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },195 observedAt: raw.fetchedAt,196 confidence: 0.7,197 parserVersion: PARSER_VERSION,198 listingType: 'fixed_price',199 price: it.price,200 currency: 'TWD',201 seller: it.storeName,202 location: 'Taiwan',203 quantity: null,204 listedAt: listedAt ?? isoDateOnly(it.updateTime),205 // Ruten's default sale_end_time is a far-future placeholder (2037); only keep real deadlines.206 endsAt: saleEnd && saleEnd.getUTCFullYear() < 2036 ? saleEnd : null,207 availability,208 bidCount: null,209 }),210 );211 }212 return out;213 }214}215216export default function createConnector(meta: ConnectorMeta) {217 return new RutenConnector(meta);218}219