TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, parseJstDateTime, posNumber, refineCategory, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * Rakuten Ichiba Item Search API (Rakuten Web Service) — GATED: needs RAKUTEN_APP_ID (+ RAKUTEN_ACCESS_KEY for9 * applications created since the 2026-07-01 API version). JPY asking prices of Rakuten Ichiba shop items → `listing`.10 *11 * Endpoint (2026-07-01): https://openapi.rakuten.co.jp/ichibams/api/IchibaItem/Search/20260701?applicationId=…&accessKey=…12 * Legacy (2022-06-01): https://app.rakuten.co.jp/services/api/IchibaItem/Search/20220601?applicationId=…13 * Both return { count, page, first, last, hits, pageCount, Items: [...] }. With formatVersion=2 each element is the14 * item object itself and image lists are plain URL arrays; with formatVersion=1 elements are wrapped as { Item: {…} }15 * and images as [{ imageUrl }]. Both shapes are accepted.16 */1718const ENDPOINT_2026 = 'https://openapi.rakuten.co.jp/ichibams/api/IchibaItem/Search/20260701';19const ENDPOINT_LEGACY = 'https://app.rakuten.co.jp/services/api/IchibaItem/Search/20220601';20const PARSER_VERSION = '1.0.0';21const HITS = 30;2223export const ItemSchema = z.object({24 itemCode: z.string(),25 itemName: z.string(),26 itemPrice: z.number(),27 itemUrl: z.string(),28 itemCaption: z.string().nullable(),29 images: z.array(z.string()),30 shopName: z.string().nullable(),31 shopCode: z.string().nullable(),32 genreId: z.string().nullable(),33 /** 1 = in stock, 0 = out of stock */34 availability: z.number().nullable(),35 /** "2026/09/01 10:00" sale window when the shop set one */36 startTime: z.string().nullable(),37 endTime: z.string().nullable(),38 reviewCount: z.number().nullable(),39 reviewAverage: z.number().nullable(),40 taxFlag: z.number().nullable(),41});42export type Item = z.infer<typeof ItemSchema>;43export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, page: z.number(), count: z.number().nullable(), pageCount: z.number().nullable(), items: z.array(ItemSchema) });44export type PagePayload = z.infer<typeof PagePayloadSchema>;4546type RawItem = Record<string, unknown>;47const ResponseSchema = z.object({ count: z.number().optional(), page: z.number().optional(), pageCount: z.number().optional(), hits: z.number().optional(), Items: z.array(z.unknown()).default([]) });4849function imageList(v: unknown): string[] {50 if (!Array.isArray(v)) return [];51 return v.map((x) => (typeof x === 'string' ? x : x && typeof x === 'object' && typeof (x as { imageUrl?: unknown }).imageUrl === 'string' ? (x as { imageUrl: string }).imageUrl : null)).filter((s): s is string => Boolean(s)).map((s) => s.replace(/\?_ex=\d+x\d+$/, ''));52}5354/** Accepts formatVersion 1 ({ Item: {…} }) and 2 (flat) elements. */55export function toItem(el: unknown): Item | null {56 if (!el || typeof el !== 'object') return null;57 const raw = ('Item' in (el as RawItem) && typeof (el as RawItem).Item === 'object' ? (el as { Item: RawItem }).Item : (el as RawItem)) ?? {};58 const code = typeof raw.itemCode === 'string' ? raw.itemCode : null;59 const name = typeof raw.itemName === 'string' ? raw.itemName : null;60 const price = posNumber(raw.itemPrice);61 const url = typeof raw.itemUrl === 'string' ? raw.itemUrl : null;62 if (!code || !name || price === null || !url) return null;63 const images = imageList(raw.mediumImageUrls).length ? imageList(raw.mediumImageUrls) : imageList(raw.smallImageUrls);64 return {65 itemCode: code,66 itemName: name,67 itemPrice: price,68 itemUrl: url,69 itemCaption: typeof raw.itemCaption === 'string' && raw.itemCaption ? raw.itemCaption.slice(0, 1500) : null,70 images: images.slice(0, 4),71 shopName: typeof raw.shopName === 'string' ? raw.shopName : null,72 shopCode: typeof raw.shopCode === 'string' ? raw.shopCode : null,73 genreId: raw.genreId === undefined || raw.genreId === null ? null : String(raw.genreId),74 availability: intOrNull(raw.availability),75 startTime: typeof raw.startTime === 'string' && raw.startTime ? raw.startTime : null,76 endTime: typeof raw.endTime === 'string' && raw.endTime ? raw.endTime : null,77 reviewCount: intOrNull(raw.reviewCount),78 reviewAverage: typeof raw.reviewAverage === 'number' ? raw.reviewAverage : posNumber(raw.reviewAverage),79 taxFlag: intOrNull(raw.taxFlag),80 };81}8283export function buildUrl(params: { appId: string; accessKey?: string | null; keyword: string; page: number; genreId?: string | null; sort?: string; legacy?: boolean }): string {84 const base = params.legacy || !params.accessKey ? ENDPOINT_LEGACY : ENDPOINT_2026;85 const q = new URLSearchParams({ applicationId: params.appId, format: 'json', formatVersion: '2', keyword: params.keyword, hits: String(HITS), page: String(params.page), sort: params.sort ?? '-updateTimestamp', imageFlag: '1' });86 if (params.accessKey && !params.legacy) q.set('accessKey', params.accessKey);87 if (params.genreId) q.set('genreId', params.genreId);88 return `${base}?${q.toString()}`;89}9091/** Strip the credentials before storing a URL. */92export function redact(url: string): string {93 return url.replace(/(applicationId|accessKey)=[^&]+/g, '$1=***');94}9596export class RakutenIchibaConnector extends BaseConnector {97 readonly version = '1.0.0';98 readonly parserVersion = PARSER_VERSION;99 protected override minIntervalMs = 1100;100101 private seeds(ctx: CrawlContext): KeywordSeed[] {102 if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => KeywordSeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') }));103 const seeds = z.array(KeywordSeedSchema).parse(this.meta.config.seeds ?? []);104 const filter = ctx.options.categories;105 return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds;106 }107108 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {109 const appId = process.env.RAKUTEN_APP_ID?.trim();110 if (!appId) {111 ctx.anomaly('missing_requirement', 'RAKUTEN_APP_ID is not set');112 return;113 }114 const accessKey = process.env.RAKUTEN_ACCESS_KEY?.trim() || null;115 const legacy = Boolean(this.meta.config.legacyEndpoint ?? !accessKey);116 const seeds = this.seeds(ctx);117 const pages = ctx.options.mode === 'backfill' ? Math.min(this.policy.backfillMaxPages, 100) : Number(this.meta.config.pagesPerSeed ?? 2);118 const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number };119 let count = 0;120 for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) {121 const seed = seeds[si]!;122 let page = si === (cur.seedIndex ?? 0) && cur.page ? cur.page : 1;123 for (; page <= pages; page++) {124 if (ctx.signal?.aborted || this.reached(ctx, count)) return;125 const url = buildUrl({ appId, accessKey, keyword: seed.q, page, sort: String(this.meta.config.sort ?? '-updateTimestamp'), legacy });126 await this.throttle(url);127 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => {128 const parsed = ResponseSchema.safeParse(r.json);129 const first = parsed.success ? parsed.data.Items.map(toItem).find(Boolean) : null;130 return first ? { title: first.itemName, price: first.itemPrice } : parsed.success && parsed.data.Items.length === 0 ? { title: 'empty', price: 1 } : null;131 } });132 const parsed = ResponseSchema.safeParse(res.json);133 if (!res.success || !parsed.success) {134 ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : res.success ? 'schema_drift' : 'page_fetch_failed', `${redact(url)}: ${res.error ?? res.httpStatus}`);135 break;136 }137 const items = parsed.data.Items.map(toItem).filter((x): x is Item => Boolean(x));138 if (!items.length) break;139 const payload: PagePayload = { kind: 'search_page', seed, page, count: parsed.data.count ?? null, pageCount: parsed.data.pageCount ?? null, items };140 count++;141 yield { url: redact(url), externalId: `search:${seed.q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };142 await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() });143 await ctx.progress({ page, totalPages: payload.pageCount, itemsProcessed: count });144 if (payload.pageCount !== null && page >= payload.pageCount) break;145 }146 await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() });147 }148 await ctx.setCursor({ done: true, at: new Date().toISOString() });149 }150151 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {152 const p = PagePayloadSchema.parse(raw.payload);153 const out: NormalizedRecord[] = [];154 const seen = new Set<string>();155 for (const it of p.items) {156 if (seen.has(it.itemCode)) continue;157 seen.add(it.itemCode);158 const title = cleanTitle(it.itemName);159 const categorySlug = refineCategory(p.seed.category, title);160 const conditionRaw = cjkConditionRaw(`${title} ${it.itemCaption ?? ''}`);161 out.push(162 NormalizedListingSchema.parse({163 kind: 'listing',164 connectorId: this.meta.id,165 sourceId: this.meta.sourceId,166 sourceUrl: it.itemUrl.split('?')[0]!,167 externalId: it.itemCode,168 rawTitle: it.itemName,169 description: it.itemCaption,170 imageUrls: it.images,171 attributes: AssetAttributesSchema.parse({172 categorySlug,173 name: title,174 language: p.seed.language ?? cjkLanguage(title),175 country: 'JP',176 identifiers: { rakuten_item_code: it.itemCode },177 metadata: { rakuten_genre_id: it.genreId, shop_code: it.shopCode, review_count: it.reviewCount, review_average: it.reviewAverage, tax_included: it.taxFlag === 0 ? true : it.taxFlag === 1 ? false : null, is_bundle: isCjkBundle(title), seed_query: p.seed.q },178 }),179 grade: { ...cjkGrade(title), certificationNumber: null },180 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },181 observedAt: raw.fetchedAt,182 confidence: 0.7,183 parserVersion: PARSER_VERSION,184 listingType: 'fixed_price',185 price: it.itemPrice,186 currency: 'JPY',187 seller: it.shopName,188 location: 'Japan',189 quantity: null,190 listedAt: parseJstDateTime(it.startTime?.replace(/\//g, '-') ?? null),191 endsAt: parseJstDateTime(it.endTime?.replace(/\//g, '-') ?? null),192 availability: it.availability === 1 ? 'available' : it.availability === 0 ? 'ended' : 'unknown',193 bidCount: null,194 }),195 );196 }197 return out;198 }199}200201export default function createConnector(meta: ConnectorMeta) {202 return new RakutenIchibaConnector(meta);203}204