TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, isoDateOnly, posNumber, refineCategory, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * Yahoo! Shopping Japan — itemSearch V3 (Yahoo! JAPAN Web API) — GATED: needs a Yahoo! JAPAN application id9 * (YAHOO_JP_APP_ID). JPY asking prices of Yahoo! Shopping store items → `listing`, with JAN codes when the store10 * publishes them (identifiers.jan) and the store's new/used condition flag.11 *12 * GET https://shopping.yahooapis.jp/ShoppingWebService/V3/itemSearch?appid=…&query=…&results=50&start=N[&condition=used|new]13 * → { totalResultsAvailable, totalResultsReturned, firstResultPosition, hits: [{ name, url, code, condition, price, janCode, brand, seller, genreCategory, releaseDate, image, inStock, … }] }14 */1516const ENDPOINT = 'https://shopping.yahooapis.jp/ShoppingWebService/V3/itemSearch';17const PARSER_VERSION = '1.0.0';18const RESULTS = 50;1920export const HitSchema = z.object({21 code: z.string(),22 name: z.string(),23 url: z.string(),24 price: z.number(),25 condition: z.string().nullable(),26 inStock: z.boolean().nullable(),27 janCode: z.string().nullable(),28 brand: z.string().nullable(),29 sellerId: z.string().nullable(),30 sellerName: z.string().nullable(),31 genreCategory: z.string().nullable(),32 genreCategoryId: z.number().nullable(),33 releaseDate: z.string().nullable(),34 image: z.string().nullable(),35 description: z.string().nullable(),36 headLine: z.string().nullable(),37 reviewCount: z.number().nullable(),38});39export type Hit = z.infer<typeof HitSchema>;40export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, start: z.number(), total: z.number().nullable(), hits: z.array(HitSchema) });41export type PagePayload = z.infer<typeof PagePayloadSchema>;4243const ResponseSchema = z.object({ totalResultsAvailable: z.number().optional(), totalResultsReturned: z.number().optional(), firstResultPosition: z.number().optional(), hits: z.array(z.record(z.string(), z.unknown())).default([]) });4445export function toHit(raw: Record<string, unknown>): Hit | null {46 const code = typeof raw.code === 'string' ? raw.code : null;47 const name = typeof raw.name === 'string' ? raw.name : null;48 const url = typeof raw.url === 'string' ? raw.url : null;49 const price = posNumber(raw.price);50 if (!code || !name || !url || price === null) return null;51 const brand = raw.brand as { name?: unknown } | undefined;52 const seller = raw.seller as { sellerId?: unknown; name?: unknown } | undefined;53 const genre = raw.genreCategory as { id?: unknown; name?: unknown } | undefined;54 const image = raw.image as { medium?: unknown; small?: unknown } | undefined;55 const exImage = raw.exImage as { url?: unknown } | undefined;56 const review = raw.review as { count?: unknown } | undefined;57 return {58 code,59 name,60 url,61 price,62 condition: typeof raw.condition === 'string' ? raw.condition : null,63 inStock: typeof raw.inStock === 'boolean' ? raw.inStock : null,64 janCode: typeof raw.janCode === 'string' && /^\d{8,14}$/.test(raw.janCode) ? raw.janCode : null,65 brand: typeof brand?.name === 'string' && brand.name ? brand.name : null,66 sellerId: typeof seller?.sellerId === 'string' ? seller.sellerId : null,67 sellerName: typeof seller?.name === 'string' ? seller.name : null,68 genreCategory: typeof genre?.name === 'string' ? genre.name : null,69 genreCategoryId: intOrNull(genre?.id),70 releaseDate: typeof raw.releaseDate === 'string' && raw.releaseDate ? raw.releaseDate : null,71 image: typeof exImage?.url === 'string' ? exImage.url : typeof image?.medium === 'string' ? image.medium : null,72 description: typeof raw.description === 'string' && raw.description ? raw.description.slice(0, 1500) : null,73 headLine: typeof raw.headLine === 'string' && raw.headLine ? raw.headLine : null,74 reviewCount: intOrNull(review?.count),75 };76}7778export function buildUrl(params: { appId: string; query: string; start: number; condition?: string | null; sort?: string; genreCategoryId?: string | null }): string {79 const q = new URLSearchParams({ appid: params.appId, query: params.query, results: String(RESULTS), start: String(params.start), image_size: '600', sort: params.sort ?? '-score' });80 if (params.condition) q.set('condition', params.condition);81 if (params.genreCategoryId) q.set('genre_category_id', params.genreCategoryId);82 return `${ENDPOINT}?${q.toString()}`;83}84export function redact(url: string): string {85 return url.replace(/appid=[^&]+/g, 'appid=***');86}8788const SeedSchema = KeywordSeedSchema.extend({ condition: z.enum(['new', 'used']).nullable().default(null) });89type Seed = z.infer<typeof SeedSchema>;9091export class YahooShoppingJpConnector extends BaseConnector {92 readonly version = '1.0.0';93 readonly parserVersion = PARSER_VERSION;94 protected override minIntervalMs = 1100;9596 private seeds(ctx: CrawlContext): Seed[] {97 if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => SeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') }));98 const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);99 const filter = ctx.options.categories;100 return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds;101 }102103 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {104 const appId = process.env.YAHOO_JP_APP_ID?.trim();105 if (!appId) {106 ctx.anomaly('missing_requirement', 'YAHOO_JP_APP_ID is not set');107 return;108 }109 const seeds = this.seeds(ctx);110 // start + results must stay ≤ 1000 per the API → at most 20 pages of 50111 const pages = Math.min(20, ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 2));112 const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number };113 let count = 0;114 for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) {115 const seed = seeds[si]!;116 let page = si === (cur.seedIndex ?? 0) && cur.page ? cur.page : 1;117 for (; page <= pages; page++) {118 if (ctx.signal?.aborted || this.reached(ctx, count)) return;119 const start = (page - 1) * RESULTS + 1;120 const url = buildUrl({ appId, query: seed.q, start, condition: seed.condition, sort: String(this.meta.config.sort ?? '-score') });121 await this.throttle(url);122 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => {123 const parsed = ResponseSchema.safeParse(r.json);124 const first = parsed.success ? parsed.data.hits.map(toHit).find(Boolean) : null;125 return first ? { title: first.name, price: first.price } : parsed.success && parsed.data.hits.length === 0 ? { title: 'empty', price: 1 } : null;126 } });127 const parsed = ResponseSchema.safeParse(res.json);128 if (!res.success || !parsed.success) {129 ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : res.success ? 'schema_drift' : 'page_fetch_failed', `${redact(url)}: ${res.error ?? res.httpStatus}`);130 break;131 }132 const hits = parsed.data.hits.map(toHit).filter((x): x is Hit => Boolean(x));133 if (!hits.length) break;134 const payload: PagePayload = { kind: 'search_page', seed, start, total: parsed.data.totalResultsAvailable ?? null, hits };135 count++;136 yield { url: redact(url), externalId: `search:${seed.q}${seed.condition ? `:${seed.condition}` : ''}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };137 await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() });138 await ctx.progress({ page, totalPages: payload.total ? Math.min(pages, Math.ceil(payload.total / RESULTS)) : null, itemsProcessed: count });139 if (payload.total !== null && start - 1 + hits.length >= payload.total) break;140 }141 await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() });142 }143 await ctx.setCursor({ done: true, at: new Date().toISOString() });144 }145146 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {147 const p = PagePayloadSchema.parse(raw.payload);148 const out: NormalizedRecord[] = [];149 const seen = new Set<string>();150 for (const h of p.hits) {151 if (seen.has(h.code)) continue;152 seen.add(h.code);153 const title = cleanTitle(h.name);154 const categorySlug = refineCategory(p.seed.category, `${title} ${h.genreCategory ?? ''}`);155 const conditionRaw = h.condition === 'used' ? 'Used' : h.condition === 'new' ? 'New' : cjkConditionRaw(title);156 out.push(157 NormalizedListingSchema.parse({158 kind: 'listing',159 connectorId: this.meta.id,160 sourceId: this.meta.sourceId,161 sourceUrl: h.url,162 externalId: h.code,163 rawTitle: h.name,164 description: h.description ?? h.headLine,165 imageUrls: h.image ? [h.image] : [],166 attributes: AssetAttributesSchema.parse({167 categorySlug,168 brand: h.brand,169 name: title,170 language: p.seed.language ?? cjkLanguage(title),171 country: 'JP',172 identifiers: { yahoo_shopping_code: h.code, ...(h.janCode ? { jan: h.janCode, ean: h.janCode } : {}) },173 metadata: { genre_category: h.genreCategory, genre_category_id: h.genreCategoryId, store_id: h.sellerId, release_date: h.releaseDate, review_count: h.reviewCount, is_bundle: isCjkBundle(title), seed_query: p.seed.q },174 }),175 grade: { ...cjkGrade(title), certificationNumber: null },176 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },177 observedAt: raw.fetchedAt,178 confidence: h.janCode ? 0.85 : 0.7,179 parserVersion: PARSER_VERSION,180 listingType: 'fixed_price',181 price: h.price,182 currency: 'JPY',183 seller: h.sellerName,184 location: 'Japan',185 quantity: null,186 listedAt: isoDateOnly(h.releaseDate?.replace(/\//g, '-') ?? null) ? null : null,187 endsAt: null,188 availability: h.inStock === false ? 'ended' : h.inStock === true ? 'available' : 'unknown',189 bidCount: null,190 }),191 );192 }193 return out;194 }195}196197export default function createConnector(meta: ConnectorMeta) {198 return new YahooShoppingJpConnector(meta);199}200