TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';5import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, parseJstDateTime, posNumber, refineCategory, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * Yahoo! Auctions Japan (ヤフオク!) — live listings from the public keyword search (server-rendered `pageData`9 * JSON) and recently closed lots from the public closed search (Next.js `__NEXT_DATA__`). JPY native.10 * Live rows → `listing` (auction or fixed price), closed rows with ≥ 1 bid → `sale` (hammer, no buyer premium).11 * Historical closed results beyond the site's own window are covered by the `aucfree` connector (same12 * `yahoo_auction_id` identifier, so entity resolution merges them).13 */1415const BASE = 'https://auctions.yahoo.co.jp';16const PARSER_VERSION = '1.0.0';17const PAGE_SIZE = 50; // Yahoo's default; the `n=` parameter is disallowed by robots.txt so we never send it.1819const SeedSchema = KeywordSeedSchema.extend({ auccat: z.string().nullable().default(null) });20type Seed = z.infer<typeof SeedSchema>;2122export const LiveRowSchema = z.object({23 id: z.string(),24 title: z.string(),25 categoryId: z.string().nullable(),26 /** current price (JPY) */27 price: z.number(),28 /** buy-now price (JPY); null when none */29 buyNow: z.number().nullable(),30 bids: z.number().nullable(),31 /** "2026-09-09 18:32:00" JST wall clock as published */32 endTime: z.string().nullable(),33 image: z.string().nullable(),34 isFlea: z.boolean().nullable().default(null),35 isStore: z.boolean().nullable().default(null),36 startTime: z.string().nullable().default(null),37 isClosed: z.boolean().nullable().default(null),38 hasWinner: z.boolean().nullable().default(null),39 /** "未使用" (unused) badge shown on the card */40 isUnused: z.boolean().nullable().default(null),41 categoryPath: z.array(z.string()).default([]),42});43export type LiveRow = z.infer<typeof LiveRowSchema>;4445export const ClosedRowSchema = z.object({46 id: z.string(),47 title: z.string(),48 categoryId: z.string().nullable(),49 categoryPath: z.array(z.string()),50 /** final price (JPY) */51 price: z.number(),52 buyNow: z.number().nullable(),53 bids: z.number(),54 /** ISO 8601 with offset, e.g. "2026-09-08T17:07:22+09:00" */55 endTime: z.string(),56 image: z.string().nullable(),57 isFixedPrice: z.boolean().nullable(),58 itemCondition: z.string().nullable(),59 isFleamarketItem: z.boolean().nullable(),60 brandId: z.number().nullable(),61});62export type ClosedRow = z.infer<typeof ClosedRowSchema>;6364export const PagePayloadSchema = z.discriminatedUnion('kind', [65 z.object({ kind: z.literal('search_page'), url: z.string(), seed: SeedSchema, offset: z.number(), total: z.number().nullable(), rows: z.array(LiveRowSchema) }),66 z.object({ kind: z.literal('closed_page'), url: z.string(), seed: SeedSchema, offset: z.number(), total: z.number().nullable(), rows: z.array(ClosedRowSchema) }),67 z.object({ kind: z.literal('item_page'), url: z.string(), seed: SeedSchema, offset: z.number(), total: z.number().nullable(), rows: z.array(LiveRowSchema) }),68]);69export type PagePayload = z.infer<typeof PagePayloadSchema>;70export type ItemPagePayload = Extract<PagePayload, { kind: 'item_page' }>;71export type SearchPagePayload = Extract<PagePayload, { kind: 'search_page' }>;72export type ClosedPagePayload = Extract<PagePayload, { kind: 'closed_page' }>;7374interface PageDataItem { productID?: string; productName?: string; productCategoryID?: string; price?: string | number; winPrice?: string | number; bids?: string | number; endtime?: string; starttime?: string; isStore?: string; isClosed?: string; hasWinner?: string }7576function str(v: unknown): string | null {77 return v === null || v === undefined || v === '' ? null : String(v);78}7980/** "1,100,000円" → 1100000 */81function yen(s: string | null | undefined): number | null {82 if (!s) return null;83 const n = Number(s.replace(/[^\d]/g, ''));84 return Number.isFinite(n) && n > 0 ? n : null;85}8687/**88 * Live search / category page: one `li.Product` card per lot. The card anchors carry the id, title, category,89 * image, current price, buy-now price and end time (unix) as data attributes; the visible price block gives the90 * displayed 現在 (current) and 即決 (buy-now, tax included for store sellers) prices; `dd.Product__bid` the bid count.91 * The small `pageData` JSON (3 featured items) is used only to fill start times when present.92 */93export function parseSearchPage(htmlText: string, url: string, seed: Seed, offset: number): SearchPagePayload {94 const data = H.inlineJson(htmlText, 'pageData') as { items?: PageDataItem[] } | null;95 const featured = new Map<string, PageDataItem>();96 for (const it of data?.items ?? []) if (it.productID) featured.set(String(it.productID), it);97 const $ = H.load(htmlText);98 const rows: LiveRow[] = [];99 const seen = new Set<string>();100 $('li.Product').each((_, li) => {101 const $li = $(li);102 const link = $li.find('a.Product__titleLink').first();103 const imageLink = $li.find('a.Product__imageLink').first();104 const bonus = $li.find('.Product__bonus').first();105 const id = link.attr('data-auction-id') ?? imageLink.attr('data-auction-id') ?? link.attr('href')?.match(/\/auction\/([a-z]?\d+)/)?.[1] ?? null;106 const title = H.text(link) ?? imageLink.attr('data-auction-title') ?? null;107 if (!id || !title || seen.has(id)) return;108 let current: number | null = null;109 let buyNow: number | null = null;110 $li.find('.Product__price').each((__, p) => {111 const label = H.text($(p).find('.Product__label').first()) ?? '';112 const value = yen(H.text($(p).find('.Product__priceValue').first()));113 if (/即決/.test(label)) buyNow = value;114 else if (/現在|落札/.test(label) || current === null) current = value;115 });116 const price = current ?? yen(link.attr('data-auction-price') ?? imageLink.attr('data-auction-price'));117 if (price === null) return;118 if (buyNow === null) buyNow = yen(bonus.attr('data-auction-buynowprice'));119 const endUnix = bonus.attr('data-auction-endtime');120 const feat = featured.get(id);121 const endTime = endUnix && /^\d+$/.test(endUnix) ? new Date(Number(endUnix) * 1000).toISOString() : str(feat?.endtime);122 const flea = imageLink.attr('data-auction-isflea') ?? link.attr('data-auction-isflea');123 const catPath = (bonus.attr('data-auction-categoryidpath') ?? '').split(',').filter(Boolean);124 seen.add(id);125 rows.push({126 id,127 title,128 categoryId: link.attr('data-auction-category') ?? imageLink.attr('data-auction-category') ?? null,129 price,130 buyNow,131 bids: intOrNull(H.text($li.find('dd.Product__bid').first())) ?? intOrNull(feat?.bids),132 endTime,133 image: imageLink.attr('data-auction-img') ?? $li.find('img.Product__imageData').attr('src') ?? null,134 isFlea: flea === undefined ? null : flea === '1',135 isStore: feat?.isStore === undefined ? null : feat.isStore === '1',136 startTime: str(feat?.starttime),137 isClosed: null,138 hasWinner: null,139 isUnused: $li.find('.Product__icon--unused').length > 0,140 categoryPath: catPath,141 });142 });143 const total = htmlText.match(/(\d[\d,]*)件/)?.[1] ?? null;144 return { kind: 'search_page', url, seed, offset, total: total ? Number(total.replace(/,/g, '')) : null, rows };145}146147/** Single item page: `pageData.items` is one object. */148export function parseItemPage(htmlText: string, url: string, seed: Seed): ItemPagePayload | null {149 const data = H.inlineJson(htmlText, 'pageData') as { items?: PageDataItem } | null;150 const it = data?.items;151 const id = str(it?.productID);152 const title = str(it?.productName);153 const price = posNumber(it?.price);154 if (!it || !id || !title || price === null) return null;155 const image = H.load(htmlText)('meta[property="og:image"]').attr('content') ?? null;156 const row: LiveRow = { id, title, categoryId: str(it.productCategoryID), price, buyNow: posNumber(it.winPrice), bids: intOrNull(it.bids), endTime: str(it.endtime), image, isFlea: null, isStore: it.isStore === undefined ? null : it.isStore === '1', startTime: str(it.starttime), isClosed: it.isClosed === undefined ? null : it.isClosed === '1', hasWinner: it.hasWinner === undefined ? null : it.hasWinner === '1', isUnused: null, categoryPath: [] };157 return { kind: 'item_page', url, seed, offset: 1, total: 1, rows: [row] };158}159160interface ClosedItem { auctionId?: string; title?: string; price?: number; buyNowPrice?: number | null; bidCount?: number; endTime?: string; imageUrl?: string; isFixedPrice?: boolean; itemCondition?: string; isFleamarketItem?: boolean; brandId?: number | null; category?: { id?: number }; categoryPath?: Array<{ name?: string }> }161162function findKey(o: unknown, key: string, depth = 0): unknown {163 if (depth > 8 || !o || typeof o !== 'object') return undefined;164 if (key in (o as Record<string, unknown>)) return (o as Record<string, unknown>)[key];165 for (const v of Object.values(o as Record<string, unknown>)) {166 const r = findKey(v, key, depth + 1);167 if (r !== undefined) return r;168 }169 return undefined;170}171172/** Closed search page: Next.js state → initialState.search.items.listing.items[] (+ totalResultsAvailable). */173export function parseClosedPage(htmlText: string, url: string, seed: Seed, offset: number): ClosedPagePayload {174 const next = H.nextData(htmlText) as { props?: { pageProps?: { initialState?: { search?: { items?: { listing?: { items?: ClosedItem[]; totalResultsAvailable?: number } } } } } } } | null;175 const listing = next?.props?.pageProps?.initialState?.search?.items?.listing;176 const items = listing?.items ?? ((findKey(next?.props?.pageProps, 'items') as ClosedItem[] | undefined) ?? []);177 const rows: ClosedRow[] = [];178 for (const it of Array.isArray(items) ? items : []) {179 const id = str(it.auctionId);180 const title = str(it.title);181 const price = posNumber(it.price);182 if (!id || !title || price === null || !it.endTime) continue;183 rows.push({ id, title, categoryId: it.category?.id !== undefined ? String(it.category.id) : null, categoryPath: (it.categoryPath ?? []).map((c) => c.name ?? '').filter(Boolean), price, buyNow: posNumber(it.buyNowPrice), bids: intOrNull(it.bidCount) ?? 0, endTime: it.endTime, image: it.imageUrl ?? null, isFixedPrice: it.isFixedPrice ?? null, itemCondition: it.itemCondition ?? null, isFleamarketItem: it.isFleamarketItem ?? null, brandId: it.brandId ?? null });184 }185 const total = listing?.totalResultsAvailable ?? (findKey(next?.props?.pageProps, 'totalResultsAvailable') as number | undefined) ?? null;186 return { kind: 'closed_page', url, seed, offset, total: typeof total === 'number' ? total : null, rows };187}188189/** Only `p`, `auccat` and `b` are ever sent — robots.txt disallows `n=`, `mode=`, `s1=`, `o1=`, `min=`/`max=` filters. */190export function searchUrl(seed: Seed, offset: number): string {191 const q = `p=${encodeURIComponent(seed.q)}${seed.auccat ? `&auccat=${encodeURIComponent(seed.auccat)}` : ''}${offset > 1 ? `&b=${offset}` : ''}`;192 return `${BASE}/search/search?${q}`;193}194export function closedUrl(seed: Seed, offset: number): string {195 const q = `p=${encodeURIComponent(seed.q)}${seed.auccat ? `&auccat=${encodeURIComponent(seed.auccat)}` : ''}${offset > 1 ? `&b=${offset}` : ''}`;196 return `${BASE}/closedsearch/closedsearch?${q}`;197}198199type Phase = 'live' | 'closed';200interface Cursor { seedIndex?: number; phase?: Phase; page?: number; done?: boolean }201202export class YahooAuctionsJpConnector extends BaseConnector {203 readonly version = '1.0.0';204 readonly parserVersion = PARSER_VERSION;205 protected override minIntervalMs = 4000;206 override readonly urlPatterns = [/^https?:\/\/(?:page\.|www\.)?auctions\.yahoo\.co\.jp\/jp\/auction\/([a-z]?\d+)/i];207208 private seeds(ctx: CrawlContext): Seed[] {209 if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => SeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') }));210 const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);211 const filter = ctx.options.categories;212 return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds;213 }214215 private async fetchPage(ctx: CrawlContext, seed: Seed, phase: Phase, page: number): Promise<{ payload: PagePayload | null; res: Awaited<ReturnType<CrawlContext['fetch']>>; url: string }> {216 const offset = (page - 1) * PAGE_SIZE + 1;217 const url = phase === 'live' ? searchUrl(seed, offset) : closedUrl(seed, offset);218 await this.throttle(url);219 const res = await ctx.fetch(url, {220 responseType: 'text',221 headers: { 'accept-language': 'ja,en;q=0.8' },222 expect: ['title', 'price', 'date', 'status'],223 parse: (r) => {224 if (!r.html) return null;225 const p = phase === 'live' ? parseSearchPage(r.html, url, seed, offset) : parseClosedPage(r.html, url, seed, offset);226 const row = p.rows[0];227 return row ? { title: row.title, price: row.price, date: row.endTime, status: phase } : p.total === 0 ? { title: 'empty', price: 1, date: 'none', status: 'empty' } : null;228 },229 });230 if (!res.success || !res.html) return { payload: null, res, url };231 return { payload: phase === 'live' ? parseSearchPage(res.html, url, seed, offset) : parseClosedPage(res.html, url, seed, offset), res, url };232 }233234 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {235 const seeds = this.seeds(ctx);236 const backfill = ctx.options.mode === 'backfill';237 const livePages = Number(this.meta.config.pagesPerSeed ?? 2);238 const closedPages = backfill ? this.policy.backfillMaxPages : Number(this.meta.config.closedPagesPerSeed ?? 1);239 const includeClosed = backfill || Boolean(this.meta.config.includeClosed ?? true);240 const cur = (ctx.options.cursor ?? {}) as Cursor;241 if (cur.done && backfill) return;242 let count = 0;243 const startSeed = cur.seedIndex ?? 0;244 for (let si = startSeed; si < seeds.length; si++) {245 const seed = seeds[si]!;246 const phases: Phase[] = backfill ? ['closed'] : includeClosed ? ['live', 'closed'] : ['live'];247 for (const phase of phases) {248 if (si === startSeed && cur.phase && cur.phase !== phase && phases.indexOf(cur.phase) > phases.indexOf(phase)) continue;249 const maxPages = phase === 'live' ? livePages : closedPages;250 let page = si === startSeed && cur.phase === phase && cur.page ? cur.page : 1;251 for (; page <= maxPages; page++) {252 if (ctx.signal?.aborted || this.reached(ctx, count)) return;253 const { payload, res, url } = await this.fetchPage(ctx, seed, phase, page);254 if (!payload) {255 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);256 break;257 }258 if (payload.rows.length === 0) {259 if (page === 1 && payload.total !== 0) ctx.anomaly('parse_failure_page', `${url}: no rows parsed (total=${payload.total})`);260 break;261 }262 count++;263 yield { url, externalId: `${phase}:${seed.q}${seed.auccat ? `@${seed.auccat}` : ''}:${page}`, kind: phase === 'live' ? 'listing' : 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };264 await ctx.setCursor({ seedIndex: si, phase, page: page + 1, at: new Date().toISOString() });265 if (backfill) {266 const oldest = payload.kind === 'closed_page' ? payload.rows.map((r) => parseJstDateTime(r.endTime)).filter((d): d is Date => Boolean(d)).sort((a, b) => a.getTime() - b.getTime())[0] ?? null : null;267 await ctx.progress({ page, totalPages: payload.total ? Math.min(maxPages, Math.ceil(payload.total / PAGE_SIZE)) : null, itemsProcessed: count, reachedDate: oldest });268 }269 if (payload.total !== null && (page - 1) * PAGE_SIZE + payload.rows.length >= payload.total) break;270 if (payload.rows.length < PAGE_SIZE) break;271 }272 }273 await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() });274 }275 await ctx.setCursor({ done: true, at: new Date().toISOString() });276 }277278 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {279 const id = url.match(this.urlPatterns[0]!)?.[1];280 if (!id) return [];281 const target = `${BASE}/jp/auction/${id}`;282 await this.throttle(target);283 const res = await ctx.fetch(target, { responseType: 'text', headers: { 'accept-language': 'ja,en;q=0.8' }, minQuality: 0.2 });284 if (!res.success || !res.html) return [];285 const seed: Seed = { q: id, category: String(this.meta.config.defaultCategory ?? 'trading_cards'), language: null, auccat: null };286 const payload = parseItemPage(res.html, target, seed);287 if (!payload) return [];288 const row = payload.rows[0]!;289 const sold = row.isClosed === true && row.hasWinner === true;290 return [{ url: target, externalId: `item:${id}`, kind: sold ? 'sale' : 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];291 }292293 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {294 const p = PagePayloadSchema.parse(raw.payload);295 const out: NormalizedRecord[] = [];296 if (p.kind === 'closed_page') {297 for (const r of p.rows) {298 if (r.bids < 1) continue; // ended without a winning bid → not a transaction299 const saleDate = parseJstDateTime(r.endTime);300 if (!saleDate) continue;301 const title = cleanTitle(r.title);302 const categorySlug = refineCategory(p.seed.category, title);303 const conditionRaw = r.itemCondition === 'NEW' ? 'New' : r.itemCondition === 'USED' ? 'Used' : cjkConditionRaw(title);304 out.push(305 NormalizedSaleSchema.parse({306 kind: 'sale',307 connectorId: this.meta.id,308 sourceId: this.meta.sourceId,309 sourceUrl: `${BASE}/jp/auction/${r.id}`,310 externalId: r.id,311 rawTitle: r.title,312 imageUrls: r.image ? [r.image] : [],313 attributes: this.attributes(categorySlug, title, r.id, p.seed, { yahoo_category_id: r.categoryId, category_path: r.categoryPath, buy_now_price: r.buyNow, bids: r.bids, flea_market: r.isFleamarketItem, seed_query: p.seed.q }),314 grade: { ...cjkGrade(title), certificationNumber: null },315 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },316 observedAt: raw.fetchedAt,317 confidence: 0.7,318 parserVersion: PARSER_VERSION,319 saleType: r.isFixedPrice ? 'fixed_price' : 'auction',320 saleDate,321 price: r.price,322 currency: 'JPY',323 buyerPremiumIncluded: false,324 quantity: 1,325 isBundle: isCjkBundle(title),326 location: 'Japan',327 auctionHouse: 'Yahoo! Auctions Japan',328 lotNumber: r.id,329 }),330 );331 }332 return out;333 }334 const liveRows: LiveRow[] = p.rows;335 for (const r of liveRows) {336 const title = cleanTitle(r.title);337 const categorySlug = refineCategory(p.seed.category, title);338 const endsAt = parseJstDateTime(r.endTime);339 const fixed = r.buyNow !== null && r.buyNow === r.price && (r.bids ?? 0) === 0;340 const closed = r.isClosed === true;341 const conditionRaw = r.isUnused ? 'New' : cjkConditionRaw(title);342 if (closed && r.hasWinner === true && endsAt) {343 out.push(344 NormalizedSaleSchema.parse({345 kind: 'sale',346 connectorId: this.meta.id,347 sourceId: this.meta.sourceId,348 sourceUrl: `${BASE}/jp/auction/${r.id}`,349 externalId: r.id,350 rawTitle: r.title,351 imageUrls: r.image ? [r.image] : [],352 attributes: this.attributes(categorySlug, title, r.id, p.seed, { yahoo_category_id: r.categoryId, buy_now_price: r.buyNow, bids: r.bids, seed_query: p.seed.q }),353 grade: { ...cjkGrade(title), certificationNumber: null },354 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },355 observedAt: raw.fetchedAt,356 confidence: 0.7,357 parserVersion: PARSER_VERSION,358 saleType: fixed ? 'fixed_price' : 'auction',359 saleDate: endsAt,360 price: r.price,361 currency: 'JPY',362 buyerPremiumIncluded: false,363 quantity: 1,364 isBundle: isCjkBundle(title),365 location: 'Japan',366 auctionHouse: 'Yahoo! Auctions Japan',367 lotNumber: r.id,368 }),369 );370 continue;371 }372 out.push(373 NormalizedListingSchema.parse({374 kind: 'listing',375 connectorId: this.meta.id,376 sourceId: this.meta.sourceId,377 sourceUrl: `${BASE}/jp/auction/${r.id}`,378 externalId: r.id,379 rawTitle: r.title,380 imageUrls: r.image ? [r.image] : [],381 attributes: this.attributes(categorySlug, title, r.id, p.seed, { yahoo_category_id: r.categoryId, category_id_path: r.categoryPath, buy_now_price: r.buyNow, flea_market: r.isFlea, store_seller: r.isStore, seed_query: p.seed.q }),382 grade: { ...cjkGrade(title), certificationNumber: null },383 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },384 observedAt: raw.fetchedAt,385 confidence: 0.7,386 parserVersion: PARSER_VERSION,387 listingType: fixed ? 'fixed_price' : 'auction',388 price: r.price,389 currency: 'JPY',390 seller: null,391 location: 'Japan',392 quantity: 1,393 listedAt: parseJstDateTime(r.startTime),394 endsAt,395 availability: closed ? 'ended' : 'available',396 bidCount: r.bids,397 }),398 );399 }400 return out;401 }402403 private attributes(categorySlug: string, title: string, id: string, seed: KeywordSeed, metadata: Record<string, unknown>) {404 return AssetAttributesSchema.parse({405 categorySlug,406 name: title,407 language: seed.language ?? cjkLanguage(title),408 country: 'JP',409 identifiers: { yahoo_auction_id: id },410 metadata,411 });412 }413}414415export default function createConnector(meta: ConnectorMeta) {416 return new YahooAuctionsJpConnector(meta);417}418