import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, posNumber, refineCategory, unixToDate, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js'; /** * Bunjang (번개장터, Korea) — public search JSON used by m.bunjang.co.kr. KRW asking prices → `listing`. * Titles are Korean; PSA/BGS grades and JP/EN/KR language hints are parsed from them. */ const API = 'https://api.bunjang.co.kr/api/1/find_v2.json'; const SITE = 'https://m.bunjang.co.kr'; const PARSER_VERSION = '1.0.0'; const PAGE_SIZE = 100; export const ItemSchema = z.object({ pid: z.string(), name: z.string(), price: z.number(), image: z.string().nullable(), /** "0" = on sale, "1" = reserved, "3" = sold (as used by the mobile site) */ status: z.string().nullable(), /** unix seconds of the last update/bump */ updateTime: z.number().nullable(), /** 1 = new, 2 = used (source enum) */ used: z.number().nullable(), categoryId: z.string().nullable(), tag: z.string().nullable(), location: z.string().nullable(), freeShipping: z.boolean().nullable(), bizseller: z.boolean().nullable(), numFaved: z.number().nullable(), ad: z.boolean().nullable(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) }); export type PagePayload = z.infer; const Response = z.object({ result: z.string().optional(), num_found: z.number().nullable().optional(), list: z.array(z.record(z.string(), z.unknown())).default([]) }); export function toItem(raw: Record): Item | null { const pid = raw.pid !== undefined && raw.pid !== null ? String(raw.pid) : null; const name = typeof raw.name === 'string' ? raw.name : null; const price = posNumber(raw.price); if (!pid || !name || price === null) return null; if (raw.type !== undefined && raw.type !== 'PRODUCT') return null; const img = typeof raw.product_image === 'string' ? raw.product_image.replace('{res}', '600').replace('{cnt}', '1') : null; return { pid, name, price, image: img, status: raw.status === undefined || raw.status === null ? null : String(raw.status), updateTime: intOrNull(raw.update_time), used: intOrNull(raw.used), categoryId: raw.category_id === undefined || raw.category_id === null ? null : String(raw.category_id), tag: typeof raw.tag === 'string' ? raw.tag : null, location: typeof raw.location === 'string' && raw.location ? raw.location : null, freeShipping: typeof raw.free_shipping === 'boolean' ? raw.free_shipping : null, bizseller: typeof raw.bizseller === 'boolean' ? raw.bizseller : null, numFaved: intOrNull(raw.num_faved), ad: typeof raw.ad === 'boolean' ? raw.ad : null, }; } export function searchUrl(q: string, page: number, n = PAGE_SIZE): string { return `${API}?q=${encodeURIComponent(q)}&order=date&page=${page}&n=${n}&req_ref=search&stat_device=w&version=5`; } export class BunjangConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2500; private seeds(ctx: CrawlContext): KeywordSeed[] { if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => KeywordSeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') })); const seeds = z.array(KeywordSeedSchema).parse(this.meta.config.seeds ?? []); const filter = ctx.options.categories; return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds; } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = this.seeds(ctx); const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 1); const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number }; let count = 0; for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) { const seed = seeds[si]!; let page = si === (cur.seedIndex ?? 0) && cur.page !== undefined ? cur.page : 0; // Bunjang pages are 0-based for (; page < pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = searchUrl(seed.q, page); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => { const parsed = Response.safeParse(r.json); const first = parsed.success ? parsed.data.list.map(toItem).find(Boolean) : null; return first ? { title: first.name, price: first.price } : parsed.success && parsed.data.list.length === 0 ? { title: 'empty', price: 1 } : null; } }); const parsed = Response.safeParse(res.json); if (!res.success || !parsed.success) { ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const items = parsed.data.list.map(toItem).filter((x): x is Item => Boolean(x) && !(x as Item).ad); if (!items.length) break; const payload: PagePayload = { kind: 'search_page', seed, page, total: parsed.data.num_found ?? null, items }; count++; yield { url, externalId: `search:${seed.q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() }); await ctx.progress({ page: page + 1, totalPages: payload.total ? Math.min(pages, Math.ceil(payload.total / PAGE_SIZE)) : null, itemsProcessed: count }); if (parsed.data.list.length < PAGE_SIZE) break; } await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() }); } await ctx.setCursor({ done: true, at: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const seen = new Set(); for (const it of p.items) { if (seen.has(it.pid)) continue; seen.add(it.pid); const title = cleanTitle(it.name); // Title first; seller hashtags (often listing several brands) only when the title alone stays at the family level. let categorySlug = refineCategory(p.seed.category, title); if (categorySlug === p.seed.category && it.tag) categorySlug = refineCategory(p.seed.category, it.tag); const conditionRaw = it.used === 1 ? 'New' : it.used === 2 ? 'Used' : cjkConditionRaw(title); const availability = it.status === '0' ? 'available' : it.status === '3' ? 'sold' : it.status === '1' ? 'available' : 'unknown'; out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${SITE}/products/${it.pid}`, externalId: it.pid, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes: AssetAttributesSchema.parse({ categorySlug, name: title, language: p.seed.language ?? cjkLanguage(title), country: 'KR', identifiers: { bunjang_pid: it.pid }, // update_time is the last bump/edit, not the original listing time → metadata only metadata: { bunjang_category_id: it.categoryId, tags: it.tag, favourites: it.numFaved, business_seller: it.bizseller, free_shipping: it.freeShipping, region: it.location, status_code: it.status, seed_query: p.seed.q, updated_at: unixToDate(it.updateTime)?.toISOString() ?? null, is_bundle: isCjkBundle(title) }, }), grade: { ...cjkGrade(title), certificationNumber: null }, condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.65, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.price, currency: 'KRW', seller: null, location: it.location ? `${it.location}, South Korea` : 'South Korea', quantity: null, listedAt: null, endsAt: null, availability, bidCount: null, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new BunjangConnector(meta); }