TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, posNumber, refineCategory, unixToDate, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * Bunjang (번개장터, Korea) — public search JSON used by m.bunjang.co.kr. KRW asking prices → `listing`.9 * Titles are Korean; PSA/BGS grades and JP/EN/KR language hints are parsed from them.10 */1112const API = 'https://api.bunjang.co.kr/api/1/find_v2.json';13const SITE = 'https://m.bunjang.co.kr';14const PARSER_VERSION = '1.0.0';15const PAGE_SIZE = 100;1617export const ItemSchema = z.object({18 pid: z.string(),19 name: z.string(),20 price: z.number(),21 image: z.string().nullable(),22 /** "0" = on sale, "1" = reserved, "3" = sold (as used by the mobile site) */23 status: z.string().nullable(),24 /** unix seconds of the last update/bump */25 updateTime: z.number().nullable(),26 /** 1 = new, 2 = used (source enum) */27 used: z.number().nullable(),28 categoryId: z.string().nullable(),29 tag: z.string().nullable(),30 location: z.string().nullable(),31 freeShipping: z.boolean().nullable(),32 bizseller: z.boolean().nullable(),33 numFaved: z.number().nullable(),34 ad: z.boolean().nullable(),35});36export type Item = z.infer<typeof ItemSchema>;37export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) });38export type PagePayload = z.infer<typeof PagePayloadSchema>;3940const Response = z.object({ result: z.string().optional(), num_found: z.number().nullable().optional(), list: z.array(z.record(z.string(), z.unknown())).default([]) });4142export function toItem(raw: Record<string, unknown>): Item | null {43 const pid = raw.pid !== undefined && raw.pid !== null ? String(raw.pid) : null;44 const name = typeof raw.name === 'string' ? raw.name : null;45 const price = posNumber(raw.price);46 if (!pid || !name || price === null) return null;47 if (raw.type !== undefined && raw.type !== 'PRODUCT') return null;48 const img = typeof raw.product_image === 'string' ? raw.product_image.replace('{res}', '600').replace('{cnt}', '1') : null;49 return {50 pid,51 name,52 price,53 image: img,54 status: raw.status === undefined || raw.status === null ? null : String(raw.status),55 updateTime: intOrNull(raw.update_time),56 used: intOrNull(raw.used),57 categoryId: raw.category_id === undefined || raw.category_id === null ? null : String(raw.category_id),58 tag: typeof raw.tag === 'string' ? raw.tag : null,59 location: typeof raw.location === 'string' && raw.location ? raw.location : null,60 freeShipping: typeof raw.free_shipping === 'boolean' ? raw.free_shipping : null,61 bizseller: typeof raw.bizseller === 'boolean' ? raw.bizseller : null,62 numFaved: intOrNull(raw.num_faved),63 ad: typeof raw.ad === 'boolean' ? raw.ad : null,64 };65}6667export function searchUrl(q: string, page: number, n = PAGE_SIZE): string {68 return `${API}?q=${encodeURIComponent(q)}&order=date&page=${page}&n=${n}&req_ref=search&stat_device=w&version=5`;69}7071export class BunjangConnector extends BaseConnector {72 readonly version = '1.0.0';73 readonly parserVersion = PARSER_VERSION;74 protected override minIntervalMs = 2500;7576 private seeds(ctx: CrawlContext): KeywordSeed[] {77 if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => KeywordSeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') }));78 const seeds = z.array(KeywordSeedSchema).parse(this.meta.config.seeds ?? []);79 const filter = ctx.options.categories;80 return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds;81 }8283 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {84 const seeds = this.seeds(ctx);85 const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 1);86 const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number };87 let count = 0;88 for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) {89 const seed = seeds[si]!;90 let page = si === (cur.seedIndex ?? 0) && cur.page !== undefined ? cur.page : 0; // Bunjang pages are 0-based91 for (; page < pages; page++) {92 if (ctx.signal?.aborted || this.reached(ctx, count)) return;93 const url = searchUrl(seed.q, page);94 await this.throttle(url);95 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => {96 const parsed = Response.safeParse(r.json);97 const first = parsed.success ? parsed.data.list.map(toItem).find(Boolean) : null;98 return first ? { title: first.name, price: first.price } : parsed.success && parsed.data.list.length === 0 ? { title: 'empty', price: 1 } : null;99 } });100 const parsed = Response.safeParse(res.json);101 if (!res.success || !parsed.success) {102 ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);103 break;104 }105 const items = parsed.data.list.map(toItem).filter((x): x is Item => Boolean(x) && !(x as Item).ad);106 if (!items.length) break;107 const payload: PagePayload = { kind: 'search_page', seed, page, total: parsed.data.num_found ?? null, items };108 count++;109 yield { url, externalId: `search:${seed.q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };110 await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() });111 await ctx.progress({ page: page + 1, totalPages: payload.total ? Math.min(pages, Math.ceil(payload.total / PAGE_SIZE)) : null, itemsProcessed: count });112 if (parsed.data.list.length < PAGE_SIZE) break;113 }114 await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() });115 }116 await ctx.setCursor({ done: true, at: new Date().toISOString() });117 }118119 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {120 const p = PagePayloadSchema.parse(raw.payload);121 const out: NormalizedRecord[] = [];122 const seen = new Set<string>();123 for (const it of p.items) {124 if (seen.has(it.pid)) continue;125 seen.add(it.pid);126 const title = cleanTitle(it.name);127 // Title first; seller hashtags (often listing several brands) only when the title alone stays at the family level.128 let categorySlug = refineCategory(p.seed.category, title);129 if (categorySlug === p.seed.category && it.tag) categorySlug = refineCategory(p.seed.category, it.tag);130 const conditionRaw = it.used === 1 ? 'New' : it.used === 2 ? 'Used' : cjkConditionRaw(title);131 const availability = it.status === '0' ? 'available' : it.status === '3' ? 'sold' : it.status === '1' ? 'available' : 'unknown';132 out.push(133 NormalizedListingSchema.parse({134 kind: 'listing',135 connectorId: this.meta.id,136 sourceId: this.meta.sourceId,137 sourceUrl: `${SITE}/products/${it.pid}`,138 externalId: it.pid,139 rawTitle: it.name,140 imageUrls: it.image ? [it.image] : [],141 attributes: AssetAttributesSchema.parse({142 categorySlug,143 name: title,144 language: p.seed.language ?? cjkLanguage(title),145 country: 'KR',146 identifiers: { bunjang_pid: it.pid },147 // update_time is the last bump/edit, not the original listing time → metadata only148 metadata: { bunjang_category_id: it.categoryId, tags: it.tag, favourites: it.numFaved, business_seller: it.bizseller, free_shipping: it.freeShipping, region: it.location, status_code: it.status, seed_query: p.seed.q, updated_at: unixToDate(it.updateTime)?.toISOString() ?? null, is_bundle: isCjkBundle(title) },149 }),150 grade: { ...cjkGrade(title), certificationNumber: null },151 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null },152 observedAt: raw.fetchedAt,153 confidence: 0.65,154 parserVersion: PARSER_VERSION,155 listingType: 'fixed_price',156 price: it.price,157 currency: 'KRW',158 seller: null,159 location: it.location ? `${it.location}, South Korea` : 'South Korea',160 quantity: null,161 listedAt: null,162 endsAt: null,163 availability,164 bidCount: null,165 }),166 );167 }168 return out;169 }170}171172export default function createConnector(meta: ConnectorMeta) {173 return new BunjangConnector(meta);174}175