TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';45const BASE = 'https://www.1999.co.jp';6const PARSER_VERSION = '1.0.0';78export const CardSchema = z.object({9 id: z.string(),10 url: z.string(),11 title: z.string(),12 image: z.string().nullable(),13 price: z.number().nullable(),14 listPrice: z.number().nullable(),15 stock: z.string().nullable(),16 released: z.string().nullable(),17});18export type Card = z.infer<typeof CardSchema>;19export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) });20export type PagePayload = z.infer<typeof PagePayloadSchema>;2122export function jpy(s: string | null | undefined): number | null {23 const m = s?.replace(/,/g, '').match(/(\d+)\s*JPY/i) ?? s?.replace(/,/g, '').match(/(\d+)/);24 return m ? Number(m[1]) : null;25}2627export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {28 const $ = H.load(htmlText);29 const items: Card[] = [];30 $('.c-product-list__item').each((_, el) => {31 const e = $(el);32 const a = e.find('a[href*="1999.co.jp/eng/"]').filter((__, x) => /\/eng\/\d{5,9}$/.test($(x).attr('href') ?? '')).first();33 const url = a.attr('href');34 const id = url?.match(/\/eng\/(\d+)$/)?.[1];35 if (!url || !id) return;36 const img = e.find('img').first();37 const title = (img.attr('alt') || img.attr('title') || H.text(a) || '').replace(/\s+/g, ' ').trim();38 if (!title) return;39 const image = img.attr('src') ?? img.attr('data-src') ?? null;40 const price = jpy(H.text(e.find('.c-card__price-element').first()));41 const listPrice = jpy(H.text(e.find('.c-card__price-proper').first()));42 const text = e.text().replace(/\s+/g, ' ');43 const stock = text.match(/(In Stock|Sold Out|Pre-Order|Back-?order|Order Stop|Reservation)/i)?.[1] ?? null;44 const released = text.match(/((?:Early|Mid|Late)\s+[A-Z][a-z]{2}\.?,?\s+\d{4}|[A-Z][a-z]{2}\.?,?\s+\d{4})\s+Released/)?.[1] ?? null;45 items.push({ id, url, title, image: image ? (image.startsWith('http') ? image : BASE + image) : null, price, listPrice, stock, released });46 });47 return { kind: 'search_page', url: pageUrl, categorySlug, items };48}4950const GUNDAM_RE = /gundam|gunpla|\bHG(UC|CE|AC|BF|IBO|GTO|BD)?\b|\bMG\b|\bRG\b|\bPG\b|\bMGEX\b|\bSDCS\b|zaku|zeta|char's|unicorn|barbatos|strike freedom|nu\b/i;5152export class HobbySearchConnector extends BaseConnector {53 readonly version = '1.0.0';54 readonly parserVersion = PARSER_VERSION;55 protected override minIntervalMs = 2000;56 override readonly urlPatterns = [/^https?:\/\/(www\.)?1999\.co\.jp\/eng\/(\d{5,9})/i];5758 private searchUrl(key: string, page: number): string {59 return `${BASE}/eng/search?typ1_c=100&cat=&state=&sold=0&sortid=7&searchkey=${encodeURIComponent(key)}${page > 1 ? `&page=${page}` : ''}`;60 }6162 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {63 const seeds = (this.meta.config.seeds as Array<{ key: string; categorySlug: string }> | undefined) ?? [];64 const pages = Number(this.meta.config.pagesPerSeed ?? 1);65 const cap = ctx.options.limit;66 let count = 0;67 const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;68 for (let i = startSeed; i < seeds.length; i++) {69 const seed = seeds[i]!;70 for (let page = 1; page <= pages; page++) {71 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;72 const url = this.searchUrl(seed.key, page);73 await this.throttle();74 const res = await ctx.fetch(url, {75 engines: ['firecrawl', 'scrapfly'],76 expect: ['title', 'price'],77 parse: (r) => {78 const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null;79 return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;80 },81 });82 const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null;83 if (!payload?.items.length) {84 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`);85 break;86 }87 count++;88 yield { url, externalId: `${seed.key}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };89 }90 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });91 }92 }9394 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {95 const p = PagePayloadSchema.parse(raw.payload);96 const out: NormalizedRecord[] = [];97 for (const c of p.items) {98 const isGundam = p.categorySlug === 'gundam' && GUNDAM_RE.test(c.title);99 const categorySlug = p.categorySlug === 'gundam' ? (isGundam ? 'gundam' : 'action_figures') : p.categorySlug;100 const scale = c.title.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null;101 const year = c.released?.match(/(\d{4})/)?.[1] ? Number(c.released.match(/(\d{4})/)![1]) : null;102 const name = c.title.replace(/\s*\((Plastic model|Completed|Figure|Action Figure|PVC Figure)\)\s*$/i, '').trim();103 const attributes = AssetAttributesSchema.parse({104 categorySlug,105 brand: isGundam ? 'Bandai' : null,106 franchise: isGundam ? 'Gundam' : null,107 name,108 year,109 size: scale,110 region: 'JP',111 originalMsrp: c.listPrice ?? c.price,112 originalMsrpCurrency: c.listPrice ?? c.price ? 'JPY' : null,113 identifiers: { hobbysearch_id: c.id },114 metadata: { product_type: c.title.match(/\(([^)]+)\)\s*$/)?.[1] ?? null, released: c.released, stock: c.stock },115 });116 const rawTitle = `${name}${scale ? ` ${scale}` : ''}${year ? ` (${year})` : ''}`;117 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };118 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: null }));119 if (c.price) {120 const availability = /sold out|order stop/i.test(c.stock ?? '') ? 'sold' : 'available';121 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'JPY', seller: 'HobbySearch', location: 'Japan', condition: { condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' }, availability }));122 }123 }124 return out;125 }126}127128export default function createConnector(meta: ConnectorMeta) {129 return new HobbySearchConnector(meta);130}131