import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; const BASE = 'https://www.1999.co.jp'; const PARSER_VERSION = '1.0.0'; export const CardSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), image: z.string().nullable(), price: z.number().nullable(), listPrice: z.number().nullable(), stock: z.string().nullable(), released: z.string().nullable(), }); export type Card = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) }); export type PagePayload = z.infer; export function jpy(s: string | null | undefined): number | null { const m = s?.replace(/,/g, '').match(/(\d+)\s*JPY/i) ?? s?.replace(/,/g, '').match(/(\d+)/); return m ? Number(m[1]) : null; } export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload { const $ = H.load(htmlText); const items: Card[] = []; $('.c-product-list__item').each((_, el) => { const e = $(el); const a = e.find('a[href*="1999.co.jp/eng/"]').filter((__, x) => /\/eng\/\d{5,9}$/.test($(x).attr('href') ?? '')).first(); const url = a.attr('href'); const id = url?.match(/\/eng\/(\d+)$/)?.[1]; if (!url || !id) return; const img = e.find('img').first(); const title = (img.attr('alt') || img.attr('title') || H.text(a) || '').replace(/\s+/g, ' ').trim(); if (!title) return; const image = img.attr('src') ?? img.attr('data-src') ?? null; const price = jpy(H.text(e.find('.c-card__price-element').first())); const listPrice = jpy(H.text(e.find('.c-card__price-proper').first())); const text = e.text().replace(/\s+/g, ' '); const stock = text.match(/(In Stock|Sold Out|Pre-Order|Back-?order|Order Stop|Reservation)/i)?.[1] ?? null; const released = text.match(/((?:Early|Mid|Late)\s+[A-Z][a-z]{2}\.?,?\s+\d{4}|[A-Z][a-z]{2}\.?,?\s+\d{4})\s+Released/)?.[1] ?? null; items.push({ id, url, title, image: image ? (image.startsWith('http') ? image : BASE + image) : null, price, listPrice, stock, released }); }); return { kind: 'search_page', url: pageUrl, categorySlug, items }; } const GUNDAM_RE = /gundam|gunpla|\bHG(UC|CE|AC|BF|IBO|GTO|BD)?\b|\bMG\b|\bRG\b|\bPG\b|\bMGEX\b|\bSDCS\b|zaku|zeta|char's|unicorn|barbatos|strike freedom|nu\b/i; export class HobbySearchConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; override readonly urlPatterns = [/^https?:\/\/(www\.)?1999\.co\.jp\/eng\/(\d{5,9})/i]; private searchUrl(key: string, page: number): string { return `${BASE}/eng/search?typ1_c=100&cat=&state=&sold=0&sortid=7&searchkey=${encodeURIComponent(key)}${page > 1 ? `&page=${page}` : ''}`; } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.seeds as Array<{ key: string; categorySlug: string }> | undefined) ?? []; const pages = Number(this.meta.config.pagesPerSeed ?? 1); const cap = ctx.options.limit; let count = 0; const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = startSeed; i < seeds.length; i++) { const seed = seeds[i]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return; const url = this.searchUrl(seed.key, page); await this.throttle(); const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], expect: ['title', 'price'], parse: (r) => { const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null; return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null; }, }); const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null; if (!payload?.items.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`); break; } count++; yield { url, externalId: `${seed.key}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const c of p.items) { const isGundam = p.categorySlug === 'gundam' && GUNDAM_RE.test(c.title); const categorySlug = p.categorySlug === 'gundam' ? (isGundam ? 'gundam' : 'action_figures') : p.categorySlug; const scale = c.title.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null; const year = c.released?.match(/(\d{4})/)?.[1] ? Number(c.released.match(/(\d{4})/)![1]) : null; const name = c.title.replace(/\s*\((Plastic model|Completed|Figure|Action Figure|PVC Figure)\)\s*$/i, '').trim(); const attributes = AssetAttributesSchema.parse({ categorySlug, brand: isGundam ? 'Bandai' : null, franchise: isGundam ? 'Gundam' : null, name, year, size: scale, region: 'JP', originalMsrp: c.listPrice ?? c.price, originalMsrpCurrency: c.listPrice ?? c.price ? 'JPY' : null, identifiers: { hobbysearch_id: c.id }, metadata: { product_type: c.title.match(/\(([^)]+)\)\s*$/)?.[1] ?? null, released: c.released, stock: c.stock }, }); const rawTitle = `${name}${scale ? ` ${scale}` : ''}${year ? ` (${year})` : ''}`; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: null })); if (c.price) { const availability = /sold out|order stop/i.test(c.stock ?? '') ? 'sold' : 'available'; out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'JPY', seller: 'HobbySearch', location: 'Japan', condition: { condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' }, availability })); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new HobbySearchConnector(meta); }