import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { lotAttributes, makeCatalogItem, makeListing, money } from '../../api/_memorabilia-lib/index.js'; import { md, splitMarkdownItems } from '../_carlib/index.js'; const BASE = 'https://www.bigbadtoystore.com'; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ productId: z.string(), variation: z.string().nullable(), title: z.string(), url: z.string(), image: z.string().nullable(), brand: z.string().nullable(), status: z.string().nullable(), price: z.number().nullable(), listPrice: z.number().nullable() }); export const PayloadSchema = z.object({ kind: z.literal('search_page'), query: z.string(), page: z.number(), items: z.array(ItemSchema) }); export type Payload = z.infer; /** Firecrawl markdown of a search/department page → product cards. */ export function parseSearchMarkdown(markdown: string, query: string, page: number): Payload { const start = markdown.indexOf('## Product Results'); const body = start >= 0 ? markdown.slice(start) : markdown; const chunks = splitMarkdownItems(body, /^- !\[/m); const items: Payload['items'] = []; for (const c of chunks) { const link = c.match(/###\s*\[([^\]]+)\]\((https:\/\/www\.bigbadtoystore\.com\/product\/[^)\s]+)\)/); if (!link) continue; const url = link[2]!; const productId = url.match(/-(\d+)(?:\?|$)/)?.[1] ?? url.match(/product\/[^/]*?(\d+)/)?.[1]; if (!productId) continue; const prices = [...c.matchAll(/\$([\d,]+\.\d{2})/g)].map((m) => money(`$${m[1]}`, 'USD')?.amount ?? null).filter((n): n is number => n !== null); const status = c.match(/\b(PRE-ORDER|IN STOCK|SOLD OUT|WAITLIST|BACKORDER|COMING SOON|LOW STOCK)\b/i)?.[1]?.toUpperCase() ?? null; items.push({ productId, variation: url.match(/variation=(\d+)/)?.[1] ?? null, title: md.clean(link[1]!), url: url.split('?')[0]!, image: md.image(c), brand: c.match(/By:\s*([^\n]+)/)?.[1]?.trim() ?? null, status, price: prices.length ? Math.min(...prices) : null, listPrice: prices.length > 1 ? Math.max(...prices) : null, }); } return { kind: 'search_page', query, page, items }; } export function bbtsCategory(title: string, brand: string | null): string { const t = `${title} ${brand ?? ''}`.toLowerCase(); if (/\bfunko\b|\bpop!\b|\bsoda\b.*funko/.test(t)) return 'funko'; if (/\b(gundam|gunpla|zaku|master grade|real grade|high grade|perfect grade|\bhg\b|\bmg\b|\brg\b|\bpg\b)\b/.test(t)) return 'gundam'; if (/\b(bearbrick|be@rbrick|medicom|kaws|pop mart|labubu|kidrobot|superplastic|mighty jaxx|designer toy|vinyl figure|art toy)\b/.test(t)) return 'designer_toys'; if (/\b(lego)\b/.test(t)) return 'lego_sets'; if (/\b(plush|plushie)\b/.test(t)) return 'plush'; if (/\b(model kit|1\/24|1\/18 scale|diecast|die-cast)\b/.test(t)) return 'model_cars'; return 'action_figures'; } /** * BigBadToyStore (major US collectibles retailer). Search/department pages rendered through * Firecrawl (plain HTTP gets a bot challenge; not bypassed): title, brand line, stock status and * price per product. Emits catalog items (retail reference) and retailer listings. */ export class BbtsConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.queries as string[] | undefined) ?? ['hot toys']; const pages = Number(this.meta.config.pagesPerQuery ?? 1); const perRun = Number(this.meta.config.queriesPerRun ?? 4); const start = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length; let count = 0; for (let k = 0; k < Math.min(perRun, seeds.length); k++) { const q = seeds[(start + k) % seeds.length]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}/Search?SearchText=${encodeURIComponent(q)}&PageSize=50&SortOrder=NewAndPopular${page > 1 ? `&PageIndex=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['firecrawl'], expect: ['title', 'price', 'status'], parse: (r) => { const p = r.markdown ? parseSearchMarkdown(r.markdown, q, page).items[0] : null; return p ? { title: p.title, price: p.price, status: p.status } : null; } }); if (!res.success || !res.markdown) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parseSearchMarkdown(res.markdown, q, page); if (payload.items.length === 0) break; count++; yield { url, externalId: `${q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (payload.items.length < 50) break; } } await ctx.setCursor({ seedIndex: (start + perRun) % seeds.length, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const it of p.items) { const slug = bbtsCategory(it.title, it.brand); const attributes = lotAttributes({ categorySlug: slug, name: it.title, brand: it.brand, identifiers: { bbts_product_id: it.productId }, metadata: { status: it.status, list_price: it.listPrice, variation: it.variation, query: p.query } }); if (it.listPrice && it.status && /PRE-ORDER|IN STOCK/.test(it.status)) attributes.originalMsrp = it.listPrice, (attributes.originalMsrpCurrency = 'USD'); const common = { meta: this.meta, sourceUrl: it.url, externalId: it.productId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' }; out.push(makeCatalogItem({ ...common, confidence: 0.8 })); if (it.price !== null && it.price > 0) { const availability = it.status === 'SOLD OUT' ? 'ended' : it.status === 'WAITLIST' ? 'unknown' : 'available'; out.push(makeListing({ ...common, price: it.price, currency: 'USD', listingType: 'fixed_price', seller: 'BigBadToyStore', location: 'US', availability, quantity: null })); } } return out; } } export default (meta: ConnectorMeta) => new BbtsConnector(meta);