spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { lotAttributes, makeCatalogItem, makeListing, money } from '../../api/_memorabilia-lib/index.js';5import { md, splitMarkdownItems } from '../_carlib/index.js';67const BASE = 'https://www.bigbadtoystore.com';8const PARSER_VERSION = '1.0.0';910export const ItemSchema = z.object({ productId: z.string(), variation: z.string().nullable(), title: z.string(), url: z.string(), image: z.string().nullable(), brand: z.string().nullable(), status: z.string().nullable(), price: z.number().nullable(), listPrice: z.number().nullable() });11export const PayloadSchema = z.object({ kind: z.literal('search_page'), query: z.string(), page: z.number(), items: z.array(ItemSchema) });12export type Payload = z.infer<typeof PayloadSchema>;1314/** Firecrawl markdown of a search/department page → product cards. */15export function parseSearchMarkdown(markdown: string, query: string, page: number): Payload {16  const start = markdown.indexOf('## Product Results');17  const body = start >= 0 ? markdown.slice(start) : markdown;18  const chunks = splitMarkdownItems(body, /^- !\[/m);19  const items: Payload['items'] = [];20  for (const c of chunks) {21    const link = c.match(/###\s*\[([^\]]+)\]\((https:\/\/www\.bigbadtoystore\.com\/product\/[^)\s]+)\)/);22    if (!link) continue;23    const url = link[2]!;24    const productId = url.match(/-(\d+)(?:\?|$)/)?.[1] ?? url.match(/product\/[^/]*?(\d+)/)?.[1];25    if (!productId) continue;26    const prices = [...c.matchAll(/\$([\d,]+\.\d{2})/g)].map((m) => money(`$${m[1]}`, 'USD')?.amount ?? null).filter((n): n is number => n !== null);27    const status = c.match(/\b(PRE-ORDER|IN STOCK|SOLD OUT|WAITLIST|BACKORDER|COMING SOON|LOW STOCK)\b/i)?.[1]?.toUpperCase() ?? null;28    items.push({29      productId,30      variation: url.match(/variation=(\d+)/)?.[1] ?? null,31      title: md.clean(link[1]!),32      url: url.split('?')[0]!,33      image: md.image(c),34      brand: c.match(/By:\s*([^\n]+)/)?.[1]?.trim() ?? null,35      status,36      price: prices.length ? Math.min(...prices) : null,37      listPrice: prices.length > 1 ? Math.max(...prices) : null,38    });39  }40  return { kind: 'search_page', query, page, items };41}4243export function bbtsCategory(title: string, brand: string | null): string {44  const t = `${title} ${brand ?? ''}`.toLowerCase();45  if (/\bfunko\b|\bpop!\b|\bsoda\b.*funko/.test(t)) return 'funko';46  if (/\b(gundam|gunpla|zaku|master grade|real grade|high grade|perfect grade|\bhg\b|\bmg\b|\brg\b|\bpg\b)\b/.test(t)) return 'gundam';47  if (/\b(bearbrick|be@rbrick|medicom|kaws|pop mart|labubu|kidrobot|superplastic|mighty jaxx|designer toy|vinyl figure|art toy)\b/.test(t)) return 'designer_toys';48  if (/\b(lego)\b/.test(t)) return 'lego_sets';49  if (/\b(plush|plushie)\b/.test(t)) return 'plush';50  if (/\b(model kit|1\/24|1\/18 scale|diecast|die-cast)\b/.test(t)) return 'model_cars';51  return 'action_figures';52}5354/**55 * BigBadToyStore (major US collectibles retailer). Search/department pages rendered through56 * Firecrawl (plain HTTP gets a bot challenge; not bypassed): title, brand line, stock status and57 * price per product. Emits catalog items (retail reference) and retailer listings.58 */59export class BbtsConnector extends BaseConnector {60  readonly version = '1.0.0';61  readonly parserVersion = PARSER_VERSION;62  protected override minIntervalMs = 2000;6364  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {65    const seeds = (this.meta.config.queries as string[] | undefined) ?? ['hot toys'];66    const pages = Number(this.meta.config.pagesPerQuery ?? 1);67    const perRun = Number(this.meta.config.queriesPerRun ?? 4);68    const start = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length;69    let count = 0;70    for (let k = 0; k < Math.min(perRun, seeds.length); k++) {71      const q = seeds[(start + k) % seeds.length]!;72      for (let page = 1; page <= pages; page++) {73        if (ctx.signal?.aborted || this.reached(ctx, count)) break;74        const url = `${BASE}/Search?SearchText=${encodeURIComponent(q)}&PageSize=50&SortOrder=NewAndPopular${page > 1 ? `&PageIndex=${page}` : ''}`;75        await this.throttle();76        const res = await ctx.fetch(url, { engines: ['firecrawl'], expect: ['title', 'price', 'status'], parse: (r) => { const p = r.markdown ? parseSearchMarkdown(r.markdown, q, page).items[0] : null; return p ? { title: p.title, price: p.price, status: p.status } : null; } });77        if (!res.success || !res.markdown) {78          ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);79          break;80        }81        const payload = parseSearchMarkdown(res.markdown, q, page);82        if (payload.items.length === 0) break;83        count++;84        yield { url, externalId: `${q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };85        if (payload.items.length < 50) break;86      }87    }88    await ctx.setCursor({ seedIndex: (start + perRun) % seeds.length, updatedAt: new Date().toISOString() });89  }9091  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {92    const p = PayloadSchema.parse(raw.payload);93    const out: NormalizedRecord[] = [];94    for (const it of p.items) {95      const slug = bbtsCategory(it.title, it.brand);96      const attributes = lotAttributes({ categorySlug: slug, name: it.title, brand: it.brand, identifiers: { bbts_product_id: it.productId }, metadata: { status: it.status, list_price: it.listPrice, variation: it.variation, query: p.query } });97      if (it.listPrice && it.status && /PRE-ORDER|IN STOCK/.test(it.status)) attributes.originalMsrp = it.listPrice, (attributes.originalMsrpCurrency = 'USD');98      const common = { meta: this.meta, sourceUrl: it.url, externalId: it.productId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' };99      out.push(makeCatalogItem({ ...common, confidence: 0.8 }));100      if (it.price !== null && it.price > 0) {101        const availability = it.status === 'SOLD OUT' ? 'ended' : it.status === 'WAITLIST' ? 'unknown' : 'available';102        out.push(makeListing({ ...common, price: it.price, currency: 'USD', listingType: 'fixed_price', seller: 'BigBadToyStore', location: 'US', availability, quantity: null }));103      }104    }105    return out;106  }107}108109export default (meta: ConnectorMeta) => new BbtsConnector(meta);110