spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { brandFromName, compactMoney, fetchFirecrawl, stripBrand } from '../../api/_wlib/index.js';56/**7 * MPB — used camera & lens retailer (US/UK/EU storefronts). Category pages render client-side;8 * Firecrawl returns markdown cards "**Name** · N available, $min-$max" linking to the model page.9 * Prices are dealer asks for the cheapest unit of each model (range kept in metadata).10 */11const BASE = 'https://www.mpb.com';12const PARSER_VERSION = '1.0.0';1314export const ModelSchema = z.object({15  name: z.string(),16  url: z.string(),17  slug: z.string(),18  available: z.number().nullable(),19  priceMin: z.number().nullable(),20  priceMax: z.number().nullable(),21  currency: z.string(),22  image: z.string().nullable(),23});24export type Model = z.infer<typeof ModelSchema>;25export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), market: z.string(), category: z.string(), total: z.number().nullable(), models: z.array(ModelSchema) });26export type PagePayload = z.infer<typeof PagePayloadSchema>;2728const CURRENCY: Record<string, string> = { 'en-us': 'USD', 'en-uk': 'GBP', 'en-eu': 'EUR', 'de-de': 'EUR', 'fr-fr': 'EUR', 'nl-nl': 'EUR', 'es-es': 'EUR', 'it-it': 'EUR' };2930export function parseCategoryMarkdown(md: string, url: string): PagePayload {31  const market = url.match(/mpb\.com\/([a-z]{2}-[a-z]{2})\//)?.[1] ?? 'en-us';32  const category = url.match(/\/category\/(.+?)(?:[?#]|$)/)?.[1] ?? url;33  const currency = CURRENCY[market] ?? 'USD';34  const models: Model[] = [];35  const seen = new Set<string>();36  const re = /\[!\[([^\]]*)\]\(([^)\s]+)\)[^\]]*?\*\*([^*]+)\*\*[^\]]*?(\d+\+?|10\+)\s*available,\s*([$£€][\d,]+)(?:\s*-\s*([$£€][\d,]+))?\]\((https?:\/\/[^)\s]+\/product\/([a-z0-9-]+)[^)\s]*)\)/g;37  for (const m of md.matchAll(re)) {38    const slug = m[8]!;39    if (seen.has(slug)) continue;40    seen.add(slug);41    models.push({ name: m[3]!.replace(/\s+/g, ' ').trim(), url: m[7]!.split('?')[0]!, slug, available: Number(m[4]!.replace('+', '')) || null, priceMin: compactMoney(m[5]!), priceMax: m[6] ? compactMoney(m[6]) : compactMoney(m[5]!), currency, image: m[2] ?? null });42  }43  const total = md.match(/Showing\s+\d+\s+of\s+(\d+)\s+results/i)?.[1];44  return { kind: 'category_page', url, market, category, total: total ? Number(total) : null, models };45}4647export class MpbConnector extends BaseConnector {48  readonly version = '1.0.0';49  readonly parserVersion = PARSER_VERSION;50  protected override minIntervalMs = 3000;5152  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {53    const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];54    let count = 0;55    const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;56    for (let i = start; i < seeds.length; i++) {57      const seed = seeds[i]!;58      if (ctx.signal?.aborted || this.reached(ctx, count)) return;59      const url = seed.startsWith('http') ? seed : `${BASE}${seed}`;60      await this.throttle();61      const res = await fetchFirecrawl(ctx, url, { timeoutMs: 90_000, waitForMs: 2500, parse: (r) => (r.markdown ? parseCategoryMarkdown(r.markdown, url).models.length : 0) });62      const payload = res.success && res.markdown ? parseCategoryMarkdown(res.markdown, url) : null;63      if (!payload?.models.length) {64        ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no model cards'}`);65        continue;66      }67      count++;68      yield { url, externalId: `${payload.market}:${payload.category}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };69      await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });70    }71  }7273  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {74    const p = PagePayloadSchema.parse(raw.payload);75    const out: NormalizedRecord[] = [];76    for (const m of p.models) {77      if (!m.priceMin) continue;78      const brand = brandFromName(m.name);79      const attributes = AssetAttributesSchema.parse({80        categorySlug: 'cameras',81        brand,82        model: stripBrand(m.name, brand),83        name: m.name,84        identifiers: { mpb_model: m.slug },85        metadata: { units_available: m.available, price_max: m.priceMax, market: p.market, mpb_category: p.category },86      });87      out.push(88        NormalizedListingSchema.parse({89          kind: 'listing',90          connectorId: this.meta.id,91          sourceId: this.meta.sourceId,92          sourceUrl: m.url,93          externalId: `${p.market}:${m.slug}`,94          rawTitle: m.name,95          imageUrls: m.image ? [m.image] : [],96          attributes,97          condition: { condition: null, conditionRaw: 'MPB graded per unit (Like New / Excellent / Good / Well Used) — cheapest unit shown', completeness: null },98          observedAt: raw.fetchedAt,99          confidence: 0.75,100          parserVersion: PARSER_VERSION,101          listingType: 'fixed_price',102          price: m.priceMin,103          currency: m.currency,104          seller: 'MPB',105          location: p.market.split('-')[1]?.toUpperCase() === 'UK' ? 'GB' : p.market.split('-')[1]?.toUpperCase() ?? null,106          quantity: m.available,107          availability: 'available',108        }),109      );110    }111    return out;112  }113}114115export default function createConnector(meta: ConnectorMeta) {116  return new MpbConnector(meta);117}118