import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { brandFromName, compactMoney, fetchFirecrawl, stripBrand } from '../../api/_wlib/index.js'; /** * MPB — used camera & lens retailer (US/UK/EU storefronts). Category pages render client-side; * Firecrawl returns markdown cards "**Name** · N available, $min-$max" linking to the model page. * Prices are dealer asks for the cheapest unit of each model (range kept in metadata). */ const BASE = 'https://www.mpb.com'; const PARSER_VERSION = '1.0.0'; export const ModelSchema = z.object({ name: z.string(), url: z.string(), slug: z.string(), available: z.number().nullable(), priceMin: z.number().nullable(), priceMax: z.number().nullable(), currency: z.string(), image: z.string().nullable(), }); export type Model = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), market: z.string(), category: z.string(), total: z.number().nullable(), models: z.array(ModelSchema) }); export type PagePayload = z.infer; const CURRENCY: Record = { 'en-us': 'USD', 'en-uk': 'GBP', 'en-eu': 'EUR', 'de-de': 'EUR', 'fr-fr': 'EUR', 'nl-nl': 'EUR', 'es-es': 'EUR', 'it-it': 'EUR' }; export function parseCategoryMarkdown(md: string, url: string): PagePayload { const market = url.match(/mpb\.com\/([a-z]{2}-[a-z]{2})\//)?.[1] ?? 'en-us'; const category = url.match(/\/category\/(.+?)(?:[?#]|$)/)?.[1] ?? url; const currency = CURRENCY[market] ?? 'USD'; const models: Model[] = []; const seen = new Set(); const re = /\[!\[([^\]]*)\]\(([^)\s]+)\)[^\]]*?\*\*([^*]+)\*\*[^\]]*?(\d+\+?|10\+)\s*available,\s*([$£€][\d,]+)(?:\s*-\s*([$£€][\d,]+))?\]\((https?:\/\/[^)\s]+\/product\/([a-z0-9-]+)[^)\s]*)\)/g; for (const m of md.matchAll(re)) { const slug = m[8]!; if (seen.has(slug)) continue; seen.add(slug); models.push({ name: m[3]!.replace(/\s+/g, ' ').trim(), url: m[7]!.split('?')[0]!, slug, available: Number(m[4]!.replace('+', '')) || null, priceMin: compactMoney(m[5]!), priceMax: m[6] ? compactMoney(m[6]) : compactMoney(m[5]!), currency, image: m[2] ?? null }); } const total = md.match(/Showing\s+\d+\s+of\s+(\d+)\s+results/i)?.[1]; return { kind: 'category_page', url, market, category, total: total ? Number(total) : null, models }; } export class MpbConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = start; i < seeds.length; i++) { const seed = seeds[i]!; if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = seed.startsWith('http') ? seed : `${BASE}${seed}`; await this.throttle(); const res = await fetchFirecrawl(ctx, url, { timeoutMs: 90_000, waitForMs: 2500, parse: (r) => (r.markdown ? parseCategoryMarkdown(r.markdown, url).models.length : 0) }); const payload = res.success && res.markdown ? parseCategoryMarkdown(res.markdown, url) : null; if (!payload?.models.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no model cards'}`); continue; } count++; yield { url, externalId: `${payload.market}:${payload.category}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const m of p.models) { if (!m.priceMin) continue; const brand = brandFromName(m.name); const attributes = AssetAttributesSchema.parse({ categorySlug: 'cameras', brand, model: stripBrand(m.name, brand), name: m.name, identifiers: { mpb_model: m.slug }, metadata: { units_available: m.available, price_max: m.priceMax, market: p.market, mpb_category: p.category }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: m.url, externalId: `${p.market}:${m.slug}`, rawTitle: m.name, imageUrls: m.image ? [m.image] : [], attributes, condition: { condition: null, conditionRaw: 'MPB graded per unit (Like New / Excellent / Good / Well Used) — cheapest unit shown', completeness: null }, observedAt: raw.fetchedAt, confidence: 0.75, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: m.priceMin, currency: m.currency, seller: 'MPB', location: p.market.split('-')[1]?.toUpperCase() === 'UK' ? 'GB' : p.market.split('-')[1]?.toUpperCase() ?? null, quantity: m.available, availability: 'available', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new MpbConnector(meta); }