import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { brandFromName, compactMoney, compactMoneyCents, fetchFirecrawl, stripBrand } from '../../api/_wlib/index.js'; /** * B&H Photo — Used Department. Category result pages are rendered through Firecrawl (plain HTTPS * gets an Akamai challenge) and parsed from markdown: product name, B&H #, MFR #, B&H used grade * (10 / 9+ / 9 / 8+ / 8 / 7), shutter count, used price (cents flattened), stock state. */ const BASE = 'https://www.bhphotovideo.com'; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ name: z.string(), url: z.string(), bhSku: z.string().nullable(), mfr: z.string().nullable(), grade: z.string().nullable(), gradeText: z.string().nullable(), shutterCount: z.number().nullable(), warranty: z.string().nullable(), usedPrice: z.number().nullable(), newPrice: z.number().nullable(), inStock: z.boolean().nullable(), image: z.string().nullable(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('used_category_page'), url: z.string(), category: z.string(), total: z.number().nullable(), items: z.array(ItemSchema) }); export type PagePayload = z.infer; /** Split the markdown into product blocks starting at "### [Name](url)". */ export function parseUsedMarkdown(md: string, url: string, category: string): PagePayload { const items: Item[] = []; const total = md.match(/(\d[\d,]*)\s*Items?\s*Found/i)?.[1]; const parts = md.split(/\n(?=###\s*\[)/g); for (const part of parts) { const head = part.match(/^###\s*\[([^\]]+)\]\((https?:\/\/[^)\s]+)\)/); if (!head) continue; const name = head[1]!.replace(/\s+/g, ' ').trim(); const body = part.slice(head[0].length); const bhSku = body.match(/BH\s*#\s*(\d+)/i)?.[1] ?? null; const mfr = body.match(/MFR\s*#\s*([A-Za-z0-9\-./]+)/)?.[1] ?? null; const cond = body.match(/Condition\s*:?\s*(10|9\+|9|8\+|8|7\+|7|6)\s*([A-Za-z][^|\n]*)?/); const shutter = body.match(/Shutter Count\s*:?\s*([\d,]+)/i)?.[1]; const warranty = body.match(/Warranty\s*:?\s*([^|\n]+)/i)?.[1]?.trim() ?? null; const newPrice = compactMoney(body.match(/New\s*\$([\d,]+\.\d{2})/)?.[1] ? `$${body.match(/New\s*\$([\d,]+\.\d{2})/)![1]}` : null); // the used price is the bare "$1,79995" token (no decimal point) on its own line const used = body.match(/(?:^|\|)\s*\$(\d[\d,]*)\s*(?:\||$)/m)?.[1] ?? null; const usedPrice = used ? compactMoneyCents(`$${used}`) : null; const inStock = /\bIn Stock\b/.test(body) ? true : /Out of Stock|Temporarily Out/i.test(body) ? false : null; const image = part.match(/!\[[^\]]*\]\((https?:\/\/[^)\s]+\.(?:jpg|jpeg|png|webp)[^)\s]*)\)/i)?.[1] ?? null; items.push({ name, url: head[2]!.split('?')[0]!, bhSku, mfr, grade: cond?.[1] ?? null, gradeText: cond?.[2]?.trim() ?? null, shutterCount: shutter ? Number(shutter.replace(/,/g, '')) : null, warranty, usedPrice, newPrice, inStock, image }); } return { kind: 'used_category_page', url, category, total: total ? Number(total.replace(/,/g, '')) : null, items }; } /** B&H used grades → 'electronics' condition scale. */ export function conditionFor(grade: string | null): string | null { switch (grade) { case '10': return 'mint'; case '9+': case '9': return 'excellent'; case '8+': case '8': return 'good'; case '7+': case '7': case '6': return 'fair'; default: return null; } } export class BhUsedConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = start; i < seeds.length; i++) { const seed = seeds[i]!; if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = seed.startsWith('http') ? seed : `${BASE}${seed}`; const category = url.match(/\/c\/buy\/([^/]+)/)?.[1] ?? url; await this.throttle(); const res = await fetchFirecrawl(ctx, url, { timeoutMs: 90_000, parse: (r) => (r.markdown ? parseUsedMarkdown(r.markdown, url, category).items.length : 0) }); const payload = res.success && res.markdown ? parseUsedMarkdown(res.markdown, url, category) : null; if (!payload?.items.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no products'}`); continue; } count++; yield { url, externalId: `used:${category}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const it of p.items) { if (!it.bhSku) continue; const brand = brandFromName(it.name); const attributes = AssetAttributesSchema.parse({ categorySlug: 'cameras', brand, model: stripBrand(it.name, brand), name: it.name, originalMsrp: it.newPrice, originalMsrpCurrency: it.newPrice ? 'USD' : null, identifiers: { bh_sku: it.bhSku, ...(it.mfr ? { mfr: it.mfr } : {}) }, metadata: { bh_grade: it.grade, bh_grade_text: it.gradeText, shutter_count: it.shutterCount, warranty: it.warranty, category: p.category, new_price_usd: it.newPrice }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: it.bhSku, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes, condition: { condition: conditionFor(it.grade), conditionRaw: it.grade ? `B&H used grade ${it.grade}${it.gradeText ? ` – ${it.gradeText}` : ''}` : null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.usedPrice, currency: it.usedPrice ? 'USD' : null, seller: 'B&H Photo (Used Department)', location: 'US', quantity: 1, availability: it.inStock === false ? 'ended' : it.usedPrice ? 'available' : 'unknown', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new BhUsedConnector(meta); }