TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { brandFromName, compactMoney, compactMoneyCents, fetchFirecrawl, stripBrand } from '../../api/_wlib/index.js';56/**7 * B&H Photo — Used Department. Category result pages are rendered through Firecrawl (plain HTTPS8 * gets an Akamai challenge) and parsed from markdown: product name, B&H #, MFR #, B&H used grade9 * (10 / 9+ / 9 / 8+ / 8 / 7), shutter count, used price (cents flattened), stock state.10 */11const BASE = 'https://www.bhphotovideo.com';12const PARSER_VERSION = '1.0.0';1314export const ItemSchema = z.object({15 name: z.string(),16 url: z.string(),17 bhSku: z.string().nullable(),18 mfr: z.string().nullable(),19 grade: z.string().nullable(),20 gradeText: z.string().nullable(),21 shutterCount: z.number().nullable(),22 warranty: z.string().nullable(),23 usedPrice: z.number().nullable(),24 newPrice: z.number().nullable(),25 inStock: z.boolean().nullable(),26 image: z.string().nullable(),27});28export type Item = z.infer<typeof ItemSchema>;29export const PagePayloadSchema = z.object({ kind: z.literal('used_category_page'), url: z.string(), category: z.string(), total: z.number().nullable(), items: z.array(ItemSchema) });30export type PagePayload = z.infer<typeof PagePayloadSchema>;3132/** Split the markdown into product blocks starting at "### [Name](url)". */33export function parseUsedMarkdown(md: string, url: string, category: string): PagePayload {34 const items: Item[] = [];35 const total = md.match(/(\d[\d,]*)\s*Items?\s*Found/i)?.[1];36 const parts = md.split(/\n(?=###\s*\[)/g);37 for (const part of parts) {38 const head = part.match(/^###\s*\[([^\]]+)\]\((https?:\/\/[^)\s]+)\)/);39 if (!head) continue;40 const name = head[1]!.replace(/\s+/g, ' ').trim();41 const body = part.slice(head[0].length);42 const bhSku = body.match(/BH\s*#\s*(\d+)/i)?.[1] ?? null;43 const mfr = body.match(/MFR\s*#\s*([A-Za-z0-9\-./]+)/)?.[1] ?? null;44 const cond = body.match(/Condition\s*:?\s*(10|9\+|9|8\+|8|7\+|7|6)\s*([A-Za-z][^|\n]*)?/);45 const shutter = body.match(/Shutter Count\s*:?\s*([\d,]+)/i)?.[1];46 const warranty = body.match(/Warranty\s*:?\s*([^|\n]+)/i)?.[1]?.trim() ?? null;47 const newPrice = compactMoney(body.match(/New\s*\$([\d,]+\.\d{2})/)?.[1] ? `$${body.match(/New\s*\$([\d,]+\.\d{2})/)![1]}` : null);48 // the used price is the bare "$1,79995" token (no decimal point) on its own line49 const used = body.match(/(?:^|\|)\s*\$(\d[\d,]*)\s*(?:\||$)/m)?.[1] ?? null;50 const usedPrice = used ? compactMoneyCents(`$${used}`) : null;51 const inStock = /\bIn Stock\b/.test(body) ? true : /Out of Stock|Temporarily Out/i.test(body) ? false : null;52 const image = part.match(/!\[[^\]]*\]\((https?:\/\/[^)\s]+\.(?:jpg|jpeg|png|webp)[^)\s]*)\)/i)?.[1] ?? null;53 items.push({ name, url: head[2]!.split('?')[0]!, bhSku, mfr, grade: cond?.[1] ?? null, gradeText: cond?.[2]?.trim() ?? null, shutterCount: shutter ? Number(shutter.replace(/,/g, '')) : null, warranty, usedPrice, newPrice, inStock, image });54 }55 return { kind: 'used_category_page', url, category, total: total ? Number(total.replace(/,/g, '')) : null, items };56}5758/** B&H used grades → 'electronics' condition scale. */59export function conditionFor(grade: string | null): string | null {60 switch (grade) {61 case '10':62 return 'mint';63 case '9+':64 case '9':65 return 'excellent';66 case '8+':67 case '8':68 return 'good';69 case '7+':70 case '7':71 case '6':72 return 'fair';73 default:74 return null;75 }76}7778export class BhUsedConnector extends BaseConnector {79 readonly version = '1.0.0';80 readonly parserVersion = PARSER_VERSION;81 protected override minIntervalMs = 3000;8283 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {84 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];85 let count = 0;86 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;87 for (let i = start; i < seeds.length; i++) {88 const seed = seeds[i]!;89 if (ctx.signal?.aborted || this.reached(ctx, count)) return;90 const url = seed.startsWith('http') ? seed : `${BASE}${seed}`;91 const category = url.match(/\/c\/buy\/([^/]+)/)?.[1] ?? url;92 await this.throttle();93 const res = await fetchFirecrawl(ctx, url, { timeoutMs: 90_000, parse: (r) => (r.markdown ? parseUsedMarkdown(r.markdown, url, category).items.length : 0) });94 const payload = res.success && res.markdown ? parseUsedMarkdown(res.markdown, url, category) : null;95 if (!payload?.items.length) {96 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no products'}`);97 continue;98 }99 count++;100 yield { url, externalId: `used:${category}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };101 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });102 }103 }104105 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {106 const p = PagePayloadSchema.parse(raw.payload);107 const out: NormalizedRecord[] = [];108 for (const it of p.items) {109 if (!it.bhSku) continue;110 const brand = brandFromName(it.name);111 const attributes = AssetAttributesSchema.parse({112 categorySlug: 'cameras',113 brand,114 model: stripBrand(it.name, brand),115 name: it.name,116 originalMsrp: it.newPrice,117 originalMsrpCurrency: it.newPrice ? 'USD' : null,118 identifiers: { bh_sku: it.bhSku, ...(it.mfr ? { mfr: it.mfr } : {}) },119 metadata: { bh_grade: it.grade, bh_grade_text: it.gradeText, shutter_count: it.shutterCount, warranty: it.warranty, category: p.category, new_price_usd: it.newPrice },120 });121 out.push(122 NormalizedListingSchema.parse({123 kind: 'listing',124 connectorId: this.meta.id,125 sourceId: this.meta.sourceId,126 sourceUrl: it.url,127 externalId: it.bhSku,128 rawTitle: it.name,129 imageUrls: it.image ? [it.image] : [],130 attributes,131 condition: { condition: conditionFor(it.grade), conditionRaw: it.grade ? `B&H used grade ${it.grade}${it.gradeText ? ` – ${it.gradeText}` : ''}` : null, completeness: null },132 observedAt: raw.fetchedAt,133 confidence: 0.8,134 parserVersion: PARSER_VERSION,135 listingType: 'fixed_price',136 price: it.usedPrice,137 currency: it.usedPrice ? 'USD' : null,138 seller: 'B&H Photo (Used Department)',139 location: 'US',140 quantity: 1,141 availability: it.inStock === false ? 'ended' : it.usedPrice ? 'available' : 'unknown',142 }),143 );144 }145 return out;146 }147}148149export default function createConnector(meta: ConnectorMeta) {150 return new BhUsedConnector(meta);151}152