import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; /** * Reverb — public listings API (api.reverb.com/api/listings, HAL JSON, version 3.0, no token needed * for read-only listing search). Instruments, amps, pedals, synths and hi-fi ("Home Audio"). * Prices are returned converted to USD by Reverb; the original listing currency is kept in metadata. */ const API = 'https://api.reverb.com/api'; const PARSER_VERSION = '1.0.0'; const HEADERS = { accept: 'application/hal+json', 'accept-version': '3.0', 'content-type': 'application/hal+json' }; export const ListingSchema = z.object({ id: z.number(), make: z.string().nullable().default(null), model: z.string().nullable().default(null), finish: z.string().nullable().default(null), year: z.string().nullable().default(null), title: z.string(), condition: z.string().nullable().default(null), price: z.number().nullable().default(null), currency: z.string().nullable().default(null), listingCurrency: z.string().nullable().default(null), categories: z.array(z.string()).default([]), state: z.string().nullable().default(null), createdAt: z.string().nullable().default(null), publishedAt: z.string().nullable().default(null), shop: z.string().nullable().default(null), offersEnabled: z.boolean().default(false), auction: z.boolean().default(false), photo: z.string().nullable().default(null), webUrl: z.string().nullable().default(null), description: z.string().nullable().default(null), }); export type Listing = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), query: z.string(), page: z.number(), total: z.number().nullable(), listings: z.array(ListingSchema) }); export type PagePayload = z.infer; type ApiListing = Record; export function trimListing(l: ApiListing): Listing { const photos = Array.isArray(l.photos) ? l.photos : []; const photo = photos[0]?._links?.large_crop?.href ?? photos[0]?._links?.full?.href ?? null; return ListingSchema.parse({ id: Number(l.id), make: l.make ?? null, model: l.model ?? null, finish: l.finish ?? null, year: l.year ?? null, title: String(l.title ?? ''), condition: l.condition?.display_name ?? null, price: l.price?.amount ? Number(l.price.amount) : null, currency: l.price?.currency ?? null, listingCurrency: l.listing_currency ?? null, categories: Array.isArray(l.categories) ? l.categories.map((c: { full_name?: string }) => String(c.full_name ?? '')).filter(Boolean) : [], state: l.state?.slug ?? null, createdAt: l.created_at ?? null, publishedAt: l.published_at ?? null, shop: l.shop_name ?? l.shop?.name ?? null, offersEnabled: Boolean(l.offers_enabled), auction: Boolean(l.auction), photo, webUrl: l._links?.web?.href ?? null, description: typeof l.description === 'string' ? l.description.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 600) : null, }); } export function parseApiPage(json: unknown, url: string, query: string, page: number): PagePayload | null { const j = json as { listings?: ApiListing[]; total?: number } | null; if (!j || !Array.isArray(j.listings)) return null; return { kind: 'listing_page', url, query, page, total: typeof j.total === 'number' ? j.total : null, listings: j.listings.map(trimListing) }; } const CONDITION_MAP: Record = { 'brand new': 'excellent', 'b-stock': 'excellent', mint: 'excellent', excellent: 'excellent', 'very good': 'very_good', good: 'good', fair: 'fair', poor: 'poor', 'non functioning': 'poor' }; export function categoryFor(categories: string[]): 'musical_instruments' | 'audio_equipment' { return categories.some((c) => /^(Home Audio|Pro Audio|DJ and Lighting Gear)/i.test(c)) ? 'audio_equipment' : 'musical_instruments'; } export class ReverbConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1200; override readonly urlPatterns = [/^https?:\/\/(www\.)?reverb\.com\/(?:[a-z-]+\/)?item\/(\d+)/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const queries = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.queries as string[] | undefined)) ?? []; const pages = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillPages ?? 6) : Number(this.meta.config.pagesPerQuery ?? 2); const perPage = Number(this.meta.config.perPage ?? 50); let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.queryIndex ?? 0) : 0; for (let qi = start; qi < queries.length; qi++) { const query = queries[qi]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${API}/listings?query=${encodeURIComponent(query)}&per_page=${perPage}&page=${page}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: HEADERS, expect: ['title', 'price'], parse: (r) => (parseApiPage(r.json, url, query, page)?.listings.length ? { title: 'ok', price: 1 } : null) }); const payload = res.success ? parseApiPage(res.json, url, query, page) : null; if (!payload) { ctx.anomaly('page_fetch_failed', `${query} p${page}: ${res.error ?? res.httpStatus}`); break; } if (!payload.listings.length) break; count++; yield { url, externalId: `q:${query}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (payload.listings.length < perPage) break; } await ctx.setCursor({ queryIndex: qi + 1 >= queries.length ? 0 : qi + 1, updatedAt: new Date().toISOString() }); } } async lookup(url: string, ctx: CrawlContext): Promise { const id = url.match(this.urlPatterns[0]!)?.[2]; if (!id) return []; await this.throttle(); const apiUrl = `${API}/listings/${id}`; const res = await ctx.fetch(apiUrl, { engines: ['api'], responseType: 'json', headers: HEADERS, minQuality: 0.2 }); if (!res.success || !res.json || typeof res.json !== 'object' || !('id' in (res.json as object))) return []; const payload: PagePayload = { kind: 'listing_page', url: apiUrl, query: `item:${id}`, page: 1, total: 1, listings: [trimListing(res.json as ApiListing)] }; return [{ url: apiUrl, externalId: `item:${id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const l of p.listings) { if (!l.price || !l.title) continue; // parts, cases and accessories are not collectible assets in their own right if (l.categories.length && l.categories.every((c) => /^(Parts|Accessories)\b/i.test(c))) continue; const categorySlug = categoryFor(l.categories); const year = l.year && /^\d{4}$/.test(l.year.trim()) ? Number(l.year.trim()) : null; const brand = l.make?.trim() || null; const model = l.model?.trim() || null; const condKey = (l.condition ?? '').toLowerCase(); const attributes = AssetAttributesSchema.parse({ categorySlug, brand, model, name: brand && model ? (model.toLowerCase().startsWith(brand.toLowerCase()) ? model : `${brand} ${model}`) : l.title, year, color: l.finish && l.finish.length <= 60 ? l.finish : null, identifiers: { reverb_listing_id: String(l.id) }, metadata: { reverb_categories: l.categories, listing_currency: l.listingCurrency, price_converted_by_reverb: l.currency === 'USD' && l.listingCurrency && l.listingCurrency !== 'USD', shop: l.shop, auction: l.auction }, }); const listedAt = l.publishedAt ? new Date(l.publishedAt) : l.createdAt ? new Date(l.createdAt) : null; out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: l.webUrl ?? `https://reverb.com/item/${l.id}`, externalId: String(l.id), rawTitle: l.title, description: l.description, imageUrls: l.photo ? [l.photo] : [], attributes, condition: { condition: CONDITION_MAP[condKey] ?? null, conditionRaw: l.condition, completeness: null }, observedAt: raw.fetchedAt, confidence: l.currency === 'USD' && l.listingCurrency && l.listingCurrency !== 'USD' ? 0.75 : 0.85, parserVersion: PARSER_VERSION, listingType: l.auction ? 'auction' : l.offersEnabled ? 'best_offer' : 'fixed_price', price: l.price, currency: l.currency && /^[A-Z]{3}$/.test(l.currency) ? l.currency : 'USD', seller: l.shop, listedAt: listedAt && !Number.isNaN(listedAt.getTime()) ? listedAt : null, availability: l.state === 'live' || l.state === null ? 'available' : l.state === 'sold' ? 'sold' : 'ended', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new ReverbConnector(meta); }