TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * Reverb — public listings API (api.reverb.com/api/listings, HAL JSON, version 3.0, no token needed7 * for read-only listing search). Instruments, amps, pedals, synths and hi-fi ("Home Audio").8 * Prices are returned converted to USD by Reverb; the original listing currency is kept in metadata.9 */10const API = 'https://api.reverb.com/api';11const PARSER_VERSION = '1.0.0';12const HEADERS = { accept: 'application/hal+json', 'accept-version': '3.0', 'content-type': 'application/hal+json' };1314export const ListingSchema = z.object({15 id: z.number(),16 make: z.string().nullable().default(null),17 model: z.string().nullable().default(null),18 finish: z.string().nullable().default(null),19 year: z.string().nullable().default(null),20 title: z.string(),21 condition: z.string().nullable().default(null),22 price: z.number().nullable().default(null),23 currency: z.string().nullable().default(null),24 listingCurrency: z.string().nullable().default(null),25 categories: z.array(z.string()).default([]),26 state: z.string().nullable().default(null),27 createdAt: z.string().nullable().default(null),28 publishedAt: z.string().nullable().default(null),29 shop: z.string().nullable().default(null),30 offersEnabled: z.boolean().default(false),31 auction: z.boolean().default(false),32 photo: z.string().nullable().default(null),33 webUrl: z.string().nullable().default(null),34 description: z.string().nullable().default(null),35});36export type Listing = z.infer<typeof ListingSchema>;37export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), query: z.string(), page: z.number(), total: z.number().nullable(), listings: z.array(ListingSchema) });38export type PagePayload = z.infer<typeof PagePayloadSchema>;3940type ApiListing = Record<string, any>;4142export function trimListing(l: ApiListing): Listing {43 const photos = Array.isArray(l.photos) ? l.photos : [];44 const photo = photos[0]?._links?.large_crop?.href ?? photos[0]?._links?.full?.href ?? null;45 return ListingSchema.parse({46 id: Number(l.id),47 make: l.make ?? null,48 model: l.model ?? null,49 finish: l.finish ?? null,50 year: l.year ?? null,51 title: String(l.title ?? ''),52 condition: l.condition?.display_name ?? null,53 price: l.price?.amount ? Number(l.price.amount) : null,54 currency: l.price?.currency ?? null,55 listingCurrency: l.listing_currency ?? null,56 categories: Array.isArray(l.categories) ? l.categories.map((c: { full_name?: string }) => String(c.full_name ?? '')).filter(Boolean) : [],57 state: l.state?.slug ?? null,58 createdAt: l.created_at ?? null,59 publishedAt: l.published_at ?? null,60 shop: l.shop_name ?? l.shop?.name ?? null,61 offersEnabled: Boolean(l.offers_enabled),62 auction: Boolean(l.auction),63 photo,64 webUrl: l._links?.web?.href ?? null,65 description: typeof l.description === 'string' ? l.description.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 600) : null,66 });67}6869export function parseApiPage(json: unknown, url: string, query: string, page: number): PagePayload | null {70 const j = json as { listings?: ApiListing[]; total?: number } | null;71 if (!j || !Array.isArray(j.listings)) return null;72 return { kind: 'listing_page', url, query, page, total: typeof j.total === 'number' ? j.total : null, listings: j.listings.map(trimListing) };73}7475const CONDITION_MAP: Record<string, string> = { 'brand new': 'excellent', 'b-stock': 'excellent', mint: 'excellent', excellent: 'excellent', 'very good': 'very_good', good: 'good', fair: 'fair', poor: 'poor', 'non functioning': 'poor' };7677export function categoryFor(categories: string[]): 'musical_instruments' | 'audio_equipment' {78 return categories.some((c) => /^(Home Audio|Pro Audio|DJ and Lighting Gear)/i.test(c)) ? 'audio_equipment' : 'musical_instruments';79}8081export class ReverbConnector extends BaseConnector {82 readonly version = '1.0.0';83 readonly parserVersion = PARSER_VERSION;84 protected override minIntervalMs = 1200;85 override readonly urlPatterns = [/^https?:\/\/(www\.)?reverb\.com\/(?:[a-z-]+\/)?item\/(\d+)/i];8687 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {88 const queries = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.queries as string[] | undefined)) ?? [];89 const pages = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillPages ?? 6) : Number(this.meta.config.pagesPerQuery ?? 2);90 const perPage = Number(this.meta.config.perPage ?? 50);91 let count = 0;92 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.queryIndex ?? 0) : 0;93 for (let qi = start; qi < queries.length; qi++) {94 const query = queries[qi]!;95 for (let page = 1; page <= pages; page++) {96 if (ctx.signal?.aborted || this.reached(ctx, count)) return;97 const url = `${API}/listings?query=${encodeURIComponent(query)}&per_page=${perPage}&page=${page}`;98 await this.throttle();99 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: HEADERS, expect: ['title', 'price'], parse: (r) => (parseApiPage(r.json, url, query, page)?.listings.length ? { title: 'ok', price: 1 } : null) });100 const payload = res.success ? parseApiPage(res.json, url, query, page) : null;101 if (!payload) {102 ctx.anomaly('page_fetch_failed', `${query} p${page}: ${res.error ?? res.httpStatus}`);103 break;104 }105 if (!payload.listings.length) break;106 count++;107 yield { url, externalId: `q:${query}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };108 if (payload.listings.length < perPage) break;109 }110 await ctx.setCursor({ queryIndex: qi + 1 >= queries.length ? 0 : qi + 1, updatedAt: new Date().toISOString() });111 }112 }113114 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {115 const id = url.match(this.urlPatterns[0]!)?.[2];116 if (!id) return [];117 await this.throttle();118 const apiUrl = `${API}/listings/${id}`;119 const res = await ctx.fetch(apiUrl, { engines: ['api'], responseType: 'json', headers: HEADERS, minQuality: 0.2 });120 if (!res.success || !res.json || typeof res.json !== 'object' || !('id' in (res.json as object))) return [];121 const payload: PagePayload = { kind: 'listing_page', url: apiUrl, query: `item:${id}`, page: 1, total: 1, listings: [trimListing(res.json as ApiListing)] };122 return [{ url: apiUrl, externalId: `item:${id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];123 }124125 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {126 const p = PagePayloadSchema.parse(raw.payload);127 const out: NormalizedRecord[] = [];128 for (const l of p.listings) {129 if (!l.price || !l.title) continue;130 // parts, cases and accessories are not collectible assets in their own right131 if (l.categories.length && l.categories.every((c) => /^(Parts|Accessories)\b/i.test(c))) continue;132 const categorySlug = categoryFor(l.categories);133 const year = l.year && /^\d{4}$/.test(l.year.trim()) ? Number(l.year.trim()) : null;134 const brand = l.make?.trim() || null;135 const model = l.model?.trim() || null;136 const condKey = (l.condition ?? '').toLowerCase();137 const attributes = AssetAttributesSchema.parse({138 categorySlug,139 brand,140 model,141 name: brand && model ? (model.toLowerCase().startsWith(brand.toLowerCase()) ? model : `${brand} ${model}`) : l.title,142 year,143 color: l.finish && l.finish.length <= 60 ? l.finish : null,144 identifiers: { reverb_listing_id: String(l.id) },145 metadata: { reverb_categories: l.categories, listing_currency: l.listingCurrency, price_converted_by_reverb: l.currency === 'USD' && l.listingCurrency && l.listingCurrency !== 'USD', shop: l.shop, auction: l.auction },146 });147 const listedAt = l.publishedAt ? new Date(l.publishedAt) : l.createdAt ? new Date(l.createdAt) : null;148 out.push(149 NormalizedListingSchema.parse({150 kind: 'listing',151 connectorId: this.meta.id,152 sourceId: this.meta.sourceId,153 sourceUrl: l.webUrl ?? `https://reverb.com/item/${l.id}`,154 externalId: String(l.id),155 rawTitle: l.title,156 description: l.description,157 imageUrls: l.photo ? [l.photo] : [],158 attributes,159 condition: { condition: CONDITION_MAP[condKey] ?? null, conditionRaw: l.condition, completeness: null },160 observedAt: raw.fetchedAt,161 confidence: l.currency === 'USD' && l.listingCurrency && l.listingCurrency !== 'USD' ? 0.75 : 0.85,162 parserVersion: PARSER_VERSION,163 listingType: l.auction ? 'auction' : l.offersEnabled ? 'best_offer' : 'fixed_price',164 price: l.price,165 currency: l.currency && /^[A-Z]{3}$/.test(l.currency) ? l.currency : 'USD',166 seller: l.shop,167 listedAt: listedAt && !Number.isNaN(listedAt.getTime()) ? listedAt : null,168 availability: l.state === 'live' || l.state === null ? 'available' : l.state === 'sold' ? 'sold' : 'ended',169 }),170 );171 }172 return out;173 }174}175176export default function createConnector(meta: ConnectorMeta) {177 return new ReverbConnector(meta);178}179