import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; import { dateMDY, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js'; /** * Bring a Trailer completed auctions. One raw record per results page (compact items); one sale per * "Sold for" item. Source: the public JSON feeding /auctions/results/. */ const BASE = 'https://bringatrailer.com'; const FILTER_URL = `${BASE}/wp-json/bringatrailer/1.0/data/listings-filter`; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ id: z.number(), title: z.string(), url: z.string(), year: z.union([z.string(), z.number()]).nullable().optional(), currency: z.string().nullable().optional(), current_bid: z.number().nullable().optional(), sold_text: z.string().nullable().optional(), timestamp_end: z.number().nullable().optional(), country_code: z.string().nullable().optional(), noreserve: z.boolean().nullable().optional(), premium: z.boolean().nullable().optional(), thumbnail_url: z.string().nullable().optional(), excerpt: z.string().nullable().optional(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), itemsTotal: z.number().nullable(), pagesTotal: z.number().nullable(), items: z.array(ItemSchema) }); export const ListingPayloadSchema = z.object({ kind: z.literal('listing'), url: z.string(), title: z.string(), soldLine: z.string().nullable(), vin: z.string().nullable(), mileage: z.string().nullable(), lotNumber: z.string().nullable(), image: z.string().nullable() }); const KEEP: Array = ['id', 'title', 'url', 'year', 'currency', 'current_bid', 'sold_text', 'timestamp_end', 'country_code', 'noreserve', 'premium', 'thumbnail_url', 'excerpt']; export function trimItem(raw: Record): Item | null { const out: Record = {}; for (const k of KEEP) if (raw[k] !== undefined) out[k] = raw[k]; const p = ItemSchema.safeParse(out); return p.success ? p.data : null; } /** "Sold for USD $23,250 on 9/6/2026 " → { sold: true, date } ; "Bid to …" → sold false */ export function parseSoldText(s: string | null | undefined): { sold: boolean; date: Date | null } { if (!s) return { sold: false, date: null }; const text = s.replace(/<[^>]+>/g, ' '); return { sold: /^\s*Sold for/i.test(text), date: dateMDY(text) }; } export function parseListingHtml(html: string, url: string) { const title = html.match(/]*class="[^"]*post-title[^"]*"[^>]*>([\s\S]*?)<\/h1>/)?.[1]?.replace(/<[^>]+>/g, '').trim() ?? html.match(/([^<|]+)/)?.[1]?.trim() ?? ''; const soldLine = html.match(/(Sold for[^<]{0,80}on\s+\d{1,2}\/\d{1,2}\/\d{2,4})/)?.[1] ?? null; const vin = html.match(/Chassis:\s*(?:<a[^>]*>)?\s*([A-HJ-NPR-Z0-9]{6,20})/i)?.[1] ?? null; const mileage = html.match(/<li>\s*([\d,]+k?\s*(?:Miles|Kilometers)[^<]{0,40})<\/li>/i)?.[1]?.trim() ?? null; const lotNumber = html.match(/Lot #(\d+)/)?.[1] ?? null; const image = html.match(/property="og:image"\s+content="([^"]+)"/)?.[1] ?? null; return ListingPayloadSchema.parse({ kind: 'listing', url, title, soldLine, vin, mileage, lotNumber, image }); } export class BringATrailerConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/bringatrailer\.com\/listing\/[a-z0-9-]+\/?$/i]; async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { const perPage = Number(this.meta.config.perPage ?? 36); const pages = Number(this.meta.config.pagesPerRun ?? 20); const backfill = ctx.options.mode === 'backfill'; const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1; const newestSeen = !backfill && typeof ctx.options.cursor?.newestEnd === 'number' ? Number(ctx.options.cursor.newestEnd) : 0; let count = 0; let maxEnd = newestSeen; for (let page = start; page < start + pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; await this.throttle(); const res = await ctx.fetch(FILTER_URL, { engines: ['api'], method: 'POST', body: { per_page: perPage, page, get_items: 1, get_stats: 0, sort: 'td' }, expect: ['title', 'price', 'date', 'status'], parse: (r) => { const d = r.json as { items?: Array<Record<string, unknown>> } | null; const first = d?.items?.[0]; return first ? { title: first.title, price: first.current_bid, date: first.timestamp_end, status: first.sold_text } : null; }, }); const data = res.json as { items?: Array<Record<string, unknown>>; items_total?: number; pages_total?: number } | null; if (!res.success || !data?.items) { ctx.anomaly('page_fetch_failed', `${FILTER_URL} page ${page}: ${res.error ?? res.httpStatus}`); break; } const items = data.items.map(trimItem).filter((x): x is Item => Boolean(x)); if (items.length === 0) { ctx.anomaly('empty_page', `page ${page}`); break; } for (const it of items) if (it.timestamp_end && it.timestamp_end > maxEnd) maxEnd = it.timestamp_end; const payload = { kind: 'results_page' as const, page, itemsTotal: data.items_total ?? null, pagesTotal: data.pages_total ?? null, items }; count++; yield { url: `${BASE}/auctions/results/?page=${page}`, externalId: `results:${page}:${items[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; const oldest = Math.min(...items.map((i) => i.timestamp_end ?? Number.MAX_SAFE_INTEGER)); if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() }); else if (newestSeen && oldest <= newestSeen) break; // caught up with the previous run } if (!backfill && maxEnd) await ctx.setCursor({ newestEnd: maxEnd, updatedAt: new Date().toISOString() }); } async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title'], parse: (r) => (r.html ? { title: parseListingHtml(r.html, url).title } : null) }); if (!res.success || !res.html) return []; const payload = parseListingHtml(res.html, url); return [{ url, externalId: url.replace(/\/$/, '').split('/').pop() ?? url, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { const payload = raw.payload as { kind?: string }; if (payload?.kind === 'listing') return this.normalizeListing(raw); const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedSale[] = []; for (const it of p.items) { const { sold, date } = parseSoldText(it.sold_text); if (!sold || !it.current_bid || it.current_bid <= 0) continue; const saleDate = it.timestamp_end ? new Date(it.timestamp_end * 1000) : date; if (!saleDate) continue; const currency = (it.currency ?? 'USD').toUpperCase(); const m = money(`${it.current_bid} ${currency}`, 'USD'); if (!m) continue; const attributes = vehicleAttributes(it.title, { country: it.country_code ?? null, identifiers: { bat_listing_id: String(it.id) }, metadata: { no_reserve: it.noreserve ?? null, premium_listing: it.premium ?? null, year_field: it.year ?? null } }); out.push( makeSale({ meta: this.meta, sourceUrl: it.url, externalId: String(it.id), rawTitle: it.title, attributes, price: it.current_bid, currency: m.currency, saleDate, buyerPremiumIncluded: false, auctionHouse: 'Bring a Trailer', imageUrls: it.thumbnail_url ? [it.thumbnail_url.replace(/\?.*$/, '')] : [], description: it.excerpt ?? null, location: it.country_code ?? null, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, confidence: 0.92, }), ); } return out; } private async normalizeListing(raw: RawRecordLike): Promise<NormalizedRecord[]> { const p = ListingPayloadSchema.parse(raw.payload); if (!p.soldLine) return []; const m = money(p.soldLine, 'USD'); const date = dateMDY(p.soldLine); if (!m || !date) return []; const identifiers: Record<string, string> = {}; if (p.vin) identifiers.vin = p.vin; if (p.lotNumber) identifiers.bat_lot = p.lotNumber; const attributes = vehicleAttributes(p.title, { identifiers, metadata: { mileage: p.mileage } }); return [makeSale({ meta: this.meta, sourceUrl: p.url, externalId: raw.externalId ?? p.url, rawTitle: p.title, attributes, price: m.amount, currency: m.currency, saleDate: date, buyerPremiumIncluded: false, auctionHouse: 'Bring a Trailer', lotNumber: p.lotNumber, imageUrls: p.image ? [p.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION })]; } } export default (meta: ConnectorMeta) => new BringATrailerConnector(meta);