TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateMDY, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js';56/**7 * Bring a Trailer completed auctions. One raw record per results page (compact items); one sale per8 * "Sold for" item. Source: the public JSON feeding /auctions/results/.9 */10const BASE = 'https://bringatrailer.com';11const FILTER_URL = `${BASE}/wp-json/bringatrailer/1.0/data/listings-filter`;12const PARSER_VERSION = '1.0.0';1314export const ItemSchema = z.object({15 id: z.number(),16 title: z.string(),17 url: z.string(),18 year: z.union([z.string(), z.number()]).nullable().optional(),19 currency: z.string().nullable().optional(),20 current_bid: z.number().nullable().optional(),21 sold_text: z.string().nullable().optional(),22 timestamp_end: z.number().nullable().optional(),23 country_code: z.string().nullable().optional(),24 noreserve: z.boolean().nullable().optional(),25 premium: z.boolean().nullable().optional(),26 thumbnail_url: z.string().nullable().optional(),27 excerpt: z.string().nullable().optional(),28});29export type Item = z.infer<typeof ItemSchema>;30export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), itemsTotal: z.number().nullable(), pagesTotal: z.number().nullable(), items: z.array(ItemSchema) });31export const ListingPayloadSchema = z.object({ kind: z.literal('listing'), url: z.string(), title: z.string(), soldLine: z.string().nullable(), vin: z.string().nullable(), mileage: z.string().nullable(), lotNumber: z.string().nullable(), image: z.string().nullable() });3233const KEEP: Array<keyof Item> = ['id', 'title', 'url', 'year', 'currency', 'current_bid', 'sold_text', 'timestamp_end', 'country_code', 'noreserve', 'premium', 'thumbnail_url', 'excerpt'];3435export function trimItem(raw: Record<string, unknown>): Item | null {36 const out: Record<string, unknown> = {};37 for (const k of KEEP) if (raw[k] !== undefined) out[k] = raw[k];38 const p = ItemSchema.safeParse(out);39 return p.success ? p.data : null;40}4142/** "Sold for USD $23,250 <span> on 9/6/2026 </span>" → { sold: true, date } ; "Bid to …" → sold false */43export function parseSoldText(s: string | null | undefined): { sold: boolean; date: Date | null } {44 if (!s) return { sold: false, date: null };45 const text = s.replace(/<[^>]+>/g, ' ');46 return { sold: /^\s*Sold for/i.test(text), date: dateMDY(text) };47}4849export function parseListingHtml(html: string, url: string) {50 const title = html.match(/<h1[^>]*class="[^"]*post-title[^"]*"[^>]*>([\s\S]*?)<\/h1>/)?.[1]?.replace(/<[^>]+>/g, '').trim() ?? html.match(/<title>([^<|]+)/)?.[1]?.trim() ?? '';51 const soldLine = html.match(/(Sold for[^<]{0,80}on\s+\d{1,2}\/\d{1,2}\/\d{2,4})/)?.[1] ?? null;52 const vin = html.match(/Chassis:\s*(?:<a[^>]*>)?\s*([A-HJ-NPR-Z0-9]{6,20})/i)?.[1] ?? null;53 const mileage = html.match(/<li>\s*([\d,]+k?\s*(?:Miles|Kilometers)[^<]{0,40})<\/li>/i)?.[1]?.trim() ?? null;54 const lotNumber = html.match(/Lot #(\d+)/)?.[1] ?? null;55 const image = html.match(/property="og:image"\s+content="([^"]+)"/)?.[1] ?? null;56 return ListingPayloadSchema.parse({ kind: 'listing', url, title, soldLine, vin, mileage, lotNumber, image });57}5859export class BringATrailerConnector extends BaseConnector {60 readonly version = '1.0.0';61 readonly parserVersion = PARSER_VERSION;62 protected override minIntervalMs = 1500;63 override readonly urlPatterns = [/^https?:\/\/bringatrailer\.com\/listing\/[a-z0-9-]+\/?$/i];6465 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {66 const perPage = Number(this.meta.config.perPage ?? 36);67 const pages = Number(this.meta.config.pagesPerRun ?? 20);68 const backfill = ctx.options.mode === 'backfill';69 const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;70 const newestSeen = !backfill && typeof ctx.options.cursor?.newestEnd === 'number' ? Number(ctx.options.cursor.newestEnd) : 0;71 let count = 0;72 let maxEnd = newestSeen;73 for (let page = start; page < start + pages; page++) {74 if (ctx.signal?.aborted || this.reached(ctx, count)) break;75 await this.throttle();76 const res = await ctx.fetch(FILTER_URL, {77 engines: ['api'],78 method: 'POST',79 body: { per_page: perPage, page, get_items: 1, get_stats: 0, sort: 'td' },80 expect: ['title', 'price', 'date', 'status'],81 parse: (r) => {82 const d = r.json as { items?: Array<Record<string, unknown>> } | null;83 const first = d?.items?.[0];84 return first ? { title: first.title, price: first.current_bid, date: first.timestamp_end, status: first.sold_text } : null;85 },86 });87 const data = res.json as { items?: Array<Record<string, unknown>>; items_total?: number; pages_total?: number } | null;88 if (!res.success || !data?.items) {89 ctx.anomaly('page_fetch_failed', `${FILTER_URL} page ${page}: ${res.error ?? res.httpStatus}`);90 break;91 }92 const items = data.items.map(trimItem).filter((x): x is Item => Boolean(x));93 if (items.length === 0) {94 ctx.anomaly('empty_page', `page ${page}`);95 break;96 }97 for (const it of items) if (it.timestamp_end && it.timestamp_end > maxEnd) maxEnd = it.timestamp_end;98 const payload = { kind: 'results_page' as const, page, itemsTotal: data.items_total ?? null, pagesTotal: data.pages_total ?? null, items };99 count++;100 yield { url: `${BASE}/auctions/results/?page=${page}`, externalId: `results:${page}:${items[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };101 const oldest = Math.min(...items.map((i) => i.timestamp_end ?? Number.MAX_SAFE_INTEGER));102 if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });103 else if (newestSeen && oldest <= newestSeen) break; // caught up with the previous run104 }105 if (!backfill && maxEnd) await ctx.setCursor({ newestEnd: maxEnd, updatedAt: new Date().toISOString() });106 }107108 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {109 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title'], parse: (r) => (r.html ? { title: parseListingHtml(r.html, url).title } : null) });110 if (!res.success || !res.html) return [];111 const payload = parseListingHtml(res.html, url);112 return [{ url, externalId: url.replace(/\/$/, '').split('/').pop() ?? url, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];113 }114115 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {116 const payload = raw.payload as { kind?: string };117 if (payload?.kind === 'listing') return this.normalizeListing(raw);118 const p = PagePayloadSchema.parse(raw.payload);119 const out: NormalizedSale[] = [];120 for (const it of p.items) {121 const { sold, date } = parseSoldText(it.sold_text);122 if (!sold || !it.current_bid || it.current_bid <= 0) continue;123 const saleDate = it.timestamp_end ? new Date(it.timestamp_end * 1000) : date;124 if (!saleDate) continue;125 const currency = (it.currency ?? 'USD').toUpperCase();126 const m = money(`${it.current_bid} ${currency}`, 'USD');127 if (!m) continue;128 const attributes = vehicleAttributes(it.title, { country: it.country_code ?? null, identifiers: { bat_listing_id: String(it.id) }, metadata: { no_reserve: it.noreserve ?? null, premium_listing: it.premium ?? null, year_field: it.year ?? null } });129 out.push(130 makeSale({131 meta: this.meta,132 sourceUrl: it.url,133 externalId: String(it.id),134 rawTitle: it.title,135 attributes,136 price: it.current_bid,137 currency: m.currency,138 saleDate,139 buyerPremiumIncluded: false,140 auctionHouse: 'Bring a Trailer',141 imageUrls: it.thumbnail_url ? [it.thumbnail_url.replace(/\?.*$/, '')] : [],142 description: it.excerpt ?? null,143 location: it.country_code ?? null,144 observedAt: raw.fetchedAt,145 parserVersion: PARSER_VERSION,146 confidence: 0.92,147 }),148 );149 }150 return out;151 }152153 private async normalizeListing(raw: RawRecordLike): Promise<NormalizedRecord[]> {154 const p = ListingPayloadSchema.parse(raw.payload);155 if (!p.soldLine) return [];156 const m = money(p.soldLine, 'USD');157 const date = dateMDY(p.soldLine);158 if (!m || !date) return [];159 const identifiers: Record<string, string> = {};160 if (p.vin) identifiers.vin = p.vin;161 if (p.lotNumber) identifiers.bat_lot = p.lotNumber;162 const attributes = vehicleAttributes(p.title, { identifiers, metadata: { mileage: p.mileage } });163 return [makeSale({ meta: this.meta, sourceUrl: p.url, externalId: raw.externalId ?? p.url, rawTitle: p.title, attributes, price: m.amount, currency: m.currency, saleDate: date, buyerPremiumIncluded: false, auctionHouse: 'Bring a Trailer', lotNumber: p.lotNumber, imageUrls: p.image ? [p.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION })];164 }165}166167export default (meta: ConnectorMeta) => new BringATrailerConnector(meta);168