import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; import { dateMDY, dateWords, lotAttributes, makeSale, md, money, splitMarkdownItems } from '../_carlib/index.js'; const BASE = 'https://www.rrauction.com'; const PARSER_VERSION = '1.0.0'; export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), dateText: z.string().nullable(), realizedText: z.string().nullable() }); export const LotSchema = z.object({ lotNumber: z.string().nullable(), title: z.string(), url: z.string(), soldText: z.string().nullable(), estimateText: z.string().nullable(), auctionLine: z.string().nullable(), image: z.string().nullable(), }); export const LotsPayloadSchema = z.object({ kind: z.literal('lots_page'), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) }); export type LotsPayload = z.infer; export function parseCalendar(markdown: string): z.infer[] { const out: z.infer[] = []; const chunks = splitMarkdownItems(markdown, /^## /m); for (const c of chunks) { const link = c.match(/\((https:\/\/www\.rrauction\.com\/auctions\/details\/(\d+)-([a-z0-9-]+))\)/); if (!link) continue; const title = md.clean(c.split('\n')[0]!.replace(/^##\s*/, '')); const dateText = c.match(/#####\s*(\d{1,2}\/\d{1,2}\/\d{4})/)?.[1] ?? null; const realizedText = c.match(/Realized\s+(\$[\d,]+)/)?.[1] ?? null; if (!out.some((a) => a.id === link[2])) out.push({ id: link[2]!, slug: link[3]!, title, dateText, realizedText }); } return out; } export function parseLotsPage(markdown: string, auction: z.infer, page: number): LotsPayload { const chunks = splitMarkdownItems(markdown, /^\[!\[Lot #/m); const lots: z.infer[] = []; for (const c of chunks) { const t = c.match(/\[\*\*(\d+)\\?\.\s*([^\]]+?)\*\*\]\((https:\/\/www\.rrauction\.com\/auctions\/lot-detail\/[^)\s"]+)/); if (!t) continue; lots.push({ lotNumber: t[1]!, title: md.clean(t[2]!), url: t[3]!.replace(/\?cat=\d+$/, ''), soldText: c.match(/Sold For:\s*(\$[\d,]+(?:\.\d+)?)\s*(\(w\/BP\))?/)?.[0] ?? null, estimateText: c.match(/Estimate:\s*([^\n]+)/)?.[1]?.trim() ?? null, auctionLine: c.match(/Auction #\d+\s*-\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})/)?.[1] ?? null, image: md.image(c), }); } return { kind: 'lots_page', auction, page, lots }; } /** Map RR auction/lot vocabulary to taxonomy slugs (never guesses beyond keywords; defaults to autographs for RR's core business). */ export function rrCategory(auctionTitle: string, lotTitle: string): string { const t = `${auctionTitle} ${lotTitle}`.toLowerCase(); if (/meteorite|tektite|moldavite/.test(t)) return 'meteorites'; if (/apple|steve jobs|macintosh|iphone|wozniak/.test(t)) return 'apple_collectibles'; if (/space|apollo|nasa|astronaut|flown|shuttle|gemini|mercury program|cosmonaut|soyuz/.test(t)) return 'space'; if (/aviation|aircraft|pilot|lindbergh|wright brothers/.test(t)) return 'aviation'; if (/computer|commodore|altair|enigma|typewriter/.test(t)) return 'vintage_computers'; if (/animation|cel\b|disney|production cel/.test(t)) return 'animation_art'; if (/guitar|beatles|elvis|rolling stones|concert|album|music|jimi hendrix|bob dylan|led zeppelin/.test(t)) return 'music_memorabilia'; if (/sports|baseball|basketball|football|hockey|boxing|game-used|game used|jersey|olympic|babe ruth|jordan/.test(t)) return 'sports_memorabilia'; if (/hollywood|movie|film|screen-used|prop|star wars|star trek|marilyn monroe|costume/.test(t)) return 'movie_memorabilia'; if (/document|manuscript|letter signed|treaty|declaration|presidential|lincoln|washington|jefferson|kennedy|constitution/.test(t)) return 'historical_documents'; return 'autographs'; } export class RRAuctionConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2); const lotPages = Number(this.meta.config.lotPagesPerAuction ?? 6); const backfill = ctx.options.mode === 'backfill'; const done = new Set(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []); const calPage = backfill ? Number(ctx.options.cursor?.calendarPage ?? 1) : 1; const calUrl = `${BASE}/auctions/auction-calendar/cron/past/${calPage > 1 ? `?page=${calPage}` : ''}`; await this.throttle(); const cal = await ctx.fetch(calUrl, { expect: ['title', 'date'], parse: (r) => (r.markdown ? { title: parseCalendar(r.markdown)[0]?.title ?? null, date: parseCalendar(r.markdown)[0]?.dateText ?? null } : null) }); if (!cal.success || !cal.markdown) { ctx.anomaly('page_fetch_failed', `${calUrl}: ${cal.error ?? cal.httpStatus}`); return; } const auctions = parseCalendar(cal.markdown).filter((a) => !done.has(a.id)); let count = 0; let processed = 0; for (const auction of auctions) { if (processed >= auctionsPerRun || ctx.signal?.aborted) break; for (let page = 1; page <= lotPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}/auctions/auction-details/${auction.id}?page=${page}&itemQty=96&view=gallery&sort=time&cat=0`; await this.throttle(); const res = await ctx.fetch(url, { expect: ['title', 'price', 'date', 'status'], parse: (r) => { const f = r.markdown ? parseLotsPage(r.markdown, auction, page).lots.find((l) => l.soldText) : null; return f ? { title: f.title, price: money(f.soldText, 'USD')?.amount ?? null, date: f.auctionLine ?? auction.dateText, status: f.soldText } : null; }, }); if (!res.success || !res.markdown) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parseLotsPage(res.markdown, auction, page); if (payload.lots.length === 0) break; count++; yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (payload.lots.length < 96) break; } processed++; done.add(auction.id); await ctx.setCursor({ doneAuctions: [...done].slice(-400), calendarPage: backfill && auctions.every((a) => done.has(a.id)) ? calPage + 1 : calPage, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = LotsPayloadSchema.parse(raw.payload); const out: NormalizedSale[] = []; for (const lot of p.lots) { if (!lot.soldText) continue; const m = money(lot.soldText, 'USD'); const saleDate = dateWords(lot.auctionLine) ?? dateMDY(p.auction.dateText); if (!m || !saleDate) continue; const g = parseGradeFromTitle(lot.title); const categorySlug = rrCategory(p.auction.title, lot.title); const attributes = lotAttributes({ categorySlug, name: lot.title, identifiers: { rr_lot: lot.url.match(/lot-detail\/(\d+)/)?.[1] ?? `${p.auction.id}-${lot.lotNumber}` }, metadata: { auction_id: p.auction.id, auction_title: p.auction.title, estimate: lot.estimateText } }); out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.id}-${lot.lotNumber}`, rawTitle: lot.title, attributes, price: m.amount, currency: 'USD', saleDate, buyerPremiumIncluded: /w\/BP/.test(lot.soldText), auctionHouse: 'RR Auction', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, location: 'US' })); } return out; } } export default (meta: ConnectorMeta) => new RRAuctionConnector(meta);