TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';5import { dateMDY, dateWords, lotAttributes, makeSale, md, money, splitMarkdownItems } from '../_carlib/index.js';67const BASE = 'https://www.rrauction.com';8const PARSER_VERSION = '1.0.0';910export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), dateText: z.string().nullable(), realizedText: z.string().nullable() });11export const LotSchema = z.object({12 lotNumber: z.string().nullable(),13 title: z.string(),14 url: z.string(),15 soldText: z.string().nullable(),16 estimateText: z.string().nullable(),17 auctionLine: z.string().nullable(),18 image: z.string().nullable(),19});20export const LotsPayloadSchema = z.object({ kind: z.literal('lots_page'), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });21export type LotsPayload = z.infer<typeof LotsPayloadSchema>;2223export function parseCalendar(markdown: string): z.infer<typeof AuctionSchema>[] {24 const out: z.infer<typeof AuctionSchema>[] = [];25 const chunks = splitMarkdownItems(markdown, /^## /m);26 for (const c of chunks) {27 const link = c.match(/\((https:\/\/www\.rrauction\.com\/auctions\/details\/(\d+)-([a-z0-9-]+))\)/);28 if (!link) continue;29 const title = md.clean(c.split('\n')[0]!.replace(/^##\s*/, ''));30 const dateText = c.match(/#####\s*(\d{1,2}\/\d{1,2}\/\d{4})/)?.[1] ?? null;31 const realizedText = c.match(/Realized\s+(\$[\d,]+)/)?.[1] ?? null;32 if (!out.some((a) => a.id === link[2])) out.push({ id: link[2]!, slug: link[3]!, title, dateText, realizedText });33 }34 return out;35}3637export function parseLotsPage(markdown: string, auction: z.infer<typeof AuctionSchema>, page: number): LotsPayload {38 const chunks = splitMarkdownItems(markdown, /^\[!\[Lot #/m);39 const lots: z.infer<typeof LotSchema>[] = [];40 for (const c of chunks) {41 const t = c.match(/\[\*\*(\d+)\\?\.\s*([^\]]+?)\*\*\]\((https:\/\/www\.rrauction\.com\/auctions\/lot-detail\/[^)\s"]+)/);42 if (!t) continue;43 lots.push({44 lotNumber: t[1]!,45 title: md.clean(t[2]!),46 url: t[3]!.replace(/\?cat=\d+$/, ''),47 soldText: c.match(/Sold For:\s*(\$[\d,]+(?:\.\d+)?)\s*(\(w\/BP\))?/)?.[0] ?? null,48 estimateText: c.match(/Estimate:\s*([^\n]+)/)?.[1]?.trim() ?? null,49 auctionLine: c.match(/Auction #\d+\s*-\s*([A-Za-z]+\s+\d{1,2},\s+\d{4})/)?.[1] ?? null,50 image: md.image(c),51 });52 }53 return { kind: 'lots_page', auction, page, lots };54}5556/** Map RR auction/lot vocabulary to taxonomy slugs (never guesses beyond keywords; defaults to autographs for RR's core business). */57export function rrCategory(auctionTitle: string, lotTitle: string): string {58 const t = `${auctionTitle} ${lotTitle}`.toLowerCase();59 if (/meteorite|tektite|moldavite/.test(t)) return 'meteorites';60 if (/apple|steve jobs|macintosh|iphone|wozniak/.test(t)) return 'apple_collectibles';61 if (/space|apollo|nasa|astronaut|flown|shuttle|gemini|mercury program|cosmonaut|soyuz/.test(t)) return 'space';62 if (/aviation|aircraft|pilot|lindbergh|wright brothers/.test(t)) return 'aviation';63 if (/computer|commodore|altair|enigma|typewriter/.test(t)) return 'vintage_computers';64 if (/animation|cel\b|disney|production cel/.test(t)) return 'animation_art';65 if (/guitar|beatles|elvis|rolling stones|concert|album|music|jimi hendrix|bob dylan|led zeppelin/.test(t)) return 'music_memorabilia';66 if (/sports|baseball|basketball|football|hockey|boxing|game-used|game used|jersey|olympic|babe ruth|jordan/.test(t)) return 'sports_memorabilia';67 if (/hollywood|movie|film|screen-used|prop|star wars|star trek|marilyn monroe|costume/.test(t)) return 'movie_memorabilia';68 if (/document|manuscript|letter signed|treaty|declaration|presidential|lincoln|washington|jefferson|kennedy|constitution/.test(t)) return 'historical_documents';69 return 'autographs';70}7172export class RRAuctionConnector extends BaseConnector {73 readonly version = '1.0.0';74 readonly parserVersion = PARSER_VERSION;75 protected override minIntervalMs = 2000;7677 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {78 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);79 const lotPages = Number(this.meta.config.lotPagesPerAuction ?? 6);80 const backfill = ctx.options.mode === 'backfill';81 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);82 const calPage = backfill ? Number(ctx.options.cursor?.calendarPage ?? 1) : 1;83 const calUrl = `${BASE}/auctions/auction-calendar/cron/past/${calPage > 1 ? `?page=${calPage}` : ''}`;84 await this.throttle();85 const cal = await ctx.fetch(calUrl, { expect: ['title', 'date'], parse: (r) => (r.markdown ? { title: parseCalendar(r.markdown)[0]?.title ?? null, date: parseCalendar(r.markdown)[0]?.dateText ?? null } : null) });86 if (!cal.success || !cal.markdown) {87 ctx.anomaly('page_fetch_failed', `${calUrl}: ${cal.error ?? cal.httpStatus}`);88 return;89 }90 const auctions = parseCalendar(cal.markdown).filter((a) => !done.has(a.id));91 let count = 0;92 let processed = 0;93 for (const auction of auctions) {94 if (processed >= auctionsPerRun || ctx.signal?.aborted) break;95 for (let page = 1; page <= lotPages; page++) {96 if (ctx.signal?.aborted || this.reached(ctx, count)) break;97 const url = `${BASE}/auctions/auction-details/${auction.id}?page=${page}&itemQty=96&view=gallery&sort=time&cat=0`;98 await this.throttle();99 const res = await ctx.fetch(url, {100 expect: ['title', 'price', 'date', 'status'],101 parse: (r) => {102 const f = r.markdown ? parseLotsPage(r.markdown, auction, page).lots.find((l) => l.soldText) : null;103 return f ? { title: f.title, price: money(f.soldText, 'USD')?.amount ?? null, date: f.auctionLine ?? auction.dateText, status: f.soldText } : null;104 },105 });106 if (!res.success || !res.markdown) {107 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);108 break;109 }110 const payload = parseLotsPage(res.markdown, auction, page);111 if (payload.lots.length === 0) break;112 count++;113 yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };114 if (payload.lots.length < 96) break;115 }116 processed++;117 done.add(auction.id);118 await ctx.setCursor({ doneAuctions: [...done].slice(-400), calendarPage: backfill && auctions.every((a) => done.has(a.id)) ? calPage + 1 : calPage, updatedAt: new Date().toISOString() });119 }120 }121122 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {123 const p = LotsPayloadSchema.parse(raw.payload);124 const out: NormalizedSale[] = [];125 for (const lot of p.lots) {126 if (!lot.soldText) continue;127 const m = money(lot.soldText, 'USD');128 const saleDate = dateWords(lot.auctionLine) ?? dateMDY(p.auction.dateText);129 if (!m || !saleDate) continue;130 const g = parseGradeFromTitle(lot.title);131 const categorySlug = rrCategory(p.auction.title, lot.title);132 const attributes = lotAttributes({ categorySlug, name: lot.title, identifiers: { rr_lot: lot.url.match(/lot-detail\/(\d+)/)?.[1] ?? `${p.auction.id}-${lot.lotNumber}` }, metadata: { auction_id: p.auction.id, auction_title: p.auction.title, estimate: lot.estimateText } });133 out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.id}-${lot.lotNumber}`, rawTitle: lot.title, attributes, price: m.amount, currency: 'USD', saleDate, buyerPremiumIncluded: /w\/BP/.test(lot.soldText), auctionHouse: 'RR Auction', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, location: 'US' }));134 }135 return out;136 }137}138139export default (meta: ConnectorMeta) => new RRAuctionConnector(meta);140